[ROCm][Perf][Kimi-K3] Use AITER Gluon KDA kernels for decode and spec decode (#57392)
Signed-off-by: omuhamma <omar.muhammad@amd.com> Signed-off-by: Nicholas Susanto <nicholas.susanto@amd.com> Co-authored-by: Nicholas Susanto <nicholas.susanto@amd.com> Co-authored-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com>
This commit is contained in:
co-authored by
Nicholas Susanto
Simon Danielsson
parent
de7d70676d
commit
3f1e57225a
@@ -1,11 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The ROCm fused KDA decode kernel must match the Triton chain it replaces.
|
||||
"""The ROCm fused KDA decode kernels must match the Triton chain they replace.
|
||||
|
||||
The fused kernel folds the packed causal conv1d update, the gated delta-rule
|
||||
recurrence and the gated output RMSNorm into one launch, and updates both the
|
||||
conv state and the recurrent state in place. Every one of those outputs is
|
||||
compared against the three-kernel path the AMD layer falls back to.
|
||||
Two fused paths fold the packed causal conv1d update, the gated delta-rule
|
||||
recurrence and the gated output RMSNorm into one launch, updating the conv
|
||||
state and the recurrent state in place:
|
||||
|
||||
* the vLLM HIP kernel ``ops.fused_kda_decode`` (gfx942 / gfx950), and
|
||||
* AITER's Gluon kernels behind ``rocm_aiter_ops`` (CDNA4+): the packed
|
||||
non-spec decode and the speculative-decode ``fused_recurrent_kda``.
|
||||
|
||||
Every output of each is compared against the unfused path the AMD layer falls
|
||||
back to, with the argument layout the layer itself uses.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -14,7 +20,7 @@ import torch
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _on_supported_arch() -> bool:
|
||||
def _hip_kernel_arch() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx942, on_gfx950
|
||||
@@ -22,10 +28,26 @@ def _on_supported_arch() -> bool:
|
||||
return on_gfx950() or on_gfx942()
|
||||
|
||||
|
||||
def _aiter_gluon_kda_available() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
return get_cdna_version() >= 4 and rocm_aiter_ops._kda_gluon_decode_importable()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _on_supported_arch(),
|
||||
not current_platform.is_rocm(), reason="ROCm-only KDA decode kernels"
|
||||
)
|
||||
requires_hip_kernel_arch = pytest.mark.skipif(
|
||||
not _hip_kernel_arch(),
|
||||
reason="The fused KDA decode kernel is only built for gfx942 / gfx950",
|
||||
)
|
||||
requires_aiter_gluon_kda = pytest.mark.skipif(
|
||||
not _aiter_gluon_kda_available(),
|
||||
reason="AITER Gluon KDA decode needs CDNA4+ and an AITER build that ships it",
|
||||
)
|
||||
|
||||
# Kimi-K3 KDA: 96 heads x 128, conv width 4, gate_lower_bound -5.0.
|
||||
HEAD_DIM = 128
|
||||
@@ -41,7 +63,11 @@ def _requires_kernel() -> None:
|
||||
|
||||
|
||||
class KdaDecodeInputs:
|
||||
"""One decode step of a Kimi-K3 KDA layer, in the layer's own layout."""
|
||||
"""One decode step of a Kimi-K3 KDA layer, in the layer's own layout.
|
||||
|
||||
``num_spec`` > 0 sizes the conv state for speculative decoding
|
||||
(``conv_width - 1 + num_spec`` taps, as ``kda_state_shape`` does).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -49,6 +75,7 @@ class KdaDecodeInputs:
|
||||
num_heads: int,
|
||||
num_slots: int,
|
||||
seed: int = 0,
|
||||
num_spec: int = 0,
|
||||
) -> None:
|
||||
torch.manual_seed(seed)
|
||||
device = "cuda"
|
||||
@@ -70,9 +97,15 @@ class KdaDecodeInputs:
|
||||
for i in range(3)
|
||||
]
|
||||
)
|
||||
# SD cache layout: [slots, width - 1, 3 * dim]; the layer transposes it.
|
||||
# SD cache layout: [slots, taps, 3 * dim]; the layer transposes it.
|
||||
self.conv_state = (
|
||||
torch.randn(num_slots, CONV_WIDTH - 1, 3 * dim, device=device, dtype=DTYPE)
|
||||
torch.randn(
|
||||
num_slots,
|
||||
CONV_WIDTH - 1 + num_spec,
|
||||
3 * dim,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
* 0.5
|
||||
)
|
||||
self.recurrent_state = (
|
||||
@@ -112,6 +145,12 @@ class KdaDecodeInputs:
|
||||
def conv_state_view(self, state: torch.Tensor) -> torch.Tensor:
|
||||
return state.transpose(-1, -2)
|
||||
|
||||
def empty_out(self) -> torch.Tensor:
|
||||
num_tokens = self.mixed_qkv.shape[0]
|
||||
return torch.empty(
|
||||
1, num_tokens, self.num_heads, HEAD_DIM, device="cuda", dtype=DTYPE
|
||||
)
|
||||
|
||||
|
||||
def _gated_rmsnorm(
|
||||
x: torch.Tensor, gate: torch.Tensor, weight: torch.Tensor, eps: float
|
||||
@@ -196,6 +235,7 @@ def _run_fused(
|
||||
return out, conv_state, recurrent_state
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@pytest.mark.parametrize("num_heads", [12, 24, 96])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
|
||||
@torch.inference_mode()
|
||||
@@ -214,6 +254,7 @@ def test_fused_kda_decode_matches_triton_chain(num_heads: int, num_tokens: int)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_leaves_untouched_slots_alone() -> None:
|
||||
"""Only the slots named by state_indices may be written."""
|
||||
@@ -234,6 +275,7 @@ def test_fused_kda_decode_leaves_untouched_slots_alone() -> None:
|
||||
)
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_without_output_norm() -> None:
|
||||
"""Omitting the gate/norm pair returns the raw recurrent output."""
|
||||
@@ -265,6 +307,7 @@ def test_fused_kda_decode_without_output_norm() -> None:
|
||||
assert torch.isfinite(out).all()
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_skips_null_block_padding() -> None:
|
||||
"""A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0).
|
||||
@@ -291,6 +334,7 @@ def test_fused_kda_decode_skips_null_block_padding() -> None:
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
def test_decode_weights_staged_after_loading(dist_init) -> None:
|
||||
"""The fused-decode weight copies must not depend on the weight loaders.
|
||||
|
||||
@@ -323,11 +367,15 @@ def test_decode_weights_staged_after_loading(dist_init) -> None:
|
||||
config, vllm_config, prefix="model.layers.0.self_attn"
|
||||
)
|
||||
assert layer.decode_conv1d_weight is not None
|
||||
# The AITER Gluon decode path takes o_norm.weight directly, so only the HIP
|
||||
# kernel gets an fp32 norm copy.
|
||||
assert (layer.decode_norm_weight is None) == layer.use_aiter_decode
|
||||
with torch.no_grad():
|
||||
for param in layer.parameters():
|
||||
param.normal_()
|
||||
layer.decode_conv1d_weight.fill_(float("nan"))
|
||||
layer.decode_norm_weight.fill_(float("nan"))
|
||||
if layer.decode_norm_weight is not None:
|
||||
layer.decode_norm_weight.fill_(float("nan"))
|
||||
|
||||
process_weights_after_loading(
|
||||
torch.nn.ModuleList([layer]), vllm_config.model_config, torch.device("cuda")
|
||||
@@ -341,7 +389,271 @@ def test_decode_weights_staged_after_loading(dist_init) -> None:
|
||||
torch.testing.assert_close(
|
||||
layer.decode_conv1d_weight, expected_conv, atol=0, rtol=0
|
||||
)
|
||||
if layer.decode_norm_weight is not None:
|
||||
torch.testing.assert_close(
|
||||
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
|
||||
)
|
||||
assert not layer.decode_norm_weight.requires_grad
|
||||
assert not layer.decode_conv1d_weight.requires_grad
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# non-spec decode: fused_recurrent_kda_packed_decode
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _run_aiter_packed_decode(
|
||||
inp: KdaDecodeInputs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Exactly the call KimiK3DeltaAttention._forward makes for decode batches."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
num_tokens = inp.mixed_qkv.shape[0]
|
||||
core_attn_out = inp.empty_out()
|
||||
rocm_aiter_ops.kda_fused_recurrent_packed_decode(
|
||||
mixed_qkv=inp.mixed_qkv,
|
||||
g=inp.g1[0],
|
||||
beta=inp.beta[0],
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
initial_state=recurrent_state,
|
||||
ssm_state_indices=inp.state_indices[:num_tokens],
|
||||
conv_state=inp.conv_state_view(conv_state),
|
||||
conv_weight=inp.decode_conv1d_weight,
|
||||
out_gate=inp.g2[:num_tokens],
|
||||
norm_weight=inp.norm_weight_bf16,
|
||||
norm_eps=NORM_EPS,
|
||||
out=core_attn_out[0, :num_tokens],
|
||||
)
|
||||
return core_attn_out, conv_state, recurrent_state
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@pytest.mark.parametrize("num_heads", [12, 24])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
|
||||
@torch.inference_mode()
|
||||
def test_aiter_packed_decode_matches_triton_chain(
|
||||
num_heads: int, num_tokens: int
|
||||
) -> None:
|
||||
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots=max(num_tokens, 4) + 3)
|
||||
|
||||
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
|
||||
actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp)
|
||||
|
||||
# The fused kernel keeps the recurrent output in fp32 through the norm,
|
||||
# while the Triton chain rounds it to BF16 in between.
|
||||
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
|
||||
# The conv state is a pure shift-and-append: it must be bit-exact.
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@torch.inference_mode()
|
||||
def test_aiter_packed_decode_leaves_untouched_slots_alone() -> None:
|
||||
"""Only the slots named by the state indices may be written."""
|
||||
num_tokens, num_heads, num_slots = 4, 12, 9
|
||||
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots, seed=3)
|
||||
|
||||
_, actual_conv, actual_state = _run_aiter_packed_decode(inp)
|
||||
|
||||
touched = set(inp.state_indices.tolist())
|
||||
untouched = [slot for slot in range(num_slots) if slot not in touched]
|
||||
assert untouched, "test needs at least one unused slot"
|
||||
torch.testing.assert_close(
|
||||
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
|
||||
actual_conv[untouched], inp.conv_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual_state[untouched], inp.recurrent_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@torch.inference_mode()
|
||||
def test_aiter_packed_decode_skips_null_block_padding() -> None:
|
||||
"""A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0).
|
||||
|
||||
gdn_attn.py fills the tail of ``non_spec_state_indices_tensor`` with 0 and
|
||||
the Triton chain leaves slot 0 alone; the fused kernel's pad_slot_guard
|
||||
must do the same rather than read-modify-write slot 0 once per padded row.
|
||||
"""
|
||||
num_real, num_padded, num_heads = 3, 5, 12
|
||||
inp = KdaDecodeInputs(num_real + num_padded, num_heads, num_slots=12, seed=11)
|
||||
inp.state_indices[num_real:] = 0
|
||||
|
||||
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
|
||||
actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp)
|
||||
|
||||
torch.testing.assert_close(
|
||||
actual_out[0, :num_real], expected_out[0, :num_real], atol=3e-2, rtol=3e-2
|
||||
)
|
||||
assert torch.isfinite(actual_out).all()
|
||||
# Slot 0 must be untouched by both paths.
|
||||
torch.testing.assert_close(actual_conv[0], inp.conv_state[0], atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state[0], inp.recurrent_state[0], atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# spec-only decode: fused_recurrent_kda with conv / gate / norm fused in
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SpecBatch:
|
||||
"""A spec-only KDA batch: ``num_spec + 1`` query tokens per request at most.
|
||||
|
||||
``seq_lens`` are the per-request query lengths, ``num_accepted`` the tokens
|
||||
verified in the previous step; ``state_indices`` is the [N, num_spec + 1]
|
||||
per-token slot table the GDN metadata hands the layer.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
seq_lens: list[int],
|
||||
num_accepted: list[int],
|
||||
num_heads: int,
|
||||
num_spec: int,
|
||||
seed: int = 0,
|
||||
) -> None:
|
||||
assert len(seq_lens) == len(num_accepted)
|
||||
assert max(seq_lens) <= num_spec + 1
|
||||
n_seqs = len(seq_lens)
|
||||
slots_per_seq = num_spec + 1
|
||||
num_slots = n_seqs * slots_per_seq + 2
|
||||
self.inp = KdaDecodeInputs(
|
||||
sum(seq_lens), num_heads, num_slots, seed=seed, num_spec=num_spec
|
||||
)
|
||||
# Distinct slots per (request, token), shuffled, none at slot 0.
|
||||
perm = torch.randperm(num_slots - 1, device="cuda")[: n_seqs * slots_per_seq]
|
||||
self.state_indices = (perm + 1).to(torch.int32).view(n_seqs, slots_per_seq)
|
||||
self.query_start_loc = torch.tensor(
|
||||
[0] + torch.tensor(seq_lens).cumsum(0).tolist(),
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
self.num_accepted_tokens = torch.tensor(
|
||||
num_accepted, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
self.max_query_len = slots_per_seq
|
||||
|
||||
|
||||
def _run_spec_triton_chain(
|
||||
sb: SpecBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""The layer's unfused spec path: conv update -> recurrent KDA -> norm."""
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import fused_recurrent_kda
|
||||
|
||||
inp = sb.inp
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
conv_out = causal_conv1d_update(
|
||||
inp.mixed_qkv,
|
||||
inp.conv_state_view(conv_state),
|
||||
inp.conv_weights,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_state_indices=sb.state_indices[:, 0],
|
||||
num_accepted_tokens=sb.num_accepted_tokens,
|
||||
query_start_loc=sb.query_start_loc,
|
||||
max_query_len=sb.max_query_len,
|
||||
validate_data=False,
|
||||
out=torch.empty_like(inp.mixed_qkv),
|
||||
)
|
||||
q, k, v = (
|
||||
x.view(1, -1, inp.num_heads, HEAD_DIM) for x in conv_out.split(inp.dim, dim=-1)
|
||||
)
|
||||
core_attn_out, _ = fused_recurrent_kda(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=inp.g1,
|
||||
raw_beta=inp.beta,
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=sb.query_start_loc,
|
||||
ssm_state_indices=sb.state_indices,
|
||||
num_accepted_tokens=sb.num_accepted_tokens,
|
||||
)
|
||||
out = _gated_rmsnorm(core_attn_out, inp.g2, inp.norm_weight_bf16, NORM_EPS)
|
||||
return out, conv_state, recurrent_state
|
||||
|
||||
|
||||
def _run_aiter_spec(
|
||||
sb: SpecBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Exactly the call KimiK3DeltaAttention._forward makes for spec-only batches."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
inp = sb.inp
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
q, k, v = (
|
||||
x.view(1, -1, inp.num_heads, HEAD_DIM)
|
||||
for x in inp.mixed_qkv.split(inp.dim, dim=-1)
|
||||
)
|
||||
n_spec = q.shape[1]
|
||||
core_attn_out = inp.empty_out()
|
||||
rocm_aiter_ops.kda_fused_recurrent_spec_decode(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=inp.g1,
|
||||
beta=inp.beta,
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=sb.query_start_loc,
|
||||
ssm_state_indices=sb.state_indices,
|
||||
num_accepted_tokens=sb.num_accepted_tokens,
|
||||
conv_state=inp.conv_state_view(conv_state),
|
||||
conv_weight=inp.decode_conv1d_weight,
|
||||
out_gate=inp.g2[:n_spec].unsqueeze(0),
|
||||
norm_weight=inp.norm_weight_bf16,
|
||||
norm_eps=NORM_EPS,
|
||||
out=core_attn_out[:, :n_spec],
|
||||
)
|
||||
return core_attn_out, conv_state, recurrent_state
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@pytest.mark.parametrize("num_heads", [12, 24])
|
||||
@pytest.mark.parametrize(
|
||||
"num_spec, seq_lens, num_accepted",
|
||||
[
|
||||
(3, [4, 4, 4], [1, 2, 4]), # Kimi-K3 recipe: 3 draft tokens, uniform
|
||||
(3, [4, 2, 1, 4], [4, 1, 1, 3]), # ragged after partial acceptance
|
||||
(6, [7, 3], [1, 4]), # longer window, two requests
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_aiter_spec_decode_matches_triton_chain(
|
||||
num_heads: int, num_spec: int, seq_lens: list[int], num_accepted: list[int]
|
||||
) -> None:
|
||||
sb = SpecBatch(seq_lens, num_accepted, num_heads, num_spec, seed=5)
|
||||
|
||||
expected_out, expected_conv, expected_state = _run_spec_triton_chain(sb)
|
||||
actual_out, actual_conv, actual_state = _run_aiter_spec(sb)
|
||||
|
||||
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
|
||||
# Nothing outside the slot table may change.
|
||||
touched = set(sb.state_indices.flatten().tolist())
|
||||
untouched = [s for s in range(sb.inp.conv_state.shape[0]) if s not in touched]
|
||||
assert untouched
|
||||
torch.testing.assert_close(
|
||||
actual_conv[untouched], sb.inp.conv_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual_state[untouched], sb.inp.recurrent_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
assert not layer.decode_norm_weight.requires_grad
|
||||
|
||||
@@ -2626,6 +2626,32 @@ class rocm_aiter_ops:
|
||||
"""
|
||||
return cls._AITER_ENABLED and cls._gdn_triton_kernels_importable()
|
||||
|
||||
@staticmethod
|
||||
def _kda_gluon_decode_importable() -> bool:
|
||||
try:
|
||||
from aiter.ops.triton.attention import kda # noqa: F401
|
||||
|
||||
return True
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_kda_gluon_decode_available(cls) -> bool:
|
||||
"""Check if AITER's Gluon KDA decode kernels can run here.
|
||||
|
||||
``aiter.ops.triton.attention.kda`` (conv + gated delta rule + gated
|
||||
RMSNorm in one launch, plain and speculative decode) ships CDNA4+
|
||||
(gfx950, gfx1250) builds only, and is absent in older aiter releases.
|
||||
"""
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
return (
|
||||
cls._AITER_ENABLED
|
||||
and get_cdna_version() >= 4
|
||||
and cls._kda_gluon_decode_importable()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@functools.cache
|
||||
def is_gdn_flydsl_prefill_available(cls) -> bool:
|
||||
@@ -4051,6 +4077,90 @@ class rocm_aiter_ops:
|
||||
output_zeros=False,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def kda_fused_recurrent_packed_decode(
|
||||
mixed_qkv: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
lower_bound: float | None,
|
||||
initial_state: torch.Tensor,
|
||||
ssm_state_indices: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
conv_weight: torch.Tensor,
|
||||
out_gate: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
norm_eps: float,
|
||||
out: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from aiter.ops.triton.attention.kda import fused_recurrent_kda_packed_decode
|
||||
|
||||
return fused_recurrent_kda_packed_decode(
|
||||
mixed_qkv=mixed_qkv,
|
||||
g=g,
|
||||
beta=beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
initial_state=initial_state,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
conv_state=conv_state,
|
||||
conv_weight=conv_weight,
|
||||
out_gate=out_gate,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
out=out,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def kda_fused_recurrent_spec_decode(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
lower_bound: float | None,
|
||||
initial_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
ssm_state_indices: torch.Tensor,
|
||||
num_accepted_tokens: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
conv_weight: torch.Tensor,
|
||||
out_gate: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
norm_eps: float,
|
||||
out: torch.Tensor,
|
||||
) -> None:
|
||||
from aiter.ops.triton.attention.kda import fused_recurrent_kda
|
||||
|
||||
fused_recurrent_kda(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
initial_state=initial_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
use_gate_in_kernel=True,
|
||||
use_beta_sigmoid_in_kernel=True,
|
||||
pad_slot_guard=True,
|
||||
conv_state=conv_state,
|
||||
conv_weight=conv_weight,
|
||||
out_gate=out_gate,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
out=out,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def batched_gemm_a16wfp4(
|
||||
X: torch.Tensor,
|
||||
|
||||
@@ -8,6 +8,7 @@ from einops import rearrange
|
||||
from torch import nn
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import divide
|
||||
@@ -182,16 +183,19 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
# into one kernel, which wants a width-major fp32 conv weight staged at
|
||||
# load time. Everything else keeps the [channel, width] layout.
|
||||
conv_state_dtype, _ = self.get_state_dtype()
|
||||
decode_conv1d_weight = None
|
||||
if is_fused_kda_decode_supported(
|
||||
self.use_aiter_decode = bool(rocm_aiter_ops.is_kda_gluon_decode_available())
|
||||
use_hip_decode = not self.use_aiter_decode and is_fused_kda_decode_supported(
|
||||
self.local_num_heads,
|
||||
self.head_dim,
|
||||
self.conv_size,
|
||||
self.num_spec,
|
||||
vllm_config.model_config.dtype,
|
||||
conv_state_dtype,
|
||||
):
|
||||
)
|
||||
if use_hip_decode:
|
||||
logger.info_once("Fused KDA decode kernel (conv+KDA+norm) is enabled.")
|
||||
decode_conv1d_weight = None
|
||||
if use_hip_decode or self.use_aiter_decode:
|
||||
decode_conv1d_weight = torch.empty(
|
||||
3,
|
||||
self.conv_size,
|
||||
@@ -257,7 +261,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
|
||||
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
|
||||
decode_norm_weight = None
|
||||
if decode_conv1d_weight is not None:
|
||||
if use_hip_decode:
|
||||
# Upcast once at load time; a BF16 norm weight slows the fused
|
||||
# decode kernel's epilogue.
|
||||
decode_norm_weight = torch.empty(
|
||||
@@ -388,6 +392,66 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
if not is_conv_state_dim_first():
|
||||
conv_state = conv_state.transpose(-1, -2)
|
||||
|
||||
if (
|
||||
self.use_aiter_decode
|
||||
and spec_sequence_masks is None
|
||||
and m.num_prefills == 0
|
||||
and m.num_decodes > 0
|
||||
):
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
rocm_aiter_ops.kda_fused_recurrent_packed_decode(
|
||||
mixed_qkv=mixed_qkv,
|
||||
g=g1[0],
|
||||
beta=beta[0],
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
ssm_state_indices=non_spec_state_indices_tensor[:num_actual_tokens],
|
||||
conv_state=conv_state,
|
||||
conv_weight=self.decode_conv1d_weight,
|
||||
out_gate=g2[:num_actual_tokens],
|
||||
norm_weight=self.o_norm.weight,
|
||||
norm_eps=self.o_norm.eps,
|
||||
out=core_attn_out[0, :num_actual_tokens],
|
||||
)
|
||||
return
|
||||
|
||||
if (
|
||||
self.use_aiter_decode
|
||||
and spec_sequence_masks is not None
|
||||
and m.num_prefills == 0
|
||||
):
|
||||
# Spec-only batch (gdn_attn never mixes spec and plain decodes).
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert spec_query_start_loc is not None
|
||||
q, k, v = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in mixed_qkv.split(self.local_projection_size, dim=-1)
|
||||
)
|
||||
n_spec = q.shape[1]
|
||||
rocm_aiter_ops.kda_fused_recurrent_spec_decode(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g1,
|
||||
beta=beta,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=spec_query_start_loc[: m.num_spec_decodes + 1],
|
||||
ssm_state_indices=spec_state_indices_tensor,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
conv_state=conv_state,
|
||||
conv_weight=self.decode_conv1d_weight,
|
||||
out_gate=g2[:n_spec].unsqueeze(0),
|
||||
norm_weight=self.o_norm.weight,
|
||||
norm_eps=self.o_norm.eps,
|
||||
out=core_attn_out[:, :n_spec],
|
||||
)
|
||||
return
|
||||
|
||||
if (
|
||||
self.decode_conv1d_weight is not None
|
||||
and self.decode_norm_weight is not None
|
||||
|
||||
Reference in New Issue
Block a user