diff --git a/tests/models/kimi_k3/test_amd_kda_decode.py b/tests/models/kimi_k3/test_amd_kda_decode.py index 647c982c7d..fc6039ee54 100644 --- a/tests/models/kimi_k3/test_amd_kda_decode.py +++ b/tests/models/kimi_k3/test_amd_kda_decode.py @@ -1,11 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""The ROCm fused KDA decode kernel must match the Triton chain it replaces. +"""The ROCm fused KDA decode kernels must match the Triton chain they replace. -The fused kernel folds the packed causal conv1d update, the gated delta-rule -recurrence and the gated output RMSNorm into one launch, and updates both the -conv state and the recurrent state in place. Every one of those outputs is -compared against the three-kernel path the AMD layer falls back to. +Two fused paths fold the packed causal conv1d update, the gated delta-rule +recurrence and the gated output RMSNorm into one launch, updating the conv +state and the recurrent state in place: + +* the vLLM HIP kernel ``ops.fused_kda_decode`` (gfx942 / gfx950), and +* AITER's Gluon kernels behind ``rocm_aiter_ops`` (CDNA4+): the packed + non-spec decode and the speculative-decode ``fused_recurrent_kda``. + +Every output of each is compared against the unfused path the AMD layer falls +back to, with the argument layout the layer itself uses. """ import pytest @@ -14,7 +20,7 @@ import torch from vllm.platforms import current_platform -def _on_supported_arch() -> bool: +def _hip_kernel_arch() -> bool: if not current_platform.is_rocm(): return False from vllm.platforms.rocm import on_gfx942, on_gfx950 @@ -22,10 +28,26 @@ def _on_supported_arch() -> bool: return on_gfx950() or on_gfx942() +def _aiter_gluon_kda_available() -> bool: + if not current_platform.is_rocm(): + return False + from vllm._aiter_ops import rocm_aiter_ops + from vllm.platforms.rocm import get_cdna_version + + return get_cdna_version() >= 4 and rocm_aiter_ops._kda_gluon_decode_importable() + + pytestmark = pytest.mark.skipif( - not _on_supported_arch(), + not current_platform.is_rocm(), reason="ROCm-only KDA decode kernels" +) +requires_hip_kernel_arch = pytest.mark.skipif( + not _hip_kernel_arch(), reason="The fused KDA decode kernel is only built for gfx942 / gfx950", ) +requires_aiter_gluon_kda = pytest.mark.skipif( + not _aiter_gluon_kda_available(), + reason="AITER Gluon KDA decode needs CDNA4+ and an AITER build that ships it", +) # Kimi-K3 KDA: 96 heads x 128, conv width 4, gate_lower_bound -5.0. HEAD_DIM = 128 @@ -41,7 +63,11 @@ def _requires_kernel() -> None: class KdaDecodeInputs: - """One decode step of a Kimi-K3 KDA layer, in the layer's own layout.""" + """One decode step of a Kimi-K3 KDA layer, in the layer's own layout. + + ``num_spec`` > 0 sizes the conv state for speculative decoding + (``conv_width - 1 + num_spec`` taps, as ``kda_state_shape`` does). + """ def __init__( self, @@ -49,6 +75,7 @@ class KdaDecodeInputs: num_heads: int, num_slots: int, seed: int = 0, + num_spec: int = 0, ) -> None: torch.manual_seed(seed) device = "cuda" @@ -70,9 +97,15 @@ class KdaDecodeInputs: for i in range(3) ] ) - # SD cache layout: [slots, width - 1, 3 * dim]; the layer transposes it. + # SD cache layout: [slots, taps, 3 * dim]; the layer transposes it. self.conv_state = ( - torch.randn(num_slots, CONV_WIDTH - 1, 3 * dim, device=device, dtype=DTYPE) + torch.randn( + num_slots, + CONV_WIDTH - 1 + num_spec, + 3 * dim, + device=device, + dtype=DTYPE, + ) * 0.5 ) self.recurrent_state = ( @@ -112,6 +145,12 @@ class KdaDecodeInputs: def conv_state_view(self, state: torch.Tensor) -> torch.Tensor: return state.transpose(-1, -2) + def empty_out(self) -> torch.Tensor: + num_tokens = self.mixed_qkv.shape[0] + return torch.empty( + 1, num_tokens, self.num_heads, HEAD_DIM, device="cuda", dtype=DTYPE + ) + def _gated_rmsnorm( x: torch.Tensor, gate: torch.Tensor, weight: torch.Tensor, eps: float @@ -196,6 +235,7 @@ def _run_fused( return out, conv_state, recurrent_state +@requires_hip_kernel_arch @pytest.mark.parametrize("num_heads", [12, 24, 96]) @pytest.mark.parametrize("num_tokens", [1, 7, 128]) @torch.inference_mode() @@ -214,6 +254,7 @@ def test_fused_kda_decode_matches_triton_chain(num_heads: int, num_tokens: int) torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3) +@requires_hip_kernel_arch @torch.inference_mode() def test_fused_kda_decode_leaves_untouched_slots_alone() -> None: """Only the slots named by state_indices may be written.""" @@ -234,6 +275,7 @@ def test_fused_kda_decode_leaves_untouched_slots_alone() -> None: ) +@requires_hip_kernel_arch @torch.inference_mode() def test_fused_kda_decode_without_output_norm() -> None: """Omitting the gate/norm pair returns the raw recurrent output.""" @@ -265,6 +307,7 @@ def test_fused_kda_decode_without_output_norm() -> None: assert torch.isfinite(out).all() +@requires_hip_kernel_arch @torch.inference_mode() def test_fused_kda_decode_skips_null_block_padding() -> None: """A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0). @@ -291,6 +334,7 @@ def test_fused_kda_decode_skips_null_block_padding() -> None: torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3) +@requires_hip_kernel_arch def test_decode_weights_staged_after_loading(dist_init) -> None: """The fused-decode weight copies must not depend on the weight loaders. @@ -323,11 +367,15 @@ def test_decode_weights_staged_after_loading(dist_init) -> None: config, vllm_config, prefix="model.layers.0.self_attn" ) assert layer.decode_conv1d_weight is not None + # The AITER Gluon decode path takes o_norm.weight directly, so only the HIP + # kernel gets an fp32 norm copy. + assert (layer.decode_norm_weight is None) == layer.use_aiter_decode with torch.no_grad(): for param in layer.parameters(): param.normal_() layer.decode_conv1d_weight.fill_(float("nan")) - layer.decode_norm_weight.fill_(float("nan")) + if layer.decode_norm_weight is not None: + layer.decode_norm_weight.fill_(float("nan")) process_weights_after_loading( torch.nn.ModuleList([layer]), vllm_config.model_config, torch.device("cuda") @@ -341,7 +389,271 @@ def test_decode_weights_staged_after_loading(dist_init) -> None: torch.testing.assert_close( layer.decode_conv1d_weight, expected_conv, atol=0, rtol=0 ) + if layer.decode_norm_weight is not None: + torch.testing.assert_close( + layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0 + ) + assert not layer.decode_norm_weight.requires_grad + assert not layer.decode_conv1d_weight.requires_grad + + +# -------------------------------------------------------------------------- +# non-spec decode: fused_recurrent_kda_packed_decode +# -------------------------------------------------------------------------- + + +def _run_aiter_packed_decode( + inp: KdaDecodeInputs, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Exactly the call KimiK3DeltaAttention._forward makes for decode batches.""" + from vllm._aiter_ops import rocm_aiter_ops + + conv_state = inp.conv_state.clone() + recurrent_state = inp.recurrent_state.clone() + num_tokens = inp.mixed_qkv.shape[0] + core_attn_out = inp.empty_out() + rocm_aiter_ops.kda_fused_recurrent_packed_decode( + mixed_qkv=inp.mixed_qkv, + g=inp.g1[0], + beta=inp.beta[0], + A_log=inp.A_log, + dt_bias=inp.dt_bias, + lower_bound=GATE_LOWER_BOUND, + initial_state=recurrent_state, + ssm_state_indices=inp.state_indices[:num_tokens], + conv_state=inp.conv_state_view(conv_state), + conv_weight=inp.decode_conv1d_weight, + out_gate=inp.g2[:num_tokens], + norm_weight=inp.norm_weight_bf16, + norm_eps=NORM_EPS, + out=core_attn_out[0, :num_tokens], + ) + return core_attn_out, conv_state, recurrent_state + + +@requires_aiter_gluon_kda +@pytest.mark.parametrize("num_heads", [12, 24]) +@pytest.mark.parametrize("num_tokens", [1, 7, 128]) +@torch.inference_mode() +def test_aiter_packed_decode_matches_triton_chain( + num_heads: int, num_tokens: int +) -> None: + inp = KdaDecodeInputs(num_tokens, num_heads, num_slots=max(num_tokens, 4) + 3) + + expected_out, expected_conv, expected_state = _run_triton_chain(inp) + actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp) + + # The fused kernel keeps the recurrent output in fp32 through the norm, + # while the Triton chain rounds it to BF16 in between. + torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2) + # The conv state is a pure shift-and-append: it must be bit-exact. + torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0) + torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3) + + +@requires_aiter_gluon_kda +@torch.inference_mode() +def test_aiter_packed_decode_leaves_untouched_slots_alone() -> None: + """Only the slots named by the state indices may be written.""" + num_tokens, num_heads, num_slots = 4, 12, 9 + inp = KdaDecodeInputs(num_tokens, num_heads, num_slots, seed=3) + + _, actual_conv, actual_state = _run_aiter_packed_decode(inp) + + touched = set(inp.state_indices.tolist()) + untouched = [slot for slot in range(num_slots) if slot not in touched] + assert untouched, "test needs at least one unused slot" torch.testing.assert_close( - layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0 + actual_conv[untouched], inp.conv_state[untouched], atol=0, rtol=0 + ) + torch.testing.assert_close( + actual_state[untouched], inp.recurrent_state[untouched], atol=0, rtol=0 + ) + + +@requires_aiter_gluon_kda +@torch.inference_mode() +def test_aiter_packed_decode_skips_null_block_padding() -> None: + """A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0). + + gdn_attn.py fills the tail of ``non_spec_state_indices_tensor`` with 0 and + the Triton chain leaves slot 0 alone; the fused kernel's pad_slot_guard + must do the same rather than read-modify-write slot 0 once per padded row. + """ + num_real, num_padded, num_heads = 3, 5, 12 + inp = KdaDecodeInputs(num_real + num_padded, num_heads, num_slots=12, seed=11) + inp.state_indices[num_real:] = 0 + + expected_out, expected_conv, expected_state = _run_triton_chain(inp) + actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp) + + torch.testing.assert_close( + actual_out[0, :num_real], expected_out[0, :num_real], atol=3e-2, rtol=3e-2 + ) + assert torch.isfinite(actual_out).all() + # Slot 0 must be untouched by both paths. + torch.testing.assert_close(actual_conv[0], inp.conv_state[0], atol=0, rtol=0) + torch.testing.assert_close(actual_state[0], inp.recurrent_state[0], atol=0, rtol=0) + torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0) + torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3) + + +# -------------------------------------------------------------------------- +# spec-only decode: fused_recurrent_kda with conv / gate / norm fused in +# -------------------------------------------------------------------------- + + +class SpecBatch: + """A spec-only KDA batch: ``num_spec + 1`` query tokens per request at most. + + ``seq_lens`` are the per-request query lengths, ``num_accepted`` the tokens + verified in the previous step; ``state_indices`` is the [N, num_spec + 1] + per-token slot table the GDN metadata hands the layer. + """ + + def __init__( + self, + seq_lens: list[int], + num_accepted: list[int], + num_heads: int, + num_spec: int, + seed: int = 0, + ) -> None: + assert len(seq_lens) == len(num_accepted) + assert max(seq_lens) <= num_spec + 1 + n_seqs = len(seq_lens) + slots_per_seq = num_spec + 1 + num_slots = n_seqs * slots_per_seq + 2 + self.inp = KdaDecodeInputs( + sum(seq_lens), num_heads, num_slots, seed=seed, num_spec=num_spec + ) + # Distinct slots per (request, token), shuffled, none at slot 0. + perm = torch.randperm(num_slots - 1, device="cuda")[: n_seqs * slots_per_seq] + self.state_indices = (perm + 1).to(torch.int32).view(n_seqs, slots_per_seq) + self.query_start_loc = torch.tensor( + [0] + torch.tensor(seq_lens).cumsum(0).tolist(), + dtype=torch.int32, + device="cuda", + ) + self.num_accepted_tokens = torch.tensor( + num_accepted, dtype=torch.int32, device="cuda" + ) + self.max_query_len = slots_per_seq + + +def _run_spec_triton_chain( + sb: SpecBatch, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """The layer's unfused spec path: conv update -> recurrent KDA -> norm.""" + from vllm.model_executor.layers.mamba.ops.causal_conv1d import ( + causal_conv1d_update, + ) + from vllm.models.kimi_k3.amd.ops.third_party.kda import fused_recurrent_kda + + inp = sb.inp + conv_state = inp.conv_state.clone() + recurrent_state = inp.recurrent_state.clone() + conv_out = causal_conv1d_update( + inp.mixed_qkv, + inp.conv_state_view(conv_state), + inp.conv_weights, + None, + activation="silu", + conv_state_indices=sb.state_indices[:, 0], + num_accepted_tokens=sb.num_accepted_tokens, + query_start_loc=sb.query_start_loc, + max_query_len=sb.max_query_len, + validate_data=False, + out=torch.empty_like(inp.mixed_qkv), + ) + q, k, v = ( + x.view(1, -1, inp.num_heads, HEAD_DIM) for x in conv_out.split(inp.dim, dim=-1) + ) + core_attn_out, _ = fused_recurrent_kda( + q=q, + k=k, + v=v, + raw_g=inp.g1, + raw_beta=inp.beta, + A_log=inp.A_log, + dt_bias=inp.dt_bias, + lower_bound=GATE_LOWER_BOUND, + initial_state=recurrent_state, + cu_seqlens=sb.query_start_loc, + ssm_state_indices=sb.state_indices, + num_accepted_tokens=sb.num_accepted_tokens, + ) + out = _gated_rmsnorm(core_attn_out, inp.g2, inp.norm_weight_bf16, NORM_EPS) + return out, conv_state, recurrent_state + + +def _run_aiter_spec( + sb: SpecBatch, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Exactly the call KimiK3DeltaAttention._forward makes for spec-only batches.""" + from vllm._aiter_ops import rocm_aiter_ops + + inp = sb.inp + conv_state = inp.conv_state.clone() + recurrent_state = inp.recurrent_state.clone() + q, k, v = ( + x.view(1, -1, inp.num_heads, HEAD_DIM) + for x in inp.mixed_qkv.split(inp.dim, dim=-1) + ) + n_spec = q.shape[1] + core_attn_out = inp.empty_out() + rocm_aiter_ops.kda_fused_recurrent_spec_decode( + q=q, + k=k, + v=v, + g=inp.g1, + beta=inp.beta, + A_log=inp.A_log, + dt_bias=inp.dt_bias, + lower_bound=GATE_LOWER_BOUND, + initial_state=recurrent_state, + cu_seqlens=sb.query_start_loc, + ssm_state_indices=sb.state_indices, + num_accepted_tokens=sb.num_accepted_tokens, + conv_state=inp.conv_state_view(conv_state), + conv_weight=inp.decode_conv1d_weight, + out_gate=inp.g2[:n_spec].unsqueeze(0), + norm_weight=inp.norm_weight_bf16, + norm_eps=NORM_EPS, + out=core_attn_out[:, :n_spec], + ) + return core_attn_out, conv_state, recurrent_state + + +@requires_aiter_gluon_kda +@pytest.mark.parametrize("num_heads", [12, 24]) +@pytest.mark.parametrize( + "num_spec, seq_lens, num_accepted", + [ + (3, [4, 4, 4], [1, 2, 4]), # Kimi-K3 recipe: 3 draft tokens, uniform + (3, [4, 2, 1, 4], [4, 1, 1, 3]), # ragged after partial acceptance + (6, [7, 3], [1, 4]), # longer window, two requests + ], +) +@torch.inference_mode() +def test_aiter_spec_decode_matches_triton_chain( + num_heads: int, num_spec: int, seq_lens: list[int], num_accepted: list[int] +) -> None: + sb = SpecBatch(seq_lens, num_accepted, num_heads, num_spec, seed=5) + + expected_out, expected_conv, expected_state = _run_spec_triton_chain(sb) + actual_out, actual_conv, actual_state = _run_aiter_spec(sb) + + torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2) + torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0) + torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2) + # Nothing outside the slot table may change. + touched = set(sb.state_indices.flatten().tolist()) + untouched = [s for s in range(sb.inp.conv_state.shape[0]) if s not in touched] + assert untouched + torch.testing.assert_close( + actual_conv[untouched], sb.inp.conv_state[untouched], atol=0, rtol=0 + ) + torch.testing.assert_close( + actual_state[untouched], sb.inp.recurrent_state[untouched], atol=0, rtol=0 ) - assert not layer.decode_norm_weight.requires_grad diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index ee146f2a57..73f8f4d513 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -2626,6 +2626,32 @@ class rocm_aiter_ops: """ return cls._AITER_ENABLED and cls._gdn_triton_kernels_importable() + @staticmethod + def _kda_gluon_decode_importable() -> bool: + try: + from aiter.ops.triton.attention import kda # noqa: F401 + + return True + except (ImportError, ModuleNotFoundError): + return False + + @classmethod + @if_aiter_supported + def is_kda_gluon_decode_available(cls) -> bool: + """Check if AITER's Gluon KDA decode kernels can run here. + + ``aiter.ops.triton.attention.kda`` (conv + gated delta rule + gated + RMSNorm in one launch, plain and speculative decode) ships CDNA4+ + (gfx950, gfx1250) builds only, and is absent in older aiter releases. + """ + from vllm.platforms.rocm import get_cdna_version + + return ( + cls._AITER_ENABLED + and get_cdna_version() >= 4 + and cls._kda_gluon_decode_importable() + ) + @classmethod @functools.cache def is_gdn_flydsl_prefill_available(cls) -> bool: @@ -4051,6 +4077,90 @@ class rocm_aiter_ops: output_zeros=False, ) + @staticmethod + def kda_fused_recurrent_packed_decode( + mixed_qkv: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + A_log: torch.Tensor, + dt_bias: torch.Tensor, + lower_bound: float | None, + initial_state: torch.Tensor, + ssm_state_indices: torch.Tensor, + conv_state: torch.Tensor, + conv_weight: torch.Tensor, + out_gate: torch.Tensor, + norm_weight: torch.Tensor, + norm_eps: float, + out: torch.Tensor, + ) -> torch.Tensor: + from aiter.ops.triton.attention.kda import fused_recurrent_kda_packed_decode + + return fused_recurrent_kda_packed_decode( + mixed_qkv=mixed_qkv, + g=g, + beta=beta, + A_log=A_log, + dt_bias=dt_bias, + lower_bound=lower_bound, + initial_state=initial_state, + ssm_state_indices=ssm_state_indices, + conv_state=conv_state, + conv_weight=conv_weight, + out_gate=out_gate, + norm_weight=norm_weight, + norm_eps=norm_eps, + out=out, + ) + + @staticmethod + def kda_fused_recurrent_spec_decode( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + A_log: torch.Tensor, + dt_bias: torch.Tensor, + lower_bound: float | None, + initial_state: torch.Tensor, + cu_seqlens: torch.Tensor, + ssm_state_indices: torch.Tensor, + num_accepted_tokens: torch.Tensor, + conv_state: torch.Tensor, + conv_weight: torch.Tensor, + out_gate: torch.Tensor, + norm_weight: torch.Tensor, + norm_eps: float, + out: torch.Tensor, + ) -> None: + from aiter.ops.triton.attention.kda import fused_recurrent_kda + + fused_recurrent_kda( + q=q, + k=k, + v=v, + g=g, + beta=beta, + A_log=A_log, + dt_bias=dt_bias, + lower_bound=lower_bound, + initial_state=initial_state, + cu_seqlens=cu_seqlens, + ssm_state_indices=ssm_state_indices, + num_accepted_tokens=num_accepted_tokens, + use_qk_l2norm_in_kernel=True, + use_gate_in_kernel=True, + use_beta_sigmoid_in_kernel=True, + pad_slot_guard=True, + conv_state=conv_state, + conv_weight=conv_weight, + out_gate=out_gate, + norm_weight=norm_weight, + norm_eps=norm_eps, + out=out, + ) + @staticmethod def batched_gemm_a16wfp4( X: torch.Tensor, diff --git a/vllm/models/kimi_k3/amd/kda.py b/vllm/models/kimi_k3/amd/kda.py index 96529734bb..611a0c5d96 100644 --- a/vllm/models/kimi_k3/amd/kda.py +++ b/vllm/models/kimi_k3/amd/kda.py @@ -8,6 +8,7 @@ from einops import rearrange from torch import nn from vllm import _custom_ops as ops +from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import VllmConfig from vllm.distributed import divide @@ -182,16 +183,19 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): # into one kernel, which wants a width-major fp32 conv weight staged at # load time. Everything else keeps the [channel, width] layout. conv_state_dtype, _ = self.get_state_dtype() - decode_conv1d_weight = None - if is_fused_kda_decode_supported( + self.use_aiter_decode = bool(rocm_aiter_ops.is_kda_gluon_decode_available()) + use_hip_decode = not self.use_aiter_decode and is_fused_kda_decode_supported( self.local_num_heads, self.head_dim, self.conv_size, self.num_spec, vllm_config.model_config.dtype, conv_state_dtype, - ): + ) + if use_hip_decode: logger.info_once("Fused KDA decode kernel (conv+KDA+norm) is enabled.") + decode_conv1d_weight = None + if use_hip_decode or self.use_aiter_decode: decode_conv1d_weight = torch.empty( 3, self.conv_size, @@ -257,7 +261,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid") decode_norm_weight = None - if decode_conv1d_weight is not None: + if use_hip_decode: # Upcast once at load time; a BF16 norm weight slows the fused # decode kernel's epilogue. decode_norm_weight = torch.empty( @@ -388,6 +392,66 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): if not is_conv_state_dim_first(): conv_state = conv_state.transpose(-1, -2) + if ( + self.use_aiter_decode + and spec_sequence_masks is None + and m.num_prefills == 0 + and m.num_decodes > 0 + ): + assert non_spec_state_indices_tensor is not None + rocm_aiter_ops.kda_fused_recurrent_packed_decode( + mixed_qkv=mixed_qkv, + g=g1[0], + beta=beta[0], + A_log=self.A_log, + dt_bias=self.dt_bias, + lower_bound=self.gate_lower_bound, + initial_state=recurrent_state, + ssm_state_indices=non_spec_state_indices_tensor[:num_actual_tokens], + conv_state=conv_state, + conv_weight=self.decode_conv1d_weight, + out_gate=g2[:num_actual_tokens], + norm_weight=self.o_norm.weight, + norm_eps=self.o_norm.eps, + out=core_attn_out[0, :num_actual_tokens], + ) + return + + if ( + self.use_aiter_decode + and spec_sequence_masks is not None + and m.num_prefills == 0 + ): + # Spec-only batch (gdn_attn never mixes spec and plain decodes). + assert spec_state_indices_tensor is not None + assert spec_query_start_loc is not None + q, k, v = ( + rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim) + for x in mixed_qkv.split(self.local_projection_size, dim=-1) + ) + n_spec = q.shape[1] + rocm_aiter_ops.kda_fused_recurrent_spec_decode( + q=q, + k=k, + v=v, + g=g1, + beta=beta, + A_log=self.A_log, + dt_bias=self.dt_bias, + lower_bound=self.gate_lower_bound, + initial_state=recurrent_state, + cu_seqlens=spec_query_start_loc[: m.num_spec_decodes + 1], + ssm_state_indices=spec_state_indices_tensor, + num_accepted_tokens=num_accepted_tokens, + conv_state=conv_state, + conv_weight=self.decode_conv1d_weight, + out_gate=g2[:n_spec].unsqueeze(0), + norm_weight=self.o_norm.weight, + norm_eps=self.o_norm.eps, + out=core_attn_out[:, :n_spec], + ) + return + if ( self.decode_conv1d_weight is not None and self.decode_norm_weight is not None