diff --git a/tests/models/kimi_k3/test_kda.py b/tests/models/kimi_k3/test_kda.py index f03d2cdcae..eaf4b69b40 100644 --- a/tests/models/kimi_k3/test_kda.py +++ b/tests/models/kimi_k3/test_kda.py @@ -22,6 +22,9 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_upd from vllm.model_executor.layers.mamba.ops.gather_initial_states import ( gather_initial_states, ) +from vllm.models.kimi_k3.amd.ops.third_party.kda import ( + fused_recurrent_kda as fused_recurrent_kda_amd, +) from vllm.models.kimi_k3.amd.ops.third_party.kda import ( fused_recurrent_kda_packed_decode as fused_recurrent_kda_packed_decode_amd, ) @@ -44,10 +47,12 @@ from vllm.models.kimi_k3.nvidia.ops.third_party.kda import ( chunk_kda, chunk_kda_with_fused_gate, fused_kda_gate, - fused_recurrent_kda, fused_recurrent_kda_fwd, fused_recurrent_kda_packed_decode, ) +from vllm.models.kimi_k3.nvidia.ops.third_party.kda import ( + fused_recurrent_kda as fused_recurrent_kda_nvidia, +) from vllm.platforms import current_platform from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd from vllm.utils.flashinfer import flashinfer_fused_kda_decode @@ -66,6 +71,10 @@ PACKED_DECODE_IMPLS = { "nvidia": fused_recurrent_kda_packed_decode, "amd": fused_recurrent_kda_packed_decode_amd, } +SPEC_DECODE_IMPLS = { + "nvidia": fused_recurrent_kda_nvidia, + "amd": fused_recurrent_kda_amd, +} def test_kda_warmup_skips_missing_metadata(monkeypatch): @@ -468,17 +477,51 @@ def test_packed_kda_decode_correctness( @pytest.mark.parametrize( - ("H", "fuse_gate"), - [(12, True), (12, False), (12, None), (96, None)], + ("impl", "H", "fuse_gate", "num_seqs", "query_len"), + [ + *[ + pytest.param( + impl, + H, + fuse_gate, + 3, + 3, + id=f"{impl}-H{H}-fuse-{fuse_gate}", + ) + for impl in SPEC_DECODE_IMPLS + for H, fuse_gate in [ + (12, True), + (12, False), + (12, None), + (96, None), + ] + ], + pytest.param("amd", 12, True, 1, 1, id="amd-single-token"), + pytest.param("amd", 12, True, 1, 8, id="amd-single-sequence-short"), + pytest.param( + "amd", + 96, + None, + 1, + 8, + id="amd-single-sequence-many-heads", + ), + pytest.param("amd", 12, True, 2, 4, id="amd-uniform-two-sequences"), + pytest.param("amd", 12, True, 4, 4, id="amd-uniform-four-sequences"), + pytest.param("amd", 12, True, 8, 7, id="amd-uniform-eight-sequences"), + ], ) @pytest.mark.parametrize("lower_bound", [-5.0, None]) @torch.inference_mode() def test_kda_spec_decode_correctness( H: int, fuse_gate: bool | None, + num_seqs: int, + query_len: int, lower_bound: float | None, + impl: str, ): - num_seqs, query_len, D = 3, 3, 128 + D = 128 T = num_seqs * query_len torch.manual_seed(1234) @@ -522,10 +565,8 @@ def test_kda_spec_decode_correctness( dtype=torch.int32, device=DEVICE, ).view(num_seqs, query_len) - num_accepted_tokens = torch.tensor( - [1, 2, 3], - dtype=torch.int32, - device=DEVICE, + num_accepted_tokens = ( + torch.arange(num_seqs, dtype=torch.int32, device=DEVICE) % query_len + 1 ) state_storage = 0.01 * torch.randn( T + 1, @@ -578,7 +619,12 @@ def test_kda_spec_decode_correctness( expected = torch.cat(expected_outputs, dim=1) actual_state = state.clone() - actual, _ = fused_recurrent_kda( + extra_args = ( + {"uniform_sequence_length": query_len} + if impl == "amd" and num_seqs in (2, 4, 8) + else {} + ) + actual, _ = SPEC_DECODE_IMPLS[impl]( q=q, k=k, v=v, @@ -593,6 +639,7 @@ def test_kda_spec_decode_correctness( num_accepted_tokens=num_accepted_tokens, out=output, fuse_gate=fuse_gate, + **extra_args, ) assert actual.data_ptr() == output.data_ptr() diff --git a/tests/models/kimi_k3/test_kda_metadata.py b/tests/models/kimi_k3/test_kda_metadata.py index fa810217b1..6268cb9bcc 100644 --- a/tests/models/kimi_k3/test_kda_metadata.py +++ b/tests/models/kimi_k3/test_kda_metadata.py @@ -49,6 +49,7 @@ PRUNED_METADATA_FIELDS = { "prefill_state_indices", "prefill_has_initial_state", "spec_sequence_masks", + "uniform_spec_sequence_length", "flashinfer_prefill_query_start_loc", "flashinfer_prefill_seq_order", } @@ -409,6 +410,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset(): "num_speculative_tokens", "full_cuda_graph", "is_prefilling", + "expected_uniform_spec_sequence_length", ), [ pytest.param( @@ -417,6 +419,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset(): 2, False, [False, False], + 3, id="pure-spec-decode", ), pytest.param( @@ -425,6 +428,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset(): 2, False, [True, False, False], + 3, id="mixed-prefill-and-spec-decode", ), pytest.param( @@ -433,6 +437,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset(): 0, False, [False, False], + None, id="regular-decode", ), pytest.param( @@ -441,8 +446,18 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset(): 2, False, [False, False], + None, id="no-scheduled-draft-tokens", ), + pytest.param( + BatchSpec(seq_lens=[50, 30], query_lens=[3, 2]), + [2, 1], + 2, + False, + [False, False], + None, + id="ragged-spec-decode", + ), ], ) def test_kimi_k3_kda_metadata_matches_shared_gdn( @@ -451,6 +466,7 @@ def test_kimi_k3_kda_metadata_matches_shared_gdn( num_speculative_tokens: int, full_cuda_graph: bool, is_prefilling: list[bool], + expected_uniform_spec_sequence_length: int | None, ): kwargs: dict[str, torch.Tensor] = {} if num_decode_draft_tokens is not None: @@ -483,6 +499,9 @@ def test_kimi_k3_kda_metadata_matches_shared_gdn( assert isinstance(actual, KimiK3KDAMetadata) _assert_matches_shared_gdn(reference, actual) + assert ( + reference.uniform_spec_sequence_length == expected_uniform_spec_sequence_length + ) def test_mixed_regular_and_spec_decode_uses_packed_decode_metadata(): diff --git a/vllm/models/kimi_k3/amd/kda.py b/vllm/models/kimi_k3/amd/kda.py index ed631aea7e..d98d16521f 100644 --- a/vllm/models/kimi_k3/amd/kda.py +++ b/vllm/models/kimi_k3/amd/kda.py @@ -438,6 +438,14 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): for x in mixed_qkv_spec.split(self.local_projection_size, dim=-1) ) spec_cu_seqlens = spec_query_start_loc[: m.num_spec_decodes + 1] + uniform_sequence_length = m.uniform_spec_sequence_length + # Token padding can make the packed tensor larger than the + # request-local metadata represented by this length. + if ( + uniform_sequence_length is not None + and q_spec.shape[1] != m.num_spec_decodes * uniform_sequence_length + ): + uniform_sequence_length = None # Spec-only batches write directly into core_attn_out. spec_out = ( core_attn_out[:, : q_spec.shape[1]] @@ -457,6 +465,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): cu_seqlens=spec_cu_seqlens, ssm_state_indices=spec_state_indices_tensor, num_accepted_tokens=num_accepted_tokens, + uniform_sequence_length=uniform_sequence_length, out=spec_out, ) diff --git a/vllm/models/kimi_k3/amd/ops/third_party/kda/__init__.py b/vllm/models/kimi_k3/amd/ops/third_party/kda/__init__.py index 77ed09d489..15f446e2eb 100644 --- a/vllm/models/kimi_k3/amd/ops/third_party/kda/__init__.py +++ b/vllm/models/kimi_k3/amd/ops/third_party/kda/__init__.py @@ -16,9 +16,10 @@ # - OOB-mask correctness fix: present (all tl.load use mask=..., other=0). # Validated on gfx950: no core-dump, gsm8k 94.1%. # -# AMD-specific deltas vs the NVIDIA copy: NONE yet (byte-identical). Keep in sync -# with the NVIDIA copy on FLA updates; any divergence should be an intentional, -# documented gfx950-specific change (a #869-style AMD-only fix). +# AMD-specific deltas vs the NVIDIA copy: +# - fused recurrent gated speculative-decode launch geometry is tuned for +# gfx950. +# Keep the copies in sync on FLA updates outside documented platform tuning. from .chunk import ( chunk_kda, diff --git a/vllm/models/kimi_k3/amd/ops/third_party/kda/fused_recurrent.py b/vllm/models/kimi_k3/amd/ops/third_party/kda/fused_recurrent.py index db519fb6f0..9db2912458 100644 --- a/vllm/models/kimi_k3/amd/ops/third_party/kda/fused_recurrent.py +++ b/vllm/models/kimi_k3/amd/ops/third_party/kda/fused_recurrent.py @@ -164,15 +164,24 @@ def fused_recurrent_kda_fwd_kernel( APPLY_BETA_SIGMOID: tl.constexpr, HAS_DT_BIAS: tl.constexpr, USE_LOWER_BOUND: tl.constexpr, + IS_SINGLE_SEQUENCE: tl.constexpr, + SEQUENCE_LENGTH: tl.constexpr, num_stages: tl.constexpr, ): pid = tl.program_id(0) i_v = pid % tl.cdiv(V, BV) i_nh = pid // tl.cdiv(V, BV) - i_n, i_h = i_nh // H, i_nh % H - bos = tl.load(cu_seqlens + i_n).to(tl.int64) - eos = tl.load(cu_seqlens + i_n + 1).to(tl.int64) - sequence_length = eos - bos + if IS_SINGLE_SEQUENCE: + i_n, i_h = 0, i_nh + else: + i_n, i_h = i_nh // H, i_nh % H + if SEQUENCE_LENGTH: + bos = i_n * SEQUENCE_LENGTH + sequence_length = SEQUENCE_LENGTH + else: + bos = tl.load(cu_seqlens + i_n).to(tl.int64) + eos = tl.load(cu_seqlens + i_n + 1).to(tl.int64) + sequence_length = eos - bos if sequence_length == 0: return @@ -208,6 +217,14 @@ def fused_recurrent_kda_fwd_kernel( p_v = v + bos * stride_qkv_token + i_h * V + o_v p_g = g + bos * stride_g_token + i_h * K + o_k p_beta = beta + bos * stride_beta_token + i_h + if USE_GATE_IN_KERNEL: + b_a = exp(tl.load(A_log + i_h).to(tl.float32)) + if HAS_DT_BIAS: + b_bias = tl.load( + dt_bias + i_h * K + o_k, + mask=m_k, + other=0.0, + ).to(tl.float32) for i_t in tl.range(0, sequence_length, num_stages=num_stages): b_q = tl.load(p_q, mask=m_k, other=0.0, eviction_policy="evict_last").to( tl.float32 @@ -219,8 +236,8 @@ def fused_recurrent_kda_fwd_kernel( tl.float32 ) if USE_QK_L2NORM_IN_KERNEL: - b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6) - b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6) + b_q *= tl.rsqrt(tl.sum(b_q * b_q) + 1e-6) + b_k *= tl.rsqrt(tl.sum(b_k * b_k) + 1e-6) b_q *= scale b_gate = tl.load( @@ -231,13 +248,7 @@ def fused_recurrent_kda_fwd_kernel( ).to(tl.float32) if USE_GATE_IN_KERNEL: if HAS_DT_BIAS: - b_bias = tl.load( - dt_bias + i_h * K + o_k, - mask=m_k, - other=0.0, - ).to(tl.float32) b_gate += b_bias - b_a = exp(tl.load(A_log + i_h).to(tl.float32)) if USE_LOWER_BOUND: b_gate = lower_bound * tl.sigmoid(b_a * b_gate) else: @@ -288,6 +299,60 @@ def fused_recurrent_kda_fwd_kernel( p_out += stride_out_token +def _get_static_sequence_length( + total_tokens: int, + head_sequences: int, + is_single_sequence: bool, + uniform_sequence_length: int | None, +) -> int: + if is_single_sequence: + return total_tokens + # MI355X performance gate: 192 head-sequences is 16 sequences at H=12. + # Static loops win through 16; dynamic loops win at 32 for short queries. + if ( + uniform_sequence_length is not None + and uniform_sequence_length > 1 + and head_sequences <= 192 + ): + return uniform_sequence_length + return 0 + + +def _select_kda_launch_config( + use_gate_in_kernel: bool, + is_spec_decoding: bool, + head_sequences: int, + static_sequence_length: int, + has_multi_token_sequence: bool, +) -> tuple[int, int, int]: + if static_sequence_length: + BV = 4 if head_sequences <= 24 else 8 + if static_sequence_length == 1: + num_stages = 2 + elif static_sequence_length <= 3: + num_stages = 8 if head_sequences <= 24 else 4 + elif static_sequence_length <= 6: + num_stages = 6 + elif static_sequence_length <= 10: + num_stages = 10 + else: + num_stages = 12 + return BV, 1, num_stages + + if use_gate_in_kernel and is_spec_decoding: + if head_sequences <= 24: + return 4, 1, 4 if has_multi_token_sequence else 2 + if has_multi_token_sequence and head_sequences <= 48: + return 8, 1, 4 + if has_multi_token_sequence: + return 8, 1, 2 + return 32, 4, 2 + + if use_gate_in_kernel: + return 32, 4, 2 + return 8, 1, 2 + + def fused_recurrent_kda_fwd( q: torch.Tensor, k: torch.Tensor, @@ -300,6 +365,7 @@ def fused_recurrent_kda_fwd( cu_seqlens: torch.Tensor | None = None, ssm_state_indices: torch.Tensor | None = None, num_accepted_tokens: torch.Tensor | None = None, + uniform_sequence_length: int | None = None, use_qk_l2norm_in_kernel: bool = True, A_log: torch.Tensor | None = None, dt_bias: torch.Tensor | None = None, @@ -336,6 +402,10 @@ def fused_recurrent_kda_fwd( assert num_accepted_tokens is None else: assert ssm_state_indices.stride(1) == 1 + if uniform_sequence_length is not None: + assert num_accepted_tokens is not None + assert uniform_sequence_length > 0 + assert N * uniform_sequence_length == T assert cu_seqlens.is_contiguous() if use_gate_in_kernel: assert A_log is not None and A_log.is_contiguous() @@ -344,8 +414,22 @@ def fused_recurrent_kda_fwd( if scale is None: scale = K**-0.5 - BV = 32 if use_gate_in_kernel else 8 - num_warps = 4 if use_gate_in_kernel else 1 + is_spec_decoding = num_accepted_tokens is not None + is_single_sequence = is_spec_decoding and N == 1 + head_sequences = H * N + static_sequence_length = _get_static_sequence_length( + total_tokens=T, + head_sequences=head_sequences, + is_single_sequence=is_single_sequence, + uniform_sequence_length=uniform_sequence_length, + ) + BV, num_warps, num_stages = _select_kda_launch_config( + use_gate_in_kernel=use_gate_in_kernel, + is_spec_decoding=is_spec_decoding, + head_sequences=head_sequences, + static_sequence_length=static_sequence_length, + has_multi_token_sequence=T > N, + ) grid = (cdiv(V, BV) * N * H,) fused_recurrent_kda_fwd_kernel[grid]( q=q, @@ -379,8 +463,10 @@ def fused_recurrent_kda_fwd( USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, USE_GATE_IN_KERNEL=use_gate_in_kernel, APPLY_BETA_SIGMOID=use_beta_sigmoid_in_kernel, + IS_SINGLE_SEQUENCE=is_single_sequence, + SEQUENCE_LENGTH=static_sequence_length, num_warps=num_warps, - num_stages=2, + num_stages=num_stages, ) return out, initial_state @@ -398,6 +484,7 @@ def fused_recurrent_kda( cu_seqlens: torch.Tensor, ssm_state_indices: torch.Tensor, num_accepted_tokens: torch.Tensor | None = None, + uniform_sequence_length: int | None = None, out: torch.Tensor | None = None, fuse_gate: bool | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: @@ -433,6 +520,7 @@ def fused_recurrent_kda( cu_seqlens=cu_seqlens, ssm_state_indices=ssm_state_indices, num_accepted_tokens=num_accepted_tokens, + uniform_sequence_length=uniform_sequence_length, use_qk_l2norm_in_kernel=True, A_log=A_log if fuse_gate else None, dt_bias=dt_bias if fuse_gate else None, diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index c2cfd003d9..85173dc2e4 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -64,6 +64,7 @@ class GDNAttentionMetadata: non_spec_token_indx: torch.Tensor | None = None num_accepted_tokens: torch.Tensor | None = None # shape: [batch,] + uniform_spec_sequence_length: int | None = None # None for ragged batches # Pre-computed FLA chunk metadata (avoids GPU->CPU sync in prepare_chunk_indices) chunk_indices: torch.Tensor | None = None @@ -226,6 +227,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] self.vllm_config.cache_config.mamba_cache_mode, ) + uniform_spec_sequence_length = None spec_sequence_masks_cpu: torch.Tensor | None = None if not self.use_spec_decode or num_decode_draft_tokens_cpu is None: spec_sequence_masks = None @@ -289,6 +291,13 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] assert spec_sequence_masks_cpu is not None non_spec_sequence_masks_cpu = ~spec_sequence_masks_cpu query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] + spec_query_lens_cpu = query_lens_cpu[spec_sequence_masks_cpu] + if spec_query_lens_cpu.numel() > 0: + first_spec_sequence_length = int(spec_query_lens_cpu[0]) + if first_spec_sequence_length > 0 and bool( + torch.all(spec_query_lens_cpu == first_spec_sequence_length) + ): + uniform_spec_sequence_length = first_spec_sequence_length # Use CPU tensors to avoid CPU-GPU sync non_spec_query_lens_cpu = query_lens_cpu[non_spec_sequence_masks_cpu] @@ -540,6 +549,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] spec_token_indx=spec_token_indx, non_spec_token_indx=non_spec_token_indx, num_accepted_tokens=num_accepted_tokens, + uniform_spec_sequence_length=uniform_spec_sequence_length, nums_dict=nums_dict, batch_ptr=batch_ptr, token_chunk_offset_ptr=token_chunk_offset_ptr,