[Spec decode] Support variable-length decode for Kimi-K3 adaptive ver (#52988)

Signed-off-by: Albert Cheng <albecheng@nvidia.com>
Signed-off-by: Albert Cheng (Engrg-Hardware 1) <albecheng@login-bia01.bia.clusters.nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
Albert Cheng
2026-09-22 20:43:32 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Benjamin Chislett
parent 355d789f0d
commit 88aa0d287d
13 changed files with 203 additions and 22 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import MethodType
from unittest.mock import MagicMock
import pytest
@@ -61,6 +62,7 @@ def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch):
attn_metadata.max_seq_len = 1
attn_metadata.decode.block_table = torch.zeros(2, 1, dtype=torch.int32)
attn_metadata.decode.seq_lens = torch.ones(2, dtype=torch.int32)
attn_metadata.decode.max_query_len = 1
query = torch.ones(2, 24, 576, dtype=torch.bfloat16)
kv_cache = torch.ones(1, 128, 576, dtype=torch.bfloat16)
@@ -100,6 +102,9 @@ def test_flashinfer_mla_forward_uses_native_dcp_api(monkeypatch, causal):
impl.dcp_world_size = 8
impl.dcp_rank = 3
impl.cp_kv_cache_interleave_size = 1
impl._flattened_decode_metadata = MethodType(
flashinfer_mla.FlashInferMLAImpl._flattened_decode_metadata, impl
)
num_reqs, query_len = 2, 3
num_tokens = num_reqs * query_len
@@ -114,6 +119,11 @@ def test_flashinfer_mla_forward_uses_native_dcp_api(monkeypatch, causal):
attn_metadata.decode.block_table = block_table
attn_metadata.decode.seq_lens = seq_lens
attn_metadata.decode.dcp_tot_seq_lens = global_causal_seq_lens
attn_metadata.decode.max_query_len = query_len
attn_metadata.decode.query_start_loc = torch.arange(
0, num_tokens + 1, query_len, dtype=torch.int32
)
attn_metadata.decode.query_len = 0
query = torch.ones(num_tokens, 24, 576, dtype=torch.bfloat16)
kv_cache = torch.ones(2, 128, 576, dtype=torch.bfloat16)
+1
View File
@@ -1267,6 +1267,7 @@ def test_flashmla_dcp_decode_metadata_uses_gathered_query_heads(
query_start_loc_cpu=query_start_loc,
query_start_loc_device=query_start_loc,
num_decode_tokens=2,
max_query_len=1,
dcp_tot_seq_lens_device=None,
)
@@ -278,6 +278,7 @@ def test_dcp_fp8_verify_build_uses_segmented(monkeypatch):
query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
num_decode_tokens=2 * qlen,
max_query_len=qlen,
dcp_tot_seq_lens_device=torch.tensor([10, 12], dtype=torch.int32),
)
@@ -633,6 +634,7 @@ def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch):
query_start_loc_cpu=torch.tensor([0, 4, 8], dtype=torch.int32),
query_start_loc_device=torch.tensor([0, 4, 8], dtype=torch.int32),
num_decode_tokens=8,
max_query_len=4,
dcp_tot_seq_lens_device=None,
)
@@ -683,6 +685,7 @@ def test_min_kv_seq_len_ignores_cudagraph_padding_rows(monkeypatch):
query_start_loc_cpu=query_start_loc,
query_start_loc_device=query_start_loc,
num_decode_tokens=num_reqs * mtp_qlen,
max_query_len=mtp_qlen,
dcp_tot_seq_lens_device=None,
)
@@ -733,6 +736,7 @@ def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr(
query_start_loc_cpu=torch.tensor([0, mtp_qlen, mtp_qlen], dtype=torch.int32),
query_start_loc_device=torch.tensor([0, mtp_qlen, mtp_qlen], dtype=torch.int32),
num_decode_tokens=seq_lens.numel() * mtp_qlen,
max_query_len=mtp_qlen,
dcp_tot_seq_lens_device=None,
)
@@ -801,6 +805,7 @@ def test_decode_expands_kernel_block_page_indices(monkeypatch):
query_start_loc_cpu=torch.tensor([0, 1, 2], dtype=torch.int32),
query_start_loc_device=torch.tensor([0, 1, 2], dtype=torch.int32),
num_decode_tokens=seq_lens.numel(),
max_query_len=1,
dcp_tot_seq_lens_device=None,
)
@@ -892,6 +897,7 @@ def test_persistent_metadata_gate(
query_start_loc_cpu=query_start_loc,
query_start_loc_device=query_start_loc,
num_decode_tokens=num_reqs * qo_len,
max_query_len=qo_len,
dcp_tot_seq_lens_device=None,
)
@@ -961,6 +967,7 @@ def test_persistent_metadata_gate_without_gluon_build(
query_start_loc_cpu=query_start_loc,
query_start_loc_device=query_start_loc,
num_decode_tokens=num_reqs * qo_len,
max_query_len=qo_len,
dcp_tot_seq_lens_device=None,
)
@@ -993,6 +1000,7 @@ def _build_non_causal(monkeypatch, *, num_heads, kv_cache_dtype, qlen, mtp_qlen)
query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
num_decode_tokens=2 * qlen,
max_query_len=qlen,
dcp_tot_seq_lens_device=None,
)
return metadata, get_mla_metadata_v1
@@ -34,6 +34,7 @@ def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> Non
query_start_loc_np=torch.tensor([0, 1], dtype=torch.int32).numpy(),
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
num_scheduled_tokens=torch.tensor([1], dtype=torch.int32),
max_query_len=None,
seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32),
seq_lens=torch.tensor([1537], dtype=torch.int32),
is_prefilling_np=torch.tensor([False]).numpy(),