[Spec decode] Support variable-length decode for Kimi-K3 adaptive ver (#52988)
Signed-off-by: Albert Cheng <albecheng@nvidia.com> Signed-off-by: Albert Cheng (Engrg-Hardware 1) <albecheng@login-bia01.bia.clusters.nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Benjamin Chislett
parent
355d789f0d
commit
88aa0d287d
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import MethodType
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
@@ -61,6 +62,7 @@ def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch):
|
||||
attn_metadata.max_seq_len = 1
|
||||
attn_metadata.decode.block_table = torch.zeros(2, 1, dtype=torch.int32)
|
||||
attn_metadata.decode.seq_lens = torch.ones(2, dtype=torch.int32)
|
||||
attn_metadata.decode.max_query_len = 1
|
||||
|
||||
query = torch.ones(2, 24, 576, dtype=torch.bfloat16)
|
||||
kv_cache = torch.ones(1, 128, 576, dtype=torch.bfloat16)
|
||||
@@ -100,6 +102,9 @@ def test_flashinfer_mla_forward_uses_native_dcp_api(monkeypatch, causal):
|
||||
impl.dcp_world_size = 8
|
||||
impl.dcp_rank = 3
|
||||
impl.cp_kv_cache_interleave_size = 1
|
||||
impl._flattened_decode_metadata = MethodType(
|
||||
flashinfer_mla.FlashInferMLAImpl._flattened_decode_metadata, impl
|
||||
)
|
||||
|
||||
num_reqs, query_len = 2, 3
|
||||
num_tokens = num_reqs * query_len
|
||||
@@ -114,6 +119,11 @@ def test_flashinfer_mla_forward_uses_native_dcp_api(monkeypatch, causal):
|
||||
attn_metadata.decode.block_table = block_table
|
||||
attn_metadata.decode.seq_lens = seq_lens
|
||||
attn_metadata.decode.dcp_tot_seq_lens = global_causal_seq_lens
|
||||
attn_metadata.decode.max_query_len = query_len
|
||||
attn_metadata.decode.query_start_loc = torch.arange(
|
||||
0, num_tokens + 1, query_len, dtype=torch.int32
|
||||
)
|
||||
attn_metadata.decode.query_len = 0
|
||||
|
||||
query = torch.ones(num_tokens, 24, 576, dtype=torch.bfloat16)
|
||||
kv_cache = torch.ones(2, 128, 576, dtype=torch.bfloat16)
|
||||
|
||||
@@ -1267,6 +1267,7 @@ def test_flashmla_dcp_decode_metadata_uses_gathered_query_heads(
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
query_start_loc_device=query_start_loc,
|
||||
num_decode_tokens=2,
|
||||
max_query_len=1,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -278,6 +278,7 @@ def test_dcp_fp8_verify_build_uses_segmented(monkeypatch):
|
||||
query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
|
||||
query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
|
||||
num_decode_tokens=2 * qlen,
|
||||
max_query_len=qlen,
|
||||
dcp_tot_seq_lens_device=torch.tensor([10, 12], dtype=torch.int32),
|
||||
)
|
||||
|
||||
@@ -633,6 +634,7 @@ def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch):
|
||||
query_start_loc_cpu=torch.tensor([0, 4, 8], dtype=torch.int32),
|
||||
query_start_loc_device=torch.tensor([0, 4, 8], dtype=torch.int32),
|
||||
num_decode_tokens=8,
|
||||
max_query_len=4,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
@@ -683,6 +685,7 @@ def test_min_kv_seq_len_ignores_cudagraph_padding_rows(monkeypatch):
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
query_start_loc_device=query_start_loc,
|
||||
num_decode_tokens=num_reqs * mtp_qlen,
|
||||
max_query_len=mtp_qlen,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
@@ -733,6 +736,7 @@ def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr(
|
||||
query_start_loc_cpu=torch.tensor([0, mtp_qlen, mtp_qlen], dtype=torch.int32),
|
||||
query_start_loc_device=torch.tensor([0, mtp_qlen, mtp_qlen], dtype=torch.int32),
|
||||
num_decode_tokens=seq_lens.numel() * mtp_qlen,
|
||||
max_query_len=mtp_qlen,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
@@ -801,6 +805,7 @@ def test_decode_expands_kernel_block_page_indices(monkeypatch):
|
||||
query_start_loc_cpu=torch.tensor([0, 1, 2], dtype=torch.int32),
|
||||
query_start_loc_device=torch.tensor([0, 1, 2], dtype=torch.int32),
|
||||
num_decode_tokens=seq_lens.numel(),
|
||||
max_query_len=1,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
@@ -892,6 +897,7 @@ def test_persistent_metadata_gate(
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
query_start_loc_device=query_start_loc,
|
||||
num_decode_tokens=num_reqs * qo_len,
|
||||
max_query_len=qo_len,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
@@ -961,6 +967,7 @@ def test_persistent_metadata_gate_without_gluon_build(
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
query_start_loc_device=query_start_loc,
|
||||
num_decode_tokens=num_reqs * qo_len,
|
||||
max_query_len=qo_len,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
@@ -993,6 +1000,7 @@ def _build_non_causal(monkeypatch, *, num_heads, kv_cache_dtype, qlen, mtp_qlen)
|
||||
query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
|
||||
query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
|
||||
num_decode_tokens=2 * qlen,
|
||||
max_query_len=qlen,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
return metadata, get_mla_metadata_v1
|
||||
|
||||
@@ -34,6 +34,7 @@ def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> Non
|
||||
query_start_loc_np=torch.tensor([0, 1], dtype=torch.int32).numpy(),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
|
||||
num_scheduled_tokens=torch.tensor([1], dtype=torch.int32),
|
||||
max_query_len=None,
|
||||
seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([1537], dtype=torch.int32),
|
||||
is_prefilling_np=torch.tensor([False]).numpy(),
|
||||
|
||||
Reference in New Issue
Block a user