[ROCm][Kimi-K3] Support DSpark adaptive verification on MI355X (#59824)
Signed-off-by: Oxana Korzh <okorzh@amd.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
307c5844ad
commit
5bf3d9f665
@@ -51,6 +51,7 @@ def _run(block_table, cu_num_tokens, kernel_block_size, out_len, device):
|
||||
block_table.stride(0),
|
||||
block_table.stride(1),
|
||||
cu_num_tokens,
|
||||
cu_num_tokens[1:] - cu_num_tokens[:-1],
|
||||
KERNEL_BLOCK_SIZE=kernel_block_size,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
@@ -109,6 +110,7 @@ def test_ragged_batch_leaves_other_rows_untouched():
|
||||
bt.stride(0),
|
||||
bt.stride(1),
|
||||
cu,
|
||||
cu[1:] - cu[:-1],
|
||||
KERNEL_BLOCK_SIZE=kbs,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
@@ -153,6 +155,7 @@ def test_over_provisioned_chunks_do_not_change_result():
|
||||
bt.stride(0),
|
||||
bt.stride(1),
|
||||
cu,
|
||||
cu[1:] - cu[:-1],
|
||||
KERNEL_BLOCK_SIZE=kbs,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
|
||||
@@ -418,6 +418,11 @@ for backend in BACKENDS_TO_TEST:
|
||||
if query_len_support != QueryLenSupport.SINGLE_ONLY:
|
||||
SPEC_DECODE_BACKENDS.append(backend)
|
||||
|
||||
# Backends whose decode pipeline replays device-trimmed verify batches.
|
||||
ADAPTIVE_VARLEN_BACKENDS = [
|
||||
b for b in BACKENDS_TO_TEST if b == AttentionBackendEnum.ROCM_AITER_MLA
|
||||
]
|
||||
|
||||
BACKEND_BLOCK_SIZES = {}
|
||||
for backend in BACKENDS_TO_TEST:
|
||||
supported_sizes = backend.get_class().get_supported_kernel_block_sizes()
|
||||
@@ -470,6 +475,10 @@ BATCH_SPECS = {
|
||||
"spec_decode_small": BatchSpec(
|
||||
seq_lens=[128, 256, 512, 1024], query_lens=[4, 4, 4, 4]
|
||||
),
|
||||
# Adaptive verification: verify requests trimmed to 1..k+1 tokens on device.
|
||||
"spec_decode_ragged_small": BatchSpec(
|
||||
seq_lens=[128, 256, 512, 1024], query_lens=[8, 3, 1, 5]
|
||||
),
|
||||
"spec_decode_medium": BatchSpec(
|
||||
seq_lens=[512, 1024, 2048, 512, 1024, 2048], query_lens=[8, 8, 8, 8, 8, 8]
|
||||
),
|
||||
@@ -1581,6 +1590,34 @@ def run_attention_backend(
|
||||
return output
|
||||
|
||||
|
||||
def _with_even_cpu_query_split(
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
) -> CommonAttentionMetadata:
|
||||
"""Host query lengths as adaptive verification leaves them.
|
||||
|
||||
The device keeps the trimmed per-request lengths; the host sees the same
|
||||
total spread evenly, the first requests taking the remainder
|
||||
(AdaptiveVerificationManager.compact_batch).
|
||||
"""
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
total = int(common_attn_metadata.query_start_loc_cpu[num_reqs])
|
||||
cpu_lens = torch.full((num_reqs,), total // num_reqs, dtype=torch.int32)
|
||||
cpu_lens[: total % num_reqs] += 1
|
||||
query_start_loc_cpu = torch.zeros_like(common_attn_metadata.query_start_loc_cpu)
|
||||
torch.cumsum(cpu_lens, dim=0, out=query_start_loc_cpu[1 : num_reqs + 1])
|
||||
return common_attn_metadata.replace(
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=int(
|
||||
(
|
||||
common_attn_metadata.query_start_loc[1:]
|
||||
- common_attn_metadata.query_start_loc[:-1]
|
||||
)
|
||||
.max()
|
||||
.item()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _run_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
@@ -1595,6 +1632,7 @@ def _run_backend_correctness(
|
||||
qk_nope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
chunked_prefill_workspace_size: int | None = None,
|
||||
adaptive_verification: bool = False,
|
||||
):
|
||||
"""Test that all backends produce similar outputs to a reference implementation
|
||||
using torch.nn.functional.scaled_dot_product_attention.
|
||||
@@ -1626,6 +1664,10 @@ def _run_backend_correctness(
|
||||
) and AttentionBackendEnum.CUTLASS_MLA in backends_to_test:
|
||||
# CUTLASS_MLA does not support non-1 Q/K scales
|
||||
backends_to_test.remove(AttentionBackendEnum.CUTLASS_MLA)
|
||||
if adaptive_verification:
|
||||
backends_to_test = [
|
||||
b for b in backends_to_test if b in ADAPTIVE_VARLEN_BACKENDS
|
||||
]
|
||||
if not backends_to_test:
|
||||
pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}")
|
||||
|
||||
@@ -1675,13 +1717,16 @@ def _run_backend_correctness(
|
||||
if is_spec_decode_test:
|
||||
from vllm.config import SpeculativeConfig
|
||||
|
||||
# Get the query length from the batch spec (they should all be uniform)
|
||||
query_len = batch_spec.query_lens[0]
|
||||
# Get the query length from the batch spec (they should all be uniform,
|
||||
# except under adaptive verification, where the longest is k+1)
|
||||
query_len = max(batch_spec.query_lens)
|
||||
# Set num_speculative_tokens to query_len - 1
|
||||
# (since threshold is 1 + num_spec_tokens)
|
||||
# Use ngram method which doesn't require a draft model
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=query_len - 1
|
||||
method="ngram",
|
||||
num_speculative_tokens=query_len - 1,
|
||||
enable_adaptive_verification=adaptive_verification,
|
||||
)
|
||||
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
@@ -1951,6 +1996,13 @@ def _run_backend_correctness(
|
||||
)
|
||||
kv_cache_per_block_size[block_size] = kv_cache
|
||||
|
||||
if adaptive_verification:
|
||||
# Only after the cache is filled: prepopulation reads the context
|
||||
# lengths off the host query lengths.
|
||||
metadata_per_block_size[block_size] = _with_even_cpu_query_split(
|
||||
common_attn_metadata
|
||||
)
|
||||
|
||||
# 4. Run vLLM backends and compare
|
||||
rtol = 1e-2
|
||||
atol = {
|
||||
@@ -2136,3 +2188,29 @@ def test_chunked_context_backend_correctness(
|
||||
v_head_dim=v_head_dim,
|
||||
chunked_prefill_workspace_size=1024,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not ADAPTIVE_VARLEN_BACKENDS, reason="No adaptive varlen decode backend"
|
||||
)
|
||||
def test_adaptive_varlen_decode_correctness(
|
||||
default_vllm_config, dist_init, workspace_init
|
||||
):
|
||||
"""Device query lengths 1..k+1 against an even host split, as adaptive
|
||||
verification hands them to the target model's attention. TP 8 gives 16
|
||||
heads, the padded Kimi-K3 count."""
|
||||
_run_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
"spec_decode_ragged_small",
|
||||
"deepseek-ai/DeepSeek-R1",
|
||||
8,
|
||||
"fp8",
|
||||
1.0,
|
||||
1.0,
|
||||
None,
|
||||
128,
|
||||
128,
|
||||
adaptive_verification=True,
|
||||
)
|
||||
|
||||
@@ -63,9 +63,11 @@ class _ExpandPageIndicesKernel:
|
||||
stride_0,
|
||||
stride_1,
|
||||
paged_kv_indptr,
|
||||
num_real_tokens,
|
||||
*,
|
||||
KERNEL_BLOCK_SIZE,
|
||||
BLOCK_SIZE,
|
||||
CLAMP_TAIL=False,
|
||||
):
|
||||
# Both block-table strides are passed explicitly: the kernel no longer
|
||||
# assumes the column stride is 1. This stand-in indexes the tensor
|
||||
@@ -145,6 +147,7 @@ def _builder(
|
||||
)
|
||||
),
|
||||
_mtp_decode_qlen=mtp_decode_qlen,
|
||||
_varlen_decode_qlen=0,
|
||||
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
|
||||
_use_persistent_metadata=False,
|
||||
kernel_block_size=kernel_block_size,
|
||||
@@ -500,6 +503,7 @@ def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch):
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.scale = 576**-0.5
|
||||
decode = SimpleNamespace(
|
||||
varlen_q_rows=None,
|
||||
max_qo_len=4,
|
||||
qo_indptr=torch.tensor([0, 4], dtype=torch.int32),
|
||||
paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32),
|
||||
@@ -563,6 +567,7 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.scale = head_dim**-0.5
|
||||
decode = SimpleNamespace(
|
||||
varlen_q_rows=None,
|
||||
max_qo_len=1,
|
||||
qo_indptr=torch.arange(num_tokens + 1, dtype=torch.int32),
|
||||
paged_kv_indptr=torch.zeros(num_tokens + 1, dtype=torch.int32),
|
||||
@@ -787,6 +792,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
|
||||
torch.tensor([global_seq_len], dtype=torch.int32, device=device),
|
||||
)
|
||||
decode = SimpleNamespace(
|
||||
varlen_q_rows=None,
|
||||
max_qo_len=qlen,
|
||||
paged_kv_indptr=torch.tensor(
|
||||
[0, local_kv.shape[0]], dtype=torch.int32, device=device
|
||||
@@ -1452,3 +1458,29 @@ def test_a_two_token_bf16_block_is_allowed(monkeypatch):
|
||||
monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=2, mtp_qlen=8
|
||||
)
|
||||
assert metadata.has_persistent_metadata
|
||||
|
||||
|
||||
def _adaptive_config(
|
||||
*, adaptive: bool = True, dcp: int = 1, num_speculative_tokens: int = 7
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
enable_adaptive_verification=adaptive,
|
||||
parallel_drafting=False,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp),
|
||||
use_v2_model_runner=True,
|
||||
)
|
||||
|
||||
|
||||
def test_varlen_cudagraph_bound_follows_adaptive_verification():
|
||||
bound = AiterMLAMetadataBuilder.get_varlen_cudagraph_max_query_len
|
||||
assert bound(_adaptive_config(), SimpleNamespace()) == 8
|
||||
assert bound(_adaptive_config(adaptive=False), SimpleNamespace()) is None
|
||||
# DCP routes plan per-row windows from host lengths.
|
||||
assert bound(_adaptive_config(dcp=2), SimpleNamespace()) is None
|
||||
# Kimi-K3 DSpark draft layers share the target's group; causality is
|
||||
# decided per batch, so the merged non-causal spec keeps the bound.
|
||||
shared_draft_group = SimpleNamespace(non_causal_multi_token_decode=True)
|
||||
assert bound(_adaptive_config(), shared_draft_group) == 8
|
||||
|
||||
Reference in New Issue
Block a user