[ROCm][Kimi-K3] Support DSpark adaptive verification on MI355X (#59824)

Signed-off-by: Oxana Korzh <okorzh@amd.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oxana Korzh
2026-10-16 13:53:49 -05:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 307c5844ad
commit 5bf3d9f665
6 changed files with 383 additions and 13 deletions
@@ -0,0 +1,93 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm Kimi-K3 KDA metadata under adaptive verification.
Adaptive verification splits the draft budget evenly on the host, then trims
each verify request on device. The KDA spec path must therefore plan from the
device offsets, never from host lengths that merely look uniform.
"""
import torch
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import SpeculativeConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.models.kimi_k3.amd.kda_metadata import (
KimiK3ROCmKDABackend,
KimiK3ROCmKDAMetadataBuilder,
)
from vllm.v1.kv_cache_interface import MambaSpec
BLOCK_SIZE = 16
DEVICE = torch.device("cpu")
NUM_SPEC = 3
def _config(adaptive: bool):
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
block_size=BLOCK_SIZE,
)
vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
vllm_config.speculative_config = SpeculativeConfig(
method="ngram",
num_speculative_tokens=NUM_SPEC,
enable_adaptive_verification=adaptive,
)
vllm_config.cache_config.mamba_cache_mode = "none"
return vllm_config
def _spec() -> MambaSpec:
return MambaSpec(
block_size=BLOCK_SIZE,
shapes=((16, 64),),
dtypes=(torch.float16,),
num_speculative_blocks=NUM_SPEC,
)
def _builder(adaptive: bool) -> KimiK3ROCmKDAMetadataBuilder:
return KimiK3ROCmKDAMetadataBuilder(
kv_cache_spec=_spec(),
layer_names=["layer.0"],
vllm_config=_config(adaptive),
device=DEVICE,
)
def _trimmed_verify_batch(builder: KimiK3ROCmKDAMetadataBuilder):
"""Host sees [3, 3]; the device was trimmed to [4, 2] (same total)."""
batch = BatchSpec(seq_lens=[64, 64], query_lens=[4, 2])
common = create_common_attn_metadata(
batch, BLOCK_SIZE, DEVICE, arange_block_indices=True
)
common = common.replace(
query_start_loc_cpu=torch.tensor([0, 3, 6], dtype=torch.int32)
)
accepted = torch.ones(2, dtype=torch.int32, device=DEVICE)
num_decode_draft_tokens_cpu = torch.tensor([NUM_SPEC, NUM_SPEC], dtype=torch.int32)
return builder.build(0, common, accepted, num_decode_draft_tokens_cpu)
def test_adaptive_verification_plans_from_device_offsets():
"""Equal host lengths would select the fixed-length recurrent kernel,
which places sequence i at i * L and ignores the device cu_seqlens."""
assert KimiK3ROCmKDABackend.supports_device_cpu_query_lens_mismatch()
bound = KimiK3ROCmKDAMetadataBuilder.get_varlen_cudagraph_max_query_len
assert bound(_config(adaptive=True), _spec()) == NUM_SPEC + 1
assert bound(_config(adaptive=False), _spec()) is None
meta = _trimmed_verify_batch(_builder(adaptive=True))
assert meta.num_spec_decodes == 2
assert meta.uniform_spec_sequence_length is None
assert torch.equal(
meta.spec_query_start_loc.cpu(), torch.tensor([0, 4, 6], dtype=torch.int32)
)
# Without adaptive verification the host lengths are exact and kept.
meta = _trimmed_verify_batch(_builder(adaptive=False))
assert meta.uniform_spec_sequence_length == 3
@@ -51,6 +51,7 @@ def _run(block_table, cu_num_tokens, kernel_block_size, out_len, device):
block_table.stride(0),
block_table.stride(1),
cu_num_tokens,
cu_num_tokens[1:] - cu_num_tokens[:-1],
KERNEL_BLOCK_SIZE=kernel_block_size,
BLOCK_SIZE=1024,
)
@@ -109,6 +110,7 @@ def test_ragged_batch_leaves_other_rows_untouched():
bt.stride(0),
bt.stride(1),
cu,
cu[1:] - cu[:-1],
KERNEL_BLOCK_SIZE=kbs,
BLOCK_SIZE=1024,
)
@@ -153,6 +155,7 @@ def test_over_provisioned_chunks_do_not_change_result():
bt.stride(0),
bt.stride(1),
cu,
cu[1:] - cu[:-1],
KERNEL_BLOCK_SIZE=kbs,
BLOCK_SIZE=1024,
)
+81 -3
View File
@@ -418,6 +418,11 @@ for backend in BACKENDS_TO_TEST:
if query_len_support != QueryLenSupport.SINGLE_ONLY:
SPEC_DECODE_BACKENDS.append(backend)
# Backends whose decode pipeline replays device-trimmed verify batches.
ADAPTIVE_VARLEN_BACKENDS = [
b for b in BACKENDS_TO_TEST if b == AttentionBackendEnum.ROCM_AITER_MLA
]
BACKEND_BLOCK_SIZES = {}
for backend in BACKENDS_TO_TEST:
supported_sizes = backend.get_class().get_supported_kernel_block_sizes()
@@ -470,6 +475,10 @@ BATCH_SPECS = {
"spec_decode_small": BatchSpec(
seq_lens=[128, 256, 512, 1024], query_lens=[4, 4, 4, 4]
),
# Adaptive verification: verify requests trimmed to 1..k+1 tokens on device.
"spec_decode_ragged_small": BatchSpec(
seq_lens=[128, 256, 512, 1024], query_lens=[8, 3, 1, 5]
),
"spec_decode_medium": BatchSpec(
seq_lens=[512, 1024, 2048, 512, 1024, 2048], query_lens=[8, 8, 8, 8, 8, 8]
),
@@ -1581,6 +1590,34 @@ def run_attention_backend(
return output
def _with_even_cpu_query_split(
common_attn_metadata: CommonAttentionMetadata,
) -> CommonAttentionMetadata:
"""Host query lengths as adaptive verification leaves them.
The device keeps the trimmed per-request lengths; the host sees the same
total spread evenly, the first requests taking the remainder
(AdaptiveVerificationManager.compact_batch).
"""
num_reqs = common_attn_metadata.num_reqs
total = int(common_attn_metadata.query_start_loc_cpu[num_reqs])
cpu_lens = torch.full((num_reqs,), total // num_reqs, dtype=torch.int32)
cpu_lens[: total % num_reqs] += 1
query_start_loc_cpu = torch.zeros_like(common_attn_metadata.query_start_loc_cpu)
torch.cumsum(cpu_lens, dim=0, out=query_start_loc_cpu[1 : num_reqs + 1])
return common_attn_metadata.replace(
query_start_loc_cpu=query_start_loc_cpu,
max_query_len=int(
(
common_attn_metadata.query_start_loc[1:]
- common_attn_metadata.query_start_loc[:-1]
)
.max()
.item()
),
)
def _run_backend_correctness(
default_vllm_config,
dist_init,
@@ -1595,6 +1632,7 @@ def _run_backend_correctness(
qk_nope_head_dim: int,
v_head_dim: int,
chunked_prefill_workspace_size: int | None = None,
adaptive_verification: bool = False,
):
"""Test that all backends produce similar outputs to a reference implementation
using torch.nn.functional.scaled_dot_product_attention.
@@ -1626,6 +1664,10 @@ def _run_backend_correctness(
) and AttentionBackendEnum.CUTLASS_MLA in backends_to_test:
# CUTLASS_MLA does not support non-1 Q/K scales
backends_to_test.remove(AttentionBackendEnum.CUTLASS_MLA)
if adaptive_verification:
backends_to_test = [
b for b in backends_to_test if b in ADAPTIVE_VARLEN_BACKENDS
]
if not backends_to_test:
pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}")
@@ -1675,13 +1717,16 @@ def _run_backend_correctness(
if is_spec_decode_test:
from vllm.config import SpeculativeConfig
# Get the query length from the batch spec (they should all be uniform)
query_len = batch_spec.query_lens[0]
# Get the query length from the batch spec (they should all be uniform,
# except under adaptive verification, where the longest is k+1)
query_len = max(batch_spec.query_lens)
# Set num_speculative_tokens to query_len - 1
# (since threshold is 1 + num_spec_tokens)
# Use ngram method which doesn't require a draft model
vllm_config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=query_len - 1
method="ngram",
num_speculative_tokens=query_len - 1,
enable_adaptive_verification=adaptive_verification,
)
device = torch.device(f"{DEVICE_TYPE}:0")
@@ -1951,6 +1996,13 @@ def _run_backend_correctness(
)
kv_cache_per_block_size[block_size] = kv_cache
if adaptive_verification:
# Only after the cache is filled: prepopulation reads the context
# lengths off the host query lengths.
metadata_per_block_size[block_size] = _with_even_cpu_query_split(
common_attn_metadata
)
# 4. Run vLLM backends and compare
rtol = 1e-2
atol = {
@@ -2136,3 +2188,29 @@ def test_chunked_context_backend_correctness(
v_head_dim=v_head_dim,
chunked_prefill_workspace_size=1024,
)
@pytest.mark.skipif(
not ADAPTIVE_VARLEN_BACKENDS, reason="No adaptive varlen decode backend"
)
def test_adaptive_varlen_decode_correctness(
default_vllm_config, dist_init, workspace_init
):
"""Device query lengths 1..k+1 against an even host split, as adaptive
verification hands them to the target model's attention. TP 8 gives 16
heads, the padded Kimi-K3 count."""
_run_backend_correctness(
default_vllm_config,
dist_init,
workspace_init,
"spec_decode_ragged_small",
"deepseek-ai/DeepSeek-R1",
8,
"fp8",
1.0,
1.0,
None,
128,
128,
adaptive_verification=True,
)
@@ -63,9 +63,11 @@ class _ExpandPageIndicesKernel:
stride_0,
stride_1,
paged_kv_indptr,
num_real_tokens,
*,
KERNEL_BLOCK_SIZE,
BLOCK_SIZE,
CLAMP_TAIL=False,
):
# Both block-table strides are passed explicitly: the kernel no longer
# assumes the column stride is 1. This stand-in indexes the tensor
@@ -145,6 +147,7 @@ def _builder(
)
),
_mtp_decode_qlen=mtp_decode_qlen,
_varlen_decode_qlen=0,
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
_use_persistent_metadata=False,
kernel_block_size=kernel_block_size,
@@ -500,6 +503,7 @@ def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch):
impl.qk_rope_head_dim = 64
impl.scale = 576**-0.5
decode = SimpleNamespace(
varlen_q_rows=None,
max_qo_len=4,
qo_indptr=torch.tensor([0, 4], dtype=torch.int32),
paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32),
@@ -563,6 +567,7 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
impl.qk_rope_head_dim = 64
impl.scale = head_dim**-0.5
decode = SimpleNamespace(
varlen_q_rows=None,
max_qo_len=1,
qo_indptr=torch.arange(num_tokens + 1, dtype=torch.int32),
paged_kv_indptr=torch.zeros(num_tokens + 1, dtype=torch.int32),
@@ -787,6 +792,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
torch.tensor([global_seq_len], dtype=torch.int32, device=device),
)
decode = SimpleNamespace(
varlen_q_rows=None,
max_qo_len=qlen,
paged_kv_indptr=torch.tensor(
[0, local_kv.shape[0]], dtype=torch.int32, device=device
@@ -1452,3 +1458,29 @@ def test_a_two_token_bf16_block_is_allowed(monkeypatch):
monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=2, mtp_qlen=8
)
assert metadata.has_persistent_metadata
def _adaptive_config(
*, adaptive: bool = True, dcp: int = 1, num_speculative_tokens: int = 7
) -> SimpleNamespace:
return SimpleNamespace(
speculative_config=SimpleNamespace(
num_speculative_tokens=num_speculative_tokens,
enable_adaptive_verification=adaptive,
parallel_drafting=False,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp),
use_v2_model_runner=True,
)
def test_varlen_cudagraph_bound_follows_adaptive_verification():
bound = AiterMLAMetadataBuilder.get_varlen_cudagraph_max_query_len
assert bound(_adaptive_config(), SimpleNamespace()) == 8
assert bound(_adaptive_config(adaptive=False), SimpleNamespace()) is None
# DCP routes plan per-row windows from host lengths.
assert bound(_adaptive_config(dcp=2), SimpleNamespace()) is None
# Kimi-K3 DSpark draft layers share the target's group; causality is
# decided per batch, so the merged non-causal spec keeps the bound.
shared_draft_group = SimpleNamespace(non_causal_multi_token_decode=True)
assert bound(_adaptive_config(), shared_draft_group) == 8