[ROCm][Kimi-K3] Support DSpark adaptive verification on MI355X (#59824)
Signed-off-by: Oxana Korzh <okorzh@amd.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
307c5844ad
commit
5bf3d9f665
@@ -0,0 +1,93 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm Kimi-K3 KDA metadata under adaptive verification.
|
||||
|
||||
Adaptive verification splits the draft budget evenly on the host, then trims
|
||||
each verify request on device. The KDA spec path must therefore plan from the
|
||||
device offsets, never from host lengths that merely look uniform.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import SpeculativeConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import (
|
||||
KimiK3ROCmKDABackend,
|
||||
KimiK3ROCmKDAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
DEVICE = torch.device("cpu")
|
||||
NUM_SPEC = 3
|
||||
|
||||
|
||||
def _config(adaptive: bool):
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=NUM_SPEC,
|
||||
enable_adaptive_verification=adaptive,
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = "none"
|
||||
return vllm_config
|
||||
|
||||
|
||||
def _spec() -> MambaSpec:
|
||||
return MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
num_speculative_blocks=NUM_SPEC,
|
||||
)
|
||||
|
||||
|
||||
def _builder(adaptive: bool) -> KimiK3ROCmKDAMetadataBuilder:
|
||||
return KimiK3ROCmKDAMetadataBuilder(
|
||||
kv_cache_spec=_spec(),
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=_config(adaptive),
|
||||
device=DEVICE,
|
||||
)
|
||||
|
||||
|
||||
def _trimmed_verify_batch(builder: KimiK3ROCmKDAMetadataBuilder):
|
||||
"""Host sees [3, 3]; the device was trimmed to [4, 2] (same total)."""
|
||||
batch = BatchSpec(seq_lens=[64, 64], query_lens=[4, 2])
|
||||
common = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, DEVICE, arange_block_indices=True
|
||||
)
|
||||
common = common.replace(
|
||||
query_start_loc_cpu=torch.tensor([0, 3, 6], dtype=torch.int32)
|
||||
)
|
||||
accepted = torch.ones(2, dtype=torch.int32, device=DEVICE)
|
||||
num_decode_draft_tokens_cpu = torch.tensor([NUM_SPEC, NUM_SPEC], dtype=torch.int32)
|
||||
return builder.build(0, common, accepted, num_decode_draft_tokens_cpu)
|
||||
|
||||
|
||||
def test_adaptive_verification_plans_from_device_offsets():
|
||||
"""Equal host lengths would select the fixed-length recurrent kernel,
|
||||
which places sequence i at i * L and ignores the device cu_seqlens."""
|
||||
assert KimiK3ROCmKDABackend.supports_device_cpu_query_lens_mismatch()
|
||||
bound = KimiK3ROCmKDAMetadataBuilder.get_varlen_cudagraph_max_query_len
|
||||
assert bound(_config(adaptive=True), _spec()) == NUM_SPEC + 1
|
||||
assert bound(_config(adaptive=False), _spec()) is None
|
||||
|
||||
meta = _trimmed_verify_batch(_builder(adaptive=True))
|
||||
assert meta.num_spec_decodes == 2
|
||||
assert meta.uniform_spec_sequence_length is None
|
||||
assert torch.equal(
|
||||
meta.spec_query_start_loc.cpu(), torch.tensor([0, 4, 6], dtype=torch.int32)
|
||||
)
|
||||
# Without adaptive verification the host lengths are exact and kept.
|
||||
meta = _trimmed_verify_batch(_builder(adaptive=False))
|
||||
assert meta.uniform_spec_sequence_length == 3
|
||||
@@ -51,6 +51,7 @@ def _run(block_table, cu_num_tokens, kernel_block_size, out_len, device):
|
||||
block_table.stride(0),
|
||||
block_table.stride(1),
|
||||
cu_num_tokens,
|
||||
cu_num_tokens[1:] - cu_num_tokens[:-1],
|
||||
KERNEL_BLOCK_SIZE=kernel_block_size,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
@@ -109,6 +110,7 @@ def test_ragged_batch_leaves_other_rows_untouched():
|
||||
bt.stride(0),
|
||||
bt.stride(1),
|
||||
cu,
|
||||
cu[1:] - cu[:-1],
|
||||
KERNEL_BLOCK_SIZE=kbs,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
@@ -153,6 +155,7 @@ def test_over_provisioned_chunks_do_not_change_result():
|
||||
bt.stride(0),
|
||||
bt.stride(1),
|
||||
cu,
|
||||
cu[1:] - cu[:-1],
|
||||
KERNEL_BLOCK_SIZE=kbs,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
|
||||
@@ -418,6 +418,11 @@ for backend in BACKENDS_TO_TEST:
|
||||
if query_len_support != QueryLenSupport.SINGLE_ONLY:
|
||||
SPEC_DECODE_BACKENDS.append(backend)
|
||||
|
||||
# Backends whose decode pipeline replays device-trimmed verify batches.
|
||||
ADAPTIVE_VARLEN_BACKENDS = [
|
||||
b for b in BACKENDS_TO_TEST if b == AttentionBackendEnum.ROCM_AITER_MLA
|
||||
]
|
||||
|
||||
BACKEND_BLOCK_SIZES = {}
|
||||
for backend in BACKENDS_TO_TEST:
|
||||
supported_sizes = backend.get_class().get_supported_kernel_block_sizes()
|
||||
@@ -470,6 +475,10 @@ BATCH_SPECS = {
|
||||
"spec_decode_small": BatchSpec(
|
||||
seq_lens=[128, 256, 512, 1024], query_lens=[4, 4, 4, 4]
|
||||
),
|
||||
# Adaptive verification: verify requests trimmed to 1..k+1 tokens on device.
|
||||
"spec_decode_ragged_small": BatchSpec(
|
||||
seq_lens=[128, 256, 512, 1024], query_lens=[8, 3, 1, 5]
|
||||
),
|
||||
"spec_decode_medium": BatchSpec(
|
||||
seq_lens=[512, 1024, 2048, 512, 1024, 2048], query_lens=[8, 8, 8, 8, 8, 8]
|
||||
),
|
||||
@@ -1581,6 +1590,34 @@ def run_attention_backend(
|
||||
return output
|
||||
|
||||
|
||||
def _with_even_cpu_query_split(
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
) -> CommonAttentionMetadata:
|
||||
"""Host query lengths as adaptive verification leaves them.
|
||||
|
||||
The device keeps the trimmed per-request lengths; the host sees the same
|
||||
total spread evenly, the first requests taking the remainder
|
||||
(AdaptiveVerificationManager.compact_batch).
|
||||
"""
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
total = int(common_attn_metadata.query_start_loc_cpu[num_reqs])
|
||||
cpu_lens = torch.full((num_reqs,), total // num_reqs, dtype=torch.int32)
|
||||
cpu_lens[: total % num_reqs] += 1
|
||||
query_start_loc_cpu = torch.zeros_like(common_attn_metadata.query_start_loc_cpu)
|
||||
torch.cumsum(cpu_lens, dim=0, out=query_start_loc_cpu[1 : num_reqs + 1])
|
||||
return common_attn_metadata.replace(
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=int(
|
||||
(
|
||||
common_attn_metadata.query_start_loc[1:]
|
||||
- common_attn_metadata.query_start_loc[:-1]
|
||||
)
|
||||
.max()
|
||||
.item()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _run_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
@@ -1595,6 +1632,7 @@ def _run_backend_correctness(
|
||||
qk_nope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
chunked_prefill_workspace_size: int | None = None,
|
||||
adaptive_verification: bool = False,
|
||||
):
|
||||
"""Test that all backends produce similar outputs to a reference implementation
|
||||
using torch.nn.functional.scaled_dot_product_attention.
|
||||
@@ -1626,6 +1664,10 @@ def _run_backend_correctness(
|
||||
) and AttentionBackendEnum.CUTLASS_MLA in backends_to_test:
|
||||
# CUTLASS_MLA does not support non-1 Q/K scales
|
||||
backends_to_test.remove(AttentionBackendEnum.CUTLASS_MLA)
|
||||
if adaptive_verification:
|
||||
backends_to_test = [
|
||||
b for b in backends_to_test if b in ADAPTIVE_VARLEN_BACKENDS
|
||||
]
|
||||
if not backends_to_test:
|
||||
pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}")
|
||||
|
||||
@@ -1675,13 +1717,16 @@ def _run_backend_correctness(
|
||||
if is_spec_decode_test:
|
||||
from vllm.config import SpeculativeConfig
|
||||
|
||||
# Get the query length from the batch spec (they should all be uniform)
|
||||
query_len = batch_spec.query_lens[0]
|
||||
# Get the query length from the batch spec (they should all be uniform,
|
||||
# except under adaptive verification, where the longest is k+1)
|
||||
query_len = max(batch_spec.query_lens)
|
||||
# Set num_speculative_tokens to query_len - 1
|
||||
# (since threshold is 1 + num_spec_tokens)
|
||||
# Use ngram method which doesn't require a draft model
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=query_len - 1
|
||||
method="ngram",
|
||||
num_speculative_tokens=query_len - 1,
|
||||
enable_adaptive_verification=adaptive_verification,
|
||||
)
|
||||
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
@@ -1951,6 +1996,13 @@ def _run_backend_correctness(
|
||||
)
|
||||
kv_cache_per_block_size[block_size] = kv_cache
|
||||
|
||||
if adaptive_verification:
|
||||
# Only after the cache is filled: prepopulation reads the context
|
||||
# lengths off the host query lengths.
|
||||
metadata_per_block_size[block_size] = _with_even_cpu_query_split(
|
||||
common_attn_metadata
|
||||
)
|
||||
|
||||
# 4. Run vLLM backends and compare
|
||||
rtol = 1e-2
|
||||
atol = {
|
||||
@@ -2136,3 +2188,29 @@ def test_chunked_context_backend_correctness(
|
||||
v_head_dim=v_head_dim,
|
||||
chunked_prefill_workspace_size=1024,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not ADAPTIVE_VARLEN_BACKENDS, reason="No adaptive varlen decode backend"
|
||||
)
|
||||
def test_adaptive_varlen_decode_correctness(
|
||||
default_vllm_config, dist_init, workspace_init
|
||||
):
|
||||
"""Device query lengths 1..k+1 against an even host split, as adaptive
|
||||
verification hands them to the target model's attention. TP 8 gives 16
|
||||
heads, the padded Kimi-K3 count."""
|
||||
_run_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
"spec_decode_ragged_small",
|
||||
"deepseek-ai/DeepSeek-R1",
|
||||
8,
|
||||
"fp8",
|
||||
1.0,
|
||||
1.0,
|
||||
None,
|
||||
128,
|
||||
128,
|
||||
adaptive_verification=True,
|
||||
)
|
||||
|
||||
@@ -63,9 +63,11 @@ class _ExpandPageIndicesKernel:
|
||||
stride_0,
|
||||
stride_1,
|
||||
paged_kv_indptr,
|
||||
num_real_tokens,
|
||||
*,
|
||||
KERNEL_BLOCK_SIZE,
|
||||
BLOCK_SIZE,
|
||||
CLAMP_TAIL=False,
|
||||
):
|
||||
# Both block-table strides are passed explicitly: the kernel no longer
|
||||
# assumes the column stride is 1. This stand-in indexes the tensor
|
||||
@@ -145,6 +147,7 @@ def _builder(
|
||||
)
|
||||
),
|
||||
_mtp_decode_qlen=mtp_decode_qlen,
|
||||
_varlen_decode_qlen=0,
|
||||
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
|
||||
_use_persistent_metadata=False,
|
||||
kernel_block_size=kernel_block_size,
|
||||
@@ -500,6 +503,7 @@ def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch):
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.scale = 576**-0.5
|
||||
decode = SimpleNamespace(
|
||||
varlen_q_rows=None,
|
||||
max_qo_len=4,
|
||||
qo_indptr=torch.tensor([0, 4], dtype=torch.int32),
|
||||
paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32),
|
||||
@@ -563,6 +567,7 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.scale = head_dim**-0.5
|
||||
decode = SimpleNamespace(
|
||||
varlen_q_rows=None,
|
||||
max_qo_len=1,
|
||||
qo_indptr=torch.arange(num_tokens + 1, dtype=torch.int32),
|
||||
paged_kv_indptr=torch.zeros(num_tokens + 1, dtype=torch.int32),
|
||||
@@ -787,6 +792,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
|
||||
torch.tensor([global_seq_len], dtype=torch.int32, device=device),
|
||||
)
|
||||
decode = SimpleNamespace(
|
||||
varlen_q_rows=None,
|
||||
max_qo_len=qlen,
|
||||
paged_kv_indptr=torch.tensor(
|
||||
[0, local_kv.shape[0]], dtype=torch.int32, device=device
|
||||
@@ -1452,3 +1458,29 @@ def test_a_two_token_bf16_block_is_allowed(monkeypatch):
|
||||
monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=2, mtp_qlen=8
|
||||
)
|
||||
assert metadata.has_persistent_metadata
|
||||
|
||||
|
||||
def _adaptive_config(
|
||||
*, adaptive: bool = True, dcp: int = 1, num_speculative_tokens: int = 7
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
enable_adaptive_verification=adaptive,
|
||||
parallel_drafting=False,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp),
|
||||
use_v2_model_runner=True,
|
||||
)
|
||||
|
||||
|
||||
def test_varlen_cudagraph_bound_follows_adaptive_verification():
|
||||
bound = AiterMLAMetadataBuilder.get_varlen_cudagraph_max_query_len
|
||||
assert bound(_adaptive_config(), SimpleNamespace()) == 8
|
||||
assert bound(_adaptive_config(adaptive=False), SimpleNamespace()) is None
|
||||
# DCP routes plan per-row windows from host lengths.
|
||||
assert bound(_adaptive_config(dcp=2), SimpleNamespace()) is None
|
||||
# Kimi-K3 DSpark draft layers share the target's group; causality is
|
||||
# decided per batch, so the merged non-causal spec keeps the bound.
|
||||
shared_draft_group = SimpleNamespace(non_causal_multi_token_decode=True)
|
||||
assert bound(_adaptive_config(), shared_draft_group) == 8
|
||||
|
||||
Reference in New Issue
Block a user