[ROCm][Kimi-K3] Support DSpark adaptive verification on MI355X (#59824)

Signed-off-by: Oxana Korzh <okorzh@amd.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oxana Korzh
2026-10-16 13:53:49 -05:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 307c5844ad
commit 5bf3d9f665
6 changed files with 383 additions and 13 deletions
@@ -0,0 +1,93 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm Kimi-K3 KDA metadata under adaptive verification.
Adaptive verification splits the draft budget evenly on the host, then trims
each verify request on device. The KDA spec path must therefore plan from the
device offsets, never from host lengths that merely look uniform.
"""
import torch
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import SpeculativeConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.models.kimi_k3.amd.kda_metadata import (
KimiK3ROCmKDABackend,
KimiK3ROCmKDAMetadataBuilder,
)
from vllm.v1.kv_cache_interface import MambaSpec
BLOCK_SIZE = 16
DEVICE = torch.device("cpu")
NUM_SPEC = 3
def _config(adaptive: bool):
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
block_size=BLOCK_SIZE,
)
vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
vllm_config.speculative_config = SpeculativeConfig(
method="ngram",
num_speculative_tokens=NUM_SPEC,
enable_adaptive_verification=adaptive,
)
vllm_config.cache_config.mamba_cache_mode = "none"
return vllm_config
def _spec() -> MambaSpec:
return MambaSpec(
block_size=BLOCK_SIZE,
shapes=((16, 64),),
dtypes=(torch.float16,),
num_speculative_blocks=NUM_SPEC,
)
def _builder(adaptive: bool) -> KimiK3ROCmKDAMetadataBuilder:
return KimiK3ROCmKDAMetadataBuilder(
kv_cache_spec=_spec(),
layer_names=["layer.0"],
vllm_config=_config(adaptive),
device=DEVICE,
)
def _trimmed_verify_batch(builder: KimiK3ROCmKDAMetadataBuilder):
"""Host sees [3, 3]; the device was trimmed to [4, 2] (same total)."""
batch = BatchSpec(seq_lens=[64, 64], query_lens=[4, 2])
common = create_common_attn_metadata(
batch, BLOCK_SIZE, DEVICE, arange_block_indices=True
)
common = common.replace(
query_start_loc_cpu=torch.tensor([0, 3, 6], dtype=torch.int32)
)
accepted = torch.ones(2, dtype=torch.int32, device=DEVICE)
num_decode_draft_tokens_cpu = torch.tensor([NUM_SPEC, NUM_SPEC], dtype=torch.int32)
return builder.build(0, common, accepted, num_decode_draft_tokens_cpu)
def test_adaptive_verification_plans_from_device_offsets():
"""Equal host lengths would select the fixed-length recurrent kernel,
which places sequence i at i * L and ignores the device cu_seqlens."""
assert KimiK3ROCmKDABackend.supports_device_cpu_query_lens_mismatch()
bound = KimiK3ROCmKDAMetadataBuilder.get_varlen_cudagraph_max_query_len
assert bound(_config(adaptive=True), _spec()) == NUM_SPEC + 1
assert bound(_config(adaptive=False), _spec()) is None
meta = _trimmed_verify_batch(_builder(adaptive=True))
assert meta.num_spec_decodes == 2
assert meta.uniform_spec_sequence_length is None
assert torch.equal(
meta.spec_query_start_loc.cpu(), torch.tensor([0, 4, 6], dtype=torch.int32)
)
# Without adaptive verification the host lengths are exact and kept.
meta = _trimmed_verify_batch(_builder(adaptive=False))
assert meta.uniform_spec_sequence_length == 3
@@ -51,6 +51,7 @@ def _run(block_table, cu_num_tokens, kernel_block_size, out_len, device):
block_table.stride(0), block_table.stride(0),
block_table.stride(1), block_table.stride(1),
cu_num_tokens, cu_num_tokens,
cu_num_tokens[1:] - cu_num_tokens[:-1],
KERNEL_BLOCK_SIZE=kernel_block_size, KERNEL_BLOCK_SIZE=kernel_block_size,
BLOCK_SIZE=1024, BLOCK_SIZE=1024,
) )
@@ -109,6 +110,7 @@ def test_ragged_batch_leaves_other_rows_untouched():
bt.stride(0), bt.stride(0),
bt.stride(1), bt.stride(1),
cu, cu,
cu[1:] - cu[:-1],
KERNEL_BLOCK_SIZE=kbs, KERNEL_BLOCK_SIZE=kbs,
BLOCK_SIZE=1024, BLOCK_SIZE=1024,
) )
@@ -153,6 +155,7 @@ def test_over_provisioned_chunks_do_not_change_result():
bt.stride(0), bt.stride(0),
bt.stride(1), bt.stride(1),
cu, cu,
cu[1:] - cu[:-1],
KERNEL_BLOCK_SIZE=kbs, KERNEL_BLOCK_SIZE=kbs,
BLOCK_SIZE=1024, BLOCK_SIZE=1024,
) )
+81 -3
View File
@@ -418,6 +418,11 @@ for backend in BACKENDS_TO_TEST:
if query_len_support != QueryLenSupport.SINGLE_ONLY: if query_len_support != QueryLenSupport.SINGLE_ONLY:
SPEC_DECODE_BACKENDS.append(backend) SPEC_DECODE_BACKENDS.append(backend)
# Backends whose decode pipeline replays device-trimmed verify batches.
ADAPTIVE_VARLEN_BACKENDS = [
b for b in BACKENDS_TO_TEST if b == AttentionBackendEnum.ROCM_AITER_MLA
]
BACKEND_BLOCK_SIZES = {} BACKEND_BLOCK_SIZES = {}
for backend in BACKENDS_TO_TEST: for backend in BACKENDS_TO_TEST:
supported_sizes = backend.get_class().get_supported_kernel_block_sizes() supported_sizes = backend.get_class().get_supported_kernel_block_sizes()
@@ -470,6 +475,10 @@ BATCH_SPECS = {
"spec_decode_small": BatchSpec( "spec_decode_small": BatchSpec(
seq_lens=[128, 256, 512, 1024], query_lens=[4, 4, 4, 4] seq_lens=[128, 256, 512, 1024], query_lens=[4, 4, 4, 4]
), ),
# Adaptive verification: verify requests trimmed to 1..k+1 tokens on device.
"spec_decode_ragged_small": BatchSpec(
seq_lens=[128, 256, 512, 1024], query_lens=[8, 3, 1, 5]
),
"spec_decode_medium": BatchSpec( "spec_decode_medium": BatchSpec(
seq_lens=[512, 1024, 2048, 512, 1024, 2048], query_lens=[8, 8, 8, 8, 8, 8] seq_lens=[512, 1024, 2048, 512, 1024, 2048], query_lens=[8, 8, 8, 8, 8, 8]
), ),
@@ -1581,6 +1590,34 @@ def run_attention_backend(
return output return output
def _with_even_cpu_query_split(
common_attn_metadata: CommonAttentionMetadata,
) -> CommonAttentionMetadata:
"""Host query lengths as adaptive verification leaves them.
The device keeps the trimmed per-request lengths; the host sees the same
total spread evenly, the first requests taking the remainder
(AdaptiveVerificationManager.compact_batch).
"""
num_reqs = common_attn_metadata.num_reqs
total = int(common_attn_metadata.query_start_loc_cpu[num_reqs])
cpu_lens = torch.full((num_reqs,), total // num_reqs, dtype=torch.int32)
cpu_lens[: total % num_reqs] += 1
query_start_loc_cpu = torch.zeros_like(common_attn_metadata.query_start_loc_cpu)
torch.cumsum(cpu_lens, dim=0, out=query_start_loc_cpu[1 : num_reqs + 1])
return common_attn_metadata.replace(
query_start_loc_cpu=query_start_loc_cpu,
max_query_len=int(
(
common_attn_metadata.query_start_loc[1:]
- common_attn_metadata.query_start_loc[:-1]
)
.max()
.item()
),
)
def _run_backend_correctness( def _run_backend_correctness(
default_vllm_config, default_vllm_config,
dist_init, dist_init,
@@ -1595,6 +1632,7 @@ def _run_backend_correctness(
qk_nope_head_dim: int, qk_nope_head_dim: int,
v_head_dim: int, v_head_dim: int,
chunked_prefill_workspace_size: int | None = None, chunked_prefill_workspace_size: int | None = None,
adaptive_verification: bool = False,
): ):
"""Test that all backends produce similar outputs to a reference implementation """Test that all backends produce similar outputs to a reference implementation
using torch.nn.functional.scaled_dot_product_attention. using torch.nn.functional.scaled_dot_product_attention.
@@ -1626,6 +1664,10 @@ def _run_backend_correctness(
) and AttentionBackendEnum.CUTLASS_MLA in backends_to_test: ) and AttentionBackendEnum.CUTLASS_MLA in backends_to_test:
# CUTLASS_MLA does not support non-1 Q/K scales # CUTLASS_MLA does not support non-1 Q/K scales
backends_to_test.remove(AttentionBackendEnum.CUTLASS_MLA) backends_to_test.remove(AttentionBackendEnum.CUTLASS_MLA)
if adaptive_verification:
backends_to_test = [
b for b in backends_to_test if b in ADAPTIVE_VARLEN_BACKENDS
]
if not backends_to_test: if not backends_to_test:
pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}") pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}")
@@ -1675,13 +1717,16 @@ def _run_backend_correctness(
if is_spec_decode_test: if is_spec_decode_test:
from vllm.config import SpeculativeConfig from vllm.config import SpeculativeConfig
# Get the query length from the batch spec (they should all be uniform) # Get the query length from the batch spec (they should all be uniform,
query_len = batch_spec.query_lens[0] # except under adaptive verification, where the longest is k+1)
query_len = max(batch_spec.query_lens)
# Set num_speculative_tokens to query_len - 1 # Set num_speculative_tokens to query_len - 1
# (since threshold is 1 + num_spec_tokens) # (since threshold is 1 + num_spec_tokens)
# Use ngram method which doesn't require a draft model # Use ngram method which doesn't require a draft model
vllm_config.speculative_config = SpeculativeConfig( vllm_config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=query_len - 1 method="ngram",
num_speculative_tokens=query_len - 1,
enable_adaptive_verification=adaptive_verification,
) )
device = torch.device(f"{DEVICE_TYPE}:0") device = torch.device(f"{DEVICE_TYPE}:0")
@@ -1951,6 +1996,13 @@ def _run_backend_correctness(
) )
kv_cache_per_block_size[block_size] = kv_cache kv_cache_per_block_size[block_size] = kv_cache
if adaptive_verification:
# Only after the cache is filled: prepopulation reads the context
# lengths off the host query lengths.
metadata_per_block_size[block_size] = _with_even_cpu_query_split(
common_attn_metadata
)
# 4. Run vLLM backends and compare # 4. Run vLLM backends and compare
rtol = 1e-2 rtol = 1e-2
atol = { atol = {
@@ -2136,3 +2188,29 @@ def test_chunked_context_backend_correctness(
v_head_dim=v_head_dim, v_head_dim=v_head_dim,
chunked_prefill_workspace_size=1024, chunked_prefill_workspace_size=1024,
) )
@pytest.mark.skipif(
not ADAPTIVE_VARLEN_BACKENDS, reason="No adaptive varlen decode backend"
)
def test_adaptive_varlen_decode_correctness(
default_vllm_config, dist_init, workspace_init
):
"""Device query lengths 1..k+1 against an even host split, as adaptive
verification hands them to the target model's attention. TP 8 gives 16
heads, the padded Kimi-K3 count."""
_run_backend_correctness(
default_vllm_config,
dist_init,
workspace_init,
"spec_decode_ragged_small",
"deepseek-ai/DeepSeek-R1",
8,
"fp8",
1.0,
1.0,
None,
128,
128,
adaptive_verification=True,
)
@@ -63,9 +63,11 @@ class _ExpandPageIndicesKernel:
stride_0, stride_0,
stride_1, stride_1,
paged_kv_indptr, paged_kv_indptr,
num_real_tokens,
*, *,
KERNEL_BLOCK_SIZE, KERNEL_BLOCK_SIZE,
BLOCK_SIZE, BLOCK_SIZE,
CLAMP_TAIL=False,
): ):
# Both block-table strides are passed explicitly: the kernel no longer # Both block-table strides are passed explicitly: the kernel no longer
# assumes the column stride is 1. This stand-in indexes the tensor # assumes the column stride is 1. This stand-in indexes the tensor
@@ -145,6 +147,7 @@ def _builder(
) )
), ),
_mtp_decode_qlen=mtp_decode_qlen, _mtp_decode_qlen=mtp_decode_qlen,
_varlen_decode_qlen=0,
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len), _uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
_use_persistent_metadata=False, _use_persistent_metadata=False,
kernel_block_size=kernel_block_size, kernel_block_size=kernel_block_size,
@@ -500,6 +503,7 @@ def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch):
impl.qk_rope_head_dim = 64 impl.qk_rope_head_dim = 64
impl.scale = 576**-0.5 impl.scale = 576**-0.5
decode = SimpleNamespace( decode = SimpleNamespace(
varlen_q_rows=None,
max_qo_len=4, max_qo_len=4,
qo_indptr=torch.tensor([0, 4], dtype=torch.int32), qo_indptr=torch.tensor([0, 4], dtype=torch.int32),
paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32), paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32),
@@ -563,6 +567,7 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
impl.qk_rope_head_dim = 64 impl.qk_rope_head_dim = 64
impl.scale = head_dim**-0.5 impl.scale = head_dim**-0.5
decode = SimpleNamespace( decode = SimpleNamespace(
varlen_q_rows=None,
max_qo_len=1, max_qo_len=1,
qo_indptr=torch.arange(num_tokens + 1, dtype=torch.int32), qo_indptr=torch.arange(num_tokens + 1, dtype=torch.int32),
paged_kv_indptr=torch.zeros(num_tokens + 1, dtype=torch.int32), paged_kv_indptr=torch.zeros(num_tokens + 1, dtype=torch.int32),
@@ -787,6 +792,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
torch.tensor([global_seq_len], dtype=torch.int32, device=device), torch.tensor([global_seq_len], dtype=torch.int32, device=device),
) )
decode = SimpleNamespace( decode = SimpleNamespace(
varlen_q_rows=None,
max_qo_len=qlen, max_qo_len=qlen,
paged_kv_indptr=torch.tensor( paged_kv_indptr=torch.tensor(
[0, local_kv.shape[0]], dtype=torch.int32, device=device [0, local_kv.shape[0]], dtype=torch.int32, device=device
@@ -1452,3 +1458,29 @@ def test_a_two_token_bf16_block_is_allowed(monkeypatch):
monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=2, mtp_qlen=8 monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=2, mtp_qlen=8
) )
assert metadata.has_persistent_metadata assert metadata.has_persistent_metadata
def _adaptive_config(
*, adaptive: bool = True, dcp: int = 1, num_speculative_tokens: int = 7
) -> SimpleNamespace:
return SimpleNamespace(
speculative_config=SimpleNamespace(
num_speculative_tokens=num_speculative_tokens,
enable_adaptive_verification=adaptive,
parallel_drafting=False,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp),
use_v2_model_runner=True,
)
def test_varlen_cudagraph_bound_follows_adaptive_verification():
bound = AiterMLAMetadataBuilder.get_varlen_cudagraph_max_query_len
assert bound(_adaptive_config(), SimpleNamespace()) == 8
assert bound(_adaptive_config(adaptive=False), SimpleNamespace()) is None
# DCP routes plan per-row windows from host lengths.
assert bound(_adaptive_config(dcp=2), SimpleNamespace()) is None
# Kimi-K3 DSpark draft layers share the target's group; causality is
# decided per batch, so the merged non-causal spec keeps the bound.
shared_draft_group = SimpleNamespace(non_causal_multi_token_decode=True)
assert bound(_adaptive_config(), shared_draft_group) == 8
+29
View File
@@ -11,6 +11,7 @@ from dataclasses import replace
import torch import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.checkpoint import ( from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata, MambaPrefillCheckpointMetadata,
@@ -18,12 +19,14 @@ from vllm.model_executor.layers.mamba.checkpoint import (
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
from vllm.triton_utils import tl, triton from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2 from vllm.utils.math_utils import next_power_of_2
from vllm.v1.attention.backend import max_decode_query_len
from vllm.v1.attention.backends.gdn_attn import ( from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionBackend, GDNAttentionBackend,
GDNAttentionMetadata, GDNAttentionMetadata,
GDNAttentionMetadataBuilder, GDNAttentionMetadataBuilder,
) )
from vllm.v1.attention.backends.utils import CommonAttentionMetadata from vllm.v1.attention.backends.utils import CommonAttentionMetadata
from vllm.v1.kv_cache_interface import KVCacheSpec
logger = init_logger(__name__) logger = init_logger(__name__)
@@ -92,7 +95,24 @@ def prepare_chunk_metadata_device(
return chunk_indices, chunk_offsets return chunk_indices, chunk_offsets
def _adaptive_verification_enabled(vllm_config: VllmConfig) -> bool:
speculative_config = vllm_config.speculative_config
return speculative_config is not None and bool(
getattr(speculative_config, "enable_adaptive_verification", False)
)
class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder): class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
@classmethod
def get_varlen_cudagraph_max_query_len(
cls: type["KimiK3ROCmKDAMetadataBuilder"],
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> int | None:
if not _adaptive_verification_enabled(vllm_config):
return None
return max_decode_query_len(vllm_config)
def build( # type: ignore[override] def build( # type: ignore[override]
self, self,
common_prefix_len: int, common_prefix_len: int,
@@ -108,6 +128,11 @@ class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
num_decode_draft_tokens_cpu, num_decode_draft_tokens_cpu,
fast_build, fast_build,
) )
if (
attn_metadata.uniform_spec_sequence_length is not None
and _adaptive_verification_enabled(self.vllm_config)
):
attn_metadata = replace(attn_metadata, uniform_spec_sequence_length=None)
if attn_metadata.checkpoint is None: if attn_metadata.checkpoint is None:
return attn_metadata return attn_metadata
# The shared builder keys checkpoint rows on every request; the ROCm # The shared builder keys checkpoint rows on every request; the ROCm
@@ -163,3 +188,7 @@ class KimiK3ROCmKDABackend(GDNAttentionBackend):
@staticmethod @staticmethod
def get_builder_cls() -> type[KimiK3ROCmKDAMetadataBuilder]: def get_builder_cls() -> type[KimiK3ROCmKDAMetadataBuilder]:
return KimiK3ROCmKDAMetadataBuilder return KimiK3ROCmKDAMetadataBuilder
@classmethod
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
return True
+145 -10
View File
@@ -35,6 +35,7 @@ from vllm.v1.attention.backend import (
AttentionLayer, AttentionLayer,
CommonAttentionMetadata, CommonAttentionMetadata,
MultipleOf, MultipleOf,
max_decode_query_len,
) )
from vllm.v1.attention.backends.mla.triton_mla import _compute_num_kv_splits from vllm.v1.attention.backends.mla.triton_mla import _compute_num_kv_splits
from vllm.v1.attention.backends.utils import ( from vllm.v1.attention.backends.utils import (
@@ -49,7 +50,11 @@ from vllm.v1.attention.ops.rocm_aiter_mla_prefill import (
gather_compressed_context, gather_compressed_context,
) )
from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd
from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheSpec,
is_quantized_kv_cache,
)
from vllm.v1.worker.workspace import ( from vllm.v1.worker.workspace import (
current_workspace_manager, current_workspace_manager,
is_workspace_manager_initialized, is_workspace_manager_initialized,
@@ -457,6 +462,16 @@ def _aiter_mla_small_head_mode() -> str:
return mode return mode
def _adaptive_varlen_verify_enabled(vllm_config: VllmConfig) -> bool:
speculative_config = vllm_config.speculative_config
return (
speculative_config is not None
and bool(getattr(speculative_config, "enable_adaptive_verification", False))
and vllm_config.parallel_config.decode_context_parallel_size == 1
and max_decode_query_len(vllm_config) > 1
)
class AiterMLABackend(MLACommonBackend): class AiterMLABackend(MLACommonBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16] supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [ supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
@@ -620,6 +635,8 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
use_gluon_verify: bool = False use_gluon_verify: bool = False
# Whether persistent MLA metadata was computed # Whether persistent MLA metadata was computed
has_persistent_metadata: bool = False has_persistent_metadata: bool = False
varlen_q_rows: torch.Tensor | None = None
varlen_o_rows: torch.Tensor | None = None
@dataclass @dataclass
@@ -720,6 +737,21 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
return uniform_qo_len return uniform_qo_len
@classmethod
def get_varlen_cudagraph_max_query_len(
cls: type["AiterMLAMetadataBuilder"],
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> int | None:
if not _adaptive_varlen_verify_enabled(vllm_config):
return None
if (
cls.get_cudagraph_support(vllm_config, kv_cache_spec)
!= AttentionCGSupport.UNIFORM_BATCH
):
return None
return max_decode_query_len(vllm_config)
def __init__( def __init__(
self, self,
kv_cache_spec: AttentionSpec, kv_cache_spec: AttentionSpec,
@@ -823,6 +855,26 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
# persistent gate below and makes aiter raise a KeyError mid-run. # persistent gate below and makes aiter raise a KeyError mid-run.
self._mtp_decode_qlen = self.reorder_batch_threshold or 1 self._mtp_decode_qlen = self.reorder_batch_threshold or 1
self._varlen_decode_qlen = 0
if _adaptive_varlen_verify_enabled(vllm_config):
assert self._mtp_decode_qlen == max_decode_query_len(vllm_config), (
self._mtp_decode_qlen,
max_decode_query_len(vllm_config),
)
self._varlen_decode_qlen = self._mtp_decode_qlen
self.query_len_support = QueryLenSupport.VARLEN # type: ignore[misc]
max_rows = vllm_config.scheduler_config.max_num_seqs * (
self._varlen_decode_qlen
)
row_ids = torch.arange(max_rows, dtype=torch.int64, device=device)
self._varlen_row_req = row_ids // self._varlen_decode_qlen
self._varlen_row_pos = row_ids % self._varlen_decode_qlen
self._varlen_row_ids = row_ids
self._varlen_q_rows = torch.zeros_like(row_ids)
self._varlen_o_rows = torch.zeros(
max_rows + 1, dtype=torch.int64, device=device
)
# Store the kernel block size from the spec. When kernel_block_size=1 # Store the kernel block size from the spec. When kernel_block_size=1
# (no spec-dec), behavior is identical to the original. When > 1 # (no spec-dec), behavior is identical to the original. When > 1
# (e.g. 16 with Eagle3), we expand block-level indices into per-token # (e.g. 16 with Eagle3), we expand block-level indices into per-token
@@ -850,6 +902,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
max_num_pages_per_req = vllm_config.model_config.max_model_len max_num_pages_per_req = vllm_config.model_config.max_model_len
max_num_reqs = vllm_config.scheduler_config.max_num_seqs max_num_reqs = vllm_config.scheduler_config.max_num_seqs
max_num_pages = max_num_reqs * max_num_pages_per_req max_num_pages = max_num_reqs * max_num_pages_per_req
speculative_config = vllm_config.speculative_config
if _adaptive_varlen_verify_enabled(vllm_config):
assert speculative_config is not None
max_num_pages += max_num_reqs * (
speculative_config.num_speculative_tokens or 0
)
# Preparing persistent buffers # Preparing persistent buffers
# TODO: we can disambiguate between decode and mixed-prefill decode here # TODO: we can disambiguate between decode and mixed-prefill decode here
@@ -1437,13 +1495,18 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
device = self.device device = self.device
num_reqs = seq_lens_device.size(0) num_reqs = seq_lens_device.size(0)
qo_len = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] qo_len = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
max_qo_len = qo_len.max().item() varlen_decode = self._varlen_decode_qlen > 0 and causal
padded_mtp_qo_len = self._uniform_padded_mtp_qo_len( if varlen_decode:
qo_len, max_qo_len, num_decode_tokens max_qo_len = self._varlen_decode_qlen
) pad_uniform_mtp = True
if padded_mtp_qo_len > 0: else:
max_qo_len = padded_mtp_qo_len max_qo_len = qo_len.max().item()
pad_uniform_mtp = padded_mtp_qo_len > 0 padded_mtp_qo_len = self._uniform_padded_mtp_qo_len(
qo_len, max_qo_len, num_decode_tokens
)
if padded_mtp_qo_len > 0:
max_qo_len = padded_mtp_qo_len
pad_uniform_mtp = padded_mtp_qo_len > 0
seq_lens_for_kernel = seq_lens_device seq_lens_for_kernel = seq_lens_device
# the global lengths need the same dummy-row # the global lengths need the same dummy-row
@@ -1477,6 +1540,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
g_tot_seq_lens.new_full((), max_qo_len * self.dcp_world_size), g_tot_seq_lens.new_full((), max_qo_len * self.dcp_world_size),
) )
varlen_q_rows = varlen_o_rows = None
if varlen_decode:
seq_lens_for_kernel = seq_lens_device + (max_qo_len - qo_lens_device)
varlen_q_rows, varlen_o_rows = self._fill_varlen_rows(
query_start_loc_device, qo_lens_device, num_reqs
)
if self._graph_seq_lens is not None: if self._graph_seq_lens is not None:
self._graph_seq_lens[:num_kernel_reqs].copy_( self._graph_seq_lens[:num_kernel_reqs].copy_(
seq_lens_for_kernel, non_blocking=True seq_lens_for_kernel, non_blocking=True
@@ -1537,6 +1607,8 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
# device synchronisation. Programs whose chunk lies past # device synchronisation. Programs whose chunk lies past
# num_tokens mask out entirely. # num_tokens mask out entirely.
max_tokens_per_req = block_table_tensor.shape[1] * self.kernel_block_size max_tokens_per_req = block_table_tensor.shape[1] * self.kernel_block_size
if varlen_decode:
max_tokens_per_req += max_qo_len - 1
num_chunks = max(1, cdiv(max_tokens_per_req, 1024)) num_chunks = max(1, cdiv(max_tokens_per_req, 1024))
_expand_page_indices_kernel[(num_reqs, num_chunks)]( _expand_page_indices_kernel[(num_reqs, num_chunks)](
self.paged_kv_indices, self.paged_kv_indices,
@@ -1544,8 +1616,10 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
block_table_tensor.stride(0), block_table_tensor.stride(0),
block_table_tensor.stride(1), block_table_tensor.stride(1),
paged_kv_indptr, paged_kv_indptr,
seq_lens_device,
KERNEL_BLOCK_SIZE=self.kernel_block_size, KERNEL_BLOCK_SIZE=self.kernel_block_size,
BLOCK_SIZE=1024, BLOCK_SIZE=1024,
CLAMP_TAIL=varlen_decode,
) )
paged_kv_indices = self.paged_kv_indices paged_kv_indices = self.paged_kv_indices
@@ -1740,10 +1814,38 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
use_gluon_verify=use_gluon_verify, use_gluon_verify=use_gluon_verify,
attn_out_dtype=self.decode_attn_out_dtype, attn_out_dtype=self.decode_attn_out_dtype,
has_persistent_metadata=has_persistent_metadata, has_persistent_metadata=has_persistent_metadata,
varlen_q_rows=varlen_q_rows,
varlen_o_rows=varlen_o_rows,
) )
return attn_metadata return attn_metadata
def _fill_varlen_rows(
self,
query_start_loc_device: torch.Tensor,
qo_lens_device: torch.Tensor,
num_reqs: int,
) -> tuple[torch.Tensor, torch.Tensor]:
width = self._varlen_decode_qlen
num_rows = num_reqs * width
row_req = self._varlen_row_req[:num_rows]
row_pos = self._varlen_row_pos[:num_rows]
row_qlen = qo_lens_device.to(torch.int64).index_select(0, row_req)
row_start = query_start_loc_device.to(torch.int64).index_select(0, row_req)
is_real = row_pos < row_qlen
packed_row = row_start + row_pos
q_rows = self._varlen_q_rows[:num_rows]
q_rows.copy_(torch.where(is_real, packed_row, 0))
o_rows = self._varlen_o_rows
o_rows.zero_()
dump_slot = o_rows.size(0) - 1
o_rows.scatter_(
0,
torch.where(is_real, packed_row, dump_slot),
self._varlen_row_ids[:num_rows],
)
return q_rows, o_rows
def build( def build(
self, self,
common_prefix_len: int, common_prefix_len: int,
@@ -1795,8 +1897,10 @@ def _expand_page_indices_kernel(
block_table_stride_0, block_table_stride_0,
block_table_stride_1, block_table_stride_1,
cu_num_tokens, cu_num_tokens,
num_real_tokens,
KERNEL_BLOCK_SIZE: tl.constexpr, KERNEL_BLOCK_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr, BLOCK_SIZE: tl.constexpr,
CLAMP_TAIL: tl.constexpr = False,
): ):
"""Expand block table entries into per-token flat page indices. """Expand block table entries into per-token flat page indices.
@@ -1810,6 +1914,10 @@ def _expand_page_indices_kernel(
When KERNEL_BLOCK_SIZE=K: block table entry b (covering K tokens) When KERNEL_BLOCK_SIZE=K: block table entry b (covering K tokens)
is expanded to flat indices b*K, b*K+1, ..., b*K+(K-1). is expanded to flat indices b*K, b*K+1, ..., b*K+(K-1).
With CLAMP_TAIL, entries past num_real_tokens[req] (padding a request's
page list out to its kernel length) repeat the request's first page, so
they never index past its block-table row.
""" """
# One program per (request, token-chunk). Parallelising over requests # One program per (request, token-chunk). Parallelising over requests
# alone degenerates at low concurrency: with num_reqs == 1 a single # alone degenerates at low concurrency: with num_reqs == 1 a single
@@ -1819,6 +1927,8 @@ def _expand_page_indices_kernel(
row_ptr = block_table + req_idx * block_table_stride_0 row_ptr = block_table + req_idx * block_table_stride_0
start_idx = tl.load(cu_num_tokens + req_idx) start_idx = tl.load(cu_num_tokens + req_idx)
num_tokens = tl.load(cu_num_tokens + req_idx + 1) - start_idx num_tokens = tl.load(cu_num_tokens + req_idx + 1) - start_idx
if CLAMP_TAIL:
real_tokens = tl.load(num_real_tokens + req_idx)
# The grid is sized from the block table width, an upper bound over all # The grid is sized from the block table width, an upper bound over all
# requests, so a ragged batch launches chunks past a short request's end. # requests, so a ragged batch launches chunks past a short request's end.
@@ -1830,11 +1940,14 @@ def _expand_page_indices_kernel(
token_offsets = chunk_start + tl.arange(0, BLOCK_SIZE) token_offsets = chunk_start + tl.arange(0, BLOCK_SIZE)
mask = token_offsets < num_tokens mask = token_offsets < num_tokens
source_offsets = token_offsets
if CLAMP_TAIL:
source_offsets = tl.where(token_offsets < real_tokens, token_offsets, 0)
# Which block in the block table does this token belong to? # Which block in the block table does this token belong to?
block_idx = token_offsets // KERNEL_BLOCK_SIZE block_idx = source_offsets // KERNEL_BLOCK_SIZE
# Offset within that block # Offset within that block
offset_in_block = token_offsets % KERNEL_BLOCK_SIZE offset_in_block = source_offsets % KERNEL_BLOCK_SIZE
# Load the block ID from the block table # Load the block ID from the block table
# Both strides are taken from the caller: the block table is a view owned # Both strides are taken from the caller: the block table is a view owned
@@ -2600,6 +2713,28 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
kv_c_and_k_pe_cache: torch.Tensor, kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: AiterMLAMetadata, attn_metadata: AiterMLAMetadata,
layer: AttentionLayer, layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
decode = attn_metadata.decode
if decode is None or decode.varlen_q_rows is None:
return self._forward_mqa(q, kv_c_and_k_pe_cache, attn_metadata, layer)
assert decode.varlen_o_rows is not None
q_rows = decode.varlen_q_rows
if isinstance(q, tuple):
num_tokens = q[0].shape[0]
q = (q[0].index_select(0, q_rows), q[1].index_select(0, q_rows))
else:
num_tokens = q.shape[0]
q = q.index_select(0, q_rows)
o, lse = self._forward_mqa(q, kv_c_and_k_pe_cache, attn_metadata, layer)
assert lse is None
return o.index_select(0, decode.varlen_o_rows[:num_tokens]), None
def _forward_mqa(
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: AiterMLAMetadata,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]: ) -> tuple[torch.Tensor, torch.Tensor | None]:
assert kv_c_and_k_pe_cache.numel() > 0 assert kv_c_and_k_pe_cache.numel() > 0
assert attn_metadata.decode is not None assert attn_metadata.decode is not None