[ROCm][Kimi-K3] Support DSpark adaptive verification on MI355X (#59824)

Signed-off-by: Oxana Korzh <okorzh@amd.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oxana Korzh
2026-10-16 13:53:49 -05:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 307c5844ad
commit 5bf3d9f665
6 changed files with 383 additions and 13 deletions
+29
View File
@@ -11,6 +11,7 @@ from dataclasses import replace
import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
@@ -18,12 +19,14 @@ from vllm.model_executor.layers.mamba.checkpoint import (
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
from vllm.v1.attention.backend import max_decode_query_len
from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionBackend,
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
from vllm.v1.kv_cache_interface import KVCacheSpec
logger = init_logger(__name__)
@@ -92,7 +95,24 @@ def prepare_chunk_metadata_device(
return chunk_indices, chunk_offsets
def _adaptive_verification_enabled(vllm_config: VllmConfig) -> bool:
speculative_config = vllm_config.speculative_config
return speculative_config is not None and bool(
getattr(speculative_config, "enable_adaptive_verification", False)
)
class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
@classmethod
def get_varlen_cudagraph_max_query_len(
cls: type["KimiK3ROCmKDAMetadataBuilder"],
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> int | None:
if not _adaptive_verification_enabled(vllm_config):
return None
return max_decode_query_len(vllm_config)
def build( # type: ignore[override]
self,
common_prefix_len: int,
@@ -108,6 +128,11 @@ class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
num_decode_draft_tokens_cpu,
fast_build,
)
if (
attn_metadata.uniform_spec_sequence_length is not None
and _adaptive_verification_enabled(self.vllm_config)
):
attn_metadata = replace(attn_metadata, uniform_spec_sequence_length=None)
if attn_metadata.checkpoint is None:
return attn_metadata
# The shared builder keys checkpoint rows on every request; the ROCm
@@ -163,3 +188,7 @@ class KimiK3ROCmKDABackend(GDNAttentionBackend):
@staticmethod
def get_builder_cls() -> type[KimiK3ROCmKDAMetadataBuilder]:
return KimiK3ROCmKDAMetadataBuilder
@classmethod
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
return True