[ROCm][DSv4] Enable DSpark adaptive verification (#52362)

Signed-off-by: Tuukka Sarvi <tuukka.sarvi@amd.com>
Signed-off-by: larryli2-amd <larryli2@amd.com>
Co-authored-by: larryli2-amd <larryli2@amd.com>
This commit is contained in:
Tuukka Sarvi
2026-09-21 10:03:41 -05:00
committed by GitHub
co-authored by larryli2-amd
parent 21aa17c128
commit e34685dfc0
6 changed files with 725 additions and 7 deletions
+28 -5
View File
@@ -44,6 +44,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.qwen3_dspark import (
DSparkConfidenceHead,
DSparkMarkovHead,
)
from vllm.model_executor.models.utils import maybe_prefix
@@ -102,7 +103,7 @@ class DSparkDeepseekV4Model(nn.Module):
]
)
# Heads: final norm + hc_head, and the Markov head
# Heads: final norm + hc_head, and the Markov + confidence heads
# Loaded from the "final" MTP layer weights (mtp.*) in the target checkpoint
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
hc_dim = self.hc_mult * config.hidden_size
@@ -125,6 +126,12 @@ class DSparkDeepseekV4Model(nn.Module):
config.dspark_markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
)
self.confidence_head: DSparkConfidenceHead | None = None
if getattr(config, "enable_confidence_head", True):
self.confidence_head = DSparkConfidenceHead(
config.hidden_size + config.dspark_markov_rank,
prefix=maybe_prefix(prefix, "confidence_head"),
)
# MHC head CustomOp dispatcher (aiter / tilelang / triton / torch),
# replacing the direct nvidia tilelang kernel call.
@@ -358,6 +365,17 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.bias(markov_embed, self.logits_processor)
def compute_confidence(
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
) -> torch.Tensor:
"""Per-position acceptance probability for each drafted token."""
if self.model.confidence_head is None:
raise RuntimeError(
"compute_confidence() requires a confidence head, but the "
"checkpoint did not provide confidence_head weights."
)
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
# --- Weight loading ----------------------------------------------------
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -391,6 +409,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
loaded_confidence_head = False
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
@@ -403,6 +422,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
if mapped is None:
continue
name = mapped
if "confidence_head." in name:
loaded_confidence_head = True
# ``.scale`` -> per-method scale suffix.
if name.endswith(".scale"):
@@ -439,8 +460,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
continue
# Stacked rules only apply to decoder-layer weights. Head-stack params
# (main_proj/norm/hc_head/markov_head) load directly — otherwise e.g.
# "markov_w1" would collide with the "w1" shard rule.
# (main_proj/norm/hc_head/markov_head/confidence_head) load directly —
# otherwise e.g. "markov_w1" would collide with the "w1" shard rule.
is_layer_param = name.startswith("model.layers.")
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
if not is_layer_param or weight_name not in name:
@@ -469,6 +490,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
weight_loader(param, loaded_weight)
loaded_params.add(name)
if self.model.confidence_head is not None and not loaded_confidence_head:
self.model.confidence_head = None
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
return loaded_params
@@ -482,8 +505,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
return None
stage = int(m.group(1))
rest = m.group(2)
# The confidence head is not wired into inference yet; drop its weights.
if rest.startswith("confidence_head."):
if rest.startswith("confidence_head.") and self.model.confidence_head is None:
return None
# Head-stack params live at model level (mtp.last), context combiner at
# model level (mtp.0); everything else is a per-layer decoder block.
@@ -493,6 +515,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
"hc_head_base",
"hc_head_scale",
"markov_head.",
"confidence_head.",
)
if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith(
head_prefixes
+31
View File
@@ -8,6 +8,7 @@ from typing import cast
import torch
from vllm import envs
from vllm.config import VllmConfig
from vllm.distributed import (
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_reduce,
@@ -26,6 +27,7 @@ from vllm.platforms.rocm import _ON_GFX950
from vllm.triton_utils import tl, triton
from vllm.utils.multi_stream_utils import execute_in_parallel
from vllm.v1.attention.backend import (
AttentionCGSupport,
CommonAttentionMetadata,
)
from vllm.v1.attention.backends.mla.sparse_swa import (
@@ -38,6 +40,7 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
rocm_sparse_attn_decode,
rocm_sparse_attn_prefill,
)
from vllm.v1.kv_cache_interface import KVCacheSpec
from vllm.v1.worker.workspace import current_workspace_manager
logger = init_logger(__name__)
@@ -445,6 +448,20 @@ class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata):
class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
@classmethod
def get_cudagraph_support(
cls,
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> AttentionCGSupport:
spec_config = vllm_config.speculative_config
if spec_config is not None and spec_config.enable_adaptive_verification:
# All per-token metadata is built from device query boundaries into
# persistent buffers, so adaptive verification can replay varlen
# FULL decode graphs after reallocating drafts across requests.
return AttentionCGSupport.ALWAYS
return super().get_cudagraph_support(vllm_config, kv_cache_spec)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.c128a_decode_topk_ragged_indices_buffer: torch.Tensor | None = None
@@ -512,6 +529,20 @@ class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4SparseMLAMetadataBui
class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
@classmethod
def get_cudagraph_support(
cls,
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> AttentionCGSupport:
spec_config = vllm_config.speculative_config
if spec_config is not None and spec_config.enable_adaptive_verification:
# SWA indices, lengths, and token-to-request mappings are built from
# device boundaries into persistent buffers, so adaptive verification
# can replay varlen FULL decode graphs safely.
return AttentionCGSupport.ALWAYS
return super().get_cudagraph_support(vllm_config, kv_cache_spec)
# Keep fused multi-step decode disabled until update_draft_decode_metadata()
# also refreshes the ROCm-specific ragged SWA indices and indptrs.
supports_draft_decode_metadata_update = False
+28 -1
View File
@@ -237,6 +237,15 @@ class KpoolTailBackend(DeepseekV32IndexerBackend):
class DeepseekV4IndexerBackend(DeepseekV32IndexerBackend):
@classmethod
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
# ROCm runs adaptive verification through the per-token flattened
# indexer path, which derives row ownership from device decode lengths.
return (
current_platform.is_rocm()
or super().supports_device_cpu_query_lens_mismatch()
)
@staticmethod
def get_name() -> str:
return "DEEPSEEK_V4_INDEXER"
@@ -254,6 +263,12 @@ class DeepseekV4IndexerBackend(DeepseekV32IndexerBackend):
class DeepseekV41IndexerBackend(DeepseekV4IndexerBackend):
@classmethod
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
# The ROCm flattened-query support above is validated for the
# DeepSeek-V4 adaptive DSpark path only.
return DeepseekV32IndexerBackend.supports_device_cpu_query_lens_mismatch()
@staticmethod
def get_name() -> str:
return "DEEPSEEK_V41_INDEXER"
@@ -802,6 +817,15 @@ def _supports_flattened_device_query_lens() -> bool:
)
def _rocm_supports_flattened_device_query_lens(vllm_config: VllmConfig) -> bool:
model_config = vllm_config.model_config
return (
current_platform.is_rocm()
and model_config is not None
and "DeepseekV4ForCausalLM" in model_config.architectures
)
def _supports_native_decode(next_n: int) -> bool:
"""Whether decode can pass `next_n` Q rows per request to the kernel
instead of flattening to one single-token row per query, which re-reads
@@ -822,7 +846,10 @@ def _use_flattening(vllm_config: VllmConfig) -> bool:
return not _supports_native_decode(next_n) or (
speculative_config is not None
and speculative_config.enable_adaptive_verification
and _supports_flattened_device_query_lens()
and (
_supports_flattened_device_query_lens()
or _rocm_supports_flattened_device_query_lens(vllm_config)
)
)