[ROCm][DSv4] Enable DSpark adaptive verification (#52362)
Signed-off-by: Tuukka Sarvi <tuukka.sarvi@amd.com> Signed-off-by: larryli2-amd <larryli2@amd.com> Co-authored-by: larryli2-amd <larryli2@amd.com>
This commit is contained in:
co-authored by
larryli2-amd
parent
21aa17c128
commit
e34685dfc0
@@ -44,6 +44,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.qwen3_dspark import (
|
||||
DSparkConfidenceHead,
|
||||
DSparkMarkovHead,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
@@ -102,7 +103,7 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
]
|
||||
)
|
||||
|
||||
# Heads: final norm + hc_head, and the Markov head
|
||||
# Heads: final norm + hc_head, and the Markov + confidence heads
|
||||
# Loaded from the "final" MTP layer weights (mtp.*) in the target checkpoint
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
hc_dim = self.hc_mult * config.hidden_size
|
||||
@@ -125,6 +126,12 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
config.dspark_markov_rank,
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
)
|
||||
self.confidence_head: DSparkConfidenceHead | None = None
|
||||
if getattr(config, "enable_confidence_head", True):
|
||||
self.confidence_head = DSparkConfidenceHead(
|
||||
config.hidden_size + config.dspark_markov_rank,
|
||||
prefix=maybe_prefix(prefix, "confidence_head"),
|
||||
)
|
||||
|
||||
# MHC head CustomOp dispatcher (aiter / tilelang / triton / torch),
|
||||
# replacing the direct nvidia tilelang kernel call.
|
||||
@@ -358,6 +365,17 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.bias(markov_embed, self.logits_processor)
|
||||
|
||||
def compute_confidence(
|
||||
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Per-position acceptance probability for each drafted token."""
|
||||
if self.model.confidence_head is None:
|
||||
raise RuntimeError(
|
||||
"compute_confidence() requires a confidence head, but the "
|
||||
"checkpoint did not provide confidence_head weights."
|
||||
)
|
||||
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
|
||||
|
||||
# --- Weight loading ----------------------------------------------------
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
@@ -391,6 +409,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
loaded_confidence_head = False
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
@@ -403,6 +422,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
if mapped is None:
|
||||
continue
|
||||
name = mapped
|
||||
if "confidence_head." in name:
|
||||
loaded_confidence_head = True
|
||||
|
||||
# ``.scale`` -> per-method scale suffix.
|
||||
if name.endswith(".scale"):
|
||||
@@ -439,8 +460,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
continue
|
||||
|
||||
# Stacked rules only apply to decoder-layer weights. Head-stack params
|
||||
# (main_proj/norm/hc_head/markov_head) load directly — otherwise e.g.
|
||||
# "markov_w1" would collide with the "w1" shard rule.
|
||||
# (main_proj/norm/hc_head/markov_head/confidence_head) load directly —
|
||||
# otherwise e.g. "markov_w1" would collide with the "w1" shard rule.
|
||||
is_layer_param = name.startswith("model.layers.")
|
||||
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
|
||||
if not is_layer_param or weight_name not in name:
|
||||
@@ -469,6 +490,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
if self.model.confidence_head is not None and not loaded_confidence_head:
|
||||
self.model.confidence_head = None
|
||||
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
@@ -482,8 +505,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
return None
|
||||
stage = int(m.group(1))
|
||||
rest = m.group(2)
|
||||
# The confidence head is not wired into inference yet; drop its weights.
|
||||
if rest.startswith("confidence_head."):
|
||||
if rest.startswith("confidence_head.") and self.model.confidence_head is None:
|
||||
return None
|
||||
# Head-stack params live at model level (mtp.last), context combiner at
|
||||
# model level (mtp.0); everything else is a per-layer decoder block.
|
||||
@@ -493,6 +515,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
"hc_head_base",
|
||||
"hc_head_scale",
|
||||
"markov_head.",
|
||||
"confidence_head.",
|
||||
)
|
||||
if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith(
|
||||
head_prefixes
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import cast
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
@@ -26,6 +27,7 @@ from vllm.platforms.rocm import _ON_GFX950
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.multi_stream_utils import execute_in_parallel
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionCGSupport,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
@@ -38,6 +40,7 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_sparse_attn_decode,
|
||||
rocm_sparse_attn_prefill,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -445,6 +448,20 @@ class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata):
|
||||
|
||||
|
||||
class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
|
||||
@classmethod
|
||||
def get_cudagraph_support(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
) -> AttentionCGSupport:
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is not None and spec_config.enable_adaptive_verification:
|
||||
# All per-token metadata is built from device query boundaries into
|
||||
# persistent buffers, so adaptive verification can replay varlen
|
||||
# FULL decode graphs after reallocating drafts across requests.
|
||||
return AttentionCGSupport.ALWAYS
|
||||
return super().get_cudagraph_support(vllm_config, kv_cache_spec)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.c128a_decode_topk_ragged_indices_buffer: torch.Tensor | None = None
|
||||
@@ -512,6 +529,20 @@ class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4SparseMLAMetadataBui
|
||||
|
||||
|
||||
class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
|
||||
@classmethod
|
||||
def get_cudagraph_support(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
) -> AttentionCGSupport:
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is not None and spec_config.enable_adaptive_verification:
|
||||
# SWA indices, lengths, and token-to-request mappings are built from
|
||||
# device boundaries into persistent buffers, so adaptive verification
|
||||
# can replay varlen FULL decode graphs safely.
|
||||
return AttentionCGSupport.ALWAYS
|
||||
return super().get_cudagraph_support(vllm_config, kv_cache_spec)
|
||||
|
||||
# Keep fused multi-step decode disabled until update_draft_decode_metadata()
|
||||
# also refreshes the ROCm-specific ragged SWA indices and indptrs.
|
||||
supports_draft_decode_metadata_update = False
|
||||
|
||||
Reference in New Issue
Block a user