[Spec Decode] DSpark confidence-scheduled verification (#47808)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com>
Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-08-12 12:27:48 -07:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5 Benjamin Chislett mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Nick Hill OpenAI Codex
parent fe889ac925
commit 7f7a32cfec
40 changed files with 1643 additions and 111 deletions
+4 -4
View File
@@ -14,9 +14,9 @@ from vllm.forward_context import get_forward_context
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
from vllm.models.deepseek_v4.common.ops import dequantize_and_gather_k_cache
from vllm.models.deepseek_v4.sparse_mla import (
DeepseekV4FlashMLABackend,
DeepseekV4FlashMLAMetadata,
DeepseekV4FlashMLAMetadataBuilder,
DeepseekV4SparseMLABackend,
DeepseekV4SparseMLAMetadataBuilder,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@@ -314,7 +314,7 @@ class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata):
decode_swa_ragged_indptr: torch.Tensor | None = None
class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4FlashMLAMetadataBuilder):
class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.c128a_decode_topk_ragged_indices_buffer: torch.Tensor | None = None
@@ -435,7 +435,7 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild
)
class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4FlashMLABackend):
class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4SparseMLABackend):
@staticmethod
def get_name() -> str:
return "ROCM_FLASHMLA_SPARSE_DSV4"
+3
View File
@@ -134,6 +134,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# Provided by the platform subclass.
backend_cls: ClassVar[type[AttentionBackend]]
# Backend for the SWA cache layer; None uses the default SWA backend.
swa_backend_cls: ClassVar[type[AttentionBackend] | None] = None
# KV-cache per-token block format (both layouts are paged). True (default)
# = fp8_ds_mla (UE8M0 block-scaled fp8 packed as uint8); False = plain
# bf16 / per-tensor fp8 KV row. Backends can override the instance hook when
@@ -322,6 +324,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
dtype=self.kv_cache_torch_dtype,
prefix=f"{prefix}.swa_cache",
cache_config=cache_config,
backend_cls=self.swa_backend_cls,
)
# Register with compilation context for metadata lookup.
+24 -5
View File
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.qwen3_dspark import (
DSparkConfidenceHead,
DSparkMarkovHead,
)
from vllm.model_executor.models.utils import maybe_prefix
@@ -113,7 +114,7 @@ class DSparkDeepseekV4Model(nn.Module):
]
)
# Heads: final norm + hc_head, and the Markov head
# Heads: final norm + hc_head, and the Markov + confidence heads
# Loaded from the "final" MTP layer weights (mtp.*) in the target checkpoint
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
hc_dim = self.hc_mult * config.hidden_size
@@ -136,6 +137,12 @@ class DSparkDeepseekV4Model(nn.Module):
config.dspark_markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
)
self.confidence_head: DSparkConfidenceHead | None = None
if getattr(config, "enable_confidence_head", True):
self.confidence_head = DSparkConfidenceHead(
config.hidden_size + config.dspark_markov_rank,
prefix=maybe_prefix(prefix, "confidence_head"),
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
@@ -369,6 +376,13 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.bias(markov_embed, self.logits_processor)
def compute_confidence(
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
) -> torch.Tensor:
"""Per-position acceptance probability for each drafted token."""
assert self.model.confidence_head is not None
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
# --- Weight loading ----------------------------------------------------
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -407,6 +421,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
loaded_confidence_head = False
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
@@ -419,6 +434,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
if mapped is None:
continue
name = mapped
if "confidence_head." in name:
loaded_confidence_head = True
# ``.scale`` -> per-method scale suffix.
if name.endswith(".scale"):
@@ -461,8 +478,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
continue
# Stacked rules only apply to decoder-layer weights. Head-stack params
# (main_proj/norm/hc_head/markov_head) load directly — otherwise e.g.
# "markov_w1" would collide with the "w1" shard rule.
# (main_proj/norm/hc_head/markov_head/confidence_head) load directly —
# otherwise e.g. "markov_w1" would collide with the "w1" shard rule.
is_layer_param = name.startswith("model.layers.")
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
if not is_layer_param or weight_name not in name:
@@ -488,6 +505,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params.add(name)
self._finalize_moe()
if self.model.confidence_head is not None and not loaded_confidence_head:
self.model.confidence_head = None
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
return loaded_params
@@ -505,8 +524,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
return None
stage = int(m.group(1))
rest = m.group(2)
# The confidence head is not wired into inference yet; drop its weights.
if rest.startswith("confidence_head."):
if rest.startswith("confidence_head.") and self.model.confidence_head is None:
return None
# Head-stack params live at model level (mtp.last), context combiner at
# model level (mtp.0); everything else is a per-layer decoder block.
@@ -516,6 +534,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
"hc_head_base",
"hc_head_scale",
"markov_head.",
"confidence_head.",
)
if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith(
head_prefixes
@@ -18,8 +18,8 @@ from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
deep_gemm_fp8_o_proj,
)
from vllm.models.deepseek_v4.sparse_mla import (
DeepseekV4FlashMLABackend,
DeepseekV4FlashMLAMetadata,
DeepseekV4SparseMLABackend,
)
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
@@ -74,11 +74,10 @@ def _pad_to_supported_q_heads(num_heads: int) -> int:
)
class DeepseekV4FlashInferMLASparseBackend(DeepseekV4FlashMLABackend):
class DeepseekV4FlashInferMLASparseBackend(DeepseekV4SparseMLABackend):
"""FlashInfer backend using the DSv4 sparse metadata/cache layout.
Inheriting from the FlashMLA V4 backend reuses its
``DeepseekV4FlashMLAMetadata`` builder.
Inherits the base and backend reuses its``DeepseekV4SparseMLAMetadataBuilder``
"""
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
@@ -159,7 +158,7 @@ class DeepseekV4FlashInferMLASparseBackend(DeepseekV4FlashMLABackend):
) -> tuple[int, ...]:
device_capability = current_platform.get_device_capability()
if device_capability is not None and device_capability.major == 12:
return DeepseekV4FlashMLABackend.get_kv_cache_shape(
return DeepseekV4SparseMLABackend.get_kv_cache_shape(
num_blocks,
block_size,
num_kv_heads,
@@ -489,8 +488,6 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
# uniform-q batches, and this avoids flattening mixed batches into one call.
if num_decode_tokens > 0:
decode_cu = query_start_loc[: num_decodes + 1]
decode_cu_cpu = query_start_loc_cpu[: num_decodes + 1]
decode_lens_cpu = decode_cu_cpu[1:] - decode_cu_cpu[:-1]
flashinfer_trtllm_batch_decode_sparse_mla_dsv4(
query=query[:num_decode_tokens],
swa_kv_cache=swa_k_cache,
@@ -504,7 +501,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
bmm2_scale=bmm2_scale,
sinks=self.attn_sink,
cum_seq_lens_q=decode_cu,
max_q_len=int(decode_lens_cpu.max().item()),
max_q_len=swa_metadata.max_decode_query_len,
)
if num_prefill_tokens > 0:
+19 -1
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, cast
from typing import TYPE_CHECKING, ClassVar, cast
import torch
@@ -21,6 +21,11 @@ from vllm.models.deepseek_v4.sparse_mla import (
DeepseekV4FlashMLAMetadata,
)
from vllm.utils.math_utils import round_up
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.attention.backends.mla.sparse_swa import (
DeepseekSparseSWABackend,
DeepseekSparseSWAMetadataBuilder,
)
from vllm.v1.attention.ops.flashmla import (
flash_mla_sparse_fwd,
flash_mla_with_kvcache,
@@ -31,10 +36,23 @@ if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
class DeepseekSparseSWAFlashMLAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
"""SWA metadata for the FlashMLA decode path, which allows varlen decode."""
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
class DeepseekSparseSWAFlashMLABackend(DeepseekSparseSWABackend):
@staticmethod
def get_builder_cls() -> type[DeepseekSparseSWAFlashMLAMetadataBuilder]:
return DeepseekSparseSWAFlashMLAMetadataBuilder
class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
"""FlashMLA sparse MLA attention layer for DeepSeek V4 (CUDA)."""
backend_cls = DeepseekV4FlashMLABackend
swa_backend_cls = DeepseekSparseSWAFlashMLABackend
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
+20 -10
View File
@@ -32,8 +32,8 @@ from vllm.v1.kv_cache_interface import AttentionSpec
_C128A_TOPK_ALIGNMENT = 128
class DeepseekV4FlashMLABackend(AttentionBackend):
"""DeepSeek-V4 sparse-MLA backend.
class DeepseekV4SparseMLABackend(AttentionBackend):
"""DeepSeek-V4 sparse-MLA backend base.
Subclasses ``AttentionBackend`` directly (not the V3.2
``FlashMLASparseBackend``): DeepSeek-V4 runs its own attention layer
@@ -54,12 +54,8 @@ class DeepseekV4FlashMLABackend(AttentionBackend):
return [256]
@staticmethod
def get_name() -> str:
return "FLASHMLA_SPARSE_DSV4"
@staticmethod
def get_builder_cls() -> type["DeepseekV4FlashMLAMetadataBuilder"]:
return DeepseekV4FlashMLAMetadataBuilder
def get_builder_cls() -> type["DeepseekV4SparseMLAMetadataBuilder"]:
return DeepseekV4SparseMLAMetadataBuilder
@staticmethod
def get_impl_cls() -> type[Any]:
@@ -67,7 +63,7 @@ class DeepseekV4FlashMLABackend(AttentionBackend):
# not the generic ``Attention``/``MLAAttention`` layer, so the backend's
# impl class is never instantiated.
raise NotImplementedError(
"DeepseekV4FlashMLABackend has no separate impl class; DeepSeek-V4 "
"DeepseekV4SparseMLABackend has no separate impl class; DeepSeek-V4 "
"attention runs through DeepseekV4Attention."
)
@@ -131,7 +127,7 @@ class DeepseekV4FlashMLAMetadata(AttentionMetadata):
c128a_prefill_topk_indices: torch.Tensor | None = None
class DeepseekV4FlashMLAMetadataBuilder(
class DeepseekV4SparseMLAMetadataBuilder(
AttentionMetadataBuilder[DeepseekV4FlashMLAMetadata]
):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
@@ -294,6 +290,20 @@ class DeepseekV4FlashMLAMetadataBuilder(
return result
class DeepseekV4FlashMLAMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
class DeepseekV4FlashMLABackend(DeepseekV4SparseMLABackend):
@staticmethod
def get_name() -> str:
return "FLASHMLA_SPARSE_DSV4"
@staticmethod
def get_builder_cls() -> type[DeepseekV4FlashMLAMetadataBuilder]:
return DeepseekV4FlashMLAMetadataBuilder
def build_c128a_topk_metadata(
positions: torch.Tensor,
compress_ratio: int,
+2 -2
View File
@@ -19,8 +19,8 @@ from vllm.models.deepseek_v4.common.ops import (
dequantize_and_gather_k_cache,
)
from vllm.models.deepseek_v4.sparse_mla import (
DeepseekV4FlashMLABackend,
DeepseekV4FlashMLAMetadata,
DeepseekV4SparseMLABackend,
)
from vllm.models.deepseek_v4.xpu.xpu_sparse_decode_fp8 import (
xpu_sparse_decode_fp8,
@@ -32,7 +32,7 @@ if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
class DeepseekV4XPUSparseBackend(DeepseekV4FlashMLABackend):
class DeepseekV4XPUSparseBackend(DeepseekV4SparseMLABackend):
@staticmethod
def get_name() -> str:
return "XPU_V4_MLA_SPARSE"