[Spec Decode] DSpark confidence-scheduled verification (#47808)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Nick Hill
OpenAI Codex
parent
fe889ac925
commit
7f7a32cfec
@@ -14,9 +14,9 @@ from vllm.forward_context import get_forward_context
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v4.common.ops import dequantize_and_gather_k_cache
|
||||
from vllm.models.deepseek_v4.sparse_mla import (
|
||||
DeepseekV4FlashMLABackend,
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
DeepseekV4FlashMLAMetadataBuilder,
|
||||
DeepseekV4SparseMLABackend,
|
||||
DeepseekV4SparseMLAMetadataBuilder,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
@@ -314,7 +314,7 @@ class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata):
|
||||
decode_swa_ragged_indptr: torch.Tensor | None = None
|
||||
|
||||
|
||||
class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4FlashMLAMetadataBuilder):
|
||||
class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.c128a_decode_topk_ragged_indices_buffer: torch.Tensor | None = None
|
||||
@@ -435,7 +435,7 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4FlashMLABackend):
|
||||
class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4SparseMLABackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "ROCM_FLASHMLA_SPARSE_DSV4"
|
||||
|
||||
@@ -134,6 +134,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
|
||||
# Provided by the platform subclass.
|
||||
backend_cls: ClassVar[type[AttentionBackend]]
|
||||
# Backend for the SWA cache layer; None uses the default SWA backend.
|
||||
swa_backend_cls: ClassVar[type[AttentionBackend] | None] = None
|
||||
# KV-cache per-token block format (both layouts are paged). True (default)
|
||||
# = fp8_ds_mla (UE8M0 block-scaled fp8 packed as uint8); False = plain
|
||||
# bf16 / per-tensor fp8 KV row. Backends can override the instance hook when
|
||||
@@ -322,6 +324,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
prefix=f"{prefix}.swa_cache",
|
||||
cache_config=cache_config,
|
||||
backend_cls=self.swa_backend_cls,
|
||||
)
|
||||
|
||||
# Register with compilation context for metadata lookup.
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.qwen3_dspark import (
|
||||
DSparkConfidenceHead,
|
||||
DSparkMarkovHead,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
@@ -113,7 +114,7 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
]
|
||||
)
|
||||
|
||||
# Heads: final norm + hc_head, and the Markov head
|
||||
# Heads: final norm + hc_head, and the Markov + confidence heads
|
||||
# Loaded from the "final" MTP layer weights (mtp.*) in the target checkpoint
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
hc_dim = self.hc_mult * config.hidden_size
|
||||
@@ -136,6 +137,12 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
config.dspark_markov_rank,
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
)
|
||||
self.confidence_head: DSparkConfidenceHead | None = None
|
||||
if getattr(config, "enable_confidence_head", True):
|
||||
self.confidence_head = DSparkConfidenceHead(
|
||||
config.hidden_size + config.dspark_markov_rank,
|
||||
prefix=maybe_prefix(prefix, "confidence_head"),
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
@@ -369,6 +376,13 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.bias(markov_embed, self.logits_processor)
|
||||
|
||||
def compute_confidence(
|
||||
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Per-position acceptance probability for each drafted token."""
|
||||
assert self.model.confidence_head is not None
|
||||
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
|
||||
|
||||
# --- Weight loading ----------------------------------------------------
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
@@ -407,6 +421,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
loaded_confidence_head = False
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
@@ -419,6 +434,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
if mapped is None:
|
||||
continue
|
||||
name = mapped
|
||||
if "confidence_head." in name:
|
||||
loaded_confidence_head = True
|
||||
|
||||
# ``.scale`` -> per-method scale suffix.
|
||||
if name.endswith(".scale"):
|
||||
@@ -461,8 +478,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
continue
|
||||
|
||||
# Stacked rules only apply to decoder-layer weights. Head-stack params
|
||||
# (main_proj/norm/hc_head/markov_head) load directly — otherwise e.g.
|
||||
# "markov_w1" would collide with the "w1" shard rule.
|
||||
# (main_proj/norm/hc_head/markov_head/confidence_head) load directly —
|
||||
# otherwise e.g. "markov_w1" would collide with the "w1" shard rule.
|
||||
is_layer_param = name.startswith("model.layers.")
|
||||
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
|
||||
if not is_layer_param or weight_name not in name:
|
||||
@@ -488,6 +505,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params.add(name)
|
||||
|
||||
self._finalize_moe()
|
||||
if self.model.confidence_head is not None and not loaded_confidence_head:
|
||||
self.model.confidence_head = None
|
||||
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
@@ -505,8 +524,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
return None
|
||||
stage = int(m.group(1))
|
||||
rest = m.group(2)
|
||||
# The confidence head is not wired into inference yet; drop its weights.
|
||||
if rest.startswith("confidence_head."):
|
||||
if rest.startswith("confidence_head.") and self.model.confidence_head is None:
|
||||
return None
|
||||
# Head-stack params live at model level (mtp.last), context combiner at
|
||||
# model level (mtp.0); everything else is a per-layer decoder block.
|
||||
@@ -516,6 +534,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
"hc_head_base",
|
||||
"hc_head_scale",
|
||||
"markov_head.",
|
||||
"confidence_head.",
|
||||
)
|
||||
if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith(
|
||||
head_prefixes
|
||||
|
||||
@@ -18,8 +18,8 @@ from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
|
||||
deep_gemm_fp8_o_proj,
|
||||
)
|
||||
from vllm.models.deepseek_v4.sparse_mla import (
|
||||
DeepseekV4FlashMLABackend,
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
DeepseekV4SparseMLABackend,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
@@ -74,11 +74,10 @@ def _pad_to_supported_q_heads(num_heads: int) -> int:
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4FlashInferMLASparseBackend(DeepseekV4FlashMLABackend):
|
||||
class DeepseekV4FlashInferMLASparseBackend(DeepseekV4SparseMLABackend):
|
||||
"""FlashInfer backend using the DSv4 sparse metadata/cache layout.
|
||||
|
||||
Inheriting from the FlashMLA V4 backend reuses its
|
||||
``DeepseekV4FlashMLAMetadata`` builder.
|
||||
Inherits the base and backend reuses its``DeepseekV4SparseMLAMetadataBuilder``
|
||||
"""
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||
@@ -159,7 +158,7 @@ class DeepseekV4FlashInferMLASparseBackend(DeepseekV4FlashMLABackend):
|
||||
) -> tuple[int, ...]:
|
||||
device_capability = current_platform.get_device_capability()
|
||||
if device_capability is not None and device_capability.major == 12:
|
||||
return DeepseekV4FlashMLABackend.get_kv_cache_shape(
|
||||
return DeepseekV4SparseMLABackend.get_kv_cache_shape(
|
||||
num_blocks,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
@@ -489,8 +488,6 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
# uniform-q batches, and this avoids flattening mixed batches into one call.
|
||||
if num_decode_tokens > 0:
|
||||
decode_cu = query_start_loc[: num_decodes + 1]
|
||||
decode_cu_cpu = query_start_loc_cpu[: num_decodes + 1]
|
||||
decode_lens_cpu = decode_cu_cpu[1:] - decode_cu_cpu[:-1]
|
||||
flashinfer_trtllm_batch_decode_sparse_mla_dsv4(
|
||||
query=query[:num_decode_tokens],
|
||||
swa_kv_cache=swa_k_cache,
|
||||
@@ -504,7 +501,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
bmm2_scale=bmm2_scale,
|
||||
sinks=self.attn_sink,
|
||||
cum_seq_lens_q=decode_cu,
|
||||
max_q_len=int(decode_lens_cpu.max().item()),
|
||||
max_q_len=swa_metadata.max_decode_query_len,
|
||||
)
|
||||
|
||||
if num_prefill_tokens > 0:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING, cast
|
||||
from typing import TYPE_CHECKING, ClassVar, cast
|
||||
|
||||
import torch
|
||||
|
||||
@@ -21,6 +21,11 @@ from vllm.models.deepseek_v4.sparse_mla import (
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
)
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
DeepseekSparseSWABackend,
|
||||
DeepseekSparseSWAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.ops.flashmla import (
|
||||
flash_mla_sparse_fwd,
|
||||
flash_mla_with_kvcache,
|
||||
@@ -31,10 +36,23 @@ if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
|
||||
class DeepseekSparseSWAFlashMLAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
|
||||
"""SWA metadata for the FlashMLA decode path, which allows varlen decode."""
|
||||
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
|
||||
|
||||
|
||||
class DeepseekSparseSWAFlashMLABackend(DeepseekSparseSWABackend):
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[DeepseekSparseSWAFlashMLAMetadataBuilder]:
|
||||
return DeepseekSparseSWAFlashMLAMetadataBuilder
|
||||
|
||||
|
||||
class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
|
||||
"""FlashMLA sparse MLA attention layer for DeepSeek V4 (CUDA)."""
|
||||
|
||||
backend_cls = DeepseekV4FlashMLABackend
|
||||
swa_backend_cls = DeepseekSparseSWAFlashMLABackend
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
@@ -32,8 +32,8 @@ from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
_C128A_TOPK_ALIGNMENT = 128
|
||||
|
||||
|
||||
class DeepseekV4FlashMLABackend(AttentionBackend):
|
||||
"""DeepSeek-V4 sparse-MLA backend.
|
||||
class DeepseekV4SparseMLABackend(AttentionBackend):
|
||||
"""DeepSeek-V4 sparse-MLA backend base.
|
||||
|
||||
Subclasses ``AttentionBackend`` directly (not the V3.2
|
||||
``FlashMLASparseBackend``): DeepSeek-V4 runs its own attention layer
|
||||
@@ -54,12 +54,8 @@ class DeepseekV4FlashMLABackend(AttentionBackend):
|
||||
return [256]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHMLA_SPARSE_DSV4"
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["DeepseekV4FlashMLAMetadataBuilder"]:
|
||||
return DeepseekV4FlashMLAMetadataBuilder
|
||||
def get_builder_cls() -> type["DeepseekV4SparseMLAMetadataBuilder"]:
|
||||
return DeepseekV4SparseMLAMetadataBuilder
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type[Any]:
|
||||
@@ -67,7 +63,7 @@ class DeepseekV4FlashMLABackend(AttentionBackend):
|
||||
# not the generic ``Attention``/``MLAAttention`` layer, so the backend's
|
||||
# impl class is never instantiated.
|
||||
raise NotImplementedError(
|
||||
"DeepseekV4FlashMLABackend has no separate impl class; DeepSeek-V4 "
|
||||
"DeepseekV4SparseMLABackend has no separate impl class; DeepSeek-V4 "
|
||||
"attention runs through DeepseekV4Attention."
|
||||
)
|
||||
|
||||
@@ -131,7 +127,7 @@ class DeepseekV4FlashMLAMetadata(AttentionMetadata):
|
||||
c128a_prefill_topk_indices: torch.Tensor | None = None
|
||||
|
||||
|
||||
class DeepseekV4FlashMLAMetadataBuilder(
|
||||
class DeepseekV4SparseMLAMetadataBuilder(
|
||||
AttentionMetadataBuilder[DeepseekV4FlashMLAMetadata]
|
||||
):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
@@ -294,6 +290,20 @@ class DeepseekV4FlashMLAMetadataBuilder(
|
||||
return result
|
||||
|
||||
|
||||
class DeepseekV4FlashMLAMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
|
||||
|
||||
|
||||
class DeepseekV4FlashMLABackend(DeepseekV4SparseMLABackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHMLA_SPARSE_DSV4"
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[DeepseekV4FlashMLAMetadataBuilder]:
|
||||
return DeepseekV4FlashMLAMetadataBuilder
|
||||
|
||||
|
||||
def build_c128a_topk_metadata(
|
||||
positions: torch.Tensor,
|
||||
compress_ratio: int,
|
||||
|
||||
@@ -19,8 +19,8 @@ from vllm.models.deepseek_v4.common.ops import (
|
||||
dequantize_and_gather_k_cache,
|
||||
)
|
||||
from vllm.models.deepseek_v4.sparse_mla import (
|
||||
DeepseekV4FlashMLABackend,
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
DeepseekV4SparseMLABackend,
|
||||
)
|
||||
from vllm.models.deepseek_v4.xpu.xpu_sparse_decode_fp8 import (
|
||||
xpu_sparse_decode_fp8,
|
||||
@@ -32,7 +32,7 @@ if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
|
||||
class DeepseekV4XPUSparseBackend(DeepseekV4FlashMLABackend):
|
||||
class DeepseekV4XPUSparseBackend(DeepseekV4SparseMLABackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "XPU_V4_MLA_SPARSE"
|
||||
|
||||
Reference in New Issue
Block a user