[Spec Decode] DSpark confidence-scheduled verification (#47808)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com>
Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-08-12 12:27:48 -07:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5 Benjamin Chislett mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Nick Hill OpenAI Codex
parent fe889ac925
commit 7f7a32cfec
40 changed files with 1643 additions and 111 deletions
@@ -609,6 +609,7 @@ def sparse_attn_indexer(
decode_metadata.schedule_metadata,
max_model_len=max_model_len,
clean_logits=False,
indices=decode_metadata.indices,
)
num_rows = logits.shape[0]
topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
+58 -4
View File
@@ -10,8 +10,9 @@ The parallel backbone is a standard Qwen3 decoder stack reused from the
DFlash Qwen3 draft (see qwen3_dflash.py). DSpark adds:
* ``markov_head``: low-rank V x r / r x V transition bias added to the base
logits, sampled left-to-right by the speculator (the sequential stage).
* ``confidence_head``: per-position acceptance-probability estimate.
DSparkMarkovHead is shared with the DSV4-style DSpark model.
DSparkMarkovHead and DSparkConfidenceHead are shared with the DSV4-style DSpark model.
"""
from collections.abc import Iterable
@@ -21,6 +22,7 @@ import torch.nn as nn
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -102,8 +104,36 @@ class DSparkMarkovHead(nn.Module):
return logits.scatter_(1, index, corrected.squeeze(-1))
class DSparkConfidenceHead(nn.Module):
"""DSpark acceptance-confidence head."""
def __init__(
self,
input_dim: int,
prefix: str,
bias: bool = False,
with_markov: bool = True,
) -> None:
super().__init__()
self.with_markov = with_markov
self.proj = ReplicatedLinear(
input_dim,
1,
bias=bias,
return_bias=False,
params_dtype=torch.float32,
prefix=maybe_prefix(prefix, "proj"),
)
def forward(self, hidden: torch.Tensor, markov_embed: torch.Tensor) -> torch.Tensor:
x = (
torch.cat([hidden, markov_embed], dim=-1) if self.with_markov else hidden
).float()
return self.proj(x).squeeze(-1)
class Qwen3DSparkModel(DFlashQwen3Model):
"""DFlash Qwen3 backbone + DSpark Markov head."""
"""DFlash Qwen3 backbone + DSpark Markov / confidence heads."""
def __init__(
self,
@@ -126,6 +156,18 @@ class Qwen3DSparkModel(DFlashQwen3Model):
prefix=maybe_prefix(prefix, "markov_head"),
quant_config=self.quant_config,
)
self.confidence_head: DSparkConfidenceHead | None = None
if getattr(config, "enable_confidence_head", False):
with_markov = getattr(config, "confidence_head_with_markov", False)
input_dim = config.hidden_size
if with_markov:
input_dim += config.markov_rank
self.confidence_head = DSparkConfidenceHead(
input_dim,
prefix=maybe_prefix(prefix, "confidence_head"),
bias=True,
with_markov=with_markov,
)
class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
@@ -197,11 +239,19 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
self.logits_processor.scale,
)
def compute_confidence(
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
) -> torch.Tensor:
"""Per-position acceptance probability for each drafted token."""
assert self.model.confidence_head is not None
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
model_weights = {}
includes_embed_tokens = False
includes_lm_head = False
includes_draft_id_mapping = False
includes_confidence_head = False
for name, loaded_weight in weights:
# t2d is training-only; the draft remaps via d2t at sampling time.
if "t2d" in name:
@@ -215,22 +265,26 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
includes_embed_tokens = True
if "lm_head" in name:
includes_lm_head = True
if "confidence_head" in name:
includes_confidence_head = True
model_weights[name] = loaded_weight
# Sets has_own_embed_tokens / has_own_lm_head so load_dspark_model
# knows whether to keep these or alias the target's.
process_eagle_weight(self, name)
# mask_embedding is an unused placeholder param; DSpark masks via the vocab row.
# confidence_head is not wired into inference yet; skip its weights.
# embed_tokens / lm_head are optional; when omitted they are shared from
# the target by load_dspark_model, so skip the unloaded params here.
skip_substrs = ["mask_embedding", "confidence_head"]
skip_substrs = ["mask_embedding"]
if not includes_embed_tokens:
skip_substrs.append("embed_tokens")
if not includes_lm_head:
skip_substrs.append("lm_head")
if not includes_draft_id_mapping:
skip_substrs.append("draft_id_to_target_id")
if self.model.confidence_head is None or not includes_confidence_head:
self.model.confidence_head = None
skip_substrs.append("confidence_head")
loader = AutoWeightsLoader(self, skip_substrs=skip_substrs)
loader.load_weights(model_weights.items())
self.model._build_fused_kv_buffers()