[Spec Decode] DSpark confidence-scheduled verification (#47808)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Nick Hill
OpenAI Codex
parent
fe889ac925
commit
7f7a32cfec
@@ -10,8 +10,9 @@ The parallel backbone is a standard Qwen3 decoder stack reused from the
|
||||
DFlash Qwen3 draft (see qwen3_dflash.py). DSpark adds:
|
||||
* ``markov_head``: low-rank V x r / r x V transition bias added to the base
|
||||
logits, sampled left-to-right by the speculator (the sequential stage).
|
||||
* ``confidence_head``: per-position acceptance-probability estimate.
|
||||
|
||||
DSparkMarkovHead is shared with the DSV4-style DSpark model.
|
||||
DSparkMarkovHead and DSparkConfidenceHead are shared with the DSV4-style DSpark model.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
@@ -21,6 +22,7 @@ import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
@@ -102,8 +104,36 @@ class DSparkMarkovHead(nn.Module):
|
||||
return logits.scatter_(1, index, corrected.squeeze(-1))
|
||||
|
||||
|
||||
class DSparkConfidenceHead(nn.Module):
|
||||
"""DSpark acceptance-confidence head."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_dim: int,
|
||||
prefix: str,
|
||||
bias: bool = False,
|
||||
with_markov: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.with_markov = with_markov
|
||||
self.proj = ReplicatedLinear(
|
||||
input_dim,
|
||||
1,
|
||||
bias=bias,
|
||||
return_bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=maybe_prefix(prefix, "proj"),
|
||||
)
|
||||
|
||||
def forward(self, hidden: torch.Tensor, markov_embed: torch.Tensor) -> torch.Tensor:
|
||||
x = (
|
||||
torch.cat([hidden, markov_embed], dim=-1) if self.with_markov else hidden
|
||||
).float()
|
||||
return self.proj(x).squeeze(-1)
|
||||
|
||||
|
||||
class Qwen3DSparkModel(DFlashQwen3Model):
|
||||
"""DFlash Qwen3 backbone + DSpark Markov head."""
|
||||
"""DFlash Qwen3 backbone + DSpark Markov / confidence heads."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -126,6 +156,18 @@ class Qwen3DSparkModel(DFlashQwen3Model):
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
quant_config=self.quant_config,
|
||||
)
|
||||
self.confidence_head: DSparkConfidenceHead | None = None
|
||||
if getattr(config, "enable_confidence_head", False):
|
||||
with_markov = getattr(config, "confidence_head_with_markov", False)
|
||||
input_dim = config.hidden_size
|
||||
if with_markov:
|
||||
input_dim += config.markov_rank
|
||||
self.confidence_head = DSparkConfidenceHead(
|
||||
input_dim,
|
||||
prefix=maybe_prefix(prefix, "confidence_head"),
|
||||
bias=True,
|
||||
with_markov=with_markov,
|
||||
)
|
||||
|
||||
|
||||
class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
|
||||
@@ -197,11 +239,19 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
|
||||
self.logits_processor.scale,
|
||||
)
|
||||
|
||||
def compute_confidence(
|
||||
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Per-position acceptance probability for each drafted token."""
|
||||
assert self.model.confidence_head is not None
|
||||
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
model_weights = {}
|
||||
includes_embed_tokens = False
|
||||
includes_lm_head = False
|
||||
includes_draft_id_mapping = False
|
||||
includes_confidence_head = False
|
||||
for name, loaded_weight in weights:
|
||||
# t2d is training-only; the draft remaps via d2t at sampling time.
|
||||
if "t2d" in name:
|
||||
@@ -215,22 +265,26 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
|
||||
includes_embed_tokens = True
|
||||
if "lm_head" in name:
|
||||
includes_lm_head = True
|
||||
if "confidence_head" in name:
|
||||
includes_confidence_head = True
|
||||
model_weights[name] = loaded_weight
|
||||
# Sets has_own_embed_tokens / has_own_lm_head so load_dspark_model
|
||||
# knows whether to keep these or alias the target's.
|
||||
process_eagle_weight(self, name)
|
||||
|
||||
# mask_embedding is an unused placeholder param; DSpark masks via the vocab row.
|
||||
# confidence_head is not wired into inference yet; skip its weights.
|
||||
# embed_tokens / lm_head are optional; when omitted they are shared from
|
||||
# the target by load_dspark_model, so skip the unloaded params here.
|
||||
skip_substrs = ["mask_embedding", "confidence_head"]
|
||||
skip_substrs = ["mask_embedding"]
|
||||
if not includes_embed_tokens:
|
||||
skip_substrs.append("embed_tokens")
|
||||
if not includes_lm_head:
|
||||
skip_substrs.append("lm_head")
|
||||
if not includes_draft_id_mapping:
|
||||
skip_substrs.append("draft_id_to_target_id")
|
||||
if self.model.confidence_head is None or not includes_confidence_head:
|
||||
self.model.confidence_head = None
|
||||
skip_substrs.append("confidence_head")
|
||||
loader = AutoWeightsLoader(self, skip_substrs=skip_substrs)
|
||||
loader.load_weights(model_weights.items())
|
||||
self.model._build_fused_kv_buffers()
|
||||
|
||||
Reference in New Issue
Block a user