[Spec Decode] DSpark speculators checkpoint support (#47093)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-07-01 17:32:27 -07:00
committed by GitHub
parent e196268bad
commit 2b753ad200
4 changed files with 140 additions and 16 deletions
+37 -5
View File
@@ -36,18 +36,26 @@ logger = init_logger(__name__)
class DSparkMarkovHead(nn.Module):
"""Sequential transition-bias head (low-rank V x r, r x V).
``markov_w1[token]`` is an r-dim embedding of the previously sampled token;
``markov_w2`` projects it back to a vocab-size bias added to the base logits.
``markov_w1[token]`` embeds the previously sampled token (target vocab,
``vocab_size``); ``markov_w2`` projects it to a draft-vocab bias
(``draft_vocab_size``) added to the base draft logits. The two sizes
coincide for full-vocab drafts.
"""
def __init__(self, vocab_size: int, markov_rank: int, prefix: str) -> None:
def __init__(
self,
vocab_size: int,
draft_vocab_size: int,
markov_rank: int,
prefix: str,
) -> None:
super().__init__()
# TODO(ben): profile for which (if any) it makes sense to replicate or TP-shard
self.markov_w1 = VocabParallelEmbedding(
vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w1")
)
self.markov_w2 = ParallelLMHead(
vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w2")
draft_vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w2")
)
def embed(self, token_ids: torch.Tensor) -> torch.Tensor:
@@ -73,8 +81,12 @@ class Qwen3DSparkModel(DFlashQwen3Model):
vllm_config=vllm_config, start_layer_id=start_layer_id, prefix=prefix
)
config = self.config
draft_vocab_size = (
getattr(config, "draft_vocab_size", None) or config.vocab_size
)
self.markov_head = DSparkMarkovHead(
config.vocab_size,
draft_vocab_size,
config.markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
)
@@ -117,6 +129,17 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
def get_draft_kv_cache_layer_names(self) -> list[str]:
return [layer.self_attn.attn.layer_name for layer in self.model.layers]
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
# Draft-vocab logits without the d2t scatter: the speculator adds the
# Markov bias in draft space, then remaps via map_draft_to_target.
return self.logits_processor(self.lm_head, hidden_states)
def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor:
# Map draft-vocab ids to target ids (identity for full-vocab drafts).
if self.draft_id_to_target_id is None:
return draft_ids
return draft_ids + self.draft_id_to_target_id[draft_ids]
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.embed(token_ids)
@@ -127,8 +150,15 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
model_weights = {}
includes_embed_tokens = False
includes_lm_head = False
includes_draft_id_mapping = False
for name, loaded_weight in weights:
if "lm_head" not in name:
# t2d is training-only; the draft remaps via d2t at sampling time.
if "t2d" in name:
continue
if "d2t" in name:
name = name.replace("d2t", "draft_id_to_target_id")
includes_draft_id_mapping = True
elif "lm_head" not in name:
name = "model." + name
if "embed_tokens" in name:
includes_embed_tokens = True
@@ -148,6 +178,8 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
skip_substrs.append("embed_tokens")
if not includes_lm_head:
skip_substrs.append("lm_head")
if not includes_draft_id_mapping:
skip_substrs.append("draft_id_to_target_id")
loader = AutoWeightsLoader(self, skip_substrs=skip_substrs)
loader.load_weights(model_weights.items())
self.model._build_fused_kv_buffers()
+11
View File
@@ -110,8 +110,12 @@ class DSparkDeepseekV4Model(nn.Module):
self.hc_head_scale = nn.Parameter(
torch.empty(1, dtype=torch.float32), requires_grad=False
)
draft_vocab_size = (
getattr(config, "draft_vocab_size", None) or config.vocab_size
)
self.markov_head = DSparkMarkovHead(
config.vocab_size,
draft_vocab_size,
config.dspark_markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
)
@@ -318,6 +322,13 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
"""Base logits U_k = lm_head(norm(head_hidden))."""
return self.logits_processor(self.lm_head, self.model.norm(hidden_states))
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
# Full-vocab draft: base logits, no d2t scatter.
return self.compute_logits(hidden_states)
def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor:
return draft_ids # full-vocab: draft ids are target ids
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.embed(token_ids)
@@ -120,3 +120,47 @@ def update_dflash(config_dict: dict, pre_trained_config: dict) -> None:
"mask_token_id": config_dict["mask_token_id"],
"target_layer_ids": [i - 1 for i in aux_layer_ids],
}
@register_speculator("dspark")
def update_dspark(config_dict: dict, pre_trained_config: dict) -> None:
"""
Apply DSpark specific configuration transformations to the `dict` used to
construct the Transformers PreTrainedConfig.
DSpark extends DFlash with a Markov logit-bias head, reusing the same
Qwen3DSparkModel loader and DSparkSpeculator runtime as the dense DSpark
checkpoints (e.g. deepseek-ai/dspark_qwen3_8b_block7).
DSpark specific fields:
- draft_vocab_size: draft vocab size; when smaller than the target vocab the
checkpoint also ships d2t/t2d remap tables.
- mask_token_id (required): token id for parallel-drafting mask slots.
- markov_rank / markov_head_type: low-rank Markov logit-bias head.
- block_size: semi-autoregressive draft block size.
- enable_confidence_head / confidence_head_with_markov: confidence head.
- aux_hidden_state_layer_ids (required): target layer indices feeding the
drafter. Mapped to both eagle_aux_hidden_state_layer_ids and
target_layer_ids (DSpark's i-1 layer semantics).
"""
pre_trained_config["architectures"] = ["Qwen3DSparkModel"]
# Speculators DSpark uses the 1+N fill-in block (anchor is a bonus token).
pre_trained_config["dspark_bonus_anchor"] = True
aux_layer_ids = config_dict["aux_hidden_state_layer_ids"]
pre_trained_config["eagle_aux_hidden_state_layer_ids"] = aux_layer_ids
# DSpark indexes target layers as aux_id - 1 (matches the dense configs).
pre_trained_config["target_layer_ids"] = [i - 1 for i in aux_layer_ids]
for key in (
"draft_vocab_size",
"target_hidden_size",
"mask_token_id",
"markov_rank",
"markov_head_type",
"block_size",
"enable_confidence_head",
"confidence_head_with_markov",
):
if config_dict.get(key) is not None:
pre_trained_config[key] = config_dict[key]
@@ -13,6 +13,8 @@ Differences from DFlash:
token), so we sample at all N positions and ``sample_pos = query_pos + 1``
(standard next-token), whereas DFlash's masks sit AT the predicted position.
This is the ``sample_from_anchor`` path in the shared prepare-inputs kernel.
Speculators-format checkpoints instead use the DFlash ``1 + N`` fill-in
layout (anchor is the bonus token).
* Sequential Markov sampling: instead of DFlash's single parallel sample, we
sample left-to-right, adding a prefix-dependent Markov bias derived from the
previously sampled token at each step.
@@ -38,8 +40,15 @@ class DSparkSpeculator(DFlashSpeculator):
def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)
# Anchor-first: N query tokens per request (anchor + N-1 noise), not 1+N.
self.num_query_per_req = self.num_speculative_steps
# Anchor-as-first (N slots) unless the checkpoint uses the 1+N fill-in
# block, where the anchor is a separate bonus token.
self.sample_from_anchor = not getattr(
self.draft_model_config.hf_config, "dspark_bonus_anchor", False
)
if self.sample_from_anchor:
self.num_query_per_req = self.num_speculative_steps
else:
self.num_query_per_req = 1 + self.num_speculative_steps
# DSpark consumes mean-pooled target aux hidden states at the target
# layers, combined to hidden_size via main_proj. Store that combined
@@ -52,9 +61,6 @@ class DSparkSpeculator(DFlashSpeculator):
self.dflash_causal = False
# The anchor query position is itself a prediction (see module docstring).
self.sample_from_anchor = True
self._step_cols = torch.arange(
self.num_speculative_steps, dtype=torch.int32, device=device
)
@@ -64,12 +70,33 @@ class DSparkSpeculator(DFlashSpeculator):
* self.num_query_per_req
)
# Reduced-vocab probabilistic drafting only; set in load_draft_model.
self._d2t_scatter_index: torch.Tensor | None = None
self._draft_scatter_buf: torch.Tensor | None = None
def load_draft_model(
self,
target_model: torch.nn.Module,
target_attn_layer_names: set[str],
) -> torch.nn.Module:
return load_dspark_model(target_model, self.vllm_config)
model = load_dspark_model(target_model, self.vllm_config)
# Reduced draft vocab: probabilistic rejection sampling indexes draft
# logits by target id, so precompute the draft->target column map and a
# scratch buffer to scatter logits into target vocab before sampling.
if self.draft_logits is not None and model.draft_id_to_target_id is not None:
d2t = model.draft_id_to_target_id
self._d2t_scatter_index = (
torch.arange(d2t.shape[0], device=d2t.device) + d2t
)
# -inf once; the per-step scatter overwrites the draft->target
# columns. Kept separate from draft_logits to avoid aliasing.
self._draft_scatter_buf = torch.full(
(self.max_num_reqs, self.vocab_size),
float("-inf"),
dtype=self.draft_logits.dtype,
device=self.device,
)
return model
def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None:
# Sequential Markov sampling over the backbone's output hidden states.
@@ -77,7 +104,8 @@ class DSparkSpeculator(DFlashSpeculator):
num_sample = num_reqs * n_spec
# Per-(req, position) head hidden, ordered (req, step).
sample_hidden = head_hidden[self.sample_indices[:num_sample]]
base_logits = self.model.compute_logits(sample_hidden)
# Draft-vocab logits; sampled ids are remapped to target vocab below.
base_logits = self.model.compute_draft_logits(sample_hidden)
vocab_size = base_logits.shape[-1]
base_logits = base_logits.view(num_reqs, n_spec, vocab_size)
@@ -94,9 +122,16 @@ class DSparkSpeculator(DFlashSpeculator):
bias = self.model.markov_bias(markov_embed)
logits_i = base_logits[:, i] + bias
if self.draft_logits is not None:
# Probabilistic: sample in target vocab (a reduced draft vocab is
# scattered into its target columns; full vocab is already there).
if self._d2t_scatter_index is not None:
assert self._draft_scatter_buf is not None
buf = self._draft_scatter_buf[:num_reqs]
buf.index_copy_(1, self._d2t_scatter_index, logits_i.to(buf.dtype))
logits_i = buf
# sample_pos is the predicted token's position Q; the target
# verifies it with the predecessor's Gumbel key (Q-1). Pass Q-1.
draft_i = gumbel_sample(
draft_sampled_i = gumbel_sample(
logits_i,
idx_map[:, i],
self.temperature,
@@ -108,9 +143,11 @@ class DSparkSpeculator(DFlashSpeculator):
use_fp64=self.use_fp64_gumbel,
)
else:
draft_i = logits_i.argmax(dim=-1)
self.draft_tokens[:num_reqs, i] = draft_i
prev = draft_i
draft_sampled_i = self.model.map_draft_to_target(
logits_i.argmax(dim=-1)
)
self.draft_tokens[:num_reqs, i] = draft_sampled_i
prev = draft_sampled_i
def _generate_draft(
self,