[Spec Decode] DSpark speculators checkpoint support (#47093)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-07-01 17:32:27 -07:00
committed by GitHub
parent e196268bad
commit 2b753ad200
4 changed files with 140 additions and 16 deletions
+37 -5
View File
@@ -36,18 +36,26 @@ logger = init_logger(__name__)
class DSparkMarkovHead(nn.Module):
"""Sequential transition-bias head (low-rank V x r, r x V).
``markov_w1[token]`` is an r-dim embedding of the previously sampled token;
``markov_w2`` projects it back to a vocab-size bias added to the base logits.
``markov_w1[token]`` embeds the previously sampled token (target vocab,
``vocab_size``); ``markov_w2`` projects it to a draft-vocab bias
(``draft_vocab_size``) added to the base draft logits. The two sizes
coincide for full-vocab drafts.
"""
def __init__(self, vocab_size: int, markov_rank: int, prefix: str) -> None:
def __init__(
self,
vocab_size: int,
draft_vocab_size: int,
markov_rank: int,
prefix: str,
) -> None:
super().__init__()
# TODO(ben): profile for which (if any) it makes sense to replicate or TP-shard
self.markov_w1 = VocabParallelEmbedding(
vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w1")
)
self.markov_w2 = ParallelLMHead(
vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w2")
draft_vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w2")
)
def embed(self, token_ids: torch.Tensor) -> torch.Tensor:
@@ -73,8 +81,12 @@ class Qwen3DSparkModel(DFlashQwen3Model):
vllm_config=vllm_config, start_layer_id=start_layer_id, prefix=prefix
)
config = self.config
draft_vocab_size = (
getattr(config, "draft_vocab_size", None) or config.vocab_size
)
self.markov_head = DSparkMarkovHead(
config.vocab_size,
draft_vocab_size,
config.markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
)
@@ -117,6 +129,17 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
def get_draft_kv_cache_layer_names(self) -> list[str]:
return [layer.self_attn.attn.layer_name for layer in self.model.layers]
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
# Draft-vocab logits without the d2t scatter: the speculator adds the
# Markov bias in draft space, then remaps via map_draft_to_target.
return self.logits_processor(self.lm_head, hidden_states)
def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor:
# Map draft-vocab ids to target ids (identity for full-vocab drafts).
if self.draft_id_to_target_id is None:
return draft_ids
return draft_ids + self.draft_id_to_target_id[draft_ids]
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.embed(token_ids)
@@ -127,8 +150,15 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
model_weights = {}
includes_embed_tokens = False
includes_lm_head = False
includes_draft_id_mapping = False
for name, loaded_weight in weights:
if "lm_head" not in name:
# t2d is training-only; the draft remaps via d2t at sampling time.
if "t2d" in name:
continue
if "d2t" in name:
name = name.replace("d2t", "draft_id_to_target_id")
includes_draft_id_mapping = True
elif "lm_head" not in name:
name = "model." + name
if "embed_tokens" in name:
includes_embed_tokens = True
@@ -148,6 +178,8 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
skip_substrs.append("embed_tokens")
if not includes_lm_head:
skip_substrs.append("lm_head")
if not includes_draft_id_mapping:
skip_substrs.append("draft_id_to_target_id")
loader = AutoWeightsLoader(self, skip_substrs=skip_substrs)
loader.load_weights(model_weights.items())
self.model._build_fused_kv_buffers()