[Spec Decode] DSpark speculators checkpoint support (#47093)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -36,18 +36,26 @@ logger = init_logger(__name__)
|
||||
class DSparkMarkovHead(nn.Module):
|
||||
"""Sequential transition-bias head (low-rank V x r, r x V).
|
||||
|
||||
``markov_w1[token]`` is an r-dim embedding of the previously sampled token;
|
||||
``markov_w2`` projects it back to a vocab-size bias added to the base logits.
|
||||
``markov_w1[token]`` embeds the previously sampled token (target vocab,
|
||||
``vocab_size``); ``markov_w2`` projects it to a draft-vocab bias
|
||||
(``draft_vocab_size``) added to the base draft logits. The two sizes
|
||||
coincide for full-vocab drafts.
|
||||
"""
|
||||
|
||||
def __init__(self, vocab_size: int, markov_rank: int, prefix: str) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int,
|
||||
draft_vocab_size: int,
|
||||
markov_rank: int,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# TODO(ben): profile for which (if any) it makes sense to replicate or TP-shard
|
||||
self.markov_w1 = VocabParallelEmbedding(
|
||||
vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w1")
|
||||
)
|
||||
self.markov_w2 = ParallelLMHead(
|
||||
vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w2")
|
||||
draft_vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w2")
|
||||
)
|
||||
|
||||
def embed(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
@@ -73,8 +81,12 @@ class Qwen3DSparkModel(DFlashQwen3Model):
|
||||
vllm_config=vllm_config, start_layer_id=start_layer_id, prefix=prefix
|
||||
)
|
||||
config = self.config
|
||||
draft_vocab_size = (
|
||||
getattr(config, "draft_vocab_size", None) or config.vocab_size
|
||||
)
|
||||
self.markov_head = DSparkMarkovHead(
|
||||
config.vocab_size,
|
||||
draft_vocab_size,
|
||||
config.markov_rank,
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
)
|
||||
@@ -117,6 +129,17 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
|
||||
def get_draft_kv_cache_layer_names(self) -> list[str]:
|
||||
return [layer.self_attn.attn.layer_name for layer in self.model.layers]
|
||||
|
||||
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
# Draft-vocab logits without the d2t scatter: the speculator adds the
|
||||
# Markov bias in draft space, then remaps via map_draft_to_target.
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor:
|
||||
# Map draft-vocab ids to target ids (identity for full-vocab drafts).
|
||||
if self.draft_id_to_target_id is None:
|
||||
return draft_ids
|
||||
return draft_ids + self.draft_id_to_target_id[draft_ids]
|
||||
|
||||
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.embed(token_ids)
|
||||
|
||||
@@ -127,8 +150,15 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
|
||||
model_weights = {}
|
||||
includes_embed_tokens = False
|
||||
includes_lm_head = False
|
||||
includes_draft_id_mapping = False
|
||||
for name, loaded_weight in weights:
|
||||
if "lm_head" not in name:
|
||||
# t2d is training-only; the draft remaps via d2t at sampling time.
|
||||
if "t2d" in name:
|
||||
continue
|
||||
if "d2t" in name:
|
||||
name = name.replace("d2t", "draft_id_to_target_id")
|
||||
includes_draft_id_mapping = True
|
||||
elif "lm_head" not in name:
|
||||
name = "model." + name
|
||||
if "embed_tokens" in name:
|
||||
includes_embed_tokens = True
|
||||
@@ -148,6 +178,8 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
|
||||
skip_substrs.append("embed_tokens")
|
||||
if not includes_lm_head:
|
||||
skip_substrs.append("lm_head")
|
||||
if not includes_draft_id_mapping:
|
||||
skip_substrs.append("draft_id_to_target_id")
|
||||
loader = AutoWeightsLoader(self, skip_substrs=skip_substrs)
|
||||
loader.load_weights(model_weights.items())
|
||||
self.model._build_fused_kv_buffers()
|
||||
|
||||
@@ -110,8 +110,12 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
self.hc_head_scale = nn.Parameter(
|
||||
torch.empty(1, dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
draft_vocab_size = (
|
||||
getattr(config, "draft_vocab_size", None) or config.vocab_size
|
||||
)
|
||||
self.markov_head = DSparkMarkovHead(
|
||||
config.vocab_size,
|
||||
draft_vocab_size,
|
||||
config.dspark_markov_rank,
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
)
|
||||
@@ -318,6 +322,13 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
"""Base logits U_k = lm_head(norm(head_hidden))."""
|
||||
return self.logits_processor(self.lm_head, self.model.norm(hidden_states))
|
||||
|
||||
def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
# Full-vocab draft: base logits, no d2t scatter.
|
||||
return self.compute_logits(hidden_states)
|
||||
|
||||
def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor:
|
||||
return draft_ids # full-vocab: draft ids are target ids
|
||||
|
||||
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.embed(token_ids)
|
||||
|
||||
|
||||
@@ -120,3 +120,47 @@ def update_dflash(config_dict: dict, pre_trained_config: dict) -> None:
|
||||
"mask_token_id": config_dict["mask_token_id"],
|
||||
"target_layer_ids": [i - 1 for i in aux_layer_ids],
|
||||
}
|
||||
|
||||
|
||||
@register_speculator("dspark")
|
||||
def update_dspark(config_dict: dict, pre_trained_config: dict) -> None:
|
||||
"""
|
||||
Apply DSpark specific configuration transformations to the `dict` used to
|
||||
construct the Transformers PreTrainedConfig.
|
||||
|
||||
DSpark extends DFlash with a Markov logit-bias head, reusing the same
|
||||
Qwen3DSparkModel loader and DSparkSpeculator runtime as the dense DSpark
|
||||
checkpoints (e.g. deepseek-ai/dspark_qwen3_8b_block7).
|
||||
|
||||
DSpark specific fields:
|
||||
- draft_vocab_size: draft vocab size; when smaller than the target vocab the
|
||||
checkpoint also ships d2t/t2d remap tables.
|
||||
- mask_token_id (required): token id for parallel-drafting mask slots.
|
||||
- markov_rank / markov_head_type: low-rank Markov logit-bias head.
|
||||
- block_size: semi-autoregressive draft block size.
|
||||
- enable_confidence_head / confidence_head_with_markov: confidence head.
|
||||
- aux_hidden_state_layer_ids (required): target layer indices feeding the
|
||||
drafter. Mapped to both eagle_aux_hidden_state_layer_ids and
|
||||
target_layer_ids (DSpark's i-1 layer semantics).
|
||||
"""
|
||||
pre_trained_config["architectures"] = ["Qwen3DSparkModel"]
|
||||
# Speculators DSpark uses the 1+N fill-in block (anchor is a bonus token).
|
||||
pre_trained_config["dspark_bonus_anchor"] = True
|
||||
|
||||
aux_layer_ids = config_dict["aux_hidden_state_layer_ids"]
|
||||
pre_trained_config["eagle_aux_hidden_state_layer_ids"] = aux_layer_ids
|
||||
# DSpark indexes target layers as aux_id - 1 (matches the dense configs).
|
||||
pre_trained_config["target_layer_ids"] = [i - 1 for i in aux_layer_ids]
|
||||
|
||||
for key in (
|
||||
"draft_vocab_size",
|
||||
"target_hidden_size",
|
||||
"mask_token_id",
|
||||
"markov_rank",
|
||||
"markov_head_type",
|
||||
"block_size",
|
||||
"enable_confidence_head",
|
||||
"confidence_head_with_markov",
|
||||
):
|
||||
if config_dict.get(key) is not None:
|
||||
pre_trained_config[key] = config_dict[key]
|
||||
|
||||
@@ -13,6 +13,8 @@ Differences from DFlash:
|
||||
token), so we sample at all N positions and ``sample_pos = query_pos + 1``
|
||||
(standard next-token), whereas DFlash's masks sit AT the predicted position.
|
||||
This is the ``sample_from_anchor`` path in the shared prepare-inputs kernel.
|
||||
Speculators-format checkpoints instead use the DFlash ``1 + N`` fill-in
|
||||
layout (anchor is the bonus token).
|
||||
* Sequential Markov sampling: instead of DFlash's single parallel sample, we
|
||||
sample left-to-right, adding a prefix-dependent Markov bias derived from the
|
||||
previously sampled token at each step.
|
||||
@@ -38,8 +40,15 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
super().__init__(vllm_config, device)
|
||||
|
||||
# Anchor-first: N query tokens per request (anchor + N-1 noise), not 1+N.
|
||||
self.num_query_per_req = self.num_speculative_steps
|
||||
# Anchor-as-first (N slots) unless the checkpoint uses the 1+N fill-in
|
||||
# block, where the anchor is a separate bonus token.
|
||||
self.sample_from_anchor = not getattr(
|
||||
self.draft_model_config.hf_config, "dspark_bonus_anchor", False
|
||||
)
|
||||
if self.sample_from_anchor:
|
||||
self.num_query_per_req = self.num_speculative_steps
|
||||
else:
|
||||
self.num_query_per_req = 1 + self.num_speculative_steps
|
||||
|
||||
# DSpark consumes mean-pooled target aux hidden states at the target
|
||||
# layers, combined to hidden_size via main_proj. Store that combined
|
||||
@@ -52,9 +61,6 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
|
||||
self.dflash_causal = False
|
||||
|
||||
# The anchor query position is itself a prediction (see module docstring).
|
||||
self.sample_from_anchor = True
|
||||
|
||||
self._step_cols = torch.arange(
|
||||
self.num_speculative_steps, dtype=torch.int32, device=device
|
||||
)
|
||||
@@ -64,12 +70,33 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
* self.num_query_per_req
|
||||
)
|
||||
|
||||
# Reduced-vocab probabilistic drafting only; set in load_draft_model.
|
||||
self._d2t_scatter_index: torch.Tensor | None = None
|
||||
self._draft_scatter_buf: torch.Tensor | None = None
|
||||
|
||||
def load_draft_model(
|
||||
self,
|
||||
target_model: torch.nn.Module,
|
||||
target_attn_layer_names: set[str],
|
||||
) -> torch.nn.Module:
|
||||
return load_dspark_model(target_model, self.vllm_config)
|
||||
model = load_dspark_model(target_model, self.vllm_config)
|
||||
# Reduced draft vocab: probabilistic rejection sampling indexes draft
|
||||
# logits by target id, so precompute the draft->target column map and a
|
||||
# scratch buffer to scatter logits into target vocab before sampling.
|
||||
if self.draft_logits is not None and model.draft_id_to_target_id is not None:
|
||||
d2t = model.draft_id_to_target_id
|
||||
self._d2t_scatter_index = (
|
||||
torch.arange(d2t.shape[0], device=d2t.device) + d2t
|
||||
)
|
||||
# -inf once; the per-step scatter overwrites the draft->target
|
||||
# columns. Kept separate from draft_logits to avoid aliasing.
|
||||
self._draft_scatter_buf = torch.full(
|
||||
(self.max_num_reqs, self.vocab_size),
|
||||
float("-inf"),
|
||||
dtype=self.draft_logits.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
return model
|
||||
|
||||
def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None:
|
||||
# Sequential Markov sampling over the backbone's output hidden states.
|
||||
@@ -77,7 +104,8 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
num_sample = num_reqs * n_spec
|
||||
# Per-(req, position) head hidden, ordered (req, step).
|
||||
sample_hidden = head_hidden[self.sample_indices[:num_sample]]
|
||||
base_logits = self.model.compute_logits(sample_hidden)
|
||||
# Draft-vocab logits; sampled ids are remapped to target vocab below.
|
||||
base_logits = self.model.compute_draft_logits(sample_hidden)
|
||||
vocab_size = base_logits.shape[-1]
|
||||
base_logits = base_logits.view(num_reqs, n_spec, vocab_size)
|
||||
|
||||
@@ -94,9 +122,16 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
bias = self.model.markov_bias(markov_embed)
|
||||
logits_i = base_logits[:, i] + bias
|
||||
if self.draft_logits is not None:
|
||||
# Probabilistic: sample in target vocab (a reduced draft vocab is
|
||||
# scattered into its target columns; full vocab is already there).
|
||||
if self._d2t_scatter_index is not None:
|
||||
assert self._draft_scatter_buf is not None
|
||||
buf = self._draft_scatter_buf[:num_reqs]
|
||||
buf.index_copy_(1, self._d2t_scatter_index, logits_i.to(buf.dtype))
|
||||
logits_i = buf
|
||||
# sample_pos is the predicted token's position Q; the target
|
||||
# verifies it with the predecessor's Gumbel key (Q-1). Pass Q-1.
|
||||
draft_i = gumbel_sample(
|
||||
draft_sampled_i = gumbel_sample(
|
||||
logits_i,
|
||||
idx_map[:, i],
|
||||
self.temperature,
|
||||
@@ -108,9 +143,11 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
else:
|
||||
draft_i = logits_i.argmax(dim=-1)
|
||||
self.draft_tokens[:num_reqs, i] = draft_i
|
||||
prev = draft_i
|
||||
draft_sampled_i = self.model.map_draft_to_target(
|
||||
logits_i.argmax(dim=-1)
|
||||
)
|
||||
self.draft_tokens[:num_reqs, i] = draft_sampled_i
|
||||
prev = draft_sampled_i
|
||||
|
||||
def _generate_draft(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user