From 2b753ad200d52a2dc16e61ff3c92a45711e2750c Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Wed, 1 Jul 2026 20:32:27 -0400 Subject: [PATCH] [Spec Decode] DSpark speculators checkpoint support (#47093) Signed-off-by: mgoin --- vllm/model_executor/models/qwen3_dspark.py | 42 +++++++++++-- vllm/models/deepseek_v4/nvidia/dspark.py | 11 ++++ .../configs/speculators/algos.py | 44 ++++++++++++++ .../gpu/spec_decode/dspark/speculator.py | 59 +++++++++++++++---- 4 files changed, 140 insertions(+), 16 deletions(-) diff --git a/vllm/model_executor/models/qwen3_dspark.py b/vllm/model_executor/models/qwen3_dspark.py index 276b90358b..219819759a 100644 --- a/vllm/model_executor/models/qwen3_dspark.py +++ b/vllm/model_executor/models/qwen3_dspark.py @@ -36,18 +36,26 @@ logger = init_logger(__name__) class DSparkMarkovHead(nn.Module): """Sequential transition-bias head (low-rank V x r, r x V). - ``markov_w1[token]`` is an r-dim embedding of the previously sampled token; - ``markov_w2`` projects it back to a vocab-size bias added to the base logits. + ``markov_w1[token]`` embeds the previously sampled token (target vocab, + ``vocab_size``); ``markov_w2`` projects it to a draft-vocab bias + (``draft_vocab_size``) added to the base draft logits. The two sizes + coincide for full-vocab drafts. """ - def __init__(self, vocab_size: int, markov_rank: int, prefix: str) -> None: + def __init__( + self, + vocab_size: int, + draft_vocab_size: int, + markov_rank: int, + prefix: str, + ) -> None: super().__init__() # TODO(ben): profile for which (if any) it makes sense to replicate or TP-shard self.markov_w1 = VocabParallelEmbedding( vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w1") ) self.markov_w2 = ParallelLMHead( - vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w2") + draft_vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w2") ) def embed(self, token_ids: torch.Tensor) -> torch.Tensor: @@ -73,8 +81,12 @@ class Qwen3DSparkModel(DFlashQwen3Model): vllm_config=vllm_config, start_layer_id=start_layer_id, prefix=prefix ) config = self.config + draft_vocab_size = ( + getattr(config, "draft_vocab_size", None) or config.vocab_size + ) self.markov_head = DSparkMarkovHead( config.vocab_size, + draft_vocab_size, config.markov_rank, prefix=maybe_prefix(prefix, "markov_head"), ) @@ -117,6 +129,17 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM): def get_draft_kv_cache_layer_names(self) -> list[str]: return [layer.self_attn.attn.layer_name for layer in self.model.layers] + def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor: + # Draft-vocab logits without the d2t scatter: the speculator adds the + # Markov bias in draft space, then remaps via map_draft_to_target. + return self.logits_processor(self.lm_head, hidden_states) + + def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor: + # Map draft-vocab ids to target ids (identity for full-vocab drafts). + if self.draft_id_to_target_id is None: + return draft_ids + return draft_ids + self.draft_id_to_target_id[draft_ids] + def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor: return self.model.markov_head.embed(token_ids) @@ -127,8 +150,15 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM): model_weights = {} includes_embed_tokens = False includes_lm_head = False + includes_draft_id_mapping = False for name, loaded_weight in weights: - if "lm_head" not in name: + # t2d is training-only; the draft remaps via d2t at sampling time. + if "t2d" in name: + continue + if "d2t" in name: + name = name.replace("d2t", "draft_id_to_target_id") + includes_draft_id_mapping = True + elif "lm_head" not in name: name = "model." + name if "embed_tokens" in name: includes_embed_tokens = True @@ -148,6 +178,8 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM): skip_substrs.append("embed_tokens") if not includes_lm_head: skip_substrs.append("lm_head") + if not includes_draft_id_mapping: + skip_substrs.append("draft_id_to_target_id") loader = AutoWeightsLoader(self, skip_substrs=skip_substrs) loader.load_weights(model_weights.items()) self.model._build_fused_kv_buffers() diff --git a/vllm/models/deepseek_v4/nvidia/dspark.py b/vllm/models/deepseek_v4/nvidia/dspark.py index ab5d1a4b47..be4a87b323 100644 --- a/vllm/models/deepseek_v4/nvidia/dspark.py +++ b/vllm/models/deepseek_v4/nvidia/dspark.py @@ -110,8 +110,12 @@ class DSparkDeepseekV4Model(nn.Module): self.hc_head_scale = nn.Parameter( torch.empty(1, dtype=torch.float32), requires_grad=False ) + draft_vocab_size = ( + getattr(config, "draft_vocab_size", None) or config.vocab_size + ) self.markov_head = DSparkMarkovHead( config.vocab_size, + draft_vocab_size, config.dspark_markov_rank, prefix=maybe_prefix(prefix, "markov_head"), ) @@ -318,6 +322,13 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): """Base logits U_k = lm_head(norm(head_hidden)).""" return self.logits_processor(self.lm_head, self.model.norm(hidden_states)) + def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor: + # Full-vocab draft: base logits, no d2t scatter. + return self.compute_logits(hidden_states) + + def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor: + return draft_ids # full-vocab: draft ids are target ids + def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor: return self.model.markov_head.embed(token_ids) diff --git a/vllm/transformers_utils/configs/speculators/algos.py b/vllm/transformers_utils/configs/speculators/algos.py index f1dfc8878f..f4b0f73f62 100644 --- a/vllm/transformers_utils/configs/speculators/algos.py +++ b/vllm/transformers_utils/configs/speculators/algos.py @@ -120,3 +120,47 @@ def update_dflash(config_dict: dict, pre_trained_config: dict) -> None: "mask_token_id": config_dict["mask_token_id"], "target_layer_ids": [i - 1 for i in aux_layer_ids], } + + +@register_speculator("dspark") +def update_dspark(config_dict: dict, pre_trained_config: dict) -> None: + """ + Apply DSpark specific configuration transformations to the `dict` used to + construct the Transformers PreTrainedConfig. + + DSpark extends DFlash with a Markov logit-bias head, reusing the same + Qwen3DSparkModel loader and DSparkSpeculator runtime as the dense DSpark + checkpoints (e.g. deepseek-ai/dspark_qwen3_8b_block7). + + DSpark specific fields: + - draft_vocab_size: draft vocab size; when smaller than the target vocab the + checkpoint also ships d2t/t2d remap tables. + - mask_token_id (required): token id for parallel-drafting mask slots. + - markov_rank / markov_head_type: low-rank Markov logit-bias head. + - block_size: semi-autoregressive draft block size. + - enable_confidence_head / confidence_head_with_markov: confidence head. + - aux_hidden_state_layer_ids (required): target layer indices feeding the + drafter. Mapped to both eagle_aux_hidden_state_layer_ids and + target_layer_ids (DSpark's i-1 layer semantics). + """ + pre_trained_config["architectures"] = ["Qwen3DSparkModel"] + # Speculators DSpark uses the 1+N fill-in block (anchor is a bonus token). + pre_trained_config["dspark_bonus_anchor"] = True + + aux_layer_ids = config_dict["aux_hidden_state_layer_ids"] + pre_trained_config["eagle_aux_hidden_state_layer_ids"] = aux_layer_ids + # DSpark indexes target layers as aux_id - 1 (matches the dense configs). + pre_trained_config["target_layer_ids"] = [i - 1 for i in aux_layer_ids] + + for key in ( + "draft_vocab_size", + "target_hidden_size", + "mask_token_id", + "markov_rank", + "markov_head_type", + "block_size", + "enable_confidence_head", + "confidence_head_with_markov", + ): + if config_dict.get(key) is not None: + pre_trained_config[key] = config_dict[key] diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py index 90012239bc..0236017cf2 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py @@ -13,6 +13,8 @@ Differences from DFlash: token), so we sample at all N positions and ``sample_pos = query_pos + 1`` (standard next-token), whereas DFlash's masks sit AT the predicted position. This is the ``sample_from_anchor`` path in the shared prepare-inputs kernel. + Speculators-format checkpoints instead use the DFlash ``1 + N`` fill-in + layout (anchor is the bonus token). * Sequential Markov sampling: instead of DFlash's single parallel sample, we sample left-to-right, adding a prefix-dependent Markov bias derived from the previously sampled token at each step. @@ -38,8 +40,15 @@ class DSparkSpeculator(DFlashSpeculator): def __init__(self, vllm_config: VllmConfig, device: torch.device): super().__init__(vllm_config, device) - # Anchor-first: N query tokens per request (anchor + N-1 noise), not 1+N. - self.num_query_per_req = self.num_speculative_steps + # Anchor-as-first (N slots) unless the checkpoint uses the 1+N fill-in + # block, where the anchor is a separate bonus token. + self.sample_from_anchor = not getattr( + self.draft_model_config.hf_config, "dspark_bonus_anchor", False + ) + if self.sample_from_anchor: + self.num_query_per_req = self.num_speculative_steps + else: + self.num_query_per_req = 1 + self.num_speculative_steps # DSpark consumes mean-pooled target aux hidden states at the target # layers, combined to hidden_size via main_proj. Store that combined @@ -52,9 +61,6 @@ class DSparkSpeculator(DFlashSpeculator): self.dflash_causal = False - # The anchor query position is itself a prediction (see module docstring). - self.sample_from_anchor = True - self._step_cols = torch.arange( self.num_speculative_steps, dtype=torch.int32, device=device ) @@ -64,12 +70,33 @@ class DSparkSpeculator(DFlashSpeculator): * self.num_query_per_req ) + # Reduced-vocab probabilistic drafting only; set in load_draft_model. + self._d2t_scatter_index: torch.Tensor | None = None + self._draft_scatter_buf: torch.Tensor | None = None + def load_draft_model( self, target_model: torch.nn.Module, target_attn_layer_names: set[str], ) -> torch.nn.Module: - return load_dspark_model(target_model, self.vllm_config) + model = load_dspark_model(target_model, self.vllm_config) + # Reduced draft vocab: probabilistic rejection sampling indexes draft + # logits by target id, so precompute the draft->target column map and a + # scratch buffer to scatter logits into target vocab before sampling. + if self.draft_logits is not None and model.draft_id_to_target_id is not None: + d2t = model.draft_id_to_target_id + self._d2t_scatter_index = ( + torch.arange(d2t.shape[0], device=d2t.device) + d2t + ) + # -inf once; the per-step scatter overwrites the draft->target + # columns. Kept separate from draft_logits to avoid aliasing. + self._draft_scatter_buf = torch.full( + (self.max_num_reqs, self.vocab_size), + float("-inf"), + dtype=self.draft_logits.dtype, + device=self.device, + ) + return model def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None: # Sequential Markov sampling over the backbone's output hidden states. @@ -77,7 +104,8 @@ class DSparkSpeculator(DFlashSpeculator): num_sample = num_reqs * n_spec # Per-(req, position) head hidden, ordered (req, step). sample_hidden = head_hidden[self.sample_indices[:num_sample]] - base_logits = self.model.compute_logits(sample_hidden) + # Draft-vocab logits; sampled ids are remapped to target vocab below. + base_logits = self.model.compute_draft_logits(sample_hidden) vocab_size = base_logits.shape[-1] base_logits = base_logits.view(num_reqs, n_spec, vocab_size) @@ -94,9 +122,16 @@ class DSparkSpeculator(DFlashSpeculator): bias = self.model.markov_bias(markov_embed) logits_i = base_logits[:, i] + bias if self.draft_logits is not None: + # Probabilistic: sample in target vocab (a reduced draft vocab is + # scattered into its target columns; full vocab is already there). + if self._d2t_scatter_index is not None: + assert self._draft_scatter_buf is not None + buf = self._draft_scatter_buf[:num_reqs] + buf.index_copy_(1, self._d2t_scatter_index, logits_i.to(buf.dtype)) + logits_i = buf # sample_pos is the predicted token's position Q; the target # verifies it with the predecessor's Gumbel key (Q-1). Pass Q-1. - draft_i = gumbel_sample( + draft_sampled_i = gumbel_sample( logits_i, idx_map[:, i], self.temperature, @@ -108,9 +143,11 @@ class DSparkSpeculator(DFlashSpeculator): use_fp64=self.use_fp64_gumbel, ) else: - draft_i = logits_i.argmax(dim=-1) - self.draft_tokens[:num_reqs, i] = draft_i - prev = draft_i + draft_sampled_i = self.model.map_draft_to_target( + logits_i.argmax(dim=-1) + ) + self.draft_tokens[:num_reqs, i] = draft_sampled_i + prev = draft_sampled_i def _generate_draft( self,