[Spec Decode] DSpark (#46995)

Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Benjamin Chislett
2026-07-01 08:30:24 -07:00
committed by GitHub
co-authored by Giancarlo Delfin mgoin
parent 63fcce4de1
commit f5a8d73377
24 changed files with 1821 additions and 94 deletions
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,133 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DSpark speculator: semi-autoregressive parallel drafting.
DSpark drafts a block of ``num_speculative_tokens`` tokens in one parallel pass
(reusing the DFlash machinery: context-KV precompute + a query-block forward),
then injects intra-block dependency with a lightweight sequential Markov head.
Differences from DFlash:
* Anchor-as-first-prediction: each request emits exactly ``N =
num_speculative_tokens`` query tokens (anchor + N-1 noise), NOT ``1 + N``.
Every query position is a prediction (the anchor predicts the first draft
token), so we sample at all N positions and ``sample_pos = query_pos + 1``
(standard next-token), whereas DFlash's masks sit AT the predicted position.
This is the ``sample_from_anchor`` path in the shared prepare-inputs kernel.
* Sequential Markov sampling: instead of DFlash's single parallel sample, we
sample left-to-right, adding a prefix-dependent Markov bias derived from the
previously sampled token at each step.
CUDA graphs (FULL, mirroring DFlash) cover the whole draft step: the parallel
backbone forward AND the sequential Markov sampling.
"""
from typing import Any
import torch
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
from vllm.v1.worker.gpu.spec_decode.dspark.utils import load_dspark_model
class DSparkSpeculator(DFlashSpeculator):
_speculator_name = "DSpark"
def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)
# Anchor-first: N query tokens per request (anchor + N-1 noise), not 1+N.
self.num_query_per_req = self.num_speculative_steps
# DSpark consumes mean-pooled target aux hidden states at the target
# layers, combined to hidden_size via main_proj. Store that combined
# main_x (hidden_size wide). DSpark does not use the same pre-allocated buffer
# that DeepSeek-V4's MTP uses.
draft_hidden = self.draft_model_config.get_hidden_size()
self.hidden_states = torch.zeros(
self.max_num_tokens, draft_hidden, dtype=self.dtype, device=device
)
self.dflash_causal = False
# The anchor query position is itself a prediction (see module docstring).
self.sample_from_anchor = True
self._step_cols = torch.arange(
self.num_speculative_steps, dtype=torch.int32, device=device
)
self._anchor_idx = (
torch.arange(self.max_num_reqs, dtype=torch.int64, device=device)
* self.num_query_per_req
)
def load_draft_model(
self,
target_model: torch.nn.Module,
target_attn_layer_names: set[str],
) -> torch.nn.Module:
return load_dspark_model(target_model, self.vllm_config)
def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None:
# Sequential Markov sampling over the backbone's output hidden states.
n_spec = self.num_speculative_steps
num_sample = num_reqs * n_spec
# Per-(req, position) head hidden, ordered (req, step).
sample_hidden = head_hidden[self.sample_indices[:num_sample]]
base_logits = self.model.compute_logits(sample_hidden)
vocab_size = base_logits.shape[-1]
base_logits = base_logits.view(num_reqs, n_spec, vocab_size)
idx_map = self.sample_idx_mapping[:num_sample].view(num_reqs, n_spec)
sample_pos = self.sample_pos[:num_sample].view(num_reqs, n_spec)
# Anchor (bonus) token per request = the input id at query offset 0,
# read via the precomputed persistent index (fixed buffer for capture).
prev = self.input_buffers.input_ids[self._anchor_idx[:num_reqs]]
for i in range(n_spec):
# Sequential stage: Markov bias from the previously sampled token.
markov_embed = self.model.markov_embed(prev)
bias = self.model.markov_bias(markov_embed)
logits_i = base_logits[:, i] + bias
if self.draft_logits is not None:
# sample_pos is the predicted token's position Q; the target
# verifies it with the predecessor's Gumbel key (Q-1). Pass Q-1.
draft_i = gumbel_sample(
logits_i,
idx_map[:, i],
self.temperature,
self.seeds,
sample_pos[:, i] - 1,
apply_temperature=True,
output_processed_logits=self.draft_logits,
output_processed_logits_col=self._step_cols[i],
use_fp64=self.use_fp64_gumbel,
)
else:
draft_i = logits_i.argmax(dim=-1)
self.draft_tokens[:num_reqs, i] = draft_i
prev = draft_i
def _generate_draft(
self,
num_reqs: int,
num_tokens_padded: int,
attn_metadata: dict[str, Any] | None,
slot_mappings: dict[str, torch.Tensor] | None,
num_tokens_across_dp: torch.Tensor | None,
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
) -> None:
# Full draft step (captured under CUDA graph): parallel backbone forward
# then sequential Markov sampling over its hidden state outputs.
head_hidden = self._run_model(
num_tokens_padded,
attn_metadata,
slot_mappings,
num_tokens_across_dp,
cudagraph_runtime_mode,
)
self._sample_sequential(num_reqs, head_hidden)
@@ -0,0 +1,64 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch.nn as nn
from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.model_loader import get_model
from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share
def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
draft_model_config = speculative_config.draft_model_config
from vllm.compilation.backends import set_model_tag
# DSpark uses non-causal attention.
causal = False
draft_vllm_config = replace(
vllm_config,
attention_config=replace(
vllm_config.attention_config,
use_non_causal=not causal,
backend=speculative_config.attention_backend,
),
)
with set_model_tag("dspark_head"):
draft_model = get_model(
vllm_config=draft_vllm_config, model_config=draft_model_config
)
if get_pp_group().world_size != 1:
raise NotImplementedError("DSpark does not support pipeline parallelism.")
target_language_model = (
target_model.get_language_model()
if hasattr(target_model, "get_language_model")
else target_model
)
target_inner = target_language_model.model
draft_inner = draft_model.model
target_embed = getattr(target_inner, "embed_tokens", None)
draft_embed = getattr(draft_inner, "embed_tokens", None)
if target_embed is not None and _should_share(
draft_model, "has_own_embed_tokens", draft_embed, target_embed
):
if draft_embed is not None:
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
target_lm_head = getattr(target_model, "lm_head", None)
draft_lm_head = getattr(draft_model, "lm_head", None)
if target_lm_head is not None and _should_share(
draft_model, "has_own_lm_head", draft_lm_head, target_lm_head
):
if draft_lm_head is not None:
del draft_model.lm_head
draft_model.lm_head = target_lm_head
return draft_model