[Spec Decode] DSpark (#46995)

Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Benjamin Chislett
2026-07-01 08:30:24 -07:00
committed by GitHub
co-authored by Giancarlo Delfin mgoin
parent 63fcce4de1
commit f5a8d73377
24 changed files with 1821 additions and 94 deletions
@@ -14,6 +14,12 @@ def init_speculator(vllm_config: VllmConfig, device: torch.device):
)
return DFlashSpeculator(vllm_config, device)
elif speculative_config.method == "dspark":
from vllm.v1.worker.gpu.spec_decode.dspark.speculator import (
DSparkSpeculator,
)
return DSparkSpeculator(vllm_config, device)
elif speculative_config.use_gemma4_mtp():
from vllm.v1.worker.gpu.spec_decode.gemma4.speculator import (
Gemma4Speculator,
@@ -29,6 +29,8 @@ logger = init_logger(__name__)
class DFlashSpeculator(DraftModelSpeculator):
_speculator_name = "DFlash" # For logging, so we can share methods with subclasses
def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)
@@ -48,15 +50,17 @@ class DFlashSpeculator(DraftModelSpeculator):
self.dflash_causal = get_dflash_causal(self.draft_model_config)
# Buffers for context K/V precomputation. Populated by prepare_dflash_inputs,
# and processed by the model's precompute_and_store_context_kv method.
# NOT captured by CUDA graphs.
# Whether the anchor query position is itself a prediction. DFlash default uses
# the anchor as the bonus token (only mask tokens predict); DSpark samples from
# the anchor and the N-1 mask token positions. See _prepare_dflash_inputs_kernel
self.sample_from_anchor = False
# Context positions for the K/V precompute. Populated by
# prepare_dflash_inputs, and processed by the model's
# precompute_and_store_context_kv method. NOT captured by CUDA graphs.
self.context_positions = torch.zeros(
self.max_num_tokens, dtype=torch.int64, device=device
)
self.context_slot_mapping = torch.zeros(
self.max_num_tokens, dtype=torch.int64, device=device
)
# Per-mask-token sampling buffers. Flattened from (num_reqs, num_spec_tokens).
max_num_sampled_tokens = self.max_num_reqs * self.num_speculative_steps
@@ -94,7 +98,7 @@ class DFlashSpeculator(DraftModelSpeculator):
)
def capture(self, attn_states: dict | None = None) -> None:
logger.info("Capturing model for DFlash speculator...")
logger.info("Capturing model for %s speculator...", self._speculator_name)
# Reset sampling indices to zero to prevent stale values from prior
# dummy runs from being baked into the captured graph.
self.sample_indices.zero_()
@@ -108,7 +112,7 @@ class DFlashSpeculator(DraftModelSpeculator):
self.attn_groups,
self.kv_cache_config,
self.max_model_len,
progress_bar_desc="Capturing dflash CUDA graphs",
progress_bar_desc=f"Capturing {self._speculator_name.lower()} CUDA graphs",
)
def load_draft_model(
@@ -126,18 +130,39 @@ class DFlashSpeculator(DraftModelSpeculator):
) -> None:
super().set_attn(model_state, kv_cache_config, block_tables)
# DFlash precomputes context K/V with a single block_size; mixing
# kv-cache groups would silently corrupt the cache for the non-matching group.
draft_groups = [gid for gid, g in enumerate(self.attn_groups) if g]
assert len(draft_groups) == 1, (
"DFlash currently requires all draft attention layers to share "
"a single kv-cache group."
)
self.draft_kv_cache_group_id = draft_groups[0]
self.draft_kv_cache_group_ids = [
gid for gid, g in enumerate(self.attn_groups) if g
]
assert self.draft_kv_cache_group_ids, "No draft attention groups found."
self.draft_kv_cache_group_id = self.draft_kv_cache_group_ids[0]
self.draft_block_size = self.block_tables.block_sizes[
self.draft_kv_cache_group_id
]
# Per-group context slot buffers for the precompute (one row per group).
self._context_slot_mappings = torch.zeros(
len(self.draft_kv_cache_group_ids),
self.max_num_tokens,
dtype=torch.int64,
device=self.device,
)
# Map each draft decoder layer to the index (within draft_kv_cache_group_ids)
# of the kv-cache group its cache belongs to. Models that share a single group
# leave this as None and share one context slot mapping.
self._layer_group_idx: list[int] | None = None
if hasattr(self.model, "get_draft_kv_cache_layer_names"):
name_to_gid = {
ln: gid
for gid, group in enumerate(kv_cache_config.kv_cache_groups)
for ln in group.layer_names
}
gid_to_idx = {gid: i for i, gid in enumerate(self.draft_kv_cache_group_ids)}
self._layer_group_idx = [
gid_to_idx[name_to_gid[name]]
for name in self.model.get_draft_kv_cache_layer_names()
]
@torch.inference_mode()
def _run_model(
self,
@@ -183,9 +208,11 @@ class DFlashSpeculator(DraftModelSpeculator):
num_sample = num_reqs * self.num_speculative_steps
sample_hidden_states = last_hidden_states[self.sample_indices[:num_sample]]
# sample_pos is the predicted token's position Q; verification keys
# Gumbel by the predecessor (Q-1). sample_draft adds +1, so pass Q-2.
draft_tokens = self.sample_draft(
sample_hidden_states,
self.sample_pos[:num_sample],
self.sample_pos[:num_sample] - 2,
self.sample_idx_mapping[:num_sample],
self.temperature,
self.seeds,
@@ -294,40 +321,49 @@ class DFlashSpeculator(DraftModelSpeculator):
# The query slot mapping is written into the shared BlockTables slot_mappings.
# That buffer's address is what the captured CUDA graph reads from at replay.
assert self.draft_kv_cache_group_id >= 0
query_slot_mapping = self.block_tables.slot_mappings[
self.draft_kv_cache_group_id
]
prepare_dflash_inputs(
self.input_buffers,
query_slot_mapping,
self.context_positions,
self.context_slot_mapping,
self.sample_indices,
self.sample_pos,
self.sample_idx_mapping,
input_batch,
num_sampled,
num_rejected,
last_sampled,
next_prefill_tokens,
self.block_tables.input_block_tables[self.draft_kv_cache_group_id],
self.draft_block_size,
self.parallel_drafting_token_id,
self.num_query_per_req,
self.num_speculative_steps,
self.max_num_reqs,
self.max_num_tokens,
)
# Support multiple draft KV cache groups by preparing inputs once for each
for i, gid in enumerate(self.draft_kv_cache_group_ids):
prepare_dflash_inputs(
self.input_buffers,
self.block_tables.slot_mappings[gid],
self.context_positions,
self._context_slot_mappings[i],
self.sample_indices,
self.sample_pos,
self.sample_idx_mapping,
input_batch,
num_sampled,
num_rejected,
last_sampled,
next_prefill_tokens,
self.block_tables.input_block_tables[gid],
self.block_tables.block_sizes[gid],
self.parallel_drafting_token_id,
self.num_query_per_req,
self.num_speculative_steps,
self.max_num_reqs,
self.max_num_tokens,
self.max_model_len,
self.sample_from_anchor,
)
# Pre-insert context K/V into the cache. Runs eagerly outside the captured graph
# because the context shape varies per step. During dummy runs the block tables
# are placeholders, so we skip the cache write to avoid clobbering real entries.
# Each layer uses the context slots of its own kv-cache group.
if dummy_run:
context_slots: torch.Tensor | list[torch.Tensor | None] | None = None
elif self._layer_group_idx is not None:
context_slots = [
self._context_slot_mappings[gidx][:num_target_tokens]
for gidx in self._layer_group_idx
]
else:
context_slots = self._context_slot_mappings[0][:num_target_tokens]
self.model.precompute_and_store_context_kv(
self.hidden_states[:num_target_tokens],
self.context_positions[:num_target_tokens],
context_slot_mapping=(
None if dummy_run else self.context_slot_mapping[:num_target_tokens]
),
context_slots,
)
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
@@ -408,6 +444,8 @@ def _prepare_dflash_inputs_kernel(
num_speculative_steps,
max_num_reqs,
max_num_tokens,
max_model_len,
SAMPLE_FROM_ANCHOR: tl.constexpr,
PAD_SLOT_ID: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
@@ -468,14 +506,21 @@ def _prepare_dflash_inputs_kernel(
q_slot = q_block_id * block_size + (query_pos % block_size)
tl.store(out_input_ids_ptr + query_idx, input_id, mask=is_query)
tl.store(out_query_positions_ptr + query_idx, query_pos, mask=is_query)
clamped_query_pos = tl.minimum(query_pos, max_model_len - 1)
tl.store(out_query_positions_ptr + query_idx, clamped_query_pos, mask=is_query)
tl.store(out_query_slot_mapping_ptr + query_idx, q_slot, mask=is_query)
# --- Sample indices / positions / idx_mapping (mask tokens only) ---
is_sample = is_query & (query_off > 0)
sample_idx = req_idx * num_speculative_steps + (query_off - 1)
# --- Sample indices / positions / idx_mapping ---
# When SAMPLE_FROM_ANCHOR (DSpark), so we sample at EVERY query position
# and each position k predicts the NEXT token (sampled position = query_pos + 1).
# Otherwise (DFlash default) the anchor is the bonus token and only the mask tokens
# at offsets > 0 are sampled from, each AT its own position.
sample_off = 0 if SAMPLE_FROM_ANCHOR else 1
is_sample = is_query & (query_off >= sample_off)
sample_idx = req_idx * num_speculative_steps + (query_off - sample_off)
sample_pos = query_pos + 1 if SAMPLE_FROM_ANCHOR else query_pos
tl.store(out_sample_indices_ptr + sample_idx, query_idx, mask=is_sample)
tl.store(out_sample_pos_ptr + sample_idx, query_pos, mask=is_sample)
tl.store(out_sample_pos_ptr + sample_idx, sample_pos, mask=is_sample)
tl.store(out_sample_idx_mapping_ptr + sample_idx, req_state_idx, mask=is_sample)
if block_idx == 0:
@@ -542,6 +587,8 @@ def prepare_dflash_inputs(
num_speculative_steps: int,
max_num_reqs: int,
max_num_tokens: int,
max_model_len: int,
sample_from_anchor: bool = False,
) -> None:
num_reqs = input_batch.num_reqs
assert num_reqs > 0
@@ -577,6 +624,8 @@ def prepare_dflash_inputs(
num_speculative_steps,
max_num_reqs,
max_num_tokens,
max_model_len,
SAMPLE_FROM_ANCHOR=sample_from_anchor,
PAD_SLOT_ID=PAD_SLOT_ID,
BLOCK_SIZE=BLOCK_SIZE,
)
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,133 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DSpark speculator: semi-autoregressive parallel drafting.
DSpark drafts a block of ``num_speculative_tokens`` tokens in one parallel pass
(reusing the DFlash machinery: context-KV precompute + a query-block forward),
then injects intra-block dependency with a lightweight sequential Markov head.
Differences from DFlash:
* Anchor-as-first-prediction: each request emits exactly ``N =
num_speculative_tokens`` query tokens (anchor + N-1 noise), NOT ``1 + N``.
Every query position is a prediction (the anchor predicts the first draft
token), so we sample at all N positions and ``sample_pos = query_pos + 1``
(standard next-token), whereas DFlash's masks sit AT the predicted position.
This is the ``sample_from_anchor`` path in the shared prepare-inputs kernel.
* Sequential Markov sampling: instead of DFlash's single parallel sample, we
sample left-to-right, adding a prefix-dependent Markov bias derived from the
previously sampled token at each step.
CUDA graphs (FULL, mirroring DFlash) cover the whole draft step: the parallel
backbone forward AND the sequential Markov sampling.
"""
from typing import Any
import torch
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
from vllm.v1.worker.gpu.spec_decode.dspark.utils import load_dspark_model
class DSparkSpeculator(DFlashSpeculator):
_speculator_name = "DSpark"
def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)
# Anchor-first: N query tokens per request (anchor + N-1 noise), not 1+N.
self.num_query_per_req = self.num_speculative_steps
# DSpark consumes mean-pooled target aux hidden states at the target
# layers, combined to hidden_size via main_proj. Store that combined
# main_x (hidden_size wide). DSpark does not use the same pre-allocated buffer
# that DeepSeek-V4's MTP uses.
draft_hidden = self.draft_model_config.get_hidden_size()
self.hidden_states = torch.zeros(
self.max_num_tokens, draft_hidden, dtype=self.dtype, device=device
)
self.dflash_causal = False
# The anchor query position is itself a prediction (see module docstring).
self.sample_from_anchor = True
self._step_cols = torch.arange(
self.num_speculative_steps, dtype=torch.int32, device=device
)
self._anchor_idx = (
torch.arange(self.max_num_reqs, dtype=torch.int64, device=device)
* self.num_query_per_req
)
def load_draft_model(
self,
target_model: torch.nn.Module,
target_attn_layer_names: set[str],
) -> torch.nn.Module:
return load_dspark_model(target_model, self.vllm_config)
def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None:
# Sequential Markov sampling over the backbone's output hidden states.
n_spec = self.num_speculative_steps
num_sample = num_reqs * n_spec
# Per-(req, position) head hidden, ordered (req, step).
sample_hidden = head_hidden[self.sample_indices[:num_sample]]
base_logits = self.model.compute_logits(sample_hidden)
vocab_size = base_logits.shape[-1]
base_logits = base_logits.view(num_reqs, n_spec, vocab_size)
idx_map = self.sample_idx_mapping[:num_sample].view(num_reqs, n_spec)
sample_pos = self.sample_pos[:num_sample].view(num_reqs, n_spec)
# Anchor (bonus) token per request = the input id at query offset 0,
# read via the precomputed persistent index (fixed buffer for capture).
prev = self.input_buffers.input_ids[self._anchor_idx[:num_reqs]]
for i in range(n_spec):
# Sequential stage: Markov bias from the previously sampled token.
markov_embed = self.model.markov_embed(prev)
bias = self.model.markov_bias(markov_embed)
logits_i = base_logits[:, i] + bias
if self.draft_logits is not None:
# sample_pos is the predicted token's position Q; the target
# verifies it with the predecessor's Gumbel key (Q-1). Pass Q-1.
draft_i = gumbel_sample(
logits_i,
idx_map[:, i],
self.temperature,
self.seeds,
sample_pos[:, i] - 1,
apply_temperature=True,
output_processed_logits=self.draft_logits,
output_processed_logits_col=self._step_cols[i],
use_fp64=self.use_fp64_gumbel,
)
else:
draft_i = logits_i.argmax(dim=-1)
self.draft_tokens[:num_reqs, i] = draft_i
prev = draft_i
def _generate_draft(
self,
num_reqs: int,
num_tokens_padded: int,
attn_metadata: dict[str, Any] | None,
slot_mappings: dict[str, torch.Tensor] | None,
num_tokens_across_dp: torch.Tensor | None,
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
) -> None:
# Full draft step (captured under CUDA graph): parallel backbone forward
# then sequential Markov sampling over its hidden state outputs.
head_hidden = self._run_model(
num_tokens_padded,
attn_metadata,
slot_mappings,
num_tokens_across_dp,
cudagraph_runtime_mode,
)
self._sample_sequential(num_reqs, head_hidden)
@@ -0,0 +1,64 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch.nn as nn
from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.model_loader import get_model
from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share
def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
draft_model_config = speculative_config.draft_model_config
from vllm.compilation.backends import set_model_tag
# DSpark uses non-causal attention.
causal = False
draft_vllm_config = replace(
vllm_config,
attention_config=replace(
vllm_config.attention_config,
use_non_causal=not causal,
backend=speculative_config.attention_backend,
),
)
with set_model_tag("dspark_head"):
draft_model = get_model(
vllm_config=draft_vllm_config, model_config=draft_model_config
)
if get_pp_group().world_size != 1:
raise NotImplementedError("DSpark does not support pipeline parallelism.")
target_language_model = (
target_model.get_language_model()
if hasattr(target_model, "get_language_model")
else target_model
)
target_inner = target_language_model.model
draft_inner = draft_model.model
target_embed = getattr(target_inner, "embed_tokens", None)
draft_embed = getattr(draft_inner, "embed_tokens", None)
if target_embed is not None and _should_share(
draft_model, "has_own_embed_tokens", draft_embed, target_embed
):
if draft_embed is not None:
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
target_lm_head = getattr(target_model, "lm_head", None)
draft_lm_head = getattr(draft_model, "lm_head", None)
if target_lm_head is not None and _should_share(
draft_model, "has_own_lm_head", draft_lm_head, target_lm_head
):
if draft_lm_head is not None:
del draft_model.lm_head
draft_model.lm_head = target_lm_head
return draft_model
@@ -44,6 +44,15 @@ def get_eagle3_aux_layers_from_config(
if dflash_config and isinstance(dflash_config, dict):
# Add 1 to convert DFlash's aux layer id semantics
layer_ids = [i + 1 for i in (dflash_config.get("target_layer_ids") or [])]
if not layer_ids:
dspark_layer_ids = getattr(hf_config, "dspark_target_layer_ids", None)
if dspark_layer_ids:
layer_ids = [i + 1 for i in dspark_layer_ids]
if not layer_ids:
# Dense DSpark (e.g. Qwen3) also uses different aux layer semantics.
target_layer_ids = getattr(hf_config, "target_layer_ids", None)
if target_layer_ids:
layer_ids = [i + 1 for i in target_layer_ids]
if layer_ids and isinstance(layer_ids, (list, tuple)):
return tuple(layer_ids)
return None
+9 -3
View File
@@ -54,17 +54,23 @@ class DraftTokensHandler:
def get_parallel_drafting_token_id(hf_config) -> int:
"""Resolve the mask token id used for parallel drafting slots.
Checks (in order): `dflash_config.mask_token_id`, `pard_token`,
`ptd_token_id`. Raises ValueError if none are present.
Checks (in order): `dflash_config.mask_token_id`, top-level `mask_token_id`,
`dspark_noise_token_id`, `pard_token`, `ptd_token_id`. Raises ValueError if
none are present.
"""
dflash_config = getattr(hf_config, "dflash_config", None) or {}
if "mask_token_id" in dflash_config:
return int(dflash_config["mask_token_id"])
if getattr(hf_config, "mask_token_id", None) is not None:
return int(hf_config.mask_token_id)
if hasattr(hf_config, "dspark_noise_token_id"):
return int(hf_config.dspark_noise_token_id)
if hasattr(hf_config, "pard_token"):
return int(hf_config.pard_token)
if hasattr(hf_config, "ptd_token_id"):
return int(hf_config.ptd_token_id)
raise ValueError(
"Model config must specify `dflash_config.mask_token_id`,"
" `pard_token`, or `ptd_token_id` for parallel drafting."
" `mask_token_id`, `dspark_noise_token_id`, `pard_token`, or"
" `ptd_token_id` for parallel drafting."
)