[Spec Decode] DSpark (#46995)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Giancarlo Delfin
mgoin
parent
63fcce4de1
commit
f5a8d73377
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,133 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DSpark speculator: semi-autoregressive parallel drafting.
|
||||
|
||||
DSpark drafts a block of ``num_speculative_tokens`` tokens in one parallel pass
|
||||
(reusing the DFlash machinery: context-KV precompute + a query-block forward),
|
||||
then injects intra-block dependency with a lightweight sequential Markov head.
|
||||
|
||||
Differences from DFlash:
|
||||
* Anchor-as-first-prediction: each request emits exactly ``N =
|
||||
num_speculative_tokens`` query tokens (anchor + N-1 noise), NOT ``1 + N``.
|
||||
Every query position is a prediction (the anchor predicts the first draft
|
||||
token), so we sample at all N positions and ``sample_pos = query_pos + 1``
|
||||
(standard next-token), whereas DFlash's masks sit AT the predicted position.
|
||||
This is the ``sample_from_anchor`` path in the shared prepare-inputs kernel.
|
||||
* Sequential Markov sampling: instead of DFlash's single parallel sample, we
|
||||
sample left-to-right, adding a prefix-dependent Markov bias derived from the
|
||||
previously sampled token at each step.
|
||||
|
||||
CUDA graphs (FULL, mirroring DFlash) cover the whole draft step: the parallel
|
||||
backbone forward AND the sequential Markov sampling.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark.utils import load_dspark_model
|
||||
|
||||
|
||||
class DSparkSpeculator(DFlashSpeculator):
|
||||
_speculator_name = "DSpark"
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
super().__init__(vllm_config, device)
|
||||
|
||||
# Anchor-first: N query tokens per request (anchor + N-1 noise), not 1+N.
|
||||
self.num_query_per_req = self.num_speculative_steps
|
||||
|
||||
# DSpark consumes mean-pooled target aux hidden states at the target
|
||||
# layers, combined to hidden_size via main_proj. Store that combined
|
||||
# main_x (hidden_size wide). DSpark does not use the same pre-allocated buffer
|
||||
# that DeepSeek-V4's MTP uses.
|
||||
draft_hidden = self.draft_model_config.get_hidden_size()
|
||||
self.hidden_states = torch.zeros(
|
||||
self.max_num_tokens, draft_hidden, dtype=self.dtype, device=device
|
||||
)
|
||||
|
||||
self.dflash_causal = False
|
||||
|
||||
# The anchor query position is itself a prediction (see module docstring).
|
||||
self.sample_from_anchor = True
|
||||
|
||||
self._step_cols = torch.arange(
|
||||
self.num_speculative_steps, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
self._anchor_idx = (
|
||||
torch.arange(self.max_num_reqs, dtype=torch.int64, device=device)
|
||||
* self.num_query_per_req
|
||||
)
|
||||
|
||||
def load_draft_model(
|
||||
self,
|
||||
target_model: torch.nn.Module,
|
||||
target_attn_layer_names: set[str],
|
||||
) -> torch.nn.Module:
|
||||
return load_dspark_model(target_model, self.vllm_config)
|
||||
|
||||
def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None:
|
||||
# Sequential Markov sampling over the backbone's output hidden states.
|
||||
n_spec = self.num_speculative_steps
|
||||
num_sample = num_reqs * n_spec
|
||||
# Per-(req, position) head hidden, ordered (req, step).
|
||||
sample_hidden = head_hidden[self.sample_indices[:num_sample]]
|
||||
base_logits = self.model.compute_logits(sample_hidden)
|
||||
vocab_size = base_logits.shape[-1]
|
||||
base_logits = base_logits.view(num_reqs, n_spec, vocab_size)
|
||||
|
||||
idx_map = self.sample_idx_mapping[:num_sample].view(num_reqs, n_spec)
|
||||
sample_pos = self.sample_pos[:num_sample].view(num_reqs, n_spec)
|
||||
|
||||
# Anchor (bonus) token per request = the input id at query offset 0,
|
||||
# read via the precomputed persistent index (fixed buffer for capture).
|
||||
prev = self.input_buffers.input_ids[self._anchor_idx[:num_reqs]]
|
||||
|
||||
for i in range(n_spec):
|
||||
# Sequential stage: Markov bias from the previously sampled token.
|
||||
markov_embed = self.model.markov_embed(prev)
|
||||
bias = self.model.markov_bias(markov_embed)
|
||||
logits_i = base_logits[:, i] + bias
|
||||
if self.draft_logits is not None:
|
||||
# sample_pos is the predicted token's position Q; the target
|
||||
# verifies it with the predecessor's Gumbel key (Q-1). Pass Q-1.
|
||||
draft_i = gumbel_sample(
|
||||
logits_i,
|
||||
idx_map[:, i],
|
||||
self.temperature,
|
||||
self.seeds,
|
||||
sample_pos[:, i] - 1,
|
||||
apply_temperature=True,
|
||||
output_processed_logits=self.draft_logits,
|
||||
output_processed_logits_col=self._step_cols[i],
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
else:
|
||||
draft_i = logits_i.argmax(dim=-1)
|
||||
self.draft_tokens[:num_reqs, i] = draft_i
|
||||
prev = draft_i
|
||||
|
||||
def _generate_draft(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens_padded: int,
|
||||
attn_metadata: dict[str, Any] | None,
|
||||
slot_mappings: dict[str, torch.Tensor] | None,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
) -> None:
|
||||
# Full draft step (captured under CUDA graph): parallel backbone forward
|
||||
# then sequential Markov sampling over its hidden state outputs.
|
||||
head_hidden = self._run_model(
|
||||
num_tokens_padded,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp,
|
||||
cudagraph_runtime_mode,
|
||||
)
|
||||
self._sample_sequential(num_reqs, head_hidden)
|
||||
@@ -0,0 +1,64 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share
|
||||
|
||||
|
||||
def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
|
||||
# DSpark uses non-causal attention.
|
||||
causal = False
|
||||
draft_vllm_config = replace(
|
||||
vllm_config,
|
||||
attention_config=replace(
|
||||
vllm_config.attention_config,
|
||||
use_non_causal=not causal,
|
||||
backend=speculative_config.attention_backend,
|
||||
),
|
||||
)
|
||||
|
||||
with set_model_tag("dspark_head"):
|
||||
draft_model = get_model(
|
||||
vllm_config=draft_vllm_config, model_config=draft_model_config
|
||||
)
|
||||
|
||||
if get_pp_group().world_size != 1:
|
||||
raise NotImplementedError("DSpark does not support pipeline parallelism.")
|
||||
|
||||
target_language_model = (
|
||||
target_model.get_language_model()
|
||||
if hasattr(target_model, "get_language_model")
|
||||
else target_model
|
||||
)
|
||||
target_inner = target_language_model.model
|
||||
draft_inner = draft_model.model
|
||||
|
||||
target_embed = getattr(target_inner, "embed_tokens", None)
|
||||
draft_embed = getattr(draft_inner, "embed_tokens", None)
|
||||
if target_embed is not None and _should_share(
|
||||
draft_model, "has_own_embed_tokens", draft_embed, target_embed
|
||||
):
|
||||
if draft_embed is not None:
|
||||
del draft_inner.embed_tokens
|
||||
draft_inner.embed_tokens = target_embed
|
||||
|
||||
target_lm_head = getattr(target_model, "lm_head", None)
|
||||
draft_lm_head = getattr(draft_model, "lm_head", None)
|
||||
if target_lm_head is not None and _should_share(
|
||||
draft_model, "has_own_lm_head", draft_lm_head, target_lm_head
|
||||
):
|
||||
if draft_lm_head is not None:
|
||||
del draft_model.lm_head
|
||||
draft_model.lm_head = target_lm_head
|
||||
|
||||
return draft_model
|
||||
Reference in New Issue
Block a user