[Spec Decode] DSpark (#46995)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Giancarlo Delfin
mgoin
parent
63fcce4de1
commit
f5a8d73377
@@ -472,7 +472,7 @@ class DFlashQwen3Model(nn.Module):
|
||||
self,
|
||||
context_states: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mapping: torch.Tensor | None = None,
|
||||
context_slot_mapping: torch.Tensor | list[torch.Tensor | None] | None = None,
|
||||
) -> None:
|
||||
"""Precompute K/V for context states write them into each layer's KV cache.
|
||||
|
||||
@@ -551,7 +551,13 @@ class DFlashQwen3Model(nn.Module):
|
||||
|
||||
# --- Per-layer cache insert ---
|
||||
all_k_final = all_k_flat.view(L, num_ctx, nkv, hd)
|
||||
per_layer = isinstance(context_slot_mapping, (list, tuple))
|
||||
for i in range(L):
|
||||
slot_mapping = (
|
||||
context_slot_mapping[i] if per_layer else context_slot_mapping
|
||||
)
|
||||
if slot_mapping is None:
|
||||
continue # dummy run: skip cache ops
|
||||
attn = self._attn_layers[i]
|
||||
kv_cache = attn.kv_cache
|
||||
attn.impl.do_kv_cache_update(
|
||||
@@ -559,7 +565,7 @@ class DFlashQwen3Model(nn.Module):
|
||||
all_k_final[i],
|
||||
all_v[i],
|
||||
kv_cache,
|
||||
context_slot_mapping,
|
||||
slot_mapping,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -701,7 +707,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
self,
|
||||
context_states: torch.Tensor,
|
||||
context_positions: torch.Tensor,
|
||||
context_slot_mapping: torch.Tensor | None = None,
|
||||
context_slot_mapping: torch.Tensor | list[torch.Tensor | None] | None = None,
|
||||
) -> None:
|
||||
"""Precompute projected + RoPE'd K/V and write to cache."""
|
||||
self.model.precompute_and_store_context_kv(
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen3 DSpark draft model for semi-autoregressive drafting.
|
||||
|
||||
DSpark drafts a whole block in one parallel pass (DFlash-style: context-KV
|
||||
precompute + a non-causal query-block forward) and then injects intra-block
|
||||
dependency with a lightweight sequential Markov head.
|
||||
|
||||
The parallel backbone is a standard Qwen3 decoder stack reused from the
|
||||
DFlash Qwen3 draft (see qwen3_dflash.py). DSpark adds:
|
||||
* ``markov_head``: low-rank V x r / r x V transition bias added to the base
|
||||
logits, sampled left-to-right by the speculator (the sequential stage).
|
||||
|
||||
DSparkMarkovHead is shared with the DSV4-style DSpark model.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
|
||||
from .qwen3_dflash import DFlashQwen3ForCausalLM, DFlashQwen3Model
|
||||
from .utils import AutoWeightsLoader, maybe_prefix, process_eagle_weight
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class DSparkMarkovHead(nn.Module):
|
||||
"""Sequential transition-bias head (low-rank V x r, r x V).
|
||||
|
||||
``markov_w1[token]`` is an r-dim embedding of the previously sampled token;
|
||||
``markov_w2`` projects it back to a vocab-size bias added to the base logits.
|
||||
"""
|
||||
|
||||
def __init__(self, vocab_size: int, markov_rank: int, prefix: str) -> None:
|
||||
super().__init__()
|
||||
# TODO(ben): profile for which (if any) it makes sense to replicate or TP-shard
|
||||
self.markov_w1 = VocabParallelEmbedding(
|
||||
vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w1")
|
||||
)
|
||||
self.markov_w2 = ParallelLMHead(
|
||||
vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w2")
|
||||
)
|
||||
|
||||
def embed(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""r-dim Markov embedding of ``token_ids`` ([B] -> [B, r])."""
|
||||
return self.markov_w1(token_ids)
|
||||
|
||||
def bias(self, markov_embed: torch.Tensor, logits_processor) -> torch.Tensor:
|
||||
"""Vocab-size transition bias from a Markov embedding ([B, r] -> [B, V])."""
|
||||
return logits_processor(self.markov_w2, markov_embed)
|
||||
|
||||
|
||||
class Qwen3DSparkModel(DFlashQwen3Model):
|
||||
"""DFlash Qwen3 backbone + DSpark Markov head."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
start_layer_id: int = 0,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(
|
||||
vllm_config=vllm_config, start_layer_id=start_layer_id, prefix=prefix
|
||||
)
|
||||
config = self.config
|
||||
self.markov_head = DSparkMarkovHead(
|
||||
config.vocab_size,
|
||||
config.markov_rank,
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
)
|
||||
|
||||
|
||||
class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
nn.Module.__init__(self)
|
||||
self.draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
self.config = self.draft_model_config.hf_config
|
||||
if getattr(self.config, "draft_vocab_size", None) is None:
|
||||
self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
|
||||
target_layer_num = vllm_config.model_config.get_num_layers(
|
||||
vllm_config.parallel_config
|
||||
)
|
||||
self.model = Qwen3DSparkModel(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "model"),
|
||||
start_layer_id=target_layer_num,
|
||||
)
|
||||
|
||||
logit_scale = getattr(self.config, "logit_scale", 1.0)
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.config.draft_vocab_size,
|
||||
self.config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.config.draft_vocab_size, scale=logit_scale
|
||||
)
|
||||
target_vocab_size = vllm_config.model_config.get_vocab_size()
|
||||
if self.config.draft_vocab_size != target_vocab_size:
|
||||
self.draft_id_to_target_id = nn.Parameter(
|
||||
torch.zeros(self.config.draft_vocab_size, dtype=torch.long),
|
||||
requires_grad=False,
|
||||
)
|
||||
else:
|
||||
self.draft_id_to_target_id = None
|
||||
|
||||
def get_draft_kv_cache_layer_names(self) -> list[str]:
|
||||
return [layer.self_attn.attn.layer_name for layer in self.model.layers]
|
||||
|
||||
def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.embed(token_ids)
|
||||
|
||||
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.bias(markov_embed, self.logits_processor)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
model_weights = {}
|
||||
includes_embed_tokens = False
|
||||
includes_lm_head = False
|
||||
for name, loaded_weight in weights:
|
||||
if "lm_head" not in name:
|
||||
name = "model." + name
|
||||
if "embed_tokens" in name:
|
||||
includes_embed_tokens = True
|
||||
if "lm_head" in name:
|
||||
includes_lm_head = True
|
||||
model_weights[name] = loaded_weight
|
||||
# Sets has_own_embed_tokens / has_own_lm_head so load_dspark_model
|
||||
# knows whether to keep these or alias the target's.
|
||||
process_eagle_weight(self, name)
|
||||
|
||||
# mask_embedding is an unused placeholder param; DSpark masks via the vocab row.
|
||||
# confidence_head is not wired into inference yet; skip its weights.
|
||||
# embed_tokens / lm_head are optional; when omitted they are shared from
|
||||
# the target by load_dspark_model, so skip the unloaded params here.
|
||||
skip_substrs = ["mask_embedding", "confidence_head"]
|
||||
if not includes_embed_tokens:
|
||||
skip_substrs.append("embed_tokens")
|
||||
if not includes_lm_head:
|
||||
skip_substrs.append("lm_head")
|
||||
loader = AutoWeightsLoader(self, skip_substrs=skip_substrs)
|
||||
loader.load_weights(model_weights.items())
|
||||
self.model._build_fused_kv_buffers()
|
||||
@@ -585,6 +585,8 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
|
||||
"EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
|
||||
"DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
|
||||
"DSparkDraftModel": ("vllm.models.deepseek_v4", "DSparkDeepseekV4ForCausalLM"),
|
||||
"Qwen3DSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
|
||||
"PEagleDraftModel": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
"PeagleLlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
"Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
|
||||
Reference in New Issue
Block a user