[Spec Decode] DFlash2: local convolution + candidate selector (#52816)

This commit is contained in:
Zihan Zhang
2026-08-20 22:27:22 -07:00
committed by GitHub
parent df344bf36f
commit b389ac2946
14 changed files with 866 additions and 44 deletions
@@ -2,6 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""A layer that compute logits from hidden_stats."""
from collections.abc import Callable
from functools import cache
import torch
import torch.nn.functional as F
@@ -10,12 +13,43 @@ from vllm.distributed import (
tensor_model_parallel_all_gather,
tensor_model_parallel_gather,
)
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.vocab_parallel_embedding import (
UnquantizedEmbeddingMethod,
VocabParallelEmbedding,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
logger = init_logger(__name__)
@cache
def _flashinfer_topk() -> Callable[..., tuple[torch.Tensor, torch.Tensor]] | None:
"""FlashInfer's radix top-k, or None for torch.topk.
The top-k spans the vocabulary, where the radix kernel is about twice
torch.topk.
"""
if not current_platform.is_cuda():
return None
if not has_flashinfer():
logger.info_once(
"flashinfer is unavailable; vocab-parallel top-k uses torch.topk, "
"at roughly half the speed."
)
return None
from flashinfer import top_k
return top_k
def _topk(scores: torch.Tensor, k: int) -> tuple[torch.Tensor, torch.Tensor]:
impl = _flashinfer_topk()
if impl is None or not scores.is_cuda:
return torch.topk(scores, k, dim=-1)
return impl(scores, k, sorted=True, deterministic=True)
# --8<-- [start:logits_processor]
@@ -204,6 +238,53 @@ class LogitsProcessor(PluggableLayer):
top_tokens = gathered[:, :, 1].gather(dim=-1, index=max_rank_idx)
return top_tokens.squeeze(-1).to(torch.int64)
def get_top_k_tokens(
self,
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
k: int,
embedding_bias: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Vocab-parallel top-k without all-gathering full logits.
The `get_top_tokens` reduction widened from one token to k, returning
the values as well as the global ids. Communication is
O(batch * 2k * tp_size) rather than O(batch * vocab_size).
Scale and soft cap are applied to the k selected values rather than
the whole vocabulary; both are monotonic, so the selection is the same
and only k entries are touched.
"""
if self.scale <= 0.0 and self.scale != 1.0:
raise ValueError(
"The local top-k reduction optimization is not supported for "
"non-positive logit scaling factors."
)
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
# Mask out padding entries beyond org_vocab_size on this shard.
num_pad = lm_head.shard_indices.num_org_vocab_padding
if num_pad > 0:
logits[..., -num_pad:] = -float("inf")
values, ids = _topk(logits, k)
# Convert shard-local indices to global vocab indices.
ids = ids.to(torch.int64) + lm_head.shard_indices.org_vocab_start_index
if lm_head.tp_size > 1:
values = tensor_model_parallel_all_gather(values, dim=-1)
ids = tensor_model_parallel_all_gather(ids, dim=-1)
values, selected = _topk(values, k)
ids = ids.gather(-1, selected)
values = values.float()
if self.scale != 1.0:
values = values * self.scale
if self.soft_cap is not None:
values = torch.tanh(values / self.soft_cap) * self.soft_cap
return ids, values
def extra_repr(self) -> str:
s = f"vocab_size={self.vocab_size}"
s += f", org_vocab_size={self.org_vocab_size}"
+11 -4
View File
@@ -56,10 +56,13 @@ _SLIDING_ATTENTION = "sliding_attention"
def _dflash_layer_causal(config: Qwen3Config, layer_idx: int) -> bool:
"""``dflash_config.causal`` overrides all layers; else only SWA layers causal."""
"""Resolve explicit causality before falling back to legacy layer defaults."""
is_causal = getattr(config, "is_causal", None)
if is_causal is not None:
return bool(is_causal)
override = (getattr(config, "dflash_config", None) or {}).get("causal")
if override is not None:
return override
return bool(override)
layer_types = getattr(config, "layer_types", None)
return bool(layer_types) and layer_types[layer_idx] == _SLIDING_ATTENTION
@@ -374,6 +377,8 @@ class DFlashQwen3DecoderLayer(nn.Module):
@support_torch_compile
class DFlashQwen3Model(nn.Module):
decoder_layer_cls = DFlashQwen3DecoderLayer
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={
"midlayer.": "layers.0.",
@@ -434,7 +439,7 @@ class DFlashQwen3Model(nn.Module):
self.layers = nn.ModuleList(
[
DFlashQwen3DecoderLayer(
self.decoder_layer_cls(
current_vllm_config,
config=self.config,
layer_idx=layer_idx,
@@ -699,6 +704,8 @@ class DFlashQwen3Model(nn.Module):
class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
model_cls = DFlashQwen3Model
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self)
self.draft_model_config = vllm_config.speculative_config.draft_model_config
@@ -708,7 +715,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
self.model = DFlashQwen3Model(
self.model = self.model_cls(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
start_layer_id=target_layer_num,
+290
View File
@@ -0,0 +1,290 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn.functional as F
from torch import nn
from vllm.compilation.backends import set_model_tag
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from .qwen3_dflash import (
DFlashQwen3DecoderLayer,
DFlashQwen3ForCausalLM,
DFlashQwen3Model,
)
from .utils import maybe_prefix
def _grouped_conv(
hidden_states: torch.Tensor,
delta: torch.Tensor,
base: torch.Tensor,
block_size: int,
num_groups: int,
group_size: int,
taps: int,
) -> torch.Tensor:
blocks = hidden_states.unflatten(-1, (num_groups, group_size))
coefficients = base.view(1, taps, num_groups, group_size) + delta.unsqueeze(-1)
output = coefficients[:, 0] * blocks
position = torch.arange(hidden_states.shape[0], device=hidden_states.device)
if block_size & (block_size - 1) == 0:
position = position & (block_size - 1)
else:
position = position % block_size
for tap in range(1, taps):
shifted = F.pad(blocks[:-tap], (0, 0, 0, 0, tap, 0))
output += coefficients[:, tap] * shifted * (position >= tap).view(-1, 1, 1)
return output.flatten(-2)
class DFlashGroupedConv(nn.Module):
def __init__(
self,
hidden_size: int,
taps: int,
group_size: int,
block_size: int,
params_dtype: torch.dtype,
prefix: str,
) -> None:
super().__init__()
if hidden_size % group_size:
raise ValueError(
f"conv_group_size={group_size} must divide hidden_size={hidden_size}."
)
self.block_size = block_size
self.taps = taps
self.group_size = group_size
self.num_groups = hidden_size // group_size
self.base_kernel = nn.Parameter(
torch.empty(2, taps, hidden_size, dtype=params_dtype),
requires_grad=False,
)
self.kernel_projection = ReplicatedLinear(
hidden_size,
2 * taps * self.num_groups,
bias=False,
params_dtype=params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "kernel_projection"),
return_bias=False,
)
def _convolve(
self, hidden_states: torch.Tensor, delta: torch.Tensor, side: int
) -> torch.Tensor:
return _grouped_conv(
hidden_states,
delta,
self.base_kernel[side],
self.block_size,
self.num_groups,
self.group_size,
self.taps,
)
def prepare(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
coefficients = self.kernel_projection(hidden_states).reshape(
hidden_states.shape[0], 2, self.taps, self.num_groups
)
return self._convolve(hidden_states, coefficients[:, 0], 0), coefficients[:, 1]
def finish(
self, hidden_states: torch.Tensor, coefficients: torch.Tensor
) -> torch.Tensor:
return self._convolve(hidden_states, coefficients, 1)
class DFlash2Qwen3DecoderLayer(DFlashQwen3DecoderLayer):
def __init__(
self,
vllm_config: VllmConfig,
*,
config,
layer_idx: int,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__(
vllm_config,
config=config,
layer_idx=layer_idx,
cache_config=cache_config,
quant_config=quant_config,
prefix=prefix,
)
draft_config = config.dflash_config
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
conv_args = dict(
hidden_size=config.hidden_size,
taps=int(draft_config["conv_kernel_size"]),
group_size=int(draft_config["conv_group_size"]),
# Query tokens per request: the bonus token plus the mask tokens.
block_size=1 + speculative_config.num_speculative_tokens,
params_dtype=vllm_config.model_config.dtype,
)
self.attention_conv = DFlashGroupedConv(
**conv_args, prefix=maybe_prefix(prefix, "attention_conv")
)
self.mlp_conv = DFlashGroupedConv(
**conv_args, prefix=maybe_prefix(prefix, "mlp_conv")
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
else:
hidden_states, residual = self.input_layernorm(hidden_states, residual)
hidden_states, coefficients = self.attention_conv.prepare(hidden_states)
hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states)
hidden_states = self.attention_conv.finish(hidden_states, coefficients)
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
hidden_states, coefficients = self.mlp_conv.prepare(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = self.mlp_conv.finish(hidden_states, coefficients)
return hidden_states, residual
def _score_edges(
predecessor_table: torch.Tensor,
successor_table: torch.Tensor,
candidate_ids: torch.Tensor,
unary_logits: torch.Tensor,
hidden: torch.Tensor,
anchor_token_ids: torch.Tensor,
top_k: int,
) -> torch.Tensor:
successors = successor_table[candidate_ids]
predecessor_ids = torch.cat(
(
anchor_token_ids[:, None, None].expand(-1, 1, top_k),
candidate_ids[:, :-1],
),
dim=1,
)
predecessors = predecessor_table[predecessor_ids]
return unary_logits[:, :, None] + torch.einsum(
"blpr,blcr->blpc", predecessors * hidden[:, :, None], successors
)
@support_torch_compile
class CandidateSelector(nn.Module):
def __init__(
self,
hidden_size: int,
vocab_size: int,
rank: int,
top_k: int,
params_dtype: torch.dtype,
prefix: str,
) -> None:
super().__init__()
self.top_k = top_k
self.predecessor_codebook = nn.Parameter(
torch.empty(vocab_size, rank, dtype=params_dtype), requires_grad=False
)
self.successor_codebook = nn.Parameter(
torch.empty(vocab_size, rank, dtype=params_dtype), requires_grad=False
)
self.hidden_projection = ReplicatedLinear(
hidden_size,
rank,
bias=False,
params_dtype=params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "hidden_projection"),
return_bias=False,
)
def forward(
self,
candidate_ids: torch.Tensor,
unary_logits: torch.Tensor,
hidden_states: torch.Tensor,
anchor_token_ids: torch.Tensor,
) -> torch.Tensor:
hidden = self.hidden_projection(hidden_states)
return _score_edges(
self.predecessor_codebook,
self.successor_codebook,
candidate_ids,
unary_logits,
hidden,
anchor_token_ids,
self.top_k,
)
class DFlash2Qwen3Model(DFlashQwen3Model):
decoder_layer_cls = DFlash2Qwen3DecoderLayer
def __init__(
self,
*,
vllm_config: VllmConfig,
start_layer_id: int = 0,
prefix: str = "",
) -> None:
super().__init__(
vllm_config=vllm_config,
start_layer_id=start_layer_id,
prefix=prefix,
)
draft_config = self.config.dflash_config
self.input_embedding_scale = float(
draft_config.get("input_embedding_scale", 1.0)
)
# Without its own tag the selector shares the draft head's compile cache.
with set_model_tag("dflash2_candidate_selector"):
self.candidate_selector = CandidateSelector(
hidden_size=self.config.hidden_size,
vocab_size=self.config.vocab_size,
rank=int(draft_config["selector_rank"]),
top_k=int(draft_config["selector_top_k"]),
params_dtype=vllm_config.model_config.dtype,
prefix=maybe_prefix(prefix, "candidate_selector"),
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return super().embed_input_ids(input_ids) * self.input_embedding_scale
class DFlash2Qwen3ForCausalLM(DFlashQwen3ForCausalLM):
model_cls = DFlash2Qwen3Model
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__(vllm_config=vllm_config, prefix=prefix)
draft_config = self.config.dflash_config
softcap = float(draft_config.get("final_logit_softcapping") or 0.0)
self.candidate_logits_processor = LogitsProcessor(
vllm_config.model_config.get_vocab_size(),
scale=float(draft_config.get("output_multiplier", 1.0)),
soft_cap=softcap if softcap > 0 else None,
)
def compute_candidates(
self, hidden_states: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
return self.candidate_logits_processor.get_top_k_tokens(
self.lm_head, hidden_states, self.model.candidate_selector.top_k
)
EntryClass = DFlash2Qwen3ForCausalLM
+1
View File
@@ -632,6 +632,7 @@ _SPECULATIVE_DECODING_MODELS = {
"EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
"EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
"DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
"DFlash2DraftModel": ("qwen3_dflash2", "DFlash2Qwen3ForCausalLM"),
# Muse Glimmer's DFlash draft head, reusing the generic qwen3_dflash
# implementation. EAGLEConfig rewrites a dflash draft's architecture to
# DFlash{arch} unless it already starts or ends with "DFlash" (see