[Spec Decode] DFlash2: local convolution + candidate selector (#52816)

This commit is contained in:
Zihan Zhang
2026-08-20 22:27:22 -07:00
committed by GitHub
parent df344bf36f
commit b389ac2946
14 changed files with 866 additions and 44 deletions
@@ -2,6 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""A layer that compute logits from hidden_stats."""
from collections.abc import Callable
from functools import cache
import torch
import torch.nn.functional as F
@@ -10,12 +13,43 @@ from vllm.distributed import (
tensor_model_parallel_all_gather,
tensor_model_parallel_gather,
)
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.vocab_parallel_embedding import (
UnquantizedEmbeddingMethod,
VocabParallelEmbedding,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
logger = init_logger(__name__)
@cache
def _flashinfer_topk() -> Callable[..., tuple[torch.Tensor, torch.Tensor]] | None:
"""FlashInfer's radix top-k, or None for torch.topk.
The top-k spans the vocabulary, where the radix kernel is about twice
torch.topk.
"""
if not current_platform.is_cuda():
return None
if not has_flashinfer():
logger.info_once(
"flashinfer is unavailable; vocab-parallel top-k uses torch.topk, "
"at roughly half the speed."
)
return None
from flashinfer import top_k
return top_k
def _topk(scores: torch.Tensor, k: int) -> tuple[torch.Tensor, torch.Tensor]:
impl = _flashinfer_topk()
if impl is None or not scores.is_cuda:
return torch.topk(scores, k, dim=-1)
return impl(scores, k, sorted=True, deterministic=True)
# --8<-- [start:logits_processor]
@@ -204,6 +238,53 @@ class LogitsProcessor(PluggableLayer):
top_tokens = gathered[:, :, 1].gather(dim=-1, index=max_rank_idx)
return top_tokens.squeeze(-1).to(torch.int64)
def get_top_k_tokens(
self,
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
k: int,
embedding_bias: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Vocab-parallel top-k without all-gathering full logits.
The `get_top_tokens` reduction widened from one token to k, returning
the values as well as the global ids. Communication is
O(batch * 2k * tp_size) rather than O(batch * vocab_size).
Scale and soft cap are applied to the k selected values rather than
the whole vocabulary; both are monotonic, so the selection is the same
and only k entries are touched.
"""
if self.scale <= 0.0 and self.scale != 1.0:
raise ValueError(
"The local top-k reduction optimization is not supported for "
"non-positive logit scaling factors."
)
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
# Mask out padding entries beyond org_vocab_size on this shard.
num_pad = lm_head.shard_indices.num_org_vocab_padding
if num_pad > 0:
logits[..., -num_pad:] = -float("inf")
values, ids = _topk(logits, k)
# Convert shard-local indices to global vocab indices.
ids = ids.to(torch.int64) + lm_head.shard_indices.org_vocab_start_index
if lm_head.tp_size > 1:
values = tensor_model_parallel_all_gather(values, dim=-1)
ids = tensor_model_parallel_all_gather(ids, dim=-1)
values, selected = _topk(values, k)
ids = ids.gather(-1, selected)
values = values.float()
if self.scale != 1.0:
values = values * self.scale
if self.soft_cap is not None:
values = torch.tanh(values / self.soft_cap) * self.soft_cap
return ids, values
def extra_repr(self) -> str:
s = f"vocab_size={self.vocab_size}"
s += f", org_vocab_size={self.org_vocab_size}"