[Spec Decode] DFlash2: local convolution + candidate selector (#52816)
This commit is contained in:
@@ -2,6 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""A layer that compute logits from hidden_stats."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from functools import cache
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
@@ -10,12 +13,43 @@ from vllm.distributed import (
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_gather,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
UnquantizedEmbeddingMethod,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@cache
|
||||
def _flashinfer_topk() -> Callable[..., tuple[torch.Tensor, torch.Tensor]] | None:
|
||||
"""FlashInfer's radix top-k, or None for torch.topk.
|
||||
|
||||
The top-k spans the vocabulary, where the radix kernel is about twice
|
||||
torch.topk.
|
||||
"""
|
||||
if not current_platform.is_cuda():
|
||||
return None
|
||||
if not has_flashinfer():
|
||||
logger.info_once(
|
||||
"flashinfer is unavailable; vocab-parallel top-k uses torch.topk, "
|
||||
"at roughly half the speed."
|
||||
)
|
||||
return None
|
||||
from flashinfer import top_k
|
||||
|
||||
return top_k
|
||||
|
||||
|
||||
def _topk(scores: torch.Tensor, k: int) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
impl = _flashinfer_topk()
|
||||
if impl is None or not scores.is_cuda:
|
||||
return torch.topk(scores, k, dim=-1)
|
||||
return impl(scores, k, sorted=True, deterministic=True)
|
||||
|
||||
|
||||
# --8<-- [start:logits_processor]
|
||||
@@ -204,6 +238,53 @@ class LogitsProcessor(PluggableLayer):
|
||||
top_tokens = gathered[:, :, 1].gather(dim=-1, index=max_rank_idx)
|
||||
return top_tokens.squeeze(-1).to(torch.int64)
|
||||
|
||||
def get_top_k_tokens(
|
||||
self,
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
k: int,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Vocab-parallel top-k without all-gathering full logits.
|
||||
|
||||
The `get_top_tokens` reduction widened from one token to k, returning
|
||||
the values as well as the global ids. Communication is
|
||||
O(batch * 2k * tp_size) rather than O(batch * vocab_size).
|
||||
|
||||
Scale and soft cap are applied to the k selected values rather than
|
||||
the whole vocabulary; both are monotonic, so the selection is the same
|
||||
and only k entries are touched.
|
||||
"""
|
||||
if self.scale <= 0.0 and self.scale != 1.0:
|
||||
raise ValueError(
|
||||
"The local top-k reduction optimization is not supported for "
|
||||
"non-positive logit scaling factors."
|
||||
)
|
||||
|
||||
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
|
||||
|
||||
# Mask out padding entries beyond org_vocab_size on this shard.
|
||||
num_pad = lm_head.shard_indices.num_org_vocab_padding
|
||||
if num_pad > 0:
|
||||
logits[..., -num_pad:] = -float("inf")
|
||||
|
||||
values, ids = _topk(logits, k)
|
||||
# Convert shard-local indices to global vocab indices.
|
||||
ids = ids.to(torch.int64) + lm_head.shard_indices.org_vocab_start_index
|
||||
|
||||
if lm_head.tp_size > 1:
|
||||
values = tensor_model_parallel_all_gather(values, dim=-1)
|
||||
ids = tensor_model_parallel_all_gather(ids, dim=-1)
|
||||
values, selected = _topk(values, k)
|
||||
ids = ids.gather(-1, selected)
|
||||
|
||||
values = values.float()
|
||||
if self.scale != 1.0:
|
||||
values = values * self.scale
|
||||
if self.soft_cap is not None:
|
||||
values = torch.tanh(values / self.soft_cap) * self.soft_cap
|
||||
return ids, values
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
s = f"vocab_size={self.vocab_size}"
|
||||
s += f", org_vocab_size={self.org_vocab_size}"
|
||||
|
||||
Reference in New Issue
Block a user