[SpecDecode] Reduce TP communication for large-vocab draft models speculative decoding (#39419)

Signed-off-by: EanWang211123 <wangyiheng@sangfor.com.cn>
This commit is contained in:
yiheng
2026-06-10 07:59:24 +00:00
committed by GitHub
parent bb78168b21
commit bd2d83ff31
7 changed files with 53 additions and 39 deletions
@@ -31,6 +31,7 @@ from vllm.model_executor.models.deepseek_v2 import (
)
from vllm.multimodal.inputs import NestedTensors
from .interfaces import LocalArgmaxMixin
from .utils import (
AutoWeightsLoader,
get_draft_quant_config,
@@ -309,7 +310,7 @@ class DeepseekV2Eagle3Model(nn.Module):
return loaded_params
class Eagle3DeepseekV2ForCausalLM(DeepseekV2ForCausalLM):
class Eagle3DeepseekV2ForCausalLM(LocalArgmaxMixin, DeepseekV2ForCausalLM):
"""Eagle3 speculative decoding model for DeepseekV2/V3."""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+35
View File
@@ -1282,6 +1282,41 @@ def supports_any_eagle(
return supports_eagle(model) or supports_eagle3(model)
class LocalArgmaxMixin:
"""Mixin for draft model heads in speculative decoding.
Provides a D2T-aware ``get_top_tokens`` that preserves the
local-argmax communication reduction even when the draft vocabulary
is smaller than the target vocabulary.
When ``draft_id_to_target_id`` is present (shape ``(draft_vocab_size,)``,
containing per-token offset to target vocab id), the draft argmax index
``k`` is mapped to the target vocab id via::
target_id = k + draft_id_to_target_id[k]
This is mathematically equivalent to computing the full-vocab scatter
logits and taking the global argmax, but requires only
O(batch * 2 * tp_size) communication instead of O(batch * vocab_size).
Requires the subclass to expose:
``self.logits_processor``: LogitsProcessor
``self.lm_head``: ParallelLMHead
``self.draft_id_to_target_id`` (optional): nn.Parameter
"""
def get_top_tokens(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Vocab-parallel argmax with optional D2T remapping."""
top = self.logits_processor.get_top_tokens(
self.lm_head,
hidden_states,
)
d2t = getattr(self, "draft_id_to_target_id", None)
if d2t is not None:
top = top + d2t[top]
return top
class EagleModelMixin:
aux_hidden_state_layers: tuple[int, ...] = ()
+2 -1
View File
@@ -62,6 +62,7 @@ from vllm.v1.attention.backend import AttentionType
from .adapters import as_embedding_model, as_seq_cls_model
from .interfaces import (
EagleModelMixin,
LocalArgmaxMixin,
SupportsEagle,
SupportsEagle3,
SupportsLoRA,
@@ -487,7 +488,7 @@ class LlamaModel(nn.Module, EagleModelMixin):
class LlamaForCausalLM(
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
LocalArgmaxMixin, nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
@@ -208,23 +208,6 @@ class EagleLlama4ForCausalLM(Llama4ForCausalLM):
) -> tuple[torch.Tensor, torch.Tensor]:
return self.model(input_ids, positions, hidden_states, inputs_embeds)
def get_top_tokens(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor:
"""Vocab-parallel argmax without all-gathering full logits.
Falls back to full logits when draft_id_to_target_id remapping is
active, since the shared lm_head covers the full target vocab but
the draft model only predicts over a subset (draft_vocab_size).
"""
if (
hasattr(self, "draft_id_to_target_id")
and self.draft_id_to_target_id is not None
):
return self.compute_logits(hidden_states).argmax(dim=-1)
return self.logits_processor.get_top_tokens(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> None:
def transform(inputs):
name, loaded_weight = inputs
+8 -2
View File
@@ -48,7 +48,13 @@ from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.config import set_default_rope_theta
from vllm.v1.attention.backend import AttentionType
from .interfaces import SupportsEagle, SupportsEagle3, SupportsLoRA, SupportsPP
from .interfaces import (
LocalArgmaxMixin,
SupportsEagle,
SupportsEagle3,
SupportsLoRA,
SupportsPP,
)
from .qwen2 import Qwen2MLP as Qwen3MLP
from .qwen2 import Qwen2Model
from .utils import AutoWeightsLoader, PPMissingLayer, extract_layer_index, maybe_prefix
@@ -259,7 +265,7 @@ class Qwen3Model(Qwen2Model):
class Qwen3ForCausalLM(
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
LocalArgmaxMixin, nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
):
packed_modules_mapping = {
"qkv_proj": [
+2 -1
View File
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import LocalArgmaxMixin
from vllm.model_executor.models.qwen3_5 import Qwen3_5DecoderLayer, Qwen3_5RMSNorm
from vllm.model_executor.models.qwen3_next import QwenNextMixtureOfExperts
from vllm.sequence import IntermediateTensors
@@ -353,7 +354,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
"hidden_states": 0,
}
)
class Qwen3_5MTP(nn.Module, SupportsMultiModal):
class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal):
packed_modules_mapping = {
"qkv_proj": [
"q_proj",