[SpecDecode] Reduce TP communication for large-vocab draft models speculative decoding (#39419)

Signed-off-by: EanWang211123 <wangyiheng@sangfor.com.cn>
This commit is contained in:
yiheng
2026-06-10 07:59:24 +00:00
committed by GitHub
parent bb78168b21
commit bd2d83ff31
7 changed files with 53 additions and 39 deletions
+35
View File
@@ -1282,6 +1282,41 @@ def supports_any_eagle(
return supports_eagle(model) or supports_eagle3(model)
class LocalArgmaxMixin:
"""Mixin for draft model heads in speculative decoding.
Provides a D2T-aware ``get_top_tokens`` that preserves the
local-argmax communication reduction even when the draft vocabulary
is smaller than the target vocabulary.
When ``draft_id_to_target_id`` is present (shape ``(draft_vocab_size,)``,
containing per-token offset to target vocab id), the draft argmax index
``k`` is mapped to the target vocab id via::
target_id = k + draft_id_to_target_id[k]
This is mathematically equivalent to computing the full-vocab scatter
logits and taking the global argmax, but requires only
O(batch * 2 * tp_size) communication instead of O(batch * vocab_size).
Requires the subclass to expose:
``self.logits_processor``: LogitsProcessor
``self.lm_head``: ParallelLMHead
``self.draft_id_to_target_id`` (optional): nn.Parameter
"""
def get_top_tokens(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Vocab-parallel argmax with optional D2T remapping."""
top = self.logits_processor.get_top_tokens(
self.lm_head,
hidden_states,
)
d2t = getattr(self, "draft_id_to_target_id", None)
if d2t is not None:
top = top + d2t[top]
return top
class EagleModelMixin:
aux_hidden_state_layers: tuple[int, ...] = ()