[SpecDecode] Reduce TP communication for large-vocab draft models speculative decoding (#39419)
Signed-off-by: EanWang211123 <wangyiheng@sangfor.com.cn>
This commit is contained in:
@@ -1282,6 +1282,41 @@ def supports_any_eagle(
|
||||
return supports_eagle(model) or supports_eagle3(model)
|
||||
|
||||
|
||||
class LocalArgmaxMixin:
|
||||
"""Mixin for draft model heads in speculative decoding.
|
||||
|
||||
Provides a D2T-aware ``get_top_tokens`` that preserves the
|
||||
local-argmax communication reduction even when the draft vocabulary
|
||||
is smaller than the target vocabulary.
|
||||
|
||||
When ``draft_id_to_target_id`` is present (shape ``(draft_vocab_size,)``,
|
||||
containing per-token offset to target vocab id), the draft argmax index
|
||||
``k`` is mapped to the target vocab id via::
|
||||
|
||||
target_id = k + draft_id_to_target_id[k]
|
||||
|
||||
This is mathematically equivalent to computing the full-vocab scatter
|
||||
logits and taking the global argmax, but requires only
|
||||
O(batch * 2 * tp_size) communication instead of O(batch * vocab_size).
|
||||
|
||||
Requires the subclass to expose:
|
||||
``self.logits_processor``: LogitsProcessor
|
||||
``self.lm_head``: ParallelLMHead
|
||||
``self.draft_id_to_target_id`` (optional): nn.Parameter
|
||||
"""
|
||||
|
||||
def get_top_tokens(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Vocab-parallel argmax with optional D2T remapping."""
|
||||
top = self.logits_processor.get_top_tokens(
|
||||
self.lm_head,
|
||||
hidden_states,
|
||||
)
|
||||
d2t = getattr(self, "draft_id_to_target_id", None)
|
||||
if d2t is not None:
|
||||
top = top + d2t[top]
|
||||
return top
|
||||
|
||||
|
||||
class EagleModelMixin:
|
||||
aux_hidden_state_layers: tuple[int, ...] = ()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user