[Spec Decode][Perf] Replicate DSpark Markov head across TP ranks (#49731)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Michael Goin
2026-07-29 11:43:51 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent a0c092ee72
commit d6247d7173
4 changed files with 94 additions and 23 deletions
@@ -7,7 +7,6 @@ import torch.nn.functional as F
from vllm.config import get_current_vllm_config
from vllm.distributed import (
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_gather,
tensor_model_parallel_gather,
)
@@ -145,7 +144,8 @@ class LogitsProcessor(PluggableLayer):
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
# Gather logits for TP
logits = self._gather_logits(logits)
if lm_head.tp_size > 1:
logits = self._gather_logits(logits)
# Remove paddings in vocab (if any).
if logits is not None:
@@ -169,7 +169,7 @@ class LogitsProcessor(PluggableLayer):
"The local argmax reduction optimization is not supported for "
"non-positive logit scaling factors."
)
tp_size = get_tensor_model_parallel_world_size()
tp_size = lm_head.tp_size
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
if self.soft_cap is not None: