[Spec Decode][Perf] Replicate DSpark Markov head across TP ranks (#49731)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
a0c092ee72
commit
d6247d7173
@@ -7,7 +7,6 @@ import torch.nn.functional as F
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_gather,
|
||||
)
|
||||
@@ -145,7 +144,8 @@ class LogitsProcessor(PluggableLayer):
|
||||
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
|
||||
|
||||
# Gather logits for TP
|
||||
logits = self._gather_logits(logits)
|
||||
if lm_head.tp_size > 1:
|
||||
logits = self._gather_logits(logits)
|
||||
|
||||
# Remove paddings in vocab (if any).
|
||||
if logits is not None:
|
||||
@@ -169,7 +169,7 @@ class LogitsProcessor(PluggableLayer):
|
||||
"The local argmax reduction optimization is not supported for "
|
||||
"non-positive logit scaling factors."
|
||||
)
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_size = lm_head.tp_size
|
||||
|
||||
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
|
||||
if self.soft_cap is not None:
|
||||
|
||||
Reference in New Issue
Block a user