[Spec Decode][Perf] Replicate DSpark Markov head across TP ranks (#49731)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Michael Goin
2026-07-29 11:43:51 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent a0c092ee72
commit d6247d7173
4 changed files with 94 additions and 23 deletions
@@ -7,7 +7,6 @@ import torch.nn.functional as F
from vllm.config import get_current_vllm_config
from vllm.distributed import (
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_gather,
tensor_model_parallel_gather,
)
@@ -145,7 +144,8 @@ class LogitsProcessor(PluggableLayer):
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
# Gather logits for TP
logits = self._gather_logits(logits)
if lm_head.tp_size > 1:
logits = self._gather_logits(logits)
# Remove paddings in vocab (if any).
if logits is not None:
@@ -169,7 +169,7 @@ class LogitsProcessor(PluggableLayer):
"The local argmax reduction optimization is not supported for "
"non-positive logit scaling factors."
)
tp_size = get_tensor_model_parallel_world_size()
tp_size = lm_head.tp_size
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
if self.soft_cap is not None:
@@ -232,6 +232,7 @@ class VocabParallelEmbedding(PluggableLayer):
padding_size: padding size for the vocabulary.
quant_config: quant config for the layer
prefix: full name of the layer in the state dict
disable_tp: If true, tensor parallelism will be disabled for this layer.
""" # noqa: E501
# --8<-- [end:vocab_parallel_embedding]
@@ -245,12 +246,19 @@ class VocabParallelEmbedding(PluggableLayer):
padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
*,
disable_tp: bool = False,
):
super().__init__()
# Keep the input dimensions.
tp_rank = get_tensor_model_parallel_rank()
self.tp_size = get_tensor_model_parallel_world_size()
self.disable_tp = disable_tp
if disable_tp:
tp_rank, self.tp_size = 0, 1
else:
tp_rank = get_tensor_model_parallel_rank()
self.tp_size = get_tensor_model_parallel_world_size()
self.tp_rank = tp_rank
self.num_embeddings = num_embeddings
self.padding_size = padding_size
self.org_vocab_size = org_num_embeddings or num_embeddings
@@ -323,6 +331,13 @@ class VocabParallelEmbedding(PluggableLayer):
params_dtype=params_dtype,
weight_loader=self.weight_loader,
)
self.update_param_tp_status()
def update_param_tp_status(self):
for param in self.parameters():
if isinstance(param, BasevLLMParameter):
param.tp_rank = self.tp_rank
param.tp_size = self.tp_size
@classmethod
def _get_indices(
@@ -487,9 +502,9 @@ class VocabParallelEmbedding(PluggableLayer):
# Mask the output embedding.
if self.tp_size > 1:
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
# Reduce across all the model parallel GPUs.
output = tensor_model_parallel_all_reduce(output_parallel)
return output
# Reduce across all the model parallel GPUs.
return tensor_model_parallel_all_reduce(output_parallel)
return output_parallel
def extra_repr(self) -> str:
s = f"num_embeddings={self.num_embeddings_per_partition}"
@@ -516,6 +531,7 @@ class ParallelLMHead(VocabParallelEmbedding):
params_dtype: type of the parameters.
org_num_embeddings: original vocabulary size (without LoRA).
padding_size: padding size for the vocabulary.
disable_tp: If true, tensor parallelism will be disabled for this layer.
"""
# --8<-- [end:parallel_lm_head]
@@ -530,6 +546,8 @@ class ParallelLMHead(VocabParallelEmbedding):
padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
*,
disable_tp: bool = False,
):
super().__init__(
num_embeddings,
@@ -539,6 +557,7 @@ class ParallelLMHead(VocabParallelEmbedding):
padding_size,
quant_config,
prefix,
disable_tp=disable_tp,
)
self.quant_config = quant_config
if bias: