[Spec Decode][Perf] Replicate DSpark Markov head across TP ranks (#49731)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
a0c092ee72
commit
d6247d7173
@@ -7,7 +7,6 @@ import torch.nn.functional as F
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_gather,
|
||||
)
|
||||
@@ -145,7 +144,8 @@ class LogitsProcessor(PluggableLayer):
|
||||
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
|
||||
|
||||
# Gather logits for TP
|
||||
logits = self._gather_logits(logits)
|
||||
if lm_head.tp_size > 1:
|
||||
logits = self._gather_logits(logits)
|
||||
|
||||
# Remove paddings in vocab (if any).
|
||||
if logits is not None:
|
||||
@@ -169,7 +169,7 @@ class LogitsProcessor(PluggableLayer):
|
||||
"The local argmax reduction optimization is not supported for "
|
||||
"non-positive logit scaling factors."
|
||||
)
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_size = lm_head.tp_size
|
||||
|
||||
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
|
||||
if self.soft_cap is not None:
|
||||
|
||||
@@ -232,6 +232,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
padding_size: padding size for the vocabulary.
|
||||
quant_config: quant config for the layer
|
||||
prefix: full name of the layer in the state dict
|
||||
disable_tp: If true, tensor parallelism will be disabled for this layer.
|
||||
""" # noqa: E501
|
||||
|
||||
# --8<-- [end:vocab_parallel_embedding]
|
||||
@@ -245,12 +246,19 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
*,
|
||||
disable_tp: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
# Keep the input dimensions.
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.disable_tp = disable_tp
|
||||
if disable_tp:
|
||||
tp_rank, self.tp_size = 0, 1
|
||||
else:
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.tp_rank = tp_rank
|
||||
self.num_embeddings = num_embeddings
|
||||
self.padding_size = padding_size
|
||||
self.org_vocab_size = org_num_embeddings or num_embeddings
|
||||
@@ -323,6 +331,13 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
params_dtype=params_dtype,
|
||||
weight_loader=self.weight_loader,
|
||||
)
|
||||
self.update_param_tp_status()
|
||||
|
||||
def update_param_tp_status(self):
|
||||
for param in self.parameters():
|
||||
if isinstance(param, BasevLLMParameter):
|
||||
param.tp_rank = self.tp_rank
|
||||
param.tp_size = self.tp_size
|
||||
|
||||
@classmethod
|
||||
def _get_indices(
|
||||
@@ -487,9 +502,9 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
# Mask the output embedding.
|
||||
if self.tp_size > 1:
|
||||
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
# Reduce across all the model parallel GPUs.
|
||||
output = tensor_model_parallel_all_reduce(output_parallel)
|
||||
return output
|
||||
# Reduce across all the model parallel GPUs.
|
||||
return tensor_model_parallel_all_reduce(output_parallel)
|
||||
return output_parallel
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
s = f"num_embeddings={self.num_embeddings_per_partition}"
|
||||
@@ -516,6 +531,7 @@ class ParallelLMHead(VocabParallelEmbedding):
|
||||
params_dtype: type of the parameters.
|
||||
org_num_embeddings: original vocabulary size (without LoRA).
|
||||
padding_size: padding size for the vocabulary.
|
||||
disable_tp: If true, tensor parallelism will be disabled for this layer.
|
||||
"""
|
||||
|
||||
# --8<-- [end:parallel_lm_head]
|
||||
@@ -530,6 +546,8 @@ class ParallelLMHead(VocabParallelEmbedding):
|
||||
padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
*,
|
||||
disable_tp: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
num_embeddings,
|
||||
@@ -539,6 +557,7 @@ class ParallelLMHead(VocabParallelEmbedding):
|
||||
padding_size,
|
||||
quant_config,
|
||||
prefix,
|
||||
disable_tp=disable_tp,
|
||||
)
|
||||
self.quant_config = quant_config
|
||||
if bias:
|
||||
|
||||
Reference in New Issue
Block a user