From 7fe6d3c76b6f44b54ba629e9accf572adfe86f6a Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Sat, 25 Jul 2026 12:36:19 -0400 Subject: [PATCH] [Perf] Fix moe `reduce_scatter` perf regression by removing additional comm, 5% E2E throughput gain back. (#48763) Signed-off-by: yewentao256 --- vllm/model_executor/models/deepseek_mtp.py | 28 +++------------------- vllm/models/deepseek_v32/nvidia/mtp.py | 22 ++++++++--------- 2 files changed, 13 insertions(+), 37 deletions(-) diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py index 65f860a4a3..35cfce3cb9 100644 --- a/vllm/model_executor/models/deepseek_mtp.py +++ b/vllm/model_executor/models/deepseek_mtp.py @@ -44,25 +44,6 @@ from .utils import ( ) -def _restore_full_token_layout_if_needed( - hidden_states: torch.Tensor, - residual: torch.Tensor, - num_tokens: int, - is_sequence_parallel: bool = False, -) -> tuple[torch.Tensor, torch.Tensor]: - """Restore full token rows for the MTP proposer after SP MoE layers.""" - if not is_sequence_parallel and hidden_states.shape[0] == num_tokens: - return hidden_states, residual - - combined_states = torch.cat([hidden_states, residual], dim=-1) - combined_states = tensor_model_parallel_all_gather(combined_states, 0) - combined_states = combined_states[:num_tokens] - hidden_states, residual = combined_states.split( - [hidden_states.shape[-1], residual.shape[-1]], dim=-1 - ) - return hidden_states, residual - - class SharedHead(nn.Module): def __init__( self, @@ -142,13 +123,10 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module): hidden_states=hidden_states, residual=None, ) - hidden_states, residual = _restore_full_token_layout_if_needed( - hidden_states, - residual, - positions.shape[0], - is_sequence_parallel=self.mtp_block.use_sequence_parallel_moe, - ) hidden_states = residual + hidden_states # pre-final-norm (logits hidden) + if self.mtp_block.use_sequence_parallel_moe: + hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) + hidden_states = hidden_states[: positions.shape[0]] # Recycle the post-final-norm hidden into the next draft step. # compute_logits applies shared_head (== final norm) to the pre-norm # element, so logits and the recycle each get exactly one final-norm. diff --git a/vllm/models/deepseek_v32/nvidia/mtp.py b/vllm/models/deepseek_v32/nvidia/mtp.py index c8f2fcc5ff..7130bb5d19 100644 --- a/vllm/models/deepseek_v32/nvidia/mtp.py +++ b/vllm/models/deepseek_v32/nvidia/mtp.py @@ -8,7 +8,10 @@ import torch.nn as nn from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig -from vllm.distributed import tensor_model_parallel_all_reduce +from vllm.distributed import ( + tensor_model_parallel_all_gather, + tensor_model_parallel_all_reduce, +) from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) @@ -24,10 +27,7 @@ from vllm.model_executor.model_loader.weight_utils import ( default_weight_loader, maybe_remap_kv_scale_name, ) -from vllm.model_executor.models.deepseek_mtp import ( - SharedHead, - _restore_full_token_layout_if_needed, -) +from vllm.model_executor.models.deepseek_mtp import SharedHead from vllm.model_executor.models.deepseek_v2 import ( DeepseekV2MixtureOfExperts, DeepseekV2MoE, @@ -95,13 +95,8 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): hidden_states, residual = self.mtp_block( positions=positions, hidden_states=hidden_states, residual=None ) - hidden_states, residual = _restore_full_token_layout_if_needed( - hidden_states, - residual, - positions.shape[0], - is_sequence_parallel=self.mtp_block.use_sequence_parallel_moe, - ) - if not self.mtp_block.use_sequence_parallel_moe: + is_sequence_parallel = self.mtp_block.use_sequence_parallel_moe + if not is_sequence_parallel: # Without sequence parallelism, the MoE output is left un-reduced. hidden_states = tensor_model_parallel_all_reduce(hidden_states) # Recycle the POST-final-norm hidden into the next draft step. The @@ -115,6 +110,9 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): # the legacy proposer (model_returns_tuple is True for the # DeepSeekMTPModel architecture). hidden_states, _ = self.shared_head.norm(hidden_states, residual) + if is_sequence_parallel: + hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) + hidden_states = hidden_states[: positions.shape[0]] return hidden_states, hidden_states