From 4238b011a76629104b00ecf999d5390b94a1d289 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Wed, 15 Jul 2026 19:33:15 -0400 Subject: [PATCH] [Feature] Migrate moe sp support to non-torch compiled path for GLM5.2 (#47881) Signed-off-by: yewentao256 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../fused_moe/experts/trtllm_fp8_moe.py | 4 +- vllm/model_executor/models/deepseek_mtp.py | 4 +- vllm/models/deepseek_v32/nvidia/model.py | 92 +++++++++++++++++-- vllm/models/deepseek_v32/nvidia/mtp.py | 8 +- 4 files changed, 92 insertions(+), 16 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py index edcea5361e..55ddd1a2c9 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py @@ -122,7 +122,9 @@ class TrtLlmFp8ExpertsBase: return ( not moe_parallel_config.use_all2all_kernels or moe_parallel_config.use_ag_rs_all2all_kernels - ) and not moe_parallel_config.enable_eplb + ) and not ( + moe_parallel_config.enable_eplb or moe_parallel_config.is_sequence_parallel + ) class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular): diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py index da7f77cae0..746f20e746 100644 --- a/vllm/model_executor/models/deepseek_mtp.py +++ b/vllm/model_executor/models/deepseek_mtp.py @@ -45,9 +45,10 @@ def _restore_full_token_layout_if_needed( hidden_states: torch.Tensor, residual: torch.Tensor, num_tokens: int, + is_sequence_parallel: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: """Restore full token rows for the MTP proposer after SP MoE layers.""" - if hidden_states.shape[0] == num_tokens: + if not is_sequence_parallel and hidden_states.shape[0] == num_tokens: return hidden_states, residual combined_states = torch.cat([hidden_states, residual], dim=-1) @@ -142,6 +143,7 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module): hidden_states, residual, positions.shape[0], + is_sequence_parallel=self.mtp_block.use_sequence_parallel_moe, ) hidden_states = residual + hidden_states # pre-final-norm (logits hidden) # Recycle the post-final-norm hidden into the next draft step. diff --git a/vllm/models/deepseek_v32/nvidia/model.py b/vllm/models/deepseek_v32/nvidia/model.py index 22ced9fa94..353aedc8ce 100644 --- a/vllm/models/deepseek_v32/nvidia/model.py +++ b/vllm/models/deepseek_v32/nvidia/model.py @@ -7,7 +7,11 @@ from itertools import islice import torch from vllm.config import VllmConfig -from vllm.distributed import get_pp_group +from vllm.distributed import ( + get_pp_group, + tensor_model_parallel_all_gather, + tensor_model_parallel_reduce_scatter, +) from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) @@ -32,6 +36,7 @@ from vllm.model_executor.models.utils import ( is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, + sequence_parallel_chunk, ) from vllm.sequence import IntermediateTensors @@ -39,6 +44,18 @@ from .attention import DeepseekV32Attention from .fused_ops import fused_allreduce_rms_norm +def _all_gather_sp_states( + hidden_states: torch.Tensor, + residual: torch.Tensor, + num_tokens: int, +) -> tuple[torch.Tensor, torch.Tensor]: + # combine hidden_states and residual and all gather once + combined_states = torch.cat([hidden_states, residual], dim=-1) + combined_states = tensor_model_parallel_all_gather(combined_states, 0)[:num_tokens] + hidden_states, residual = combined_states.chunk(2, dim=-1) + return hidden_states, residual.contiguous() + + class DeepseekV32DecoderLayer(torch.nn.Module): def __init__( self, @@ -92,6 +109,12 @@ class DeepseekV32DecoderLayer(torch.nn.Module): prefix=f"{prefix}.mlp", reduce_results=False, ) + self.use_sequence_parallel_moe = ( + parallel_config.use_sequence_parallel_moe + and parallel_config.pipeline_parallel_size == 1 + and isinstance(self.mlp, DeepseekV2MoE) + ) + self.tp_size = parallel_config.tensor_parallel_size self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps @@ -104,25 +127,53 @@ class DeepseekV32DecoderLayer(torch.nn.Module): hidden_states: torch.Tensor, residual: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor]: + full_num_tokens = positions.shape[0] + input_is_sequence_parallel = ( + self.use_sequence_parallel_moe + and residual is not None + and hidden_states.shape[0] != full_num_tokens + ) + if residual is None: # First layer: hidden_states is the (already reduced) embedding. residual = hidden_states hidden_states = self.input_layernorm(hidden_states) + elif input_is_sequence_parallel: + hidden_states, residual = self.input_layernorm(hidden_states, residual) else: # The previous layer's MLP/MoE output is left un-reduced; fuse its # all-reduce into this input_layernorm. hidden_states, residual = fused_allreduce_rms_norm( hidden_states, residual, self.input_layernorm ) - # self_attn's o_proj runs reduce_results=False; fuse its all-reduce with - # the post-attention RMSNorm. + if input_is_sequence_parallel: + hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) + hidden_states = hidden_states[:full_num_tokens] + + # self_attn's o_proj runs reduce_results=False; reduce before RMSNorm. hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states) - hidden_states, residual = fused_allreduce_rms_norm( - hidden_states, residual, self.post_attention_layernorm - ) + if self.use_sequence_parallel_moe: + # small trick using minus, eg. -17 % 8 = 7 + sp_pad = (-hidden_states.shape[0]) % self.tp_size + # pad if not divisible by world size + hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad)) + hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0) + if not input_is_sequence_parallel: + residual = sequence_parallel_chunk(residual) + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual + ) + else: + hidden_states, residual = fused_allreduce_rms_norm( + hidden_states, residual, self.post_attention_layernorm + ) + # MLP/MoE runs un-reduced; its all-reduce is fused into the next layer's # input_layernorm (or the model's final norm). - hidden_states = self.mlp(hidden_states) + if self.use_sequence_parallel_moe: + hidden_states = self.mlp(hidden_states, already_sequence_parallel=True) + else: + hidden_states = self.mlp(hidden_states) return hidden_states, residual @@ -206,12 +257,26 @@ class DeepseekV32Model(torch.nn.Module): residual = intermediate_tensors["residual"] aux_hidden_states = [] + full_num_tokens = positions.shape[0] for idx, layer in enumerate( islice(self.layers, self.start_layer, self.end_layer), start=self.start_layer, ): + if ( + hidden_states.shape[0] != full_num_tokens + and not layer.use_sequence_parallel_moe + ): + hidden_states, residual = _all_gather_sp_states( + hidden_states, residual, full_num_tokens + ) if idx in self.aux_hidden_state_layers: - aux_hidden_states.append(hidden_states + residual) + aux_hidden_state = hidden_states + residual + if aux_hidden_state.shape[0] != full_num_tokens: + aux_hidden_state = tensor_model_parallel_all_gather( + aux_hidden_state, 0 + ) + aux_hidden_state = aux_hidden_state[:full_num_tokens] + aux_hidden_states.append(aux_hidden_state) hidden_states, residual = layer(positions, hidden_states, residual) if not get_pp_group().is_last_rank: @@ -219,8 +284,15 @@ class DeepseekV32Model(torch.nn.Module): {"hidden_states": hidden_states, "residual": residual} ) - # Last layer's MoE output is un-reduced; fuse its all-reduce into norm. - hidden_states, _ = fused_allreduce_rms_norm(hidden_states, residual, self.norm) + if hidden_states.shape[0] != full_num_tokens: + hidden_states, residual = _all_gather_sp_states( + hidden_states, residual, full_num_tokens + ) + hidden_states, _ = self.norm(hidden_states, residual) + else: + hidden_states, _ = fused_allreduce_rms_norm( + hidden_states, residual, self.norm + ) if len(aux_hidden_states) > 0: return hidden_states, aux_hidden_states return hidden_states diff --git a/vllm/models/deepseek_v32/nvidia/mtp.py b/vllm/models/deepseek_v32/nvidia/mtp.py index 8edb3283d0..d3d8e5aae9 100644 --- a/vllm/models/deepseek_v32/nvidia/mtp.py +++ b/vllm/models/deepseek_v32/nvidia/mtp.py @@ -96,11 +96,11 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): hidden_states, residual, positions.shape[0], + is_sequence_parallel=self.mtp_block.use_sequence_parallel_moe, ) - # mtp_block's MoE output is left un-reduced (skip_final_all_reduce); the - # main model fuses that all-reduce into the next norm, but here the - # recycle hidden is consumed directly, so reduce it now. - hidden_states = tensor_model_parallel_all_reduce(hidden_states) + if not self.mtp_block.use_sequence_parallel_moe: + # Without sequence parallelism, the MoE output is left un-reduced. + hidden_states = tensor_model_parallel_all_reduce(hidden_states) # Recycle the POST-final-norm hidden into the next draft step. The # residual-add is fused into the final RMSNorm so it is computed # exactly once, and the result is returned for both tuple positions: