From 5c91039c41bc0b6a4a4ab2dc5f62115946e38a30 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Sun, 28 Jun 2026 10:55:54 -0400 Subject: [PATCH] [GLM5.2 Perf] Replace MOE all-reduce with reduce-scatter, 3.1%~3.2 E2E Throughput improvement (#46635) Signed-off-by: yewentao256 --- vllm/model_executor/models/deepseek_v2.py | 98 ++++++++++++++++++++--- 1 file changed, 86 insertions(+), 12 deletions(-) diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index aaca07b693..2c6b075ae7 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -42,6 +42,7 @@ from vllm.distributed import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, tensor_model_parallel_all_gather, + tensor_model_parallel_reduce_scatter, ) from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul @@ -128,6 +129,7 @@ class DeepseekAttention(nn.Module): max_position_embeddings: int = 8192, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, + reduce_results: bool = True, prefix: str = "", **kwargs, ) -> None: @@ -166,6 +168,7 @@ class DeepseekAttention(nn.Module): self.total_num_heads * self.head_dim, hidden_size, bias=False, + reduce_results=reduce_results, quant_config=quant_config, ) @@ -372,15 +375,17 @@ class DeepseekV2MoE(nn.Module): self.gate.e_score_correction_bias.data.to(self.gate.out_dtype) ) - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + def forward( + self, + hidden_states: torch.Tensor, + already_sequence_parallel: bool = False, + ) -> torch.Tensor: num_tokens, hidden_dim = hidden_states.shape hidden_states = hidden_states.view(-1, hidden_dim) # Chunk the hidden states so they aren't replicated across TP ranks. # This avoids duplicate computation in self.experts. - # TODO: We can replace the all_reduce at the end of attn with a - # reduce_scatter instead of chunking here. - if self.is_sequence_parallel: + if self.is_sequence_parallel and not already_sequence_parallel: hidden_states = sequence_parallel_chunk(hidden_states) if self.experts.is_internal_router: @@ -393,7 +398,7 @@ class DeepseekV2MoE(nn.Module): hidden_states=hidden_states, router_logits=router_logits ) - if self.is_sequence_parallel: + if self.is_sequence_parallel and not already_sequence_parallel: final_hidden_states = tensor_model_parallel_all_gather( final_hidden_states, 0 ) @@ -436,6 +441,7 @@ class DeepseekV2Attention(nn.Module): cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, topk_indices_buffer: torch.Tensor | None = None, + reduce_results: bool = True, prefix: str = "", ) -> None: super().__init__() @@ -502,6 +508,7 @@ class DeepseekV2Attention(nn.Module): self.num_heads * self.v_head_dim, self.hidden_size, bias=False, + reduce_results=reduce_results, quant_config=quant_config, prefix=f"{prefix}.o_proj", ) @@ -950,6 +957,7 @@ class DeepseekV2MLAAttention(nn.Module): prefix: str = "", topk_indices_buffer: torch.Tensor | None = None, input_size: int | None = None, + reduce_results: bool = True, ) -> None: super().__init__() self.hidden_size = hidden_size @@ -1018,6 +1026,7 @@ class DeepseekV2MLAAttention(nn.Module): self.num_heads * self.v_head_dim, self.hidden_size, bias=False, + reduce_results=reduce_results, quant_config=quant_config, prefix=f"{prefix}.o_proj", ) @@ -1184,6 +1193,18 @@ class DeepseekV2DecoderLayer(nn.Module): attn_cls = DeepseekV2MLAAttention else: attn_cls = DeepseekV2Attention + is_moe_layer = ( + config.n_routed_experts is not None + and layer_idx >= config.first_k_dense_replace + and layer_idx % moe_layer_freq == 0 + ) + # TODO(wentao): enable SP MoE with PP after the PP boundary logic can safely + # send/receive sequence-parallel hidden_states across stages. + self.use_sequence_parallel_moe = ( + parallel_config.use_sequence_parallel_moe + and parallel_config.pipeline_parallel_size == 1 + and is_moe_layer + ) self.self_attn = attn_cls( vllm_config=vllm_config, config=config, @@ -1199,13 +1220,10 @@ class DeepseekV2DecoderLayer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.self_attn", topk_indices_buffer=topk_indices_buffer, + reduce_results=not self.use_sequence_parallel_moe, ) - if ( - config.n_routed_experts is not None - and layer_idx >= config.first_k_dense_replace - and layer_idx % moe_layer_freq == 0 - ): + if is_moe_layer: self.mlp = DeepseekV2MoE( config=config, parallel_config=parallel_config, @@ -1233,6 +1251,13 @@ class DeepseekV2DecoderLayer(nn.Module): residual: torch.Tensor | None, llama_4_scaling: torch.Tensor | None = None, ) -> torch.Tensor: + full_num_tokens = positions.shape[0] + input_is_sequence_parallel = ( + self.use_sequence_parallel_moe + and residual is not None + and hidden_states.shape[0] != full_num_tokens + ) + # Self Attention if residual is None: residual = hidden_states @@ -1240,6 +1265,10 @@ class DeepseekV2DecoderLayer(nn.Module): else: hidden_states, residual = self.input_layernorm(hidden_states, residual) + if input_is_sequence_parallel: + hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) + hidden_states = hidden_states[:full_num_tokens] + attn_kwargs = { "positions": positions, "hidden_states": hidden_states, @@ -1261,9 +1290,29 @@ class DeepseekV2DecoderLayer(nn.Module): # first layer. residual *= 1.0 / self.routed_scaling_factor + if self.use_sequence_parallel_moe: + sp_remainder = ( + hidden_states.shape[0] % get_tensor_model_parallel_world_size() + ) + # pad if not divisible by world size + if sp_remainder: + sp_pad = get_tensor_model_parallel_world_size() - sp_remainder + hidden_states = torch.nn.functional.pad( + hidden_states, (0, 0, 0, sp_pad) + ) + hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0) + if not input_is_sequence_parallel: + residual = sequence_parallel_chunk(residual) + # Fully Connected hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) - hidden_states = self.mlp(hidden_states) + if self.use_sequence_parallel_moe: + hidden_states = self.mlp( + hidden_states, + already_sequence_parallel=True, + ) + else: + hidden_states = self.mlp(hidden_states) if isinstance(self.mlp, DeepseekV2MLP) and hidden_states.dtype == torch.float16: # Fix FP16 overflow @@ -1385,8 +1434,25 @@ class DeepseekV2Model(nn.Module): islice(self.layers, self.start_layer, self.end_layer), start=self.start_layer, ): + # all gather if we need to use the whole states + if ( + hidden_states.shape[0] != positions.shape[0] + and not layer.use_sequence_parallel_moe + ): + combined_states = torch.cat([hidden_states, residual], dim=-1) + combined_states = tensor_model_parallel_all_gather(combined_states, 0) + combined_states = combined_states[: positions.shape[0]] + hidden_states, residual = combined_states.split( + [self.hidden_size, self.hidden_size], dim=-1 + ) if idx in self.aux_hidden_state_layers: - aux_hidden_states.append(hidden_states + residual) + aux_hidden_state = hidden_states + residual + if aux_hidden_state.shape[0] != positions.shape[0]: + aux_hidden_state = tensor_model_parallel_all_gather( + aux_hidden_state, 0 + ) + aux_hidden_state = aux_hidden_state[: positions.shape[0]] + aux_hidden_states.append(aux_hidden_state) hidden_states, residual = layer( positions, hidden_states, residual, llama_4_scaling ) @@ -1396,6 +1462,14 @@ class DeepseekV2Model(nn.Module): {"hidden_states": hidden_states, "residual": residual} ) + if hidden_states.shape[0] != positions.shape[0]: + combined_states = torch.cat([hidden_states, residual], dim=-1) + combined_states = tensor_model_parallel_all_gather(combined_states, 0) + combined_states = combined_states[: positions.shape[0]] + hidden_states, residual = combined_states.split( + [self.hidden_size, self.hidden_size], dim=-1 + ) + hidden_states, _ = self.norm(hidden_states, residual) if len(aux_hidden_states) > 0: return hidden_states, aux_hidden_states