[Perf] Minimax M3 - Support cross-layer allreduce-norm fusion (#47631)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Wei Zhao
2026-07-07 21:16:32 -07:00
committed by GitHub
co-authored by Yongye Zhu
parent 80eb01e93d
commit 2afa3f7e95
7 changed files with 65 additions and 17 deletions
+4 -4
View File
@@ -72,17 +72,17 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
and layer_idx >= config.first_k_dense_replace
and layer_idx % moe_layer_freq == 0
):
# Defer the MoE cross-rank all-reduce; it is fused into the next
# layer's input_layernorm (or the final norm) via
# fused_allreduce_rms_norm.
self.mlp = DeepseekV2MoE(
config=config,
parallel_config=parallel_config,
quant_config=quant_config,
reduce_results=False,
prefix=f"{prefix}.mlp",
apply_routed_scale_to_output=False,
)
# Defer the MoE cross-rank all-reduce; it is fused into the next
# layer's input_layernorm (or the final norm) via
# fused_allreduce_rms_norm. self.mlp.experts is the MoERunner.
self.mlp.experts.moe_config.skip_final_all_reduce = True
else:
self.mlp = DeepseekV2MLP(
hidden_size=config.hidden_size,
+1 -1
View File
@@ -89,7 +89,7 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
hidden_states, residual = self.mtp_block(
positions=positions, hidden_states=hidden_states, residual=None
)
# mtp_block's MoE output is left un-reduced (skip_final_all_reduce); the
# mtp_block's MoE output is left un-reduced (reduce_results=False); the
# main model fuses that all-reduce into the next norm, but here the
# recycle hidden is consumed directly, so reduce it now.
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
+37 -10
View File
@@ -196,6 +196,7 @@ class MiniMaxM3MoE(nn.Module):
config: PretrainedConfig,
layer_id: int,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
) -> None:
super().__init__()
@@ -261,6 +262,7 @@ class MiniMaxM3MoE(nn.Module):
router_logits_dtype=self.gate.out_dtype,
shared_experts=self.shared_experts,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=f"{prefix}.experts",
)
@@ -658,14 +660,10 @@ class MiniMaxM3DecoderLayer(nn.Module):
layer_id = int(prefix.split(sep=".")[-1])
self.layer_id = layer_id
# Complete the preceding dense MLP's deferred all-reduce
# (reduce_results=False), fused into this layer's input_layernorm.
# Disable this fusion when PP is set
self.fuse_input_allreduce = (
layer_id > 0
and not _is_moe_layer(config, layer_id - 1)
and vllm_config.parallel_config.pipeline_parallel_size == 1
)
# When set, complete the preceding layer's deferred FFN all-reduce
# fused into this layer's input_layernorm.
# Configured by MiniMaxM3Model.__init__
self.fuse_input_allreduce = False
is_sparse_attention_layer = (
force_sparse_attn or layer_id in _sparse_attention_layer_ids(config)
@@ -691,12 +689,19 @@ class MiniMaxM3DecoderLayer(nn.Module):
# Dense layers store the FFN under `mlp`; MoE layers under
# `block_sparse_moe` -- matching the checkpoint's naming.
# Leave the FFN output un-reduced so its all-reduce fuses into the
# next RMSNorm. MTP blocks add the residual directly and PP sends
# hidden states across stages, so both must reduce.
reduce_results = (
is_mtp_block or vllm_config.parallel_config.pipeline_parallel_size > 1
)
self.is_moe_layer = force_moe or _is_moe_layer(config, layer_id)
if self.is_moe_layer:
self.block_sparse_moe = MiniMaxM3MoE(
config=config,
layer_id=layer_id,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=f"{prefix}.block_sparse_moe",
)
else:
@@ -705,7 +710,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
intermediate_size=config.dense_intermediate_size,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
reduce_results=vllm_config.parallel_config.pipeline_parallel_size > 1,
reduce_results=reduce_results,
)
# config.use_gemma_norm is True for M3 -> Gemma-style RMSNorm.
@@ -744,6 +749,14 @@ class MiniMaxM3DecoderLayer(nn.Module):
hidden_states = ffn(hidden_states)
return hidden_states, residual
@property
def ffn_all_reduce_deferred(self) -> bool:
"""This layer's FFN output is left un-reduced; the caller fuses the
all-reduce into the next RMSNorm."""
if self.is_moe_layer:
return self.block_sparse_moe.experts.moe_config.skip_final_all_reduce
return not self.mlp.down_proj.reduce_results
class MiniMaxM3Model(nn.Module, EagleModelMixin):
fall_back_to_pt_during_load = False
@@ -807,6 +820,15 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
["hidden_states", "residual"], config.hidden_size
)
# Configure cross-layer all-reduce/RMSNorm fusion: a layer whose FFN output
# is left un-reduced has that all-reduce fused into the next layer's
# input_layernorm (or the final norm).
prev_defers = False
for idx, layer in enumerate(self.layers[self.start_layer : self.end_layer]):
layer.fuse_input_allreduce = idx > 0 and prev_defers
prev_defers = layer.ffn_all_reduce_deferred
self.fuse_final_norm_allreduce = prev_defers
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
@@ -841,7 +863,12 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
{"hidden_states": hidden_states, "residual": residual}
)
hidden_states, _ = self.norm(hidden_states, residual)
if self.fuse_final_norm_allreduce:
hidden_states, _ = fused_allreduce_gemma_rms_norm(
hidden_states, residual, self.norm
)
else:
hidden_states, _ = self.norm(hidden_states, residual)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states