[Perf] Minimax M3 - Support cross-layer allreduce-norm fusion (#47631)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Wei Zhao
2026-07-07 21:16:32 -07:00
committed by GitHub
co-authored by Yongye Zhu
parent 80eb01e93d
commit 2afa3f7e95
7 changed files with 65 additions and 17 deletions
@@ -428,7 +428,6 @@ class MoERunner(MoERunnerInterface):
if (
shared_output is not None
and not self.moe_config.is_sequence_parallel
and not self.moe_config.skip_final_all_reduce
and self._fused_output_is_reduced
):
shared_output = tensor_model_parallel_all_reduce(shared_output)
@@ -446,6 +445,13 @@ class MoERunner(MoERunnerInterface):
here. Skipped when sequence-parallel is active (SP handles its
own reduction) or when the early path already reduced both outputs.
"""
# skip_final_all_reduce must not coexist with a pre-reduced fused
# output. This should be enforced by MoE config initialization.
if self.moe_config.skip_final_all_reduce:
assert not self._fused_output_is_reduced, (
"skip_final_all_reduce requires an un-reduced fused output"
)
# We don't need to reduce the final output if:
# - We are not running with TP or DP
# - The MK already reduced the fused output itself.