[Perf] Minimax M3 - Support cross-layer allreduce-norm fusion (#47631)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -428,7 +428,6 @@ class MoERunner(MoERunnerInterface):
|
||||
if (
|
||||
shared_output is not None
|
||||
and not self.moe_config.is_sequence_parallel
|
||||
and not self.moe_config.skip_final_all_reduce
|
||||
and self._fused_output_is_reduced
|
||||
):
|
||||
shared_output = tensor_model_parallel_all_reduce(shared_output)
|
||||
@@ -446,6 +445,13 @@ class MoERunner(MoERunnerInterface):
|
||||
here. Skipped when sequence-parallel is active (SP handles its
|
||||
own reduction) or when the early path already reduced both outputs.
|
||||
"""
|
||||
# skip_final_all_reduce must not coexist with a pre-reduced fused
|
||||
# output. This should be enforced by MoE config initialization.
|
||||
if self.moe_config.skip_final_all_reduce:
|
||||
assert not self._fused_output_is_reduced, (
|
||||
"skip_final_all_reduce requires an un-reduced fused output"
|
||||
)
|
||||
|
||||
# We don't need to reduce the final output if:
|
||||
# - We are not running with TP or DP
|
||||
# - The MK already reduced the fused output itself.
|
||||
|
||||
Reference in New Issue
Block a user