[Perf] Minimax M3 - Support cross-layer allreduce-norm fusion (#47631)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -279,6 +279,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
config: DeepseekV2Config | DeepseekV3Config,
|
||||
parallel_config: ParallelConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
apply_routed_scale_to_output: bool = False,
|
||||
):
|
||||
@@ -379,6 +380,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
enable_eplb=self.enable_eplb,
|
||||
num_redundant_experts=self.n_redundant_experts,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
reduce_results=reduce_results,
|
||||
n_shared_experts=config.n_shared_experts
|
||||
if self.is_fusion_moe_shared_experts_enabled
|
||||
else None,
|
||||
|
||||
Reference in New Issue
Block a user