[Perf] Minimax M3 - Support cross-layer allreduce-norm fusion (#47631)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -127,6 +127,7 @@ def FusedMoE(
|
||||
num_redundant_experts: int = 0,
|
||||
has_bias: bool = False,
|
||||
is_sequence_parallel: bool = False,
|
||||
reduce_results: bool = True,
|
||||
ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"),
|
||||
n_shared_experts: int | None = None,
|
||||
router_logits_dtype: torch.dtype | None = None,
|
||||
@@ -184,6 +185,9 @@ def FusedMoE(
|
||||
num_redundant_experts: Number of redundant experts for EPLB
|
||||
has_bias: Whether expert layers have bias terms
|
||||
is_sequence_parallel: Whether sequence parallelism is enabled
|
||||
reduce_results: Whether to all-reduce the final output. Setting this
|
||||
to False (to fuse the all-reduce downstream) is only honored on the
|
||||
late-AR path.
|
||||
expert_mapping: Expert parameter mapping for weight loading
|
||||
n_shared_experts: Number of shared experts to fuse into the routed
|
||||
grouped GEMM (ROCm; requires aiter FSE or the router-append path)
|
||||
@@ -220,6 +224,14 @@ def FusedMoE(
|
||||
parallel_config=vllm_config.parallel_config,
|
||||
)
|
||||
|
||||
# Resolve the deferred all-reduce request against the parallel config.
|
||||
skip_final_all_reduce = (
|
||||
not reduce_results
|
||||
and not moe_parallel_config.use_all2all_kernels
|
||||
and not moe_parallel_config.is_sequence_parallel
|
||||
and zero_expert_type is None
|
||||
)
|
||||
|
||||
global_num_experts, logical_num_experts, num_fused_shared_experts = (
|
||||
determine_expert_counts(
|
||||
num_experts,
|
||||
@@ -340,6 +352,7 @@ def FusedMoE(
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
max_capture_size=vllm_config.compilation_config.max_cudagraph_capture_size,
|
||||
skip_final_all_reduce=skip_final_all_reduce,
|
||||
)
|
||||
|
||||
logger.debug("FusedMoEConfig = %s", moe_config)
|
||||
|
||||
@@ -428,7 +428,6 @@ class MoERunner(MoERunnerInterface):
|
||||
if (
|
||||
shared_output is not None
|
||||
and not self.moe_config.is_sequence_parallel
|
||||
and not self.moe_config.skip_final_all_reduce
|
||||
and self._fused_output_is_reduced
|
||||
):
|
||||
shared_output = tensor_model_parallel_all_reduce(shared_output)
|
||||
@@ -446,6 +445,13 @@ class MoERunner(MoERunnerInterface):
|
||||
here. Skipped when sequence-parallel is active (SP handles its
|
||||
own reduction) or when the early path already reduced both outputs.
|
||||
"""
|
||||
# skip_final_all_reduce must not coexist with a pre-reduced fused
|
||||
# output. This should be enforced by MoE config initialization.
|
||||
if self.moe_config.skip_final_all_reduce:
|
||||
assert not self._fused_output_is_reduced, (
|
||||
"skip_final_all_reduce requires an un-reduced fused output"
|
||||
)
|
||||
|
||||
# We don't need to reduce the final output if:
|
||||
# - We are not running with TP or DP
|
||||
# - The MK already reduced the fused output itself.
|
||||
|
||||
@@ -279,6 +279,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
config: DeepseekV2Config | DeepseekV3Config,
|
||||
parallel_config: ParallelConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
apply_routed_scale_to_output: bool = False,
|
||||
):
|
||||
@@ -379,6 +380,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
enable_eplb=self.enable_eplb,
|
||||
num_redundant_experts=self.n_redundant_experts,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
reduce_results=reduce_results,
|
||||
n_shared_experts=config.n_shared_experts
|
||||
if self.is_fusion_moe_shared_experts_enabled
|
||||
else None,
|
||||
|
||||
Reference in New Issue
Block a user