diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py index 2868690ebe..6e63563ab8 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py @@ -457,9 +457,18 @@ class MoERunner(MoERunnerInterface): Latent MoE output transforms may contain non-linear ops, e.g. RMSNorm. TP partial routed outputs must be summed in latent space before such transforms are applied. + + A transform that commutes with the TP sum is exempt: if + ``sum_r T(x_r) == T(sum_r x_r)``, applying the transform to the local + partial output and letting the existing late all-reduce sum the + combined result is equivalent, and costs one collective instead of two. + Such a transform opts out by setting ``reduce_commutative = True``. + The default is False, so transforms that do not declare themselves + keep being reduced early. """ if ( self.routed_output_transform is not None + and not getattr(self.routed_output_transform, "reduce_commutative", False) and not self.moe_config.is_sequence_parallel and (self.moe_config.tp_size > 1 or self.moe_config.ep_size > 1) and not fused_output_is_reduced diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index c1bc4c6823..ea8a28ebd9 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -49,6 +49,7 @@ from vllm.model_executor.layers.linear import ( QKVParallelLinear, ReplicatedLinear, RowParallelLinear, + UnquantizedLinearMethod, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2 @@ -204,6 +205,17 @@ class NemotronHMoE(nn.Module): disable_tp=self.is_sequence_parallel, prefix=f"{prefix}.fc2_latent_proj", ) + # A bias-free, unquantized linear commutes with the TP sum + # (sum_r W x_r == W sum_r x_r), so one reduce after the transform + # suffices. Test the layer, not the model-wide `quant_config`: ModelOpt + # excludes the latent projections, so quantized checkpoints still get + # an UnquantizedLinearMethod here. + self.fc2_latent_proj.reduce_commutative = ( + not config.mlp_bias + and isinstance( + self.fc2_latent_proj.quant_method, UnquantizedLinearMethod + ) + ) else: self.fc1_latent_proj = None self.fc2_latent_proj = None