[Perf][Nemotron] Skip redundant latent-MoE all-reduce at TP>1 (~13% decode win) (#52301)
Signed-off-by: Junhao Shen <junshen@nvidia.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
Misha Goin
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
a2e8f1ea94
commit
4bac767695
@@ -49,6 +49,7 @@ from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
|
||||
@@ -204,6 +205,17 @@ class NemotronHMoE(nn.Module):
|
||||
disable_tp=self.is_sequence_parallel,
|
||||
prefix=f"{prefix}.fc2_latent_proj",
|
||||
)
|
||||
# A bias-free, unquantized linear commutes with the TP sum
|
||||
# (sum_r W x_r == W sum_r x_r), so one reduce after the transform
|
||||
# suffices. Test the layer, not the model-wide `quant_config`: ModelOpt
|
||||
# excludes the latent projections, so quantized checkpoints still get
|
||||
# an UnquantizedLinearMethod here.
|
||||
self.fc2_latent_proj.reduce_commutative = (
|
||||
not config.mlp_bias
|
||||
and isinstance(
|
||||
self.fc2_latent_proj.quant_method, UnquantizedLinearMethod
|
||||
)
|
||||
)
|
||||
else:
|
||||
self.fc1_latent_proj = None
|
||||
self.fc2_latent_proj = None
|
||||
|
||||
Reference in New Issue
Block a user