[MoE Refactor] Rename FusedMoE to FusedMoEFactory (#44941)

Signed-off-by: Bill Nell <bnell@redhat.com>
Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
bnellnm
2026-07-31 11:28:18 +01:00
committed by GitHub
co-authored by Harry Mellor
parent c911120a6f
commit 6e311c6e20
80 changed files with 243 additions and 234 deletions
+9 -7
View File
@@ -19,7 +19,7 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -174,7 +174,7 @@ class LagunaMoE(nn.Module):
prefix=f"{prefix}.gate",
)
# Shared expert (optional) - passed to FusedMoE for overlap optimization
# Shared expert (optional) - passed to FusedMoEFactory for overlap optimization
self.shared_expert: LagunaMLP | None
if config.shared_expert_intermediate_size > 0:
self.shared_expert = LagunaMLP(
@@ -191,20 +191,22 @@ class LagunaMoE(nn.Module):
# Auxiliary-loss-free load-balancing bias (arXiv:2408.15664). The
# checkpoint stores one [num_experts] tensor per MoE layer at
# `mlp.experts.e_score_correction_bias`; registering it as a Parameter
# on the FusedMoE lets the weight loader pick it up and the router
# on the MoERunner lets the weight loader pick it up and the router
# add it during top-k selection. The fused top-k bias router requires
# float32 regardless of model dtype.
e_score_correction_bias = torch.nn.Parameter(
torch.zeros(config.num_experts, dtype=torch.float32),
requires_grad=False,
)
# FusedMoE with SIGMOID routing. Passing `shared_experts=` lets the
# MoERunner with SIGMOID routing. Passing `shared_experts=` lets the
# layer overlap the shared-expert compute with the all2all dispatch.
# `apply_routed_scale_to_output=True` makes FusedMoE handle the
# `apply_routed_scale_to_output=True` makes MoERunner handle the
# FusedMoEFactory with SIGMOID routing. Passing `shared_experts=` lets the
# layer overlap the shared-expert compute with the all2all dispatch.
# `apply_routed_scale_to_output=True` makes FusedMoEFactory handle the
# routed_scaling_factor, shared+routed combine, and TP all-reduce
# internally, so forward() just returns the final hidden states.
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_expert,
num_experts=config.num_experts,
top_k=config.num_experts_per_tok,