[MoE Refactor] Rename FusedMoE to FusedMoEFactory (#44941)

Signed-off-by: Bill Nell <bnell@redhat.com>
Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
bnellnm
2026-07-31 11:28:18 +01:00
committed by GitHub
co-authored by Harry Mellor
parent c911120a6f
commit 6e311c6e20
80 changed files with 243 additions and 234 deletions
@@ -19,7 +19,7 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.layer import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import (
@@ -63,7 +63,7 @@ def get_config() -> dict[str, Any] | None:
__all__ = [
"FusedMoE",
"FusedMoEFactory",
"FusedMoERouter",
"FusedMoEConfig",
"FusedMoEQuantConfig",
@@ -1134,9 +1134,9 @@ class FusedMoEParallelConfig:
level's of parallelism to use in the fused moe layer.
Args:
tp_size_ (int): `tp_size` passed into the FusedMoE constructor.
pcp_size_ (int): `pcp_size` passed into the FusedMoE constructor.
dp_size_ (int): `dp_size` passed into the FusedMoE constructor.
tp_size_ (int): `tp_size` passed into the FusedMoEFactory constructor.
pcp_size_ (int): `pcp_size` passed into the FusedMoEFactory constructor.
dp_size_ (int): `dp_size` passed into the FusedMoEFactory constructor.
vllm_parallel_config (ParallelConfig): vLLM's parallel config
object which contains the `enable_expert_parallel` flag.
@@ -96,8 +96,7 @@ def determine_expert_counts(
return global_num_experts, logical_num_experts, num_fused_shared_experts
# TODO: rename this
def FusedMoE(
def FusedMoEFactory(
num_experts: int, # Global number of experts
top_k: int,
hidden_size: int,
@@ -1016,7 +1016,7 @@ def _process_weights_xpu(
w2: [E, K, N] int4 (uint8 storage [E, K, N // 2])
w2_scales: [E, K, N // group_size] params_dtype
Input GPTQ layout from FusedMoE.weight_loader:
Input GPTQ layout from MoERunner.weight_loader:
w13: [E, K // 8, 2*N] int32 (8 nibbles per int32 along the input dim)
w13_scales: [E, K // group_size, 2*N] params_dtype
w2: [E, N // 8, K] int32
@@ -1419,7 +1419,7 @@ def convert_to_wna16_moe_kernel_format(
Args:
backend: the selected ``WNA16MoEBackend``.
layer: the ``FusedMoE`` layer whose parameters are being prepared.
layer: the ``MoERunner`` layer whose parameters are being prepared.
quant_config: the ``QuantizationConfig`` for this layer.
input_dtype: optional activation dtype, usually should be 16 bit.
"""
@@ -180,7 +180,7 @@ class RoutedExpertsCapturer:
# (trailing rows are SP ceil-div padding). The TP group
# is always initialized on real rollout workers, and
# every rank in the group reaches this branch in
# lockstep (bind is per-FusedMoE layer, SP is a global
# lockstep (bind is per-FusedMoEFactory layer, SP is a global
# condition), so a bare all_gather here will not
# deadlock -- let it raise if the precondition is
# violated rather than skip silently.
@@ -353,7 +353,7 @@ class MoERunner(MoERunnerInterface):
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Apply transform for routed experts (e.g., latent projection).
This is called by FusedMoE.forward_native. The original hidden_states
This is called by MoERunner.forward_native. The original hidden_states
is saved separately so shared experts get [S, hidden_size] while
routed experts get the transformed [S, moe_latent_size].
@@ -646,7 +646,7 @@ class MoERunner(MoERunnerInterface):
):
# If router/gate provided, then apply it here.
# (Note: This code runs only when "overlapped mode" is on to allow
# parallel execution of shared experts with the FusedMoE via
# parallel execution of shared experts with the RoutedExperts via
# separate cuda stream)
if self._shared_experts is not None:
assert shared_experts_input is not None
@@ -62,7 +62,7 @@ class MoERunnerInterface(PluggableLayer, ABC):
########################################################################
#
# FusedMoE layer methods
# FusedMoEFactory layer methods
#
########################################################################
@@ -215,7 +215,7 @@ class CompressedTensorsConfig(QuantizationConfig):
def _add_fused_moe_to_target_scheme_map(self): # XXXXXXXXXXXXXXXXXXXXXX
"""
Helper function to update target_scheme_map
since linear layers get fused into FusedMoE
since linear layers get fused into RoutedExperts
targeting 'Linear' needs to also match
RoutedExperts modules.
"""
@@ -2363,7 +2363,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
if key.startswith(prefix_dot):
return info["quant_algo"].upper()
# FusedMoE expert prefix is e.g. "...moe.experts", while ModelOpt's
# RoutedExperts expert prefix is e.g. "...moe.experts", while ModelOpt's
# quantized_layers entries use "...moe.gate_proj" / "...moe.up_proj".
if prefix.endswith(".experts"):
parent_dot = prefix.rsplit(".experts", 1)[0] + "."
@@ -467,7 +467,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
elif isinstance(module, RoutedExperts) and hasattr(
module.quant_method, "quant_config"
):
# TODO: support FusedMoE with prequant and 8bit.
# TODO: support RoutedExperts with prequant and 8bit.
if self.pre_quant and self.load_8bit:
raise ValueError(
"Prequant BitsAndBytes 8bit models with RoutedExperts "
+2 -2
View File
@@ -43,7 +43,7 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -168,7 +168,7 @@ class AXK1MoE(nn.Module):
prefix=f"{prefix}.shared_experts",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
gate=self.gate,
num_experts=config.n_routed_experts,
+3 -3
View File
@@ -18,7 +18,7 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
MoERunner,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -122,8 +122,8 @@ class AfmoeMoE(nn.Module):
prefix=f"{prefix}.shared_experts",
)
# Routed experts using FusedMoE
self.experts = FusedMoE(
# Routed experts using FusedMoEFactory
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.num_experts,
top_k=config.num_experts_per_tok,
+4 -2
View File
@@ -13,7 +13,9 @@ from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -222,7 +224,7 @@ class AriaTextMoELayer(nn.Module):
bias=config.mlp_bias,
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.moe_num_experts,
top_k=config.moe_topk,
+2 -2
View File
@@ -41,7 +41,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -289,7 +289,7 @@ class BailingMoE(nn.Module):
else:
self.shared_experts = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=self.num_experts,
top_k=self.top_k,
@@ -18,7 +18,7 @@ from vllm.distributed import (
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -329,8 +329,8 @@ class BailingMoeV25(nn.Module):
else:
self.shared_experts = None
# Routed experts using FusedMoE
self.experts = FusedMoE(
# Routed experts using FusedMoEFactory
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=self.num_experts,
top_k=self.top_k,
@@ -602,7 +602,7 @@ class BailingMoeV25Model(nn.Module):
(".gate_up_proj", ".up_proj", 1),
]
# Expert parameter mappings from FusedMoE
# Expert parameter mappings from FusedMoEFactory
expert_mappings = list(self.get_expert_mapping())
def load_param(name: str, tensor: torch.Tensor, shard_id=None) -> bool:
+1 -1
View File
@@ -320,7 +320,7 @@ class NomicMoE(nn.Module):
# router_logits: (num_tokens, n_experts)
router_logits, _ = self.router(hidden_states)
# FIXME(Isotr0py): This implementation is too tricky,
# we should use FusedMoE instead in the future
# we should use FusedMoEFactory instead in the future
# after supporting ungated activation for it.
topk_weights, topk_ids, _ = fused_topk(
hidden_states, router_logits, self.top_k, renormalize=False
+3 -3
View File
@@ -16,7 +16,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
@@ -300,7 +300,7 @@ class Cohere2Moe(nn.Module):
self.shared_experts = None
self.shared_expert_combination_strategy = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=config.num_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
@@ -318,7 +318,7 @@ class Cohere2Moe(nn.Module):
orig_shape = hidden_states.shape
hidden_states = hidden_states.view(-1, self.hidden_size)
router_logits, _ = self.gate(hidden_states)
# FusedMoE handles shared expert overlap internally and returns
# FusedMoEFactory handles shared expert overlap internally and returns
# shared_output + routed_output when shared_experts is set.
final_hidden_states = self.experts(hidden_states, router_logits)
if self.shared_expert_combination_strategy == "average":
+2 -2
View File
@@ -16,7 +16,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
RoutedExperts,
)
from vllm.model_executor.layers.linear import (
@@ -159,7 +159,7 @@ class DbrxMoE(nn.Module):
self.router = DbrxRouter(config, self.params_dtype)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=config.ffn_config.moe_num_experts,
top_k=config.ffn_config.moe_top_k,
hidden_size=config.d_model,
+2 -2
View File
@@ -50,7 +50,7 @@ from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention, RSWAAttention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
@@ -361,7 +361,7 @@ class DeepseekV2MoE(nn.Module):
prefix=f"{prefix}.shared_experts",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
gate=self.gate,
num_experts=config.n_routed_experts,
+2 -2
View File
@@ -41,7 +41,7 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -183,7 +183,7 @@ class Ernie4_5_MoeMoE(nn.Module):
else:
self.shared_experts = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.moe_num_experts,
top_k=config.moe_k,
+3 -3
View File
@@ -37,7 +37,7 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -260,7 +260,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
prefix=f"{prefix}.text_experts_gate",
)
self.text_experts = FusedMoE(
self.text_experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.moe_num_experts[0],
top_k=config.moe_k,
@@ -297,7 +297,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
prefix=f"{prefix}.vision_experts_gate",
)
self.vision_experts = FusedMoE(
self.vision_experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.moe_num_experts[1],
top_k=config.moe_k,
+2 -4
View File
@@ -29,9 +29,7 @@ from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -126,7 +124,7 @@ class ExaoneMoe(nn.Module):
else:
self.shared_experts = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
gate=self.gate,
num_experts=self.n_routed_experts,
+2 -2
View File
@@ -21,7 +21,7 @@ from vllm.config import VllmConfig
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
@@ -73,7 +73,7 @@ class FlexOlmoMoE(nn.Module):
prefix=f"{prefix}.gate",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=hf_config.num_experts,
top_k=hf_config.num_experts_per_tok,
hidden_size=hf_config.hidden_size,
+16 -16
View File
@@ -38,7 +38,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import get_act_and_mul_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
@@ -305,14 +305,14 @@ class Gemma4Router(nn.Module):
class Gemma4MoE(nn.Module):
"""Mixture of Experts for Gemma4 using vLLM's FusedMoE.
"""Mixture of Experts for Gemma4 using vLLM's MoERunner.
Wraps FusedMoE with custom routing. The router projection is
Wraps MoERunner with custom routing. The router projection is
external (Gemma4Router) — this class only handles expert dispatch.
Gemma4 routing: softmax over ALL experts → top-k → renormalize.
per_expert_scale is folded into routing weights for mathematical
correctness with FusedMoE's fused kernel.
correctness with MoERunner's fused kernel.
"""
def __init__(
@@ -326,11 +326,11 @@ class Gemma4MoE(nn.Module):
self.num_experts = config.num_experts
# Per-expert output scale folded into routing weights so that
# FusedMoE's fused kernel computes: Σ_e (expert_e * w_e * scale_e)
# MoERunner's fused kernel computes: Σ_e (expert_e * w_e * scale_e)
self.per_expert_scale = nn.Parameter(torch.ones(config.num_experts))
# Gemma4 routing: softmax over ALL experts → top-k → renormalize.
# FusedMoE's built-in fused_topk scopes softmax differently, so
# MoERunner's built-in fused_topk scopes softmax differently, so
# a custom routing function is needed for numerical correctness.
# NOTE: self.per_expert_scale is read at call time (not captured into
# a local) so that torch.func.functional_call parameter substitution
@@ -350,8 +350,8 @@ class Gemma4MoE(nn.Module):
gating_output, topk, self.per_expert_scale
)
# FusedMoE experts with custom Gemma4 routing
self.experts = FusedMoE(
# MoERunner experts with custom Gemma4 routing
self.experts = FusedMoEFactory(
num_experts=config.num_experts,
top_k=config.top_k_experts,
hidden_size=config.hidden_size,
@@ -1382,10 +1382,10 @@ class Gemma4Model(nn.Module, EagleModelMixin):
# MoE expert weight mapping: checkpoint can have either:
# 1. 3D packed tensors (exploded in _weight_iterator to per-expert 2D)
# 2. Already per-expert 2D weights (if quantized)
# Map to FusedMoE parameters:
# moe.experts.{id}.gate_proj → FusedMoE w1 (shard of w13)
# moe.experts.{id}.up_proj → FusedMoE w3 (shard of w13)
# moe.experts.{id}.down_proj → FusedMoE w2
# Map to MoERunner parameters:
# moe.experts.{id}.gate_proj → MoERunner w1 (shard of w13)
# moe.experts.{id}.up_proj → MoERunner w3 (shard of w13)
# moe.experts.{id}.down_proj → MoERunner w2
num_experts = getattr(self.config, "num_experts", None) or 0
# Strategy A: dot-separated suffix
# (standard AWQ/GPTQ e.g. .qweight, .scales, .weight)
@@ -1475,7 +1475,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
continue
param = params_dict[moe_name]
# Expert weights are already in the correct
# orientation for FusedMoE after _weight_iterator:
# orientation for MoERunner after _weight_iterator:
# gate/up: [I, H] → w1/w3 expects [I, H]
# down: [H, I] → w2 expects [H, I]
# Scales and other quantization params may be 1D or scalar.
@@ -1668,19 +1668,19 @@ class Gemma4ForCausalLM(
# MoE expert weights: checkpoint stores as 3D packed
# tensors. Explode into per-expert 2D weights for
# FusedMoE weight_loader.
# MoERunner weight_loader.
#
# Checkpoint format:
# moe.gate_up_proj: [E, 2*I, H] (fused gate + up)
# moe.down_proj: [E, H, I]
#
# FusedMoE expects per-expert:
# MoERunner expects per-expert:
# w1 (gate): [I, H] — first half of gate_up
# w3 (up): [I, H] — second half of gate_up
# w2 (down): [H, I] — as-is from checkpoint
#
# No transpose needed: checkpoint orientation already
# matches FusedMoE's expected layout.
# matches MoERunner's expected layout.
if "moe.gate_up_proj" in name and weight.dim() == 3:
num_experts = weight.size(0)
intermediate_size = weight.size(1) // 2
+5 -3
View File
@@ -42,7 +42,9 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -164,7 +166,7 @@ class Glm4MoE(nn.Module):
)
# AITER fused shared-expert (FSE) gate; mirrors the deepseek_v2.py
# pattern (see Glm4MoE / FusedMoE wiring there).
# pattern (see Glm4MoE / MoERunner wiring there).
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
self.is_fusion_moe_shared_experts_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
@@ -183,7 +185,7 @@ class Glm4MoE(nn.Module):
prefix=f"{prefix}.shared_experts",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.n_routed_experts,
top_k=config.num_experts_per_tok,
+3 -3
View File
@@ -21,7 +21,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
@@ -211,7 +211,7 @@ class MLPBlock(torch.nn.Module):
return_bias=False,
)
assert config.intermediate_size % self.world_size == 0
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=config.num_local_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
@@ -1010,7 +1010,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, intermediate_size)
# Use centralized weight remapping for MoE expert parameters.
# The FusedMoE refactor moved expert params under
# The MoERunner refactor moved expert params under
# `mlp.experts.routed_experts.*`; this remaps checkpoint names so
# MoE weight/bias keys resolve against params_dict.
for name, weight in remap_moe_expert_weights(weights, params_dict):
+2 -2
View File
@@ -40,7 +40,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -101,7 +101,7 @@ class GraniteMoeMoE(nn.Module):
prefix=f"{prefix}.gate",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
+2 -2
View File
@@ -43,7 +43,7 @@ from vllm.distributed import (
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -441,7 +441,7 @@ class HunYuanSparseMoeBlock(nn.Module):
else:
self.shared_mlp = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_mlp,
num_experts=self.n_routed_experts,
top_k=top_k,
+2 -2
View File
@@ -44,7 +44,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
@@ -184,7 +184,7 @@ class HYV3MoEFused(nn.Module):
scoring_func = "sigmoid"
e_score_correction_bias = self.expert_bias
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=self.n_routed_experts,
top_k=top_k,
hidden_size=config.hidden_size,
+2 -2
View File
@@ -42,7 +42,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -173,7 +173,7 @@ class InternS1ProMoeSparseMoeBlock(nn.Module):
# For custom routing function
self.n_groups = getattr(config, "router_n_groups", -1)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=self.n_routed_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
+4 -2
View File
@@ -14,7 +14,9 @@ from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -81,7 +83,7 @@ class JambaMoE(nn.Module):
prefix=f"{prefix}.router",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
self.num_total_experts,
self.top_k,
self.hidden_size,
+9 -7
View File
@@ -19,7 +19,7 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -174,7 +174,7 @@ class LagunaMoE(nn.Module):
prefix=f"{prefix}.gate",
)
# Shared expert (optional) - passed to FusedMoE for overlap optimization
# Shared expert (optional) - passed to FusedMoEFactory for overlap optimization
self.shared_expert: LagunaMLP | None
if config.shared_expert_intermediate_size > 0:
self.shared_expert = LagunaMLP(
@@ -191,20 +191,22 @@ class LagunaMoE(nn.Module):
# Auxiliary-loss-free load-balancing bias (arXiv:2408.15664). The
# checkpoint stores one [num_experts] tensor per MoE layer at
# `mlp.experts.e_score_correction_bias`; registering it as a Parameter
# on the FusedMoE lets the weight loader pick it up and the router
# on the MoERunner lets the weight loader pick it up and the router
# add it during top-k selection. The fused top-k bias router requires
# float32 regardless of model dtype.
e_score_correction_bias = torch.nn.Parameter(
torch.zeros(config.num_experts, dtype=torch.float32),
requires_grad=False,
)
# FusedMoE with SIGMOID routing. Passing `shared_experts=` lets the
# MoERunner with SIGMOID routing. Passing `shared_experts=` lets the
# layer overlap the shared-expert compute with the all2all dispatch.
# `apply_routed_scale_to_output=True` makes FusedMoE handle the
# `apply_routed_scale_to_output=True` makes MoERunner handle the
# FusedMoEFactory with SIGMOID routing. Passing `shared_experts=` lets the
# layer overlap the shared-expert compute with the all2all dispatch.
# `apply_routed_scale_to_output=True` makes FusedMoEFactory handle the
# routed_scaling_factor, shared+routed combine, and TP all-reduce
# internally, so forward() just returns the final hidden states.
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_expert,
num_experts=config.num_experts,
top_k=config.num_experts_per_tok,
+4 -2
View File
@@ -15,7 +15,9 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -143,7 +145,7 @@ class Lfm2MoeSparseMoeBlock(nn.Module):
else:
self.gate.e_score_correction_bias = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=self.n_routed_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
+2 -2
View File
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.attention import (
ChunkedLocalAttention,
)
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -131,7 +131,7 @@ class Llama4MoE(nn.Module):
self.n_physical_experts = self.n_local_experts + self.n_redundant_experts
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_expert,
num_experts=config.num_local_experts,
top_k=config.num_experts_per_tok,
+4 -4
View File
@@ -47,7 +47,7 @@ from vllm.distributed import get_pp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -297,7 +297,7 @@ class LongcatMoe(nn.Module):
)
assert config.zero_expert_type is not None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
zero_expert_type=config.zero_expert_type,
e_score_correction_bias=self.router.e_score_correction_bias,
num_experts=num_experts,
@@ -317,7 +317,7 @@ class LongcatMoe(nn.Module):
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
# Align to FusedMoE padded hidden size to avoid dim mismatch
# Align to MoERunner padded hidden size to avoid dim mismatch
padded_hidden = self.experts.moe_config.hidden_dim
if hidden_dim < padded_hidden:
hidden_states_padded = torch.nn.functional.pad(
@@ -333,7 +333,7 @@ class LongcatMoe(nn.Module):
hidden_states_padded.to(self.router_params_dtype)
)
# FusedMoE handles routing memoization and zero expert computation
# MoERunner handles routing memoization and zero expert computation
# internally. Pass full router_logits (including zero experts) so that
# zero experts can be properly identified in routing.
final_hidden_states = self.experts(
+2 -2
View File
@@ -24,7 +24,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -171,7 +171,7 @@ class MiMoV2MoE(nn.Module):
torch.empty(config.n_routed_experts, dtype=self.gate_dtype)
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=self.n_routed_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
+2 -4
View File
@@ -39,9 +39,7 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -95,7 +93,7 @@ class MiniMaxM2MoE(nn.Module):
else:
self.e_score_correction_bias = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=config.num_local_experts,
top_k=config.num_experts_per_tok,
scoring_func=config.scoring_func,
+2 -4
View File
@@ -39,9 +39,7 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -123,7 +121,7 @@ class MixtralMoE(nn.Module):
prefix=f"{prefix}.gate",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
+2 -2
View File
@@ -33,7 +33,7 @@ from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.layers.activation import ReLUSquaredActivation
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
GateLinear,
activation_without_mul,
fused_moe_make_expert_params_mapping,
@@ -209,7 +209,7 @@ class NemotronHMoE(nn.Module):
self.fc1_latent_proj = None
self.fc2_latent_proj = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.n_routed_experts,
top_k=config.num_experts_per_tok,
+2 -4
View File
@@ -32,9 +32,7 @@ from vllm.distributed import (
from vllm.distributed.utils import split_tensor_along_last_dim
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -94,7 +92,7 @@ class OlmoeMoE(nn.Module):
prefix=f"{prefix}.gate",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
+2 -2
View File
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.attention import (
Attention,
StaticSinkAttention,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -195,7 +195,7 @@ class OpenPanguMoE(nn.Module):
else:
self.shared_experts = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.n_routed_experts,
top_k=config.num_experts_per_tok,
+5 -3
View File
@@ -32,7 +32,9 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -325,7 +327,7 @@ class Param2MoEMoEBlock(nn.Module):
else:
self.shared_experts = None # type: ignore[assignment]
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=self.num_experts,
top_k=self.top_k,
@@ -342,7 +344,7 @@ class Param2MoEMoEBlock(nn.Module):
routed_scaling_factor=self.routed_scaling_factor,
)
def maybe_get_fused_moe(self) -> FusedMoE:
def maybe_get_fused_moe(self) -> FusedMoEFactory:
return self.experts
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+2 -4
View File
@@ -35,9 +35,7 @@ from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
@@ -273,7 +271,7 @@ class PhiMoE(nn.Module):
prefix=f"{prefix}.gate",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
+2 -4
View File
@@ -40,9 +40,7 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -165,7 +163,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
else:
self.shared_expert = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_expert,
num_experts=config.num_experts,
top_k=config.num_experts_per_tok,
+4 -4
View File
@@ -42,7 +42,7 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -201,7 +201,7 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
self.shared_expert_gate = None
self.shared_expert = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_expert,
gate=self.gate,
num_experts=self.n_routed_experts,
@@ -228,13 +228,13 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
hidden_states = sequence_parallel_chunk(hidden_states)
if self.experts.is_internal_router:
# In this case, the gate/router runs inside the FusedMoE class
# In this case, the gate/router runs inside the MoERunner class
final_hidden_states = self.experts(
hidden_states=hidden_states, router_logits=hidden_states
)
else:
# Actually this will be dead code, since we always pass gate into
# FusedMoE in the current implementation. But we keep this code
# MoERunner in the current implementation. But we keep this code
# here for clarity and future flexibility.
router_logits, _ = self.gate(hidden_states)
final_hidden_states = self.experts(
+3 -5
View File
@@ -20,9 +20,7 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate
from vllm.model_executor.layers.layernorm import (
GemmaRMSNorm as Qwen3NextRMSNorm,
@@ -175,7 +173,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
prefix=f"{prefix}.shared_expert",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_expert,
gate=self.gate,
num_experts=self.n_routed_experts,
@@ -208,7 +206,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
hidden_states = sequence_parallel_chunk(hidden_states)
if self.experts.is_internal_router:
# In this case, the gate/router runs inside the FusedMoE class
# In this case, the gate/router runs inside the MoERunner class
final_hidden_states = self.experts(
hidden_states=hidden_states, router_logits=hidden_states
)
+2 -2
View File
@@ -35,7 +35,7 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -334,7 +334,7 @@ class SarvamMLAMoE(nn.Module):
else:
self.shared_experts = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=self.num_experts,
top_k=self.top_k,
+2 -2
View File
@@ -19,7 +19,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -68,7 +68,7 @@ class FusedMoEBlock(nn.Module):
f"the number of experts {config.moe_num_experts}."
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=config.moe_num_experts,
top_k=config.moe_top_k,
hidden_size=config.hidden_size,
+3 -3
View File
@@ -24,7 +24,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
MoERunner,
fused_moe_make_expert_params_mapping,
)
@@ -376,7 +376,7 @@ class FusedMoEBlock(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.share_expert",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.share_expert,
gate=self.gate,
num_experts=config.moe_num_experts,
@@ -404,7 +404,7 @@ class FusedMoEBlock(nn.Module):
hidden_states=hidden_states, router_logits=hidden_states
)
else:
# TODO(bnell): this gate could be moved into the FusedMoE?
# TODO(bnell): this gate could be moved into the MoERunner?
router_logits, _ = self.gate(hidden_states)
final_hidden_states = self.experts(
hidden_states=hidden_states, router_logits=router_logits
+10 -6
View File
@@ -29,7 +29,11 @@ from vllm.distributed import get_dp_group, get_ep_group
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
MoERunner,
RoutedExperts,
)
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
@@ -53,7 +57,7 @@ class TransformersMoEState:
# --8<-- [start:transformers_fused_moe]
@PluggableLayer.register("transformers_fused_moe")
class TransformersMoERunner(MoERunner):
"""Custom FusedMoE for the Transformers modeling backend."""
"""Custom MoERunner for the Transformers modeling backend."""
# --8<-- [end:transformers_fused_moe]
def __init__(self, *args, moe_state: TransformersMoEState, **kwargs):
@@ -297,7 +301,7 @@ class MoEMixin(MixtureOfExperts):
)
if fuser is not None and not reaches_fp16_trick:
# MoE block forward is fully replaced.
# gate/router and shared expert (if any) runs in FusedMoE.
# gate/router and shared expert (if any) runs in MoERunner.
shared_experts = fuser.shared_experts(moe_block, prefix)
# Store shared experts for later down projection adjustment
if shared_experts is not None:
@@ -331,7 +335,7 @@ class MoEMixin(MixtureOfExperts):
if fuser.shared_name:
routed += " + shared experts"
logger.info_once(
"Fused: %s (%s) -> FusedMoE (internal routing)",
"Fused: %s (%s) -> MoERunner (internal routing)",
routed,
moe_block_cls,
)
@@ -373,10 +377,10 @@ class MoEMixin(MixtureOfExperts):
runner_args={"moe_state": moe_state},
)
logger.info_once(
"Fused: experts (%s) -> FusedMoE (external routing)",
"Fused: experts (%s) -> MoERunner (external routing)",
experts_cls,
)
fused_experts = FusedMoE(**kwargs)
fused_experts = FusedMoEFactory(**kwargs)
moe_block.experts = fused_experts
log_replacement(qual_name, experts, fused_experts)
# Update MixtureOfExperts mixin state
@@ -108,7 +108,7 @@ def _extract_data_from_fused_moe_module(
m_: torch.nn.Module,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]:
"""
Extract weights, weight scales and num_topk from FusedMoE module.
Extract weights, weight scales and num_topk from MoERunner module.
"""
assert isinstance(m_, MoERunner)
m = m_.routed_experts