[MoE Refactor] Rename FusedMoE to FusedMoEFactory (#44941)

Signed-off-by: Bill Nell <bnell@redhat.com>
Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
bnellnm
2026-07-31 11:28:18 +01:00
committed by GitHub
co-authored by Harry Mellor
parent c911120a6f
commit 6e311c6e20
80 changed files with 243 additions and 234 deletions
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Test that the interaction between EPLB and FusedMoE Layer is okay
# Test that the interaction between EPLB and FusedMoEFactory Layer is okay
from dataclasses import dataclass
@@ -16,7 +16,7 @@ from vllm.distributed.parallel_state import (
get_eplb_group,
get_tp_group,
)
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
from .eplb_utils import distributed_run, set_env_vars_and_device
@@ -69,8 +69,8 @@ def make_fused_moe_layer(
rank: int,
layer_idx: int,
test_config: TestConfig,
) -> FusedMoE:
fml = FusedMoE(
) -> MoERunner:
fml = FusedMoEFactory(
num_experts=test_config.num_experts,
top_k=test_config.num_topk,
hidden_size=test_config.hidden_size,
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Test that the interaction between EPLB and FusedMoE Layer is okay for DP w/ NVFP4
# Test that the interaction between EPLB and MoERunner Layer is okay for DP w/ NVFP4
from dataclasses import dataclass
@@ -19,7 +19,7 @@ from vllm.distributed.parallel_state import (
get_eplb_group,
)
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory, MoERunner
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
ModelOptNvFp4FusedMoE,
@@ -44,7 +44,7 @@ def make_fused_moe_layer(
rank: int,
layer_idx: int,
test_config: TestConfig,
) -> FusedMoE:
) -> MoERunner:
quant_config = None
device = torch.device(f"cuda:{rank}")
@@ -55,7 +55,7 @@ def make_fused_moe_layer(
exclude_modules=[],
)
fml = FusedMoE(
fml = FusedMoEFactory(
num_experts=test_config.num_experts,
top_k=test_config.num_topk,
hidden_size=test_config.hidden_size,
+1 -1
View File
@@ -409,7 +409,7 @@ def _deep_ep_v2_moe_cudagraph(
vllm_cfg.kernel_config = KernelConfig(moe_backend="flashinfer_trtllm")
with set_current_vllm_config(vllm_cfg):
# Initialize vLLM parallel state (needed by FusedMoE layer)
# Initialize vLLM parallel state (needed by MoERunner layer)
temp_file = tempfile.mktemp()
init_distributed_environment(
world_size=pgi.world_size,
+12 -7
View File
@@ -42,7 +42,11 @@ from vllm.distributed.device_communicators.all_reduce_utils import (
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
from vllm.distributed.eplb.rebalance_execute import rearrange_expert_weights_inplace
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, fused_experts
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
MoERunner,
fused_experts,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.router.router_factory import (
@@ -80,7 +84,7 @@ NUM_EXPERTS = [8, 64]
TOP_KS = [2, 6]
# dp_size, tp_size, use_ep
# Note: DP+TP is not yet supported in the FusedMoE layer.
# Note: DP+TP is not yet supported in the FusedMoEFactory layer.
PARALLEL_COMBOS = [
[1, 2, False],
[1, 4, False],
@@ -1023,7 +1027,7 @@ def make_fused_moe_layer(
kwargs["routed_input_transform"] = routed_input_transform
kwargs["routed_output_transform"] = routed_output_transform
layer = FusedMoE(
layer = FusedMoEFactory(
num_experts=global_num_experts,
top_k=top_k,
hidden_size=hidden_size,
@@ -1267,7 +1271,7 @@ def _test_body_eplb(
):
output_before = sp_wrapper(moe_layer)(hidden_states, router_logits)
# Create a fresh FusedMoE layer with enable_eplb=True
# Create a fresh MoERunner layer with enable_eplb=True
# Delete the original layer's registration so the constructor can
# re-use the same "from_forward_context" prefix
cc = vllm_config.compilation_config
@@ -1401,7 +1405,8 @@ def _run_one_config(
- When is_sequence_parallel=True (EP + sequence splitting):
* ep_size: Number of expert parallel ranks (equals dp_size * tp_size)
* tp_size: Number of ranks to split sequence across (becomes sp_size in FusedMoE)
* tp_size: Number of ranks to split sequence across (becomes sp_size in
MoERunner)
* Weights are chunked by ep_size (experts) but NOT by tp_size
* Input sequences are chunked by tp_size (via sp_wrapper)
"""
@@ -1476,8 +1481,8 @@ def _run_one_config(
torch.accelerator.empty_cache()
with set_current_vllm_config(vllm_config):
# Chunk weights for EP BEFORE creating FusedMoE
# FusedMoE uses EP-chunked weights and handles reductions internally
# Chunk weights for EP BEFORE creating MoERunner.
# MoERunner uses EP-chunked weights and handles reductions internally.
if ep_size > 1:
# Split experts across ranks (dimension 0 is the expert dimension)
# When EP is enabled, use EP group rank and ep_size for chunking
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for FusedMoE weight loading with padded hidden dimensions.
"""Tests for FusedMoEFactory weight loading with padded hidden dimensions.
When using DeepEP backends or NIXL EP with models like nemotron_h,
hidden_size may be rounded up (e.g., 2688 -> 3072) for backend requirements.
+4 -4
View File
@@ -6,7 +6,7 @@ Test script for the token-to-expert routing simulator.
This script demonstrates how to use the routing simulator to test
different routing strategies and analyze their performance, including
integration tests with FusedMoE layer.
integration tests with FusedMoEFactory layer.
"""
import tempfile
@@ -77,11 +77,11 @@ def test_basic_functionality(
def test_routing_strategy_integration(monkeypatch, device):
"""Test that the routing strategy environment variable works with
FusedMoE."""
FusedMoEFactory."""
pytest.importorskip("vllm.model_executor.layers.fused_moe.layer")
import vllm.envs as envs
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
# Test parameters
num_tokens = 32
@@ -111,7 +111,7 @@ def test_routing_strategy_integration(monkeypatch, device):
)
for strategy in strategies:
fused_moe = FusedMoE(
fused_moe = FusedMoEFactory(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
@@ -1,9 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for FusedMoE with routed_input_transform.
Tests for FusedMoEFactory with routed_input_transform.
Verifies that applying routed_input_transform inside FusedMoE
Verifies that applying routed_input_transform inside FusedMoEFactory
produces the same results as applying the transform manually outside.
"""
@@ -13,7 +13,7 @@ import torch.nn as nn
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_torch_equal_or_newer, set_random_seed
@@ -133,9 +133,9 @@ def test_routed_input_transform_inside_vs_outside(
workspace_init,
monkeypatch,
):
"""Compare FusedMoE with transform inside vs manually applying outside.
Method A (inside): FusedMoE with routed_input_transform
Method B (outside): Manually transform, then FusedMoE without transform
"""Compare FusedMoEFactory with transform inside vs manually applying outside.
Method A (inside): FusedMoEFactory with routed_input_transform
Method B (outside): Manually transform, then FusedMoEFactory without transform
"""
if current_platform.is_rocm():
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_rocm_aiter else "0")
@@ -157,8 +157,8 @@ def test_routed_input_transform_inside_vs_outside(
routed_transform = SimpleLinear(hidden_size, latent_size, dtype)
with set_current_vllm_config(vllm_config):
# Method A: FusedMoE WITH routed_input_transform
moe_with_transform = FusedMoE(
# Method A: FusedMoEFactory WITH routed_input_transform
moe_with_transform = FusedMoEFactory(
shared_experts=shared_experts,
routed_input_transform=routed_transform,
num_experts=num_experts,
@@ -173,9 +173,9 @@ def test_routed_input_transform_inside_vs_outside(
prefix="moe_with_transform",
)
# Method B: FusedMoE WITHOUT routed_input_transform
# Method B: FusedMoEFactory WITHOUT routed_input_transform
# Note: shared_experts=None because when transform is done outside,
moe_without_transform = FusedMoE(
moe_without_transform = FusedMoEFactory(
shared_experts=None,
routed_input_transform=None,
num_experts=num_experts,
+16 -13
View File
@@ -1,10 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for FusedMoE with zero experts.
"""Tests for FusedMoEFactory with zero experts.
Verifies that:
- The ZeroExpertRouter is properly created and used as the layer router.
- A forward pass through FusedMoE with zero experts produces correct output.
- A forward pass through FusedMoEFactory with zero experts produces correct output.
- The output decomposes correctly into real expert + zero expert contributions.
Note: tests generated with Claude.
@@ -15,7 +15,7 @@ import torch
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.router.zero_expert_router import (
ZeroExpertRouter,
)
@@ -24,7 +24,7 @@ from vllm.v1.worker.workspace import init_workspace_manager
@pytest.fixture
def zero_expert_moe(dist_init, default_vllm_config):
"""Create a FusedMoE layer with zero experts."""
"""Create a FusedMoEFactory layer with zero experts."""
num_experts = 4
top_k = 2
# hidden_size must be >= 256 for the zero expert identity kernel to
@@ -45,7 +45,7 @@ def zero_expert_moe(dist_init, default_vllm_config):
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
init_workspace_manager(torch.accelerator.current_device_index())
layer = FusedMoE(
layer = FusedMoEFactory(
zero_expert_type="identity",
e_score_correction_bias=e_score_correction_bias,
num_experts=num_experts,
@@ -66,7 +66,7 @@ def zero_expert_moe(dist_init, default_vllm_config):
@pytest.mark.parametrize("num_tokens", [1, 32])
def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_tokens):
"""Verify that FusedMoE with zero_expert_type creates a ZeroExpertRouter."""
"""Verify that FusedMoEFactory with zero_expert_type creates a ZeroExpertRouter."""
layer, _ = zero_expert_moe
assert isinstance(layer.router, ZeroExpertRouter), (
f"Expected ZeroExpertRouter but got {type(layer.router).__name__}."
@@ -83,7 +83,10 @@ def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_token
@pytest.mark.parametrize("num_tokens", [1, 32])
def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
"""Run a forward pass through FusedMoE with zero experts and verify output shape."""
"""
Run a forward pass through FusedMoEFactory with zero experts
and verify output shape.
"""
layer, vllm_config = zero_expert_moe
hidden_size = layer.routed_experts.hidden_size
@@ -118,16 +121,16 @@ def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
@pytest.mark.parametrize("num_tokens", [1, 32])
def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
"""Validate that the FusedMoE output equals a plain FusedMoE
"""Validate that the FusedMoEFactory output equals a plain FusedMoEFactory
output (real experts only) plus the zero expert contribution.
The key invariant is:
zero_layer.forward(h, r_full) == plain_layer.forward(h, r_real)
+ zero_expert_output
We create a plain FusedMoE layer with the same weights and real-expert-only
We create a plain FusedMoEFactory layer with the same weights and real-expert-only
router logits, compute the zero expert output via the ZeroExpertRouter, and
verify the sum matches the FusedMoE output.
verify the sum matches the FusedMoEFactory output.
"""
layer, vllm_config = zero_expert_moe
num_experts = 4
@@ -152,9 +155,9 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
get_forward_context().all_moe_layers = None
# Create a plain FusedMoE layer with the same config but no zero
# Create a plain FusedMoEFactory layer with the same config but no zero
# experts. Use a separate prefix to avoid collision.
plain_layer = FusedMoE(
plain_layer = FusedMoEFactory(
num_experts=num_experts,
top_k=layer.routed_experts.top_k,
hidden_size=layer.routed_experts.hidden_size,
@@ -210,7 +213,7 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
expected,
atol=4e-3,
rtol=4e-3,
msg="FusedMoE output should equal plain FusedMoE output "
msg="FusedMoEFactory output should equal plain FusedMoEFactory output "
"plus zero expert contribution",
)
+1 -1
View File
@@ -198,7 +198,7 @@ def test_dedup_shared_module_across_paths(default_vllm_config, dist_init, dummy_
self.gate = gate # canonical path: "moe.gate"
# Inner submodule holding the SAME gate instance under another
# path. This mirrors how FusedMoE.runner.gate references the
# path. This mirrors how MoERunner.runner.gate references the
# block's gate in qwen3_moe.
class _Runner(nn.Module):
def __init__(self, g):
+2 -2
View File
@@ -21,7 +21,7 @@ from vllm.model_executor.layers.attention.attention import (
Attention,
set_default_quant_scales,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.quantization.fp8 import (
Fp8Config,
Fp8KVCacheMethod,
@@ -393,7 +393,7 @@ def test_fp8_reloading(
method.use_marlin = use_marlin
else:
layer = FusedMoE(
layer = FusedMoEFactory(
num_experts=1,
top_k=1,
hidden_size=1,
+1 -1
View File
@@ -1022,7 +1022,7 @@ def cutlass_fp4_moe_mm(
An FP4 Blockscaled Group Gemm that takes in a_tensors, b_tensors and runs
the gemms for each combination based on the specified problem sizes.
This is used as the MoE gemm during NVFP4 Quantized FusedMoE forward.
This is used as the MoE gemm during NVFP4 Quantized MoERunner forward.
- a/b_tensors: the NVFP4 a_ptrs and b_ptrs tensors which are quantized
input and expert weights.
- a_/b_scales: The blockscales in FP8-E4M3 precision
+1 -1
View File
@@ -88,7 +88,7 @@ class QuantizationConfigArgs:
"""Spec applied to ``LinearBase`` layers."""
moe: QuantSpec | None = None
"""Spec applied to ``FusedMoE`` layers."""
"""Spec applied to ``FusedMoEFactory`` layers."""
ignore: list[str] = Field(default_factory=list)
"""Layers to skip quantization for."""
@@ -170,7 +170,7 @@ class ExampleConnector(KVConnectorBase_V1):
# Only process layers that have kv_cache
# attribute (attention layers) Skip non-attention
# layers like FusedMoE/MLP etc.
# layers like FusedMoEFactory/MLP etc.
kv_cache_layer = getattr(layer, "kv_cache", None)
if kv_cache_layer is None:
continue
+1 -1
View File
@@ -44,7 +44,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
"Monolithic kernels are not supported for Fused MoE LoRA."
)
# Use the MoE-aware TP rank/size: when EP is active, FusedMoE collapses
# Use the MoE-aware TP rank/size: when EP is active, MoERunner collapses
# moe_parallel_config.tp_size to 1 (experts are sharded across the
# TP group instead).
moe_parallel_config = self.moe_config.moe_parallel_config
+2 -2
View File
@@ -24,7 +24,7 @@ logger = init_logger(__name__)
@dataclass(frozen=True)
class MoEEPLoadSpec:
"""Per-expert-parallel slicing metadata for one FusedMoE LoRA module.
"""Per-expert-parallel slicing metadata for one FusedMoEFactory LoRA module.
Threaded into the LoRA loader so per-expert weights from EP ranks
other than this one can be skipped before they ever hit CPU memory.
@@ -193,7 +193,7 @@ class LoRAModel:
skip_prefixes: List of module name prefixes to skip during loading.
Models can define this to skip modules not used in inference
(e.g., MTP layers). Format: ["mtp."]
moe_ep_spec: When 2D FusedMoE LoRA modules are present with
moe_ep_spec: When 2D FusedMoEFactory LoRA modules are present with
expert parallelism enabled, the (ep_rank, local, global)
slicing metadata shared across all MoE layers. Non-local
expert weights are skipped at read time instead of being
+2 -2
View File
@@ -739,7 +739,7 @@ class LoRAModelManager:
def _create_merged_loras_inplace(self, lora_model: LoRAModel) -> None:
for module_name, new_module_names in self.packed_modules.items():
# For 2D FusedMoE modules with EP, narrow the per-expert
# For 2D RoutedExperts modules with EP, narrow the per-expert
# sub-module list to this rank's owned experts so pack_moe
# produces a tensor sized to local_num_experts directly.
packed_module_names = new_module_names
@@ -1096,7 +1096,7 @@ class LoRAModelManager:
def _build_moe_ep_load_spec(self) -> MoEEPLoadSpec | None:
"""
Per-rank slicing metadata for 2D FusedMoE LoRA modules.
Per-rank slicing metadata for 2D RoutedEXperts LoRA modules.
"""
if not self._use_ep or not self._is_moe:
return None
@@ -19,7 +19,7 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.layer import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import (
@@ -63,7 +63,7 @@ def get_config() -> dict[str, Any] | None:
__all__ = [
"FusedMoE",
"FusedMoEFactory",
"FusedMoERouter",
"FusedMoEConfig",
"FusedMoEQuantConfig",
@@ -1134,9 +1134,9 @@ class FusedMoEParallelConfig:
level's of parallelism to use in the fused moe layer.
Args:
tp_size_ (int): `tp_size` passed into the FusedMoE constructor.
pcp_size_ (int): `pcp_size` passed into the FusedMoE constructor.
dp_size_ (int): `dp_size` passed into the FusedMoE constructor.
tp_size_ (int): `tp_size` passed into the FusedMoEFactory constructor.
pcp_size_ (int): `pcp_size` passed into the FusedMoEFactory constructor.
dp_size_ (int): `dp_size` passed into the FusedMoEFactory constructor.
vllm_parallel_config (ParallelConfig): vLLM's parallel config
object which contains the `enable_expert_parallel` flag.
@@ -96,8 +96,7 @@ def determine_expert_counts(
return global_num_experts, logical_num_experts, num_fused_shared_experts
# TODO: rename this
def FusedMoE(
def FusedMoEFactory(
num_experts: int, # Global number of experts
top_k: int,
hidden_size: int,
@@ -1016,7 +1016,7 @@ def _process_weights_xpu(
w2: [E, K, N] int4 (uint8 storage [E, K, N // 2])
w2_scales: [E, K, N // group_size] params_dtype
Input GPTQ layout from FusedMoE.weight_loader:
Input GPTQ layout from MoERunner.weight_loader:
w13: [E, K // 8, 2*N] int32 (8 nibbles per int32 along the input dim)
w13_scales: [E, K // group_size, 2*N] params_dtype
w2: [E, N // 8, K] int32
@@ -1419,7 +1419,7 @@ def convert_to_wna16_moe_kernel_format(
Args:
backend: the selected ``WNA16MoEBackend``.
layer: the ``FusedMoE`` layer whose parameters are being prepared.
layer: the ``MoERunner`` layer whose parameters are being prepared.
quant_config: the ``QuantizationConfig`` for this layer.
input_dtype: optional activation dtype, usually should be 16 bit.
"""
@@ -180,7 +180,7 @@ class RoutedExpertsCapturer:
# (trailing rows are SP ceil-div padding). The TP group
# is always initialized on real rollout workers, and
# every rank in the group reaches this branch in
# lockstep (bind is per-FusedMoE layer, SP is a global
# lockstep (bind is per-FusedMoEFactory layer, SP is a global
# condition), so a bare all_gather here will not
# deadlock -- let it raise if the precondition is
# violated rather than skip silently.
@@ -353,7 +353,7 @@ class MoERunner(MoERunnerInterface):
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Apply transform for routed experts (e.g., latent projection).
This is called by FusedMoE.forward_native. The original hidden_states
This is called by MoERunner.forward_native. The original hidden_states
is saved separately so shared experts get [S, hidden_size] while
routed experts get the transformed [S, moe_latent_size].
@@ -646,7 +646,7 @@ class MoERunner(MoERunnerInterface):
):
# If router/gate provided, then apply it here.
# (Note: This code runs only when "overlapped mode" is on to allow
# parallel execution of shared experts with the FusedMoE via
# parallel execution of shared experts with the RoutedExperts via
# separate cuda stream)
if self._shared_experts is not None:
assert shared_experts_input is not None
@@ -62,7 +62,7 @@ class MoERunnerInterface(PluggableLayer, ABC):
########################################################################
#
# FusedMoE layer methods
# FusedMoEFactory layer methods
#
########################################################################
@@ -215,7 +215,7 @@ class CompressedTensorsConfig(QuantizationConfig):
def _add_fused_moe_to_target_scheme_map(self): # XXXXXXXXXXXXXXXXXXXXXX
"""
Helper function to update target_scheme_map
since linear layers get fused into FusedMoE
since linear layers get fused into RoutedExperts
targeting 'Linear' needs to also match
RoutedExperts modules.
"""
@@ -2363,7 +2363,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
if key.startswith(prefix_dot):
return info["quant_algo"].upper()
# FusedMoE expert prefix is e.g. "...moe.experts", while ModelOpt's
# RoutedExperts expert prefix is e.g. "...moe.experts", while ModelOpt's
# quantized_layers entries use "...moe.gate_proj" / "...moe.up_proj".
if prefix.endswith(".experts"):
parent_dot = prefix.rsplit(".experts", 1)[0] + "."
@@ -467,7 +467,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
elif isinstance(module, RoutedExperts) and hasattr(
module.quant_method, "quant_config"
):
# TODO: support FusedMoE with prequant and 8bit.
# TODO: support RoutedExperts with prequant and 8bit.
if self.pre_quant and self.load_8bit:
raise ValueError(
"Prequant BitsAndBytes 8bit models with RoutedExperts "
+2 -2
View File
@@ -43,7 +43,7 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -168,7 +168,7 @@ class AXK1MoE(nn.Module):
prefix=f"{prefix}.shared_experts",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
gate=self.gate,
num_experts=config.n_routed_experts,
+3 -3
View File
@@ -18,7 +18,7 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
MoERunner,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -122,8 +122,8 @@ class AfmoeMoE(nn.Module):
prefix=f"{prefix}.shared_experts",
)
# Routed experts using FusedMoE
self.experts = FusedMoE(
# Routed experts using FusedMoEFactory
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.num_experts,
top_k=config.num_experts_per_tok,
+4 -2
View File
@@ -13,7 +13,9 @@ from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -222,7 +224,7 @@ class AriaTextMoELayer(nn.Module):
bias=config.mlp_bias,
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.moe_num_experts,
top_k=config.moe_topk,
+2 -2
View File
@@ -41,7 +41,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -289,7 +289,7 @@ class BailingMoE(nn.Module):
else:
self.shared_experts = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=self.num_experts,
top_k=self.top_k,
@@ -18,7 +18,7 @@ from vllm.distributed import (
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -329,8 +329,8 @@ class BailingMoeV25(nn.Module):
else:
self.shared_experts = None
# Routed experts using FusedMoE
self.experts = FusedMoE(
# Routed experts using FusedMoEFactory
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=self.num_experts,
top_k=self.top_k,
@@ -602,7 +602,7 @@ class BailingMoeV25Model(nn.Module):
(".gate_up_proj", ".up_proj", 1),
]
# Expert parameter mappings from FusedMoE
# Expert parameter mappings from FusedMoEFactory
expert_mappings = list(self.get_expert_mapping())
def load_param(name: str, tensor: torch.Tensor, shard_id=None) -> bool:
+1 -1
View File
@@ -320,7 +320,7 @@ class NomicMoE(nn.Module):
# router_logits: (num_tokens, n_experts)
router_logits, _ = self.router(hidden_states)
# FIXME(Isotr0py): This implementation is too tricky,
# we should use FusedMoE instead in the future
# we should use FusedMoEFactory instead in the future
# after supporting ungated activation for it.
topk_weights, topk_ids, _ = fused_topk(
hidden_states, router_logits, self.top_k, renormalize=False
+3 -3
View File
@@ -16,7 +16,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
@@ -300,7 +300,7 @@ class Cohere2Moe(nn.Module):
self.shared_experts = None
self.shared_expert_combination_strategy = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=config.num_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
@@ -318,7 +318,7 @@ class Cohere2Moe(nn.Module):
orig_shape = hidden_states.shape
hidden_states = hidden_states.view(-1, self.hidden_size)
router_logits, _ = self.gate(hidden_states)
# FusedMoE handles shared expert overlap internally and returns
# FusedMoEFactory handles shared expert overlap internally and returns
# shared_output + routed_output when shared_experts is set.
final_hidden_states = self.experts(hidden_states, router_logits)
if self.shared_expert_combination_strategy == "average":
+2 -2
View File
@@ -16,7 +16,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
RoutedExperts,
)
from vllm.model_executor.layers.linear import (
@@ -159,7 +159,7 @@ class DbrxMoE(nn.Module):
self.router = DbrxRouter(config, self.params_dtype)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=config.ffn_config.moe_num_experts,
top_k=config.ffn_config.moe_top_k,
hidden_size=config.d_model,
+2 -2
View File
@@ -50,7 +50,7 @@ from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention, RSWAAttention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
@@ -361,7 +361,7 @@ class DeepseekV2MoE(nn.Module):
prefix=f"{prefix}.shared_experts",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
gate=self.gate,
num_experts=config.n_routed_experts,
+2 -2
View File
@@ -41,7 +41,7 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -183,7 +183,7 @@ class Ernie4_5_MoeMoE(nn.Module):
else:
self.shared_experts = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.moe_num_experts,
top_k=config.moe_k,
+3 -3
View File
@@ -37,7 +37,7 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -260,7 +260,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
prefix=f"{prefix}.text_experts_gate",
)
self.text_experts = FusedMoE(
self.text_experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.moe_num_experts[0],
top_k=config.moe_k,
@@ -297,7 +297,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
prefix=f"{prefix}.vision_experts_gate",
)
self.vision_experts = FusedMoE(
self.vision_experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.moe_num_experts[1],
top_k=config.moe_k,
+2 -4
View File
@@ -29,9 +29,7 @@ from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -126,7 +124,7 @@ class ExaoneMoe(nn.Module):
else:
self.shared_experts = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
gate=self.gate,
num_experts=self.n_routed_experts,
+2 -2
View File
@@ -21,7 +21,7 @@ from vllm.config import VllmConfig
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
@@ -73,7 +73,7 @@ class FlexOlmoMoE(nn.Module):
prefix=f"{prefix}.gate",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=hf_config.num_experts,
top_k=hf_config.num_experts_per_tok,
hidden_size=hf_config.hidden_size,
+16 -16
View File
@@ -38,7 +38,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import get_act_and_mul_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
@@ -305,14 +305,14 @@ class Gemma4Router(nn.Module):
class Gemma4MoE(nn.Module):
"""Mixture of Experts for Gemma4 using vLLM's FusedMoE.
"""Mixture of Experts for Gemma4 using vLLM's MoERunner.
Wraps FusedMoE with custom routing. The router projection is
Wraps MoERunner with custom routing. The router projection is
external (Gemma4Router) — this class only handles expert dispatch.
Gemma4 routing: softmax over ALL experts → top-k → renormalize.
per_expert_scale is folded into routing weights for mathematical
correctness with FusedMoE's fused kernel.
correctness with MoERunner's fused kernel.
"""
def __init__(
@@ -326,11 +326,11 @@ class Gemma4MoE(nn.Module):
self.num_experts = config.num_experts
# Per-expert output scale folded into routing weights so that
# FusedMoE's fused kernel computes: Σ_e (expert_e * w_e * scale_e)
# MoERunner's fused kernel computes: Σ_e (expert_e * w_e * scale_e)
self.per_expert_scale = nn.Parameter(torch.ones(config.num_experts))
# Gemma4 routing: softmax over ALL experts → top-k → renormalize.
# FusedMoE's built-in fused_topk scopes softmax differently, so
# MoERunner's built-in fused_topk scopes softmax differently, so
# a custom routing function is needed for numerical correctness.
# NOTE: self.per_expert_scale is read at call time (not captured into
# a local) so that torch.func.functional_call parameter substitution
@@ -350,8 +350,8 @@ class Gemma4MoE(nn.Module):
gating_output, topk, self.per_expert_scale
)
# FusedMoE experts with custom Gemma4 routing
self.experts = FusedMoE(
# MoERunner experts with custom Gemma4 routing
self.experts = FusedMoEFactory(
num_experts=config.num_experts,
top_k=config.top_k_experts,
hidden_size=config.hidden_size,
@@ -1382,10 +1382,10 @@ class Gemma4Model(nn.Module, EagleModelMixin):
# MoE expert weight mapping: checkpoint can have either:
# 1. 3D packed tensors (exploded in _weight_iterator to per-expert 2D)
# 2. Already per-expert 2D weights (if quantized)
# Map to FusedMoE parameters:
# moe.experts.{id}.gate_proj → FusedMoE w1 (shard of w13)
# moe.experts.{id}.up_proj → FusedMoE w3 (shard of w13)
# moe.experts.{id}.down_proj → FusedMoE w2
# Map to MoERunner parameters:
# moe.experts.{id}.gate_proj → MoERunner w1 (shard of w13)
# moe.experts.{id}.up_proj → MoERunner w3 (shard of w13)
# moe.experts.{id}.down_proj → MoERunner w2
num_experts = getattr(self.config, "num_experts", None) or 0
# Strategy A: dot-separated suffix
# (standard AWQ/GPTQ e.g. .qweight, .scales, .weight)
@@ -1475,7 +1475,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
continue
param = params_dict[moe_name]
# Expert weights are already in the correct
# orientation for FusedMoE after _weight_iterator:
# orientation for MoERunner after _weight_iterator:
# gate/up: [I, H] → w1/w3 expects [I, H]
# down: [H, I] → w2 expects [H, I]
# Scales and other quantization params may be 1D or scalar.
@@ -1668,19 +1668,19 @@ class Gemma4ForCausalLM(
# MoE expert weights: checkpoint stores as 3D packed
# tensors. Explode into per-expert 2D weights for
# FusedMoE weight_loader.
# MoERunner weight_loader.
#
# Checkpoint format:
# moe.gate_up_proj: [E, 2*I, H] (fused gate + up)
# moe.down_proj: [E, H, I]
#
# FusedMoE expects per-expert:
# MoERunner expects per-expert:
# w1 (gate): [I, H] — first half of gate_up
# w3 (up): [I, H] — second half of gate_up
# w2 (down): [H, I] — as-is from checkpoint
#
# No transpose needed: checkpoint orientation already
# matches FusedMoE's expected layout.
# matches MoERunner's expected layout.
if "moe.gate_up_proj" in name and weight.dim() == 3:
num_experts = weight.size(0)
intermediate_size = weight.size(1) // 2
+5 -3
View File
@@ -42,7 +42,9 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -164,7 +166,7 @@ class Glm4MoE(nn.Module):
)
# AITER fused shared-expert (FSE) gate; mirrors the deepseek_v2.py
# pattern (see Glm4MoE / FusedMoE wiring there).
# pattern (see Glm4MoE / MoERunner wiring there).
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
self.is_fusion_moe_shared_experts_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
@@ -183,7 +185,7 @@ class Glm4MoE(nn.Module):
prefix=f"{prefix}.shared_experts",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.n_routed_experts,
top_k=config.num_experts_per_tok,
+3 -3
View File
@@ -21,7 +21,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
@@ -211,7 +211,7 @@ class MLPBlock(torch.nn.Module):
return_bias=False,
)
assert config.intermediate_size % self.world_size == 0
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=config.num_local_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
@@ -1010,7 +1010,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, intermediate_size)
# Use centralized weight remapping for MoE expert parameters.
# The FusedMoE refactor moved expert params under
# The MoERunner refactor moved expert params under
# `mlp.experts.routed_experts.*`; this remaps checkpoint names so
# MoE weight/bias keys resolve against params_dict.
for name, weight in remap_moe_expert_weights(weights, params_dict):
+2 -2
View File
@@ -40,7 +40,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -101,7 +101,7 @@ class GraniteMoeMoE(nn.Module):
prefix=f"{prefix}.gate",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
+2 -2
View File
@@ -43,7 +43,7 @@ from vllm.distributed import (
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -441,7 +441,7 @@ class HunYuanSparseMoeBlock(nn.Module):
else:
self.shared_mlp = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_mlp,
num_experts=self.n_routed_experts,
top_k=top_k,
+2 -2
View File
@@ -44,7 +44,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
@@ -184,7 +184,7 @@ class HYV3MoEFused(nn.Module):
scoring_func = "sigmoid"
e_score_correction_bias = self.expert_bias
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=self.n_routed_experts,
top_k=top_k,
hidden_size=config.hidden_size,
+2 -2
View File
@@ -42,7 +42,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -173,7 +173,7 @@ class InternS1ProMoeSparseMoeBlock(nn.Module):
# For custom routing function
self.n_groups = getattr(config, "router_n_groups", -1)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=self.n_routed_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
+4 -2
View File
@@ -14,7 +14,9 @@ from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -81,7 +83,7 @@ class JambaMoE(nn.Module):
prefix=f"{prefix}.router",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
self.num_total_experts,
self.top_k,
self.hidden_size,
+9 -7
View File
@@ -19,7 +19,7 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -174,7 +174,7 @@ class LagunaMoE(nn.Module):
prefix=f"{prefix}.gate",
)
# Shared expert (optional) - passed to FusedMoE for overlap optimization
# Shared expert (optional) - passed to FusedMoEFactory for overlap optimization
self.shared_expert: LagunaMLP | None
if config.shared_expert_intermediate_size > 0:
self.shared_expert = LagunaMLP(
@@ -191,20 +191,22 @@ class LagunaMoE(nn.Module):
# Auxiliary-loss-free load-balancing bias (arXiv:2408.15664). The
# checkpoint stores one [num_experts] tensor per MoE layer at
# `mlp.experts.e_score_correction_bias`; registering it as a Parameter
# on the FusedMoE lets the weight loader pick it up and the router
# on the MoERunner lets the weight loader pick it up and the router
# add it during top-k selection. The fused top-k bias router requires
# float32 regardless of model dtype.
e_score_correction_bias = torch.nn.Parameter(
torch.zeros(config.num_experts, dtype=torch.float32),
requires_grad=False,
)
# FusedMoE with SIGMOID routing. Passing `shared_experts=` lets the
# MoERunner with SIGMOID routing. Passing `shared_experts=` lets the
# layer overlap the shared-expert compute with the all2all dispatch.
# `apply_routed_scale_to_output=True` makes FusedMoE handle the
# `apply_routed_scale_to_output=True` makes MoERunner handle the
# FusedMoEFactory with SIGMOID routing. Passing `shared_experts=` lets the
# layer overlap the shared-expert compute with the all2all dispatch.
# `apply_routed_scale_to_output=True` makes FusedMoEFactory handle the
# routed_scaling_factor, shared+routed combine, and TP all-reduce
# internally, so forward() just returns the final hidden states.
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_expert,
num_experts=config.num_experts,
top_k=config.num_experts_per_tok,
+4 -2
View File
@@ -15,7 +15,9 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -143,7 +145,7 @@ class Lfm2MoeSparseMoeBlock(nn.Module):
else:
self.gate.e_score_correction_bias = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=self.n_routed_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
+2 -2
View File
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.attention import (
ChunkedLocalAttention,
)
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -131,7 +131,7 @@ class Llama4MoE(nn.Module):
self.n_physical_experts = self.n_local_experts + self.n_redundant_experts
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_expert,
num_experts=config.num_local_experts,
top_k=config.num_experts_per_tok,
+4 -4
View File
@@ -47,7 +47,7 @@ from vllm.distributed import get_pp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -297,7 +297,7 @@ class LongcatMoe(nn.Module):
)
assert config.zero_expert_type is not None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
zero_expert_type=config.zero_expert_type,
e_score_correction_bias=self.router.e_score_correction_bias,
num_experts=num_experts,
@@ -317,7 +317,7 @@ class LongcatMoe(nn.Module):
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
# Align to FusedMoE padded hidden size to avoid dim mismatch
# Align to MoERunner padded hidden size to avoid dim mismatch
padded_hidden = self.experts.moe_config.hidden_dim
if hidden_dim < padded_hidden:
hidden_states_padded = torch.nn.functional.pad(
@@ -333,7 +333,7 @@ class LongcatMoe(nn.Module):
hidden_states_padded.to(self.router_params_dtype)
)
# FusedMoE handles routing memoization and zero expert computation
# MoERunner handles routing memoization and zero expert computation
# internally. Pass full router_logits (including zero experts) so that
# zero experts can be properly identified in routing.
final_hidden_states = self.experts(
+2 -2
View File
@@ -24,7 +24,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -171,7 +171,7 @@ class MiMoV2MoE(nn.Module):
torch.empty(config.n_routed_experts, dtype=self.gate_dtype)
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=self.n_routed_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
+2 -4
View File
@@ -39,9 +39,7 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -95,7 +93,7 @@ class MiniMaxM2MoE(nn.Module):
else:
self.e_score_correction_bias = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=config.num_local_experts,
top_k=config.num_experts_per_tok,
scoring_func=config.scoring_func,
+2 -4
View File
@@ -39,9 +39,7 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -123,7 +121,7 @@ class MixtralMoE(nn.Module):
prefix=f"{prefix}.gate",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
+2 -2
View File
@@ -33,7 +33,7 @@ from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.layers.activation import ReLUSquaredActivation
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
GateLinear,
activation_without_mul,
fused_moe_make_expert_params_mapping,
@@ -209,7 +209,7 @@ class NemotronHMoE(nn.Module):
self.fc1_latent_proj = None
self.fc2_latent_proj = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.n_routed_experts,
top_k=config.num_experts_per_tok,
+2 -4
View File
@@ -32,9 +32,7 @@ from vllm.distributed import (
from vllm.distributed.utils import split_tensor_along_last_dim
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -94,7 +92,7 @@ class OlmoeMoE(nn.Module):
prefix=f"{prefix}.gate",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
+2 -2
View File
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.attention import (
Attention,
StaticSinkAttention,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -195,7 +195,7 @@ class OpenPanguMoE(nn.Module):
else:
self.shared_experts = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=config.n_routed_experts,
top_k=config.num_experts_per_tok,
+5 -3
View File
@@ -32,7 +32,9 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -325,7 +327,7 @@ class Param2MoEMoEBlock(nn.Module):
else:
self.shared_experts = None # type: ignore[assignment]
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=self.num_experts,
top_k=self.top_k,
@@ -342,7 +344,7 @@ class Param2MoEMoEBlock(nn.Module):
routed_scaling_factor=self.routed_scaling_factor,
)
def maybe_get_fused_moe(self) -> FusedMoE:
def maybe_get_fused_moe(self) -> FusedMoEFactory:
return self.experts
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+2 -4
View File
@@ -35,9 +35,7 @@ from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
@@ -273,7 +271,7 @@ class PhiMoE(nn.Module):
prefix=f"{prefix}.gate",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
+2 -4
View File
@@ -40,9 +40,7 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -165,7 +163,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
else:
self.shared_expert = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_expert,
num_experts=config.num_experts,
top_k=config.num_experts_per_tok,
+4 -4
View File
@@ -42,7 +42,7 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -201,7 +201,7 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
self.shared_expert_gate = None
self.shared_expert = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_expert,
gate=self.gate,
num_experts=self.n_routed_experts,
@@ -228,13 +228,13 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
hidden_states = sequence_parallel_chunk(hidden_states)
if self.experts.is_internal_router:
# In this case, the gate/router runs inside the FusedMoE class
# In this case, the gate/router runs inside the MoERunner class
final_hidden_states = self.experts(
hidden_states=hidden_states, router_logits=hidden_states
)
else:
# Actually this will be dead code, since we always pass gate into
# FusedMoE in the current implementation. But we keep this code
# MoERunner in the current implementation. But we keep this code
# here for clarity and future flexibility.
router_logits, _ = self.gate(hidden_states)
final_hidden_states = self.experts(
+3 -5
View File
@@ -20,9 +20,7 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate
from vllm.model_executor.layers.layernorm import (
GemmaRMSNorm as Qwen3NextRMSNorm,
@@ -175,7 +173,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
prefix=f"{prefix}.shared_expert",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_expert,
gate=self.gate,
num_experts=self.n_routed_experts,
@@ -208,7 +206,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
hidden_states = sequence_parallel_chunk(hidden_states)
if self.experts.is_internal_router:
# In this case, the gate/router runs inside the FusedMoE class
# In this case, the gate/router runs inside the MoERunner class
final_hidden_states = self.experts(
hidden_states=hidden_states, router_logits=hidden_states
)
+2 -2
View File
@@ -35,7 +35,7 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -334,7 +334,7 @@ class SarvamMLAMoE(nn.Module):
else:
self.shared_experts = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=self.num_experts,
top_k=self.top_k,
+2 -2
View File
@@ -19,7 +19,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -68,7 +68,7 @@ class FusedMoEBlock(nn.Module):
f"the number of experts {config.moe_num_experts}."
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=config.moe_num_experts,
top_k=config.moe_top_k,
hidden_size=config.hidden_size,
+3 -3
View File
@@ -24,7 +24,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
MoERunner,
fused_moe_make_expert_params_mapping,
)
@@ -376,7 +376,7 @@ class FusedMoEBlock(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.share_expert",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.share_expert,
gate=self.gate,
num_experts=config.moe_num_experts,
@@ -404,7 +404,7 @@ class FusedMoEBlock(nn.Module):
hidden_states=hidden_states, router_logits=hidden_states
)
else:
# TODO(bnell): this gate could be moved into the FusedMoE?
# TODO(bnell): this gate could be moved into the MoERunner?
router_logits, _ = self.gate(hidden_states)
final_hidden_states = self.experts(
hidden_states=hidden_states, router_logits=router_logits
+10 -6
View File
@@ -29,7 +29,11 @@ from vllm.distributed import get_dp_group, get_ep_group
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
MoERunner,
RoutedExperts,
)
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
@@ -53,7 +57,7 @@ class TransformersMoEState:
# --8<-- [start:transformers_fused_moe]
@PluggableLayer.register("transformers_fused_moe")
class TransformersMoERunner(MoERunner):
"""Custom FusedMoE for the Transformers modeling backend."""
"""Custom MoERunner for the Transformers modeling backend."""
# --8<-- [end:transformers_fused_moe]
def __init__(self, *args, moe_state: TransformersMoEState, **kwargs):
@@ -297,7 +301,7 @@ class MoEMixin(MixtureOfExperts):
)
if fuser is not None and not reaches_fp16_trick:
# MoE block forward is fully replaced.
# gate/router and shared expert (if any) runs in FusedMoE.
# gate/router and shared expert (if any) runs in MoERunner.
shared_experts = fuser.shared_experts(moe_block, prefix)
# Store shared experts for later down projection adjustment
if shared_experts is not None:
@@ -331,7 +335,7 @@ class MoEMixin(MixtureOfExperts):
if fuser.shared_name:
routed += " + shared experts"
logger.info_once(
"Fused: %s (%s) -> FusedMoE (internal routing)",
"Fused: %s (%s) -> MoERunner (internal routing)",
routed,
moe_block_cls,
)
@@ -373,10 +377,10 @@ class MoEMixin(MixtureOfExperts):
runner_args={"moe_state": moe_state},
)
logger.info_once(
"Fused: experts (%s) -> FusedMoE (external routing)",
"Fused: experts (%s) -> MoERunner (external routing)",
experts_cls,
)
fused_experts = FusedMoE(**kwargs)
fused_experts = FusedMoEFactory(**kwargs)
moe_block.experts = fused_experts
log_replacement(qual_name, experts, fused_experts)
# Update MixtureOfExperts mixin state
@@ -108,7 +108,7 @@ def _extract_data_from_fused_moe_module(
m_: torch.nn.Module,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]:
"""
Extract weights, weight scales and num_topk from FusedMoE module.
Extract weights, weight scales and num_topk from MoERunner module.
"""
assert isinstance(m_, MoERunner)
m = m_.routed_experts
+3 -3
View File
@@ -18,7 +18,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
@@ -279,7 +279,7 @@ class DeepseekV4MoE(nn.Module):
self.experts_start_idx = self.tp_rank * self.n_local_experts
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
n_shared_experts=(
config.n_shared_experts if self.fuse_shared_experts else None
@@ -308,7 +308,7 @@ class DeepseekV4MoE(nn.Module):
org_shape = hidden_states.shape
if self.experts.is_internal_router:
# In this case, the gate/router runs inside the FusedMoE class
# In this case, the gate/router runs inside the MoERunner class
final_hidden_states = self.experts(
hidden_states=hidden_states,
router_logits=hidden_states,
+1 -1
View File
@@ -51,7 +51,7 @@ from .model import DeepseekV4DecoderLayer
logger = init_logger(__name__)
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
# parameter suffix depends on which FusedMoE method handles the experts:
# parameter suffix depends on which MoERunner method handles the experts:
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
# ``w{1,2,3}_weight_scale_inv``.
+3 -3
View File
@@ -27,7 +27,7 @@ from vllm.model_executor.kernels.mhc.tilelang import (
)
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.router.base_router import (
@@ -670,7 +670,7 @@ class DeepseekV4MoE(nn.Module):
self.physical_expert_start = self.experts_start_idx
self.physical_expert_end = self.experts_end_idx
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
gate=self.gate,
num_experts=config.n_routed_experts,
@@ -736,7 +736,7 @@ class DeepseekV4MoE(nn.Module):
) -> torch.Tensor:
org_shape = hidden_states.shape
if self.experts.is_internal_router:
# In this case, the gate/router runs inside the FusedMoE class
# In this case, the gate/router runs inside the MoERunner class
final_hidden_states = self.experts(
hidden_states=hidden_states,
router_logits=hidden_states,
+1 -1
View File
@@ -59,7 +59,7 @@ from .model import (
logger = init_logger(__name__)
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
# parameter suffix depends on which FusedMoE method handles the experts:
# parameter suffix depends on which FusedMoEFactory method handles the experts:
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
# ``w{1,2,3}_weight_scale_inv``.
+2 -2
View File
@@ -18,7 +18,7 @@ from vllm.distributed import (
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
@@ -729,7 +729,7 @@ class DeepseekV4MoE(nn.Module):
self.n_local_experts = config.n_routed_experts // self.tp_size
self.experts_start_idx = self.tp_rank * self.n_local_experts
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
gate=self.gate,
num_experts=config.n_routed_experts,
+1 -1
View File
@@ -54,7 +54,7 @@ from .model import (
logger = init_logger(__name__)
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
# parameter suffix depends on which FusedMoE method handles the experts:
# parameter suffix depends on which FusedMoEFactory method handles the experts:
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
# ``w{1,2,3}_weight_scale_inv``.
+1 -1
View File
@@ -646,7 +646,7 @@ def _load_inkling_weights(
weight.shard_id = shard_id
# MoE expert tensors (fused stacked, routed + shared sink): translate
# the checkpoint layout to per-expert FusedMoE loads.
# the checkpoint layout to per-expert MoERunner loads.
moe_match = _MOE_EXPERT_WEIGHT_RE.match(name)
if moe_match is not None and moe_match.group("mlp") in moe_modules:
moe = moe_modules[moe_match.group("mlp")]
+8 -8
View File
@@ -1,11 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inkling mixture-of-experts on vLLM's FusedMoE abstraction.
"""Inkling mixture-of-experts on vLLM's MoERunner abstraction.
Overfit to the served checkpoint: sigmoid gate (+ selection bias) top-k over
the routed experts, log-sigmoid renormalization over the k routed + S shared
"sink" logits, scaled by route_scale * global_scale. The routed top-k goes
through vLLM's FusedMoE (which handles TP/EP); the sink experts run in
through vLLM's MoERunner (which handles TP/EP); the sink experts run in
:class:`InklingSinkExperts` -- replicated across EP ranks (every token
activates every sink) and always bf16 (the checkpoint excludes every
``shared_experts`` from quantization).
@@ -35,7 +35,7 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
@@ -265,7 +265,7 @@ class InklingGate(nn.Module):
def _inkling_moe_ep_size() -> int:
"""EP size the FusedMoE layer will run with (mirrors
"""EP size the MoERunner layer will run with (mirrors
FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when
expert parallelism is enabled)."""
parallel_config = get_current_vllm_config().parallel_config
@@ -444,7 +444,7 @@ class InklingMoE(nn.Module):
# The released MXFP4 checkpoint keeps the first routed-expert layer
# in bf16 and lists its two expert weights explicitly in Quark's
# exclusion list. FusedMoE asks for a quant method at the module
# exclusion list. FusedMoEFactory asks for a quant method at the module
# prefix, so an exact weight exclusion would otherwise be missed and
# the bf16 tensors would be loaded into MXFP4 parameters.
routed_quant_config = quant_config
@@ -458,7 +458,7 @@ class InklingMoE(nn.Module):
if routed_weights <= excluded:
routed_quant_config = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=num_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
@@ -503,7 +503,7 @@ class InklingMoE(nn.Module):
topk: int,
renormalize: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
"""FusedMoE ``custom_routing_function``: the routed top-k slice of the
"""MoERunner ``custom_routing_function``: the routed top-k slice of the
full (routed + sink) selection.
forward() stashes its selection (keyed by logits identity) so the
@@ -523,7 +523,7 @@ class InklingMoE(nn.Module):
router_logits = self.gate.compute_logits(x)
num_tokens = x.shape[0]
# One gate select per layer: the routed slice is stashed for the
# routing function inside the FusedMoE op; the sink gammas are the
# routing function inside the MoERunner op; the sink gammas are the
# trailing columns.
k = self.gate.topk
weights, ids = self.gate.select_experts(router_logits)
+2 -2
View File
@@ -34,7 +34,7 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.triton_utils import tl, tldevice, triton
@@ -439,7 +439,7 @@ class InklingMoE(nn.Module):
# power-of-two EP sizes (n_routed is a power of two).
num_experts = n_routed + (-n_routed) % _inkling_moe_ep_size()
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=num_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
+2 -2
View File
@@ -15,7 +15,7 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul, SituAndMul
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
@@ -260,7 +260,7 @@ class KimiMoE(nn.Module):
self.routed_expert_up_proj = None
self.routed_output_transform = None
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=num_experts,
top_k=num_experts_per_token,
+4 -4
View File
@@ -20,7 +20,7 @@ from vllm.forward_context import get_forward_context, is_forward_context_availab
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul, SituAndMul
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.router.base_router import (
@@ -560,7 +560,7 @@ class KimiMoE(nn.Module):
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=num_experts,
top_k=num_experts_per_token,
@@ -578,7 +578,7 @@ class KimiMoE(nn.Module):
scoring_func=config.moe_router_activation_func,
e_score_correction_bias=self.gate.e_score_correction_bias,
routed_scaling_factor=self.routed_scaling_factor,
# Down projection runs outside FusedMoE so it can overlap the
# Down projection runs outside MoERunner so it can overlap the
# router gate on the aux stream (see forward()); the original
# hidden states are passed to forward() as shared_experts_input
# so shared experts still see the untransformed input.
@@ -689,7 +689,7 @@ class KimiMoE(nn.Module):
)
else:
# Routed experts consume the down-projected latent; shared experts
# (inside FusedMoE) get the original hidden states via
# (inside MoERunner) get the original hidden states via
# shared_experts_input.
final_hidden_states = self.experts(
hidden_states=routed_hidden_states,
+2 -2
View File
@@ -41,7 +41,7 @@ from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
fused_allreduce_gemma_rms_norm,
)
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
@@ -391,7 +391,7 @@ class MiniMaxM3MoE(nn.Module):
# always-on shared expert; aiter applies the routed scaling internally.
# Every other path (vLLM top-k bias router, or no fusion) applies the
# routed scaling to the MoE output here.
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=config.num_local_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
+2 -2
View File
@@ -31,7 +31,7 @@ from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
fused_allreduce_gemma_rms_norm,
)
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
@@ -244,7 +244,7 @@ class MiniMaxM3MoE(nn.Module):
prefix=f"{prefix}.shared_experts",
)
self.experts = FusedMoE(
self.experts = FusedMoEFactory(
num_experts=config.num_local_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,