[MoE Refactor] Rename FusedMoE to FusedMoEFactory (#44941)

Signed-off-by: Bill Nell <bnell@redhat.com>
Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
bnellnm
2026-07-31 11:28:18 +01:00
committed by GitHub
co-authored by Harry Mellor
parent c911120a6f
commit 6e311c6e20
80 changed files with 243 additions and 234 deletions
+1 -1
View File
@@ -409,7 +409,7 @@ def _deep_ep_v2_moe_cudagraph(
vllm_cfg.kernel_config = KernelConfig(moe_backend="flashinfer_trtllm")
with set_current_vllm_config(vllm_cfg):
# Initialize vLLM parallel state (needed by FusedMoE layer)
# Initialize vLLM parallel state (needed by MoERunner layer)
temp_file = tempfile.mktemp()
init_distributed_environment(
world_size=pgi.world_size,
+12 -7
View File
@@ -42,7 +42,11 @@ from vllm.distributed.device_communicators.all_reduce_utils import (
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
from vllm.distributed.eplb.rebalance_execute import rearrange_expert_weights_inplace
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, fused_experts
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
MoERunner,
fused_experts,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.router.router_factory import (
@@ -80,7 +84,7 @@ NUM_EXPERTS = [8, 64]
TOP_KS = [2, 6]
# dp_size, tp_size, use_ep
# Note: DP+TP is not yet supported in the FusedMoE layer.
# Note: DP+TP is not yet supported in the FusedMoEFactory layer.
PARALLEL_COMBOS = [
[1, 2, False],
[1, 4, False],
@@ -1023,7 +1027,7 @@ def make_fused_moe_layer(
kwargs["routed_input_transform"] = routed_input_transform
kwargs["routed_output_transform"] = routed_output_transform
layer = FusedMoE(
layer = FusedMoEFactory(
num_experts=global_num_experts,
top_k=top_k,
hidden_size=hidden_size,
@@ -1267,7 +1271,7 @@ def _test_body_eplb(
):
output_before = sp_wrapper(moe_layer)(hidden_states, router_logits)
# Create a fresh FusedMoE layer with enable_eplb=True
# Create a fresh MoERunner layer with enable_eplb=True
# Delete the original layer's registration so the constructor can
# re-use the same "from_forward_context" prefix
cc = vllm_config.compilation_config
@@ -1401,7 +1405,8 @@ def _run_one_config(
- When is_sequence_parallel=True (EP + sequence splitting):
* ep_size: Number of expert parallel ranks (equals dp_size * tp_size)
* tp_size: Number of ranks to split sequence across (becomes sp_size in FusedMoE)
* tp_size: Number of ranks to split sequence across (becomes sp_size in
MoERunner)
* Weights are chunked by ep_size (experts) but NOT by tp_size
* Input sequences are chunked by tp_size (via sp_wrapper)
"""
@@ -1476,8 +1481,8 @@ def _run_one_config(
torch.accelerator.empty_cache()
with set_current_vllm_config(vllm_config):
# Chunk weights for EP BEFORE creating FusedMoE
# FusedMoE uses EP-chunked weights and handles reductions internally
# Chunk weights for EP BEFORE creating MoERunner.
# MoERunner uses EP-chunked weights and handles reductions internally.
if ep_size > 1:
# Split experts across ranks (dimension 0 is the expert dimension)
# When EP is enabled, use EP group rank and ep_size for chunking
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for FusedMoE weight loading with padded hidden dimensions.
"""Tests for FusedMoEFactory weight loading with padded hidden dimensions.
When using DeepEP backends or NIXL EP with models like nemotron_h,
hidden_size may be rounded up (e.g., 2688 -> 3072) for backend requirements.
+4 -4
View File
@@ -6,7 +6,7 @@ Test script for the token-to-expert routing simulator.
This script demonstrates how to use the routing simulator to test
different routing strategies and analyze their performance, including
integration tests with FusedMoE layer.
integration tests with FusedMoEFactory layer.
"""
import tempfile
@@ -77,11 +77,11 @@ def test_basic_functionality(
def test_routing_strategy_integration(monkeypatch, device):
"""Test that the routing strategy environment variable works with
FusedMoE."""
FusedMoEFactory."""
pytest.importorskip("vllm.model_executor.layers.fused_moe.layer")
import vllm.envs as envs
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
# Test parameters
num_tokens = 32
@@ -111,7 +111,7 @@ def test_routing_strategy_integration(monkeypatch, device):
)
for strategy in strategies:
fused_moe = FusedMoE(
fused_moe = FusedMoEFactory(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
@@ -1,9 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for FusedMoE with routed_input_transform.
Tests for FusedMoEFactory with routed_input_transform.
Verifies that applying routed_input_transform inside FusedMoE
Verifies that applying routed_input_transform inside FusedMoEFactory
produces the same results as applying the transform manually outside.
"""
@@ -13,7 +13,7 @@ import torch.nn as nn
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_torch_equal_or_newer, set_random_seed
@@ -133,9 +133,9 @@ def test_routed_input_transform_inside_vs_outside(
workspace_init,
monkeypatch,
):
"""Compare FusedMoE with transform inside vs manually applying outside.
Method A (inside): FusedMoE with routed_input_transform
Method B (outside): Manually transform, then FusedMoE without transform
"""Compare FusedMoEFactory with transform inside vs manually applying outside.
Method A (inside): FusedMoEFactory with routed_input_transform
Method B (outside): Manually transform, then FusedMoEFactory without transform
"""
if current_platform.is_rocm():
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_rocm_aiter else "0")
@@ -157,8 +157,8 @@ def test_routed_input_transform_inside_vs_outside(
routed_transform = SimpleLinear(hidden_size, latent_size, dtype)
with set_current_vllm_config(vllm_config):
# Method A: FusedMoE WITH routed_input_transform
moe_with_transform = FusedMoE(
# Method A: FusedMoEFactory WITH routed_input_transform
moe_with_transform = FusedMoEFactory(
shared_experts=shared_experts,
routed_input_transform=routed_transform,
num_experts=num_experts,
@@ -173,9 +173,9 @@ def test_routed_input_transform_inside_vs_outside(
prefix="moe_with_transform",
)
# Method B: FusedMoE WITHOUT routed_input_transform
# Method B: FusedMoEFactory WITHOUT routed_input_transform
# Note: shared_experts=None because when transform is done outside,
moe_without_transform = FusedMoE(
moe_without_transform = FusedMoEFactory(
shared_experts=None,
routed_input_transform=None,
num_experts=num_experts,
+16 -13
View File
@@ -1,10 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for FusedMoE with zero experts.
"""Tests for FusedMoEFactory with zero experts.
Verifies that:
- The ZeroExpertRouter is properly created and used as the layer router.
- A forward pass through FusedMoE with zero experts produces correct output.
- A forward pass through FusedMoEFactory with zero experts produces correct output.
- The output decomposes correctly into real expert + zero expert contributions.
Note: tests generated with Claude.
@@ -15,7 +15,7 @@ import torch
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.router.zero_expert_router import (
ZeroExpertRouter,
)
@@ -24,7 +24,7 @@ from vllm.v1.worker.workspace import init_workspace_manager
@pytest.fixture
def zero_expert_moe(dist_init, default_vllm_config):
"""Create a FusedMoE layer with zero experts."""
"""Create a FusedMoEFactory layer with zero experts."""
num_experts = 4
top_k = 2
# hidden_size must be >= 256 for the zero expert identity kernel to
@@ -45,7 +45,7 @@ def zero_expert_moe(dist_init, default_vllm_config):
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
init_workspace_manager(torch.accelerator.current_device_index())
layer = FusedMoE(
layer = FusedMoEFactory(
zero_expert_type="identity",
e_score_correction_bias=e_score_correction_bias,
num_experts=num_experts,
@@ -66,7 +66,7 @@ def zero_expert_moe(dist_init, default_vllm_config):
@pytest.mark.parametrize("num_tokens", [1, 32])
def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_tokens):
"""Verify that FusedMoE with zero_expert_type creates a ZeroExpertRouter."""
"""Verify that FusedMoEFactory with zero_expert_type creates a ZeroExpertRouter."""
layer, _ = zero_expert_moe
assert isinstance(layer.router, ZeroExpertRouter), (
f"Expected ZeroExpertRouter but got {type(layer.router).__name__}."
@@ -83,7 +83,10 @@ def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_token
@pytest.mark.parametrize("num_tokens", [1, 32])
def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
"""Run a forward pass through FusedMoE with zero experts and verify output shape."""
"""
Run a forward pass through FusedMoEFactory with zero experts
and verify output shape.
"""
layer, vllm_config = zero_expert_moe
hidden_size = layer.routed_experts.hidden_size
@@ -118,16 +121,16 @@ def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
@pytest.mark.parametrize("num_tokens", [1, 32])
def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
"""Validate that the FusedMoE output equals a plain FusedMoE
"""Validate that the FusedMoEFactory output equals a plain FusedMoEFactory
output (real experts only) plus the zero expert contribution.
The key invariant is:
zero_layer.forward(h, r_full) == plain_layer.forward(h, r_real)
+ zero_expert_output
We create a plain FusedMoE layer with the same weights and real-expert-only
We create a plain FusedMoEFactory layer with the same weights and real-expert-only
router logits, compute the zero expert output via the ZeroExpertRouter, and
verify the sum matches the FusedMoE output.
verify the sum matches the FusedMoEFactory output.
"""
layer, vllm_config = zero_expert_moe
num_experts = 4
@@ -152,9 +155,9 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
get_forward_context().all_moe_layers = None
# Create a plain FusedMoE layer with the same config but no zero
# Create a plain FusedMoEFactory layer with the same config but no zero
# experts. Use a separate prefix to avoid collision.
plain_layer = FusedMoE(
plain_layer = FusedMoEFactory(
num_experts=num_experts,
top_k=layer.routed_experts.top_k,
hidden_size=layer.routed_experts.hidden_size,
@@ -210,7 +213,7 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
expected,
atol=4e-3,
rtol=4e-3,
msg="FusedMoE output should equal plain FusedMoE output "
msg="FusedMoEFactory output should equal plain FusedMoEFactory output "
"plus zero expert contribution",
)