[MoE Refactor] Rename FusedMoE to FusedMoEFactory (#44941)

Signed-off-by: Bill Nell <bnell@redhat.com>
Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
bnellnm
2026-07-31 11:28:18 +01:00
committed by GitHub
co-authored by Harry Mellor
parent c911120a6f
commit 6e311c6e20
80 changed files with 243 additions and 234 deletions
+16 -13
View File
@@ -1,10 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for FusedMoE with zero experts.
"""Tests for FusedMoEFactory with zero experts.
Verifies that:
- The ZeroExpertRouter is properly created and used as the layer router.
- A forward pass through FusedMoE with zero experts produces correct output.
- A forward pass through FusedMoEFactory with zero experts produces correct output.
- The output decomposes correctly into real expert + zero expert contributions.
Note: tests generated with Claude.
@@ -15,7 +15,7 @@ import torch
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.router.zero_expert_router import (
ZeroExpertRouter,
)
@@ -24,7 +24,7 @@ from vllm.v1.worker.workspace import init_workspace_manager
@pytest.fixture
def zero_expert_moe(dist_init, default_vllm_config):
"""Create a FusedMoE layer with zero experts."""
"""Create a FusedMoEFactory layer with zero experts."""
num_experts = 4
top_k = 2
# hidden_size must be >= 256 for the zero expert identity kernel to
@@ -45,7 +45,7 @@ def zero_expert_moe(dist_init, default_vllm_config):
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
init_workspace_manager(torch.accelerator.current_device_index())
layer = FusedMoE(
layer = FusedMoEFactory(
zero_expert_type="identity",
e_score_correction_bias=e_score_correction_bias,
num_experts=num_experts,
@@ -66,7 +66,7 @@ def zero_expert_moe(dist_init, default_vllm_config):
@pytest.mark.parametrize("num_tokens", [1, 32])
def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_tokens):
"""Verify that FusedMoE with zero_expert_type creates a ZeroExpertRouter."""
"""Verify that FusedMoEFactory with zero_expert_type creates a ZeroExpertRouter."""
layer, _ = zero_expert_moe
assert isinstance(layer.router, ZeroExpertRouter), (
f"Expected ZeroExpertRouter but got {type(layer.router).__name__}."
@@ -83,7 +83,10 @@ def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_token
@pytest.mark.parametrize("num_tokens", [1, 32])
def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
"""Run a forward pass through FusedMoE with zero experts and verify output shape."""
"""
Run a forward pass through FusedMoEFactory with zero experts
and verify output shape.
"""
layer, vllm_config = zero_expert_moe
hidden_size = layer.routed_experts.hidden_size
@@ -118,16 +121,16 @@ def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
@pytest.mark.parametrize("num_tokens", [1, 32])
def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
"""Validate that the FusedMoE output equals a plain FusedMoE
"""Validate that the FusedMoEFactory output equals a plain FusedMoEFactory
output (real experts only) plus the zero expert contribution.
The key invariant is:
zero_layer.forward(h, r_full) == plain_layer.forward(h, r_real)
+ zero_expert_output
We create a plain FusedMoE layer with the same weights and real-expert-only
We create a plain FusedMoEFactory layer with the same weights and real-expert-only
router logits, compute the zero expert output via the ZeroExpertRouter, and
verify the sum matches the FusedMoE output.
verify the sum matches the FusedMoEFactory output.
"""
layer, vllm_config = zero_expert_moe
num_experts = 4
@@ -152,9 +155,9 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
get_forward_context().all_moe_layers = None
# Create a plain FusedMoE layer with the same config but no zero
# Create a plain FusedMoEFactory layer with the same config but no zero
# experts. Use a separate prefix to avoid collision.
plain_layer = FusedMoE(
plain_layer = FusedMoEFactory(
num_experts=num_experts,
top_k=layer.routed_experts.top_k,
hidden_size=layer.routed_experts.hidden_size,
@@ -210,7 +213,7 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
expected,
atol=4e-3,
rtol=4e-3,
msg="FusedMoE output should equal plain FusedMoE output "
msg="FusedMoEFactory output should equal plain FusedMoEFactory output "
"plus zero expert contribution",
)