[MoE Refactor] Rename FusedMoE to FusedMoEFactory (#44941)
Signed-off-by: Bill Nell <bnell@redhat.com> Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -1,10 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for FusedMoE with zero experts.
|
||||
"""Tests for FusedMoEFactory with zero experts.
|
||||
|
||||
Verifies that:
|
||||
- The ZeroExpertRouter is properly created and used as the layer router.
|
||||
- A forward pass through FusedMoE with zero experts produces correct output.
|
||||
- A forward pass through FusedMoEFactory with zero experts produces correct output.
|
||||
- The output decomposes correctly into real expert + zero expert contributions.
|
||||
|
||||
Note: tests generated with Claude.
|
||||
@@ -15,7 +15,7 @@ import torch
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_moe.router.zero_expert_router import (
|
||||
ZeroExpertRouter,
|
||||
)
|
||||
@@ -24,7 +24,7 @@ from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
@pytest.fixture
|
||||
def zero_expert_moe(dist_init, default_vllm_config):
|
||||
"""Create a FusedMoE layer with zero experts."""
|
||||
"""Create a FusedMoEFactory layer with zero experts."""
|
||||
num_experts = 4
|
||||
top_k = 2
|
||||
# hidden_size must be >= 256 for the zero expert identity kernel to
|
||||
@@ -45,7 +45,7 @@ def zero_expert_moe(dist_init, default_vllm_config):
|
||||
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
|
||||
init_workspace_manager(torch.accelerator.current_device_index())
|
||||
|
||||
layer = FusedMoE(
|
||||
layer = FusedMoEFactory(
|
||||
zero_expert_type="identity",
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
num_experts=num_experts,
|
||||
@@ -66,7 +66,7 @@ def zero_expert_moe(dist_init, default_vllm_config):
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 32])
|
||||
def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_tokens):
|
||||
"""Verify that FusedMoE with zero_expert_type creates a ZeroExpertRouter."""
|
||||
"""Verify that FusedMoEFactory with zero_expert_type creates a ZeroExpertRouter."""
|
||||
layer, _ = zero_expert_moe
|
||||
assert isinstance(layer.router, ZeroExpertRouter), (
|
||||
f"Expected ZeroExpertRouter but got {type(layer.router).__name__}."
|
||||
@@ -83,7 +83,10 @@ def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_token
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 32])
|
||||
def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
|
||||
"""Run a forward pass through FusedMoE with zero experts and verify output shape."""
|
||||
"""
|
||||
Run a forward pass through FusedMoEFactory with zero experts
|
||||
and verify output shape.
|
||||
"""
|
||||
layer, vllm_config = zero_expert_moe
|
||||
|
||||
hidden_size = layer.routed_experts.hidden_size
|
||||
@@ -118,16 +121,16 @@ def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 32])
|
||||
def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
|
||||
"""Validate that the FusedMoE output equals a plain FusedMoE
|
||||
"""Validate that the FusedMoEFactory output equals a plain FusedMoEFactory
|
||||
output (real experts only) plus the zero expert contribution.
|
||||
|
||||
The key invariant is:
|
||||
zero_layer.forward(h, r_full) == plain_layer.forward(h, r_real)
|
||||
+ zero_expert_output
|
||||
|
||||
We create a plain FusedMoE layer with the same weights and real-expert-only
|
||||
We create a plain FusedMoEFactory layer with the same weights and real-expert-only
|
||||
router logits, compute the zero expert output via the ZeroExpertRouter, and
|
||||
verify the sum matches the FusedMoE output.
|
||||
verify the sum matches the FusedMoEFactory output.
|
||||
"""
|
||||
layer, vllm_config = zero_expert_moe
|
||||
num_experts = 4
|
||||
@@ -152,9 +155,9 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
|
||||
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
|
||||
get_forward_context().all_moe_layers = None
|
||||
|
||||
# Create a plain FusedMoE layer with the same config but no zero
|
||||
# Create a plain FusedMoEFactory layer with the same config but no zero
|
||||
# experts. Use a separate prefix to avoid collision.
|
||||
plain_layer = FusedMoE(
|
||||
plain_layer = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=layer.routed_experts.top_k,
|
||||
hidden_size=layer.routed_experts.hidden_size,
|
||||
@@ -210,7 +213,7 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
|
||||
expected,
|
||||
atol=4e-3,
|
||||
rtol=4e-3,
|
||||
msg="FusedMoE output should equal plain FusedMoE output "
|
||||
msg="FusedMoEFactory output should equal plain FusedMoEFactory output "
|
||||
"plus zero expert contribution",
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user