[MoE Refactor] Separate Router into OO Classes (#30623)

Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
bnellnm
2026-01-18 11:40:49 -05:00
committed by GitHub
parent 2f03035a61
commit 327a02d8db
45 changed files with 1750 additions and 684 deletions
@@ -21,12 +21,12 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
from vllm.utils.import_utils import has_deep_ep, has_deep_gemm, has_pplx
from .mk_objects import (
+1 -1
View File
@@ -15,10 +15,10 @@ from tests.kernels.moe.utils import (
from tests.kernels.quant_utils import native_batched_masked_quant_matmul
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.fused_batched_moe import (
invoke_moe_batched_triton_kernel,
)
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
from vllm.platforms import current_platform
from vllm.triton_utils import tl
from vllm.utils.torch_utils import set_random_seed
+4 -2
View File
@@ -11,13 +11,15 @@ from tests.kernels.quant_utils import (
)
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import fused_experts
from vllm.model_executor.layers.fused_moe import (
fused_experts,
fused_topk,
)
from vllm.model_executor.layers.fused_moe.deep_gemm_moe import (
_valid_deep_gemm_shape,
deep_gemm_moe_fp8,
)
from vllm.model_executor.layers.fused_moe.fused_moe import (
fused_topk,
modular_triton_fused_moe,
)
from vllm.platforms import current_platform
+1 -1
View File
@@ -10,6 +10,7 @@ import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import _custom_ops as ops
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk
from vllm.model_executor.layers.fused_moe.config import (
FUSED_MOE_UNQUANTIZED_CONFIG,
FusedMoEQuantConfig,
@@ -19,7 +20,6 @@ from vllm.model_executor.layers.fused_moe.cutlass_moe import (
CutlassExpertsFp8,
run_cutlass_moe_fp8,
)
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts, fused_topk
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
MoEPrepareAndFinalizeNoEP,
)
+1 -1
View File
@@ -12,6 +12,7 @@ from tests.kernels.quantization.nvfp4_utils import (
from tests.kernels.utils import torch_moe
from vllm import _custom_ops as ops
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import (
FlashInferExperts,
is_valid_flashinfer_cutlass_fused_moe,
@@ -19,7 +20,6 @@ from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import (
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_prepare_finalize import (
create_flashinfer_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEModularKernel
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe
+1 -1
View File
@@ -14,7 +14,7 @@ from vllm.config import (
get_cached_compilation_config,
set_current_vllm_config,
)
from vllm.model_executor.layers.fused_moe.fused_moe import (
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
GroupedTopk,
fused_grouped_topk,
)
+3 -1
View File
@@ -24,6 +24,9 @@ from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed.parallel_state import init_distributed_environment
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe import (
fused_topk,
)
from vllm.model_executor.layers.fused_moe.config import (
FUSED_MOE_UNQUANTIZED_CONFIG,
int4_w4a16_moe_quant_config,
@@ -34,7 +37,6 @@ from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
fused_marlin_moe,
)
from vllm.model_executor.layers.fused_moe.fused_moe import (
fused_topk,
modular_triton_fused_moe,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
@@ -9,7 +9,7 @@ import numpy as np
import pytest
import torch
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.layer import determine_expert_map
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
moe_permute,
+1 -1
View File
@@ -13,11 +13,11 @@ from tests.kernels.quantization.nvfp4_utils import (
from tests.kernels.utils import torch_moe
from vllm import _custom_ops as ops
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.config import nvfp4_moe_quant_config
from vllm.model_executor.layers.fused_moe.cutlass_moe import (
CutlassExpertsFp4,
)
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
MoEPrepareAndFinalizeNoEP,
)
+1 -1
View File
@@ -8,9 +8,9 @@ import torch
from tests.kernels.utils import torch_experts
from vllm import _custom_ops as ops
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config
from vllm.model_executor.layers.fused_moe.cutlass_moe import CutlassBatchedExpertsFp8
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEModularKernel
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
+499
View File
@@ -0,0 +1,499 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import pytest
import torch
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.router.router_factory import (
create_fused_moe_router,
)
from vllm.model_executor.models.llama4 import Llama4MoE
# Test parameters
MK_S = [(32, 256), (64, 512)]
TOP_KS = [2, 4, 6]
NUM_EXPERTS = [8, 16, 64]
def setup_eplb_state(enable_eplb: bool, global_num_experts: int) -> EplbLayerState:
if not enable_eplb:
return EplbLayerState()
# Initialize EPLB state with proper tensors for testing
# For testing purposes, we use a simple 1:1 mapping (no redundant experts)
# expert_load_view: tracks load on each expert (shape: num_experts)
expert_load_view = torch.zeros(global_num_experts, dtype=torch.int32, device="cuda")
# logical_to_physical_map: maps logical experts to physical experts
# Shape: (num_logical_experts, max_slots)
# For testing, use simple 1:1 mapping with single slot per expert
logical_to_physical_map = torch.arange(
global_num_experts, dtype=torch.int64, device="cuda"
).unsqueeze(-1)
# logical_replica_count: number of replicas per logical expert
# Shape: (num_logical_experts,)
# For testing, each logical expert has exactly 1 replica
logical_replica_count = torch.ones(
global_num_experts, dtype=torch.int64, device="cuda"
)
return EplbLayerState(
expert_load_view=expert_load_view,
logical_to_physical_map=logical_to_physical_map,
logical_replica_count=logical_replica_count,
)
def make_test_data(
m: int, k: int, num_experts: int
) -> tuple[torch.Tensor, torch.Tensor]:
hidden_states = torch.randn((m, k), device="cuda") / 10
logits = torch.randn((m, num_experts), device="cuda")
return hidden_states, logits
def make_e_score_correction_bias(
e_score_correction_bias_val: float,
num_experts: int,
) -> torch.Tensor:
# return torch.randn(num_experts, device="cuda") * e_score_correction_bias_val
return torch.full(
(num_experts,), e_score_correction_bias_val, device="cuda", dtype=torch.float32
)
def assert_routing_results_close(
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
baseline_weights: torch.Tensor,
baseline_ids: torch.Tensor,
rtol: float = 1e-3,
atol: float = 1e-3,
):
"""
Compare routing results, sorting by expert ID first to handle non-deterministic
ordering from sorted=False in topk.
"""
# Sort both results by expert IDs for consistent comparison
sorted_indices_actual = torch.argsort(topk_ids, dim=-1)
sorted_indices_baseline = torch.argsort(baseline_ids.to(topk_ids.dtype), dim=-1)
# Gather the sorted values
topk_ids_sorted = torch.gather(topk_ids, 1, sorted_indices_actual)
topk_weights_sorted = torch.gather(topk_weights, 1, sorted_indices_actual)
baseline_ids_sorted = torch.gather(
baseline_ids.to(topk_ids.dtype), 1, sorted_indices_baseline
)
baseline_weights_sorted = torch.gather(baseline_weights, 1, sorted_indices_baseline)
# Compare
torch.testing.assert_close(topk_ids_sorted, baseline_ids_sorted)
torch.testing.assert_close(
topk_weights_sorted, baseline_weights_sorted, rtol=rtol, atol=atol
)
def baseline_fused_topk(
router_logits: torch.Tensor, top_k: int, renormalize: bool
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Baseline for standard fused top-k routing.
Algorithm:
1. Apply softmax to router logits
2. Select top-k experts
3. Optionally renormalize the weights
"""
scores = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
# Use sorted=False to match vllm implementation (vllm_is_batch_invariant
# defaults to False)
topk_weights, topk_ids = torch.topk(scores, top_k, dim=-1, sorted=False)
if renormalize:
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
def baseline_fused_topk_bias(
router_logits: torch.Tensor,
top_k: int,
renormalize: bool,
e_score_correction_bias: torch.Tensor,
routed_scaling_factor: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Baseline for fused top-k with bias correction.
Algorithm:
1. Apply softmax to router logits
2. Add bias to scores for expert selection
3. Select top-k experts using biased scores
4. Get weights from original (unbiased) scores
5. Apply routed scaling factor
6. Optionally renormalize the weights
"""
# Apply softmax to get scores
scores = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
# Add bias for expert selection
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
# Select top-k using biased scores (sorted=False to match implementation)
topk_ids = torch.topk(scores_for_choice, k=top_k, dim=-1, sorted=False)[1]
# Get weights from original scores (not biased)
topk_weights = scores.gather(1, topk_ids)
# Renormalize if needed (BEFORE applying scaling factor)
if renormalize:
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
# Apply scaling factor (AFTER renormalization, if applicable)
if routed_scaling_factor != 1.0:
topk_weights *= routed_scaling_factor
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
def baseline_grouped_topk(
router_logits: torch.Tensor,
top_k: int,
num_expert_group: int,
topk_group: int,
scoring_func: str,
renormalize: bool,
e_score_correction_bias: torch.Tensor | None,
routed_scaling_factor: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Baseline for grouped top-k routing (e.g., DeepSeek).
Algorithm:
1. Apply scoring function (softmax or sigmoid)
2. Optionally add bias
3. Select top-k groups based on max scores within each group
4. Mask scores to only include selected groups
5. Select top-k experts from masked scores
6. Apply scaling factor
7. Optionally renormalize
"""
num_token = router_logits.shape[0]
# Apply scoring function
if scoring_func == "softmax":
scores = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
elif scoring_func == "sigmoid":
scores = torch.sigmoid(router_logits.float())
else:
raise ValueError(f"Unsupported scoring function: {scoring_func}")
# Handle bias correction
if e_score_correction_bias is not None:
original_scores = scores
scores = scores + e_score_correction_bias.unsqueeze(0)
# For bias case, use sum of top-2 scores in each group
group_scores = (
scores.view(num_token, num_expert_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
)
else:
# Use max score in each group
group_scores = scores.view(num_token, num_expert_group, -1).max(dim=-1).values
# Select top-k groups
group_idx = torch.topk(group_scores, k=topk_group, dim=-1, sorted=False)[1]
# Create mask for selected groups
group_mask = torch.zeros_like(group_scores)
group_mask.scatter_(1, group_idx, 1)
# Expand mask to all experts
score_mask = (
group_mask.unsqueeze(-1)
.expand(num_token, num_expert_group, scores.shape[-1] // num_expert_group)
.reshape(num_token, -1)
)
# Mask scores (set non-selected to -inf)
tmp_scores = scores.masked_fill(~score_mask.bool(), float("-inf"))
# Select top-k experts
if e_score_correction_bias is not None:
topk_ids = torch.topk(tmp_scores, k=top_k, dim=-1, sorted=False)[1]
topk_weights = original_scores.gather(1, topk_ids)
else:
topk_weights, topk_ids = torch.topk(tmp_scores, k=top_k, dim=-1, sorted=False)
# Renormalize if needed
if renormalize:
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
# Apply scaling factor
if routed_scaling_factor != 1.0:
topk_weights *= routed_scaling_factor
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
def baseline_custom_llama4(
router_logits: torch.Tensor, top_k: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Baseline for Llama4 custom routing.
Algorithm:
1. Select top-k expert indices (without softmax)
2. Apply sigmoid to the selected scores
"""
router_scores, router_indices = torch.topk(router_logits, top_k, dim=-1)
router_scores = torch.sigmoid(router_scores.float())
return router_scores.to(torch.float32), router_indices.to(torch.int32)
@pytest.mark.parametrize("m,k", MK_S)
@pytest.mark.parametrize("top_k", TOP_KS)
@pytest.mark.parametrize("global_num_experts", NUM_EXPERTS)
@pytest.mark.parametrize("renormalize", [False, True])
@pytest.mark.parametrize("enable_eplb", [False, True])
def test_fused_topk(
m: int,
k: int,
top_k: int,
global_num_experts: int,
renormalize: bool,
enable_eplb: bool,
):
if top_k > global_num_experts:
pytest.skip(f"top_k ({top_k}) > global_num_experts ({global_num_experts})")
eplb_state = setup_eplb_state(enable_eplb, global_num_experts)
router = create_fused_moe_router(
top_k=top_k,
global_num_experts=global_num_experts,
renormalize=renormalize,
enable_eplb=enable_eplb,
eplb_state=eplb_state,
)
hidden_states, router_logits = make_test_data(m, k, global_num_experts)
# Get router output
topk_weights, topk_ids = router.select_experts(hidden_states, router_logits)
# Compute baseline
baseline_weights, baseline_ids = baseline_fused_topk(
router_logits, top_k, renormalize
)
# Compare results
assert_routing_results_close(topk_weights, topk_ids, baseline_weights, baseline_ids)
@pytest.mark.parametrize("m,k", MK_S)
@pytest.mark.parametrize("top_k", TOP_KS)
@pytest.mark.parametrize("global_num_experts", NUM_EXPERTS)
@pytest.mark.parametrize("renormalize", [False, True])
@pytest.mark.parametrize("enable_eplb", [False, True])
@pytest.mark.parametrize("e_score_correction_bias_val", [0.9])
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 1.1])
def test_fused_topk_bias(
m: int,
k: int,
top_k: int,
global_num_experts: int,
renormalize: bool,
enable_eplb: bool,
e_score_correction_bias_val: float,
routed_scaling_factor: float,
):
if top_k > global_num_experts:
pytest.skip(f"top_k ({top_k}) > global_num_experts ({global_num_experts})")
eplb_state = setup_eplb_state(enable_eplb, global_num_experts)
e_score_correction_bias = make_e_score_correction_bias(
e_score_correction_bias_val,
global_num_experts,
)
router = create_fused_moe_router(
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
top_k=top_k,
global_num_experts=global_num_experts,
renormalize=renormalize,
enable_eplb=enable_eplb,
eplb_state=eplb_state,
)
hidden_states, router_logits = make_test_data(m, k, global_num_experts)
# Get router output
topk_weights, topk_ids = router.select_experts(hidden_states, router_logits)
# Compute baseline
baseline_weights, baseline_ids = baseline_fused_topk_bias(
router_logits,
top_k,
renormalize,
e_score_correction_bias,
routed_scaling_factor,
)
# Compare results
assert_routing_results_close(topk_weights, topk_ids, baseline_weights, baseline_ids)
@pytest.mark.parametrize("m,k", MK_S)
@pytest.mark.parametrize("top_k", TOP_KS)
@pytest.mark.parametrize(
"global_num_experts,num_expert_group,topk_group",
[
(64, 8, 4), # 8 groups of 8 experts, select 4 groups
(32, 4, 2), # 4 groups of 8 experts, select 2 groups
],
)
@pytest.mark.parametrize("renormalize", [False, True])
@pytest.mark.parametrize("enable_eplb", [False, True])
@pytest.mark.parametrize("e_score_correction_bias_val", [0.9])
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 1.1])
@pytest.mark.parametrize("scoring_func", ["sigmoid", "softmax"])
def test_grouped_topk(
m: int,
k: int,
top_k: int,
global_num_experts: int,
renormalize: bool,
enable_eplb: bool,
num_expert_group: int,
topk_group: int,
scoring_func: str,
e_score_correction_bias_val: float,
routed_scaling_factor: float,
):
if top_k > global_num_experts:
pytest.skip(f"top_k ({top_k}) > global_num_experts ({global_num_experts})")
eplb_state = setup_eplb_state(enable_eplb, global_num_experts)
e_score_correction_bias = make_e_score_correction_bias(
e_score_correction_bias_val,
global_num_experts,
)
routing_method_type = None
if scoring_func == "llama4":
routing_method_type = RoutingMethodType.Llama4
scoring_func = "sigmoid"
router = create_fused_moe_router(
use_grouped_topk=True,
num_expert_group=num_expert_group,
topk_group=topk_group,
scoring_func=scoring_func,
routing_method_type=routing_method_type,
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
top_k=top_k,
global_num_experts=global_num_experts,
renormalize=renormalize,
enable_eplb=enable_eplb,
eplb_state=eplb_state,
)
hidden_states, router_logits = make_test_data(m, k, global_num_experts)
# Get router output
topk_weights, topk_ids = router.select_experts(hidden_states, router_logits)
# Compute baseline
baseline_weights, baseline_ids = baseline_grouped_topk(
router_logits,
top_k,
num_expert_group,
topk_group,
scoring_func,
renormalize,
e_score_correction_bias,
routed_scaling_factor,
)
# Compare results
assert_routing_results_close(topk_weights, topk_ids, baseline_weights, baseline_ids)
@pytest.mark.parametrize("m,k", MK_S)
@pytest.mark.parametrize("top_k", TOP_KS)
@pytest.mark.parametrize("global_num_experts", NUM_EXPERTS)
@pytest.mark.parametrize("renormalize", [False, True])
@pytest.mark.parametrize("enable_eplb", [False, True])
@pytest.mark.parametrize("custom_routing_function", [Llama4MoE.custom_routing_function])
def test_custom(
m: int,
k: int,
top_k: int,
global_num_experts: int,
renormalize: bool,
enable_eplb: bool,
custom_routing_function: Callable,
):
if top_k > global_num_experts:
pytest.skip(f"top_k ({top_k}) > global_num_experts ({global_num_experts})")
eplb_state = setup_eplb_state(enable_eplb, global_num_experts)
router = create_fused_moe_router(
top_k=top_k,
global_num_experts=global_num_experts,
custom_routing_function=custom_routing_function,
renormalize=renormalize,
enable_eplb=enable_eplb,
eplb_state=eplb_state,
)
hidden_states, router_logits = make_test_data(m, k, global_num_experts)
# Get router output
topk_weights, topk_ids = router.select_experts(hidden_states, router_logits)
# Compute baseline (Llama4 uses sigmoid)
baseline_weights, baseline_ids = baseline_custom_llama4(router_logits, top_k)
# Compare results
assert_routing_results_close(topk_weights, topk_ids, baseline_weights, baseline_ids)
# TODO: is other test sufficient?
# # See tests/test_routing_simulatator.py
# @pytest.mark.parametrize("m,k", MK_S)
# @pytest.mark.parametrize("top_k", TOP_KS)
# @pytest.mark.parametrize("global_num_experts", NUM_EXPERTS)
# @pytest.mark.parametrize("renormalize", [False, True])
# @pytest.mark.parametrize("enable_eplb", [False, True])
# @pytest.mark.parameterize("strategy", ["uniform_random", "normal_routing"])
# def test_simulated(
# m: int,
# k: int,
# top_k: int,
# global_num_experts: int,
# renormalize: bool,
# enable_eplb: bool,
# strategy: str,
# monkeypatch,
# ):
# eplb_state = setup_eplb_state(enable_eplb)
# monkeypatch.setenv("VLLM_MOE_ROUTING_SIMULATION_STRATEGY", strategy)
# router = create_fused_moe_router(
# top_k=top_k,
# global_num_experts=global_num_experts,
# enable_eplb=enable_eplb,
# eplb_state=eplb_state,
# )
# hidden_states, router_logits = make_test_data(m, k, global_num_experts)
# topk_weights, topk_ids = router.select_experts(hidden_states, router_logits)
+203
View File
@@ -0,0 +1,203 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test script for the token-to-expert routing simulator.
This script demonstrates how to use the routing simulator to test
different routing strategies and analyze their performance, including
integration tests with FusedMoE layer.
"""
import tempfile
import pytest
import torch
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.layers.fused_moe.router.routing_simulator_router import (
DistributionBasedRouting,
RoutingSimulator,
)
@pytest.fixture
def device():
"""Fixture to provide the appropriate device for testing."""
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
@pytest.mark.parametrize("num_tokens", [1, 16, 256])
@pytest.mark.parametrize("hidden_size", [64, 1024])
@pytest.mark.parametrize("num_experts", [16, 128])
@pytest.mark.parametrize("top_k", [1, 4])
def test_basic_functionality(
num_tokens: int,
hidden_size: int,
num_experts: int,
top_k: int,
device,
):
"""Test basic functionality of the routing simulator."""
# Test each routing strategy
strategies = RoutingSimulator.get_available_strategies()
hidden_states = torch.randn(num_tokens, hidden_size, device=device)
router_logits = torch.randn(num_tokens, num_experts, device=device)
for strategy in strategies:
# Simulate routing
topk_weights, topk_ids = RoutingSimulator.simulate_routing(
hidden_states=hidden_states,
router_logits=router_logits,
strategy_name=strategy,
top_k=top_k,
)
# Check output shapes
assert topk_weights.shape == (
num_tokens,
top_k,
), f"Wrong weights shape for {strategy}"
assert topk_ids.shape == (
num_tokens,
top_k,
), f"Wrong ids shape for {strategy}"
# Check that expert IDs are valid
assert topk_ids.min() >= 0, f"Invalid expert ID (negative) for {strategy}"
assert topk_ids.max() < num_experts, (
f"Invalid expert ID (too large) for {strategy}"
)
def test_routing_strategy_integration(monkeypatch, device):
"""Test that the routing strategy environment variable works with
FusedMoE."""
pytest.importorskip("vllm.model_executor.layers.fused_moe.layer")
import vllm.envs as envs
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
# Test parameters
num_tokens = 32
hidden_size = 16
num_experts = 4
top_k = 2
# Create test data
hidden_states = torch.randn(num_tokens, hidden_size, device=device)
router_logits = torch.randn(num_tokens, num_experts, device=device)
# Test different routing strategies
strategies = RoutingSimulator.get_available_strategies()
vllm_config = VllmConfig()
with set_current_vllm_config(vllm_config):
temp_file = tempfile.mkstemp()[1]
init_distributed_environment(
world_size=1,
rank=0,
local_rank=0,
distributed_init_method=f"file://{temp_file}",
)
initialize_model_parallel(
tensor_model_parallel_size=1,
pipeline_model_parallel_size=1,
)
for strategy in strategies:
fused_moe = FusedMoE(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
intermediate_size=0,
use_grouped_topk=False,
renormalize=True,
prefix=strategy,
)
# Set environment variable
env_name = "VLLM_MOE_ROUTING_SIMULATION_STRATEGY"
monkeypatch.setenv(env_name, strategy)
# Force reload of environment variable
envs.environment_variables[env_name] = lambda s=strategy: s
# Test the select_experts method
topk_weights, topk_ids = fused_moe.router.select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
)
# Verify output shapes
assert topk_weights.shape == (num_tokens, top_k), (
f"Wrong weights shape for {strategy}"
)
assert topk_ids.shape == (num_tokens, top_k), (
f"Wrong ids shape for {strategy}"
)
# Verify expert IDs are valid
assert topk_ids.min() >= 0, f"Invalid expert ID (negative) for {strategy}"
assert topk_ids.max() < num_experts, (
f"Invalid expert ID (too large) for {strategy}"
)
def test_distribution_based_routing_with_custom_strategy():
"""Test registering and using DistributionBasedRouting with custom
parameters."""
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Register custom distribution-based strategy
custom_strategy = DistributionBasedRouting(distribution="normal", mean=2.0, std=0.5)
RoutingSimulator.register_strategy("custom_normal", custom_strategy)
# Test data
num_tokens = 60
hidden_size = 48
num_experts = 6
top_k = 3
hidden_states = torch.randn(num_tokens, hidden_size, device=device)
router_logits = torch.randn(num_tokens, num_experts, device=device)
# Use the custom strategy
topk_weights, topk_ids = RoutingSimulator.simulate_routing(
hidden_states=hidden_states,
router_logits=router_logits,
strategy_name="custom_normal",
top_k=top_k,
)
# Check output shapes
assert topk_weights.shape == (num_tokens, top_k)
assert topk_ids.shape == (num_tokens, top_k)
# Check that expert IDs are valid
assert topk_ids.min() >= 0
assert topk_ids.max() < num_experts
def test_instance_compatibility():
"""Test that static methods work correctly."""
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Test static method directly
hidden_states = torch.randn(10, 8, device=device)
router_logits = torch.randn(10, 4, device=device)
topk_weights, topk_ids = RoutingSimulator.simulate_routing(
hidden_states=hidden_states,
router_logits=router_logits,
strategy_name="uniform_random",
top_k=2,
)
assert topk_weights.shape == (10, 2)
assert topk_ids.shape == (10, 2)