[MoE Refactor] Separate Router into OO Classes (#30623)
Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
@@ -21,12 +21,12 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
|
||||
from vllm.utils.import_utils import has_deep_ep, has_deep_gemm, has_pplx
|
||||
|
||||
from .mk_objects import (
|
||||
|
||||
@@ -15,10 +15,10 @@ from tests.kernels.moe.utils import (
|
||||
from tests.kernels.quant_utils import native_batched_masked_quant_matmul
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.fused_batched_moe import (
|
||||
invoke_moe_batched_triton_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
@@ -11,13 +11,15 @@ from tests.kernels.quant_utils import (
|
||||
)
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import fused_experts
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_experts,
|
||||
fused_topk,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_moe import (
|
||||
_valid_deep_gemm_shape,
|
||||
deep_gemm_moe_fp8,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import (
|
||||
fused_topk,
|
||||
modular_triton_fused_moe,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -10,6 +10,7 @@ import torch
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
FusedMoEQuantConfig,
|
||||
@@ -19,7 +20,6 @@ from vllm.model_executor.layers.fused_moe.cutlass_moe import (
|
||||
CutlassExpertsFp8,
|
||||
run_cutlass_moe_fp8,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts, fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
MoEPrepareAndFinalizeNoEP,
|
||||
)
|
||||
|
||||
@@ -12,6 +12,7 @@ from tests.kernels.quantization.nvfp4_utils import (
|
||||
from tests.kernels.utils import torch_moe
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts,
|
||||
is_valid_flashinfer_cutlass_fused_moe,
|
||||
@@ -19,7 +20,6 @@ from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_prepare_finalize import (
|
||||
create_flashinfer_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEModularKernel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm.config import (
|
||||
get_cached_compilation_config,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
|
||||
GroupedTopk,
|
||||
fused_grouped_topk,
|
||||
)
|
||||
|
||||
@@ -24,6 +24,9 @@ from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed.parallel_state import init_distributed_environment
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_topk,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
int4_w4a16_moe_quant_config,
|
||||
@@ -34,7 +37,6 @@ from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
|
||||
fused_marlin_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import (
|
||||
fused_topk,
|
||||
modular_triton_fused_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
|
||||
@@ -9,7 +9,7 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.layer import determine_expert_map
|
||||
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
|
||||
moe_permute,
|
||||
|
||||
@@ -13,11 +13,11 @@ from tests.kernels.quantization.nvfp4_utils import (
|
||||
from tests.kernels.utils import torch_moe
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.config import nvfp4_moe_quant_config
|
||||
from vllm.model_executor.layers.fused_moe.cutlass_moe import (
|
||||
CutlassExpertsFp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
MoEPrepareAndFinalizeNoEP,
|
||||
)
|
||||
|
||||
@@ -8,9 +8,9 @@ import torch
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config
|
||||
from vllm.model_executor.layers.fused_moe.cutlass_moe import CutlassBatchedExpertsFp8
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEModularKernel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
@@ -0,0 +1,499 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.router.router_factory import (
|
||||
create_fused_moe_router,
|
||||
)
|
||||
from vllm.model_executor.models.llama4 import Llama4MoE
|
||||
|
||||
# Test parameters
|
||||
MK_S = [(32, 256), (64, 512)]
|
||||
TOP_KS = [2, 4, 6]
|
||||
NUM_EXPERTS = [8, 16, 64]
|
||||
|
||||
|
||||
def setup_eplb_state(enable_eplb: bool, global_num_experts: int) -> EplbLayerState:
|
||||
if not enable_eplb:
|
||||
return EplbLayerState()
|
||||
|
||||
# Initialize EPLB state with proper tensors for testing
|
||||
# For testing purposes, we use a simple 1:1 mapping (no redundant experts)
|
||||
# expert_load_view: tracks load on each expert (shape: num_experts)
|
||||
expert_load_view = torch.zeros(global_num_experts, dtype=torch.int32, device="cuda")
|
||||
|
||||
# logical_to_physical_map: maps logical experts to physical experts
|
||||
# Shape: (num_logical_experts, max_slots)
|
||||
# For testing, use simple 1:1 mapping with single slot per expert
|
||||
logical_to_physical_map = torch.arange(
|
||||
global_num_experts, dtype=torch.int64, device="cuda"
|
||||
).unsqueeze(-1)
|
||||
|
||||
# logical_replica_count: number of replicas per logical expert
|
||||
# Shape: (num_logical_experts,)
|
||||
# For testing, each logical expert has exactly 1 replica
|
||||
logical_replica_count = torch.ones(
|
||||
global_num_experts, dtype=torch.int64, device="cuda"
|
||||
)
|
||||
|
||||
return EplbLayerState(
|
||||
expert_load_view=expert_load_view,
|
||||
logical_to_physical_map=logical_to_physical_map,
|
||||
logical_replica_count=logical_replica_count,
|
||||
)
|
||||
|
||||
|
||||
def make_test_data(
|
||||
m: int, k: int, num_experts: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states = torch.randn((m, k), device="cuda") / 10
|
||||
logits = torch.randn((m, num_experts), device="cuda")
|
||||
return hidden_states, logits
|
||||
|
||||
|
||||
def make_e_score_correction_bias(
|
||||
e_score_correction_bias_val: float,
|
||||
num_experts: int,
|
||||
) -> torch.Tensor:
|
||||
# return torch.randn(num_experts, device="cuda") * e_score_correction_bias_val
|
||||
return torch.full(
|
||||
(num_experts,), e_score_correction_bias_val, device="cuda", dtype=torch.float32
|
||||
)
|
||||
|
||||
|
||||
def assert_routing_results_close(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
baseline_weights: torch.Tensor,
|
||||
baseline_ids: torch.Tensor,
|
||||
rtol: float = 1e-3,
|
||||
atol: float = 1e-3,
|
||||
):
|
||||
"""
|
||||
Compare routing results, sorting by expert ID first to handle non-deterministic
|
||||
ordering from sorted=False in topk.
|
||||
"""
|
||||
# Sort both results by expert IDs for consistent comparison
|
||||
sorted_indices_actual = torch.argsort(topk_ids, dim=-1)
|
||||
sorted_indices_baseline = torch.argsort(baseline_ids.to(topk_ids.dtype), dim=-1)
|
||||
|
||||
# Gather the sorted values
|
||||
topk_ids_sorted = torch.gather(topk_ids, 1, sorted_indices_actual)
|
||||
topk_weights_sorted = torch.gather(topk_weights, 1, sorted_indices_actual)
|
||||
baseline_ids_sorted = torch.gather(
|
||||
baseline_ids.to(topk_ids.dtype), 1, sorted_indices_baseline
|
||||
)
|
||||
baseline_weights_sorted = torch.gather(baseline_weights, 1, sorted_indices_baseline)
|
||||
|
||||
# Compare
|
||||
torch.testing.assert_close(topk_ids_sorted, baseline_ids_sorted)
|
||||
torch.testing.assert_close(
|
||||
topk_weights_sorted, baseline_weights_sorted, rtol=rtol, atol=atol
|
||||
)
|
||||
|
||||
|
||||
def baseline_fused_topk(
|
||||
router_logits: torch.Tensor, top_k: int, renormalize: bool
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Baseline for standard fused top-k routing.
|
||||
|
||||
Algorithm:
|
||||
1. Apply softmax to router logits
|
||||
2. Select top-k experts
|
||||
3. Optionally renormalize the weights
|
||||
"""
|
||||
scores = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
|
||||
# Use sorted=False to match vllm implementation (vllm_is_batch_invariant
|
||||
# defaults to False)
|
||||
topk_weights, topk_ids = torch.topk(scores, top_k, dim=-1, sorted=False)
|
||||
|
||||
if renormalize:
|
||||
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
|
||||
|
||||
|
||||
def baseline_fused_topk_bias(
|
||||
router_logits: torch.Tensor,
|
||||
top_k: int,
|
||||
renormalize: bool,
|
||||
e_score_correction_bias: torch.Tensor,
|
||||
routed_scaling_factor: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Baseline for fused top-k with bias correction.
|
||||
|
||||
Algorithm:
|
||||
1. Apply softmax to router logits
|
||||
2. Add bias to scores for expert selection
|
||||
3. Select top-k experts using biased scores
|
||||
4. Get weights from original (unbiased) scores
|
||||
5. Apply routed scaling factor
|
||||
6. Optionally renormalize the weights
|
||||
"""
|
||||
# Apply softmax to get scores
|
||||
scores = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
|
||||
|
||||
# Add bias for expert selection
|
||||
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
|
||||
|
||||
# Select top-k using biased scores (sorted=False to match implementation)
|
||||
topk_ids = torch.topk(scores_for_choice, k=top_k, dim=-1, sorted=False)[1]
|
||||
|
||||
# Get weights from original scores (not biased)
|
||||
topk_weights = scores.gather(1, topk_ids)
|
||||
|
||||
# Renormalize if needed (BEFORE applying scaling factor)
|
||||
if renormalize:
|
||||
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
# Apply scaling factor (AFTER renormalization, if applicable)
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights *= routed_scaling_factor
|
||||
|
||||
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
|
||||
|
||||
|
||||
def baseline_grouped_topk(
|
||||
router_logits: torch.Tensor,
|
||||
top_k: int,
|
||||
num_expert_group: int,
|
||||
topk_group: int,
|
||||
scoring_func: str,
|
||||
renormalize: bool,
|
||||
e_score_correction_bias: torch.Tensor | None,
|
||||
routed_scaling_factor: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Baseline for grouped top-k routing (e.g., DeepSeek).
|
||||
|
||||
Algorithm:
|
||||
1. Apply scoring function (softmax or sigmoid)
|
||||
2. Optionally add bias
|
||||
3. Select top-k groups based on max scores within each group
|
||||
4. Mask scores to only include selected groups
|
||||
5. Select top-k experts from masked scores
|
||||
6. Apply scaling factor
|
||||
7. Optionally renormalize
|
||||
"""
|
||||
num_token = router_logits.shape[0]
|
||||
|
||||
# Apply scoring function
|
||||
if scoring_func == "softmax":
|
||||
scores = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
|
||||
elif scoring_func == "sigmoid":
|
||||
scores = torch.sigmoid(router_logits.float())
|
||||
else:
|
||||
raise ValueError(f"Unsupported scoring function: {scoring_func}")
|
||||
|
||||
# Handle bias correction
|
||||
if e_score_correction_bias is not None:
|
||||
original_scores = scores
|
||||
scores = scores + e_score_correction_bias.unsqueeze(0)
|
||||
# For bias case, use sum of top-2 scores in each group
|
||||
group_scores = (
|
||||
scores.view(num_token, num_expert_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
|
||||
)
|
||||
else:
|
||||
# Use max score in each group
|
||||
group_scores = scores.view(num_token, num_expert_group, -1).max(dim=-1).values
|
||||
|
||||
# Select top-k groups
|
||||
group_idx = torch.topk(group_scores, k=topk_group, dim=-1, sorted=False)[1]
|
||||
|
||||
# Create mask for selected groups
|
||||
group_mask = torch.zeros_like(group_scores)
|
||||
group_mask.scatter_(1, group_idx, 1)
|
||||
|
||||
# Expand mask to all experts
|
||||
score_mask = (
|
||||
group_mask.unsqueeze(-1)
|
||||
.expand(num_token, num_expert_group, scores.shape[-1] // num_expert_group)
|
||||
.reshape(num_token, -1)
|
||||
)
|
||||
|
||||
# Mask scores (set non-selected to -inf)
|
||||
tmp_scores = scores.masked_fill(~score_mask.bool(), float("-inf"))
|
||||
|
||||
# Select top-k experts
|
||||
if e_score_correction_bias is not None:
|
||||
topk_ids = torch.topk(tmp_scores, k=top_k, dim=-1, sorted=False)[1]
|
||||
topk_weights = original_scores.gather(1, topk_ids)
|
||||
else:
|
||||
topk_weights, topk_ids = torch.topk(tmp_scores, k=top_k, dim=-1, sorted=False)
|
||||
|
||||
# Renormalize if needed
|
||||
if renormalize:
|
||||
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
# Apply scaling factor
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights *= routed_scaling_factor
|
||||
|
||||
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
|
||||
|
||||
|
||||
def baseline_custom_llama4(
|
||||
router_logits: torch.Tensor, top_k: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Baseline for Llama4 custom routing.
|
||||
|
||||
Algorithm:
|
||||
1. Select top-k expert indices (without softmax)
|
||||
2. Apply sigmoid to the selected scores
|
||||
"""
|
||||
router_scores, router_indices = torch.topk(router_logits, top_k, dim=-1)
|
||||
router_scores = torch.sigmoid(router_scores.float())
|
||||
return router_scores.to(torch.float32), router_indices.to(torch.int32)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,k", MK_S)
|
||||
@pytest.mark.parametrize("top_k", TOP_KS)
|
||||
@pytest.mark.parametrize("global_num_experts", NUM_EXPERTS)
|
||||
@pytest.mark.parametrize("renormalize", [False, True])
|
||||
@pytest.mark.parametrize("enable_eplb", [False, True])
|
||||
def test_fused_topk(
|
||||
m: int,
|
||||
k: int,
|
||||
top_k: int,
|
||||
global_num_experts: int,
|
||||
renormalize: bool,
|
||||
enable_eplb: bool,
|
||||
):
|
||||
if top_k > global_num_experts:
|
||||
pytest.skip(f"top_k ({top_k}) > global_num_experts ({global_num_experts})")
|
||||
|
||||
eplb_state = setup_eplb_state(enable_eplb, global_num_experts)
|
||||
router = create_fused_moe_router(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
renormalize=renormalize,
|
||||
enable_eplb=enable_eplb,
|
||||
eplb_state=eplb_state,
|
||||
)
|
||||
|
||||
hidden_states, router_logits = make_test_data(m, k, global_num_experts)
|
||||
|
||||
# Get router output
|
||||
topk_weights, topk_ids = router.select_experts(hidden_states, router_logits)
|
||||
|
||||
# Compute baseline
|
||||
baseline_weights, baseline_ids = baseline_fused_topk(
|
||||
router_logits, top_k, renormalize
|
||||
)
|
||||
|
||||
# Compare results
|
||||
assert_routing_results_close(topk_weights, topk_ids, baseline_weights, baseline_ids)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,k", MK_S)
|
||||
@pytest.mark.parametrize("top_k", TOP_KS)
|
||||
@pytest.mark.parametrize("global_num_experts", NUM_EXPERTS)
|
||||
@pytest.mark.parametrize("renormalize", [False, True])
|
||||
@pytest.mark.parametrize("enable_eplb", [False, True])
|
||||
@pytest.mark.parametrize("e_score_correction_bias_val", [0.9])
|
||||
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 1.1])
|
||||
def test_fused_topk_bias(
|
||||
m: int,
|
||||
k: int,
|
||||
top_k: int,
|
||||
global_num_experts: int,
|
||||
renormalize: bool,
|
||||
enable_eplb: bool,
|
||||
e_score_correction_bias_val: float,
|
||||
routed_scaling_factor: float,
|
||||
):
|
||||
if top_k > global_num_experts:
|
||||
pytest.skip(f"top_k ({top_k}) > global_num_experts ({global_num_experts})")
|
||||
|
||||
eplb_state = setup_eplb_state(enable_eplb, global_num_experts)
|
||||
|
||||
e_score_correction_bias = make_e_score_correction_bias(
|
||||
e_score_correction_bias_val,
|
||||
global_num_experts,
|
||||
)
|
||||
|
||||
router = create_fused_moe_router(
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
renormalize=renormalize,
|
||||
enable_eplb=enable_eplb,
|
||||
eplb_state=eplb_state,
|
||||
)
|
||||
|
||||
hidden_states, router_logits = make_test_data(m, k, global_num_experts)
|
||||
|
||||
# Get router output
|
||||
topk_weights, topk_ids = router.select_experts(hidden_states, router_logits)
|
||||
|
||||
# Compute baseline
|
||||
baseline_weights, baseline_ids = baseline_fused_topk_bias(
|
||||
router_logits,
|
||||
top_k,
|
||||
renormalize,
|
||||
e_score_correction_bias,
|
||||
routed_scaling_factor,
|
||||
)
|
||||
|
||||
# Compare results
|
||||
assert_routing_results_close(topk_weights, topk_ids, baseline_weights, baseline_ids)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,k", MK_S)
|
||||
@pytest.mark.parametrize("top_k", TOP_KS)
|
||||
@pytest.mark.parametrize(
|
||||
"global_num_experts,num_expert_group,topk_group",
|
||||
[
|
||||
(64, 8, 4), # 8 groups of 8 experts, select 4 groups
|
||||
(32, 4, 2), # 4 groups of 8 experts, select 2 groups
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("renormalize", [False, True])
|
||||
@pytest.mark.parametrize("enable_eplb", [False, True])
|
||||
@pytest.mark.parametrize("e_score_correction_bias_val", [0.9])
|
||||
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 1.1])
|
||||
@pytest.mark.parametrize("scoring_func", ["sigmoid", "softmax"])
|
||||
def test_grouped_topk(
|
||||
m: int,
|
||||
k: int,
|
||||
top_k: int,
|
||||
global_num_experts: int,
|
||||
renormalize: bool,
|
||||
enable_eplb: bool,
|
||||
num_expert_group: int,
|
||||
topk_group: int,
|
||||
scoring_func: str,
|
||||
e_score_correction_bias_val: float,
|
||||
routed_scaling_factor: float,
|
||||
):
|
||||
if top_k > global_num_experts:
|
||||
pytest.skip(f"top_k ({top_k}) > global_num_experts ({global_num_experts})")
|
||||
|
||||
eplb_state = setup_eplb_state(enable_eplb, global_num_experts)
|
||||
|
||||
e_score_correction_bias = make_e_score_correction_bias(
|
||||
e_score_correction_bias_val,
|
||||
global_num_experts,
|
||||
)
|
||||
|
||||
routing_method_type = None
|
||||
if scoring_func == "llama4":
|
||||
routing_method_type = RoutingMethodType.Llama4
|
||||
scoring_func = "sigmoid"
|
||||
|
||||
router = create_fused_moe_router(
|
||||
use_grouped_topk=True,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
scoring_func=scoring_func,
|
||||
routing_method_type=routing_method_type,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
renormalize=renormalize,
|
||||
enable_eplb=enable_eplb,
|
||||
eplb_state=eplb_state,
|
||||
)
|
||||
|
||||
hidden_states, router_logits = make_test_data(m, k, global_num_experts)
|
||||
|
||||
# Get router output
|
||||
topk_weights, topk_ids = router.select_experts(hidden_states, router_logits)
|
||||
|
||||
# Compute baseline
|
||||
baseline_weights, baseline_ids = baseline_grouped_topk(
|
||||
router_logits,
|
||||
top_k,
|
||||
num_expert_group,
|
||||
topk_group,
|
||||
scoring_func,
|
||||
renormalize,
|
||||
e_score_correction_bias,
|
||||
routed_scaling_factor,
|
||||
)
|
||||
|
||||
# Compare results
|
||||
assert_routing_results_close(topk_weights, topk_ids, baseline_weights, baseline_ids)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,k", MK_S)
|
||||
@pytest.mark.parametrize("top_k", TOP_KS)
|
||||
@pytest.mark.parametrize("global_num_experts", NUM_EXPERTS)
|
||||
@pytest.mark.parametrize("renormalize", [False, True])
|
||||
@pytest.mark.parametrize("enable_eplb", [False, True])
|
||||
@pytest.mark.parametrize("custom_routing_function", [Llama4MoE.custom_routing_function])
|
||||
def test_custom(
|
||||
m: int,
|
||||
k: int,
|
||||
top_k: int,
|
||||
global_num_experts: int,
|
||||
renormalize: bool,
|
||||
enable_eplb: bool,
|
||||
custom_routing_function: Callable,
|
||||
):
|
||||
if top_k > global_num_experts:
|
||||
pytest.skip(f"top_k ({top_k}) > global_num_experts ({global_num_experts})")
|
||||
|
||||
eplb_state = setup_eplb_state(enable_eplb, global_num_experts)
|
||||
|
||||
router = create_fused_moe_router(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
custom_routing_function=custom_routing_function,
|
||||
renormalize=renormalize,
|
||||
enable_eplb=enable_eplb,
|
||||
eplb_state=eplb_state,
|
||||
)
|
||||
|
||||
hidden_states, router_logits = make_test_data(m, k, global_num_experts)
|
||||
|
||||
# Get router output
|
||||
topk_weights, topk_ids = router.select_experts(hidden_states, router_logits)
|
||||
|
||||
# Compute baseline (Llama4 uses sigmoid)
|
||||
baseline_weights, baseline_ids = baseline_custom_llama4(router_logits, top_k)
|
||||
|
||||
# Compare results
|
||||
assert_routing_results_close(topk_weights, topk_ids, baseline_weights, baseline_ids)
|
||||
|
||||
|
||||
# TODO: is other test sufficient?
|
||||
# # See tests/test_routing_simulatator.py
|
||||
# @pytest.mark.parametrize("m,k", MK_S)
|
||||
# @pytest.mark.parametrize("top_k", TOP_KS)
|
||||
# @pytest.mark.parametrize("global_num_experts", NUM_EXPERTS)
|
||||
# @pytest.mark.parametrize("renormalize", [False, True])
|
||||
# @pytest.mark.parametrize("enable_eplb", [False, True])
|
||||
# @pytest.mark.parameterize("strategy", ["uniform_random", "normal_routing"])
|
||||
# def test_simulated(
|
||||
# m: int,
|
||||
# k: int,
|
||||
# top_k: int,
|
||||
# global_num_experts: int,
|
||||
# renormalize: bool,
|
||||
# enable_eplb: bool,
|
||||
# strategy: str,
|
||||
# monkeypatch,
|
||||
# ):
|
||||
# eplb_state = setup_eplb_state(enable_eplb)
|
||||
|
||||
# monkeypatch.setenv("VLLM_MOE_ROUTING_SIMULATION_STRATEGY", strategy)
|
||||
# router = create_fused_moe_router(
|
||||
# top_k=top_k,
|
||||
# global_num_experts=global_num_experts,
|
||||
# enable_eplb=enable_eplb,
|
||||
# eplb_state=eplb_state,
|
||||
# )
|
||||
|
||||
# hidden_states, router_logits = make_test_data(m, k, global_num_experts)
|
||||
# topk_weights, topk_ids = router.select_experts(hidden_states, router_logits)
|
||||
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test script for the token-to-expert routing simulator.
|
||||
|
||||
This script demonstrates how to use the routing simulator to test
|
||||
different routing strategies and analyze their performance, including
|
||||
integration tests with FusedMoE layer.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.routing_simulator_router import (
|
||||
DistributionBasedRouting,
|
||||
RoutingSimulator,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def device():
|
||||
"""Fixture to provide the appropriate device for testing."""
|
||||
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 16, 256])
|
||||
@pytest.mark.parametrize("hidden_size", [64, 1024])
|
||||
@pytest.mark.parametrize("num_experts", [16, 128])
|
||||
@pytest.mark.parametrize("top_k", [1, 4])
|
||||
def test_basic_functionality(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
num_experts: int,
|
||||
top_k: int,
|
||||
device,
|
||||
):
|
||||
"""Test basic functionality of the routing simulator."""
|
||||
# Test each routing strategy
|
||||
strategies = RoutingSimulator.get_available_strategies()
|
||||
|
||||
hidden_states = torch.randn(num_tokens, hidden_size, device=device)
|
||||
router_logits = torch.randn(num_tokens, num_experts, device=device)
|
||||
|
||||
for strategy in strategies:
|
||||
# Simulate routing
|
||||
topk_weights, topk_ids = RoutingSimulator.simulate_routing(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
strategy_name=strategy,
|
||||
top_k=top_k,
|
||||
)
|
||||
|
||||
# Check output shapes
|
||||
assert topk_weights.shape == (
|
||||
num_tokens,
|
||||
top_k,
|
||||
), f"Wrong weights shape for {strategy}"
|
||||
assert topk_ids.shape == (
|
||||
num_tokens,
|
||||
top_k,
|
||||
), f"Wrong ids shape for {strategy}"
|
||||
|
||||
# Check that expert IDs are valid
|
||||
assert topk_ids.min() >= 0, f"Invalid expert ID (negative) for {strategy}"
|
||||
assert topk_ids.max() < num_experts, (
|
||||
f"Invalid expert ID (too large) for {strategy}"
|
||||
)
|
||||
|
||||
|
||||
def test_routing_strategy_integration(monkeypatch, device):
|
||||
"""Test that the routing strategy environment variable works with
|
||||
FusedMoE."""
|
||||
pytest.importorskip("vllm.model_executor.layers.fused_moe.layer")
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
|
||||
# Test parameters
|
||||
num_tokens = 32
|
||||
hidden_size = 16
|
||||
num_experts = 4
|
||||
top_k = 2
|
||||
|
||||
# Create test data
|
||||
hidden_states = torch.randn(num_tokens, hidden_size, device=device)
|
||||
router_logits = torch.randn(num_tokens, num_experts, device=device)
|
||||
|
||||
# Test different routing strategies
|
||||
strategies = RoutingSimulator.get_available_strategies()
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
with set_current_vllm_config(vllm_config):
|
||||
temp_file = tempfile.mkstemp()[1]
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
local_rank=0,
|
||||
distributed_init_method=f"file://{temp_file}",
|
||||
)
|
||||
initialize_model_parallel(
|
||||
tensor_model_parallel_size=1,
|
||||
pipeline_model_parallel_size=1,
|
||||
)
|
||||
|
||||
for strategy in strategies:
|
||||
fused_moe = FusedMoE(
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=0,
|
||||
use_grouped_topk=False,
|
||||
renormalize=True,
|
||||
prefix=strategy,
|
||||
)
|
||||
|
||||
# Set environment variable
|
||||
env_name = "VLLM_MOE_ROUTING_SIMULATION_STRATEGY"
|
||||
monkeypatch.setenv(env_name, strategy)
|
||||
|
||||
# Force reload of environment variable
|
||||
envs.environment_variables[env_name] = lambda s=strategy: s
|
||||
|
||||
# Test the select_experts method
|
||||
topk_weights, topk_ids = fused_moe.router.select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
)
|
||||
|
||||
# Verify output shapes
|
||||
assert topk_weights.shape == (num_tokens, top_k), (
|
||||
f"Wrong weights shape for {strategy}"
|
||||
)
|
||||
assert topk_ids.shape == (num_tokens, top_k), (
|
||||
f"Wrong ids shape for {strategy}"
|
||||
)
|
||||
|
||||
# Verify expert IDs are valid
|
||||
assert topk_ids.min() >= 0, f"Invalid expert ID (negative) for {strategy}"
|
||||
assert topk_ids.max() < num_experts, (
|
||||
f"Invalid expert ID (too large) for {strategy}"
|
||||
)
|
||||
|
||||
|
||||
def test_distribution_based_routing_with_custom_strategy():
|
||||
"""Test registering and using DistributionBasedRouting with custom
|
||||
parameters."""
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# Register custom distribution-based strategy
|
||||
custom_strategy = DistributionBasedRouting(distribution="normal", mean=2.0, std=0.5)
|
||||
RoutingSimulator.register_strategy("custom_normal", custom_strategy)
|
||||
|
||||
# Test data
|
||||
num_tokens = 60
|
||||
hidden_size = 48
|
||||
num_experts = 6
|
||||
top_k = 3
|
||||
|
||||
hidden_states = torch.randn(num_tokens, hidden_size, device=device)
|
||||
router_logits = torch.randn(num_tokens, num_experts, device=device)
|
||||
|
||||
# Use the custom strategy
|
||||
topk_weights, topk_ids = RoutingSimulator.simulate_routing(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
strategy_name="custom_normal",
|
||||
top_k=top_k,
|
||||
)
|
||||
|
||||
# Check output shapes
|
||||
assert topk_weights.shape == (num_tokens, top_k)
|
||||
assert topk_ids.shape == (num_tokens, top_k)
|
||||
|
||||
# Check that expert IDs are valid
|
||||
assert topk_ids.min() >= 0
|
||||
assert topk_ids.max() < num_experts
|
||||
|
||||
|
||||
def test_instance_compatibility():
|
||||
"""Test that static methods work correctly."""
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# Test static method directly
|
||||
hidden_states = torch.randn(10, 8, device=device)
|
||||
router_logits = torch.randn(10, 4, device=device)
|
||||
|
||||
topk_weights, topk_ids = RoutingSimulator.simulate_routing(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
strategy_name="uniform_random",
|
||||
top_k=2,
|
||||
)
|
||||
|
||||
assert topk_weights.shape == (10, 2)
|
||||
assert topk_ids.shape == (10, 2)
|
||||
Reference in New Issue
Block a user