diff --git a/tests/kernels/moe/test_flashinfer.py b/tests/kernels/moe/test_flashinfer.py index 5d55876e81..2a46a0a56a 100644 --- a/tests/kernels/moe/test_flashinfer.py +++ b/tests/kernels/moe/test_flashinfer.py @@ -42,6 +42,9 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( convert_to_fp8_moe_kernel_format, make_fp8_moe_quant_config, ) +from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import ( + fused_topk_bias, +) from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( rotate_weights_for_fi_trtllm_fp8_per_tensor_moe, @@ -887,3 +890,97 @@ def test_trtllm_fp8_block_moe_deferred_finalize( ), router_weights=topk_weights, ) + + +@pytest.mark.parametrize("m", [1, 16]) +@pytest.mark.skipif( + not current_platform.is_device_capability_family(100), + reason="Requires TRTLLM-Gen FP8 MoE (SM100)", +) +def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_init): + """FlashInfer's fused MiniMax2 routing must apply routed_scaling_factor, + matching vLLM's own sigmoid+bias top-k routing (MiniMax-M3 uses 2.0).""" + e, topk, n, k, scale = 32, 4, 1024, 1024, 2.0 + block_shape = [1, 32] + set_random_seed(7) + with set_current_vllm_config(vllm_config): + w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format( + Fp8MoeBackend.FLASHINFER_TRTLLM, + SimpleNamespace( + weight_block_size=block_shape, + moe_config=SimpleNamespace( + is_act_and_mul=True, intermediate_size_per_partition=n + ), + activation=MoEActivation.SILU, + ), + *_make_mxfp8_moe_weights(e, n, k), + w13_input_scale=None, + w2_input_scale=None, + ) + quant_config = make_fp8_moe_quant_config( + Fp8MoeBackend.FLASHINFER_TRTLLM, + w1_scale, + w2_scale, + a1_scale=None, + a2_scale=None, + block_shape=block_shape, + ) + moe_config = FusedMoEConfig( + num_experts=e, + experts_per_token=topk, + hidden_dim=k, + intermediate_size=n, + num_local_experts=e, + num_logical_experts=e, + activation=MoEActivation.SILU, + device="cuda", + moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(), + in_dtype=torch.bfloat16, + routing_method=RoutingMethodType.MiniMax2, + max_num_tokens=next_power_of_2(m), + ) + + def make_kernel(experts_cls) -> mk.FusedMoEKernel: + return mk.FusedMoEKernel( + maybe_make_prepare_finalize( + moe=moe_config, + quant_config=quant_config, + allow_new_interface=True, + use_monolithic=experts_cls is TrtLlmFp8ExpertsMonolithic, + ), + experts_cls(moe_config=moe_config, quant_config=quant_config), + ) + + a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10 + logits = torch.randn((m, e), device="cuda", dtype=torch.float32) + bias = torch.randn(e, device="cuda", dtype=torch.float32) / 10 + common = dict( + w1=w1, + w2=w2, + activation=MoEActivation.SILU, + global_num_experts=e, + expert_map=None, + apply_router_weight_on_input=False, + ) + + fused = make_kernel(TrtLlmFp8ExpertsMonolithic).apply_monolithic( + hidden_states=a, + router_logits=logits, + e_score_correction_bias=bias, + routed_scaling_factor=scale, + **common, + ) + topk_weights, topk_ids = fused_topk_bias( + a, + logits, + scoring_func="sigmoid", + e_score_correction_bias=bias, + topk=topk, + renormalize=True, + routed_scaling_factor=scale, + ) + reference = make_kernel(TrtLlmFp8ExpertsModular).apply( + hidden_states=a, topk_weights=topk_weights, topk_ids=topk_ids, **common + ) + + torch.testing.assert_close(fused, reference, atol=2e-2, rtol=2e-2) diff --git a/tests/kernels/moe/test_routing.py b/tests/kernels/moe/test_routing.py index 4baad24cc3..b0c3e26db9 100644 --- a/tests/kernels/moe/test_routing.py +++ b/tests/kernels/moe/test_routing.py @@ -167,6 +167,38 @@ def test_single_expert_group_with_non_unit_scale_uses_grouped_topk() -> None: assert isinstance(router, GroupedTopKRouter) +def test_sigmoid_bias_routing_with_routed_scale_is_minimax2() -> None: + """FlashInfer's MiniMax2 routing applies routed_scaling_factor, so a + non-unit scale (MiniMax-M3 uses 2.0) must not block fused routing.""" + router = create_fused_moe_router( + top_k=4, + global_num_experts=128, + scoring_func="sigmoid", + renormalize=True, + routed_scaling_factor=2.0, + e_score_correction_bias=torch.empty(128), + ) + + assert isinstance(router, FusedTopKBiasRouter) + assert router.routing_method_type == RoutingMethodType.MiniMax2 + + +def test_zero_expert_routing_is_unspecified() -> None: + """Zero experts are resolved in the router, so kernels with built-in + routing must never be selected for them.""" + router = create_fused_moe_router( + top_k=4, + global_num_experts=128, + scoring_func="sigmoid", + renormalize=True, + e_score_correction_bias=torch.empty(160), + zero_expert_type="identity", + num_logical_experts=160, + ) + + assert router.routing_method_type == RoutingMethodType.Unspecified + + def setup_eplb_state( enable_eplb: bool, global_num_experts: int ) -> EplbLayerState | None: diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index 9221e93824..11e2a57520 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -118,7 +118,7 @@ class RoutingMethodType(IntEnum): # SigmoidRenorm: Sigmoid -> TopK -> Renormalize (divide by sum of top-K) SigmoidRenorm = (6,) # MiniMax2: Sigmoid + Bias -> TopK -> ScaledSumNormalize - # (routeScale=1.0, epsilon=1e-20) + # (routeScale=routed_scaling_factor, epsilon=1e-20) MiniMax2 = (7,) # Sigmoid: Sigmoid -> TopK (no renormalization) Sigmoid = (8,) @@ -137,7 +137,6 @@ def get_routing_method_type( renormalize: bool, num_expert_group: int | None, has_e_score_bias: bool, - routed_scaling_factor: float | None = 1.0, ) -> RoutingMethodType: if scoring_func == "sqrtsoftplus": # DeepSeek V4 uses sqrtsoftplus routing with optional routing bias @@ -153,9 +152,7 @@ def get_routing_method_type( return RoutingMethodType.Unspecified if (num_expert_group or 0) > 0: return RoutingMethodType.DeepSeekV3 - if routed_scaling_factor in (None, 1.0): - return RoutingMethodType.MiniMax2 - return RoutingMethodType.Unspecified + return RoutingMethodType.MiniMax2 else: return RoutingMethodType.Unspecified diff --git a/vllm/model_executor/layers/fused_moe/router/cpu_router.py b/vllm/model_executor/layers/fused_moe/router/cpu_router.py index 1f96e9107a..3d034efdf0 100644 --- a/vllm/model_executor/layers/fused_moe/router/cpu_router.py +++ b/vllm/model_executor/layers/fused_moe/router/cpu_router.py @@ -198,7 +198,6 @@ class CPURouter(BaseRouter): renormalize=self.renormalize, num_expert_group=self.num_expert_group, has_e_score_bias=self.e_score_correction_bias is not None, - routed_scaling_factor=self.routed_scaling_factor, ) def _compute_routing( diff --git a/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py b/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py index 257ede27d5..2df66d063a 100644 --- a/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py +++ b/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py @@ -391,7 +391,6 @@ class FusedTopKBiasRouter(BaseRouter): renormalize=self.renormalize, num_expert_group=None, has_e_score_bias=True, - routed_scaling_factor=self.routed_scaling_factor, ) def _compute_routing( diff --git a/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py b/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py index 9b134d385a..64a12e57cd 100644 --- a/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py +++ b/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py @@ -283,7 +283,6 @@ class GroupedTopKRouter(BaseRouter): renormalize=self.renormalize, num_expert_group=self.num_expert_group, has_e_score_bias=self.e_score_correction_bias is not None, - routed_scaling_factor=self.routed_scaling_factor, ) def _compute_routing( diff --git a/vllm/model_executor/layers/fused_moe/router/router_factory.py b/vllm/model_executor/layers/fused_moe/router/router_factory.py index f03fb106f7..96ccc853b8 100644 --- a/vllm/model_executor/layers/fused_moe/router/router_factory.py +++ b/vllm/model_executor/layers/fused_moe/router/router_factory.py @@ -188,7 +188,6 @@ def create_fused_moe_router( renormalize=renormalize, num_expert_group=groups, has_e_score_bias=e_score_correction_bias is not None, - routed_scaling_factor=routed_scaling_factor, ) routing_method_preserved = advertised_routing_method( diff --git a/vllm/model_executor/layers/fused_moe/router/zero_expert_router.py b/vllm/model_executor/layers/fused_moe/router/zero_expert_router.py index 0107f88156..4e4a562c2b 100644 --- a/vllm/model_executor/layers/fused_moe/router/zero_expert_router.py +++ b/vllm/model_executor/layers/fused_moe/router/zero_expert_router.py @@ -4,10 +4,7 @@ import torch from vllm.distributed.eplb.eplb_state import EplbLayerState -from vllm.model_executor.layers.fused_moe.config import ( - RoutingMethodType, - get_routing_method_type, -) +from vllm.model_executor.layers.fused_moe.config import RoutingMethodType from vllm.model_executor.layers.fused_moe.fused_moe import ( zero_experts_compute_triton, ) @@ -53,14 +50,9 @@ class ZeroExpertRouter(BaseRouter): @property def routing_method_type(self) -> RoutingMethodType: - return get_routing_method_type( - scoring_func=self.scoring_func, - top_k=self.top_k, - renormalize=self.renormalize, - num_expert_group=None, - has_e_score_bias=True, - routed_scaling_factor=self.routed_scaling_factor, - ) + # Zero experts are resolved by this router, which kernels with + # built-in routing (e.g. FlashInfer TRT-LLM) cannot reproduce. + return RoutingMethodType.Unspecified def _compute_routing( self,