diff --git a/tests/kernels/moe/test_routing.py b/tests/kernels/moe/test_routing.py index 62a4968a0d..9b12fef045 100644 --- a/tests/kernels/moe/test_routing.py +++ b/tests/kernels/moe/test_routing.py @@ -8,9 +8,19 @@ import torch from vllm._aiter_ops import rocm_aiter_ops from vllm.distributed.eplb.eplb_state import EplbLayerState +from vllm.model_executor.layers.fused_moe.config import RoutingMethodType from vllm.model_executor.layers.fused_moe.router.base_router import ( eplb_map_to_physical_and_record, ) +from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import ( + FusedTopKBiasRouter, +) +from vllm.model_executor.layers.fused_moe.router.fused_topk_router import ( + FusedTopKRouter, +) +from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import ( + GroupedTopKRouter, +) from vllm.model_executor.layers.fused_moe.router.router_factory import ( create_fused_moe_router, ) @@ -36,6 +46,112 @@ TOP_KS = [2, 4, 6] NUM_EXPERTS = [8, 16, 64] +def test_degenerate_grouped_config_uses_standard_topk() -> None: + router = create_fused_moe_router( + top_k=4, + global_num_experts=128, + use_grouped_topk=True, + num_expert_group=1, + topk_group=1, + scoring_func="softmax", + renormalize=True, + ) + + assert isinstance(router, FusedTopKRouter) + hidden_states, router_logits = make_test_data(32, 256, 128) + + topk_weights, topk_ids = router.select_experts(hidden_states, router_logits) + baseline_weights, baseline_ids = baseline_fused_topk( + router_logits, + top_k=4, + renormalize=True, + ) + + assert_routing_results_close( + topk_weights, + topk_ids, + baseline_weights, + baseline_ids, + ) + + +def test_multiple_expert_groups_use_grouped_topk() -> None: + router = create_fused_moe_router( + top_k=4, + global_num_experts=128, + use_grouped_topk=True, + num_expert_group=8, + topk_group=4, + scoring_func="softmax", + renormalize=True, + ) + + assert isinstance(router, GroupedTopKRouter) + + +def test_degenerate_grouped_config_with_bias_uses_topk_bias() -> None: + router = create_fused_moe_router( + top_k=4, + global_num_experts=128, + use_grouped_topk=True, + num_expert_group=1, + topk_group=1, + scoring_func="softmax", + renormalize=True, + e_score_correction_bias=torch.empty(128), + ) + + assert isinstance(router, FusedTopKBiasRouter) + + +def test_degenerate_grouped_config_with_bias_keeps_routed_scale() -> None: + router = create_fused_moe_router( + top_k=4, + global_num_experts=128, + use_grouped_topk=True, + num_expert_group=1, + topk_group=1, + scoring_func="softmax", + renormalize=True, + routed_scaling_factor=1.1, + e_score_correction_bias=torch.empty(128), + ) + + assert isinstance(router, FusedTopKBiasRouter) + assert router.routed_scaling_factor == 1.1 + + +def test_degenerate_deepseek_v3_routing_stays_grouped() -> None: + router = create_fused_moe_router( + top_k=4, + global_num_experts=128, + use_grouped_topk=True, + num_expert_group=1, + topk_group=1, + scoring_func="sigmoid", + renormalize=True, + e_score_correction_bias=torch.empty(128), + ) + + assert isinstance(router, GroupedTopKRouter) + assert router.routing_method_type == RoutingMethodType.DeepSeekV3 + + +def test_single_expert_group_with_non_unit_scale_uses_grouped_topk() -> None: + router = create_fused_moe_router( + top_k=4, + global_num_experts=128, + use_grouped_topk=True, + num_expert_group=1, + topk_group=1, + scoring_func="softmax", + renormalize=True, + routed_scaling_factor=1.1, + ) + + assert isinstance(router, GroupedTopKRouter) + + def setup_eplb_state( enable_eplb: bool, global_num_experts: int ) -> EplbLayerState | None: diff --git a/vllm/model_executor/layers/fused_moe/router/router_factory.py b/vllm/model_executor/layers/fused_moe/router/router_factory.py index c7cfccbe64..ff4874b20e 100644 --- a/vllm/model_executor/layers/fused_moe/router/router_factory.py +++ b/vllm/model_executor/layers/fused_moe/router/router_factory.py @@ -9,6 +9,7 @@ from vllm._aiter_ops import rocm_aiter_ops from vllm.distributed.eplb.eplb_state import EplbLayerState from vllm.model_executor.layers.fused_moe.config import ( RoutingMethodType, + get_routing_method_type, ) from vllm.model_executor.layers.fused_moe.router.aiter_shared_routed_fused_moe_router import ( # noqa: E501 AiterSharedRoutedFusedMoERouter, @@ -67,7 +68,8 @@ def create_fused_moe_router( The selection logic follows this priority order: 1. RoutingSimulatorRouter - if VLLM_MOE_ROUTING_SIMULATION_STRATEGY env var is set 2. ZeroExpertRouter - if zero_expert_type is not None - 3. GroupedTopKRouter - if use_grouped_topk is True + 3. GroupedTopKRouter - if use_grouped_topk is True and the grouping is not + degenerate (at most one group, with topk_group <= 1) 4. CustomRoutingRouter - if custom_routing_function is not None 5. FusedTopKBiasRouter - if e_score_correction_bias is not None 6. AiterSharedRoutedFusedMoERouter - if num_fused_shared_experts > 0 @@ -143,30 +145,48 @@ def create_fused_moe_router( "num_expert_group and topk_group must be provided when " "use_grouped_topk is True" ) - grouped_topk_router = GroupedTopKRouter( - top_k=top_k, - global_num_experts=global_num_experts, - eplb_state=eplb_state, - num_expert_group=num_expert_group, - topk_group=topk_group, - renormalize=renormalize, - scoring_func=scoring_func, - routed_scaling_factor=routed_scaling_factor, - e_score_correction_bias=e_score_correction_bias, - num_fused_shared_experts=num_fused_shared_experts, - ) - if ( - grouped_topk_router.routing_method_type != RoutingMethodType.Unspecified - or num_expert_group > 1 - or topk_group > 1 - ): - return grouped_topk_router - # If routing_method for GroupedTopKRouter is Unspecified and there is only - # one group, fallback to standard top-k routing - use_grouped_topk = False - num_expert_group = None - topk_group = None + # For topk_group <= 1, grouped implementation is pure overhead. + degenerate_grouping = num_expert_group <= 1 and topk_group <= 1 + # FusedTopKRouter cannot apply routed_scaling_factor, FusedTopKBiasRouter can. + scaling_handled_downstream = ( + routed_scaling_factor == 1.0 or e_score_correction_bias is not None + ) + + # Degenerating must not change the advertised routing method, which drives + # kernel selection. num_expert_group only affects it for biased routing. + def advertised_routing_method(groups: int | None) -> RoutingMethodType: + return get_routing_method_type( + scoring_func=scoring_func, + top_k=top_k, + renormalize=renormalize, + num_expert_group=groups, + has_e_score_bias=e_score_correction_bias is not None, + routed_scaling_factor=routed_scaling_factor, + ) + + routing_method_preserved = advertised_routing_method( + num_expert_group + ) == advertised_routing_method(None) + + if not ( + degenerate_grouping + and scaling_handled_downstream + and routing_method_preserved + ): + return GroupedTopKRouter( + top_k=top_k, + global_num_experts=global_num_experts, + eplb_state=eplb_state, + num_expert_group=num_expert_group, + topk_group=topk_group, + renormalize=renormalize, + scoring_func=scoring_func, + routed_scaling_factor=routed_scaling_factor, + e_score_correction_bias=e_score_correction_bias, + num_fused_shared_experts=num_fused_shared_experts, + ) + # Otherwise fall through to the non-grouped chain below. if custom_routing_function is not None: return CustomRoutingRouter(