[Perf][MoE] Use fused MiniMax2 routing with non-unit routed scaling (#58880)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
@@ -42,6 +42,9 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
fused_topk_bias,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
rotate_weights_for_fi_trtllm_fp8_per_tensor_moe,
|
||||
@@ -887,3 +890,97 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
|
||||
),
|
||||
router_weights=topk_weights,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", [1, 16])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
|
||||
)
|
||||
def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_init):
|
||||
"""FlashInfer's fused MiniMax2 routing must apply routed_scaling_factor,
|
||||
matching vLLM's own sigmoid+bias top-k routing (MiniMax-M3 uses 2.0)."""
|
||||
e, topk, n, k, scale = 32, 4, 1024, 1024, 2.0
|
||||
block_shape = [1, 32]
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
SimpleNamespace(
|
||||
weight_block_size=block_shape,
|
||||
moe_config=SimpleNamespace(
|
||||
is_act_and_mul=True, intermediate_size_per_partition=n
|
||||
),
|
||||
activation=MoEActivation.SILU,
|
||||
),
|
||||
*_make_mxfp8_moe_weights(e, n, k),
|
||||
w13_input_scale=None,
|
||||
w2_input_scale=None,
|
||||
)
|
||||
quant_config = make_fp8_moe_quant_config(
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
num_local_experts=e,
|
||||
num_logical_experts=e,
|
||||
activation=MoEActivation.SILU,
|
||||
device="cuda",
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
routing_method=RoutingMethodType.MiniMax2,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
|
||||
def make_kernel(experts_cls) -> mk.FusedMoEKernel:
|
||||
return mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=experts_cls is TrtLlmFp8ExpertsMonolithic,
|
||||
),
|
||||
experts_cls(moe_config=moe_config, quant_config=quant_config),
|
||||
)
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
|
||||
logits = torch.randn((m, e), device="cuda", dtype=torch.float32)
|
||||
bias = torch.randn(e, device="cuda", dtype=torch.float32) / 10
|
||||
common = dict(
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
fused = make_kernel(TrtLlmFp8ExpertsMonolithic).apply_monolithic(
|
||||
hidden_states=a,
|
||||
router_logits=logits,
|
||||
e_score_correction_bias=bias,
|
||||
routed_scaling_factor=scale,
|
||||
**common,
|
||||
)
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
a,
|
||||
logits,
|
||||
scoring_func="sigmoid",
|
||||
e_score_correction_bias=bias,
|
||||
topk=topk,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=scale,
|
||||
)
|
||||
reference = make_kernel(TrtLlmFp8ExpertsModular).apply(
|
||||
hidden_states=a, topk_weights=topk_weights, topk_ids=topk_ids, **common
|
||||
)
|
||||
|
||||
torch.testing.assert_close(fused, reference, atol=2e-2, rtol=2e-2)
|
||||
|
||||
@@ -167,6 +167,38 @@ def test_single_expert_group_with_non_unit_scale_uses_grouped_topk() -> None:
|
||||
assert isinstance(router, GroupedTopKRouter)
|
||||
|
||||
|
||||
def test_sigmoid_bias_routing_with_routed_scale_is_minimax2() -> None:
|
||||
"""FlashInfer's MiniMax2 routing applies routed_scaling_factor, so a
|
||||
non-unit scale (MiniMax-M3 uses 2.0) must not block fused routing."""
|
||||
router = create_fused_moe_router(
|
||||
top_k=4,
|
||||
global_num_experts=128,
|
||||
scoring_func="sigmoid",
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.0,
|
||||
e_score_correction_bias=torch.empty(128),
|
||||
)
|
||||
|
||||
assert isinstance(router, FusedTopKBiasRouter)
|
||||
assert router.routing_method_type == RoutingMethodType.MiniMax2
|
||||
|
||||
|
||||
def test_zero_expert_routing_is_unspecified() -> None:
|
||||
"""Zero experts are resolved in the router, so kernels with built-in
|
||||
routing must never be selected for them."""
|
||||
router = create_fused_moe_router(
|
||||
top_k=4,
|
||||
global_num_experts=128,
|
||||
scoring_func="sigmoid",
|
||||
renormalize=True,
|
||||
e_score_correction_bias=torch.empty(160),
|
||||
zero_expert_type="identity",
|
||||
num_logical_experts=160,
|
||||
)
|
||||
|
||||
assert router.routing_method_type == RoutingMethodType.Unspecified
|
||||
|
||||
|
||||
def setup_eplb_state(
|
||||
enable_eplb: bool, global_num_experts: int
|
||||
) -> EplbLayerState | None:
|
||||
|
||||
@@ -118,7 +118,7 @@ class RoutingMethodType(IntEnum):
|
||||
# SigmoidRenorm: Sigmoid -> TopK -> Renormalize (divide by sum of top-K)
|
||||
SigmoidRenorm = (6,)
|
||||
# MiniMax2: Sigmoid + Bias -> TopK -> ScaledSumNormalize
|
||||
# (routeScale=1.0, epsilon=1e-20)
|
||||
# (routeScale=routed_scaling_factor, epsilon=1e-20)
|
||||
MiniMax2 = (7,)
|
||||
# Sigmoid: Sigmoid -> TopK (no renormalization)
|
||||
Sigmoid = (8,)
|
||||
@@ -137,7 +137,6 @@ def get_routing_method_type(
|
||||
renormalize: bool,
|
||||
num_expert_group: int | None,
|
||||
has_e_score_bias: bool,
|
||||
routed_scaling_factor: float | None = 1.0,
|
||||
) -> RoutingMethodType:
|
||||
if scoring_func == "sqrtsoftplus":
|
||||
# DeepSeek V4 uses sqrtsoftplus routing with optional routing bias
|
||||
@@ -153,9 +152,7 @@ def get_routing_method_type(
|
||||
return RoutingMethodType.Unspecified
|
||||
if (num_expert_group or 0) > 0:
|
||||
return RoutingMethodType.DeepSeekV3
|
||||
if routed_scaling_factor in (None, 1.0):
|
||||
return RoutingMethodType.MiniMax2
|
||||
return RoutingMethodType.Unspecified
|
||||
return RoutingMethodType.MiniMax2
|
||||
else:
|
||||
return RoutingMethodType.Unspecified
|
||||
|
||||
|
||||
@@ -198,7 +198,6 @@ class CPURouter(BaseRouter):
|
||||
renormalize=self.renormalize,
|
||||
num_expert_group=self.num_expert_group,
|
||||
has_e_score_bias=self.e_score_correction_bias is not None,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
def _compute_routing(
|
||||
|
||||
@@ -391,7 +391,6 @@ class FusedTopKBiasRouter(BaseRouter):
|
||||
renormalize=self.renormalize,
|
||||
num_expert_group=None,
|
||||
has_e_score_bias=True,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
def _compute_routing(
|
||||
|
||||
@@ -283,7 +283,6 @@ class GroupedTopKRouter(BaseRouter):
|
||||
renormalize=self.renormalize,
|
||||
num_expert_group=self.num_expert_group,
|
||||
has_e_score_bias=self.e_score_correction_bias is not None,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
def _compute_routing(
|
||||
|
||||
@@ -188,7 +188,6 @@ def create_fused_moe_router(
|
||||
renormalize=renormalize,
|
||||
num_expert_group=groups,
|
||||
has_e_score_bias=e_score_correction_bias is not None,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
routing_method_preserved = advertised_routing_method(
|
||||
|
||||
@@ -4,10 +4,7 @@
|
||||
import torch
|
||||
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
RoutingMethodType,
|
||||
get_routing_method_type,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import (
|
||||
zero_experts_compute_triton,
|
||||
)
|
||||
@@ -53,14 +50,9 @@ class ZeroExpertRouter(BaseRouter):
|
||||
|
||||
@property
|
||||
def routing_method_type(self) -> RoutingMethodType:
|
||||
return get_routing_method_type(
|
||||
scoring_func=self.scoring_func,
|
||||
top_k=self.top_k,
|
||||
renormalize=self.renormalize,
|
||||
num_expert_group=None,
|
||||
has_e_score_bias=True,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
# Zero experts are resolved by this router, which kernels with
|
||||
# built-in routing (e.g. FlashInfer TRT-LLM) cannot reproduce.
|
||||
return RoutingMethodType.Unspecified
|
||||
|
||||
def _compute_routing(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user