[Bugfix][MoE] Tune FlashInfer experts to scheduler token limit (#52989)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Misha Goin
2026-08-20 16:26:24 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent 4f6885fffc
commit bfb6c13499
6 changed files with 12 additions and 1 deletions
+2
View File
@@ -72,6 +72,7 @@ def test_flashinfer_swigluoai_params_are_forwarded(activation, monkeypatch):
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=torch.bfloat16,
routing_method=RoutingMethodType.TopK,
max_num_tokens=16_384,
)
quant_config = FusedMoEQuantConfig.make(
gemm1_alpha=1.702,
@@ -110,6 +111,7 @@ def test_flashinfer_swigluoai_params_are_forwarded(activation, monkeypatch):
assert experts._supports_activation(activation)
assert call_args["activation_type"] == ActivationType.Swiglu
assert call_args["tune_max_num_tokens"] == 16_384
for name, value in (
("swiglu_alpha", 1.702),
("swiglu_beta", 1.0),
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
activation_to_flashinfer_type,
)
@@ -388,6 +389,7 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
use_deepseek_fp8_block_scale=self.use_deepseek_fp8_block_scale,
use_mxfp8_act_scaling=use_mxfp8_act_scaling,
use_w4_group_scaling=use_w4_group_scaling,
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
)
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
@@ -219,6 +219,7 @@ class TrtLlmBf16ExpertsModular(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsModular)
weight_layout=WeightLayout.BlockMajorK,
do_finalize=True,
activation_type=activation_to_flashinfer_int(activation),
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
)
# FlashInfer's BF16 routed wrapper does not expose an output= argument.
output.copy_(result[0] if isinstance(result, list) else result)
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import (
fi_moe_largest_bucket,
trtllm_moe_pack_topk_ids_weights,
)
from vllm.platforms import current_platform
@@ -549,6 +550,7 @@ class TrtLlmBf16LoRAExperts(_TrtLlmLoRAExpertsBase):
routing_method_type=self.routing_method_type,
do_finalize=do_finalize,
output=output if do_finalize else None,
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
)
if not do_finalize:
# [gemm2_output, expert_weights, expanded_idx, gemm1_activation]
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt4Static32,
@@ -169,6 +170,7 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias=e_score_correction_bias,
routing_method_type=self.routing_method,
routing_replay_out=routing_replay_out,
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
@@ -191,6 +191,7 @@ def flashinfer_trtllm_mxint4_moe(
e_score_correction_bias: torch.Tensor | None = None,
routing_method_type: int | None = None,
routing_replay_out: torch.Tensor | None = None,
tune_max_num_tokens: int = 8192,
) -> torch.Tensor:
"""
Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
@@ -211,6 +212,7 @@ def flashinfer_trtllm_mxint4_moe(
topk_group: Top-k within groups (default: None -> 0)
e_score_correction_bias: Optional routing bias. dtype: bfloat16
routing_method_type: FlashInfer RoutingMethodType enum value
tune_max_num_tokens: Maximum token count covered by autotuning.
Returns:
Output tensor from MoE layer. dtype: same as x (bfloat16)
@@ -262,7 +264,7 @@ def flashinfer_trtllm_mxint4_moe(
enable_pdl=None,
do_finalize=True,
output=None,
tune_max_num_tokens=8192,
tune_max_num_tokens=tune_max_num_tokens,
routing_replay_out=routing_replay_out,
)
if isinstance(out, (tuple, list)):