[Bugfix][MoE] Tune FlashInfer experts to scheduler token limit (#52989)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Misha Goin
2026-08-20 16:26:24 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent 4f6885fffc
commit bfb6c13499
6 changed files with 12 additions and 1 deletions
@@ -191,6 +191,7 @@ def flashinfer_trtllm_mxint4_moe(
e_score_correction_bias: torch.Tensor | None = None,
routing_method_type: int | None = None,
routing_replay_out: torch.Tensor | None = None,
tune_max_num_tokens: int = 8192,
) -> torch.Tensor:
"""
Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
@@ -211,6 +212,7 @@ def flashinfer_trtllm_mxint4_moe(
topk_group: Top-k within groups (default: None -> 0)
e_score_correction_bias: Optional routing bias. dtype: bfloat16
routing_method_type: FlashInfer RoutingMethodType enum value
tune_max_num_tokens: Maximum token count covered by autotuning.
Returns:
Output tensor from MoE layer. dtype: same as x (bfloat16)
@@ -262,7 +264,7 @@ def flashinfer_trtllm_mxint4_moe(
enable_pdl=None,
do_finalize=True,
output=None,
tune_max_num_tokens=8192,
tune_max_num_tokens=tune_max_num_tokens,
routing_replay_out=routing_replay_out,
)
if isinstance(out, (tuple, list)):