[MoE] FI autotuning: max bucket = max token count [e.g. DP_size*MNBT] (#47427)

Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Netanel Haber
2026-07-07 12:08:36 +03:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5d23ca47ab
commit 066f02ae94
4 changed files with 38 additions and 2 deletions
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from math import prod
from typing import TYPE_CHECKING
import torch
import torch.nn.functional as F
@@ -33,6 +34,9 @@ from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
@triton.jit
def _count_expert_num_tokens(
@@ -403,6 +407,23 @@ def _pack_topk_ids_weights_kernel(
tl.store(output_ptr + offsets, packed, mask=mask)
def fi_moe_largest_bucket(moe_config: "FusedMoEConfig") -> int:
"""Estimate FlashInfer's MoE autotuning maximum token count.
All DP ranks may contribute `max_num_tokens` to one invocation.
Keep FlashInfer's default moe `tune_max_num_tokens=8192`
floor to avoid over-underestimation.
DeepEP, SP, or PCP may make this underestimate, however overestimation
may be dangerous, increasing tuning- cost and memory use.
NOTE: The DP factor applies even when EP is disabled:
> Without `--enable-expert-parallel`, MoE layers would use tensor parallelism.
For a detailed explanation, see: `docs/serving/data_parallel_deployment.md`
"""
return max(moe_config.max_num_tokens * moe_config.dp_size, 8192)
def trtllm_moe_pack_topk_ids_weights(
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,