[Bugfix] Autotune FlashInfer deferred MoE decode kernels before CUDA graph capture (#55377)

Signed-off-by: ming <ming@inferact.ai>
Co-authored-by: Kimi Code <kimi-code@moonshot.cn>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
mingg26
2026-09-08 04:02:23 +00:00
committed by GitHub
co-authored by Kimi Code mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent eb6b619ab2
commit 7fa2c63796
3 changed files with 159 additions and 8 deletions
@@ -0,0 +1,116 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import Any
from unittest.mock import Mock, call, patch
import pytest
from vllm.model_executor.warmup.kernel_warmup import (
_flashinfer_autotune_token_counts,
_run_flashinfer_autotune_dummy_runs,
)
pytestmark = pytest.mark.cpu_test
class _FakeMoERunner:
"""Minimal MoERunner stand-in for token-count discovery tests."""
moe_config: Any
def _make_moe(*, max_deferred_tokens: int = 128, enabled: bool = True):
"""Create a fake MoE layer with a deferred-finalize token limit."""
moe = _FakeMoERunner()
moe.moe_config = SimpleNamespace(
use_deferred_moe_finalize=enabled,
defer_moe_finalize_max_num_tokens=max_deferred_tokens,
)
return moe
def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto"):
"""Create a runner carrying only the state used by FlashInfer warmup."""
return SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
vllm_config=SimpleNamespace(
kernel_config=SimpleNamespace(linear_backend=linear_backend)
),
get_model=Mock(
return_value=SimpleNamespace(modules=Mock(return_value=modules))
),
_dummy_run=Mock(),
)
def test_flashinfer_autotune_token_counts_include_deferred_moe_limits():
runner = _make_runner(
[
object(),
_make_moe(max_deferred_tokens=128),
_make_moe(max_deferred_tokens=128),
_make_moe(max_deferred_tokens=64, enabled=False),
_make_moe(max_deferred_tokens=-1),
],
linear_backend="flashinfer_cutedsl",
)
with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner):
token_counts = _flashinfer_autotune_token_counts(runner)
assert token_counts == (8192, 32, 128)
def test_flashinfer_autotune_token_counts_are_bounded_and_deduplicated():
runner = _make_runner(
[_make_moe(max_deferred_tokens=4096)],
max_tokens=32,
linear_backend="flashinfer_cutedsl",
)
with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner):
token_counts = _flashinfer_autotune_token_counts(runner)
assert token_counts == (32,)
def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run():
runner = _make_runner([])
max_buckets = (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 8192)
deferred_buckets = (1, 2, 4, 8, 16, 32, 64, 128)
with (
patch(
"vllm.model_executor.warmup.kernel_warmup."
"_flashinfer_autotune_token_counts",
return_value=(8192, 128),
),
patch(
"vllm.utils.flashinfer.flashinfer_get_hybrid_num_tokens_buckets",
side_effect=(max_buckets, deferred_buckets),
) as get_buckets,
patch("vllm.utils.flashinfer.autotune") as autotune,
):
_run_flashinfer_autotune_dummy_runs(runner)
assert get_buckets.call_args_list == [call(8192), call(128)]
assert autotune.call_args_list == [
call(tuning_buckets=max_buckets),
call(tuning_buckets=deferred_buckets),
]
assert runner._dummy_run.call_args_list == [
call(
num_tokens=8192,
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
),
call(
num_tokens=128,
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
),
]