[Bugfix] Autotune FlashInfer deferred MoE decode kernels before CUDA graph capture (#55377)
Signed-off-by: ming <ming@inferact.ai> Co-authored-by: Kimi Code <kimi-code@moonshot.cn> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Kimi Code
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
eb6b619ab2
commit
7fa2c63796
@@ -0,0 +1,116 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import Mock, call, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.warmup.kernel_warmup import (
|
||||
_flashinfer_autotune_token_counts,
|
||||
_run_flashinfer_autotune_dummy_runs,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _FakeMoERunner:
|
||||
"""Minimal MoERunner stand-in for token-count discovery tests."""
|
||||
|
||||
moe_config: Any
|
||||
|
||||
|
||||
def _make_moe(*, max_deferred_tokens: int = 128, enabled: bool = True):
|
||||
"""Create a fake MoE layer with a deferred-finalize token limit."""
|
||||
moe = _FakeMoERunner()
|
||||
moe.moe_config = SimpleNamespace(
|
||||
use_deferred_moe_finalize=enabled,
|
||||
defer_moe_finalize_max_num_tokens=max_deferred_tokens,
|
||||
)
|
||||
return moe
|
||||
|
||||
|
||||
def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto"):
|
||||
"""Create a runner carrying only the state used by FlashInfer warmup."""
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
|
||||
vllm_config=SimpleNamespace(
|
||||
kernel_config=SimpleNamespace(linear_backend=linear_backend)
|
||||
),
|
||||
get_model=Mock(
|
||||
return_value=SimpleNamespace(modules=Mock(return_value=modules))
|
||||
),
|
||||
_dummy_run=Mock(),
|
||||
)
|
||||
|
||||
|
||||
def test_flashinfer_autotune_token_counts_include_deferred_moe_limits():
|
||||
runner = _make_runner(
|
||||
[
|
||||
object(),
|
||||
_make_moe(max_deferred_tokens=128),
|
||||
_make_moe(max_deferred_tokens=128),
|
||||
_make_moe(max_deferred_tokens=64, enabled=False),
|
||||
_make_moe(max_deferred_tokens=-1),
|
||||
],
|
||||
linear_backend="flashinfer_cutedsl",
|
||||
)
|
||||
|
||||
with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner):
|
||||
token_counts = _flashinfer_autotune_token_counts(runner)
|
||||
|
||||
assert token_counts == (8192, 32, 128)
|
||||
|
||||
|
||||
def test_flashinfer_autotune_token_counts_are_bounded_and_deduplicated():
|
||||
runner = _make_runner(
|
||||
[_make_moe(max_deferred_tokens=4096)],
|
||||
max_tokens=32,
|
||||
linear_backend="flashinfer_cutedsl",
|
||||
)
|
||||
|
||||
with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner):
|
||||
token_counts = _flashinfer_autotune_token_counts(runner)
|
||||
|
||||
assert token_counts == (32,)
|
||||
|
||||
|
||||
def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run():
|
||||
runner = _make_runner([])
|
||||
max_buckets = (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 8192)
|
||||
deferred_buckets = (1, 2, 4, 8, 16, 32, 64, 128)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.model_executor.warmup.kernel_warmup."
|
||||
"_flashinfer_autotune_token_counts",
|
||||
return_value=(8192, 128),
|
||||
),
|
||||
patch(
|
||||
"vllm.utils.flashinfer.flashinfer_get_hybrid_num_tokens_buckets",
|
||||
side_effect=(max_buckets, deferred_buckets),
|
||||
) as get_buckets,
|
||||
patch("vllm.utils.flashinfer.autotune") as autotune,
|
||||
):
|
||||
_run_flashinfer_autotune_dummy_runs(runner)
|
||||
|
||||
assert get_buckets.call_args_list == [call(8192), call(128)]
|
||||
assert autotune.call_args_list == [
|
||||
call(tuning_buckets=max_buckets),
|
||||
call(tuning_buckets=deferred_buckets),
|
||||
]
|
||||
assert runner._dummy_run.call_args_list == [
|
||||
call(
|
||||
num_tokens=8192,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
),
|
||||
call(
|
||||
num_tokens=128,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
),
|
||||
]
|
||||
Reference in New Issue
Block a user