diff --git a/tests/kernels/test_flashinfer_bf16_gemm.py b/tests/kernels/test_flashinfer_bf16_gemm.py index 63a5a39c2e..28520a6083 100644 --- a/tests/kernels/test_flashinfer_bf16_gemm.py +++ b/tests/kernels/test_flashinfer_bf16_gemm.py @@ -25,6 +25,8 @@ def require_flashinfer_bf16_cutedsl() -> None: (8, True, False), (16, False, True), (32, True, True), + (64, True, True), + (512, False, False), ], ) def test_flashinfer_bf16_cutedsl_correctness( diff --git a/tests/model_executor/test_flashinfer_autotune_warmup.py b/tests/model_executor/test_flashinfer_autotune_warmup.py index fe89b09d09..f7ea6847f2 100644 --- a/tests/model_executor/test_flashinfer_autotune_warmup.py +++ b/tests/model_executor/test_flashinfer_autotune_warmup.py @@ -39,12 +39,11 @@ def _make_moe(*, max_deferred_tokens: int = 128, enabled: bool = True): return moe -def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto"): +def _make_runner(modules, *, max_tokens: int = 8192): """Create a runner carrying only the state used by FlashInfer warmup.""" return SimpleNamespace( scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens), vllm_config=SimpleNamespace( - kernel_config=SimpleNamespace(linear_backend=linear_backend), attention_config=SimpleNamespace(hisparse_config=None), parallel_config=SimpleNamespace(data_parallel_rank=0), ), @@ -63,21 +62,19 @@ def test_flashinfer_autotune_token_counts_include_deferred_moe_limits(): _make_moe(max_deferred_tokens=128), _make_moe(max_deferred_tokens=64, enabled=False), _make_moe(max_deferred_tokens=-1), - ], - linear_backend="flashinfer_cutedsl", + ] ) with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner): token_counts = _flashinfer_autotune_token_counts(runner) - assert token_counts == (8192, 32, 128) + assert token_counts == (8192, 128) def test_flashinfer_autotune_token_counts_are_bounded_and_deduplicated(): runner = _make_runner( [_make_moe(max_deferred_tokens=4096)], max_tokens=32, - linear_backend="flashinfer_cutedsl", ) with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner): diff --git a/vllm/model_executor/layers/utils.py b/vllm/model_executor/layers/utils.py index d4ee153fb2..80eef7c75a 100644 --- a/vllm/model_executor/layers/utils.py +++ b/vllm/model_executor/layers/utils.py @@ -128,16 +128,13 @@ def _can_use_flashinfer_cutedsl_bf16( if ( k <= 0 or n <= 0 + or x.numel() == 0 or weight.shape[1] != k or k % 128 != 0 or x.data_ptr() % 32 != 0 or weight.data_ptr() % 32 != 0 ): return False - - m = x.numel() // k - if not 1 <= m <= 32: - return False return bias is None or ( bias.is_cuda and bias.device == x.device diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index d6544333be..e5751730e8 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -316,9 +316,6 @@ def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None: return None -_FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS = 32 - - def _flashinfer_deferred_moe_token_counts( runner: "GPUModelRunner", ) -> tuple[int, ...]: @@ -339,19 +336,10 @@ def _flashinfer_deferred_moe_token_counts( return tuple(dict.fromkeys(token_counts)) -def _flashinfer_autotune_token_counts( - runner: "GPUModelRunner", *, include_bf16: bool = True -) -> tuple[int, ...]: +def _flashinfer_autotune_token_counts(runner: "GPUModelRunner") -> tuple[int, ...]: max_tokens = runner.scheduler_config.max_num_batched_tokens # Tune the widest bucket set first so bounded passes reuse its configs. token_counts = [max_tokens] - linear_backend = runner.vllm_config.kernel_config.linear_backend - if ( - include_bf16 - and linear_backend == "flashinfer_cutedsl" - and max_tokens > _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS - ): - token_counts.append(_FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS) token_counts.extend(_flashinfer_deferred_moe_token_counts(runner)) return tuple(dict.fromkeys(token_counts)) @@ -362,7 +350,7 @@ def _run_flashinfer_autotune_dummy_runs( import vllm.utils.flashinfer as fi_utils dummy_run_kwargs = {"skip_attn": True} if skip_attn else {} - for num_tokens in _flashinfer_autotune_token_counts(runner, include_bf16=False): + for num_tokens in _flashinfer_autotune_token_counts(runner): # The drafter may use more tokens than this pass in the same dummy run. # Include its max M when building the autotune buckets. max_tuning_tokens = num_tokens @@ -398,36 +386,6 @@ def _run_flashinfer_autotune_dummy_runs( ) -def _run_flashinfer_bf16_autotune_dummy_run( - runner: "GPUModelRunner", - *, - skip_ops: set[str] | None = None, - skip_attn: bool = False, -) -> None: - import vllm.utils.flashinfer as fi_utils - - if ( - runner.vllm_config.kernel_config.linear_backend != "flashinfer_cutedsl" - or runner.scheduler_config.max_num_batched_tokens - <= _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS - ): - return - - num_tokens = _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS - tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(num_tokens) - logger.info("Running FlashInfer BF16-only autotune with %d tokens.", num_tokens) - # Other ops execute from their existing cache or fallback, without the - # BF16 bucket cap. No full-model autotune context may enclose this pass. - with fi_utils.autotune_bf16_only(tuning_buckets, skip_ops=skip_ops): - runner._dummy_run( - num_tokens=num_tokens, - skip_eplb=True, - is_profile=True, - randomize_inputs=True, - **({"skip_attn": True} if skip_attn else {}), - ) - - def _autotune_cache_fingerprint(path: Path) -> tuple[str, int] | None: """Identify a saved autotune file by its FlashInfer metadata and size.""" try: @@ -534,10 +492,6 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: _run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled) replayssm_autotune_warmup(runner) _autotune_kimi_k3_kda_qkvg(runner.get_model()) - with torch.inference_mode(): - _run_flashinfer_bf16_autotune_dummy_run( - runner, skip_ops=skip_ops, skip_attn=hisparse_enabled - ) finally: set_autotune_process_group(None) diff --git a/vllm/utils/flashinfer.py b/vllm/utils/flashinfer.py index d7fa807c33..f9d6c4439f 100644 --- a/vllm/utils/flashinfer.py +++ b/vllm/utils/flashinfer.py @@ -11,8 +11,7 @@ import importlib import importlib.util import os import shutil -from collections.abc import Callable, Iterator -from contextvars import ContextVar +from collections.abc import Callable from typing import Any, NoReturn import requests @@ -27,24 +26,6 @@ from vllm.utils.torch_utils import PIN_MEMORY logger = init_logger(__name__) -_bf16_autotune_buckets: ContextVar[tuple[int, ...] | None] = ContextVar( - "flashinfer_bf16_autotune_buckets", default=None -) - - -@contextlib.contextmanager -def autotune_bf16_only( - tuning_buckets: tuple[int, ...], *, skip_ops: set[str] | None = None -) -> Iterator[None]: - """Tune BF16 calls with bounded buckets, outside full-model autotuning.""" - token = _bf16_autotune_buckets.set(tuning_buckets) - try: - with autotune(tune_mode=False, skip_ops=skip_ops): - yield - finally: - _bf16_autotune_buckets.reset(token) - - def flashinfer_bf16_mm( a: torch.Tensor, b: torch.Tensor, @@ -54,21 +35,14 @@ def flashinfer_bf16_mm( ) -> torch.Tensor: from flashinfer import mm_bf16 - tuning_buckets = _bf16_autotune_buckets.get() - tuning = ( - autotune(tune_mode=True, tuning_buckets=tuning_buckets) - if tuning_buckets is not None - else contextlib.nullcontext() + return mm_bf16( + a, + b, + bias=bias, + pdl=pdl, + out_dtype=torch.bfloat16, + backend=backend, ) - with tuning: - return mm_bf16( - a, - b, - bias=bias, - pdl=pdl, - out_dtype=torch.bfloat16, - backend=backend, - ) # This is the storage path for the cubins, it can be replaced @@ -1295,7 +1269,6 @@ def is_flashinfer_cudnn_fp8_prefill_attn_supported() -> bool: __all__ = [ "has_flashinfer", "flashinfer_bf16_mm", - "autotune_bf16_only", "has_flashinfer_bf16_gemm", "is_flashinfer_bf16_gemm_supported", "is_flashinfer_cutedsl_bf16_gemm_supported",