Add VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS and skip CuTeDSL fp4_gemm autotuning by default (#48268)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-07-10 18:13:08 -07:00
committed by GitHub
co-authored by Claude Fable 5
parent ed908cf0a0
commit 29fd688892
2 changed files with 46 additions and 2 deletions
+32 -2
View File
@@ -130,6 +130,25 @@ def kernel_warmup(worker: "Worker"):
cutedsl_warmup()
def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
if envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS is not None:
return set(envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS) or None
from vllm.model_executor.kernels.linear import (
FlashInferCuteDslNvFp4LinearKernel,
)
for module in runner.get_model().modules():
for holder_name in ("quant_method", "scheme"):
kernel = getattr(getattr(module, holder_name, None), "kernel", None)
# CuTe-DSL mm_fp4 tuning JIT-compiles every tactic and its
# fallback is already the heuristic; all mm_fp4 backends share
# the "fp4_gemm" op name, so skip only when cute-dsl is selected.
if isinstance(kernel, FlashInferCuteDslNvFp4LinearKernel):
return {"fp4_gemm"}
return None
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
"""
Autotune FlashInfer operations.
@@ -146,6 +165,15 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
import vllm.utils.flashinfer as fi_utils
from vllm.distributed.parallel_state import get_world_group
autotune_kwargs: dict = {}
skip_ops = _flashinfer_autotune_skip_ops(runner)
if skip_ops:
logger.info(
"Skipping FlashInfer autotuning for ops %s",
sorted(skip_ops),
)
autotune_kwargs["skip_ops"] = skip_ops
use_persistent_cache = True
# When distributed, tune on every rank so the collectives stay synchronized.
@@ -153,7 +181,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
use_persistent_cache = False
if not use_persistent_cache:
with torch.inference_mode(), fi_utils.autotune():
with torch.inference_mode(), fi_utils.autotune(**autotune_kwargs):
runner._dummy_run(
num_tokens=runner.scheduler_config.max_num_batched_tokens,
skip_eplb=True,
@@ -181,7 +209,9 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
with torch.inference_mode():
if is_leader:
with fi_utils.autotune(tune_mode=True, cache=str(cache_path)):
with fi_utils.autotune(
tune_mode=True, cache=str(cache_path), **autotune_kwargs
):
runner._dummy_run(**dummy_run_kwargs)
else:
runner._dummy_run(**dummy_run_kwargs)