Add VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS and skip CuTeDSL fp4_gemm autotuning by default (#48268)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-07-10 18:13:08 -07:00
committed by GitHub
co-authored by Claude Fable 5
parent ed908cf0a0
commit 29fd688892
2 changed files with 46 additions and 2 deletions
+14
View File
@@ -197,6 +197,7 @@ if TYPE_CHECKING:
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS: list[str] | None = None
VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto"
VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024
VLLM_XGRAMMAR_CACHE_MB: int = 0
@@ -1594,6 +1595,18 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR": lambda: os.getenv(
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", None
),
# Comma-separated FlashInfer op names to exclude from autotuning, using
# the heuristic fallback tactic instead. Unset: skip "fp4_gemm" when the
# CuTe-DSL NVFP4 linear kernel is selected. Empty: skip nothing.
"VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS": lambda: (
None
if "VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS" not in os.environ
else [
v.strip()
for v in os.environ["VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS"].split(",")
if v.strip()
]
),
# Flashinfer fused allreduce backend.
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": env_with_choices(
"VLLM_FLASHINFER_ALLREDUCE_BACKEND",
@@ -2110,6 +2123,7 @@ def compile_factors() -> dict[str, object]:
"VLLM_DEBUG_LOG_API_SERVER_RESPONSE",
"VLLM_TUNED_CONFIG_FOLDER",
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR",
"VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS",
"VLLM_ENGINE_ITERATION_TIMEOUT_S",
"VLLM_HTTP_TIMEOUT_KEEP_ALIVE",
"VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS",
+32 -2
View File
@@ -130,6 +130,25 @@ def kernel_warmup(worker: "Worker"):
cutedsl_warmup()
def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
if envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS is not None:
return set(envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS) or None
from vllm.model_executor.kernels.linear import (
FlashInferCuteDslNvFp4LinearKernel,
)
for module in runner.get_model().modules():
for holder_name in ("quant_method", "scheme"):
kernel = getattr(getattr(module, holder_name, None), "kernel", None)
# CuTe-DSL mm_fp4 tuning JIT-compiles every tactic and its
# fallback is already the heuristic; all mm_fp4 backends share
# the "fp4_gemm" op name, so skip only when cute-dsl is selected.
if isinstance(kernel, FlashInferCuteDslNvFp4LinearKernel):
return {"fp4_gemm"}
return None
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
"""
Autotune FlashInfer operations.
@@ -146,6 +165,15 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
import vllm.utils.flashinfer as fi_utils
from vllm.distributed.parallel_state import get_world_group
autotune_kwargs: dict = {}
skip_ops = _flashinfer_autotune_skip_ops(runner)
if skip_ops:
logger.info(
"Skipping FlashInfer autotuning for ops %s",
sorted(skip_ops),
)
autotune_kwargs["skip_ops"] = skip_ops
use_persistent_cache = True
# When distributed, tune on every rank so the collectives stay synchronized.
@@ -153,7 +181,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
use_persistent_cache = False
if not use_persistent_cache:
with torch.inference_mode(), fi_utils.autotune():
with torch.inference_mode(), fi_utils.autotune(**autotune_kwargs):
runner._dummy_run(
num_tokens=runner.scheduler_config.max_num_batched_tokens,
skip_eplb=True,
@@ -181,7 +209,9 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
with torch.inference_mode():
if is_leader:
with fi_utils.autotune(tune_mode=True, cache=str(cache_path)):
with fi_utils.autotune(
tune_mode=True, cache=str(cache_path), **autotune_kwargs
):
runner._dummy_run(**dummy_run_kwargs)
else:
runner._dummy_run(**dummy_run_kwargs)