diff --git a/vllm/envs.py b/vllm/envs.py index 9305bb9cbe..33b9f2f14f 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -197,6 +197,7 @@ if TYPE_CHECKING: VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True VLLM_USE_FLASHINFER_MOE_INT4: bool = False VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None + VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS: list[str] | None = None VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto" VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024 VLLM_XGRAMMAR_CACHE_MB: int = 0 @@ -1594,6 +1595,18 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR": lambda: os.getenv( "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", None ), + # Comma-separated FlashInfer op names to exclude from autotuning, using + # the heuristic fallback tactic instead. Unset: skip "fp4_gemm" when the + # CuTe-DSL NVFP4 linear kernel is selected. Empty: skip nothing. + "VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS": lambda: ( + None + if "VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS" not in os.environ + else [ + v.strip() + for v in os.environ["VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS"].split(",") + if v.strip() + ] + ), # Flashinfer fused allreduce backend. "VLLM_FLASHINFER_ALLREDUCE_BACKEND": env_with_choices( "VLLM_FLASHINFER_ALLREDUCE_BACKEND", @@ -2110,6 +2123,7 @@ def compile_factors() -> dict[str, object]: "VLLM_DEBUG_LOG_API_SERVER_RESPONSE", "VLLM_TUNED_CONFIG_FOLDER", "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", + "VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS", "VLLM_ENGINE_ITERATION_TIMEOUT_S", "VLLM_HTTP_TIMEOUT_KEEP_ALIVE", "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index b7f3c26570..d96a2e594b 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -130,6 +130,25 @@ def kernel_warmup(worker: "Worker"): cutedsl_warmup() +def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None: + if envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS is not None: + return set(envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS) or None + + from vllm.model_executor.kernels.linear import ( + FlashInferCuteDslNvFp4LinearKernel, + ) + + for module in runner.get_model().modules(): + for holder_name in ("quant_method", "scheme"): + kernel = getattr(getattr(module, holder_name, None), "kernel", None) + # CuTe-DSL mm_fp4 tuning JIT-compiles every tactic and its + # fallback is already the heuristic; all mm_fp4 backends share + # the "fp4_gemm" op name, so skip only when cute-dsl is selected. + if isinstance(kernel, FlashInferCuteDslNvFp4LinearKernel): + return {"fp4_gemm"} + return None + + def flashinfer_autotune(runner: "GPUModelRunner") -> None: """ Autotune FlashInfer operations. @@ -146,6 +165,15 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: import vllm.utils.flashinfer as fi_utils from vllm.distributed.parallel_state import get_world_group + autotune_kwargs: dict = {} + skip_ops = _flashinfer_autotune_skip_ops(runner) + if skip_ops: + logger.info( + "Skipping FlashInfer autotuning for ops %s", + sorted(skip_ops), + ) + autotune_kwargs["skip_ops"] = skip_ops + use_persistent_cache = True # When distributed, tune on every rank so the collectives stay synchronized. @@ -153,7 +181,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: use_persistent_cache = False if not use_persistent_cache: - with torch.inference_mode(), fi_utils.autotune(): + with torch.inference_mode(), fi_utils.autotune(**autotune_kwargs): runner._dummy_run( num_tokens=runner.scheduler_config.max_num_batched_tokens, skip_eplb=True, @@ -181,7 +209,9 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: with torch.inference_mode(): if is_leader: - with fi_utils.autotune(tune_mode=True, cache=str(cache_path)): + with fi_utils.autotune( + tune_mode=True, cache=str(cache_path), **autotune_kwargs + ): runner._dummy_run(**dummy_run_kwargs) else: runner._dummy_run(**dummy_run_kwargs)