Add VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS and skip CuTeDSL fp4_gemm autotuning by default (#48268)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
ed908cf0a0
commit
29fd688892
@@ -197,6 +197,7 @@ if TYPE_CHECKING:
|
||||
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
|
||||
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
|
||||
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
|
||||
VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS: list[str] | None = None
|
||||
VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto"
|
||||
VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024
|
||||
VLLM_XGRAMMAR_CACHE_MB: int = 0
|
||||
@@ -1594,6 +1595,18 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR": lambda: os.getenv(
|
||||
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", None
|
||||
),
|
||||
# Comma-separated FlashInfer op names to exclude from autotuning, using
|
||||
# the heuristic fallback tactic instead. Unset: skip "fp4_gemm" when the
|
||||
# CuTe-DSL NVFP4 linear kernel is selected. Empty: skip nothing.
|
||||
"VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS": lambda: (
|
||||
None
|
||||
if "VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS" not in os.environ
|
||||
else [
|
||||
v.strip()
|
||||
for v in os.environ["VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS"].split(",")
|
||||
if v.strip()
|
||||
]
|
||||
),
|
||||
# Flashinfer fused allreduce backend.
|
||||
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": env_with_choices(
|
||||
"VLLM_FLASHINFER_ALLREDUCE_BACKEND",
|
||||
@@ -2110,6 +2123,7 @@ def compile_factors() -> dict[str, object]:
|
||||
"VLLM_DEBUG_LOG_API_SERVER_RESPONSE",
|
||||
"VLLM_TUNED_CONFIG_FOLDER",
|
||||
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR",
|
||||
"VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS",
|
||||
"VLLM_ENGINE_ITERATION_TIMEOUT_S",
|
||||
"VLLM_HTTP_TIMEOUT_KEEP_ALIVE",
|
||||
"VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS",
|
||||
|
||||
@@ -130,6 +130,25 @@ def kernel_warmup(worker: "Worker"):
|
||||
cutedsl_warmup()
|
||||
|
||||
|
||||
def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
|
||||
if envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS is not None:
|
||||
return set(envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS) or None
|
||||
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
FlashInferCuteDslNvFp4LinearKernel,
|
||||
)
|
||||
|
||||
for module in runner.get_model().modules():
|
||||
for holder_name in ("quant_method", "scheme"):
|
||||
kernel = getattr(getattr(module, holder_name, None), "kernel", None)
|
||||
# CuTe-DSL mm_fp4 tuning JIT-compiles every tactic and its
|
||||
# fallback is already the heuristic; all mm_fp4 backends share
|
||||
# the "fp4_gemm" op name, so skip only when cute-dsl is selected.
|
||||
if isinstance(kernel, FlashInferCuteDslNvFp4LinearKernel):
|
||||
return {"fp4_gemm"}
|
||||
return None
|
||||
|
||||
|
||||
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
"""
|
||||
Autotune FlashInfer operations.
|
||||
@@ -146,6 +165,15 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
autotune_kwargs: dict = {}
|
||||
skip_ops = _flashinfer_autotune_skip_ops(runner)
|
||||
if skip_ops:
|
||||
logger.info(
|
||||
"Skipping FlashInfer autotuning for ops %s",
|
||||
sorted(skip_ops),
|
||||
)
|
||||
autotune_kwargs["skip_ops"] = skip_ops
|
||||
|
||||
use_persistent_cache = True
|
||||
|
||||
# When distributed, tune on every rank so the collectives stay synchronized.
|
||||
@@ -153,7 +181,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
use_persistent_cache = False
|
||||
|
||||
if not use_persistent_cache:
|
||||
with torch.inference_mode(), fi_utils.autotune():
|
||||
with torch.inference_mode(), fi_utils.autotune(**autotune_kwargs):
|
||||
runner._dummy_run(
|
||||
num_tokens=runner.scheduler_config.max_num_batched_tokens,
|
||||
skip_eplb=True,
|
||||
@@ -181,7 +209,9 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
|
||||
with torch.inference_mode():
|
||||
if is_leader:
|
||||
with fi_utils.autotune(tune_mode=True, cache=str(cache_path)):
|
||||
with fi_utils.autotune(
|
||||
tune_mode=True, cache=str(cache_path), **autotune_kwargs
|
||||
):
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
else:
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
|
||||
Reference in New Issue
Block a user