Add VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS and skip CuTeDSL fp4_gemm autotuning by default (#48268)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
ed908cf0a0
commit
29fd688892
@@ -130,6 +130,25 @@ def kernel_warmup(worker: "Worker"):
|
||||
cutedsl_warmup()
|
||||
|
||||
|
||||
def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
|
||||
if envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS is not None:
|
||||
return set(envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS) or None
|
||||
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
FlashInferCuteDslNvFp4LinearKernel,
|
||||
)
|
||||
|
||||
for module in runner.get_model().modules():
|
||||
for holder_name in ("quant_method", "scheme"):
|
||||
kernel = getattr(getattr(module, holder_name, None), "kernel", None)
|
||||
# CuTe-DSL mm_fp4 tuning JIT-compiles every tactic and its
|
||||
# fallback is already the heuristic; all mm_fp4 backends share
|
||||
# the "fp4_gemm" op name, so skip only when cute-dsl is selected.
|
||||
if isinstance(kernel, FlashInferCuteDslNvFp4LinearKernel):
|
||||
return {"fp4_gemm"}
|
||||
return None
|
||||
|
||||
|
||||
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
"""
|
||||
Autotune FlashInfer operations.
|
||||
@@ -146,6 +165,15 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
autotune_kwargs: dict = {}
|
||||
skip_ops = _flashinfer_autotune_skip_ops(runner)
|
||||
if skip_ops:
|
||||
logger.info(
|
||||
"Skipping FlashInfer autotuning for ops %s",
|
||||
sorted(skip_ops),
|
||||
)
|
||||
autotune_kwargs["skip_ops"] = skip_ops
|
||||
|
||||
use_persistent_cache = True
|
||||
|
||||
# When distributed, tune on every rank so the collectives stay synchronized.
|
||||
@@ -153,7 +181,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
use_persistent_cache = False
|
||||
|
||||
if not use_persistent_cache:
|
||||
with torch.inference_mode(), fi_utils.autotune():
|
||||
with torch.inference_mode(), fi_utils.autotune(**autotune_kwargs):
|
||||
runner._dummy_run(
|
||||
num_tokens=runner.scheduler_config.max_num_batched_tokens,
|
||||
skip_eplb=True,
|
||||
@@ -181,7 +209,9 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
|
||||
with torch.inference_mode():
|
||||
if is_leader:
|
||||
with fi_utils.autotune(tune_mode=True, cache=str(cache_path)):
|
||||
with fi_utils.autotune(
|
||||
tune_mode=True, cache=str(cache_path), **autotune_kwargs
|
||||
):
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
else:
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
|
||||
Reference in New Issue
Block a user