[Fast Start] Preload the FlashInfer autotune table on the weight cache daemon (#60085)

Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
siyu
2026-10-10 15:48:06 +00:00
committed by GitHub
co-authored by Claude Opus 5 Isotr0py
parent 3709632ff2
commit f2376cdc4d
10 changed files with 253 additions and 17 deletions
+8 -2
View File
@@ -406,7 +406,7 @@ def _all_ranks_have_matching_cache(path: Path, group) -> bool:
return fingerprint is not None and all(f == fingerprint for f in gathered)
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) -> None:
"""Autotune FlashInfer operations.
FlashInfer have many implementations for the same operation,
autotuning runs benchmarks for each implementation and stores
@@ -415,6 +415,10 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
Without autotuning, FlashInfer will rely on heuristics, which may
be significantly slower.
Args:
runner: The model runner to tune.
skip_attn: Additionally skip attention in the tuning dummy runs.
With PP > 1, stages run different layers and may profile different ops,
so each stage's TP group tunes separately with its own cache file;
otherwise the world group tunes together. Per-tactic timings are
@@ -425,6 +429,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
rank. A rank with a cache hit skips the per-tactic reduce the others block
in, so ranks keep loaded configs only if every rank in the tuning group
has a matching file and successfully loads it.
"""
from flashinfer.autotuner import AutoTuner, set_autotune_process_group
@@ -485,11 +490,12 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
hisparse_enabled = (
runner.vllm_config.attention_config.hisparse_config is not None
)
skip_attn = skip_attn or hisparse_enabled
if hisparse_enabled:
# HiSparse hot-buffer attention is bounded by decode batch
# size, not the prefill-sized batch used for the full model.
autotune_hisparse_flashinfer_attention(runner)
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled)
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
replayssm_autotune_warmup(runner)
_autotune_kimi_k3_kda_qkvg(runner.get_model())
finally: