[Fast Start] Preload the FlashInfer autotune table on the weight cache daemon (#60085)

Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
siyu
2026-10-10 15:48:06 +00:00
committed by GitHub
co-authored by Claude Opus 5 Isotr0py
parent 3709632ff2
commit f2376cdc4d
10 changed files with 253 additions and 17 deletions
+8 -1
View File
@@ -39,6 +39,7 @@ from vllm.logger import init_logger
from vllm.model_executor.model_loader.weight_cache.protocol import (
check_ipc_platform_support,
)
from vllm.usage.usage_lib import UsageContext
from vllm.utils.network_utils import get_distributed_init_method, get_open_port
if typing.TYPE_CHECKING:
@@ -157,7 +158,13 @@ class PreloadSubcommand(CLISubcommand):
)
engine_args = EngineArgs.from_cli_args(args)
vllm_config = engine_args.create_engine_config()
# Warmup context for flashinfer autotune.
usage_context = (
UsageContext.OPENAI_API_SERVER
if engine_args.kernel_config.enable_flashinfer_autotune is not False
else None
)
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
if vllm_config.load_config.load_format == "ipc_cache":
raise ValueError(
"The weight cache daemon itself must load from disk; use the "
@@ -112,11 +112,14 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
verify_peer_is_owner,
)
from vllm.model_executor.model_loader.weight_cache.utils import (
build_warmup_runner,
export_model_attrs,
format_daemon_role,
is_draft_model_cacheable,
)
from vllm.model_executor.warmup.kernel_warmup import flashinfer_autotune
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.utils.mem_utils import format_gib
from vllm.utils.torch_utils import set_default_torch_dtype
from vllm.v1.worker.workspace import init_workspace_manager
@@ -236,6 +239,41 @@ class WeightCacheDaemon:
is_draft=is_draft,
)
def warmup(self) -> None:
"""Run the FlashInfer autotune pass against the cached model."""
vllm_config = self.vllm_config
# Mirrors the gate in the engine's kernel warmup.
if (
vllm_config.kernel_config.enable_flashinfer_autotune is False
or not has_flashinfer()
or not current_platform.has_device_capability(90)
or is_draft_model_cacheable(vllm_config.speculative_config)
):
return
try:
current_platform.update_block_size_for_backend(vllm_config)
runner = build_warmup_runner(
vllm_config,
self.local_rank,
is_draft=self.is_draft,
model_config=self.model_config,
)
assert self.model is not None, "warmup ran before load_model"
runner.load_model(model=self.model)
# The V2 runner requires skip_attn when no KV cache exists
flashinfer_autotune(runner, skip_attn=vllm_config.use_v2_model_runner)
logger.info(
"Weight cache %s daemon rank %d tuned FlashInfer; the tuned "
"configs are in the on-disk autotune cache",
self.role,
self.global_rank,
)
finally:
# Free the runner and the dummy-run activations; the daemon's
# weights are untouched.
gc.collect()
torch.accelerator.empty_cache()
def load_model(self) -> None:
torch.accelerator.set_device_index(self.local_rank)
# Outside the config context so the daemon keeps its own rendezvous
@@ -457,6 +495,7 @@ def _run_daemon(
pp_rank,
)
daemon.load_model()
daemon.warmup()
daemon.serve_forever(
ready_callback=lambda: ready_queue.put((daemon.role, daemon.global_rank))
)
@@ -7,10 +7,17 @@ draft is cached, or how a daemon group is named, do not have to import the
wire format.
"""
from typing import Any
from typing import TYPE_CHECKING, Any, cast
from vllm.config import SpeculativeConfig
import torch
from vllm.config import SpeculativeConfig, VllmConfig, replace, set_current_vllm_config
from vllm.model_executor.models.interfaces import SupportsEagleBase
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.config import ModelConfig
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
# Speculative methods whose draft model the daemon caches in its own group.
# Other drafts keep loading from disk in the engine.
@@ -46,3 +53,27 @@ def format_daemon_role(is_draft: bool) -> str:
def format_socket_role_suffix(is_draft: bool) -> str:
"""Socket-name suffix keeping the draft group distinct from the target."""
return "_draft" if is_draft else ""
def build_warmup_runner(
vllm_config: VllmConfig,
local_rank: int,
*,
is_draft: bool,
model_config: "ModelConfig",
) -> "GPUModelRunner":
"""Create a model runner for the daemon's warmup dummy runs."""
if is_draft:
vllm_config = replace(
vllm_config, model_config=model_config, speculative_config=None
)
device = torch.device(current_platform.device_type, local_rank)
with set_current_vllm_config(vllm_config):
if vllm_config.use_v2_model_runner:
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as Runner
else:
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as Runner
runner = Runner(vllm_config, device)
# The two runner classes share the warmup surface; the nominal type
# stays V1's GPUModelRunner.
return cast("GPUModelRunner", runner)
@@ -10,14 +10,28 @@ from pathlib import Path
from typing import TYPE_CHECKING
import vllm.envs as envs
from vllm.config import CacheConfig, VllmConfig, replace
if TYPE_CHECKING:
from vllm.distributed.parallel_state import GroupCoordinator
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
def _normalize_cache_config_for_hash(cache_config: CacheConfig) -> CacheConfig:
"""Normalize cache config for hashing"""
if cache_config.mamba_block_size is None and cache_config.kv_cache_layout is None:
return cache_config
normalized = replace(cache_config, mamba_block_size=None)
assert normalized.kv_cache_layout is None
return normalized
def flashinfer_autotune_cache_hash(runner: "GPUModelRunner") -> str:
config_hash = runner.vllm_config.compute_hash(include_version=False)
vllm_config: VllmConfig = runner.vllm_config
cache_config = _normalize_cache_config_for_hash(vllm_config.cache_config)
if cache_config is not vllm_config.cache_config:
vllm_config = replace(vllm_config, cache_config=cache_config)
config_hash = vllm_config.compute_hash(include_version=False)
return hashlib.sha256(config_hash.encode()).hexdigest()
+8 -2
View File
@@ -406,7 +406,7 @@ def _all_ranks_have_matching_cache(path: Path, group) -> bool:
return fingerprint is not None and all(f == fingerprint for f in gathered)
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) -> None:
"""Autotune FlashInfer operations.
FlashInfer have many implementations for the same operation,
autotuning runs benchmarks for each implementation and stores
@@ -415,6 +415,10 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
Without autotuning, FlashInfer will rely on heuristics, which may
be significantly slower.
Args:
runner: The model runner to tune.
skip_attn: Additionally skip attention in the tuning dummy runs.
With PP > 1, stages run different layers and may profile different ops,
so each stage's TP group tunes separately with its own cache file;
otherwise the world group tunes together. Per-tactic timings are
@@ -425,6 +429,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
rank. A rank with a cache hit skips the per-tactic reduce the others block
in, so ranks keep loaded configs only if every rank in the tuning group
has a matching file and successfully loads it.
"""
from flashinfer.autotuner import AutoTuner, set_autotune_process_group
@@ -485,11 +490,12 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
hisparse_enabled = (
runner.vllm_config.attention_config.hisparse_config is not None
)
skip_attn = skip_attn or hisparse_enabled
if hisparse_enabled:
# HiSparse hot-buffer attention is bounded by decode batch
# size, not the prefill-sized batch used for the full model.
autotune_hisparse_flashinfer_attention(runner)
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled)
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
replayssm_autotune_warmup(runner)
_autotune_kimi_k3_kda_qkvg(runner.get_model())
finally:
+9 -3
View File
@@ -373,9 +373,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
tasks.extend(PoolingRunner.get_supported_tasks(self.model))
return tuple(tasks)
def load_model(self, load_dummy_weights: bool = False, *args, **kwargs) -> None:
def load_model(
self,
load_dummy_weights: bool = False,
*args,
model: nn.Module | None = None,
**kwargs,
) -> None:
time_before_load = time.perf_counter()
if load_dummy_weights:
if model is None and load_dummy_weights:
self.load_config.load_format = "dummy"
self.eplb.prepare_load()
eplb_models_added = False
@@ -385,7 +391,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Capture warmup providers selected while constructing the model.
with self.jit_warmup_registry.activate():
self.model = model_loader.load_model(
self.model = model or model_loader.load_model(
vllm_config=self.vllm_config,
model_config=self.vllm_config.model_config,
)
+11 -4
View File
@@ -5217,9 +5217,15 @@ class GPUModelRunner(
setattr(self, config_name, new_config)
@instrument(span_name="Loading (GPU)")
def load_model(self, load_dummy_weights: bool = False) -> None:
def load_model(
self,
load_dummy_weights: bool = False,
*,
model: nn.Module | None = None,
) -> None:
"""Args:
load_dummy_weights: load dummy weights instead of real weights.
model: a pre-loaded model to skip the model loader.
"""
logger.info_once(
@@ -5235,13 +5241,14 @@ class GPUModelRunner(
try:
with DeviceMemoryProfiler() as m:
time_before_load = time.perf_counter()
if load_dummy_weights:
if model is None and load_dummy_weights:
self.load_config.load_format = "dummy"
model_loader = get_model_loader(self.load_config)
# Capture warmup providers selected while constructing the model.
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
self.model = model_loader.load_model(
vllm_config=self.vllm_config, model_config=self.model_config
self.model = model or model_loader.load_model(
vllm_config=self.vllm_config,
model_config=self.model_config,
)
lookback_depth = getattr(self.model, "token_lookback_depth", 0)
if lookback_depth > 0: