[Fast Start] Preload the FlashInfer autotune table on the weight cache daemon (#60085)

Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
siyu
2026-10-10 15:48:06 +00:00
committed by GitHub
co-authored by Claude Opus 5 Isotr0py
parent 3709632ff2
commit f2376cdc4d
10 changed files with 253 additions and 17 deletions
@@ -112,11 +112,14 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
verify_peer_is_owner,
)
from vllm.model_executor.model_loader.weight_cache.utils import (
build_warmup_runner,
export_model_attrs,
format_daemon_role,
is_draft_model_cacheable,
)
from vllm.model_executor.warmup.kernel_warmup import flashinfer_autotune
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.utils.mem_utils import format_gib
from vllm.utils.torch_utils import set_default_torch_dtype
from vllm.v1.worker.workspace import init_workspace_manager
@@ -236,6 +239,41 @@ class WeightCacheDaemon:
is_draft=is_draft,
)
def warmup(self) -> None:
"""Run the FlashInfer autotune pass against the cached model."""
vllm_config = self.vllm_config
# Mirrors the gate in the engine's kernel warmup.
if (
vllm_config.kernel_config.enable_flashinfer_autotune is False
or not has_flashinfer()
or not current_platform.has_device_capability(90)
or is_draft_model_cacheable(vllm_config.speculative_config)
):
return
try:
current_platform.update_block_size_for_backend(vllm_config)
runner = build_warmup_runner(
vllm_config,
self.local_rank,
is_draft=self.is_draft,
model_config=self.model_config,
)
assert self.model is not None, "warmup ran before load_model"
runner.load_model(model=self.model)
# The V2 runner requires skip_attn when no KV cache exists
flashinfer_autotune(runner, skip_attn=vllm_config.use_v2_model_runner)
logger.info(
"Weight cache %s daemon rank %d tuned FlashInfer; the tuned "
"configs are in the on-disk autotune cache",
self.role,
self.global_rank,
)
finally:
# Free the runner and the dummy-run activations; the daemon's
# weights are untouched.
gc.collect()
torch.accelerator.empty_cache()
def load_model(self) -> None:
torch.accelerator.set_device_index(self.local_rank)
# Outside the config context so the daemon keeps its own rendezvous
@@ -457,6 +495,7 @@ def _run_daemon(
pp_rank,
)
daemon.load_model()
daemon.warmup()
daemon.serve_forever(
ready_callback=lambda: ready_queue.put((daemon.role, daemon.global_rank))
)
@@ -7,10 +7,17 @@ draft is cached, or how a daemon group is named, do not have to import the
wire format.
"""
from typing import Any
from typing import TYPE_CHECKING, Any, cast
from vllm.config import SpeculativeConfig
import torch
from vllm.config import SpeculativeConfig, VllmConfig, replace, set_current_vllm_config
from vllm.model_executor.models.interfaces import SupportsEagleBase
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.config import ModelConfig
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
# Speculative methods whose draft model the daemon caches in its own group.
# Other drafts keep loading from disk in the engine.
@@ -46,3 +53,27 @@ def format_daemon_role(is_draft: bool) -> str:
def format_socket_role_suffix(is_draft: bool) -> str:
"""Socket-name suffix keeping the draft group distinct from the target."""
return "_draft" if is_draft else ""
def build_warmup_runner(
vllm_config: VllmConfig,
local_rank: int,
*,
is_draft: bool,
model_config: "ModelConfig",
) -> "GPUModelRunner":
"""Create a model runner for the daemon's warmup dummy runs."""
if is_draft:
vllm_config = replace(
vllm_config, model_config=model_config, speculative_config=None
)
device = torch.device(current_platform.device_type, local_rank)
with set_current_vllm_config(vllm_config):
if vllm_config.use_v2_model_runner:
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as Runner
else:
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as Runner
runner = Runner(vllm_config, device)
# The two runner classes share the warmup surface; the nominal type
# stays V1's GPUModelRunner.
return cast("GPUModelRunner", runner)