[Fast Start] Preload the FlashInfer autotune table on the weight cache daemon (#60085)
Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Claude Opus 5
Isotr0py
parent
3709632ff2
commit
f2376cdc4d
@@ -112,11 +112,14 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
|
||||
verify_peer_is_owner,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_cache.utils import (
|
||||
build_warmup_runner,
|
||||
export_model_attrs,
|
||||
format_daemon_role,
|
||||
is_draft_model_cacheable,
|
||||
)
|
||||
from vllm.model_executor.warmup.kernel_warmup import flashinfer_autotune
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.mem_utils import format_gib
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
@@ -236,6 +239,41 @@ class WeightCacheDaemon:
|
||||
is_draft=is_draft,
|
||||
)
|
||||
|
||||
def warmup(self) -> None:
|
||||
"""Run the FlashInfer autotune pass against the cached model."""
|
||||
vllm_config = self.vllm_config
|
||||
# Mirrors the gate in the engine's kernel warmup.
|
||||
if (
|
||||
vllm_config.kernel_config.enable_flashinfer_autotune is False
|
||||
or not has_flashinfer()
|
||||
or not current_platform.has_device_capability(90)
|
||||
or is_draft_model_cacheable(vllm_config.speculative_config)
|
||||
):
|
||||
return
|
||||
try:
|
||||
current_platform.update_block_size_for_backend(vllm_config)
|
||||
runner = build_warmup_runner(
|
||||
vllm_config,
|
||||
self.local_rank,
|
||||
is_draft=self.is_draft,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
assert self.model is not None, "warmup ran before load_model"
|
||||
runner.load_model(model=self.model)
|
||||
# The V2 runner requires skip_attn when no KV cache exists
|
||||
flashinfer_autotune(runner, skip_attn=vllm_config.use_v2_model_runner)
|
||||
logger.info(
|
||||
"Weight cache %s daemon rank %d tuned FlashInfer; the tuned "
|
||||
"configs are in the on-disk autotune cache",
|
||||
self.role,
|
||||
self.global_rank,
|
||||
)
|
||||
finally:
|
||||
# Free the runner and the dummy-run activations; the daemon's
|
||||
# weights are untouched.
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
def load_model(self) -> None:
|
||||
torch.accelerator.set_device_index(self.local_rank)
|
||||
# Outside the config context so the daemon keeps its own rendezvous
|
||||
@@ -457,6 +495,7 @@ def _run_daemon(
|
||||
pp_rank,
|
||||
)
|
||||
daemon.load_model()
|
||||
daemon.warmup()
|
||||
daemon.serve_forever(
|
||||
ready_callback=lambda: ready_queue.put((daemon.role, daemon.global_rank))
|
||||
)
|
||||
|
||||
@@ -7,10 +7,17 @@ draft is cached, or how a daemon group is named, do not have to import the
|
||||
wire format.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from vllm.config import SpeculativeConfig
|
||||
import torch
|
||||
|
||||
from vllm.config import SpeculativeConfig, VllmConfig, replace, set_current_vllm_config
|
||||
from vllm.model_executor.models.interfaces import SupportsEagleBase
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
# Speculative methods whose draft model the daemon caches in its own group.
|
||||
# Other drafts keep loading from disk in the engine.
|
||||
@@ -46,3 +53,27 @@ def format_daemon_role(is_draft: bool) -> str:
|
||||
def format_socket_role_suffix(is_draft: bool) -> str:
|
||||
"""Socket-name suffix keeping the draft group distinct from the target."""
|
||||
return "_draft" if is_draft else ""
|
||||
|
||||
|
||||
def build_warmup_runner(
|
||||
vllm_config: VllmConfig,
|
||||
local_rank: int,
|
||||
*,
|
||||
is_draft: bool,
|
||||
model_config: "ModelConfig",
|
||||
) -> "GPUModelRunner":
|
||||
"""Create a model runner for the daemon's warmup dummy runs."""
|
||||
if is_draft:
|
||||
vllm_config = replace(
|
||||
vllm_config, model_config=model_config, speculative_config=None
|
||||
)
|
||||
device = torch.device(current_platform.device_type, local_rank)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
if vllm_config.use_v2_model_runner:
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as Runner
|
||||
else:
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as Runner
|
||||
runner = Runner(vllm_config, device)
|
||||
# The two runner classes share the warmup surface; the nominal type
|
||||
# stays V1's GPUModelRunner.
|
||||
return cast("GPUModelRunner", runner)
|
||||
|
||||
Reference in New Issue
Block a user