[Fast Start] Preload the FlashInfer autotune table on the weight cache daemon (#60085)
Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Claude Opus 5
Isotr0py
parent
3709632ff2
commit
f2376cdc4d
@@ -39,6 +39,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import (
|
||||
check_ipc_platform_support,
|
||||
)
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils.network_utils import get_distributed_init_method, get_open_port
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -157,7 +158,13 @@ class PreloadSubcommand(CLISubcommand):
|
||||
)
|
||||
|
||||
engine_args = EngineArgs.from_cli_args(args)
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
# Warmup context for flashinfer autotune.
|
||||
usage_context = (
|
||||
UsageContext.OPENAI_API_SERVER
|
||||
if engine_args.kernel_config.enable_flashinfer_autotune is not False
|
||||
else None
|
||||
)
|
||||
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
|
||||
if vllm_config.load_config.load_format == "ipc_cache":
|
||||
raise ValueError(
|
||||
"The weight cache daemon itself must load from disk; use the "
|
||||
|
||||
@@ -112,11 +112,14 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
|
||||
verify_peer_is_owner,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_cache.utils import (
|
||||
build_warmup_runner,
|
||||
export_model_attrs,
|
||||
format_daemon_role,
|
||||
is_draft_model_cacheable,
|
||||
)
|
||||
from vllm.model_executor.warmup.kernel_warmup import flashinfer_autotune
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.mem_utils import format_gib
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
@@ -236,6 +239,41 @@ class WeightCacheDaemon:
|
||||
is_draft=is_draft,
|
||||
)
|
||||
|
||||
def warmup(self) -> None:
|
||||
"""Run the FlashInfer autotune pass against the cached model."""
|
||||
vllm_config = self.vllm_config
|
||||
# Mirrors the gate in the engine's kernel warmup.
|
||||
if (
|
||||
vllm_config.kernel_config.enable_flashinfer_autotune is False
|
||||
or not has_flashinfer()
|
||||
or not current_platform.has_device_capability(90)
|
||||
or is_draft_model_cacheable(vllm_config.speculative_config)
|
||||
):
|
||||
return
|
||||
try:
|
||||
current_platform.update_block_size_for_backend(vllm_config)
|
||||
runner = build_warmup_runner(
|
||||
vllm_config,
|
||||
self.local_rank,
|
||||
is_draft=self.is_draft,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
assert self.model is not None, "warmup ran before load_model"
|
||||
runner.load_model(model=self.model)
|
||||
# The V2 runner requires skip_attn when no KV cache exists
|
||||
flashinfer_autotune(runner, skip_attn=vllm_config.use_v2_model_runner)
|
||||
logger.info(
|
||||
"Weight cache %s daemon rank %d tuned FlashInfer; the tuned "
|
||||
"configs are in the on-disk autotune cache",
|
||||
self.role,
|
||||
self.global_rank,
|
||||
)
|
||||
finally:
|
||||
# Free the runner and the dummy-run activations; the daemon's
|
||||
# weights are untouched.
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
def load_model(self) -> None:
|
||||
torch.accelerator.set_device_index(self.local_rank)
|
||||
# Outside the config context so the daemon keeps its own rendezvous
|
||||
@@ -457,6 +495,7 @@ def _run_daemon(
|
||||
pp_rank,
|
||||
)
|
||||
daemon.load_model()
|
||||
daemon.warmup()
|
||||
daemon.serve_forever(
|
||||
ready_callback=lambda: ready_queue.put((daemon.role, daemon.global_rank))
|
||||
)
|
||||
|
||||
@@ -7,10 +7,17 @@ draft is cached, or how a daemon group is named, do not have to import the
|
||||
wire format.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from vllm.config import SpeculativeConfig
|
||||
import torch
|
||||
|
||||
from vllm.config import SpeculativeConfig, VllmConfig, replace, set_current_vllm_config
|
||||
from vllm.model_executor.models.interfaces import SupportsEagleBase
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
# Speculative methods whose draft model the daemon caches in its own group.
|
||||
# Other drafts keep loading from disk in the engine.
|
||||
@@ -46,3 +53,27 @@ def format_daemon_role(is_draft: bool) -> str:
|
||||
def format_socket_role_suffix(is_draft: bool) -> str:
|
||||
"""Socket-name suffix keeping the draft group distinct from the target."""
|
||||
return "_draft" if is_draft else ""
|
||||
|
||||
|
||||
def build_warmup_runner(
|
||||
vllm_config: VllmConfig,
|
||||
local_rank: int,
|
||||
*,
|
||||
is_draft: bool,
|
||||
model_config: "ModelConfig",
|
||||
) -> "GPUModelRunner":
|
||||
"""Create a model runner for the daemon's warmup dummy runs."""
|
||||
if is_draft:
|
||||
vllm_config = replace(
|
||||
vllm_config, model_config=model_config, speculative_config=None
|
||||
)
|
||||
device = torch.device(current_platform.device_type, local_rank)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
if vllm_config.use_v2_model_runner:
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as Runner
|
||||
else:
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as Runner
|
||||
runner = Runner(vllm_config, device)
|
||||
# The two runner classes share the warmup surface; the nominal type
|
||||
# stays V1's GPUModelRunner.
|
||||
return cast("GPUModelRunner", runner)
|
||||
|
||||
@@ -10,14 +10,28 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import CacheConfig, VllmConfig, replace
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
|
||||
def _normalize_cache_config_for_hash(cache_config: CacheConfig) -> CacheConfig:
|
||||
"""Normalize cache config for hashing"""
|
||||
if cache_config.mamba_block_size is None and cache_config.kv_cache_layout is None:
|
||||
return cache_config
|
||||
normalized = replace(cache_config, mamba_block_size=None)
|
||||
assert normalized.kv_cache_layout is None
|
||||
return normalized
|
||||
|
||||
|
||||
def flashinfer_autotune_cache_hash(runner: "GPUModelRunner") -> str:
|
||||
config_hash = runner.vllm_config.compute_hash(include_version=False)
|
||||
vllm_config: VllmConfig = runner.vllm_config
|
||||
cache_config = _normalize_cache_config_for_hash(vllm_config.cache_config)
|
||||
if cache_config is not vllm_config.cache_config:
|
||||
vllm_config = replace(vllm_config, cache_config=cache_config)
|
||||
config_hash = vllm_config.compute_hash(include_version=False)
|
||||
return hashlib.sha256(config_hash.encode()).hexdigest()
|
||||
|
||||
|
||||
|
||||
@@ -406,7 +406,7 @@ def _all_ranks_have_matching_cache(path: Path, group) -> bool:
|
||||
return fingerprint is not None and all(f == fingerprint for f in gathered)
|
||||
|
||||
|
||||
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) -> None:
|
||||
"""Autotune FlashInfer operations.
|
||||
FlashInfer have many implementations for the same operation,
|
||||
autotuning runs benchmarks for each implementation and stores
|
||||
@@ -415,6 +415,10 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
Without autotuning, FlashInfer will rely on heuristics, which may
|
||||
be significantly slower.
|
||||
|
||||
Args:
|
||||
runner: The model runner to tune.
|
||||
skip_attn: Additionally skip attention in the tuning dummy runs.
|
||||
|
||||
With PP > 1, stages run different layers and may profile different ops,
|
||||
so each stage's TP group tunes separately with its own cache file;
|
||||
otherwise the world group tunes together. Per-tactic timings are
|
||||
@@ -425,6 +429,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
rank. A rank with a cache hit skips the per-tactic reduce the others block
|
||||
in, so ranks keep loaded configs only if every rank in the tuning group
|
||||
has a matching file and successfully loads it.
|
||||
|
||||
"""
|
||||
from flashinfer.autotuner import AutoTuner, set_autotune_process_group
|
||||
|
||||
@@ -485,11 +490,12 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
hisparse_enabled = (
|
||||
runner.vllm_config.attention_config.hisparse_config is not None
|
||||
)
|
||||
skip_attn = skip_attn or hisparse_enabled
|
||||
if hisparse_enabled:
|
||||
# HiSparse hot-buffer attention is bounded by decode batch
|
||||
# size, not the prefill-sized batch used for the full model.
|
||||
autotune_hisparse_flashinfer_attention(runner)
|
||||
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled)
|
||||
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
|
||||
replayssm_autotune_warmup(runner)
|
||||
_autotune_kimi_k3_kda_qkvg(runner.get_model())
|
||||
finally:
|
||||
|
||||
@@ -373,9 +373,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
tasks.extend(PoolingRunner.get_supported_tasks(self.model))
|
||||
return tuple(tasks)
|
||||
|
||||
def load_model(self, load_dummy_weights: bool = False, *args, **kwargs) -> None:
|
||||
def load_model(
|
||||
self,
|
||||
load_dummy_weights: bool = False,
|
||||
*args,
|
||||
model: nn.Module | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
time_before_load = time.perf_counter()
|
||||
if load_dummy_weights:
|
||||
if model is None and load_dummy_weights:
|
||||
self.load_config.load_format = "dummy"
|
||||
self.eplb.prepare_load()
|
||||
eplb_models_added = False
|
||||
@@ -385,7 +391,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
# Capture warmup providers selected while constructing the model.
|
||||
with self.jit_warmup_registry.activate():
|
||||
self.model = model_loader.load_model(
|
||||
self.model = model or model_loader.load_model(
|
||||
vllm_config=self.vllm_config,
|
||||
model_config=self.vllm_config.model_config,
|
||||
)
|
||||
|
||||
@@ -5217,9 +5217,15 @@ class GPUModelRunner(
|
||||
setattr(self, config_name, new_config)
|
||||
|
||||
@instrument(span_name="Loading (GPU)")
|
||||
def load_model(self, load_dummy_weights: bool = False) -> None:
|
||||
def load_model(
|
||||
self,
|
||||
load_dummy_weights: bool = False,
|
||||
*,
|
||||
model: nn.Module | None = None,
|
||||
) -> None:
|
||||
"""Args:
|
||||
load_dummy_weights: load dummy weights instead of real weights.
|
||||
model: a pre-loaded model to skip the model loader.
|
||||
|
||||
"""
|
||||
logger.info_once(
|
||||
@@ -5235,13 +5241,14 @@ class GPUModelRunner(
|
||||
try:
|
||||
with DeviceMemoryProfiler() as m:
|
||||
time_before_load = time.perf_counter()
|
||||
if load_dummy_weights:
|
||||
if model is None and load_dummy_weights:
|
||||
self.load_config.load_format = "dummy"
|
||||
model_loader = get_model_loader(self.load_config)
|
||||
# Capture warmup providers selected while constructing the model.
|
||||
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
|
||||
self.model = model_loader.load_model(
|
||||
vllm_config=self.vllm_config, model_config=self.model_config
|
||||
self.model = model or model_loader.load_model(
|
||||
vllm_config=self.vllm_config,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
lookback_depth = getattr(self.model, "token_lookback_depth", 0)
|
||||
if lookback_depth > 0:
|
||||
|
||||
Reference in New Issue
Block a user