[Core] Persist and reuse the memory-profiling result across boots (opt-in) (#47388)
Signed-off-by: Nils Matteson <nils@thaw.sh> Signed-off-by: Nils Matteson <nilsmatteson@icloud.com> Co-authored-by: Nils Matteson <nils@thaw.sh> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Nils Matteson
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4aceabf8c1
commit
e97c3cb303
@@ -0,0 +1,191 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Persist and reuse the memory-profiling result across engine boots.
|
||||
|
||||
On startup, vLLM measures how much GPU memory the KV cache can use and
|
||||
computes the ``--kv-cache-memory`` value that reproduces that allocation.
|
||||
For a fixed (model, config, hardware, library) combination the result is
|
||||
deterministic, yet it is re-measured on every boot.
|
||||
|
||||
When ``VLLM_ENABLE_STARTUP_PLAN=1``, each worker persists that value under
|
||||
``{VLLM_CACHE_ROOT}/startup_plan/`` (regenerable derived state, alongside
|
||||
the torch.compile cache), keyed by a fingerprint of everything the value
|
||||
depends on, and later boots apply it automatically -- skipping the
|
||||
memory-profiling measurement and the CUDA-graph memory estimation pass --
|
||||
if and only if the fingerprint matches and the device has at least as much
|
||||
free memory as when the plan was recorded. On any mismatch the worker
|
||||
falls back to full profiling, so a stale plan costs nothing and is never
|
||||
trusted.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
PLAN_SCHEMA_VERSION = 1
|
||||
|
||||
|
||||
def compute_plan_fingerprint(
|
||||
vllm_config: VllmConfig, rank: int, world_size: int
|
||||
) -> str:
|
||||
"""Hash everything the profiled KV-cache memory value depends on.
|
||||
|
||||
``VllmConfig.compute_hash()`` covers the vLLM version and the model,
|
||||
cache, parallel, and compilation configs, but deliberately contains no
|
||||
device identity (``DeviceConfig.compute_hash`` is empty), so device
|
||||
name, total memory, compute capability, and the torch/CUDA build are
|
||||
added here. The vLLM version is also pinned as an explicit factor so
|
||||
version invalidation holds no matter how ``compute_hash`` evolves.
|
||||
Rank is included because per-rank memory use differs under TP/PP.
|
||||
Driver-only changes are not part of the key; the free-memory gate at
|
||||
apply time bounds the residual risk.
|
||||
"""
|
||||
# Imported here (as VllmConfig.compute_hash does) to avoid a cycle with
|
||||
# the top-level vllm package.
|
||||
from vllm import __version__ as vllm_version
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
factors = {
|
||||
"schema": PLAN_SCHEMA_VERSION,
|
||||
"vllm": vllm_version,
|
||||
"vllm_config": vllm_config.compute_hash(),
|
||||
"device_name": current_platform.get_device_name(),
|
||||
"device_total_memory": current_platform.get_device_total_memory(),
|
||||
"device_capability": str(capability) if capability else "",
|
||||
"torch": torch.__version__,
|
||||
"cuda": torch.version.cuda or "",
|
||||
"rank": rank,
|
||||
"world_size": world_size,
|
||||
}
|
||||
digest = hashlib.sha256(json.dumps(factors, sort_keys=True).encode()).hexdigest()
|
||||
return digest[:16]
|
||||
|
||||
|
||||
def _plan_path(fingerprint: str) -> str:
|
||||
"""Plans are regenerable derived state, so they live under the standard
|
||||
vLLM cache root (like the torch.compile cache) and relocate with
|
||||
``VLLM_CACHE_ROOT`` instead of needing a location knob of their own."""
|
||||
# VLLM_CACHE_ROOT is already user-expanded by envs.py.
|
||||
return os.path.join(
|
||||
envs.VLLM_CACHE_ROOT, "startup_plan", f"startup_plan_{fingerprint}.json"
|
||||
)
|
||||
|
||||
|
||||
def _load_plan(fingerprint: str) -> dict | None:
|
||||
"""Load a plan for this fingerprint; None if absent or unreadable."""
|
||||
path = _plan_path(fingerprint)
|
||||
try:
|
||||
with open(path) as f:
|
||||
plan = json.load(f)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except (OSError, json.JSONDecodeError) as e:
|
||||
logger.warning("Ignoring unreadable startup plan %s: %s", path, e)
|
||||
return None
|
||||
if (
|
||||
plan.get("schema") != PLAN_SCHEMA_VERSION
|
||||
or plan.get("fingerprint") != fingerprint
|
||||
):
|
||||
return None
|
||||
return plan
|
||||
|
||||
|
||||
def _applicable_kv_cache_memory_bytes(
|
||||
plan: dict, current_free_memory: int
|
||||
) -> int | None:
|
||||
"""The apply-time OOM-safety gate.
|
||||
|
||||
The recorded value is only valid if the device has at least as much
|
||||
free memory now as when the plan was measured (co-tenants, leaked
|
||||
allocations, or MIG changes all reduce it). Outside that envelope,
|
||||
return None and let the caller re-profile.
|
||||
"""
|
||||
kv_bytes = plan.get("kv_cache_memory_bytes")
|
||||
baseline = plan.get("free_memory_baseline")
|
||||
if not isinstance(kv_bytes, int) or not isinstance(baseline, int):
|
||||
return None
|
||||
if kv_bytes <= 0:
|
||||
return None
|
||||
if current_free_memory < baseline:
|
||||
logger.info(
|
||||
"Startup plan not applied: current free memory (%.2f GiB) is "
|
||||
"below the recorded baseline (%.2f GiB); falling back to full "
|
||||
"memory profiling.",
|
||||
current_free_memory / (1 << 30),
|
||||
baseline / (1 << 30),
|
||||
)
|
||||
return None
|
||||
return kv_bytes
|
||||
|
||||
|
||||
def maybe_apply_startup_plan(worker: "Worker") -> None:
|
||||
"""If enabled and ``--kv-cache-memory`` was not set explicitly, apply a
|
||||
persisted plan by setting ``worker.cache_config.kv_cache_memory_bytes``.
|
||||
No-op unless ``VLLM_ENABLE_STARTUP_PLAN=1``."""
|
||||
if (
|
||||
not envs.VLLM_ENABLE_STARTUP_PLAN
|
||||
or worker.cache_config.kv_cache_memory_bytes is not None
|
||||
):
|
||||
return
|
||||
fingerprint = compute_plan_fingerprint(
|
||||
worker.vllm_config, worker.rank, worker.parallel_config.world_size
|
||||
)
|
||||
plan = _load_plan(fingerprint)
|
||||
if plan is None:
|
||||
return
|
||||
current_free_memory = worker.init_snapshot.free_memory
|
||||
kv_bytes = _applicable_kv_cache_memory_bytes(plan, current_free_memory)
|
||||
if kv_bytes is None:
|
||||
return
|
||||
logger.info(
|
||||
"Applying persisted startup plan (fingerprint %s): "
|
||||
"kv_cache_memory_bytes=%d (%.2f GiB), recorded free-memory "
|
||||
"baseline %.2f GiB, current %.2f GiB. Memory profiling will "
|
||||
"be skipped.",
|
||||
fingerprint,
|
||||
kv_bytes,
|
||||
kv_bytes / (1 << 30),
|
||||
plan["free_memory_baseline"] / (1 << 30),
|
||||
current_free_memory / (1 << 30),
|
||||
)
|
||||
worker.cache_config.kv_cache_memory_bytes = kv_bytes
|
||||
|
||||
|
||||
def maybe_save_startup_plan(worker: "Worker", kv_cache_memory_bytes: int) -> None:
|
||||
"""Atomically persist this boot's profiling result for future boots.
|
||||
No-op unless ``VLLM_ENABLE_STARTUP_PLAN=1``; failures are logged,
|
||||
never raised."""
|
||||
if not envs.VLLM_ENABLE_STARTUP_PLAN:
|
||||
return
|
||||
fingerprint = compute_plan_fingerprint(
|
||||
worker.vllm_config, worker.rank, worker.parallel_config.world_size
|
||||
)
|
||||
path = _plan_path(fingerprint)
|
||||
try:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
payload = {
|
||||
"schema": PLAN_SCHEMA_VERSION,
|
||||
"fingerprint": fingerprint,
|
||||
"kv_cache_memory_bytes": int(kv_cache_memory_bytes),
|
||||
"free_memory_baseline": int(worker.init_snapshot.free_memory),
|
||||
}
|
||||
tmp = f"{path}.tmp.{os.getpid()}"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(payload, f)
|
||||
os.replace(tmp, path)
|
||||
logger.info("Saved startup plan to %s", path)
|
||||
except OSError as e:
|
||||
logger.warning("Failed to save startup plan to %s: %s", path, e)
|
||||
Reference in New Issue
Block a user