[Core] Persist and reuse the memory-profiling result across boots (opt-in) (#47388)

Signed-off-by: Nils Matteson <nils@thaw.sh>
Signed-off-by: Nils Matteson <nilsmatteson@icloud.com>
Co-authored-by: Nils Matteson <nils@thaw.sh>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nils Matteson
2026-07-08 00:53:02 +00:00
committed by GitHub
co-authored by Nils Matteson mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4aceabf8c1
commit e97c3cb303
4 changed files with 284 additions and 0 deletions
+8
View File
@@ -75,6 +75,10 @@ from vllm.v1.outputs import (
ModelRunnerOutput,
)
from vllm.v1.utils import compute_iteration_details, report_usage_stats
from vllm.v1.worker.startup_plan import (
maybe_apply_startup_plan,
maybe_save_startup_plan,
)
from vllm.v1.worker.utils import is_residual_scattered_for_sp
from vllm.v1.worker.worker_base import CompilationTimes, WorkerBase
from vllm.v1.worker.workspace import init_workspace_manager
@@ -439,6 +443,8 @@ class Worker(WorkerBase):
You may limit the usage of GPU memory
by adjusting the `gpu_memory_utilization` parameter.
"""
maybe_apply_startup_plan(self)
if kv_cache_memory_bytes := self.cache_config.kv_cache_memory_bytes:
# still need a profile run which compiles the model for
# max_num_batched_tokens
@@ -834,6 +840,8 @@ class Worker(WorkerBase):
logger.info(msg)
maybe_save_startup_plan(self, kv_cache_memory_bytes_to_requested_limit)
if self.use_v2_model_runner:
# V2: Run full execute_model + sample_tokens to JIT compile triton kernels.
warmup_kernels(self.model_runner, self.execute_model, self.sample_tokens)