[Core] Persist and reuse the memory-profiling result across boots (opt-in) (#47388)
Signed-off-by: Nils Matteson <nils@thaw.sh> Signed-off-by: Nils Matteson <nilsmatteson@icloud.com> Co-authored-by: Nils Matteson <nils@thaw.sh> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Nils Matteson
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4aceabf8c1
commit
e97c3cb303
@@ -75,6 +75,10 @@ from vllm.v1.outputs import (
|
||||
ModelRunnerOutput,
|
||||
)
|
||||
from vllm.v1.utils import compute_iteration_details, report_usage_stats
|
||||
from vllm.v1.worker.startup_plan import (
|
||||
maybe_apply_startup_plan,
|
||||
maybe_save_startup_plan,
|
||||
)
|
||||
from vllm.v1.worker.utils import is_residual_scattered_for_sp
|
||||
from vllm.v1.worker.worker_base import CompilationTimes, WorkerBase
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
@@ -439,6 +443,8 @@ class Worker(WorkerBase):
|
||||
You may limit the usage of GPU memory
|
||||
by adjusting the `gpu_memory_utilization` parameter.
|
||||
"""
|
||||
maybe_apply_startup_plan(self)
|
||||
|
||||
if kv_cache_memory_bytes := self.cache_config.kv_cache_memory_bytes:
|
||||
# still need a profile run which compiles the model for
|
||||
# max_num_batched_tokens
|
||||
@@ -834,6 +840,8 @@ class Worker(WorkerBase):
|
||||
|
||||
logger.info(msg)
|
||||
|
||||
maybe_save_startup_plan(self, kv_cache_memory_bytes_to_requested_limit)
|
||||
|
||||
if self.use_v2_model_runner:
|
||||
# V2: Run full execute_model + sample_tokens to JIT compile triton kernels.
|
||||
warmup_kernels(self.model_runner, self.execute_model, self.sample_tokens)
|
||||
|
||||
Reference in New Issue
Block a user