[Core] Persist and reuse the memory-profiling result across boots (opt-in) (#47388)

Signed-off-by: Nils Matteson <nils@thaw.sh>
Signed-off-by: Nils Matteson <nilsmatteson@icloud.com>
Co-authored-by: Nils Matteson <nils@thaw.sh>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nils Matteson
2026-07-08 00:53:02 +00:00
committed by GitHub
co-authored by Nils Matteson mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4aceabf8c1
commit e97c3cb303
4 changed files with 284 additions and 0 deletions
+13
View File
@@ -234,6 +234,7 @@ if TYPE_CHECKING:
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
VLLM_ALLREDUCE_USE_FLASHINFER: bool = False
VLLM_TUNED_CONFIG_FOLDER: str | None = None
VLLM_ENABLE_STARTUP_PLAN: bool = False
VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: set[str] = set()
VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT: bool = False
VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False
@@ -1728,6 +1729,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Each component first checks this folder, then the configs shipped with
# vLLM (if any). If no JSON matches, it uses a hard-coded heuristic.
"VLLM_TUNED_CONFIG_FOLDER": lambda: os.getenv("VLLM_TUNED_CONFIG_FOLDER", None),
# Opt-in persistence of the startup plan. When enabled, each worker
# saves the memory-profiling result (the suggested --kv-cache-memory value
# and the free-memory baseline) under VLLM_CACHE_ROOT/startup_plan/,
# keyed by a hardware+config fingerprint, and later boots auto-apply it
# -- skipping memory profiling -- when the fingerprint matches and
# current free memory >= the recorded baseline.
# See vllm/v1/worker/startup_plan.py.
"VLLM_ENABLE_STARTUP_PLAN": lambda: bool(
int(os.getenv("VLLM_ENABLE_STARTUP_PLAN", "0"))
),
# Valid values are container,code_interpreter,web_search_preview
# ex VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS=container,code_interpreter
# If the server_label of your mcp tool is not in this list it will
@@ -2062,6 +2073,8 @@ def compile_factors() -> dict[str, object]:
"VLLM_DEBUG_DUMP_PATH",
"VLLM_PORT",
"VLLM_CACHE_ROOT",
# Runtime memory-plan persistence; does not affect compiled graphs.
"VLLM_ENABLE_STARTUP_PLAN",
"LD_LIBRARY_PATH",
"VLLM_SERVER_DEV_MODE",
"VLLM_DP_MASTER_IP",