[Core] Persist and reuse the memory-profiling result across boots (opt-in) (#47388)
Signed-off-by: Nils Matteson <nils@thaw.sh> Signed-off-by: Nils Matteson <nilsmatteson@icloud.com> Co-authored-by: Nils Matteson <nils@thaw.sh> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Nils Matteson
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4aceabf8c1
commit
e97c3cb303
@@ -234,6 +234,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
|
||||
VLLM_ALLREDUCE_USE_FLASHINFER: bool = False
|
||||
VLLM_TUNED_CONFIG_FOLDER: str | None = None
|
||||
VLLM_ENABLE_STARTUP_PLAN: bool = False
|
||||
VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: set[str] = set()
|
||||
VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT: bool = False
|
||||
VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False
|
||||
@@ -1728,6 +1729,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# Each component first checks this folder, then the configs shipped with
|
||||
# vLLM (if any). If no JSON matches, it uses a hard-coded heuristic.
|
||||
"VLLM_TUNED_CONFIG_FOLDER": lambda: os.getenv("VLLM_TUNED_CONFIG_FOLDER", None),
|
||||
# Opt-in persistence of the startup plan. When enabled, each worker
|
||||
# saves the memory-profiling result (the suggested --kv-cache-memory value
|
||||
# and the free-memory baseline) under VLLM_CACHE_ROOT/startup_plan/,
|
||||
# keyed by a hardware+config fingerprint, and later boots auto-apply it
|
||||
# -- skipping memory profiling -- when the fingerprint matches and
|
||||
# current free memory >= the recorded baseline.
|
||||
# See vllm/v1/worker/startup_plan.py.
|
||||
"VLLM_ENABLE_STARTUP_PLAN": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_STARTUP_PLAN", "0"))
|
||||
),
|
||||
# Valid values are container,code_interpreter,web_search_preview
|
||||
# ex VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS=container,code_interpreter
|
||||
# If the server_label of your mcp tool is not in this list it will
|
||||
@@ -2062,6 +2073,8 @@ def compile_factors() -> dict[str, object]:
|
||||
"VLLM_DEBUG_DUMP_PATH",
|
||||
"VLLM_PORT",
|
||||
"VLLM_CACHE_ROOT",
|
||||
# Runtime memory-plan persistence; does not affect compiled graphs.
|
||||
"VLLM_ENABLE_STARTUP_PLAN",
|
||||
"LD_LIBRARY_PATH",
|
||||
"VLLM_SERVER_DEV_MODE",
|
||||
"VLLM_DP_MASTER_IP",
|
||||
|
||||
Reference in New Issue
Block a user