From 69aed12cf8495f18fce6574fad19ab7603c558b4 Mon Sep 17 00:00:00 2001 From: MOVZX Date: Mon, 5 Oct 2026 00:13:13 -0400 Subject: [PATCH] config, envs: pass use_replayssm to the breakable-graph fallback; register VLLM_FAIL_ON_PROFILING_KV_LEAK CompilationConfig.resolve_cudagraph_mode_and_sizes read self.cache_config.use_replayssm, but CompilationConfig has no cache_config, so with VLLM_USE_BREAKABLE_CUDAGRAPH=1 the fallback branch raised AttributeError. Both model runners now pass cache_config.use_replayssm as an argument. The shipped launchers set VLLM_USE_BREAKABLE_CUDAGRAPH=0 and never reach this branch. VLLM_FAIL_ON_PROFILING_KV_LEAK was read with a raw os.getenv, so setting it printed the unknown-variable warning. It is now registered in vllm.envs with the same spellings (unset fails the boot; 0, false or no warn instead), listed as an ignored compile factor and read through vllm.envs. Tests cover both. Co-Authored-By: Hinata Signed-off-by: MOVZX --- vllm/config/compilation.py | 4 +++- vllm/envs.py | 13 +++++++++++++ vllm/v1/worker/gpu/cudagraph_utils.py | 6 ++---- vllm/v1/worker/gpu/model_runner.py | 1 + vllm/v1/worker/gpu_model_runner.py | 1 + 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 70d04f93fc..c13a885a9f 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -1390,6 +1390,7 @@ class CompilationConfig: max_num_reqs: int | None = None, is_profiling: bool = False, piecewise_capture_available: bool = True, + use_replayssm: bool = False, ) -> CUDAGraphMode: from vllm.v1.attention.backend import AttentionCGSupport @@ -1443,7 +1444,8 @@ class CompilationConfig: ) ) or ( envs.VLLM_USE_BREAKABLE_CUDAGRAPH - and self.cache_config.use_replayssm + # CompilationConfig has no cache_config; the runner passes the flag. + and use_replayssm ): # Breakable CUDA graphs break at the attention kernels at runtime, # with or without torch.compile. diff --git a/vllm/envs.py b/vllm/envs.py index 4f5faeb6e7..7c5b0b4694 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -471,6 +471,11 @@ if TYPE_CHECKING: # it changes the ring views' block stride and which block table the PLE # state is read through, so ON and OFF must not share a compile cache. VLLM_E2_SIDE_CACHE_LAYOUT: bool = True + # Profiling-teardown leak policy (vllm/v1/worker/gpu/cudagraph_utils.py): a profiling + # KV cache still allocated after teardown fails the boot by default; "0", "false" or + # "no" downgrade it to a warning while diagnosing. AOT: no. CAPTURE: no -- it only + # decides whether a host-side check raises. + VLLM_FAIL_ON_PROFILING_KV_LEAK: bool = True # Opt-out for the fork startup guards in vllm/v1/e2_guards.py, read at # Qwen4ExpForCausalLM construction. AOT: no. CAPTURE: no -- it changes # nothing itself; it only permits a boot whose identity IS different @@ -2712,6 +2717,11 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_E2_SIDE_CACHE_LAYOUT": lambda: _e2_strict_switch( "VLLM_E2_SIDE_CACHE_LAYOUT", default=True ), + # Profiling-teardown leak policy: fail the boot unless "0"/"false"/"no". + "VLLM_FAIL_ON_PROFILING_KV_LEAK": lambda: os.getenv( + "VLLM_FAIL_ON_PROFILING_KV_LEAK", "1" + ).strip().lower() + not in ("0", "false", "no"), # Seconds between `topk-exact-counters` lines from each GPU worker # (0, the default, disables the reader entirely). The line reports the # medium-path exact-restart counters; see @@ -2967,6 +2977,9 @@ def compile_factors() -> dict[str, object]: # compiled and the worker's tables are identical in both worlds, so an # A/B flip must reuse the same AOT cache. "VLLM_E2_MAMBA_RETAIN_CHECKPOINT", + # Boot-failure policy of a host-side profiling-teardown check; read by no + # compiled code. + "VLLM_FAIL_ON_PROFILING_KV_LEAK", # Logging cadence for an already-compiled kernel's counters: it gates a # host-side timer and a snapshot op, and is read by no compiled code, so # hashing it would invalidate every compile cache for a log interval. diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 9be87cfa82..b0c15824a3 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -3,7 +3,6 @@ import contextlib import gc import itertools -import os from collections import defaultdict from collections.abc import Callable, Iterable, Iterator from contextlib import ExitStack @@ -15,6 +14,7 @@ import torch import torch.nn as nn from tqdm import tqdm +import vllm.envs as envs from vllm.compilation.breakable_cudagraph import ( BreakableCUDAGraphWrapper, is_breakable_cudagraph_enabled, @@ -1215,8 +1215,6 @@ def _teardown_profiling_state(runner: "GPUModelRunner") -> None: # Fail boot by default: this memory is absent from the real KV # pool and can invalidate capacity planning. Operators may opt out # temporarily while diagnosing with VLLM_FAIL_ON_PROFILING_KV_LEAK=0. - if os.getenv("VLLM_FAIL_ON_PROFILING_KV_LEAK", "1").lower() not in ( - "0", "false", "no" - ): + if envs.VLLM_FAIL_ON_PROFILING_KV_LEAK: raise RuntimeError(message) logger.warning(message) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index eb1ac4f9e0..143451dc95 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -804,6 +804,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): max_num_reqs=self.max_num_reqs, is_profiling=is_profiling, piecewise_capture_available=piecewise_capture_available, + use_replayssm=self.cache_config.use_replayssm, ) self.cudagraph_manager = ModelCudaGraphManager( self.vllm_config, diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 93a30eb4e9..10449739af 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -7132,6 +7132,7 @@ class GPUModelRunner( kv_cache_config=self.kv_cache_config, max_num_reqs=self.max_num_reqs, is_profiling=is_profiling, + use_replayssm=self.cache_config.use_replayssm, ) # Trigger cudagraph dispatching keys initialization after # resolved cudagraph mode.