From 8c518d4f09245e839a5a80c00b7ccd325d48f08e Mon Sep 17 00:00:00 2001 From: MOVZX Date: Fri, 2 Oct 2026 04:07:45 +0100 Subject: [PATCH] [Worker] Gate the graph fallback and fail on profiling leaks The breakable-CUDA-graph fallback in the compilation config now applies only with --use-replayssm, and a profiling KV cache that survives the teardown fails the boot instead of only being logged (VLLM_FAIL_ON_PROFILING_KV_LEAK=0 turns it back into a warning). Co-Authored-By: Hinata Signed-off-by: MOVZX --- vllm/config/compilation.py | 5 ++++- vllm/v1/worker/gpu/cudagraph_utils.py | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index e5b0f16d0d..70d04f93fc 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -1441,7 +1441,10 @@ class CompilationConfig: self.splitting_ops_contain_attention() or self.use_inductor_graph_partition ) - ) or envs.VLLM_USE_BREAKABLE_CUDAGRAPH: + ) or ( + envs.VLLM_USE_BREAKABLE_CUDAGRAPH + and self.cache_config.use_replayssm + ): # Breakable CUDA graphs break at the attention kernels at runtime, # with or without torch.compile. msg += ( diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index de9b8d7f14..9be87cfa82 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -3,6 +3,7 @@ import contextlib import gc import itertools +import os from collections import defaultdict from collections.abc import Callable, Iterable, Iterator from contextlib import ExitStack @@ -1206,10 +1207,16 @@ def _teardown_profiling_state(runner: "GPUModelRunner") -> None: kv_bytes, allocated_before, torch.cuda.memory_allocated() ) if held > (1 << 20): - logger.warning( + message = ( "Profiling teardown: at least %.1f MiB of the %.1f MiB profiling KV " "cache is still allocated; something still references it (it will " - "be missing from the real KV cache / free-memory cushion)", - held / (1 << 20), - kv_bytes / (1 << 20), - ) + "be missing from the real KV cache / free-memory cushion)" + ) % (held / (1 << 20), kv_bytes / (1 << 20)) + # Fail boot by default: this memory is absent from the real KV + # pool and can invalidate capacity planning. Operators may opt out + # temporarily while diagnosing with VLLM_FAIL_ON_PROFILING_KV_LEAK=0. + if os.getenv("VLLM_FAIL_ON_PROFILING_KV_LEAK", "1").lower() not in ( + "0", "false", "no" + ): + raise RuntimeError(message) + logger.warning(message)