From 4cfd09ebda863817480377256c307f6d0de64f02 Mon Sep 17 00:00:00 2001 From: MOVZX Date: Thu, 1 Oct 2026 21:29:59 -0400 Subject: [PATCH] [Worker] Report a profiling KV cache that survives the teardown _teardown_profiling_state measures the storage behind the profiling KV caches on entry and the allocated memory before and after each release, and logs any profiling KV cache that is still alive afterwards. Co-Authored-By: Hinata Signed-off-by: MOVZX --- vllm/v1/worker/gpu/cudagraph_utils.py | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index cac4328c7f..de9b8d7f14 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -1127,10 +1127,35 @@ def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None: runner.cache_config.num_gpu_blocks = minimal_config.num_blocks +def _profiling_kv_cache_bytes(kv_caches: Iterable[Any]) -> int: + """Bytes of the distinct CUDA storages behind the runner's KV cache views.""" + storages: dict[int, int] = {} + for cache in kv_caches: + for t in cache if isinstance(cache, (list, tuple)) else (cache,): + if isinstance(t, torch.Tensor) and t.is_cuda: + st = t.untyped_storage() + storages[st.data_ptr()] = st.nbytes() + return sum(storages.values()) + + +def _profiling_kv_still_held( + kv_bytes: int, allocated_before: int, allocated_after: int +) -> int: + """Lower bound on the profiling KV cache bytes still allocated after the + teardown: whatever the teardown freed counts against the cache first, so a + positive result means something still references the cache (the RecoverSSM + commit-context class of leak, ~26 GiB on H200; against a sub-GiB per-card + per-card cushion even a small one matters). One-sided: never a false alarm, + may miss a leak masked by other frees.""" + return max(kv_bytes - max(allocated_before - allocated_after, 0), 0) + + def _teardown_profiling_state(runner: "GPUModelRunner") -> None: """Release the profiling KV cache and captured graphs while keeping model weights, so the real ``initialize_kv_cache`` starts from a clean slate.""" torch.accelerator.synchronize() + kv_bytes = _profiling_kv_cache_bytes(getattr(runner, "kv_caches", ())) + allocated_before = torch.cuda.memory_allocated() if kv_bytes else 0 if hasattr(runner.model_state, "_mamba_ctx"): runner.model_state._mamba_ctx = None # A PIECEWISE profiling capture records its RecoverSSM step (only FULL @@ -1173,3 +1198,18 @@ def _teardown_profiling_state(runner: "GPUModelRunner") -> None: runner.maybe_remove_all_loras(runner.lora_config) gc.collect() torch.accelerator.empty_cache() + if kv_bytes: + # last, after every release above (one-sided check) + gc.collect() + torch.accelerator.synchronize() + held = _profiling_kv_still_held( + kv_bytes, allocated_before, torch.cuda.memory_allocated() + ) + if held > (1 << 20): + logger.warning( + "Profiling teardown: at least %.1f MiB of the %.1f MiB profiling KV " + "cache is still allocated; something still references it (it will " + "be missing from the real KV cache / free-memory cushion)", + held / (1 << 20), + kv_bytes / (1 << 20), + )