diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index cac4328c7f..de9b8d7f14 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -1127,10 +1127,35 @@ def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None: runner.cache_config.num_gpu_blocks = minimal_config.num_blocks +def _profiling_kv_cache_bytes(kv_caches: Iterable[Any]) -> int: + """Bytes of the distinct CUDA storages behind the runner's KV cache views.""" + storages: dict[int, int] = {} + for cache in kv_caches: + for t in cache if isinstance(cache, (list, tuple)) else (cache,): + if isinstance(t, torch.Tensor) and t.is_cuda: + st = t.untyped_storage() + storages[st.data_ptr()] = st.nbytes() + return sum(storages.values()) + + +def _profiling_kv_still_held( + kv_bytes: int, allocated_before: int, allocated_after: int +) -> int: + """Lower bound on the profiling KV cache bytes still allocated after the + teardown: whatever the teardown freed counts against the cache first, so a + positive result means something still references the cache (the RecoverSSM + commit-context class of leak, ~26 GiB on H200; against a sub-GiB per-card + per-card cushion even a small one matters). One-sided: never a false alarm, + may miss a leak masked by other frees.""" + return max(kv_bytes - max(allocated_before - allocated_after, 0), 0) + + def _teardown_profiling_state(runner: "GPUModelRunner") -> None: """Release the profiling KV cache and captured graphs while keeping model weights, so the real ``initialize_kv_cache`` starts from a clean slate.""" torch.accelerator.synchronize() + kv_bytes = _profiling_kv_cache_bytes(getattr(runner, "kv_caches", ())) + allocated_before = torch.cuda.memory_allocated() if kv_bytes else 0 if hasattr(runner.model_state, "_mamba_ctx"): runner.model_state._mamba_ctx = None # A PIECEWISE profiling capture records its RecoverSSM step (only FULL @@ -1173,3 +1198,18 @@ def _teardown_profiling_state(runner: "GPUModelRunner") -> None: runner.maybe_remove_all_loras(runner.lora_config) gc.collect() torch.accelerator.empty_cache() + if kv_bytes: + # last, after every release above (one-sided check) + gc.collect() + torch.accelerator.synchronize() + held = _profiling_kv_still_held( + kv_bytes, allocated_before, torch.cuda.memory_allocated() + ) + if held > (1 << 20): + logger.warning( + "Profiling teardown: at least %.1f MiB of the %.1f MiB profiling KV " + "cache is still allocated; something still references it (it will " + "be missing from the real KV cache / free-memory cushion)", + held / (1 << 20), + kv_bytes / (1 << 20), + )