[Worker] Report a profiling KV cache that survives the teardown

_teardown_profiling_state measures the storage behind the profiling KV caches on entry and the allocated memory
before and after each release, and logs any profiling KV cache that is still alive afterwards.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 20:08:57 +07:00
co-authored by Hinata
parent 812d662602
commit 4cfd09ebda
+40
View File
@@ -1127,10 +1127,35 @@ def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None:
runner.cache_config.num_gpu_blocks = minimal_config.num_blocks
def _profiling_kv_cache_bytes(kv_caches: Iterable[Any]) -> int:
"""Bytes of the distinct CUDA storages behind the runner's KV cache views."""
storages: dict[int, int] = {}
for cache in kv_caches:
for t in cache if isinstance(cache, (list, tuple)) else (cache,):
if isinstance(t, torch.Tensor) and t.is_cuda:
st = t.untyped_storage()
storages[st.data_ptr()] = st.nbytes()
return sum(storages.values())
def _profiling_kv_still_held(
kv_bytes: int, allocated_before: int, allocated_after: int
) -> int:
"""Lower bound on the profiling KV cache bytes still allocated after the
teardown: whatever the teardown freed counts against the cache first, so a
positive result means something still references the cache (the RecoverSSM
commit-context class of leak, ~26 GiB on H200; against a sub-GiB per-card
per-card cushion even a small one matters). One-sided: never a false alarm,
may miss a leak masked by other frees."""
return max(kv_bytes - max(allocated_before - allocated_after, 0), 0)
def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
"""Release the profiling KV cache and captured graphs while keeping model
weights, so the real ``initialize_kv_cache`` starts from a clean slate."""
torch.accelerator.synchronize()
kv_bytes = _profiling_kv_cache_bytes(getattr(runner, "kv_caches", ()))
allocated_before = torch.cuda.memory_allocated() if kv_bytes else 0
if hasattr(runner.model_state, "_mamba_ctx"):
runner.model_state._mamba_ctx = None
# A PIECEWISE profiling capture records its RecoverSSM step (only FULL
@@ -1173,3 +1198,18 @@ def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
runner.maybe_remove_all_loras(runner.lora_config)
gc.collect()
torch.accelerator.empty_cache()
if kv_bytes:
# last, after every release above (one-sided check)
gc.collect()
torch.accelerator.synchronize()
held = _profiling_kv_still_held(
kv_bytes, allocated_before, torch.cuda.memory_allocated()
)
if held > (1 << 20):
logger.warning(
"Profiling teardown: at least %.1f MiB of the %.1f MiB profiling KV "
"cache is still allocated; something still references it (it will "
"be missing from the real KV cache / free-memory cushion)",
held / (1 << 20),
kv_bytes / (1 << 20),
)