[Worker] Gate the graph fallback and fail on profiling leaks
The breakable-CUDA-graph fallback in the compilation config now applies only with --use-replayssm, and a profiling KV cache that survives the teardown fails the boot instead of only being logged (VLLM_FAIL_ON_PROFILING_KV_LEAK=0 turns it back into a warning). Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1441,7 +1441,10 @@ class CompilationConfig:
|
||||
self.splitting_ops_contain_attention()
|
||||
or self.use_inductor_graph_partition
|
||||
)
|
||||
) or envs.VLLM_USE_BREAKABLE_CUDAGRAPH:
|
||||
) or (
|
||||
envs.VLLM_USE_BREAKABLE_CUDAGRAPH
|
||||
and self.cache_config.use_replayssm
|
||||
):
|
||||
# Breakable CUDA graphs break at the attention kernels at runtime,
|
||||
# with or without torch.compile.
|
||||
msg += (
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import contextlib
|
||||
import gc
|
||||
import itertools
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from contextlib import ExitStack
|
||||
@@ -1206,10 +1207,16 @@ def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
|
||||
kv_bytes, allocated_before, torch.cuda.memory_allocated()
|
||||
)
|
||||
if held > (1 << 20):
|
||||
logger.warning(
|
||||
message = (
|
||||
"Profiling teardown: at least %.1f MiB of the %.1f MiB profiling KV "
|
||||
"cache is still allocated; something still references it (it will "
|
||||
"be missing from the real KV cache / free-memory cushion)",
|
||||
held / (1 << 20),
|
||||
kv_bytes / (1 << 20),
|
||||
)
|
||||
"be missing from the real KV cache / free-memory cushion)"
|
||||
) % (held / (1 << 20), kv_bytes / (1 << 20))
|
||||
# Fail boot by default: this memory is absent from the real KV
|
||||
# pool and can invalidate capacity planning. Operators may opt out
|
||||
# temporarily while diagnosing with VLLM_FAIL_ON_PROFILING_KV_LEAK=0.
|
||||
if os.getenv("VLLM_FAIL_ON_PROFILING_KV_LEAK", "1").lower() not in (
|
||||
"0", "false", "no"
|
||||
):
|
||||
raise RuntimeError(message)
|
||||
logger.warning(message)
|
||||
|
||||
Reference in New Issue
Block a user