[ROCm][CI] Keep startup profiling from aborting when free memory grows (#53591)
Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
This commit is contained in:
@@ -66,7 +66,12 @@ from vllm.tracing import instrument
|
||||
from vllm.utils.gc_utils import freeze_gc_heap, maybe_attach_gc_debug_callback
|
||||
from vllm.utils.gpu_sync_debug import enable_gpu_sync_check, with_gpu_sync_check
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.utils.mem_utils import MemorySnapshot, format_gib, memory_profiling
|
||||
from vllm.utils.mem_utils import (
|
||||
MemoryProfilingResult,
|
||||
MemorySnapshot,
|
||||
format_gib,
|
||||
memory_profiling,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed, set_torch_threads_for_runtime
|
||||
from vllm.v1.attention.backends.utils import record_kv_cache_layout
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
@@ -102,6 +107,33 @@ def _num_workspace_lanes(vllm_config: VllmConfig, use_v2_model_runner: bool) ->
|
||||
)
|
||||
|
||||
|
||||
def maybe_rocm_profiling_fallback(profile_result: MemoryProfilingResult) -> int | None:
|
||||
"""Memory in bytes to size the KV cache from when profiling measured a
|
||||
release on ROCm, or None to keep what profiling measured.
|
||||
|
||||
The measurement is the drop in free device memory across startup, so it comes
|
||||
out negative when anything else on the device released memory meanwhile. Torch
|
||||
reserved is process local, so it stands in as a lower bound instead of refusing
|
||||
to start. Kept to ROCm, where the AMD CI groups hit this, following #45490.
|
||||
"""
|
||||
if profile_result.total_consumed >= 0 or not current_platform.is_rocm():
|
||||
return None
|
||||
|
||||
baseline = profile_result.before_create
|
||||
after_profile = profile_result.after_profile
|
||||
torch_reserved = max(after_profile.torch_memory - baseline.torch_memory, 0)
|
||||
logger.warning(
|
||||
"Free memory grew by %s GiB during profiling (initial %s GiB, current "
|
||||
"%s GiB). Sizing the KV cache from the %s GiB torch reserved instead of "
|
||||
"the profiling difference.",
|
||||
format_gib(-profile_result.total_consumed),
|
||||
format_gib(baseline.free_memory),
|
||||
format_gib(after_profile.free_memory),
|
||||
format_gib(torch_reserved),
|
||||
)
|
||||
return torch_reserved
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.device_allocator.sleep_mode_backend import SleepModeBackend
|
||||
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
|
||||
@@ -543,24 +575,33 @@ class Worker(WorkerBase):
|
||||
else 0
|
||||
)
|
||||
|
||||
init_free_memory = self.init_snapshot.free_memory
|
||||
free_gpu_memory = profile_result.after_profile.free_memory
|
||||
rocm_fallback = maybe_rocm_profiling_fallback(profile_result)
|
||||
if rocm_fallback is None:
|
||||
# NOTE(woosuk): Here we assume that the other processes using the same
|
||||
# GPU did not change their memory usage during the profiling.
|
||||
assert init_free_memory >= free_gpu_memory, (
|
||||
"Error in memory profiling. "
|
||||
f"Initial free memory {format_gib(init_free_memory)} GiB, "
|
||||
f"current free memory {format_gib(free_gpu_memory)} GiB. "
|
||||
"This happens when other processes sharing the same container "
|
||||
"release GPU memory while vLLM is profiling during initialization. "
|
||||
"To fix this, ensure consistent GPU memory allocation or "
|
||||
"isolate vLLM in its own container."
|
||||
)
|
||||
else:
|
||||
profile_result.total_consumed = rocm_fallback
|
||||
profile_result.non_kv_cache_memory = (
|
||||
profile_result.total_consumed + profile_result.transient_peak_headroom
|
||||
)
|
||||
|
||||
self.total_consumed = profile_result.total_consumed
|
||||
self.peak_activation_memory = (
|
||||
profile_result.transient_peak_headroom + cudagraph_memory_estimate_applied
|
||||
)
|
||||
self.cudagraph_memory_estimate = cudagraph_memory_estimate
|
||||
|
||||
free_gpu_memory = profile_result.after_profile.free_memory
|
||||
# NOTE(woosuk): Here we assume that the other processes using the same
|
||||
# GPU did not change their memory usage during the profiling.
|
||||
assert self.init_snapshot.free_memory >= free_gpu_memory, (
|
||||
"Error in memory profiling. "
|
||||
f"Initial free memory {format_gib(self.init_snapshot.free_memory)} GiB, "
|
||||
f"current free memory {format_gib(free_gpu_memory)} GiB. "
|
||||
"This happens when other processes sharing the same container "
|
||||
"release GPU memory while vLLM is profiling during initialization. "
|
||||
"To fix this, ensure consistent GPU memory allocation or "
|
||||
"isolate vLLM in its own container."
|
||||
)
|
||||
self.available_kv_cache_memory_bytes = (
|
||||
self.requested_memory
|
||||
- profile_result.non_kv_cache_memory
|
||||
|
||||
Reference in New Issue
Block a user