From 4eb227992aa2231ad538b8c90bc8191397ba3697 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Mon, 29 Jun 2026 16:26:41 -0500 Subject: [PATCH] [ROCm][CI] Make memory sampling less racy in tests and sleep mode (#45490) Signed-off-by: Andreas Karatzas Signed-off-by: Codex Co-authored-by: Codex --- tests/utils.py | 50 +++++++++++++++++++++++++++++++++--- vllm/v1/worker/gpu_worker.py | 16 +++++++++--- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 2acb971630..07cda56ce0 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1501,6 +1501,9 @@ def wait_for_gpu_memory_to_clear( threshold_bytes: int | dict[int, int] | None = None, threshold_ratio: float | dict[int, float] | None = None, timeout_s: float = 120, + stable_duration_s: float = 0, + stable_tolerance_bytes: int = 512 * 1024**2, + poll_interval_s: float = 5, ) -> None: assert threshold_bytes is not None or threshold_ratio is not None devices = get_physical_device_indices(devices) @@ -1528,8 +1531,13 @@ def wait_for_gpu_memory_to_clear( # Use nvml instead of pytorch to reduce measurement error from torch cuda # context. start_time = time.time() + stable_since: float | None = None + stable_used_bytes: dict[int, int] | None = None while True: output_raw = record_gpu_memory_usage_stats(devices=devices) + used_bytes_by_device = { + device: int(gb_used * 2**30) for device, (gb_used, _) in output_raw.items() + } output = { device: f"{gb_used:.02f}/{gb_total:.02f}" for device, (gb_used, gb_total) in output_raw.items() @@ -1577,15 +1585,45 @@ def wait_for_gpu_memory_to_clear( dur_s = time.time() - start_time if all_free: - print(f"Done waiting for free GPU memory on ({threshold=}) {dur_s=:.02f}") - break + if stable_duration_s <= 0: + print( + f"Done waiting for free GPU memory on devices {devices=} " + f"({threshold=}) {dur_s=:.02f}" + ) + break + + now = time.time() + if stable_used_bytes is None: + stable_since = now + stable_used_bytes = used_bytes_by_device + else: + memory_changed = any( + abs(used_bytes_by_device[device] - stable_used_bytes[device]) + > stable_tolerance_bytes + for device in devices + ) + if memory_changed: + stable_since = now + stable_used_bytes = used_bytes_by_device + elif ( + stable_since is not None and now - stable_since >= stable_duration_s + ): + print( + f"Done waiting for stable free GPU memory on devices " + f"{devices=} ({threshold=}) {dur_s=:.02f}" + ) + break + else: + stable_since = None + stable_used_bytes = None if dur_s >= timeout_s: raise ValueError( - f"Memory of devices not free after {dur_s=:.02f} ({threshold=})" + f"Memory of devices {devices=} not free after " + f"{dur_s=:.02f} ({threshold=})" ) - time.sleep(5) + time.sleep(poll_interval_s) def wait_for_rocm_memory_to_settle( @@ -1606,11 +1644,15 @@ def wait_for_rocm_memory_to_settle( num_gpus = current_platform.device_count() if num_gpus == 0: return + if threshold_ratio is None: + threshold_ratio = 0.1 wait_for_gpu_memory_to_clear( devices=list(range(num_gpus)), threshold_ratio=threshold_ratio, timeout_s=timeout_s, + stable_duration_s=2.0, + poll_interval_s=1.0, ) diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 9afc235252..07c3615edc 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -4,6 +4,7 @@ import gc import os +import time from collections.abc import Callable from contextlib import AbstractContextManager, contextmanager, nullcontext from datetime import timedelta @@ -170,7 +171,8 @@ class Worker(WorkerBase): self._pp_send_work: list[Handle] = [] def sleep(self, level: int = 1) -> None: - free_bytes_before_sleep = torch.cuda.mem_get_info()[0] + torch.accelerator.synchronize() + free_bytes_before_sleep = current_platform.mem_get_info()[0] # Save the buffers before level 2 sleep if level == 2: @@ -181,8 +183,16 @@ class Worker(WorkerBase): allocator = get_mem_allocator_instance() allocator.sleep(offload_tags=("weights",) if level == 1 else tuple()) - free_bytes_after_sleep, total = torch.cuda.mem_get_info() - freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep + + torch.accelerator.synchronize() + deadline = time.monotonic() + (5.0 if current_platform.is_rocm() else 0) + while True: + free_bytes_after_sleep, total = current_platform.mem_get_info() + freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep + if freed_bytes >= 0 or time.monotonic() >= deadline: + break + time.sleep(0.1) + used_bytes = total - free_bytes_after_sleep assert freed_bytes >= 0, "Memory usage increased after sleeping." logger.info(