[ROCm][CI] Make memory sampling less racy in tests and sleep mode (#45490)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Codex <codex@example.invalid>
Co-authored-by: Codex <codex@example.invalid>
This commit is contained in:
Andreas Karatzas
2026-06-29 14:26:41 -07:00
committed by GitHub
co-authored by Codex
parent ebcf511ec3
commit 4eb227992a
2 changed files with 59 additions and 7 deletions
+46 -4
View File
@@ -1501,6 +1501,9 @@ def wait_for_gpu_memory_to_clear(
threshold_bytes: int | dict[int, int] | None = None,
threshold_ratio: float | dict[int, float] | None = None,
timeout_s: float = 120,
stable_duration_s: float = 0,
stable_tolerance_bytes: int = 512 * 1024**2,
poll_interval_s: float = 5,
) -> None:
assert threshold_bytes is not None or threshold_ratio is not None
devices = get_physical_device_indices(devices)
@@ -1528,8 +1531,13 @@ def wait_for_gpu_memory_to_clear(
# Use nvml instead of pytorch to reduce measurement error from torch cuda
# context.
start_time = time.time()
stable_since: float | None = None
stable_used_bytes: dict[int, int] | None = None
while True:
output_raw = record_gpu_memory_usage_stats(devices=devices)
used_bytes_by_device = {
device: int(gb_used * 2**30) for device, (gb_used, _) in output_raw.items()
}
output = {
device: f"{gb_used:.02f}/{gb_total:.02f}"
for device, (gb_used, gb_total) in output_raw.items()
@@ -1577,15 +1585,45 @@ def wait_for_gpu_memory_to_clear(
dur_s = time.time() - start_time
if all_free:
print(f"Done waiting for free GPU memory on ({threshold=}) {dur_s=:.02f}")
break
if stable_duration_s <= 0:
print(
f"Done waiting for free GPU memory on devices {devices=} "
f"({threshold=}) {dur_s=:.02f}"
)
break
now = time.time()
if stable_used_bytes is None:
stable_since = now
stable_used_bytes = used_bytes_by_device
else:
memory_changed = any(
abs(used_bytes_by_device[device] - stable_used_bytes[device])
> stable_tolerance_bytes
for device in devices
)
if memory_changed:
stable_since = now
stable_used_bytes = used_bytes_by_device
elif (
stable_since is not None and now - stable_since >= stable_duration_s
):
print(
f"Done waiting for stable free GPU memory on devices "
f"{devices=} ({threshold=}) {dur_s=:.02f}"
)
break
else:
stable_since = None
stable_used_bytes = None
if dur_s >= timeout_s:
raise ValueError(
f"Memory of devices not free after {dur_s=:.02f} ({threshold=})"
f"Memory of devices {devices=} not free after "
f"{dur_s=:.02f} ({threshold=})"
)
time.sleep(5)
time.sleep(poll_interval_s)
def wait_for_rocm_memory_to_settle(
@@ -1606,11 +1644,15 @@ def wait_for_rocm_memory_to_settle(
num_gpus = current_platform.device_count()
if num_gpus == 0:
return
if threshold_ratio is None:
threshold_ratio = 0.1
wait_for_gpu_memory_to_clear(
devices=list(range(num_gpus)),
threshold_ratio=threshold_ratio,
timeout_s=timeout_s,
stable_duration_s=2.0,
poll_interval_s=1.0,
)
+13 -3
View File
@@ -4,6 +4,7 @@
import gc
import os
import time
from collections.abc import Callable
from contextlib import AbstractContextManager, contextmanager, nullcontext
from datetime import timedelta
@@ -170,7 +171,8 @@ class Worker(WorkerBase):
self._pp_send_work: list[Handle] = []
def sleep(self, level: int = 1) -> None:
free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
torch.accelerator.synchronize()
free_bytes_before_sleep = current_platform.mem_get_info()[0]
# Save the buffers before level 2 sleep
if level == 2:
@@ -181,8 +183,16 @@ class Worker(WorkerBase):
allocator = get_mem_allocator_instance()
allocator.sleep(offload_tags=("weights",) if level == 1 else tuple())
free_bytes_after_sleep, total = torch.cuda.mem_get_info()
freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
torch.accelerator.synchronize()
deadline = time.monotonic() + (5.0 if current_platform.is_rocm() else 0)
while True:
free_bytes_after_sleep, total = current_platform.mem_get_info()
freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
if freed_bytes >= 0 or time.monotonic() >= deadline:
break
time.sleep(0.1)
used_bytes = total - free_bytes_after_sleep
assert freed_bytes >= 0, "Memory usage increased after sleeping."
logger.info(