[ROCm][CI] Make memory sampling less racy in tests and sleep mode (#45490)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Codex <codex@example.invalid> Co-authored-by: Codex <codex@example.invalid>
This commit is contained in:
co-authored by
Codex
parent
ebcf511ec3
commit
4eb227992a
+46
-4
@@ -1501,6 +1501,9 @@ def wait_for_gpu_memory_to_clear(
|
||||
threshold_bytes: int | dict[int, int] | None = None,
|
||||
threshold_ratio: float | dict[int, float] | None = None,
|
||||
timeout_s: float = 120,
|
||||
stable_duration_s: float = 0,
|
||||
stable_tolerance_bytes: int = 512 * 1024**2,
|
||||
poll_interval_s: float = 5,
|
||||
) -> None:
|
||||
assert threshold_bytes is not None or threshold_ratio is not None
|
||||
devices = get_physical_device_indices(devices)
|
||||
@@ -1528,8 +1531,13 @@ def wait_for_gpu_memory_to_clear(
|
||||
# Use nvml instead of pytorch to reduce measurement error from torch cuda
|
||||
# context.
|
||||
start_time = time.time()
|
||||
stable_since: float | None = None
|
||||
stable_used_bytes: dict[int, int] | None = None
|
||||
while True:
|
||||
output_raw = record_gpu_memory_usage_stats(devices=devices)
|
||||
used_bytes_by_device = {
|
||||
device: int(gb_used * 2**30) for device, (gb_used, _) in output_raw.items()
|
||||
}
|
||||
output = {
|
||||
device: f"{gb_used:.02f}/{gb_total:.02f}"
|
||||
for device, (gb_used, gb_total) in output_raw.items()
|
||||
@@ -1577,15 +1585,45 @@ def wait_for_gpu_memory_to_clear(
|
||||
|
||||
dur_s = time.time() - start_time
|
||||
if all_free:
|
||||
print(f"Done waiting for free GPU memory on ({threshold=}) {dur_s=:.02f}")
|
||||
break
|
||||
if stable_duration_s <= 0:
|
||||
print(
|
||||
f"Done waiting for free GPU memory on devices {devices=} "
|
||||
f"({threshold=}) {dur_s=:.02f}"
|
||||
)
|
||||
break
|
||||
|
||||
now = time.time()
|
||||
if stable_used_bytes is None:
|
||||
stable_since = now
|
||||
stable_used_bytes = used_bytes_by_device
|
||||
else:
|
||||
memory_changed = any(
|
||||
abs(used_bytes_by_device[device] - stable_used_bytes[device])
|
||||
> stable_tolerance_bytes
|
||||
for device in devices
|
||||
)
|
||||
if memory_changed:
|
||||
stable_since = now
|
||||
stable_used_bytes = used_bytes_by_device
|
||||
elif (
|
||||
stable_since is not None and now - stable_since >= stable_duration_s
|
||||
):
|
||||
print(
|
||||
f"Done waiting for stable free GPU memory on devices "
|
||||
f"{devices=} ({threshold=}) {dur_s=:.02f}"
|
||||
)
|
||||
break
|
||||
else:
|
||||
stable_since = None
|
||||
stable_used_bytes = None
|
||||
|
||||
if dur_s >= timeout_s:
|
||||
raise ValueError(
|
||||
f"Memory of devices not free after {dur_s=:.02f} ({threshold=})"
|
||||
f"Memory of devices {devices=} not free after "
|
||||
f"{dur_s=:.02f} ({threshold=})"
|
||||
)
|
||||
|
||||
time.sleep(5)
|
||||
time.sleep(poll_interval_s)
|
||||
|
||||
|
||||
def wait_for_rocm_memory_to_settle(
|
||||
@@ -1606,11 +1644,15 @@ def wait_for_rocm_memory_to_settle(
|
||||
num_gpus = current_platform.device_count()
|
||||
if num_gpus == 0:
|
||||
return
|
||||
if threshold_ratio is None:
|
||||
threshold_ratio = 0.1
|
||||
|
||||
wait_for_gpu_memory_to_clear(
|
||||
devices=list(range(num_gpus)),
|
||||
threshold_ratio=threshold_ratio,
|
||||
timeout_s=timeout_s,
|
||||
stable_duration_s=2.0,
|
||||
poll_interval_s=1.0,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import gc
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||
from datetime import timedelta
|
||||
@@ -170,7 +171,8 @@ class Worker(WorkerBase):
|
||||
self._pp_send_work: list[Handle] = []
|
||||
|
||||
def sleep(self, level: int = 1) -> None:
|
||||
free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
|
||||
torch.accelerator.synchronize()
|
||||
free_bytes_before_sleep = current_platform.mem_get_info()[0]
|
||||
|
||||
# Save the buffers before level 2 sleep
|
||||
if level == 2:
|
||||
@@ -181,8 +183,16 @@ class Worker(WorkerBase):
|
||||
|
||||
allocator = get_mem_allocator_instance()
|
||||
allocator.sleep(offload_tags=("weights",) if level == 1 else tuple())
|
||||
free_bytes_after_sleep, total = torch.cuda.mem_get_info()
|
||||
freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
deadline = time.monotonic() + (5.0 if current_platform.is_rocm() else 0)
|
||||
while True:
|
||||
free_bytes_after_sleep, total = current_platform.mem_get_info()
|
||||
freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
|
||||
if freed_bytes >= 0 or time.monotonic() >= deadline:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
|
||||
used_bytes = total - free_bytes_after_sleep
|
||||
assert freed_bytes >= 0, "Memory usage increased after sleeping."
|
||||
logger.info(
|
||||
|
||||
Reference in New Issue
Block a user