[ROCm][CI] Make memory sampling less racy in tests and sleep mode (#45490)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Codex <codex@example.invalid> Co-authored-by: Codex <codex@example.invalid>
This commit is contained in:
co-authored by
Codex
parent
ebcf511ec3
commit
4eb227992a
+46
-4
@@ -1501,6 +1501,9 @@ def wait_for_gpu_memory_to_clear(
|
||||
threshold_bytes: int | dict[int, int] | None = None,
|
||||
threshold_ratio: float | dict[int, float] | None = None,
|
||||
timeout_s: float = 120,
|
||||
stable_duration_s: float = 0,
|
||||
stable_tolerance_bytes: int = 512 * 1024**2,
|
||||
poll_interval_s: float = 5,
|
||||
) -> None:
|
||||
assert threshold_bytes is not None or threshold_ratio is not None
|
||||
devices = get_physical_device_indices(devices)
|
||||
@@ -1528,8 +1531,13 @@ def wait_for_gpu_memory_to_clear(
|
||||
# Use nvml instead of pytorch to reduce measurement error from torch cuda
|
||||
# context.
|
||||
start_time = time.time()
|
||||
stable_since: float | None = None
|
||||
stable_used_bytes: dict[int, int] | None = None
|
||||
while True:
|
||||
output_raw = record_gpu_memory_usage_stats(devices=devices)
|
||||
used_bytes_by_device = {
|
||||
device: int(gb_used * 2**30) for device, (gb_used, _) in output_raw.items()
|
||||
}
|
||||
output = {
|
||||
device: f"{gb_used:.02f}/{gb_total:.02f}"
|
||||
for device, (gb_used, gb_total) in output_raw.items()
|
||||
@@ -1577,15 +1585,45 @@ def wait_for_gpu_memory_to_clear(
|
||||
|
||||
dur_s = time.time() - start_time
|
||||
if all_free:
|
||||
print(f"Done waiting for free GPU memory on ({threshold=}) {dur_s=:.02f}")
|
||||
break
|
||||
if stable_duration_s <= 0:
|
||||
print(
|
||||
f"Done waiting for free GPU memory on devices {devices=} "
|
||||
f"({threshold=}) {dur_s=:.02f}"
|
||||
)
|
||||
break
|
||||
|
||||
now = time.time()
|
||||
if stable_used_bytes is None:
|
||||
stable_since = now
|
||||
stable_used_bytes = used_bytes_by_device
|
||||
else:
|
||||
memory_changed = any(
|
||||
abs(used_bytes_by_device[device] - stable_used_bytes[device])
|
||||
> stable_tolerance_bytes
|
||||
for device in devices
|
||||
)
|
||||
if memory_changed:
|
||||
stable_since = now
|
||||
stable_used_bytes = used_bytes_by_device
|
||||
elif (
|
||||
stable_since is not None and now - stable_since >= stable_duration_s
|
||||
):
|
||||
print(
|
||||
f"Done waiting for stable free GPU memory on devices "
|
||||
f"{devices=} ({threshold=}) {dur_s=:.02f}"
|
||||
)
|
||||
break
|
||||
else:
|
||||
stable_since = None
|
||||
stable_used_bytes = None
|
||||
|
||||
if dur_s >= timeout_s:
|
||||
raise ValueError(
|
||||
f"Memory of devices not free after {dur_s=:.02f} ({threshold=})"
|
||||
f"Memory of devices {devices=} not free after "
|
||||
f"{dur_s=:.02f} ({threshold=})"
|
||||
)
|
||||
|
||||
time.sleep(5)
|
||||
time.sleep(poll_interval_s)
|
||||
|
||||
|
||||
def wait_for_rocm_memory_to_settle(
|
||||
@@ -1606,11 +1644,15 @@ def wait_for_rocm_memory_to_settle(
|
||||
num_gpus = current_platform.device_count()
|
||||
if num_gpus == 0:
|
||||
return
|
||||
if threshold_ratio is None:
|
||||
threshold_ratio = 0.1
|
||||
|
||||
wait_for_gpu_memory_to_clear(
|
||||
devices=list(range(num_gpus)),
|
||||
threshold_ratio=threshold_ratio,
|
||||
timeout_s=timeout_s,
|
||||
stable_duration_s=2.0,
|
||||
poll_interval_s=1.0,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user