[XPU][UT] Fix OOM and skip graph case (#49287)
Signed-off-by: mayuyuace <qiming1.zhang@intel.com> Signed-off-by: Qiming Zhang <qiming1.zhang@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
69d3335066
commit
f4b161d7fc
+11
-4
@@ -1540,6 +1540,13 @@ def record_gpu_memory_usage_stats(
|
||||
mem_info = amdsmi_get_gpu_vram_usage(dev_handle)
|
||||
gb_used = mem_info["vram_used"] / 2**10
|
||||
gb_total = mem_info["vram_total"] / 2**10
|
||||
elif current_platform.is_xpu():
|
||||
# nvml/amdsmi are unavailable on XPU. Query device memory through
|
||||
# torch.accelerator.get_memory_info, which the XPU platform patches
|
||||
# to return (free, total) bytes via Level Zero.
|
||||
free_b, total_b = torch.accelerator.get_memory_info(device)
|
||||
gb_used = (total_b - free_b) / 2**30
|
||||
gb_total = total_b / 2**30
|
||||
else:
|
||||
dev_handle = get_nvml_device_handle(device)
|
||||
mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
|
||||
@@ -1680,19 +1687,19 @@ def wait_for_gpu_memory_to_clear(
|
||||
time.sleep(poll_interval_s)
|
||||
|
||||
|
||||
def wait_for_rocm_memory_to_settle(
|
||||
def wait_for_memory_to_settle(
|
||||
*,
|
||||
threshold_ratio: float | dict[int, float] | None = 0.1,
|
||||
timeout_s: float = 240,
|
||||
) -> None:
|
||||
"""Block until ROCm device VRAM usage drops below ``threshold_ratio``.
|
||||
"""Block until ROCm or XPU device VRAM usage drops below ``threshold_ratio``.
|
||||
|
||||
ROCm reclaims GPU memory more lazily than CUDA, so back-to-back model
|
||||
ROCm and XPU reclaims GPU memory more lazily than CUDA, so back-to-back model
|
||||
loads in a single test process can OOM the *next* engine/model startup
|
||||
even after ``cleanup_dist_env_and_memory``. This gives the driver time to
|
||||
actually release VRAM before the next allocation. No-op off ROCm.
|
||||
"""
|
||||
if not current_platform.is_rocm():
|
||||
if not current_platform.is_rocm() and not current_platform.is_xpu():
|
||||
return
|
||||
|
||||
num_gpus = current_platform.device_count()
|
||||
|
||||
Reference in New Issue
Block a user