[XPU][UT] Fix OOM and skip graph case (#49287)

Signed-off-by: mayuyuace <qiming1.zhang@intel.com>
Signed-off-by: Qiming Zhang <qiming1.zhang@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Qiming Zhang
2026-08-18 09:47:35 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent 69d3335066
commit f4b161d7fc
6 changed files with 46 additions and 21 deletions
+11 -4
View File
@@ -1540,6 +1540,13 @@ def record_gpu_memory_usage_stats(
mem_info = amdsmi_get_gpu_vram_usage(dev_handle)
gb_used = mem_info["vram_used"] / 2**10
gb_total = mem_info["vram_total"] / 2**10
elif current_platform.is_xpu():
# nvml/amdsmi are unavailable on XPU. Query device memory through
# torch.accelerator.get_memory_info, which the XPU platform patches
# to return (free, total) bytes via Level Zero.
free_b, total_b = torch.accelerator.get_memory_info(device)
gb_used = (total_b - free_b) / 2**30
gb_total = total_b / 2**30
else:
dev_handle = get_nvml_device_handle(device)
mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
@@ -1680,19 +1687,19 @@ def wait_for_gpu_memory_to_clear(
time.sleep(poll_interval_s)
def wait_for_rocm_memory_to_settle(
def wait_for_memory_to_settle(
*,
threshold_ratio: float | dict[int, float] | None = 0.1,
timeout_s: float = 240,
) -> None:
"""Block until ROCm device VRAM usage drops below ``threshold_ratio``.
"""Block until ROCm or XPU device VRAM usage drops below ``threshold_ratio``.
ROCm reclaims GPU memory more lazily than CUDA, so back-to-back model
ROCm and XPU reclaims GPU memory more lazily than CUDA, so back-to-back model
loads in a single test process can OOM the *next* engine/model startup
even after ``cleanup_dist_env_and_memory``. This gives the driver time to
actually release VRAM before the next allocation. No-op off ROCm.
"""
if not current_platform.is_rocm():
if not current_platform.is_rocm() and not current_platform.is_xpu():
return
num_gpus = current_platform.device_count()