[XPU][UT] Fix OOM and skip graph case (#49287)

Signed-off-by: mayuyuace <qiming1.zhang@intel.com>
Signed-off-by: Qiming Zhang <qiming1.zhang@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Qiming Zhang
2026-08-18 09:47:35 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent 69d3335066
commit f4b161d7fc
6 changed files with 46 additions and 21 deletions
+25 -10
View File
@@ -914,13 +914,13 @@ class HfRunner:
return self
def __exit__(self, exc_type, exc_value, traceback):
from tests.utils import wait_for_rocm_memory_to_settle
from tests.utils import wait_for_memory_to_settle
del self.model
cleanup_dist_env_and_memory()
# ROCm frees VRAM lazily; wait so a runner started right after this HF
# model exits does not OOM on its startup memory guard.
wait_for_rocm_memory_to_settle(
wait_for_memory_to_settle(
threshold_ratio=getattr(self, "threshold_ratios", None)
)
if hasattr(self, "threshold_ratios"):
@@ -998,9 +998,24 @@ class VllmRunner:
# V1 startup requires free_memory >= total * gpu_memory_utilization.
# ROCm CI can hand a test a device that is still lazily releasing
# VRAM from a previous process, so wait before constructing LLM.
from tests.utils import wait_for_rocm_memory_to_settle
from tests.utils import wait_for_memory_to_settle
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
elif current_platform.is_xpu():
# The XPU/oneAPI runtime keeps ~1 GiB of context resident in the
# parent pytest process for its whole lifetime (grown by in-process
# HfRunner models), and distributed tests additionally allocate a
# CCL context in the engine subprocess. The default utilization of
# 0.92 leaves too little headroom for both, so lower it on XPU when
# the caller did not request an explicit value.
if "gpu_memory_utilization" not in kwargs:
kwargs["gpu_memory_utilization"] = 0.9
gpu_memory_utilization = kwargs["gpu_memory_utilization"]
# XPU (Level Zero) can also release device memory lazily after a
# previous engine shuts down, so wait before constructing LLM.
from tests.utils import wait_for_memory_to_settle
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
with init_ctx:
self.llm = LLM(
@@ -1326,14 +1341,14 @@ class VllmRunner:
def __enter__(self):
return self
def _wait_for_rocm_memory_release(self, gpu_memory_utilization: float) -> None:
from tests.utils import wait_for_rocm_memory_to_settle
def _wait_for_memory_release(self, gpu_memory_utilization: float) -> None:
from tests.utils import wait_for_memory_to_settle
# V1 startup requires free_memory >= total * gpu_memory_utilization.
# Wait for the complementary used-memory ratio so the next runner does
# not fail the startup guard immediately after this runner exits. The
# wait is bounded so cleanup failures fail this test instead of hanging.
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
def __exit__(self, exc_type, exc_value, traceback):
# Explicitly shutdown the engine core to release GPU resources
@@ -1359,7 +1374,7 @@ class VllmRunner:
del self.llm
torch._dynamo.reset()
cleanup_dist_env_and_memory()
self._wait_for_rocm_memory_release(gpu_memory_utilization)
self._wait_for_memory_release(gpu_memory_utilization)
@pytest.fixture(scope="session")
@@ -1746,7 +1761,7 @@ def clean_gpu_memory_between_tests():
import gc
from tests.utils import wait_for_gpu_memory_to_clear, wait_for_rocm_memory_to_settle
from tests.utils import wait_for_gpu_memory_to_clear, wait_for_memory_to_settle
num_gpus = torch.accelerator.device_count()
@@ -1755,7 +1770,7 @@ def clean_gpu_memory_between_tests():
return
try:
if current_platform.is_rocm():
wait_for_rocm_memory_to_settle()
wait_for_memory_to_settle()
else:
wait_for_gpu_memory_to_clear(
devices=list(range(num_gpus)),