[XPU][UT] Fix OOM and skip graph case (#49287)
Signed-off-by: mayuyuace <qiming1.zhang@intel.com> Signed-off-by: Qiming Zhang <qiming1.zhang@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
69d3335066
commit
f4b161d7fc
+25
-10
@@ -914,13 +914,13 @@ class HfRunner:
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
from tests.utils import wait_for_rocm_memory_to_settle
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
|
||||
del self.model
|
||||
cleanup_dist_env_and_memory()
|
||||
# ROCm frees VRAM lazily; wait so a runner started right after this HF
|
||||
# model exits does not OOM on its startup memory guard.
|
||||
wait_for_rocm_memory_to_settle(
|
||||
wait_for_memory_to_settle(
|
||||
threshold_ratio=getattr(self, "threshold_ratios", None)
|
||||
)
|
||||
if hasattr(self, "threshold_ratios"):
|
||||
@@ -998,9 +998,24 @@ class VllmRunner:
|
||||
# V1 startup requires free_memory >= total * gpu_memory_utilization.
|
||||
# ROCm CI can hand a test a device that is still lazily releasing
|
||||
# VRAM from a previous process, so wait before constructing LLM.
|
||||
from tests.utils import wait_for_rocm_memory_to_settle
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
|
||||
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
||||
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
||||
elif current_platform.is_xpu():
|
||||
# The XPU/oneAPI runtime keeps ~1 GiB of context resident in the
|
||||
# parent pytest process for its whole lifetime (grown by in-process
|
||||
# HfRunner models), and distributed tests additionally allocate a
|
||||
# CCL context in the engine subprocess. The default utilization of
|
||||
# 0.92 leaves too little headroom for both, so lower it on XPU when
|
||||
# the caller did not request an explicit value.
|
||||
if "gpu_memory_utilization" not in kwargs:
|
||||
kwargs["gpu_memory_utilization"] = 0.9
|
||||
gpu_memory_utilization = kwargs["gpu_memory_utilization"]
|
||||
# XPU (Level Zero) can also release device memory lazily after a
|
||||
# previous engine shuts down, so wait before constructing LLM.
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
|
||||
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
||||
|
||||
with init_ctx:
|
||||
self.llm = LLM(
|
||||
@@ -1326,14 +1341,14 @@ class VllmRunner:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def _wait_for_rocm_memory_release(self, gpu_memory_utilization: float) -> None:
|
||||
from tests.utils import wait_for_rocm_memory_to_settle
|
||||
def _wait_for_memory_release(self, gpu_memory_utilization: float) -> None:
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
|
||||
# V1 startup requires free_memory >= total * gpu_memory_utilization.
|
||||
# Wait for the complementary used-memory ratio so the next runner does
|
||||
# not fail the startup guard immediately after this runner exits. The
|
||||
# wait is bounded so cleanup failures fail this test instead of hanging.
|
||||
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
||||
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
# Explicitly shutdown the engine core to release GPU resources
|
||||
@@ -1359,7 +1374,7 @@ class VllmRunner:
|
||||
del self.llm
|
||||
torch._dynamo.reset()
|
||||
cleanup_dist_env_and_memory()
|
||||
self._wait_for_rocm_memory_release(gpu_memory_utilization)
|
||||
self._wait_for_memory_release(gpu_memory_utilization)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
@@ -1746,7 +1761,7 @@ def clean_gpu_memory_between_tests():
|
||||
|
||||
import gc
|
||||
|
||||
from tests.utils import wait_for_gpu_memory_to_clear, wait_for_rocm_memory_to_settle
|
||||
from tests.utils import wait_for_gpu_memory_to_clear, wait_for_memory_to_settle
|
||||
|
||||
num_gpus = torch.accelerator.device_count()
|
||||
|
||||
@@ -1755,7 +1770,7 @@ def clean_gpu_memory_between_tests():
|
||||
return
|
||||
try:
|
||||
if current_platform.is_rocm():
|
||||
wait_for_rocm_memory_to_settle()
|
||||
wait_for_memory_to_settle()
|
||||
else:
|
||||
wait_for_gpu_memory_to_clear(
|
||||
devices=list(range(num_gpus)),
|
||||
|
||||
Reference in New Issue
Block a user