[XPU][UT] Fix OOM and skip graph case (#49287)

Signed-off-by: mayuyuace <qiming1.zhang@intel.com>
Signed-off-by: Qiming Zhang <qiming1.zhang@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Qiming Zhang
2026-08-18 09:47:35 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent 69d3335066
commit f4b161d7fc
6 changed files with 46 additions and 21 deletions
+25 -10
View File
@@ -914,13 +914,13 @@ class HfRunner:
return self
def __exit__(self, exc_type, exc_value, traceback):
from tests.utils import wait_for_rocm_memory_to_settle
from tests.utils import wait_for_memory_to_settle
del self.model
cleanup_dist_env_and_memory()
# ROCm frees VRAM lazily; wait so a runner started right after this HF
# model exits does not OOM on its startup memory guard.
wait_for_rocm_memory_to_settle(
wait_for_memory_to_settle(
threshold_ratio=getattr(self, "threshold_ratios", None)
)
if hasattr(self, "threshold_ratios"):
@@ -998,9 +998,24 @@ class VllmRunner:
# V1 startup requires free_memory >= total * gpu_memory_utilization.
# ROCm CI can hand a test a device that is still lazily releasing
# VRAM from a previous process, so wait before constructing LLM.
from tests.utils import wait_for_rocm_memory_to_settle
from tests.utils import wait_for_memory_to_settle
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
elif current_platform.is_xpu():
# The XPU/oneAPI runtime keeps ~1 GiB of context resident in the
# parent pytest process for its whole lifetime (grown by in-process
# HfRunner models), and distributed tests additionally allocate a
# CCL context in the engine subprocess. The default utilization of
# 0.92 leaves too little headroom for both, so lower it on XPU when
# the caller did not request an explicit value.
if "gpu_memory_utilization" not in kwargs:
kwargs["gpu_memory_utilization"] = 0.9
gpu_memory_utilization = kwargs["gpu_memory_utilization"]
# XPU (Level Zero) can also release device memory lazily after a
# previous engine shuts down, so wait before constructing LLM.
from tests.utils import wait_for_memory_to_settle
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
with init_ctx:
self.llm = LLM(
@@ -1326,14 +1341,14 @@ class VllmRunner:
def __enter__(self):
return self
def _wait_for_rocm_memory_release(self, gpu_memory_utilization: float) -> None:
from tests.utils import wait_for_rocm_memory_to_settle
def _wait_for_memory_release(self, gpu_memory_utilization: float) -> None:
from tests.utils import wait_for_memory_to_settle
# V1 startup requires free_memory >= total * gpu_memory_utilization.
# Wait for the complementary used-memory ratio so the next runner does
# not fail the startup guard immediately after this runner exits. The
# wait is bounded so cleanup failures fail this test instead of hanging.
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
def __exit__(self, exc_type, exc_value, traceback):
# Explicitly shutdown the engine core to release GPU resources
@@ -1359,7 +1374,7 @@ class VllmRunner:
del self.llm
torch._dynamo.reset()
cleanup_dist_env_and_memory()
self._wait_for_rocm_memory_release(gpu_memory_utilization)
self._wait_for_memory_release(gpu_memory_utilization)
@pytest.fixture(scope="session")
@@ -1746,7 +1761,7 @@ def clean_gpu_memory_between_tests():
import gc
from tests.utils import wait_for_gpu_memory_to_clear, wait_for_rocm_memory_to_settle
from tests.utils import wait_for_gpu_memory_to_clear, wait_for_memory_to_settle
num_gpus = torch.accelerator.device_count()
@@ -1755,7 +1770,7 @@ def clean_gpu_memory_between_tests():
return
try:
if current_platform.is_rocm():
wait_for_rocm_memory_to_settle()
wait_for_memory_to_settle()
else:
wait_for_gpu_memory_to_clear(
devices=list(range(num_gpus)),
@@ -9,6 +9,7 @@ import pytest
from tests.models.registry import HF_EXAMPLE_MODELS
from tests.utils import multi_gpu_test
from vllm import LLM
from vllm.config import CUDAGraphMode
from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
from vllm.sampling_params import SamplingParams
@@ -210,6 +211,8 @@ def test_mamba_cache_cg_padding(
cudagraph_dispatcher.initialize_cudagraph_keys(
vllm_config.compilation_config.cudagraph_mode
)
if cudagraph_dispatcher.cudagraph_mode == CUDAGraphMode.NONE:
pytest.skip("CUDA/XPU graph is disabled.Please enable it to run this test. ")
while (
len(example_prompts)
== cudagraph_dispatcher.dispatch(len(example_prompts))[1].num_tokens
@@ -11,7 +11,7 @@ from contextlib import contextmanager
import pytest
import torch
from tests.utils import wait_for_rocm_memory_to_settle
from tests.utils import wait_for_memory_to_settle
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
@@ -165,7 +165,7 @@ def _hf_colbert_model(model_name: str, hf_spec: dict, device: torch.device):
finally:
del hf_model, linear_weight
cleanup_dist_env_and_memory()
wait_for_rocm_memory_to_settle()
wait_for_memory_to_settle()
def _assert_embeddings_close(vllm_outputs, hf_embeddings):
@@ -104,9 +104,9 @@ async def async_engine():
from vllm.platforms import current_platform
if current_platform.is_rocm():
from tests.utils import wait_for_rocm_memory_to_settle
from tests.utils import wait_for_memory_to_settle
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
engine_args = AsyncEngineArgs(**ENGINE_CONFIG)
llm = AsyncLLM.from_engine_args(engine_args)
@@ -122,9 +122,9 @@ async def async_engine():
from vllm.distributed import cleanup_dist_env_and_memory
cleanup_dist_env_and_memory()
from tests.utils import wait_for_rocm_memory_to_settle
from tests.utils import wait_for_memory_to_settle
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypatch):
+11 -4
View File
@@ -1540,6 +1540,13 @@ def record_gpu_memory_usage_stats(
mem_info = amdsmi_get_gpu_vram_usage(dev_handle)
gb_used = mem_info["vram_used"] / 2**10
gb_total = mem_info["vram_total"] / 2**10
elif current_platform.is_xpu():
# nvml/amdsmi are unavailable on XPU. Query device memory through
# torch.accelerator.get_memory_info, which the XPU platform patches
# to return (free, total) bytes via Level Zero.
free_b, total_b = torch.accelerator.get_memory_info(device)
gb_used = (total_b - free_b) / 2**30
gb_total = total_b / 2**30
else:
dev_handle = get_nvml_device_handle(device)
mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
@@ -1680,19 +1687,19 @@ def wait_for_gpu_memory_to_clear(
time.sleep(poll_interval_s)
def wait_for_rocm_memory_to_settle(
def wait_for_memory_to_settle(
*,
threshold_ratio: float | dict[int, float] | None = 0.1,
timeout_s: float = 240,
) -> None:
"""Block until ROCm device VRAM usage drops below ``threshold_ratio``.
"""Block until ROCm or XPU device VRAM usage drops below ``threshold_ratio``.
ROCm reclaims GPU memory more lazily than CUDA, so back-to-back model
ROCm and XPU reclaims GPU memory more lazily than CUDA, so back-to-back model
loads in a single test process can OOM the *next* engine/model startup
even after ``cleanup_dist_env_and_memory``. This gives the driver time to
actually release VRAM before the next allocation. No-op off ROCm.
"""
if not current_platform.is_rocm():
if not current_platform.is_rocm() and not current_platform.is_xpu():
return
num_gpus = current_platform.device_count()
@@ -112,7 +112,7 @@ wait_for_server() {
# Function to clean up previous instances
wait_for_gpu_memory_release() {
if [[ "$SMI_BIN" == *"rocm"* ]]; then
PYTHONPATH="${GIT_ROOT}" python3 -c "from tests.utils import wait_for_rocm_memory_to_settle; wait_for_rocm_memory_to_settle()"
PYTHONPATH="${GIT_ROOT}" python3 -c "from tests.utils import wait_for_memory_to_settle; wait_for_memory_to_settle()"
fi
}