[ROCm][CI] Stabilize runner teardown between sampler tests (#43023)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-05-23 12:19:54 +08:00
committed by GitHub
parent 367cb81966
commit 6a4723a2e0
2 changed files with 32 additions and 1 deletions
+25
View File
@@ -1247,11 +1247,35 @@ class VllmRunner:
def __enter__(self):
return self
def _wait_for_rocm_memory_release(self, gpu_memory_utilization: float) -> None:
from tests.utils import wait_for_gpu_memory_to_clear
from vllm.platforms import current_platform
if not current_platform.is_rocm():
return
num_gpus = torch.accelerator.device_count()
if num_gpus == 0:
return
# V1 startup requires free_memory >= total * gpu_memory_utilization.
# Wait for the complementary used-memory ratio so the next runner does
# not fail the startup guard immediately after this runner exits. Bound
# the wait so cleanup failures fail this test instead of hanging.
wait_for_gpu_memory_to_clear(
devices=list(range(num_gpus)),
threshold_ratio=1.0 - gpu_memory_utilization,
timeout_s=120,
)
def __exit__(self, exc_type, exc_value, traceback):
# Explicitly shutdown the engine core to release GPU resources
# This is needed because when executing consecutive tests, the GC
# might not be fast enough in shutting down the llm engine. This can lead to OOMs
# because when the next test starts some GPU memory is still in use.
gpu_memory_utilization = (
self.llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
)
try:
self.llm.llm_engine.engine_core.shutdown()
except Exception:
@@ -1259,6 +1283,7 @@ class VllmRunner:
pass
del self.llm
cleanup_dist_env_and_memory()
self._wait_for_rocm_memory_release(gpu_memory_utilization)
@pytest.fixture(scope="session")