[ROCm][CI] Stabilize runner teardown between sampler tests (#43023)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-05-23 12:19:54 +08:00
committed by GitHub
parent 367cb81966
commit 6a4723a2e0
2 changed files with 32 additions and 1 deletions
+25
View File
@@ -1247,11 +1247,35 @@ class VllmRunner:
def __enter__(self):
return self
def _wait_for_rocm_memory_release(self, gpu_memory_utilization: float) -> None:
from tests.utils import wait_for_gpu_memory_to_clear
from vllm.platforms import current_platform
if not current_platform.is_rocm():
return
num_gpus = torch.accelerator.device_count()
if num_gpus == 0:
return
# V1 startup requires free_memory >= total * gpu_memory_utilization.
# Wait for the complementary used-memory ratio so the next runner does
# not fail the startup guard immediately after this runner exits. Bound
# the wait so cleanup failures fail this test instead of hanging.
wait_for_gpu_memory_to_clear(
devices=list(range(num_gpus)),
threshold_ratio=1.0 - gpu_memory_utilization,
timeout_s=120,
)
def __exit__(self, exc_type, exc_value, traceback):
# Explicitly shutdown the engine core to release GPU resources
# This is needed because when executing consecutive tests, the GC
# might not be fast enough in shutting down the llm engine. This can lead to OOMs
# because when the next test starts some GPU memory is still in use.
gpu_memory_utilization = (
self.llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
)
try:
self.llm.llm_engine.engine_core.shutdown()
except Exception:
@@ -1259,6 +1283,7 @@ class VllmRunner:
pass
del self.llm
cleanup_dist_env_and_memory()
self._wait_for_rocm_memory_release(gpu_memory_utilization)
@pytest.fixture(scope="session")
+7 -1
View File
@@ -22,7 +22,10 @@ import zmq
import vllm.envs as envs
from vllm.config import ParallelConfig, VllmConfig
from vllm.distributed import stateless_destroy_torch_distributed_process_group
from vllm.distributed import (
cleanup_dist_env_and_memory,
stateless_destroy_torch_distributed_process_group,
)
from vllm.envs import enable_envs_cache
from vllm.logger import init_logger
from vllm.logging_utils.dump_input import dump_engine_exception
@@ -603,6 +606,9 @@ class EngineCore:
# visible to the garbage collector again. Without this, deleting
# the engine in-process (e.g. unit tests) leaks GPU memory.
gc.unfreeze()
# Tear down distributed state initialized in this EngineCore process
# before it exits and release cached memory.
cleanup_dist_env_and_memory()
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
self.model_executor.profile(is_start, profile_prefix)