diff --git a/tests/conftest.py b/tests/conftest.py index 5939d8bfd2..3eaebc38bc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1247,11 +1247,35 @@ class VllmRunner: def __enter__(self): return self + def _wait_for_rocm_memory_release(self, gpu_memory_utilization: float) -> None: + from tests.utils import wait_for_gpu_memory_to_clear + from vllm.platforms import current_platform + + if not current_platform.is_rocm(): + return + + num_gpus = torch.accelerator.device_count() + if num_gpus == 0: + return + + # V1 startup requires free_memory >= total * gpu_memory_utilization. + # Wait for the complementary used-memory ratio so the next runner does + # not fail the startup guard immediately after this runner exits. Bound + # the wait so cleanup failures fail this test instead of hanging. + wait_for_gpu_memory_to_clear( + devices=list(range(num_gpus)), + threshold_ratio=1.0 - gpu_memory_utilization, + timeout_s=120, + ) + def __exit__(self, exc_type, exc_value, traceback): # Explicitly shutdown the engine core to release GPU resources # This is needed because when executing consecutive tests, the GC # might not be fast enough in shutting down the llm engine. This can lead to OOMs # because when the next test starts some GPU memory is still in use. + gpu_memory_utilization = ( + self.llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization + ) try: self.llm.llm_engine.engine_core.shutdown() except Exception: @@ -1259,6 +1283,7 @@ class VllmRunner: pass del self.llm cleanup_dist_env_and_memory() + self._wait_for_rocm_memory_release(gpu_memory_utilization) @pytest.fixture(scope="session") diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 3993f77ef6..8c5fe71ed2 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -22,7 +22,10 @@ import zmq import vllm.envs as envs from vllm.config import ParallelConfig, VllmConfig -from vllm.distributed import stateless_destroy_torch_distributed_process_group +from vllm.distributed import ( + cleanup_dist_env_and_memory, + stateless_destroy_torch_distributed_process_group, +) from vllm.envs import enable_envs_cache from vllm.logger import init_logger from vllm.logging_utils.dump_input import dump_engine_exception @@ -603,6 +606,9 @@ class EngineCore: # visible to the garbage collector again. Without this, deleting # the engine in-process (e.g. unit tests) leaks GPU memory. gc.unfreeze() + # Tear down distributed state initialized in this EngineCore process + # before it exits and release cached memory. + cleanup_dist_env_and_memory() def profile(self, is_start: bool = True, profile_prefix: str | None = None): self.model_executor.profile(is_start, profile_prefix)