[ROCm][CI] Stabilize runner teardown between sampler tests (#43023)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -1247,11 +1247,35 @@ class VllmRunner:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def _wait_for_rocm_memory_release(self, gpu_memory_utilization: float) -> None:
|
||||
from tests.utils import wait_for_gpu_memory_to_clear
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
return
|
||||
|
||||
num_gpus = torch.accelerator.device_count()
|
||||
if num_gpus == 0:
|
||||
return
|
||||
|
||||
# V1 startup requires free_memory >= total * gpu_memory_utilization.
|
||||
# Wait for the complementary used-memory ratio so the next runner does
|
||||
# not fail the startup guard immediately after this runner exits. Bound
|
||||
# the wait so cleanup failures fail this test instead of hanging.
|
||||
wait_for_gpu_memory_to_clear(
|
||||
devices=list(range(num_gpus)),
|
||||
threshold_ratio=1.0 - gpu_memory_utilization,
|
||||
timeout_s=120,
|
||||
)
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
# Explicitly shutdown the engine core to release GPU resources
|
||||
# This is needed because when executing consecutive tests, the GC
|
||||
# might not be fast enough in shutting down the llm engine. This can lead to OOMs
|
||||
# because when the next test starts some GPU memory is still in use.
|
||||
gpu_memory_utilization = (
|
||||
self.llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
|
||||
)
|
||||
try:
|
||||
self.llm.llm_engine.engine_core.shutdown()
|
||||
except Exception:
|
||||
@@ -1259,6 +1283,7 @@ class VllmRunner:
|
||||
pass
|
||||
del self.llm
|
||||
cleanup_dist_env_and_memory()
|
||||
self._wait_for_rocm_memory_release(gpu_memory_utilization)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
|
||||
@@ -22,7 +22,10 @@ import zmq
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import ParallelConfig, VllmConfig
|
||||
from vllm.distributed import stateless_destroy_torch_distributed_process_group
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
stateless_destroy_torch_distributed_process_group,
|
||||
)
|
||||
from vllm.envs import enable_envs_cache
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logging_utils.dump_input import dump_engine_exception
|
||||
@@ -603,6 +606,9 @@ class EngineCore:
|
||||
# visible to the garbage collector again. Without this, deleting
|
||||
# the engine in-process (e.g. unit tests) leaks GPU memory.
|
||||
gc.unfreeze()
|
||||
# Tear down distributed state initialized in this EngineCore process
|
||||
# before it exits and release cached memory.
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
|
||||
self.model_executor.profile(is_start, profile_prefix)
|
||||
|
||||
Reference in New Issue
Block a user