From d9ff7e4e9a397c5c3ac6337116cc71969c966a23 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Mon, 8 Jun 2026 01:11:17 -0500 Subject: [PATCH] [ROCm][CI] Stabilizing teardown and timeout of flaky tests to prevent rare OOMs (#44761) Signed-off-by: Andreas Karatzas --- tests/conftest.py | 35 ++++---- .../models/language/generation/test_hybrid.py | 34 ++------ tests/models/language/pooling/test_colbert.py | 46 ++++++++--- tests/utils.py | 80 ++++++++++++++++--- 4 files changed, 130 insertions(+), 65 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3eaebc38bc..ebf9608b01 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -854,8 +854,13 @@ class HfRunner: return self def __exit__(self, exc_type, exc_value, traceback): + from tests.utils import wait_for_rocm_memory_to_settle + del self.model cleanup_dist_env_and_memory() + # ROCm frees VRAM lazily; wait so a runner started right after this HF + # model exits does not OOM on its startup memory guard. + wait_for_rocm_memory_to_settle() @pytest.fixture(scope="session") @@ -1248,25 +1253,13 @@ class VllmRunner: return self def _wait_for_rocm_memory_release(self, gpu_memory_utilization: float) -> None: - from tests.utils import wait_for_gpu_memory_to_clear - from vllm.platforms import current_platform - - if not current_platform.is_rocm(): - return - - num_gpus = torch.accelerator.device_count() - if num_gpus == 0: - return + from tests.utils import wait_for_rocm_memory_to_settle # V1 startup requires free_memory >= total * gpu_memory_utilization. # Wait for the complementary used-memory ratio so the next runner does - # not fail the startup guard immediately after this runner exits. Bound - # the wait so cleanup failures fail this test instead of hanging. - wait_for_gpu_memory_to_clear( - devices=list(range(num_gpus)), - threshold_ratio=1.0 - gpu_memory_utilization, - timeout_s=120, - ) + # not fail the startup guard immediately after this runner exits. The + # wait is bounded so cleanup failures fail this test instead of hanging. + wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization) def __exit__(self, exc_type, exc_value, traceback): # Explicitly shutdown the engine core to release GPU resources @@ -1276,8 +1269,16 @@ class VllmRunner: gpu_memory_utilization = ( self.llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization ) + from vllm.platforms import current_platform + try: - self.llm.llm_engine.engine_core.shutdown() + # Give the engine core time to run its own graceful shutdown + # (model_executor teardown + empty_cache + process-group destroy) + # before the process manager SIGKILLs it at the default 5s. On ROCm + # a hard kill leaves the whole allocation for the driver's slow async + # VRAM reclamation, which starves the next test's startup. + shutdown_timeout = 60.0 if current_platform.is_rocm() else None + self.llm.llm_engine.engine_core.shutdown(timeout=shutdown_timeout) except Exception: # Ignore shutdown errors as cleanup will still proceed pass diff --git a/tests/models/language/generation/test_hybrid.py b/tests/models/language/generation/test_hybrid.py index 7d22278cf1..cd89ca284d 100644 --- a/tests/models/language/generation/test_hybrid.py +++ b/tests/models/language/generation/test_hybrid.py @@ -7,7 +7,7 @@ from contextlib import contextmanager, nullcontext import pytest from tests.models.registry import HF_EXAMPLE_MODELS -from tests.utils import multi_gpu_test, wait_for_gpu_memory_to_clear +from tests.utils import multi_gpu_test from vllm import LLM from vllm.engine.arg_utils import EngineArgs from vllm.platforms import current_platform @@ -405,28 +405,10 @@ def _get_vllm_runner_params( } -def _wait_for_rocm_memory_to_settle() -> None: - if not current_platform.is_rocm(): - return - - num_gpus = current_platform.device_count() - if num_gpus == 0: - return - - wait_for_gpu_memory_to_clear( - devices=list(range(num_gpus)), - threshold_ratio=0.01, - timeout_s=120, - ) - - @contextmanager -def _owned_vLLM_runner(vllm_runner, kwargs): - try: - with vllm_runner(**kwargs) as runner: - yield runner - finally: - _wait_for_rocm_memory_to_settle() +def _owned_vllm_runner(vllm_runner, kwargs): + with vllm_runner(**kwargs) as runner: + yield runner def _get_vLLM_output( @@ -439,7 +421,7 @@ def _get_vLLM_output( vllm_model=None, ): runner_context = ( - _owned_vLLM_runner(vllm_runner, kwargs) + _owned_vllm_runner(vllm_runner, kwargs) if vllm_model is None else nullcontext(vllm_model) ) @@ -801,7 +783,7 @@ def test_apc_multiple_prompts_partial_cached_outputs( # Cache only part of all the prompts vllm_runner_kwargs["enable_prefix_caching"] = True - with _owned_vLLM_runner(vllm_runner, vllm_runner_kwargs) as vllm_model: + with _owned_vllm_runner(vllm_runner, vllm_runner_kwargs) as vllm_model: vllm_outputs_partial_cache, _ = _get_vLLM_output( vllm_runner, vllm_runner_kwargs, @@ -861,7 +843,7 @@ def test_same_mamba_output_apc_on_vs_off( # No prefix caching kwargs_no_apc = {**base_kwargs, "enable_prefix_caching": False} - with _owned_vLLM_runner(vllm_runner, kwargs_no_apc) as vllm_model: + with vllm_runner(**kwargs_no_apc) as vllm_model: outputs_no_apc, _ = _get_vLLM_output( vllm_runner, kwargs_no_apc, @@ -876,7 +858,7 @@ def test_same_mamba_output_apc_on_vs_off( "enable_prefix_caching": True, "mamba_block_size": 16, } - with _owned_vLLM_runner(vllm_runner, kwargs_with_apc) as vllm_model: + with vllm_runner(**kwargs_with_apc) as vllm_model: outputs_with_apc, _ = _get_vLLM_output( vllm_runner, kwargs_with_apc, diff --git a/tests/models/language/pooling/test_colbert.py b/tests/models/language/pooling/test_colbert.py index b6ff9b9ffb..6c82ad8a9c 100644 --- a/tests/models/language/pooling/test_colbert.py +++ b/tests/models/language/pooling/test_colbert.py @@ -6,15 +6,19 @@ Tests are parametrized across multiple ColBERT backbones to ensure the generic ColBERT support works with different encoder architectures. """ +from contextlib import contextmanager + import pytest import torch +from tests.utils import wait_for_rocm_memory_to_settle +from vllm.distributed import cleanup_dist_env_and_memory from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score # ----------------------------------------------------------------------- # Model definitions: (model_name, colbert_dim, extra vllm_runner kwargs) # ----------------------------------------------------------------------- -COLBERT_MODELS: dict[str, dict] = { +COLBERT_MODELS = { "bert": { "model": "answerdotai/answerai-colbert-small-v1", "colbert_dim": 96, @@ -145,6 +149,24 @@ def _compute_hf_colbert_embeddings(model, tokenizer, linear_weight, texts, devic return embeddings +@contextmanager +def _hf_colbert_model(model_name: str, hf_spec: dict, device: torch.device): + """Load the HF backbone + ColBERT projection, freeing the GPU on exit. + + These live outside any runner context, so without explicit cleanup ROCm + keeps the VRAM resident and the next backend parametrization (or test) + OOMs on startup. + """ + hf_model = _load_hf_model(model_name, hf_spec, device) + linear_weight = _load_projection_weight(model_name, hf_spec, device) + try: + yield hf_model, linear_weight + finally: + del hf_model, linear_weight + cleanup_dist_env_and_memory() + wait_for_rocm_memory_to_settle() + + def _assert_embeddings_close(vllm_outputs, hf_embeddings): """Assert that vLLM and HuggingFace embeddings match.""" for i, (hf_emb, vllm_out) in enumerate(zip(hf_embeddings, vllm_outputs)): @@ -363,9 +385,11 @@ def test_colbert_hf_comparison(vllm_runner, backend): spec = COLBERT_MODELS[backend] hf_spec = spec["hf_comparison"] + extra_kwargs = spec["extra_kwargs"] model_name = spec["model"] assert isinstance(model_name, str) assert isinstance(hf_spec, dict) + assert isinstance(extra_kwargs, dict) test_texts = [TEXTS_1[0], TEXTS_2[0]] with vllm_runner( @@ -374,7 +398,7 @@ def test_colbert_hf_comparison(vllm_runner, backend): dtype="float32", max_model_len=spec["max_model_len"], enforce_eager=True, - **spec["extra_kwargs"], + **extra_kwargs, ) as vllm_model: vllm_outputs = vllm_model.token_embed(test_texts) @@ -384,15 +408,13 @@ def test_colbert_hf_comparison(vllm_runner, backend): model_name, trust_remote_code=hf_spec.get("trust_remote_code", False), ) - hf_model = _load_hf_model(model_name, hf_spec, device) - linear_weight = _load_projection_weight(model_name, hf_spec, device) - - hf_embeddings = _compute_hf_colbert_embeddings( - hf_model, - hf_tokenizer, - linear_weight, - test_texts, - device, - ) + with _hf_colbert_model(model_name, hf_spec, device) as (hf_model, linear_weight): + hf_embeddings = _compute_hf_colbert_embeddings( + hf_model, + hf_tokenizer, + linear_weight, + test_texts, + device, + ) _assert_embeddings_close(vllm_outputs, hf_embeddings) diff --git a/tests/utils.py b/tests/utils.py index 6a32f3e2e2..5b18435398 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -538,11 +538,22 @@ class RemoteVLLMServer: if current_platform.is_rocm(): with _nvml(): handles = amdsmi_get_processor_handles() - total_used = 0 - for handle in handles: + devices = get_physical_device_indices( + list(range(current_platform.device_count())) + ) + total_used_mib = 0 + for device in devices: + handle = handles[device] vram_info = amdsmi_get_gpu_vram_usage(handle) - total_used += vram_info["vram_used"] - return total_used + total_used_mib += vram_info["vram_used"] + # amdsmi reports VRAM in MiB; convert to bytes so this + # matches the CUDA/nvml branch (already bytes) and the + # byte-based target in _wait_for_gpu_memory_release. Without + # this, that wait compares MiB against a ~2e9-byte target, + # is always satisfied instantly, and returns "released to + # 0.00 GB" while the previous server's VRAM is still + # resident -- OOMing the next server's startup on ROCm. + return total_used_mib * 1024 * 1024 elif current_platform.is_cuda(): with _nvml(): total_used = 0 @@ -1428,6 +1439,18 @@ def wait_for_gpu_memory_to_clear( timeout_s: float = 120, ) -> None: assert threshold_bytes is not None or threshold_ratio is not None + if ( + current_platform.is_rocm() + and threshold_ratio is not None + and threshold_ratio < 0.05 + ): + # ROCm can keep a small runtime/driver footprint resident even after + # all model allocations are gone. On MI300 this has been observed + # around 2.5 GiB, which is above a strict 1% idle threshold but nowhere + # near the amount of free memory needed by the next vLLM runner. + min_threshold_bytes = 4 * 1024**3 + threshold_bytes = max(threshold_bytes or 0, min_threshold_bytes) + # Use nvml instead of pytorch to reduce measurement error from torch cuda # context. devices = get_physical_device_indices(devices) @@ -1454,15 +1477,26 @@ def wait_for_gpu_memory_to_clear( print(f"{k}={v}; ", end="") print("") - if threshold_bytes is not None: - is_free = lambda used, total: used <= threshold_bytes / 2**30 - threshold = f"{threshold_bytes / 2**30} GiB" + if threshold_bytes is not None and threshold_ratio is not None: + threshold_gib = threshold_bytes / 2**30 + threshold = f"max({threshold_gib:.2f} GiB, {threshold_ratio:.3f})" + all_free = all( + used <= max(threshold_gib, total * threshold_ratio) + for used, total in output_raw.values() + ) + elif threshold_bytes is not None: + threshold_gib = threshold_bytes / 2**30 + threshold = f"{threshold_gib} GiB" + all_free = all(used <= threshold_gib for used, _ in output_raw.values()) else: - is_free = lambda used, total: used / total <= threshold_ratio - threshold = f"{threshold_ratio:.2f}" + assert threshold_ratio is not None + threshold = f"{threshold_ratio:.3f}" + all_free = all( + used / total <= threshold_ratio for used, total in output_raw.values() + ) dur_s = time.time() - start_time - if all(is_free(used, total) for used, total in output_raw.values()): + if all_free: print( f"Done waiting for free GPU memory on devices {devices=} " f"({threshold=}) {dur_s=:.02f}" @@ -1478,6 +1512,32 @@ def wait_for_gpu_memory_to_clear( time.sleep(5) +def wait_for_rocm_memory_to_settle( + *, + threshold_ratio: float = 0.1, + timeout_s: float = 240, +) -> None: + """Block until ROCm device VRAM usage drops below ``threshold_ratio``. + + ROCm reclaims GPU memory more lazily than CUDA, so back-to-back model + loads in a single test process can OOM the *next* engine/model startup + even after ``cleanup_dist_env_and_memory``. This gives the driver time to + actually release VRAM before the next allocation. No-op off ROCm. + """ + if not current_platform.is_rocm(): + return + + num_gpus = current_platform.device_count() + if num_gpus == 0: + return + + wait_for_gpu_memory_to_clear( + devices=list(range(num_gpus)), + threshold_ratio=threshold_ratio, + timeout_s=timeout_s, + ) + + _P = ParamSpec("_P")