[ROCm][CI] Stabilizing teardown and timeout of flaky tests to prevent rare OOMs (#44761)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-06-08 14:11:17 +08:00
committed by GitHub
parent 967c5c3bc3
commit d9ff7e4e9a
4 changed files with 130 additions and 65 deletions
+18 -17
View File
@@ -854,8 +854,13 @@ class HfRunner:
return self
def __exit__(self, exc_type, exc_value, traceback):
from tests.utils import wait_for_rocm_memory_to_settle
del self.model
cleanup_dist_env_and_memory()
# ROCm frees VRAM lazily; wait so a runner started right after this HF
# model exits does not OOM on its startup memory guard.
wait_for_rocm_memory_to_settle()
@pytest.fixture(scope="session")
@@ -1248,25 +1253,13 @@ class VllmRunner:
return self
def _wait_for_rocm_memory_release(self, gpu_memory_utilization: float) -> None:
from tests.utils import wait_for_gpu_memory_to_clear
from vllm.platforms import current_platform
if not current_platform.is_rocm():
return
num_gpus = torch.accelerator.device_count()
if num_gpus == 0:
return
from tests.utils import wait_for_rocm_memory_to_settle
# V1 startup requires free_memory >= total * gpu_memory_utilization.
# Wait for the complementary used-memory ratio so the next runner does
# not fail the startup guard immediately after this runner exits. Bound
# the wait so cleanup failures fail this test instead of hanging.
wait_for_gpu_memory_to_clear(
devices=list(range(num_gpus)),
threshold_ratio=1.0 - gpu_memory_utilization,
timeout_s=120,
)
# not fail the startup guard immediately after this runner exits. The
# wait is bounded so cleanup failures fail this test instead of hanging.
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
def __exit__(self, exc_type, exc_value, traceback):
# Explicitly shutdown the engine core to release GPU resources
@@ -1276,8 +1269,16 @@ class VllmRunner:
gpu_memory_utilization = (
self.llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
)
from vllm.platforms import current_platform
try:
self.llm.llm_engine.engine_core.shutdown()
# Give the engine core time to run its own graceful shutdown
# (model_executor teardown + empty_cache + process-group destroy)
# before the process manager SIGKILLs it at the default 5s. On ROCm
# a hard kill leaves the whole allocation for the driver's slow async
# VRAM reclamation, which starves the next test's startup.
shutdown_timeout = 60.0 if current_platform.is_rocm() else None
self.llm.llm_engine.engine_core.shutdown(timeout=shutdown_timeout)
except Exception:
# Ignore shutdown errors as cleanup will still proceed
pass
@@ -7,7 +7,7 @@ from contextlib import contextmanager, nullcontext
import pytest
from tests.models.registry import HF_EXAMPLE_MODELS
from tests.utils import multi_gpu_test, wait_for_gpu_memory_to_clear
from tests.utils import multi_gpu_test
from vllm import LLM
from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
@@ -405,28 +405,10 @@ def _get_vllm_runner_params(
}
def _wait_for_rocm_memory_to_settle() -> None:
if not current_platform.is_rocm():
return
num_gpus = current_platform.device_count()
if num_gpus == 0:
return
wait_for_gpu_memory_to_clear(
devices=list(range(num_gpus)),
threshold_ratio=0.01,
timeout_s=120,
)
@contextmanager
def _owned_vLLM_runner(vllm_runner, kwargs):
try:
with vllm_runner(**kwargs) as runner:
yield runner
finally:
_wait_for_rocm_memory_to_settle()
def _owned_vllm_runner(vllm_runner, kwargs):
with vllm_runner(**kwargs) as runner:
yield runner
def _get_vLLM_output(
@@ -439,7 +421,7 @@ def _get_vLLM_output(
vllm_model=None,
):
runner_context = (
_owned_vLLM_runner(vllm_runner, kwargs)
_owned_vllm_runner(vllm_runner, kwargs)
if vllm_model is None
else nullcontext(vllm_model)
)
@@ -801,7 +783,7 @@ def test_apc_multiple_prompts_partial_cached_outputs(
# Cache only part of all the prompts
vllm_runner_kwargs["enable_prefix_caching"] = True
with _owned_vLLM_runner(vllm_runner, vllm_runner_kwargs) as vllm_model:
with _owned_vllm_runner(vllm_runner, vllm_runner_kwargs) as vllm_model:
vllm_outputs_partial_cache, _ = _get_vLLM_output(
vllm_runner,
vllm_runner_kwargs,
@@ -861,7 +843,7 @@ def test_same_mamba_output_apc_on_vs_off(
# No prefix caching
kwargs_no_apc = {**base_kwargs, "enable_prefix_caching": False}
with _owned_vLLM_runner(vllm_runner, kwargs_no_apc) as vllm_model:
with vllm_runner(**kwargs_no_apc) as vllm_model:
outputs_no_apc, _ = _get_vLLM_output(
vllm_runner,
kwargs_no_apc,
@@ -876,7 +858,7 @@ def test_same_mamba_output_apc_on_vs_off(
"enable_prefix_caching": True,
"mamba_block_size": 16,
}
with _owned_vLLM_runner(vllm_runner, kwargs_with_apc) as vllm_model:
with vllm_runner(**kwargs_with_apc) as vllm_model:
outputs_with_apc, _ = _get_vLLM_output(
vllm_runner,
kwargs_with_apc,
+34 -12
View File
@@ -6,15 +6,19 @@ Tests are parametrized across multiple ColBERT backbones to ensure the
generic ColBERT support works with different encoder architectures.
"""
from contextlib import contextmanager
import pytest
import torch
from tests.utils import wait_for_rocm_memory_to_settle
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
# -----------------------------------------------------------------------
# Model definitions: (model_name, colbert_dim, extra vllm_runner kwargs)
# -----------------------------------------------------------------------
COLBERT_MODELS: dict[str, dict] = {
COLBERT_MODELS = {
"bert": {
"model": "answerdotai/answerai-colbert-small-v1",
"colbert_dim": 96,
@@ -145,6 +149,24 @@ def _compute_hf_colbert_embeddings(model, tokenizer, linear_weight, texts, devic
return embeddings
@contextmanager
def _hf_colbert_model(model_name: str, hf_spec: dict, device: torch.device):
"""Load the HF backbone + ColBERT projection, freeing the GPU on exit.
These live outside any runner context, so without explicit cleanup ROCm
keeps the VRAM resident and the next backend parametrization (or test)
OOMs on startup.
"""
hf_model = _load_hf_model(model_name, hf_spec, device)
linear_weight = _load_projection_weight(model_name, hf_spec, device)
try:
yield hf_model, linear_weight
finally:
del hf_model, linear_weight
cleanup_dist_env_and_memory()
wait_for_rocm_memory_to_settle()
def _assert_embeddings_close(vllm_outputs, hf_embeddings):
"""Assert that vLLM and HuggingFace embeddings match."""
for i, (hf_emb, vllm_out) in enumerate(zip(hf_embeddings, vllm_outputs)):
@@ -363,9 +385,11 @@ def test_colbert_hf_comparison(vllm_runner, backend):
spec = COLBERT_MODELS[backend]
hf_spec = spec["hf_comparison"]
extra_kwargs = spec["extra_kwargs"]
model_name = spec["model"]
assert isinstance(model_name, str)
assert isinstance(hf_spec, dict)
assert isinstance(extra_kwargs, dict)
test_texts = [TEXTS_1[0], TEXTS_2[0]]
with vllm_runner(
@@ -374,7 +398,7 @@ def test_colbert_hf_comparison(vllm_runner, backend):
dtype="float32",
max_model_len=spec["max_model_len"],
enforce_eager=True,
**spec["extra_kwargs"],
**extra_kwargs,
) as vllm_model:
vllm_outputs = vllm_model.token_embed(test_texts)
@@ -384,15 +408,13 @@ def test_colbert_hf_comparison(vllm_runner, backend):
model_name,
trust_remote_code=hf_spec.get("trust_remote_code", False),
)
hf_model = _load_hf_model(model_name, hf_spec, device)
linear_weight = _load_projection_weight(model_name, hf_spec, device)
hf_embeddings = _compute_hf_colbert_embeddings(
hf_model,
hf_tokenizer,
linear_weight,
test_texts,
device,
)
with _hf_colbert_model(model_name, hf_spec, device) as (hf_model, linear_weight):
hf_embeddings = _compute_hf_colbert_embeddings(
hf_model,
hf_tokenizer,
linear_weight,
test_texts,
device,
)
_assert_embeddings_close(vllm_outputs, hf_embeddings)
+70 -10
View File
@@ -538,11 +538,22 @@ class RemoteVLLMServer:
if current_platform.is_rocm():
with _nvml():
handles = amdsmi_get_processor_handles()
total_used = 0
for handle in handles:
devices = get_physical_device_indices(
list(range(current_platform.device_count()))
)
total_used_mib = 0
for device in devices:
handle = handles[device]
vram_info = amdsmi_get_gpu_vram_usage(handle)
total_used += vram_info["vram_used"]
return total_used
total_used_mib += vram_info["vram_used"]
# amdsmi reports VRAM in MiB; convert to bytes so this
# matches the CUDA/nvml branch (already bytes) and the
# byte-based target in _wait_for_gpu_memory_release. Without
# this, that wait compares MiB against a ~2e9-byte target,
# is always satisfied instantly, and returns "released to
# 0.00 GB" while the previous server's VRAM is still
# resident -- OOMing the next server's startup on ROCm.
return total_used_mib * 1024 * 1024
elif current_platform.is_cuda():
with _nvml():
total_used = 0
@@ -1428,6 +1439,18 @@ def wait_for_gpu_memory_to_clear(
timeout_s: float = 120,
) -> None:
assert threshold_bytes is not None or threshold_ratio is not None
if (
current_platform.is_rocm()
and threshold_ratio is not None
and threshold_ratio < 0.05
):
# ROCm can keep a small runtime/driver footprint resident even after
# all model allocations are gone. On MI300 this has been observed
# around 2.5 GiB, which is above a strict 1% idle threshold but nowhere
# near the amount of free memory needed by the next vLLM runner.
min_threshold_bytes = 4 * 1024**3
threshold_bytes = max(threshold_bytes or 0, min_threshold_bytes)
# Use nvml instead of pytorch to reduce measurement error from torch cuda
# context.
devices = get_physical_device_indices(devices)
@@ -1454,15 +1477,26 @@ def wait_for_gpu_memory_to_clear(
print(f"{k}={v}; ", end="")
print("")
if threshold_bytes is not None:
is_free = lambda used, total: used <= threshold_bytes / 2**30
threshold = f"{threshold_bytes / 2**30} GiB"
if threshold_bytes is not None and threshold_ratio is not None:
threshold_gib = threshold_bytes / 2**30
threshold = f"max({threshold_gib:.2f} GiB, {threshold_ratio:.3f})"
all_free = all(
used <= max(threshold_gib, total * threshold_ratio)
for used, total in output_raw.values()
)
elif threshold_bytes is not None:
threshold_gib = threshold_bytes / 2**30
threshold = f"{threshold_gib} GiB"
all_free = all(used <= threshold_gib for used, _ in output_raw.values())
else:
is_free = lambda used, total: used / total <= threshold_ratio
threshold = f"{threshold_ratio:.2f}"
assert threshold_ratio is not None
threshold = f"{threshold_ratio:.3f}"
all_free = all(
used / total <= threshold_ratio for used, total in output_raw.values()
)
dur_s = time.time() - start_time
if all(is_free(used, total) for used, total in output_raw.values()):
if all_free:
print(
f"Done waiting for free GPU memory on devices {devices=} "
f"({threshold=}) {dur_s=:.02f}"
@@ -1478,6 +1512,32 @@ def wait_for_gpu_memory_to_clear(
time.sleep(5)
def wait_for_rocm_memory_to_settle(
*,
threshold_ratio: float = 0.1,
timeout_s: float = 240,
) -> None:
"""Block until ROCm device VRAM usage drops below ``threshold_ratio``.
ROCm reclaims GPU memory more lazily than CUDA, so back-to-back model
loads in a single test process can OOM the *next* engine/model startup
even after ``cleanup_dist_env_and_memory``. This gives the driver time to
actually release VRAM before the next allocation. No-op off ROCm.
"""
if not current_platform.is_rocm():
return
num_gpus = current_platform.device_count()
if num_gpus == 0:
return
wait_for_gpu_memory_to_clear(
devices=list(range(num_gpus)),
threshold_ratio=threshold_ratio,
timeout_s=timeout_s,
)
_P = ParamSpec("_P")