[ROCm][CI] Stabilizing teardown and timeout of flaky tests to prevent rare OOMs (#44761)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
+18
-17
@@ -854,8 +854,13 @@ class HfRunner:
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
from tests.utils import wait_for_rocm_memory_to_settle
|
||||
|
||||
del self.model
|
||||
cleanup_dist_env_and_memory()
|
||||
# ROCm frees VRAM lazily; wait so a runner started right after this HF
|
||||
# model exits does not OOM on its startup memory guard.
|
||||
wait_for_rocm_memory_to_settle()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
@@ -1248,25 +1253,13 @@ class VllmRunner:
|
||||
return self
|
||||
|
||||
def _wait_for_rocm_memory_release(self, gpu_memory_utilization: float) -> None:
|
||||
from tests.utils import wait_for_gpu_memory_to_clear
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
return
|
||||
|
||||
num_gpus = torch.accelerator.device_count()
|
||||
if num_gpus == 0:
|
||||
return
|
||||
from tests.utils import wait_for_rocm_memory_to_settle
|
||||
|
||||
# V1 startup requires free_memory >= total * gpu_memory_utilization.
|
||||
# Wait for the complementary used-memory ratio so the next runner does
|
||||
# not fail the startup guard immediately after this runner exits. Bound
|
||||
# the wait so cleanup failures fail this test instead of hanging.
|
||||
wait_for_gpu_memory_to_clear(
|
||||
devices=list(range(num_gpus)),
|
||||
threshold_ratio=1.0 - gpu_memory_utilization,
|
||||
timeout_s=120,
|
||||
)
|
||||
# not fail the startup guard immediately after this runner exits. The
|
||||
# wait is bounded so cleanup failures fail this test instead of hanging.
|
||||
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
# Explicitly shutdown the engine core to release GPU resources
|
||||
@@ -1276,8 +1269,16 @@ class VllmRunner:
|
||||
gpu_memory_utilization = (
|
||||
self.llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
try:
|
||||
self.llm.llm_engine.engine_core.shutdown()
|
||||
# Give the engine core time to run its own graceful shutdown
|
||||
# (model_executor teardown + empty_cache + process-group destroy)
|
||||
# before the process manager SIGKILLs it at the default 5s. On ROCm
|
||||
# a hard kill leaves the whole allocation for the driver's slow async
|
||||
# VRAM reclamation, which starves the next test's startup.
|
||||
shutdown_timeout = 60.0 if current_platform.is_rocm() else None
|
||||
self.llm.llm_engine.engine_core.shutdown(timeout=shutdown_timeout)
|
||||
except Exception:
|
||||
# Ignore shutdown errors as cleanup will still proceed
|
||||
pass
|
||||
|
||||
@@ -7,7 +7,7 @@ from contextlib import contextmanager, nullcontext
|
||||
import pytest
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from tests.utils import multi_gpu_test, wait_for_gpu_memory_to_clear
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm import LLM
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
@@ -405,28 +405,10 @@ def _get_vllm_runner_params(
|
||||
}
|
||||
|
||||
|
||||
def _wait_for_rocm_memory_to_settle() -> None:
|
||||
if not current_platform.is_rocm():
|
||||
return
|
||||
|
||||
num_gpus = current_platform.device_count()
|
||||
if num_gpus == 0:
|
||||
return
|
||||
|
||||
wait_for_gpu_memory_to_clear(
|
||||
devices=list(range(num_gpus)),
|
||||
threshold_ratio=0.01,
|
||||
timeout_s=120,
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _owned_vLLM_runner(vllm_runner, kwargs):
|
||||
try:
|
||||
with vllm_runner(**kwargs) as runner:
|
||||
yield runner
|
||||
finally:
|
||||
_wait_for_rocm_memory_to_settle()
|
||||
def _owned_vllm_runner(vllm_runner, kwargs):
|
||||
with vllm_runner(**kwargs) as runner:
|
||||
yield runner
|
||||
|
||||
|
||||
def _get_vLLM_output(
|
||||
@@ -439,7 +421,7 @@ def _get_vLLM_output(
|
||||
vllm_model=None,
|
||||
):
|
||||
runner_context = (
|
||||
_owned_vLLM_runner(vllm_runner, kwargs)
|
||||
_owned_vllm_runner(vllm_runner, kwargs)
|
||||
if vllm_model is None
|
||||
else nullcontext(vllm_model)
|
||||
)
|
||||
@@ -801,7 +783,7 @@ def test_apc_multiple_prompts_partial_cached_outputs(
|
||||
|
||||
# Cache only part of all the prompts
|
||||
vllm_runner_kwargs["enable_prefix_caching"] = True
|
||||
with _owned_vLLM_runner(vllm_runner, vllm_runner_kwargs) as vllm_model:
|
||||
with _owned_vllm_runner(vllm_runner, vllm_runner_kwargs) as vllm_model:
|
||||
vllm_outputs_partial_cache, _ = _get_vLLM_output(
|
||||
vllm_runner,
|
||||
vllm_runner_kwargs,
|
||||
@@ -861,7 +843,7 @@ def test_same_mamba_output_apc_on_vs_off(
|
||||
|
||||
# No prefix caching
|
||||
kwargs_no_apc = {**base_kwargs, "enable_prefix_caching": False}
|
||||
with _owned_vLLM_runner(vllm_runner, kwargs_no_apc) as vllm_model:
|
||||
with vllm_runner(**kwargs_no_apc) as vllm_model:
|
||||
outputs_no_apc, _ = _get_vLLM_output(
|
||||
vllm_runner,
|
||||
kwargs_no_apc,
|
||||
@@ -876,7 +858,7 @@ def test_same_mamba_output_apc_on_vs_off(
|
||||
"enable_prefix_caching": True,
|
||||
"mamba_block_size": 16,
|
||||
}
|
||||
with _owned_vLLM_runner(vllm_runner, kwargs_with_apc) as vllm_model:
|
||||
with vllm_runner(**kwargs_with_apc) as vllm_model:
|
||||
outputs_with_apc, _ = _get_vLLM_output(
|
||||
vllm_runner,
|
||||
kwargs_with_apc,
|
||||
|
||||
@@ -6,15 +6,19 @@ Tests are parametrized across multiple ColBERT backbones to ensure the
|
||||
generic ColBERT support works with different encoder architectures.
|
||||
"""
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import wait_for_rocm_memory_to_settle
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Model definitions: (model_name, colbert_dim, extra vllm_runner kwargs)
|
||||
# -----------------------------------------------------------------------
|
||||
COLBERT_MODELS: dict[str, dict] = {
|
||||
COLBERT_MODELS = {
|
||||
"bert": {
|
||||
"model": "answerdotai/answerai-colbert-small-v1",
|
||||
"colbert_dim": 96,
|
||||
@@ -145,6 +149,24 @@ def _compute_hf_colbert_embeddings(model, tokenizer, linear_weight, texts, devic
|
||||
return embeddings
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _hf_colbert_model(model_name: str, hf_spec: dict, device: torch.device):
|
||||
"""Load the HF backbone + ColBERT projection, freeing the GPU on exit.
|
||||
|
||||
These live outside any runner context, so without explicit cleanup ROCm
|
||||
keeps the VRAM resident and the next backend parametrization (or test)
|
||||
OOMs on startup.
|
||||
"""
|
||||
hf_model = _load_hf_model(model_name, hf_spec, device)
|
||||
linear_weight = _load_projection_weight(model_name, hf_spec, device)
|
||||
try:
|
||||
yield hf_model, linear_weight
|
||||
finally:
|
||||
del hf_model, linear_weight
|
||||
cleanup_dist_env_and_memory()
|
||||
wait_for_rocm_memory_to_settle()
|
||||
|
||||
|
||||
def _assert_embeddings_close(vllm_outputs, hf_embeddings):
|
||||
"""Assert that vLLM and HuggingFace embeddings match."""
|
||||
for i, (hf_emb, vllm_out) in enumerate(zip(hf_embeddings, vllm_outputs)):
|
||||
@@ -363,9 +385,11 @@ def test_colbert_hf_comparison(vllm_runner, backend):
|
||||
|
||||
spec = COLBERT_MODELS[backend]
|
||||
hf_spec = spec["hf_comparison"]
|
||||
extra_kwargs = spec["extra_kwargs"]
|
||||
model_name = spec["model"]
|
||||
assert isinstance(model_name, str)
|
||||
assert isinstance(hf_spec, dict)
|
||||
assert isinstance(extra_kwargs, dict)
|
||||
test_texts = [TEXTS_1[0], TEXTS_2[0]]
|
||||
|
||||
with vllm_runner(
|
||||
@@ -374,7 +398,7 @@ def test_colbert_hf_comparison(vllm_runner, backend):
|
||||
dtype="float32",
|
||||
max_model_len=spec["max_model_len"],
|
||||
enforce_eager=True,
|
||||
**spec["extra_kwargs"],
|
||||
**extra_kwargs,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_embed(test_texts)
|
||||
|
||||
@@ -384,15 +408,13 @@ def test_colbert_hf_comparison(vllm_runner, backend):
|
||||
model_name,
|
||||
trust_remote_code=hf_spec.get("trust_remote_code", False),
|
||||
)
|
||||
hf_model = _load_hf_model(model_name, hf_spec, device)
|
||||
linear_weight = _load_projection_weight(model_name, hf_spec, device)
|
||||
|
||||
hf_embeddings = _compute_hf_colbert_embeddings(
|
||||
hf_model,
|
||||
hf_tokenizer,
|
||||
linear_weight,
|
||||
test_texts,
|
||||
device,
|
||||
)
|
||||
with _hf_colbert_model(model_name, hf_spec, device) as (hf_model, linear_weight):
|
||||
hf_embeddings = _compute_hf_colbert_embeddings(
|
||||
hf_model,
|
||||
hf_tokenizer,
|
||||
linear_weight,
|
||||
test_texts,
|
||||
device,
|
||||
)
|
||||
|
||||
_assert_embeddings_close(vllm_outputs, hf_embeddings)
|
||||
|
||||
+70
-10
@@ -538,11 +538,22 @@ class RemoteVLLMServer:
|
||||
if current_platform.is_rocm():
|
||||
with _nvml():
|
||||
handles = amdsmi_get_processor_handles()
|
||||
total_used = 0
|
||||
for handle in handles:
|
||||
devices = get_physical_device_indices(
|
||||
list(range(current_platform.device_count()))
|
||||
)
|
||||
total_used_mib = 0
|
||||
for device in devices:
|
||||
handle = handles[device]
|
||||
vram_info = amdsmi_get_gpu_vram_usage(handle)
|
||||
total_used += vram_info["vram_used"]
|
||||
return total_used
|
||||
total_used_mib += vram_info["vram_used"]
|
||||
# amdsmi reports VRAM in MiB; convert to bytes so this
|
||||
# matches the CUDA/nvml branch (already bytes) and the
|
||||
# byte-based target in _wait_for_gpu_memory_release. Without
|
||||
# this, that wait compares MiB against a ~2e9-byte target,
|
||||
# is always satisfied instantly, and returns "released to
|
||||
# 0.00 GB" while the previous server's VRAM is still
|
||||
# resident -- OOMing the next server's startup on ROCm.
|
||||
return total_used_mib * 1024 * 1024
|
||||
elif current_platform.is_cuda():
|
||||
with _nvml():
|
||||
total_used = 0
|
||||
@@ -1428,6 +1439,18 @@ def wait_for_gpu_memory_to_clear(
|
||||
timeout_s: float = 120,
|
||||
) -> None:
|
||||
assert threshold_bytes is not None or threshold_ratio is not None
|
||||
if (
|
||||
current_platform.is_rocm()
|
||||
and threshold_ratio is not None
|
||||
and threshold_ratio < 0.05
|
||||
):
|
||||
# ROCm can keep a small runtime/driver footprint resident even after
|
||||
# all model allocations are gone. On MI300 this has been observed
|
||||
# around 2.5 GiB, which is above a strict 1% idle threshold but nowhere
|
||||
# near the amount of free memory needed by the next vLLM runner.
|
||||
min_threshold_bytes = 4 * 1024**3
|
||||
threshold_bytes = max(threshold_bytes or 0, min_threshold_bytes)
|
||||
|
||||
# Use nvml instead of pytorch to reduce measurement error from torch cuda
|
||||
# context.
|
||||
devices = get_physical_device_indices(devices)
|
||||
@@ -1454,15 +1477,26 @@ def wait_for_gpu_memory_to_clear(
|
||||
print(f"{k}={v}; ", end="")
|
||||
print("")
|
||||
|
||||
if threshold_bytes is not None:
|
||||
is_free = lambda used, total: used <= threshold_bytes / 2**30
|
||||
threshold = f"{threshold_bytes / 2**30} GiB"
|
||||
if threshold_bytes is not None and threshold_ratio is not None:
|
||||
threshold_gib = threshold_bytes / 2**30
|
||||
threshold = f"max({threshold_gib:.2f} GiB, {threshold_ratio:.3f})"
|
||||
all_free = all(
|
||||
used <= max(threshold_gib, total * threshold_ratio)
|
||||
for used, total in output_raw.values()
|
||||
)
|
||||
elif threshold_bytes is not None:
|
||||
threshold_gib = threshold_bytes / 2**30
|
||||
threshold = f"{threshold_gib} GiB"
|
||||
all_free = all(used <= threshold_gib for used, _ in output_raw.values())
|
||||
else:
|
||||
is_free = lambda used, total: used / total <= threshold_ratio
|
||||
threshold = f"{threshold_ratio:.2f}"
|
||||
assert threshold_ratio is not None
|
||||
threshold = f"{threshold_ratio:.3f}"
|
||||
all_free = all(
|
||||
used / total <= threshold_ratio for used, total in output_raw.values()
|
||||
)
|
||||
|
||||
dur_s = time.time() - start_time
|
||||
if all(is_free(used, total) for used, total in output_raw.values()):
|
||||
if all_free:
|
||||
print(
|
||||
f"Done waiting for free GPU memory on devices {devices=} "
|
||||
f"({threshold=}) {dur_s=:.02f}"
|
||||
@@ -1478,6 +1512,32 @@ def wait_for_gpu_memory_to_clear(
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
def wait_for_rocm_memory_to_settle(
|
||||
*,
|
||||
threshold_ratio: float = 0.1,
|
||||
timeout_s: float = 240,
|
||||
) -> None:
|
||||
"""Block until ROCm device VRAM usage drops below ``threshold_ratio``.
|
||||
|
||||
ROCm reclaims GPU memory more lazily than CUDA, so back-to-back model
|
||||
loads in a single test process can OOM the *next* engine/model startup
|
||||
even after ``cleanup_dist_env_and_memory``. This gives the driver time to
|
||||
actually release VRAM before the next allocation. No-op off ROCm.
|
||||
"""
|
||||
if not current_platform.is_rocm():
|
||||
return
|
||||
|
||||
num_gpus = current_platform.device_count()
|
||||
if num_gpus == 0:
|
||||
return
|
||||
|
||||
wait_for_gpu_memory_to_clear(
|
||||
devices=list(range(num_gpus)),
|
||||
threshold_ratio=threshold_ratio,
|
||||
timeout_s=timeout_s,
|
||||
)
|
||||
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user