[ROCm][CI] Stabilizing teardown and timeout of flaky tests to prevent rare OOMs (#44761)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-06-08 14:11:17 +08:00
committed by GitHub
parent 967c5c3bc3
commit d9ff7e4e9a
4 changed files with 130 additions and 65 deletions
@@ -7,7 +7,7 @@ from contextlib import contextmanager, nullcontext
import pytest
from tests.models.registry import HF_EXAMPLE_MODELS
from tests.utils import multi_gpu_test, wait_for_gpu_memory_to_clear
from tests.utils import multi_gpu_test
from vllm import LLM
from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
@@ -405,28 +405,10 @@ def _get_vllm_runner_params(
}
def _wait_for_rocm_memory_to_settle() -> None:
if not current_platform.is_rocm():
return
num_gpus = current_platform.device_count()
if num_gpus == 0:
return
wait_for_gpu_memory_to_clear(
devices=list(range(num_gpus)),
threshold_ratio=0.01,
timeout_s=120,
)
@contextmanager
def _owned_vLLM_runner(vllm_runner, kwargs):
try:
with vllm_runner(**kwargs) as runner:
yield runner
finally:
_wait_for_rocm_memory_to_settle()
def _owned_vllm_runner(vllm_runner, kwargs):
with vllm_runner(**kwargs) as runner:
yield runner
def _get_vLLM_output(
@@ -439,7 +421,7 @@ def _get_vLLM_output(
vllm_model=None,
):
runner_context = (
_owned_vLLM_runner(vllm_runner, kwargs)
_owned_vllm_runner(vllm_runner, kwargs)
if vllm_model is None
else nullcontext(vllm_model)
)
@@ -801,7 +783,7 @@ def test_apc_multiple_prompts_partial_cached_outputs(
# Cache only part of all the prompts
vllm_runner_kwargs["enable_prefix_caching"] = True
with _owned_vLLM_runner(vllm_runner, vllm_runner_kwargs) as vllm_model:
with _owned_vllm_runner(vllm_runner, vllm_runner_kwargs) as vllm_model:
vllm_outputs_partial_cache, _ = _get_vLLM_output(
vllm_runner,
vllm_runner_kwargs,
@@ -861,7 +843,7 @@ def test_same_mamba_output_apc_on_vs_off(
# No prefix caching
kwargs_no_apc = {**base_kwargs, "enable_prefix_caching": False}
with _owned_vLLM_runner(vllm_runner, kwargs_no_apc) as vllm_model:
with vllm_runner(**kwargs_no_apc) as vllm_model:
outputs_no_apc, _ = _get_vLLM_output(
vllm_runner,
kwargs_no_apc,
@@ -876,7 +858,7 @@ def test_same_mamba_output_apc_on_vs_off(
"enable_prefix_caching": True,
"mamba_block_size": 16,
}
with _owned_vLLM_runner(vllm_runner, kwargs_with_apc) as vllm_model:
with vllm_runner(**kwargs_with_apc) as vllm_model:
outputs_with_apc, _ = _get_vLLM_output(
vllm_runner,
kwargs_with_apc,
+34 -12
View File
@@ -6,15 +6,19 @@ Tests are parametrized across multiple ColBERT backbones to ensure the
generic ColBERT support works with different encoder architectures.
"""
from contextlib import contextmanager
import pytest
import torch
from tests.utils import wait_for_rocm_memory_to_settle
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
# -----------------------------------------------------------------------
# Model definitions: (model_name, colbert_dim, extra vllm_runner kwargs)
# -----------------------------------------------------------------------
COLBERT_MODELS: dict[str, dict] = {
COLBERT_MODELS = {
"bert": {
"model": "answerdotai/answerai-colbert-small-v1",
"colbert_dim": 96,
@@ -145,6 +149,24 @@ def _compute_hf_colbert_embeddings(model, tokenizer, linear_weight, texts, devic
return embeddings
@contextmanager
def _hf_colbert_model(model_name: str, hf_spec: dict, device: torch.device):
"""Load the HF backbone + ColBERT projection, freeing the GPU on exit.
These live outside any runner context, so without explicit cleanup ROCm
keeps the VRAM resident and the next backend parametrization (or test)
OOMs on startup.
"""
hf_model = _load_hf_model(model_name, hf_spec, device)
linear_weight = _load_projection_weight(model_name, hf_spec, device)
try:
yield hf_model, linear_weight
finally:
del hf_model, linear_weight
cleanup_dist_env_and_memory()
wait_for_rocm_memory_to_settle()
def _assert_embeddings_close(vllm_outputs, hf_embeddings):
"""Assert that vLLM and HuggingFace embeddings match."""
for i, (hf_emb, vllm_out) in enumerate(zip(hf_embeddings, vllm_outputs)):
@@ -363,9 +385,11 @@ def test_colbert_hf_comparison(vllm_runner, backend):
spec = COLBERT_MODELS[backend]
hf_spec = spec["hf_comparison"]
extra_kwargs = spec["extra_kwargs"]
model_name = spec["model"]
assert isinstance(model_name, str)
assert isinstance(hf_spec, dict)
assert isinstance(extra_kwargs, dict)
test_texts = [TEXTS_1[0], TEXTS_2[0]]
with vllm_runner(
@@ -374,7 +398,7 @@ def test_colbert_hf_comparison(vllm_runner, backend):
dtype="float32",
max_model_len=spec["max_model_len"],
enforce_eager=True,
**spec["extra_kwargs"],
**extra_kwargs,
) as vllm_model:
vllm_outputs = vllm_model.token_embed(test_texts)
@@ -384,15 +408,13 @@ def test_colbert_hf_comparison(vllm_runner, backend):
model_name,
trust_remote_code=hf_spec.get("trust_remote_code", False),
)
hf_model = _load_hf_model(model_name, hf_spec, device)
linear_weight = _load_projection_weight(model_name, hf_spec, device)
hf_embeddings = _compute_hf_colbert_embeddings(
hf_model,
hf_tokenizer,
linear_weight,
test_texts,
device,
)
with _hf_colbert_model(model_name, hf_spec, device) as (hf_model, linear_weight):
hf_embeddings = _compute_hf_colbert_embeddings(
hf_model,
hf_tokenizer,
linear_weight,
test_texts,
device,
)
_assert_embeddings_close(vllm_outputs, hf_embeddings)