[ROCm][CI] Stabilizing teardown and timeout of flaky tests to prevent rare OOMs (#44761)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-06-08 14:11:17 +08:00
committed by GitHub
parent 967c5c3bc3
commit d9ff7e4e9a
4 changed files with 130 additions and 65 deletions
+34 -12
View File
@@ -6,15 +6,19 @@ Tests are parametrized across multiple ColBERT backbones to ensure the
generic ColBERT support works with different encoder architectures.
"""
from contextlib import contextmanager
import pytest
import torch
from tests.utils import wait_for_rocm_memory_to_settle
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
# -----------------------------------------------------------------------
# Model definitions: (model_name, colbert_dim, extra vllm_runner kwargs)
# -----------------------------------------------------------------------
COLBERT_MODELS: dict[str, dict] = {
COLBERT_MODELS = {
"bert": {
"model": "answerdotai/answerai-colbert-small-v1",
"colbert_dim": 96,
@@ -145,6 +149,24 @@ def _compute_hf_colbert_embeddings(model, tokenizer, linear_weight, texts, devic
return embeddings
@contextmanager
def _hf_colbert_model(model_name: str, hf_spec: dict, device: torch.device):
"""Load the HF backbone + ColBERT projection, freeing the GPU on exit.
These live outside any runner context, so without explicit cleanup ROCm
keeps the VRAM resident and the next backend parametrization (or test)
OOMs on startup.
"""
hf_model = _load_hf_model(model_name, hf_spec, device)
linear_weight = _load_projection_weight(model_name, hf_spec, device)
try:
yield hf_model, linear_weight
finally:
del hf_model, linear_weight
cleanup_dist_env_and_memory()
wait_for_rocm_memory_to_settle()
def _assert_embeddings_close(vllm_outputs, hf_embeddings):
"""Assert that vLLM and HuggingFace embeddings match."""
for i, (hf_emb, vllm_out) in enumerate(zip(hf_embeddings, vllm_outputs)):
@@ -363,9 +385,11 @@ def test_colbert_hf_comparison(vllm_runner, backend):
spec = COLBERT_MODELS[backend]
hf_spec = spec["hf_comparison"]
extra_kwargs = spec["extra_kwargs"]
model_name = spec["model"]
assert isinstance(model_name, str)
assert isinstance(hf_spec, dict)
assert isinstance(extra_kwargs, dict)
test_texts = [TEXTS_1[0], TEXTS_2[0]]
with vllm_runner(
@@ -374,7 +398,7 @@ def test_colbert_hf_comparison(vllm_runner, backend):
dtype="float32",
max_model_len=spec["max_model_len"],
enforce_eager=True,
**spec["extra_kwargs"],
**extra_kwargs,
) as vllm_model:
vllm_outputs = vllm_model.token_embed(test_texts)
@@ -384,15 +408,13 @@ def test_colbert_hf_comparison(vllm_runner, backend):
model_name,
trust_remote_code=hf_spec.get("trust_remote_code", False),
)
hf_model = _load_hf_model(model_name, hf_spec, device)
linear_weight = _load_projection_weight(model_name, hf_spec, device)
hf_embeddings = _compute_hf_colbert_embeddings(
hf_model,
hf_tokenizer,
linear_weight,
test_texts,
device,
)
with _hf_colbert_model(model_name, hf_spec, device) as (hf_model, linear_weight):
hf_embeddings = _compute_hf_colbert_embeddings(
hf_model,
hf_tokenizer,
linear_weight,
test_texts,
device,
)
_assert_embeddings_close(vllm_outputs, hf_embeddings)