[ROCm][CI] Stabilizing teardown and timeout of flaky tests to prevent rare OOMs (#44761)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -6,15 +6,19 @@ Tests are parametrized across multiple ColBERT backbones to ensure the
|
||||
generic ColBERT support works with different encoder architectures.
|
||||
"""
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import wait_for_rocm_memory_to_settle
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Model definitions: (model_name, colbert_dim, extra vllm_runner kwargs)
|
||||
# -----------------------------------------------------------------------
|
||||
COLBERT_MODELS: dict[str, dict] = {
|
||||
COLBERT_MODELS = {
|
||||
"bert": {
|
||||
"model": "answerdotai/answerai-colbert-small-v1",
|
||||
"colbert_dim": 96,
|
||||
@@ -145,6 +149,24 @@ def _compute_hf_colbert_embeddings(model, tokenizer, linear_weight, texts, devic
|
||||
return embeddings
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _hf_colbert_model(model_name: str, hf_spec: dict, device: torch.device):
|
||||
"""Load the HF backbone + ColBERT projection, freeing the GPU on exit.
|
||||
|
||||
These live outside any runner context, so without explicit cleanup ROCm
|
||||
keeps the VRAM resident and the next backend parametrization (or test)
|
||||
OOMs on startup.
|
||||
"""
|
||||
hf_model = _load_hf_model(model_name, hf_spec, device)
|
||||
linear_weight = _load_projection_weight(model_name, hf_spec, device)
|
||||
try:
|
||||
yield hf_model, linear_weight
|
||||
finally:
|
||||
del hf_model, linear_weight
|
||||
cleanup_dist_env_and_memory()
|
||||
wait_for_rocm_memory_to_settle()
|
||||
|
||||
|
||||
def _assert_embeddings_close(vllm_outputs, hf_embeddings):
|
||||
"""Assert that vLLM and HuggingFace embeddings match."""
|
||||
for i, (hf_emb, vllm_out) in enumerate(zip(hf_embeddings, vllm_outputs)):
|
||||
@@ -363,9 +385,11 @@ def test_colbert_hf_comparison(vllm_runner, backend):
|
||||
|
||||
spec = COLBERT_MODELS[backend]
|
||||
hf_spec = spec["hf_comparison"]
|
||||
extra_kwargs = spec["extra_kwargs"]
|
||||
model_name = spec["model"]
|
||||
assert isinstance(model_name, str)
|
||||
assert isinstance(hf_spec, dict)
|
||||
assert isinstance(extra_kwargs, dict)
|
||||
test_texts = [TEXTS_1[0], TEXTS_2[0]]
|
||||
|
||||
with vllm_runner(
|
||||
@@ -374,7 +398,7 @@ def test_colbert_hf_comparison(vllm_runner, backend):
|
||||
dtype="float32",
|
||||
max_model_len=spec["max_model_len"],
|
||||
enforce_eager=True,
|
||||
**spec["extra_kwargs"],
|
||||
**extra_kwargs,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_embed(test_texts)
|
||||
|
||||
@@ -384,15 +408,13 @@ def test_colbert_hf_comparison(vllm_runner, backend):
|
||||
model_name,
|
||||
trust_remote_code=hf_spec.get("trust_remote_code", False),
|
||||
)
|
||||
hf_model = _load_hf_model(model_name, hf_spec, device)
|
||||
linear_weight = _load_projection_weight(model_name, hf_spec, device)
|
||||
|
||||
hf_embeddings = _compute_hf_colbert_embeddings(
|
||||
hf_model,
|
||||
hf_tokenizer,
|
||||
linear_weight,
|
||||
test_texts,
|
||||
device,
|
||||
)
|
||||
with _hf_colbert_model(model_name, hf_spec, device) as (hf_model, linear_weight):
|
||||
hf_embeddings = _compute_hf_colbert_embeddings(
|
||||
hf_model,
|
||||
hf_tokenizer,
|
||||
linear_weight,
|
||||
test_texts,
|
||||
device,
|
||||
)
|
||||
|
||||
_assert_embeddings_close(vllm_outputs, hf_embeddings)
|
||||
|
||||
Reference in New Issue
Block a user