[ROCm][CI] Stabilizing teardown and timeout of flaky tests to prevent rare OOMs (#44761)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -7,7 +7,7 @@ from contextlib import contextmanager, nullcontext
|
||||
import pytest
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from tests.utils import multi_gpu_test, wait_for_gpu_memory_to_clear
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm import LLM
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
@@ -405,28 +405,10 @@ def _get_vllm_runner_params(
|
||||
}
|
||||
|
||||
|
||||
def _wait_for_rocm_memory_to_settle() -> None:
|
||||
if not current_platform.is_rocm():
|
||||
return
|
||||
|
||||
num_gpus = current_platform.device_count()
|
||||
if num_gpus == 0:
|
||||
return
|
||||
|
||||
wait_for_gpu_memory_to_clear(
|
||||
devices=list(range(num_gpus)),
|
||||
threshold_ratio=0.01,
|
||||
timeout_s=120,
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _owned_vLLM_runner(vllm_runner, kwargs):
|
||||
try:
|
||||
with vllm_runner(**kwargs) as runner:
|
||||
yield runner
|
||||
finally:
|
||||
_wait_for_rocm_memory_to_settle()
|
||||
def _owned_vllm_runner(vllm_runner, kwargs):
|
||||
with vllm_runner(**kwargs) as runner:
|
||||
yield runner
|
||||
|
||||
|
||||
def _get_vLLM_output(
|
||||
@@ -439,7 +421,7 @@ def _get_vLLM_output(
|
||||
vllm_model=None,
|
||||
):
|
||||
runner_context = (
|
||||
_owned_vLLM_runner(vllm_runner, kwargs)
|
||||
_owned_vllm_runner(vllm_runner, kwargs)
|
||||
if vllm_model is None
|
||||
else nullcontext(vllm_model)
|
||||
)
|
||||
@@ -801,7 +783,7 @@ def test_apc_multiple_prompts_partial_cached_outputs(
|
||||
|
||||
# Cache only part of all the prompts
|
||||
vllm_runner_kwargs["enable_prefix_caching"] = True
|
||||
with _owned_vLLM_runner(vllm_runner, vllm_runner_kwargs) as vllm_model:
|
||||
with _owned_vllm_runner(vllm_runner, vllm_runner_kwargs) as vllm_model:
|
||||
vllm_outputs_partial_cache, _ = _get_vLLM_output(
|
||||
vllm_runner,
|
||||
vllm_runner_kwargs,
|
||||
@@ -861,7 +843,7 @@ def test_same_mamba_output_apc_on_vs_off(
|
||||
|
||||
# No prefix caching
|
||||
kwargs_no_apc = {**base_kwargs, "enable_prefix_caching": False}
|
||||
with _owned_vLLM_runner(vllm_runner, kwargs_no_apc) as vllm_model:
|
||||
with vllm_runner(**kwargs_no_apc) as vllm_model:
|
||||
outputs_no_apc, _ = _get_vLLM_output(
|
||||
vllm_runner,
|
||||
kwargs_no_apc,
|
||||
@@ -876,7 +858,7 @@ def test_same_mamba_output_apc_on_vs_off(
|
||||
"enable_prefix_caching": True,
|
||||
"mamba_block_size": 16,
|
||||
}
|
||||
with _owned_vLLM_runner(vllm_runner, kwargs_with_apc) as vllm_model:
|
||||
with vllm_runner(**kwargs_with_apc) as vllm_model:
|
||||
outputs_with_apc, _ = _get_vLLM_output(
|
||||
vllm_runner,
|
||||
kwargs_with_apc,
|
||||
|
||||
@@ -6,15 +6,19 @@ Tests are parametrized across multiple ColBERT backbones to ensure the
|
||||
generic ColBERT support works with different encoder architectures.
|
||||
"""
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import wait_for_rocm_memory_to_settle
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Model definitions: (model_name, colbert_dim, extra vllm_runner kwargs)
|
||||
# -----------------------------------------------------------------------
|
||||
COLBERT_MODELS: dict[str, dict] = {
|
||||
COLBERT_MODELS = {
|
||||
"bert": {
|
||||
"model": "answerdotai/answerai-colbert-small-v1",
|
||||
"colbert_dim": 96,
|
||||
@@ -145,6 +149,24 @@ def _compute_hf_colbert_embeddings(model, tokenizer, linear_weight, texts, devic
|
||||
return embeddings
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _hf_colbert_model(model_name: str, hf_spec: dict, device: torch.device):
|
||||
"""Load the HF backbone + ColBERT projection, freeing the GPU on exit.
|
||||
|
||||
These live outside any runner context, so without explicit cleanup ROCm
|
||||
keeps the VRAM resident and the next backend parametrization (or test)
|
||||
OOMs on startup.
|
||||
"""
|
||||
hf_model = _load_hf_model(model_name, hf_spec, device)
|
||||
linear_weight = _load_projection_weight(model_name, hf_spec, device)
|
||||
try:
|
||||
yield hf_model, linear_weight
|
||||
finally:
|
||||
del hf_model, linear_weight
|
||||
cleanup_dist_env_and_memory()
|
||||
wait_for_rocm_memory_to_settle()
|
||||
|
||||
|
||||
def _assert_embeddings_close(vllm_outputs, hf_embeddings):
|
||||
"""Assert that vLLM and HuggingFace embeddings match."""
|
||||
for i, (hf_emb, vllm_out) in enumerate(zip(hf_embeddings, vllm_outputs)):
|
||||
@@ -363,9 +385,11 @@ def test_colbert_hf_comparison(vllm_runner, backend):
|
||||
|
||||
spec = COLBERT_MODELS[backend]
|
||||
hf_spec = spec["hf_comparison"]
|
||||
extra_kwargs = spec["extra_kwargs"]
|
||||
model_name = spec["model"]
|
||||
assert isinstance(model_name, str)
|
||||
assert isinstance(hf_spec, dict)
|
||||
assert isinstance(extra_kwargs, dict)
|
||||
test_texts = [TEXTS_1[0], TEXTS_2[0]]
|
||||
|
||||
with vllm_runner(
|
||||
@@ -374,7 +398,7 @@ def test_colbert_hf_comparison(vllm_runner, backend):
|
||||
dtype="float32",
|
||||
max_model_len=spec["max_model_len"],
|
||||
enforce_eager=True,
|
||||
**spec["extra_kwargs"],
|
||||
**extra_kwargs,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_embed(test_texts)
|
||||
|
||||
@@ -384,15 +408,13 @@ def test_colbert_hf_comparison(vllm_runner, backend):
|
||||
model_name,
|
||||
trust_remote_code=hf_spec.get("trust_remote_code", False),
|
||||
)
|
||||
hf_model = _load_hf_model(model_name, hf_spec, device)
|
||||
linear_weight = _load_projection_weight(model_name, hf_spec, device)
|
||||
|
||||
hf_embeddings = _compute_hf_colbert_embeddings(
|
||||
hf_model,
|
||||
hf_tokenizer,
|
||||
linear_weight,
|
||||
test_texts,
|
||||
device,
|
||||
)
|
||||
with _hf_colbert_model(model_name, hf_spec, device) as (hf_model, linear_weight):
|
||||
hf_embeddings = _compute_hf_colbert_embeddings(
|
||||
hf_model,
|
||||
hf_tokenizer,
|
||||
linear_weight,
|
||||
test_texts,
|
||||
device,
|
||||
)
|
||||
|
||||
_assert_embeddings_close(vllm_outputs, hf_embeddings)
|
||||
|
||||
Reference in New Issue
Block a user