[Bugfix][CI] Release the shared ColBERT engine before test_colbert_hf_comparison (#52608)

Signed-off-by: Stefan Koncarevic <Stefan.Koncarevic@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
stefankoncarevic
2026-08-17 17:50:26 +00:00
committed by GitHub
co-authored by Andreas Karatzas
parent ceb340e2eb
commit 402547d7f0
+71 -82
View File
@@ -186,33 +186,33 @@ def _assert_embeddings_close(vllm_outputs, hf_embeddings):
)
@pytest.fixture(params=list(COLBERT_MODELS.keys()), scope="module")
@pytest.fixture(params=list(COLBERT_MODELS.keys()), scope="class")
def colbert_spec(request):
"""Return the model spec dict for the current parametrization."""
return COLBERT_MODELS[request.param]
@pytest.fixture(scope="module")
@pytest.fixture(scope="class")
def colbert_model_name(colbert_spec):
return colbert_spec["model"]
@pytest.fixture(scope="module")
@pytest.fixture(scope="class")
def colbert_dim(colbert_spec):
return colbert_spec["colbert_dim"]
@pytest.fixture(scope="module")
@pytest.fixture(scope="class")
def colbert_max_model_len(colbert_spec):
return colbert_spec["max_model_len"]
@pytest.fixture(scope="module")
@pytest.fixture(scope="class")
def colbert_extra_kwargs(colbert_spec):
return colbert_spec["extra_kwargs"]
@pytest.fixture(scope="module")
@pytest.fixture(scope="class")
def colbert_model(
vllm_runner,
colbert_model_name,
@@ -230,103 +230,92 @@ def colbert_model(
yield vllm_model
def test_colbert_token_embed(
colbert_model,
colbert_dim,
):
"""Test that ColBERT model produces token embeddings."""
outputs = colbert_model.token_embed([TEXTS_1[0]])
class TestColbertSharedEngine:
"""Tests sharing one engine per model.
assert len(outputs) == 1
emb = torch.as_tensor(outputs[0])
assert emb.dim() == 2
assert emb.shape[1] == colbert_dim
assert emb.shape[0] > 1
Class-scoped so the engine is released before `test_colbert_hf_comparison`,
which needs a runner of its own and cannot start while this one holds VRAM.
"""
def test_colbert_token_embed(self, colbert_model, colbert_dim):
"""Test that ColBERT model produces token embeddings."""
outputs = colbert_model.token_embed([TEXTS_1[0]])
def test_colbert_late_interaction_1_to_1(
colbert_model,
):
"""Test ColBERT late interaction scoring with 1:1 query-document pair."""
q_outputs = colbert_model.token_embed([TEXTS_1[0]])
d_outputs = colbert_model.token_embed([TEXTS_2[0]])
assert len(outputs) == 1
emb = torch.as_tensor(outputs[0])
assert emb.dim() == 2
assert emb.shape[1] == colbert_dim
assert emb.shape[0] > 1
q_emb = torch.as_tensor(q_outputs[0])
d_emb = torch.as_tensor(d_outputs[0])
def test_colbert_late_interaction_1_to_1(self, colbert_model):
"""Test ColBERT late interaction scoring with 1:1 query-document pair."""
q_outputs = colbert_model.token_embed([TEXTS_1[0]])
d_outputs = colbert_model.token_embed([TEXTS_2[0]])
manual_score = compute_maxsim_score(q_emb, d_emb).item()
q_emb = torch.as_tensor(q_outputs[0])
d_emb = torch.as_tensor(d_outputs[0])
vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2[0])
manual_score = compute_maxsim_score(q_emb, d_emb).item()
assert len(vllm_scores) == 1
assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2[0])
assert len(vllm_scores) == 1
assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
def test_colbert_late_interaction_1_to_N(
colbert_model,
):
"""Test ColBERT late interaction scoring with 1:N query-documents."""
q_outputs = colbert_model.token_embed([TEXTS_1[0]])
d_outputs = colbert_model.token_embed(TEXTS_2)
def test_colbert_late_interaction_1_to_N(self, colbert_model):
"""Test ColBERT late interaction scoring with 1:N query-documents."""
q_outputs = colbert_model.token_embed([TEXTS_1[0]])
d_outputs = colbert_model.token_embed(TEXTS_2)
q_emb = torch.as_tensor(q_outputs[0])
q_emb = torch.as_tensor(q_outputs[0])
manual_scores = []
for d_out in d_outputs:
d_emb = torch.as_tensor(d_out)
manual_scores.append(compute_maxsim_score(q_emb, d_emb).item())
manual_scores = []
for d_out in d_outputs:
d_emb = torch.as_tensor(d_out)
manual_scores.append(compute_maxsim_score(q_emb, d_emb).item())
vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2)
vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2)
assert len(vllm_scores) == 2
for i in range(2):
assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01)
assert len(vllm_scores) == 2
for i in range(2):
assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01)
def test_colbert_late_interaction_N_to_N(self, colbert_model):
"""Test ColBERT late interaction scoring with N:N query-documents."""
q_outputs = colbert_model.token_embed(TEXTS_1)
d_outputs = colbert_model.token_embed(TEXTS_2)
def test_colbert_late_interaction_N_to_N(
colbert_model,
):
"""Test ColBERT late interaction scoring with N:N query-documents."""
q_outputs = colbert_model.token_embed(TEXTS_1)
d_outputs = colbert_model.token_embed(TEXTS_2)
manual_scores = []
for q_out, d_out in zip(q_outputs, d_outputs):
q_emb = torch.as_tensor(q_out)
d_emb = torch.as_tensor(d_out)
manual_scores.append(compute_maxsim_score(q_emb, d_emb).item())
manual_scores = []
for q_out, d_out in zip(q_outputs, d_outputs):
q_emb = torch.as_tensor(q_out)
d_emb = torch.as_tensor(d_out)
manual_scores.append(compute_maxsim_score(q_emb, d_emb).item())
vllm_scores = colbert_model.score(TEXTS_1, TEXTS_2)
vllm_scores = colbert_model.score(TEXTS_1, TEXTS_2)
assert len(vllm_scores) == 2
for i in range(2):
assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01)
assert len(vllm_scores) == 2
for i in range(2):
assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01)
def test_colbert_relevance_ordering(self, colbert_model):
"""Test that ColBERT scores relevant documents higher than irrelevant."""
query = "What is machine learning?"
documents = [
"Machine learning is a subset of artificial intelligence.",
"Python is a programming language.",
"Deep learning uses neural networks.",
]
scores = colbert_model.score(query, documents)
def test_colbert_relevance_ordering(
colbert_model,
):
"""Test that ColBERT scores relevant documents higher than irrelevant."""
query = "What is machine learning?"
documents = [
"Machine learning is a subset of artificial intelligence.",
"Python is a programming language.",
"Deep learning uses neural networks.",
]
assert len(scores) == 3
assert scores[0] > scores[1], "ML doc should score higher than Python doc"
assert scores[2] > scores[1], "DL doc should score higher than Python doc"
scores = colbert_model.score(query, documents)
assert len(scores) == 3
assert scores[0] > scores[1], "ML doc should score higher than Python doc"
assert scores[2] > scores[1], "DL doc should score higher than Python doc"
def test_colbert_embed_not_supported(
colbert_model,
):
"""Test that ColBERT model does not support the embed task."""
with pytest.raises(ValueError, match="Embedding API is not supported"):
colbert_model.embed([TEXTS_1[0]])
def test_colbert_embed_not_supported(self, colbert_model):
"""Test that ColBERT model does not support the embed task."""
with pytest.raises(ValueError, match="Embedding API is not supported"):
colbert_model.embed([TEXTS_1[0]])
@pytest.mark.parametrize(