diff --git a/tests/models/language/pooling/test_colbert.py b/tests/models/language/pooling/test_colbert.py index 975c4d6e8f..bb7afb2336 100644 --- a/tests/models/language/pooling/test_colbert.py +++ b/tests/models/language/pooling/test_colbert.py @@ -186,33 +186,33 @@ def _assert_embeddings_close(vllm_outputs, hf_embeddings): ) -@pytest.fixture(params=list(COLBERT_MODELS.keys()), scope="module") +@pytest.fixture(params=list(COLBERT_MODELS.keys()), scope="class") def colbert_spec(request): """Return the model spec dict for the current parametrization.""" return COLBERT_MODELS[request.param] -@pytest.fixture(scope="module") +@pytest.fixture(scope="class") def colbert_model_name(colbert_spec): return colbert_spec["model"] -@pytest.fixture(scope="module") +@pytest.fixture(scope="class") def colbert_dim(colbert_spec): return colbert_spec["colbert_dim"] -@pytest.fixture(scope="module") +@pytest.fixture(scope="class") def colbert_max_model_len(colbert_spec): return colbert_spec["max_model_len"] -@pytest.fixture(scope="module") +@pytest.fixture(scope="class") def colbert_extra_kwargs(colbert_spec): return colbert_spec["extra_kwargs"] -@pytest.fixture(scope="module") +@pytest.fixture(scope="class") def colbert_model( vllm_runner, colbert_model_name, @@ -230,103 +230,92 @@ def colbert_model( yield vllm_model -def test_colbert_token_embed( - colbert_model, - colbert_dim, -): - """Test that ColBERT model produces token embeddings.""" - outputs = colbert_model.token_embed([TEXTS_1[0]]) +class TestColbertSharedEngine: + """Tests sharing one engine per model. - assert len(outputs) == 1 - emb = torch.as_tensor(outputs[0]) - assert emb.dim() == 2 - assert emb.shape[1] == colbert_dim - assert emb.shape[0] > 1 + Class-scoped so the engine is released before `test_colbert_hf_comparison`, + which needs a runner of its own and cannot start while this one holds VRAM. + """ + def test_colbert_token_embed(self, colbert_model, colbert_dim): + """Test that ColBERT model produces token embeddings.""" + outputs = colbert_model.token_embed([TEXTS_1[0]]) -def test_colbert_late_interaction_1_to_1( - colbert_model, -): - """Test ColBERT late interaction scoring with 1:1 query-document pair.""" - q_outputs = colbert_model.token_embed([TEXTS_1[0]]) - d_outputs = colbert_model.token_embed([TEXTS_2[0]]) + assert len(outputs) == 1 + emb = torch.as_tensor(outputs[0]) + assert emb.dim() == 2 + assert emb.shape[1] == colbert_dim + assert emb.shape[0] > 1 - q_emb = torch.as_tensor(q_outputs[0]) - d_emb = torch.as_tensor(d_outputs[0]) + def test_colbert_late_interaction_1_to_1(self, colbert_model): + """Test ColBERT late interaction scoring with 1:1 query-document pair.""" + q_outputs = colbert_model.token_embed([TEXTS_1[0]]) + d_outputs = colbert_model.token_embed([TEXTS_2[0]]) - manual_score = compute_maxsim_score(q_emb, d_emb).item() + q_emb = torch.as_tensor(q_outputs[0]) + d_emb = torch.as_tensor(d_outputs[0]) - vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2[0]) + manual_score = compute_maxsim_score(q_emb, d_emb).item() - assert len(vllm_scores) == 1 - assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01) + vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2[0]) + assert len(vllm_scores) == 1 + assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01) -def test_colbert_late_interaction_1_to_N( - colbert_model, -): - """Test ColBERT late interaction scoring with 1:N query-documents.""" - q_outputs = colbert_model.token_embed([TEXTS_1[0]]) - d_outputs = colbert_model.token_embed(TEXTS_2) + def test_colbert_late_interaction_1_to_N(self, colbert_model): + """Test ColBERT late interaction scoring with 1:N query-documents.""" + q_outputs = colbert_model.token_embed([TEXTS_1[0]]) + d_outputs = colbert_model.token_embed(TEXTS_2) - q_emb = torch.as_tensor(q_outputs[0]) + q_emb = torch.as_tensor(q_outputs[0]) - manual_scores = [] - for d_out in d_outputs: - d_emb = torch.as_tensor(d_out) - manual_scores.append(compute_maxsim_score(q_emb, d_emb).item()) + manual_scores = [] + for d_out in d_outputs: + d_emb = torch.as_tensor(d_out) + manual_scores.append(compute_maxsim_score(q_emb, d_emb).item()) - vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2) + vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2) - assert len(vllm_scores) == 2 - for i in range(2): - assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01) + assert len(vllm_scores) == 2 + for i in range(2): + assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01) + def test_colbert_late_interaction_N_to_N(self, colbert_model): + """Test ColBERT late interaction scoring with N:N query-documents.""" + q_outputs = colbert_model.token_embed(TEXTS_1) + d_outputs = colbert_model.token_embed(TEXTS_2) -def test_colbert_late_interaction_N_to_N( - colbert_model, -): - """Test ColBERT late interaction scoring with N:N query-documents.""" - q_outputs = colbert_model.token_embed(TEXTS_1) - d_outputs = colbert_model.token_embed(TEXTS_2) + manual_scores = [] + for q_out, d_out in zip(q_outputs, d_outputs): + q_emb = torch.as_tensor(q_out) + d_emb = torch.as_tensor(d_out) + manual_scores.append(compute_maxsim_score(q_emb, d_emb).item()) - manual_scores = [] - for q_out, d_out in zip(q_outputs, d_outputs): - q_emb = torch.as_tensor(q_out) - d_emb = torch.as_tensor(d_out) - manual_scores.append(compute_maxsim_score(q_emb, d_emb).item()) + vllm_scores = colbert_model.score(TEXTS_1, TEXTS_2) - vllm_scores = colbert_model.score(TEXTS_1, TEXTS_2) + assert len(vllm_scores) == 2 + for i in range(2): + assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01) - assert len(vllm_scores) == 2 - for i in range(2): - assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01) + def test_colbert_relevance_ordering(self, colbert_model): + """Test that ColBERT scores relevant documents higher than irrelevant.""" + query = "What is machine learning?" + documents = [ + "Machine learning is a subset of artificial intelligence.", + "Python is a programming language.", + "Deep learning uses neural networks.", + ] + scores = colbert_model.score(query, documents) -def test_colbert_relevance_ordering( - colbert_model, -): - """Test that ColBERT scores relevant documents higher than irrelevant.""" - query = "What is machine learning?" - documents = [ - "Machine learning is a subset of artificial intelligence.", - "Python is a programming language.", - "Deep learning uses neural networks.", - ] + assert len(scores) == 3 + assert scores[0] > scores[1], "ML doc should score higher than Python doc" + assert scores[2] > scores[1], "DL doc should score higher than Python doc" - scores = colbert_model.score(query, documents) - - assert len(scores) == 3 - assert scores[0] > scores[1], "ML doc should score higher than Python doc" - assert scores[2] > scores[1], "DL doc should score higher than Python doc" - - -def test_colbert_embed_not_supported( - colbert_model, -): - """Test that ColBERT model does not support the embed task.""" - with pytest.raises(ValueError, match="Embedding API is not supported"): - colbert_model.embed([TEXTS_1[0]]) + def test_colbert_embed_not_supported(self, colbert_model): + """Test that ColBERT model does not support the embed task.""" + with pytest.raises(ValueError, match="Embedding API is not supported"): + colbert_model.embed([TEXTS_1[0]]) @pytest.mark.parametrize(