diff --git a/tests/models/language/pooling/test_colbert.py b/tests/models/language/pooling/test_colbert.py index 810d7064d7..3057e14060 100644 --- a/tests/models/language/pooling/test_colbert.py +++ b/tests/models/language/pooling/test_colbert.py @@ -379,8 +379,16 @@ def test_colbert_embed_not_supported( vllm_model.embed([TEXTS_1[0]]) -@pytest.mark.parametrize("backend", list(COLBERT_MODELS.keys())) -def test_colbert_hf_comparison(vllm_runner, backend): +@pytest.mark.parametrize( + ("backend", "use_v2"), + [ + pytest.param("bert", True, id="bert-v2"), + pytest.param("modernbert", True, id="modernbert-v2"), + pytest.param("jina", False, id="jina-v1"), + pytest.param("lfm2", False, id="lfm2-v1"), + ], +) +def test_colbert_hf_comparison(vllm_runner, monkeypatch, backend, use_v2): """Test that vLLM ColBERT embeddings match HuggingFace for each backend.""" from transformers import AutoTokenizer @@ -393,6 +401,8 @@ def test_colbert_hf_comparison(vllm_runner, backend): assert isinstance(extra_kwargs, dict) test_texts = [TEXTS_1[0], TEXTS_2[0]] + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2 else "0") + with vllm_runner( model_name, runner="pooling", @@ -401,6 +411,7 @@ def test_colbert_hf_comparison(vllm_runner, backend): enforce_eager=True, **extra_kwargs, ) as vllm_model: + assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner == use_v2 vllm_outputs = vllm_model.token_embed(test_texts) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") diff --git a/tests/models/language/pooling/test_splade_sparse_pooler.py b/tests/models/language/pooling/test_splade_sparse_pooler.py index 4a43bc1991..3f2888f048 100644 --- a/tests/models/language/pooling/test_splade_sparse_pooler.py +++ b/tests/models/language/pooling/test_splade_sparse_pooler.py @@ -15,8 +15,10 @@ from vllm.model_executor.models.bert import ( from vllm.platforms import current_platform from vllm.pooling_params import PoolingParams from vllm.utils.torch_utils import PIN_MEMORY +from vllm.v1.pool.late_interaction_runner import LateInteractionRunner from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.model_runner import GPUModelRunner from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner from vllm.v1.worker.gpu.states import RequestState @@ -138,9 +140,11 @@ def test_pooling_runner_stores_only_required_token_ids() -> None: runner.pooling_params = {} runner.pooling_states = {} runner.prompt_token_ids = {} + runner.late_interaction_runner = MagicMock() - runner.add_request(1, PoolingParams(task="embed"), [101, 102]) + runner.add_request("req-1", 1, PoolingParams(task="embed"), [101, 102]) runner.add_request( + "req-2", 2, PoolingParams(task="embed", requires_token_ids=True), [101, 11, 102], @@ -150,6 +154,42 @@ def test_pooling_runner_stores_only_required_token_ids() -> None: torch.testing.assert_close(runner.prompt_token_ids[2], torch.tensor([101, 11, 102])) +def test_pooling_runner_releases_aborted_late_interaction_doc() -> None: + runner = PoolingRunner.__new__(PoolingRunner) + runner.late_interaction_runner = LateInteractionRunner() + + query_key = "query-abort" + late_interaction_runner = runner.late_interaction_runner + late_interaction_runner._query_cache[query_key] = torch.ones(2, 4) + late_interaction_runner._query_uses[query_key] = 1 + late_interaction_runner._doc_query_keys["doc-req"] = query_key + + runner.on_requests_finished({"doc-req"}) + + assert not late_interaction_runner._query_cache + assert not late_interaction_runner._query_uses + assert not late_interaction_runner._doc_query_keys + + +def test_encoder_cache_reset_clears_late_interaction_state() -> None: + # Cached query embeddings are only invalidated by weight reloads, which + # reach the runner via reset_encoder_cache. Resetting the multi-modal + # cache is unrelated and must leave them intact. + runner = GPUModelRunner.__new__(GPUModelRunner) + runner.encoder_cache = MagicMock() + runner.pooling_runner = MagicMock() + + runner.reset_mm_cache() + + runner.encoder_cache.reset_mm_cache.assert_called_once_with() + runner.pooling_runner.clear.assert_not_called() + + runner.reset_encoder_cache() + + runner.encoder_cache.reset_encoder_cache.assert_called_once_with() + runner.pooling_runner.clear.assert_called_once_with() + + def test_pooling_runner_rejects_unsupported_selected_task() -> None: model = MagicMock() model.pooler.get_supported_tasks.return_value = { @@ -182,6 +222,19 @@ def test_pooling_runner_supports_encoder_token_classification() -> None: assert runner.supported_tasks == {"token_classify"} +def test_pooling_runner_supports_encoder_token_embedding() -> None: + model = MagicMock() + model.pooler.get_supported_tasks.return_value = {"token_embed"} + vllm_config = MagicMock() + vllm_config.scheduler_config.max_num_seqs = 2 + vllm_config.model_config.attn_type = "encoder_only" + vllm_config.model_config.get_pooling_task.return_value = "token_embed" + + runner = PoolingRunner(model, vllm_config) + + assert runner.supported_tasks == {"token_embed"} + + def test_pooling_runner_rejects_decoder_token_classification() -> None: model = MagicMock() model.pooler.get_supported_tasks.return_value = {"token_classify"} @@ -207,3 +260,16 @@ def test_pooling_runner_filters_decoder_token_classification() -> None: runner = PoolingRunner(model, vllm_config) assert runner.supported_tasks == {"embed"} + + +def test_pooling_runner_filters_decoder_token_embedding() -> None: + model = MagicMock() + model.pooler.get_supported_tasks.return_value = {"embed", "token_embed"} + vllm_config = MagicMock() + vllm_config.scheduler_config.max_num_seqs = 2 + vllm_config.model_config.attn_type = "decoder" + vllm_config.model_config.get_pooling_task.return_value = "embed" + + runner = PoolingRunner(model, vllm_config) + + assert runner.supported_tasks == {"embed"} diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 79dcc2cf20..cea735c0e8 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -743,6 +743,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): def reset_encoder_cache(self) -> None: if self.encoder_cache is not None: self.encoder_cache.reset_encoder_cache() + if self.pooling_runner is not None: + self.pooling_runner.clear() def profile_cudagraph_memory(self) -> int: # NOTE(woosuk): It is TBD whether we keep this API or not. @@ -813,6 +815,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): def finish_requests(self, scheduler_output: SchedulerOutput) -> None: finished_req_ids = scheduler_output.finished_req_ids + if self.pooling_runner is not None: + # Preempted docs keep their query-use reservation until rescheduled. + self.pooling_runner.on_requests_finished(finished_req_ids) preempted_req_ids = scheduler_output.preempted_req_ids if preempted_req_ids: finished_req_ids = finished_req_ids.union(preempted_req_ids) @@ -857,6 +862,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): if self.pooling_runner is not None: assert new_req_data.pooling_params is not None self.pooling_runner.add_request( + req_id, req_index, new_req_data.pooling_params, new_req_data.prompt_token_ids, diff --git a/vllm/v1/worker/gpu/pool/pooling_runner.py b/vllm/v1/worker/gpu/pool/pooling_runner.py index 11075ed5d9..139570303b 100644 --- a/vllm/v1/worker/gpu/pool/pooling_runner.py +++ b/vllm/v1/worker/gpu/pool/pooling_runner.py @@ -12,13 +12,15 @@ from vllm.pooling_params import PoolingParams from vllm.tasks import PoolingTask from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.outputs import PoolerOutput +from vllm.v1.pool.late_interaction_runner import LateInteractionRunner from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.states import RequestState _SUPPORTED_TASKS: frozenset[PoolingTask] = frozenset( - {"embed", "classify", "token_classify"} + {"embed", "classify", "token_embed", "token_classify"} ) +_TOKEN_TASKS: frozenset[PoolingTask] = frozenset({"token_embed", "token_classify"}) class PoolingRunner: @@ -53,13 +55,14 @@ class PoolingRunner: self.pooling_params: dict[int, PoolingParams] = {} self.pooling_states: dict[int, PoolingStates] = {} self.prompt_token_ids: dict[int, torch.Tensor] = {} + self.late_interaction_runner = LateInteractionRunner() @staticmethod def _get_enabled_tasks(model_config: ModelConfig) -> frozenset[PoolingTask]: - # Token classification is validated only for unchunked encoder-only prefill. + # Token-wise pooling is validated only for unchunked encoder-only prefill. if model_config.attn_type == "encoder_only": return _SUPPORTED_TASKS - return _SUPPORTED_TASKS - {"token_classify"} + return _SUPPORTED_TASKS - _TOKEN_TASKS @staticmethod def get_supported_tasks( @@ -72,6 +75,7 @@ class PoolingRunner: def add_request( self, + req_id: str, req_index: int, pooling_params: PoolingParams, prompt_token_ids: list[int], @@ -83,6 +87,7 @@ class PoolingRunner: f"Supported tasks: {sorted(self.supported_tasks)}" ) self.model.pooler.get_pooling_updates(task).apply(pooling_params) + self.late_interaction_runner.register_request(req_id, pooling_params) self.pooling_params[req_index] = pooling_params self.pooling_states[req_index] = PoolingStates() if pooling_params.requires_token_ids: @@ -96,6 +101,12 @@ class PoolingRunner: state.clean() self.prompt_token_ids.pop(req_index, None) + def on_requests_finished(self, req_ids: set[str]) -> None: + self.late_interaction_runner.on_requests_finished(req_ids) + + def clear(self) -> None: + self.late_interaction_runner.clear() + def _get_pooling_metadata( self, input_batch: InputBatch, @@ -153,8 +164,13 @@ class PoolingRunner: query_start_loc_gpu=input_batch.query_start_loc[: num_reqs + 1], ) pooler_output = self.model.pooler(hidden_states, pooling_metadata) - finished_mask = pooling_metadata.get_pooling_cursor().is_finished().tolist() + pooler_output = self.late_interaction_runner.postprocess_pooler_output( + raw_pooler_output=pooler_output, + pooling_params=pooling_metadata.pooling_params, + req_ids=input_batch.req_ids, + finished_mask=finished_mask, + ) return pooler_output, finished_mask def _dummy_pooler_run_task(