[Model Runner V2] Enable encoder token embedding (#50574)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-08-01 07:55:36 -04:00
committed by GitHub
parent 3986b967b2
commit 652ba59229
4 changed files with 106 additions and 7 deletions
+13 -2
View File
@@ -379,8 +379,16 @@ def test_colbert_embed_not_supported(
vllm_model.embed([TEXTS_1[0]]) vllm_model.embed([TEXTS_1[0]])
@pytest.mark.parametrize("backend", list(COLBERT_MODELS.keys())) @pytest.mark.parametrize(
def test_colbert_hf_comparison(vllm_runner, backend): ("backend", "use_v2"),
[
pytest.param("bert", True, id="bert-v2"),
pytest.param("modernbert", True, id="modernbert-v2"),
pytest.param("jina", False, id="jina-v1"),
pytest.param("lfm2", False, id="lfm2-v1"),
],
)
def test_colbert_hf_comparison(vllm_runner, monkeypatch, backend, use_v2):
"""Test that vLLM ColBERT embeddings match HuggingFace for each backend.""" """Test that vLLM ColBERT embeddings match HuggingFace for each backend."""
from transformers import AutoTokenizer from transformers import AutoTokenizer
@@ -393,6 +401,8 @@ def test_colbert_hf_comparison(vllm_runner, backend):
assert isinstance(extra_kwargs, dict) assert isinstance(extra_kwargs, dict)
test_texts = [TEXTS_1[0], TEXTS_2[0]] test_texts = [TEXTS_1[0], TEXTS_2[0]]
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2 else "0")
with vllm_runner( with vllm_runner(
model_name, model_name,
runner="pooling", runner="pooling",
@@ -401,6 +411,7 @@ def test_colbert_hf_comparison(vllm_runner, backend):
enforce_eager=True, enforce_eager=True,
**extra_kwargs, **extra_kwargs,
) as vllm_model: ) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner == use_v2
vllm_outputs = vllm_model.token_embed(test_texts) vllm_outputs = vllm_model.token_embed(test_texts)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -15,8 +15,10 @@ from vllm.model_executor.models.bert import (
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.pooling_params import PoolingParams from vllm.pooling_params import PoolingParams
from vllm.utils.torch_utils import PIN_MEMORY from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.pool.late_interaction_runner import LateInteractionRunner
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner
from vllm.v1.worker.gpu.states import RequestState from vllm.v1.worker.gpu.states import RequestState
@@ -138,9 +140,11 @@ def test_pooling_runner_stores_only_required_token_ids() -> None:
runner.pooling_params = {} runner.pooling_params = {}
runner.pooling_states = {} runner.pooling_states = {}
runner.prompt_token_ids = {} runner.prompt_token_ids = {}
runner.late_interaction_runner = MagicMock()
runner.add_request(1, PoolingParams(task="embed"), [101, 102]) runner.add_request("req-1", 1, PoolingParams(task="embed"), [101, 102])
runner.add_request( runner.add_request(
"req-2",
2, 2,
PoolingParams(task="embed", requires_token_ids=True), PoolingParams(task="embed", requires_token_ids=True),
[101, 11, 102], [101, 11, 102],
@@ -150,6 +154,42 @@ def test_pooling_runner_stores_only_required_token_ids() -> None:
torch.testing.assert_close(runner.prompt_token_ids[2], torch.tensor([101, 11, 102])) torch.testing.assert_close(runner.prompt_token_ids[2], torch.tensor([101, 11, 102]))
def test_pooling_runner_releases_aborted_late_interaction_doc() -> None:
runner = PoolingRunner.__new__(PoolingRunner)
runner.late_interaction_runner = LateInteractionRunner()
query_key = "query-abort"
late_interaction_runner = runner.late_interaction_runner
late_interaction_runner._query_cache[query_key] = torch.ones(2, 4)
late_interaction_runner._query_uses[query_key] = 1
late_interaction_runner._doc_query_keys["doc-req"] = query_key
runner.on_requests_finished({"doc-req"})
assert not late_interaction_runner._query_cache
assert not late_interaction_runner._query_uses
assert not late_interaction_runner._doc_query_keys
def test_encoder_cache_reset_clears_late_interaction_state() -> None:
# Cached query embeddings are only invalidated by weight reloads, which
# reach the runner via reset_encoder_cache. Resetting the multi-modal
# cache is unrelated and must leave them intact.
runner = GPUModelRunner.__new__(GPUModelRunner)
runner.encoder_cache = MagicMock()
runner.pooling_runner = MagicMock()
runner.reset_mm_cache()
runner.encoder_cache.reset_mm_cache.assert_called_once_with()
runner.pooling_runner.clear.assert_not_called()
runner.reset_encoder_cache()
runner.encoder_cache.reset_encoder_cache.assert_called_once_with()
runner.pooling_runner.clear.assert_called_once_with()
def test_pooling_runner_rejects_unsupported_selected_task() -> None: def test_pooling_runner_rejects_unsupported_selected_task() -> None:
model = MagicMock() model = MagicMock()
model.pooler.get_supported_tasks.return_value = { model.pooler.get_supported_tasks.return_value = {
@@ -182,6 +222,19 @@ def test_pooling_runner_supports_encoder_token_classification() -> None:
assert runner.supported_tasks == {"token_classify"} assert runner.supported_tasks == {"token_classify"}
def test_pooling_runner_supports_encoder_token_embedding() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"token_embed"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "encoder_only"
vllm_config.model_config.get_pooling_task.return_value = "token_embed"
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"token_embed"}
def test_pooling_runner_rejects_decoder_token_classification() -> None: def test_pooling_runner_rejects_decoder_token_classification() -> None:
model = MagicMock() model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"token_classify"} model.pooler.get_supported_tasks.return_value = {"token_classify"}
@@ -207,3 +260,16 @@ def test_pooling_runner_filters_decoder_token_classification() -> None:
runner = PoolingRunner(model, vllm_config) runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"embed"} assert runner.supported_tasks == {"embed"}
def test_pooling_runner_filters_decoder_token_embedding() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"embed", "token_embed"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "decoder"
vllm_config.model_config.get_pooling_task.return_value = "embed"
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"embed"}
+6
View File
@@ -743,6 +743,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def reset_encoder_cache(self) -> None: def reset_encoder_cache(self) -> None:
if self.encoder_cache is not None: if self.encoder_cache is not None:
self.encoder_cache.reset_encoder_cache() self.encoder_cache.reset_encoder_cache()
if self.pooling_runner is not None:
self.pooling_runner.clear()
def profile_cudagraph_memory(self) -> int: def profile_cudagraph_memory(self) -> int:
# NOTE(woosuk): It is TBD whether we keep this API or not. # NOTE(woosuk): It is TBD whether we keep this API or not.
@@ -813,6 +815,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def finish_requests(self, scheduler_output: SchedulerOutput) -> None: def finish_requests(self, scheduler_output: SchedulerOutput) -> None:
finished_req_ids = scheduler_output.finished_req_ids finished_req_ids = scheduler_output.finished_req_ids
if self.pooling_runner is not None:
# Preempted docs keep their query-use reservation until rescheduled.
self.pooling_runner.on_requests_finished(finished_req_ids)
preempted_req_ids = scheduler_output.preempted_req_ids preempted_req_ids = scheduler_output.preempted_req_ids
if preempted_req_ids: if preempted_req_ids:
finished_req_ids = finished_req_ids.union(preempted_req_ids) finished_req_ids = finished_req_ids.union(preempted_req_ids)
@@ -857,6 +862,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.pooling_runner is not None: if self.pooling_runner is not None:
assert new_req_data.pooling_params is not None assert new_req_data.pooling_params is not None
self.pooling_runner.add_request( self.pooling_runner.add_request(
req_id,
req_index, req_index,
new_req_data.pooling_params, new_req_data.pooling_params,
new_req_data.prompt_token_ids, new_req_data.prompt_token_ids,
+20 -4
View File
@@ -12,13 +12,15 @@ from vllm.pooling_params import PoolingParams
from vllm.tasks import PoolingTask from vllm.tasks import PoolingTask
from vllm.utils.torch_utils import PIN_MEMORY from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.outputs import PoolerOutput from vllm.v1.outputs import PoolerOutput
from vllm.v1.pool.late_interaction_runner import LateInteractionRunner
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.states import RequestState from vllm.v1.worker.gpu.states import RequestState
_SUPPORTED_TASKS: frozenset[PoolingTask] = frozenset( _SUPPORTED_TASKS: frozenset[PoolingTask] = frozenset(
{"embed", "classify", "token_classify"} {"embed", "classify", "token_embed", "token_classify"}
) )
_TOKEN_TASKS: frozenset[PoolingTask] = frozenset({"token_embed", "token_classify"})
class PoolingRunner: class PoolingRunner:
@@ -53,13 +55,14 @@ class PoolingRunner:
self.pooling_params: dict[int, PoolingParams] = {} self.pooling_params: dict[int, PoolingParams] = {}
self.pooling_states: dict[int, PoolingStates] = {} self.pooling_states: dict[int, PoolingStates] = {}
self.prompt_token_ids: dict[int, torch.Tensor] = {} self.prompt_token_ids: dict[int, torch.Tensor] = {}
self.late_interaction_runner = LateInteractionRunner()
@staticmethod @staticmethod
def _get_enabled_tasks(model_config: ModelConfig) -> frozenset[PoolingTask]: def _get_enabled_tasks(model_config: ModelConfig) -> frozenset[PoolingTask]:
# Token classification is validated only for unchunked encoder-only prefill. # Token-wise pooling is validated only for unchunked encoder-only prefill.
if model_config.attn_type == "encoder_only": if model_config.attn_type == "encoder_only":
return _SUPPORTED_TASKS return _SUPPORTED_TASKS
return _SUPPORTED_TASKS - {"token_classify"} return _SUPPORTED_TASKS - _TOKEN_TASKS
@staticmethod @staticmethod
def get_supported_tasks( def get_supported_tasks(
@@ -72,6 +75,7 @@ class PoolingRunner:
def add_request( def add_request(
self, self,
req_id: str,
req_index: int, req_index: int,
pooling_params: PoolingParams, pooling_params: PoolingParams,
prompt_token_ids: list[int], prompt_token_ids: list[int],
@@ -83,6 +87,7 @@ class PoolingRunner:
f"Supported tasks: {sorted(self.supported_tasks)}" f"Supported tasks: {sorted(self.supported_tasks)}"
) )
self.model.pooler.get_pooling_updates(task).apply(pooling_params) self.model.pooler.get_pooling_updates(task).apply(pooling_params)
self.late_interaction_runner.register_request(req_id, pooling_params)
self.pooling_params[req_index] = pooling_params self.pooling_params[req_index] = pooling_params
self.pooling_states[req_index] = PoolingStates() self.pooling_states[req_index] = PoolingStates()
if pooling_params.requires_token_ids: if pooling_params.requires_token_ids:
@@ -96,6 +101,12 @@ class PoolingRunner:
state.clean() state.clean()
self.prompt_token_ids.pop(req_index, None) self.prompt_token_ids.pop(req_index, None)
def on_requests_finished(self, req_ids: set[str]) -> None:
self.late_interaction_runner.on_requests_finished(req_ids)
def clear(self) -> None:
self.late_interaction_runner.clear()
def _get_pooling_metadata( def _get_pooling_metadata(
self, self,
input_batch: InputBatch, input_batch: InputBatch,
@@ -153,8 +164,13 @@ class PoolingRunner:
query_start_loc_gpu=input_batch.query_start_loc[: num_reqs + 1], query_start_loc_gpu=input_batch.query_start_loc[: num_reqs + 1],
) )
pooler_output = self.model.pooler(hidden_states, pooling_metadata) pooler_output = self.model.pooler(hidden_states, pooling_metadata)
finished_mask = pooling_metadata.get_pooling_cursor().is_finished().tolist() finished_mask = pooling_metadata.get_pooling_cursor().is_finished().tolist()
pooler_output = self.late_interaction_runner.postprocess_pooler_output(
raw_pooler_output=pooler_output,
pooling_params=pooling_metadata.pooling_params,
req_ids=input_batch.req_ids,
finished_mask=finished_mask,
)
return pooler_output, finished_mask return pooler_output, finished_mask
def _dummy_pooler_run_task( def _dummy_pooler_run_task(