From 3e90d015ba5bbad6f229ddb4c1f467ca33101524 Mon Sep 17 00:00:00 2001 From: "wang.yuqi" Date: Thu, 16 Jul 2026 22:00:20 +0800 Subject: [PATCH] [Frontend] Overlap preprocessing and computation for pooling models offline inference (#47699) Signed-off-by: wang.yuqi --- .buildkite/test_areas/models_language.yaml | 6 +- .../pooling/basic/test_tiling_engine.py | 108 +++++++++ .../pooling/test_multi_vector_retrieval.py | 9 +- vllm/entrypoints/llm.py | 2 +- vllm/entrypoints/pooling/base/io_processor.py | 163 +++++++++++--- .../entrypoints/pooling/embed/io_processor.py | 11 +- vllm/entrypoints/pooling/offline.py | 195 +++++++++++----- .../pooling/pooling/io_processor.py | 41 +++- .../pooling/scoring/io_processor.py | 211 +++++++++++++++--- vllm/entrypoints/pooling/typing.py | 83 ++++++- 10 files changed, 676 insertions(+), 153 deletions(-) create mode 100644 tests/entrypoints/pooling/basic/test_tiling_engine.py diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index 83d2fa26c2..6066c9bc70 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -103,10 +103,10 @@ steps: commands: - pytest -v -s models/language/generation_ppl_test -- label: Language Models Test (Extended Pooling) # 36min +- label: Language Models Test (Extended Pooling) device: h200_35gb key: language-models-test-extended-pooling - timeout_in_minutes: 70 + timeout_in_minutes: 120 optional: true source_file_dependencies: - vllm/ @@ -116,7 +116,7 @@ steps: mirror: amd: device: mi325_1 - timeout_in_minutes: 100 + timeout_in_minutes: 120 depends_on: - image-build-amd diff --git a/tests/entrypoints/pooling/basic/test_tiling_engine.py b/tests/entrypoints/pooling/basic/test_tiling_engine.py new file mode 100644 index 0000000000..329f91debb --- /dev/null +++ b/tests/entrypoints/pooling/basic/test_tiling_engine.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import weakref +from unittest import mock + +import pytest + +from vllm import LLM, PoolingParams +from vllm.distributed import cleanup_dist_env_and_memory + +MODEL_NAME = "intfloat/multilingual-e5-small" + + +@pytest.fixture(scope="module") +def llm(): + llm = LLM( + model=MODEL_NAME, + max_num_seqs=2, # small to trigger tiling + tensor_parallel_size=1, + gpu_memory_utilization=0.75, + enforce_eager=True, + seed=0, + ) + + yield weakref.proxy(llm) + + del llm + + cleanup_dist_env_and_memory() + + +@pytest.mark.skip_global_cleanup +def test_tiling_engine_basic(llm): + """ + Basic test with a small number of prompts (less than max_num_seqs). + No tiling should be triggered, but the engine still processes correctly. + """ + prompts = ["Hello", "World"] + outputs = llm.encode(prompts, pooling_task="embed") + assert len(outputs) == len(prompts) + + +@pytest.mark.skip_global_cleanup +def test_tiling_engine_many_requests(llm): + """ + Test with a large number of prompts that exceeds max_num_seqs. + This verifies that _run_tiling_engine correctly chunks requests, + processes all of them, and returns outputs in the correct order. + """ + num_prompts = 10 + prompts = [f"Prompt {i}" for i in range(num_prompts)] + outputs = llm.encode(prompts, pooling_task="embed") + assert len(outputs) == num_prompts + + +@pytest.mark.skip_global_cleanup +def test_tiling_engine_with_pooling_params(llm): + """ + Test the tiling engine when different PoolingParams are provided. + The engine must handle a list of params that matches the number of prompts. + """ + num_prompts = 10 + prompts = [f"Prompt {i}" for i in range(num_prompts)] + pooling_params = [PoolingParams() for _ in range(num_prompts)] + + outputs = llm.encode(prompts, pooling_params=pooling_params, pooling_task="embed") + assert len(outputs) == num_prompts + + # Single PoolingParams shared across all prompts + single_param = PoolingParams() + outputs = llm.encode(prompts, pooling_params=single_param, pooling_task="embed") + assert len(outputs) == num_prompts + + # None PoolingParams should fall back to default + outputs = llm.encode(prompts, pooling_params=None, pooling_task="embed") + assert len(outputs) == num_prompts + + +@pytest.mark.skip_global_cleanup +def test_tiling_engine_abort_on_exception(llm): + """ + Test that abort_request IS called with the correct arguments when an + exception occurs inside the engine's step() loop. + """ + prompts = ["Prompt 0", "Prompt 1", "Prompt 2"] + + # Mock the step method to throw an exception on the second call + original_step = llm.llm_engine.step + call_count = 0 + + def mocked_step(): + nonlocal call_count + call_count += 1 + if call_count == 2: + raise RuntimeError("Simulated engine error") + return original_step() + + with mock.patch.object(llm.llm_engine, "step", side_effect=mocked_step): + # We expect an exception to be raised from encode + with mock.patch.object(llm.llm_engine, "abort_request") as mock_abort: # noqa: SIM117 + with pytest.raises(RuntimeError, match="Simulated engine error"): + llm.encode(prompts, pooling_task="embed") + + args, kwargs = mock_abort.call_args + request_ids = args[0] + assert isinstance(request_ids, list) + assert len(request_ids) > 0 diff --git a/tests/models/language/pooling/test_multi_vector_retrieval.py b/tests/models/language/pooling/test_multi_vector_retrieval.py index 3161271be0..d72bf20881 100644 --- a/tests/models/language/pooling/test_multi_vector_retrieval.py +++ b/tests/models/language/pooling/test_multi_vector_retrieval.py @@ -12,7 +12,7 @@ from vllm.config import PoolerConfig "model", ["BAAI/bge-m3"], ) -@pytest.mark.parametrize("dtype", ["half"]) +@pytest.mark.parametrize("dtype", ["float"]) @torch.inference_mode def test_embed_models(hf_runner, vllm_runner, example_prompts, model: str, dtype: str): with vllm_runner( @@ -20,13 +20,11 @@ def test_embed_models(hf_runner, vllm_runner, example_prompts, model: str, dtype runner="pooling", pooler_config=PoolerConfig(task="token_embed"), max_model_len=None, + dtype=dtype, ) as vllm_model: vllm_outputs = vllm_model.token_embed(example_prompts) - with hf_runner( - model, - auto_cls=AutoModel, - ) as hf_model: + with hf_runner(model, auto_cls=AutoModel, dtype=dtype) as hf_model: tokenizer = hf_model.tokenizer hf_outputs = [] for prompt in example_prompts: @@ -34,7 +32,6 @@ def test_embed_models(hf_runner, vllm_runner, example_prompts, model: str, dtype inputs = hf_model.wrap_device(inputs) output = hf_model.model(**inputs) embedding = output.last_hidden_state[0].float() - # normal hf_outputs.append(embedding.cpu()) for hf_output, vllm_output in zip(hf_outputs, vllm_outputs): diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 0280c386f2..b3205728e4 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -356,7 +356,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): # path; the synchronous `LLM` entrypoint runs multimodal # preprocessing serially. Warn so the setting is not a silent # no-op. See vllm-project/vllm#42901. - if self.model_config.renderer_num_workers > 1: + if self.model_config.renderer_num_workers > 1 and self.runner_type != "pooling": logger.warning_once( "`renderer_num_workers=%d` was set, but the offline `LLM` " "entrypoint uses the synchronous renderer path and runs " diff --git a/vllm/entrypoints/pooling/base/io_processor.py b/vllm/entrypoints/pooling/base/io_processor.py index 3d672a13ec..cdc4c16cac 100644 --- a/vllm/entrypoints/pooling/base/io_processor.py +++ b/vllm/entrypoints/pooling/base/io_processor.py @@ -2,9 +2,12 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Sequence -from typing import Any, Final +from typing import Any, Final, cast -from vllm import PoolingParams, PoolingRequestOutput, PromptType +from vllm import ( + PoolingParams, + PoolingRequestOutput, +) from vllm.config import VllmConfig from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, @@ -14,18 +17,25 @@ from vllm.entrypoints.chat_utils import ( ) from vllm.entrypoints.serve.engine.typing import RendererChatRequest, RendererRequest from vllm.inputs import EngineInput, SingletonPrompt -from vllm.renderers import BaseRenderer, TokenizeParams, merge_kwargs +from vllm.lora.request import LoRARequest +from vllm.renderers import BaseRenderer, merge_kwargs from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq from vllm.tool_parsers import ToolParser from vllm.utils.mistral import is_mistral_tokenizer -from ..scoring.typing import ScoringData from ..typing import ( - OfflineInputsContext, + ALLOfflineInputsContext, + EncodeChatRenderParams, + EncodeCMPLRenderParams, + OfflineEncodeInputsContext, OfflineOutputsContext, PoolingChatLikeRequest, PoolingCompletionLikeRequest, + PoolingEngineInput, PoolingServeContext, + RequestFactory, + RequestGenerator, + ScoringRenderParams, ) @@ -98,16 +108,62 @@ class PoolingIOProcessor: ####################################### # offline APIs - def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]: - assert not isinstance(ctx.prompts, ScoringData) and not ( - isinstance(ctx.prompts, dict) and "data" in ctx.prompts - ) + def get_request_factory_offline( + self, ctx: ALLOfflineInputsContext + ) -> tuple[RequestFactory, int]: + assert isinstance(ctx, OfflineEncodeInputsContext) prompts_seq = prompt_to_seq(ctx.prompts) + num_requests = len(prompts_seq) + pooling_task = ctx.pooling_task + + parsed_prompts = [ + ( + prompt + if isinstance(prompt, bytes) + else parse_model_prompt(self.model_config, prompt) + ) + for prompt in prompts_seq + ] tok_params = self.renderer.default_cmpl_tok_params.with_kwargs( **(ctx.tokenization_kwargs or {}) ) - return self._preprocess_cmpl_offline(prompts=prompts_seq, tok_params=tok_params) + + pooling_params: PoolingParams | Sequence[PoolingParams] + if ctx.pooling_params is None: + pooling_params = PoolingParams() + else: + pooling_params = ctx.pooling_params + + params_seq = self._params_to_seq(pooling_params, num_requests) + + for param in params_seq: + if param.task is None: + param.task = pooling_task + elif pooling_task == "plugin": + # `plugin` task uses io_processor.parse_request to verify inputs. + # We actually allow plugin to overwrite pooling_task. + pass + elif param.task != pooling_task: + msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!" + raise ValueError(msg) + + seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests) + seq_priority = self._priority_to_seq(ctx.priorities, num_requests) + + def request_factory() -> RequestGenerator: + for i in range(num_requests): + yield EncodeCMPLRenderParams( + prompts=parsed_prompts[i], + tok_params=tok_params, + prompt_extras=None, + skip_mm_cache=False, + params=params_seq[i], + lora_requests=seq_lora_requests[i], + priorities=seq_priority[i], + ) + + return request_factory, num_requests def post_process_offline( self, @@ -118,6 +174,41 @@ class PoolingIOProcessor: ####################################### # helpers + def render( + self, + render_params: EncodeCMPLRenderParams + | EncodeChatRenderParams + | ScoringRenderParams, + ) -> PoolingEngineInput: + if "conversations" in render_params: + render_params = cast(EncodeChatRenderParams, render_params) + (_,), engine_input = self.renderer.render_chat( + conversations=[render_params["conversations"]], + chat_params=render_params["chat_params"], + tok_params=render_params["tok_params"], + prompt_extras=render_params["prompt_extras"], + skip_mm_cache=render_params["skip_mm_cache"], + ) + elif "prompts" in render_params: + render_params = cast(EncodeCMPLRenderParams, render_params) + engine_input = self.renderer.render_cmpl( + prompts=[render_params["prompts"]], + tok_params=render_params["tok_params"], + prompt_extras=render_params["prompt_extras"], + skip_mm_cache=render_params["skip_mm_cache"], + ) + else: + raise ValueError( + f"Unsupported render_params type {render_params.__class__.__name__}" + ) + + return PoolingEngineInput( + prompts=engine_input[0], + params=render_params["params"], + lora_requests=render_params["lora_requests"], + priorities=render_params["priorities"], + ) + def _preprocess_cmpl_online( self, request: RendererRequest, @@ -196,26 +287,6 @@ class PoolingIOProcessor: return conversation, [engine_input] - def _preprocess_cmpl_offline( - self, - prompts: PromptType | Sequence[PromptType], - tok_params: TokenizeParams, - prompt_extras: dict[str, Any] | None = None, - ) -> Sequence[EngineInput]: - prompts = prompt_to_seq(prompts) - parsed_prompts = [ - ( - prompt - if isinstance(prompt, bytes) - else parse_model_prompt(self.model_config, prompt) - ) - for prompt in prompts - ] - - return self.renderer.render_cmpl( - parsed_prompts, tok_params, prompt_extras=prompt_extras - ) - def _validate_chat_template( self, request_chat_template: str | None, @@ -251,3 +322,35 @@ class PoolingIOProcessor: return params return [params] * num_requests + + def _lora_request_to_seq( + self, + lora_request: LoRARequest | None | Sequence[LoRARequest | None], + num_requests: int, + ) -> Sequence[LoRARequest | None]: + if isinstance(lora_request, Sequence): + if len(lora_request) != num_requests: + raise ValueError( + f"The lengths of prompts ({num_requests}) " + f"and lora_request ({len(lora_request)}) must be the same." + ) + + return lora_request + + return [lora_request] * num_requests + + def _priority_to_seq( + self, + priority: Sequence[int] | None, + num_requests: int, + ) -> Sequence[int]: + if priority is not None: + if len(priority) != num_requests: + raise ValueError( + f"The lengths of prompts ({num_requests}) " + f"and priority ({len(priority)}) must be the same." + ) + + return priority + + return [0] * num_requests diff --git a/vllm/entrypoints/pooling/embed/io_processor.py b/vllm/entrypoints/pooling/embed/io_processor.py index ec52e2efd6..5c2f825a96 100644 --- a/vllm/entrypoints/pooling/embed/io_processor.py +++ b/vllm/entrypoints/pooling/embed/io_processor.py @@ -28,11 +28,13 @@ from vllm.utils.mistral import is_mistral_tokenizer from ..base.io_processor import PoolingIOProcessor from ..scoring.io_processor import JinaRankingIOProcessorMixin from ..typing import ( + ALLOfflineInputsContext, ChunkedEmbeddingMetadata, - OfflineInputsContext, + OfflineEncodeInputsContext, PoolingChatLikeRequest, PoolingCompletionLikeRequest, PoolingServeContext, + RequestFactory, ) from .protocol import ( CohereEmbedContent, @@ -665,7 +667,10 @@ class JinaRankingTokenEmbedIOProcessor( ctx.engine_inputs = engine_inputs - def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]: + def get_request_factory_offline( + self, ctx: ALLOfflineInputsContext + ) -> tuple[RequestFactory, int]: + assert isinstance(ctx, OfflineEncodeInputsContext) if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2: raise ValueError("The JinaForRanking model requires at least 2 inputs.") @@ -677,4 +682,4 @@ class JinaRankingTokenEmbedIOProcessor( query=text_prompts[-1], docs=text_prompts[:-1] ) - return super().pre_process_offline(ctx) + return super().get_request_factory_offline(ctx) diff --git a/vllm/entrypoints/pooling/offline.py b/vllm/entrypoints/pooling/offline.py index a005bb92b4..bb3be812c6 100644 --- a/vllm/entrypoints/pooling/offline.py +++ b/vllm/entrypoints/pooling/offline.py @@ -20,10 +20,18 @@ from vllm.outputs import ( from vllm.pooling_params import PoolingParams from vllm.tasks import SCORE_TYPE_MAP, PoolingTask, SupportedTask +from .base.io_processor import PoolingIOProcessor from .factories import init_pooling_io_processors from .scoring.io_processor import ScoringIOProcessor from .scoring.typing import ScoreInput -from .typing import OfflineInputsContext, OfflineOutputsContext +from .typing import ( + ALLOfflineInputsContext, + OfflineEncodeInputsContext, + OfflineOutputsContext, + OfflinePluginInputsContext, + OfflineScoringInputsContext, + RequestFactory, +) logger = init_logger(__name__) @@ -48,6 +56,9 @@ class PoolingOfflineMixin(OfflineInferenceMixin): chat_template_config=self.chat_template_config, ) + # Use thread pool executor to accelerate preprocessing. + self._executor = self.renderer._executor + def encode( self, prompts: PromptType | Sequence[PromptType] | DataPrompt, @@ -93,43 +104,30 @@ class PoolingOfflineMixin(OfflineInferenceMixin): io_processor = self.pooling_io_processors[pooling_task] - if pooling_params is None: - pooling_params = PoolingParams() + ctx: ALLOfflineInputsContext + if isinstance(prompts, dict) and "data" in prompts: + ctx = OfflinePluginInputsContext( + pooling_task=pooling_task, + prompts=prompts, # type: ignore[arg-type] + tokenization_kwargs=tokenization_kwargs, + pooling_params=pooling_params, + lora_request=lora_request, + priorities=None, + ) + else: + ctx = OfflineEncodeInputsContext( + pooling_task=pooling_task, + prompts=prompts, + tokenization_kwargs=tokenization_kwargs, + pooling_params=pooling_params, + lora_request=lora_request, + priorities=None, + ) - ctx = OfflineInputsContext( - prompts=prompts, - pooling_params=pooling_params, - tokenization_kwargs=tokenization_kwargs, + request_factory, num_requests = io_processor.get_request_factory_offline(ctx) + outputs = self._run_tiling_engine( + io_processor, request_factory, num_requests, use_tqdm=use_tqdm ) - - engine_inputs = io_processor.pre_process_offline(ctx) - n_inputs = len(engine_inputs) - assert ctx.pooling_params is not None - - params_seq = self._params_to_seq(ctx.pooling_params, n_inputs) - - for param in params_seq: - if param.task is None: - param.task = pooling_task - elif pooling_task == "plugin": - # `plugin` task uses io_processor.parse_request to verify inputs. - # We actually allow plugin to overwrite pooling_task. - pass - elif param.task != pooling_task: - msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!" - raise ValueError(msg) - - seq_lora_requests = self._lora_request_to_seq(lora_request, n_inputs) - seq_priority = self._priority_to_seq(None, n_inputs) - - self._render_and_add_requests( - prompts=engine_inputs, - params=params_seq, - lora_requests=seq_lora_requests, - priorities=seq_priority, - ) - - outputs = self._run_engine(use_tqdm=use_tqdm, output_type=PoolingRequestOutput) outputs = io_processor.post_process_offline( ctx=OfflineOutputsContext(outputs=outputs) ) @@ -357,46 +355,121 @@ class PoolingOfflineMixin(OfflineInferenceMixin): io_processor = self.pooling_io_processors[score_type] assert isinstance(io_processor, ScoringIOProcessor) - pooling_task = io_processor.pooling_task scoring_data = io_processor.valid_inputs(data_1, data_2) n_queries = len(scoring_data.data_1) if pooling_params is None: pooling_params = PoolingParams() - ctx = OfflineInputsContext( - prompts=scoring_data, + assert isinstance(pooling_params, PoolingParams) + pooling_task = io_processor.pooling_task + if pooling_params.task is None: + pooling_params.task = pooling_task + elif pooling_params.task != pooling_task: + msg = ( + f"You cannot overwrite {pooling_params.task=!r} with {pooling_task=!r}!" + ) + raise ValueError(msg) + + ctx = OfflineScoringInputsContext( + pooling_task=pooling_task, + scoring_data=scoring_data, pooling_params=pooling_params, tokenization_kwargs=tokenization_kwargs, + lora_request=lora_request, chat_template=chat_template, - n_queries=n_queries, + priorities=None, ) - engine_inputs = io_processor.pre_process_offline(ctx) - n_inputs = len(engine_inputs) - - seq_lora_requests = self._lora_request_to_seq(lora_request, n_inputs) - params_seq = self._params_to_seq(ctx.pooling_params, n_inputs) - - for param in params_seq: - if param.task is None: - param.task = pooling_task - elif param.task != pooling_task: - msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!" - raise ValueError(msg) - - seq_priority = self._priority_to_seq(None, n_inputs) - - self._render_and_add_requests( - prompts=engine_inputs, - params=params_seq, - lora_requests=seq_lora_requests, - priorities=seq_priority, + request_factory, num_requests = io_processor.get_request_factory_offline(ctx) + outputs = self._run_tiling_engine( + io_processor, request_factory, num_requests, use_tqdm=use_tqdm ) - outputs = self._run_engine(use_tqdm=use_tqdm, output_type=PoolingRequestOutput) outputs = io_processor.post_process_offline( ctx=OfflineOutputsContext(outputs=outputs, n_queries=n_queries), ) return [ScoringRequestOutput.from_base(item) for item in outputs] + + def _run_tiling_engine( + self, + io_processor: PoolingIOProcessor, + request_factory: RequestFactory, + num_requests: int, + use_tqdm: bool | Callable[..., tqdm] = True, + ): + # Keeping max_num_seqs * 2 requests in the core can already saturate the core. + # Therefore, keep most requests waiting outside the core. + max_requests_in_core = ( + self.llm_engine.vllm_config.scheduler_config.max_num_seqs * 2 + ) + num_requests_in_core = 0 + num_waited_requests = num_requests + + if use_tqdm: + tqdm_func = use_tqdm if callable(use_tqdm) else tqdm + pbar = tqdm_func( + total=num_requests, + desc="Processed prompts", + dynamic_ncols=True, + postfix=f"est. speed input: {0:.2f} toks/s, output: {0:.2f} toks/s", + ) + + outputs: list[PoolingRequestOutput] = [] + added_request_ids: set[str] = set() + + it = self._executor.map(io_processor.render, request_factory()) + + try: + while num_waited_requests or self.llm_engine.has_unfinished_requests(): + requests = [] + for _ in range(max_requests_in_core - num_requests_in_core): + if num_waited_requests == 0: + break + try: + request = next(it) + requests.append(request) + except StopIteration: + num_waited_requests = 0 + break + + num_waited_requests -= 1 + num_requests_in_core += 1 + + if requests: + request_ids = self._render_and_add_requests( + prompts=[x["prompts"] for x in requests], + params=[x["params"] for x in requests], + lora_requests=[x["lora_requests"] for x in requests], + priorities=[x["priorities"] for x in requests], + ) + + for request_id in request_ids: + # undo assign_request_id + request_id = request_id.split("-", 1)[0] + added_request_ids.add(request_id) + + step_outputs = self.llm_engine.step() + for output in step_outputs: + assert isinstance(output, PoolingRequestOutput) + assert output.finished + outputs.append(output) + added_request_ids.discard(output.request_id) + num_requests_in_core -= 1 + + if use_tqdm: + pbar.update(1) + if pbar.n == num_requests: + pbar.refresh() + + except Exception: + if added_request_ids: + self.llm_engine.abort_request(list(added_request_ids)) + raise + + finally: + if use_tqdm: + pbar.close() + + return sorted(outputs, key=lambda x: int(x.request_id)) diff --git a/vllm/entrypoints/pooling/pooling/io_processor.py b/vllm/entrypoints/pooling/pooling/io_processor.py index b07ceeede3..b0c38b63a3 100644 --- a/vllm/entrypoints/pooling/pooling/io_processor.py +++ b/vllm/entrypoints/pooling/pooling/io_processor.py @@ -4,13 +4,19 @@ from collections.abc import Sequence from typing import Any from vllm import PoolingParams, PoolingRequestOutput -from vllm.inputs import EngineInput from vllm.logger import init_logger from vllm.plugins.io_processors import get_io_processor from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq from ..base.io_processor import PoolingIOProcessor -from ..typing import OfflineInputsContext, OfflineOutputsContext, PoolingServeContext +from ..typing import ( + ALLOfflineInputsContext, + OfflineEncodeInputsContext, + OfflineOutputsContext, + OfflinePluginInputsContext, + PoolingServeContext, + RequestFactory, +) from .protocol import IOProcessorRequest, IOProcessorResponse logger = init_logger(__name__) @@ -107,9 +113,11 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor): ####################################### # offline APIs - def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]: + def get_request_factory_offline( + self, ctx: ALLOfflineInputsContext + ) -> tuple[RequestFactory, int]: + assert isinstance(ctx, OfflinePluginInputsContext) assert isinstance(ctx.prompts, dict) and "data" in ctx.prompts - assert ctx.pooling_params is not None # Validate the request data is valid for the loaded plugin prompt_data = ctx.prompts.get("data") @@ -126,20 +134,35 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor): prompts = self.io_processor.pre_process(prompt=validated_prompt) prompts_seq = prompt_to_seq(prompts) + num_requests = len(prompts_seq) + + pooling_params: PoolingParams | Sequence[PoolingParams] + if ctx.pooling_params is None: + pooling_params = PoolingParams() + else: + pooling_params = ctx.pooling_params + params_seq: list[PoolingParams] = [ self.io_processor.merge_pooling_params(param) for param in self._params_to_seq( - ctx.pooling_params, - len(prompts_seq), + pooling_params, + num_requests, ) ] for p in params_seq: if p.task is None: p.task = "plugin" - ctx.pooling_params = params_seq - ctx.prompts = prompts_seq - return super().pre_process_offline(ctx) + return super().get_request_factory_offline( + OfflineEncodeInputsContext( + prompts=prompts_seq, + pooling_params=params_seq, + pooling_task="plugin", + tokenization_kwargs=ctx.tokenization_kwargs, + lora_request=ctx.lora_request, + priorities=ctx.priorities, + ) + ) def post_process_offline( self, diff --git a/vllm/entrypoints/pooling/scoring/io_processor.py b/vllm/entrypoints/pooling/scoring/io_processor.py index db872b7f30..e8b3efc32e 100644 --- a/vllm/entrypoints/pooling/scoring/io_processor.py +++ b/vllm/entrypoints/pooling/scoring/io_processor.py @@ -2,24 +2,36 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import time from collections.abc import Sequence -from typing import Any, TypeAlias +from typing import Any, TypeAlias, cast import torch.nn.functional as F -from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt +from vllm import PoolingParams, PoolingRequestOutput, PromptType, TokensPrompt from vllm.inputs import EngineInput from vllm.renderers import TokenizeParams from vllm.renderers.hf import safe_apply_chat_template -from vllm.renderers.inputs.preprocess import extract_target_prompt +from vllm.renderers.inputs.preprocess import ( + extract_target_prompt, + parse_model_prompt, + prompt_to_seq, +) from vllm.tasks import PoolingTask from vllm.utils.mistral import is_mistral_tokenizer from ...chat_utils import ChatTemplateResolutionError from ..base.io_processor import PoolingIOProcessor from ..typing import ( - OfflineInputsContext, + ALLOfflineInputsContext, + EncodeChatRenderParams, + EncodeCMPLRenderParams, + OfflineEncodeInputsContext, OfflineOutputsContext, + OfflineScoringInputsContext, + PoolingEngineInput, PoolingServeContext, + RequestFactory, + RequestGenerator, + ScoringRenderParams, ) from .protocol import RerankRequest, ScoreRequest, ScoringRequest from .typing import ScoreData, ScoreInput, ScoringData @@ -213,25 +225,37 @@ class BiEncoderIOProcessor(ScoringIOProcessor): ####################################### # offline APIs - def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]: - assert isinstance(ctx.prompts, ScoringData) - assert not isinstance(ctx.pooling_params, Sequence) - - tok_params = self.renderer.default_cmpl_tok_params.with_kwargs( - **(ctx.tokenization_kwargs or {}) - ) + def get_request_factory_offline( + self, ctx: ALLOfflineInputsContext + ) -> tuple[RequestFactory, int]: + assert isinstance(ctx, OfflineScoringInputsContext) max_tokens_per_query, max_tokens_per_doc = self._get_token_limits( pooling_params=ctx.pooling_params ) - scoring_data = ctx.prompts + scoring_data = ctx.scoring_data if max_tokens_per_query > 0 or max_tokens_per_doc > 0: scoring_data = self._truncate_scoring_data( scoring_data, max_tokens_per_query, max_tokens_per_doc ) - return self._pre_process(scoring_data, tok_params) + data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config) + data_2 = score_data_to_prompts( + scoring_data.data_2, "document", self.model_config + ) + prompts = data_1 + data_2 + + return super().get_request_factory_offline( + OfflineEncodeInputsContext( + pooling_task=self.pooling_task, + prompts=prompts, + tokenization_kwargs=ctx.tokenization_kwargs, + pooling_params=ctx.pooling_params, + lora_request=ctx.lora_request, + priorities=ctx.priorities, + ) + ) def post_process_offline( self, @@ -258,6 +282,26 @@ class BiEncoderIOProcessor(ScoringIOProcessor): prompts=data_1 + data_2, tok_params=tok_params, prompt_extras=prompt_extras ) + def _preprocess_cmpl_offline( + self, + prompts: PromptType | Sequence[PromptType], + tok_params: TokenizeParams, + prompt_extras: dict[str, Any] | None = None, + ) -> Sequence[EngineInput]: + prompts = prompt_to_seq(prompts) + parsed_prompts = [ + ( + prompt + if isinstance(prompt, bytes) + else parse_model_prompt(self.model_config, prompt) + ) + for prompt in prompts + ] + + return self.renderer.render_cmpl( + parsed_prompts, tok_params, prompt_extras=prompt_extras + ) + def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int): emb_data_1 = outputs[:n_queries] emb_data_2 = outputs[n_queries:] @@ -429,34 +473,96 @@ class CrossEncoderIOProcessor(ScoringIOProcessor): ####################################### # offline APIs - def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]: - assert isinstance(ctx.prompts, ScoringData) - assert not isinstance(ctx.pooling_params, Sequence) + def get_request_factory_offline( + self, ctx: ALLOfflineInputsContext + ) -> tuple[RequestFactory, int]: + assert isinstance(ctx, OfflineScoringInputsContext) + + data_1 = ctx.scoring_data.data_1 + data_2 = ctx.scoring_data.data_2 + num_requests = len(data_2) + + if len(data_1) == 1: + data_1 = data_1 * num_requests tok_params = self.renderer.default_cmpl_tok_params.with_kwargs( **(ctx.tokenization_kwargs or {}) ) - max_tokens_per_query, max_tokens_per_doc = self._get_token_limits( - pooling_params=ctx.pooling_params - ) + prompt_extras = ctx.pooling_params.extra_kwargs - prompt_extras = ctx.pooling_params.extra_kwargs if ctx.pooling_params else None - engine_inputs, pooling_params_list = self._pre_process( - ctx.prompts, - tok_params, - ctx.pooling_params, - ctx.chat_template, - max_tokens_per_query=max_tokens_per_query, - max_tokens_per_doc=max_tokens_per_doc, - prompt_extras=prompt_extras, - ) - ctx.pooling_params = pooling_params_list - return engine_inputs + seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests) + seq_priority = self._priority_to_seq(ctx.priorities, num_requests) + + def request_factory() -> RequestGenerator: + for i in range(num_requests): + yield ScoringRenderParams( + data_1=data_1[i], + data_2=data_2[i], + chat_template=ctx.chat_template, + tok_params=tok_params, + prompt_extras=prompt_extras, + skip_mm_cache=False, + params=ctx.pooling_params, + lora_requests=seq_lora_requests[i], + priorities=seq_priority[i], + ) + + return request_factory, num_requests ####################################### # helpers + def render( + self, + render_params: EncodeCMPLRenderParams + | EncodeChatRenderParams + | ScoringRenderParams, + ) -> PoolingEngineInput: + if "data_1" not in render_params: + raise ValueError( + f"Unsupported render_params type {render_params.__class__.__name__}" + ) + render_params = cast(ScoringRenderParams, render_params) + + arrival_time = time.time() + + tok_params = render_params["tok_params"] + params = render_params["params"] + prompt_extras = render_params["prompt_extras"] + + max_tokens_per_query, max_tokens_per_doc = self._get_token_limits( + pooling_params=params + ) + + _, engine_prompt = self.get_score_prompt( + data_1=render_params["data_1"], + data_2=render_params["data_2"], + encode_kwargs=tok_params.get_encode_kwargs(), + chat_template=render_params["chat_template"], + max_tokens_per_query=max_tokens_per_query, + max_tokens_per_doc=max_tokens_per_doc, + chat_template_kwargs=prompt_extras.get("chat_template_kwargs") + if prompt_extras + else None, + ) + + tok_params.apply_post_tokenization(self.tokenizer, engine_prompt) + + if token_type_ids := engine_prompt.pop("token_type_ids", None): + params = params.clone() + compressed = compress_token_type_ids(token_type_ids) + params.extra_kwargs = {"compressed_token_type_ids": compressed} + + engine_input = self.renderer.process_for_engine(engine_prompt, arrival_time) + + return PoolingEngineInput( + prompts=engine_input, + params=params, + lora_requests=render_params["lora_requests"], + priorities=render_params["priorities"], + ) + def _pre_process( self, scoring_data: ScoringData, @@ -731,6 +837,49 @@ class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorM name = "jina-reranking-scoring" pooling_task: PoolingTask = "token_embed" + def get_request_factory_offline( + self, ctx: ALLOfflineInputsContext + ) -> tuple[RequestFactory, int]: + assert isinstance(ctx, OfflineScoringInputsContext) + + scoring_data = ctx.scoring_data + prompt_extras = ctx.pooling_params.extra_kwargs + + queries = self.ensure_str(scoring_data.data_1) + docs = self.ensure_str(scoring_data.data_2) + chat_template_kwargs = ( + prompt_extras.get("chat_template_kwargs") if prompt_extras else None + ) + instruction = ( + chat_template_kwargs.get("instruction") if chat_template_kwargs else None + ) + + if len(queries) == 1: + prompts = [ + self.format_docs_prompts_func( + query=queries[0], docs=docs, instruction=instruction + ) + ] + else: + prompts = [ + self.format_docs_prompts_func( + query=q, docs=[d], instruction=instruction + ) + for q, d in zip(queries, docs) + ] + + return PoolingIOProcessor.get_request_factory_offline( + self, + OfflineEncodeInputsContext( + pooling_task=self.pooling_task, + prompts=prompts, + tokenization_kwargs=ctx.tokenization_kwargs, + pooling_params=ctx.pooling_params, + lora_request=ctx.lora_request, + priorities=ctx.priorities, + ), + ) + def _pre_process( self, scoring_data: ScoringData, diff --git a/vllm/entrypoints/pooling/typing.py b/vllm/entrypoints/pooling/typing.py index 54d02c5b61..b44c9476e7 100644 --- a/vllm/entrypoints/pooling/typing.py +++ b/vllm/entrypoints/pooling/typing.py @@ -1,17 +1,21 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import time -from collections.abc import AsyncGenerator, Sequence +from collections.abc import AsyncGenerator, Callable, Generator, Sequence from dataclasses import dataclass, field -from typing import Any, Generic, TypeAlias, TypeVar +from typing import Any, Generic, TypeAlias, TypedDict, TypeVar from fastapi import Request from pydantic import ConfigDict from vllm import PoolingParams, PoolingRequestOutput, PromptType +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam from vllm.inputs import DataPrompt, EngineInput from vllm.lora.request import LoRARequest +from vllm.renderers import ChatParams, TokenizeParams +from vllm.renderers.inputs import DictPrompt +from ...tasks import PoolingTask from .classify.protocol import ( ClassificationChatRequest, ClassificationCompletionRequest, @@ -34,7 +38,7 @@ from .pooling.protocol import ( PoolingResponse, ) from .scoring.protocol import ScoringRequest, ScoringResponse -from .scoring.typing import ScoringData +from .scoring.typing import ScoreData, ScoringData PoolingCompletionLikeRequest: TypeAlias = ( EmbeddingCompletionRequest @@ -111,13 +115,36 @@ class PoolingServeContext(Generic[PoolingRequestT]): @dataclass class OfflineInputsContext: - prompts: PromptType | Sequence[PromptType] | DataPrompt | ScoringData - pooling_params: PoolingParams | Sequence[PoolingParams] - tokenization_kwargs: dict[str, Any] | None = None - chat_template: str | None = None + pooling_task: PoolingTask + tokenization_kwargs: dict[str, Any] | None + lora_request: Sequence[LoRARequest | None] | None + priorities: Sequence[int] | None - ## for bi-encoder & late-interaction - n_queries: int | None = None + +@dataclass +class OfflineEncodeInputsContext(OfflineInputsContext): + prompts: PromptType | Sequence[PromptType] + pooling_params: PoolingParams | Sequence[PoolingParams] | None + + +@dataclass +class OfflineScoringInputsContext(OfflineInputsContext): + scoring_data: ScoringData + chat_template: str | None + pooling_params: PoolingParams + + +@dataclass +class OfflinePluginInputsContext(OfflineInputsContext): + prompts: DataPrompt + pooling_params: PoolingParams | Sequence[PoolingParams] | None + + +ALLOfflineInputsContext: TypeAlias = ( + OfflineEncodeInputsContext + | OfflineScoringInputsContext + | OfflinePluginInputsContext +) @dataclass @@ -126,3 +153,41 @@ class OfflineOutputsContext: ## for bi-encoder & late-interaction n_queries: int | None = None + + +class RenderParams(TypedDict): + tok_params: TokenizeParams + prompt_extras: dict[str, Any] | None + skip_mm_cache: bool + + params: PoolingParams + lora_requests: LoRARequest | None + priorities: int + + +class EncodeCMPLRenderParams(RenderParams): + prompts: DictPrompt + + +class EncodeChatRenderParams(RenderParams): + conversations: list["ChatCompletionMessageParam"] + chat_params: ChatParams + + +class ScoringRenderParams(RenderParams): + data_1: ScoreData + data_2: ScoreData + chat_template: str | None + + +class PoolingEngineInput(TypedDict): + prompts: EngineInput + params: PoolingParams + lora_requests: LoRARequest | None + priorities: int + + +RequestGenerator: TypeAlias = Generator[ + EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams +] +RequestFactory: TypeAlias = Callable[[], RequestGenerator]