From 61c9ef986a807aa3b9c6ccd25bb223b8f4116ac7 Mon Sep 17 00:00:00 2001 From: "wang.yuqi" Date: Wed, 22 Jul 2026 18:56:23 +0800 Subject: [PATCH] [Frontend] Parallelize preprocessing within the same request for pooling models online serving. (#49153) Signed-off-by: wang.yuqi --- .../pooling/embed/test_io_processor.py | 254 ++----------- .../scoring/test_cross_encoder_offline.py | 43 --- .../serve/lora/test_serving_models.py | 14 +- vllm/entrypoints/pooling/base/io_processor.py | 196 +++++----- vllm/entrypoints/pooling/base/serving.py | 60 +-- .../pooling/classify/api_router.py | 10 +- vllm/entrypoints/pooling/embed/api_router.py | 13 +- .../entrypoints/pooling/embed/io_processor.py | 256 ++++++------- vllm/entrypoints/pooling/embed/protocol.py | 12 +- vllm/entrypoints/pooling/embed/serving.py | 7 + vllm/entrypoints/pooling/offline.py | 7 +- .../entrypoints/pooling/pooling/api_router.py | 10 +- .../pooling/pooling/io_processor.py | 39 +- .../entrypoints/pooling/scoring/api_router.py | 20 +- .../pooling/scoring/io_processor.py | 355 ++++++++---------- vllm/entrypoints/pooling/scoring/serving.py | 32 +- vllm/entrypoints/pooling/typing.py | 30 +- 17 files changed, 538 insertions(+), 820 deletions(-) diff --git a/tests/entrypoints/pooling/embed/test_io_processor.py b/tests/entrypoints/pooling/embed/test_io_processor.py index 5a7a8aab2a..f06e43d143 100644 --- a/tests/entrypoints/pooling/embed/test_io_processor.py +++ b/tests/entrypoints/pooling/embed/test_io_processor.py @@ -9,8 +9,6 @@ from pydantic import TypeAdapter, ValidationError from vllm import PoolingParams from vllm.entrypoints.pooling.embed.io_processor import EmbedIOProcessor from vllm.entrypoints.pooling.embed.protocol import ( - CohereEmbedContent, - CohereEmbedInput, CohereEmbedRequest, EmbeddingBatchChatInputRequest, EmbeddingBatchChatRequest, @@ -19,7 +17,10 @@ from vllm.entrypoints.pooling.embed.protocol import ( EmbeddingCompletionRequest, EmbeddingRequest, ) -from vllm.entrypoints.pooling.typing import PoolingServeContext +from vllm.entrypoints.pooling.typing import ( + PoolingEngineInput, + PoolingServeContext, +) from vllm.outputs import PoolingOutput, PoolingRequestOutput @@ -410,6 +411,7 @@ class TestChunkedEmbeddingProcessing: def _make_handler(cls): handler = object.__new__(EmbedIOProcessor) handler.model_config = cls._FakeModelConfig() + handler.enable_chunked_processing = True return handler @staticmethod @@ -421,15 +423,29 @@ class TestChunkedEmbeddingProcessing: } ) assert isinstance(request, EmbeddingCompletionRequest) + pooling_params = PoolingParams() return PoolingServeContext( request=request, - pooling_params=PoolingParams(), + pooling_params=pooling_params, model_name="test", request_id="embd-client-prompt-999-chunk-888", engine_inputs=[ - {"prompt_token_ids": [0, 1, 2, 3, 4]}, - {"prompt_token_ids": [10, 11]}, + PoolingEngineInput( + prompts={"prompt_token_ids": [0, 1, 2, 3, 4]}, + params=pooling_params, + lora_requests=None, + priorities=0, + ), + PoolingEngineInput( + prompts={"prompt_token_ids": [10, 11]}, + params=pooling_params, + lora_requests=None, + priorities=0, + ), ], + lora_request=None, + priorities=0, + prompt_extras=None, ) @staticmethod @@ -450,7 +466,7 @@ class TestChunkedEmbeddingProcessing: handler = self._make_handler() ctx = self._make_context() - handler._pre_process_chunked(ctx) + handler.maybe_pre_process_chunked(ctx) assert ctx.prompt_request_ids == [ "embd-client-prompt-999-chunk-888-prompt-0-chunk-0", @@ -488,227 +504,3 @@ class TestChunkedEmbeddingProcessing: ctx.final_res_batch[1].outputs.data, torch.tensor([9.0, 9.0]), ) - - -class TestPreProcessCohereOnline: - """Unit tests for EmbedIOProcessor._pre_process_cohere_online.""" - - @staticmethod - def _make_context(**request_kwargs) -> PoolingServeContext[CohereEmbedRequest]: - return PoolingServeContext( - request=CohereEmbedRequest(model="test", **request_kwargs), - pooling_params=PoolingParams(), - model_name="test", - request_id="embd-test", - ) - - @staticmethod - def _make_handler(): - handler = object.__new__(EmbedIOProcessor) - handler._validate_input_type = lambda _input_type: None - return handler - - def test_text_only_without_task_prefix_uses_completion_path(self): - handler = self._make_handler() - ctx = self._make_context(texts=["hello"]) - calls: list[tuple[str, object]] = [] - - def preprocess_cmpl_online(request, prompt_input, prompt_embeds): - calls.append(("completion", prompt_input)) - return ["completion"] - - handler._get_task_instruction_prefix = lambda _input_type: None - handler._has_chat_template = lambda: False - handler._preprocess_cmpl_online = preprocess_cmpl_online - handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail( - "text-only request should not require chat rendering" - ) - - handler._pre_process_cohere_online(ctx) - - assert ctx.engine_inputs == ["completion"] - assert calls == [("completion", ["hello"])] - - def test_text_only_falls_back_to_prefixed_completion_without_template(self): - handler = self._make_handler() - ctx = self._make_context(texts=["hello"], input_type="query") - calls: list[tuple[str, object]] = [] - - def preprocess_cmpl(request, prompt_input, prompt_embeds): - calls.append(("completion", prompt_input)) - return ["fallback"] - - handler._get_task_instruction_prefix = lambda _input_type: "query: " - handler._has_chat_template = lambda: False - handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail( - "chat rendering should be skipped without a template" - ) - handler._preprocess_cmpl_online = preprocess_cmpl - - handler._pre_process_cohere_online(ctx) - - assert ctx.engine_inputs == ["fallback"] - assert calls == [("completion", ["query: hello"])] - - def test_text_only_with_template_uses_chat_path(self): - handler = self._make_handler() - ctx = self._make_context(texts=["hello"], input_type="query") - calls: list[tuple[str, object]] = [] - - def batch_render_chat( - request, - all_messages, - truncate_prompt_tokens, - truncation_side, - ): - calls.append( - ( - "chat", - { - "request": request, - "all_messages": all_messages, - "truncate_prompt_tokens": truncate_prompt_tokens, - "truncation_side": truncation_side, - }, - ) - ) - return ["chat"] - - handler._get_task_instruction_prefix = lambda _input_type: "query: " - handler._has_chat_template = lambda: True - handler._batch_render_chat = batch_render_chat - handler._preprocess_cmpl_online = lambda *_args, **_kwargs: pytest.fail( - "completion path should be skipped when a template exists" - ) - - handler._pre_process_cohere_online(ctx) - - assert ctx.engine_inputs == ["chat"] - assert calls == [ - ( - "chat", - { - "request": ctx.request, - "all_messages": [ - handler._mixed_input_to_messages( - CohereEmbedInput( - content=[CohereEmbedContent(type="text", text="hello")] - ), - task_prefix="query: ", - ) - ], - "truncate_prompt_tokens": -1, - "truncation_side": None, - }, - ) - ] - - -class TestPreProcessOpenAIEmbeddingChatOnline: - """Unit tests for OpenAI embedding chat preprocessing.""" - - class _FakeModelConfig: - max_model_len = 128 - encoder_config: dict[str, object] = {} - pooler_config = None - multimodal_config = None - is_encoder_decoder = False - - class _FakeRenderer: - tokenizer = object() - - def __init__(self): - self.calls = [] - - def render_chat( - self, - all_messages, - chat_params, - tok_params, - prompt_extras=None, - ): - self.calls.append( - { - "all_messages": all_messages, - "chat_params": chat_params, - "tok_params": tok_params, - "prompt_extras": prompt_extras, - } - ) - return all_messages, [ - {"prompt_token_ids": [index]} for index, _ in enumerate(all_messages) - ] - - @classmethod - def _make_handler(cls, renderer): - handler = object.__new__(EmbedIOProcessor) - handler.renderer = renderer - handler.model_config = cls._FakeModelConfig() - handler.chat_template = "template" - handler.chat_template_content_format = "auto" - handler.trust_request_chat_template = False - handler.enable_chunked_processing = False - return handler - - @staticmethod - def _make_context( - request: ( - EmbeddingChatRequest - | EmbeddingBatchChatRequest - | EmbeddingChatInputRequest - | EmbeddingBatchChatInputRequest - ), - ) -> PoolingServeContext[ - EmbeddingChatRequest - | EmbeddingBatchChatRequest - | EmbeddingChatInputRequest - | EmbeddingBatchChatInputRequest - ]: - return PoolingServeContext( - request=request, - pooling_params=PoolingParams(), - model_name="test", - request_id="embd-test", - ) - - def test_chat_template_kwargs_forwarded_for_batched_input_messages(self): - request = TypeAdapter(EmbeddingRequest).validate_python( - { - "model": "test", - "input": [ - [{"role": "user", "content": "hello"}], - [{"role": "user", "content": "goodbye"}], - ], - "add_generation_prompt": True, - "chat_template_kwargs": {"instruction": "Represent the query: "}, - "mm_processor_kwargs": {"max_pixels": 1}, - "cache_salt": "salt", - } - ) - assert isinstance(request, EmbeddingBatchChatInputRequest) - - renderer = self._FakeRenderer() - handler = self._make_handler(renderer) - ctx = self._make_context(request) - - handler.pre_process_online(ctx) - - assert ctx.engine_inputs == [ - {"prompt_token_ids": [0]}, - {"prompt_token_ids": [1]}, - ] - assert len(renderer.calls) == 1 - - call = renderer.calls[0] - assert call["all_messages"] == request.messages - assert call["prompt_extras"] == { - "mm_processor_kwargs": {"max_pixels": 1}, - "cache_salt": "salt", - } - - chat_template_kwargs = call["chat_params"].chat_template_kwargs - assert chat_template_kwargs["instruction"] == "Represent the query: " - assert chat_template_kwargs["add_generation_prompt"] is True - assert chat_template_kwargs["continue_final_message"] is False - assert "tools" not in chat_template_kwargs - assert chat_template_kwargs["tokenize"] is False diff --git a/tests/entrypoints/pooling/scoring/test_cross_encoder_offline.py b/tests/entrypoints/pooling/scoring/test_cross_encoder_offline.py index df79a387af..56e83de3f7 100644 --- a/tests/entrypoints/pooling/scoring/test_cross_encoder_offline.py +++ b/tests/entrypoints/pooling/scoring/test_cross_encoder_offline.py @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import weakref -from types import SimpleNamespace import pytest import torch @@ -10,10 +9,7 @@ import torch from tests.models.utils import softmax from vllm import LLM, PoolingParams from vllm.distributed import cleanup_dist_env_and_memory -from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor -from vllm.entrypoints.pooling.scoring.typing import ScoringData from vllm.platforms import current_platform -from vllm.renderers import TokenizeParams MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls" PROMPT = "The chef prepared a delicious meal." @@ -145,45 +141,6 @@ def test_max_tokens_per_doc(llm: LLM): assert with_limit_tokens < no_limit_tokens -def test_token_type_ids_follow_post_tokenization(): - processor = object.__new__(CrossEncoderIOProcessor) - processor.tokenizer = SimpleNamespace(truncation_side="right", pad_token_id=-1) - processor.renderer = SimpleNamespace(process_for_engine=lambda prompt, _: prompt) - processor.model_config = None - processor.get_score_prompt = lambda **_: ( - "", - { - "prompt_token_ids": list(range(32)), - "token_type_ids": [0] * 16 + [1] * 16, - }, - ) - - engine_inputs, pooling_params = processor._pre_process( - ScoringData(data_1=["query"], data_2=["document"]), - TokenizeParams( - max_total_tokens=None, - truncate_prompt_tokens=16, - truncation_side="left", - ), - PoolingParams(task="classify", extra_kwargs={"cache_salt": "salt"}), - ) - - assert engine_inputs[0]["prompt_token_ids"] == list(range(16, 32)) - assert pooling_params[0].extra_kwargs == { - "cache_salt": "salt", - "compressed_token_type_ids": 0, - } - - engine_inputs, pooling_params = processor._pre_process( - ScoringData(data_1=["query"], data_2=["document"]), - TokenizeParams(max_total_tokens=None, pad_prompt_tokens=40), - PoolingParams(task="classify"), - ) - - assert engine_inputs[0]["prompt_token_ids"] == list(range(32)) + [-1] * 8 - assert pooling_params[0].extra_kwargs == {"compressed_token_type_ids": 16} - - def test_pooling_params(llm: LLM): def get_outputs(use_activation): outputs = llm.score( diff --git a/tests/entrypoints/serve/lora/test_serving_models.py b/tests/entrypoints/serve/lora/test_serving_models.py index 658d004580..e75cbe908b 100644 --- a/tests/entrypoints/serve/lora/test_serving_models.py +++ b/tests/entrypoints/serve/lora/test_serving_models.py @@ -161,7 +161,9 @@ def _make_pooling_serving(lora_name: str) -> _ConcretePoolingServing: return serving -def _make_pooling_ctx(model_name: str) -> PoolingServeContext: +def _make_pooling_ctx( + model_name: str, serving: PoolingBaseServing +) -> PoolingServeContext: mock_request = MagicMock() mock_request.model = model_name return PoolingServeContext( @@ -169,6 +171,9 @@ def _make_pooling_ctx(model_name: str) -> PoolingServeContext: model_name=MODEL_NAME, request_id="test-id", pooling_params=PoolingParams(), + lora_request=serving._maybe_get_adapters(mock_request), + priorities=0, + prompt_extras=None, ) @@ -176,9 +181,7 @@ def test_pooling_maybe_get_adapters_lora_name_sets_lora_request(): """LoRA adapter name must populate ctx.lora_request without raising.""" lora_name = "bot-embed-lora" serving = _make_pooling_serving(lora_name) - ctx = _make_pooling_ctx(lora_name) - - ctx.lora_request = serving._maybe_get_adapters(ctx.request) + ctx = _make_pooling_ctx(lora_name, serving) assert ctx.lora_request is not None assert ctx.lora_request.lora_name == lora_name @@ -187,7 +190,6 @@ def test_pooling_maybe_get_adapters_lora_name_sets_lora_request(): def test_pooling_maybe_get_adapters_unknown_model_raises(): """An unrecognised model name must still raise VLLMNotFoundError.""" serving = _make_pooling_serving("some-lora") - ctx = _make_pooling_ctx("unknown-model") with pytest.raises(VLLMNotFoundError): - serving._maybe_get_adapters(ctx.request) + _make_pooling_ctx("unknown-model", serving) diff --git a/vllm/entrypoints/pooling/base/io_processor.py b/vllm/entrypoints/pooling/base/io_processor.py index cdc4c16cac..f74b740276 100644 --- a/vllm/entrypoints/pooling/base/io_processor.py +++ b/vllm/entrypoints/pooling/base/io_processor.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Sequence +from concurrent.futures import Executor from typing import Any, Final, cast from vllm import ( @@ -10,21 +11,17 @@ from vllm import ( ) from vllm.config import VllmConfig from vllm.entrypoints.chat_utils import ( - ChatCompletionMessageParam, ChatTemplateConfig, - ChatTemplateContentFormatOption, - ConversationMessage, ) -from vllm.entrypoints.serve.engine.typing import RendererChatRequest, RendererRequest -from vllm.inputs import EngineInput, SingletonPrompt from vllm.lora.request import LoRARequest from vllm.renderers import BaseRenderer, merge_kwargs from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq -from vllm.tool_parsers import ToolParser +from vllm.utils.async_utils import make_async from vllm.utils.mistral import is_mistral_tokenizer from ..typing import ( - ALLOfflineInputsContext, + AnyOfflineInputsContext, + AnyRenderParam, EncodeChatRenderParams, EncodeCMPLRenderParams, OfflineEncodeInputsContext, @@ -66,14 +63,25 @@ class PoolingIOProcessor: chat_template_config.trust_request_chat_template ) + self.template_kwargs = None + self.tool_dicts = None + + # Shared thread pool executor for preprocessing + self._executor: Executor = self.renderer._executor + self.render_async = make_async(self.render, executor=self._executor) + ####################################### # online APIs def create_pooling_params(self, request): return request.to_pooling_params() - def pre_process_online(self, ctx: PoolingServeContext): + def get_request_factory_online( + self, ctx: PoolingServeContext + ) -> Sequence[AnyRenderParam]: request = ctx.request + renderer = self.renderer + requests: Sequence[AnyRenderParam] if isinstance(request, PoolingChatLikeRequest): self._validate_chat_template( @@ -81,24 +89,87 @@ class PoolingIOProcessor: chat_template_kwargs=request.chat_template_kwargs, trust_request_chat_template=self.trust_request_chat_template, ) - _, engine_inputs = self._preprocess_chat_online( - request, - request.messages, - default_template=self.chat_template, - default_template_content_format=self.chat_template_content_format, - default_template_kwargs=None, + + num_requests = 1 + default_template_kwargs = merge_kwargs( + self.template_kwargs, + dict( + tools=self.tool_dicts, + tokenize=is_mistral_tokenizer(renderer.tokenizer), + ), ) + + mm_config = self.model_config.multimodal_config + tok_params = request.build_tok_params(self.model_config) + chat_params = request.build_chat_params( + self.chat_template, self.chat_template_content_format + ).with_defaults( + default_template_kwargs, + default_media_io_kwargs=( + mm_config.media_io_kwargs if mm_config else None + ), + ) + + params_seq = self._params_to_seq(ctx.pooling_params, num_requests) + seq_lora_requests = self._lora_request_to_seq( + ctx.lora_request, num_requests + ) + seq_priority = self._priority_to_seq(ctx.priorities, num_requests) + + requests = [ + EncodeChatRenderParams( + conversations=request.messages, + chat_params=chat_params, + tok_params=tok_params, + prompt_extras=ctx.prompt_extras, + skip_mm_cache=False, + params=params_seq[i], + lora_requests=seq_lora_requests[i], + priorities=seq_priority[i], + ) + for i in range(num_requests) + ] + + return requests + elif isinstance(request, PoolingCompletionLikeRequest): - engine_inputs = self._preprocess_cmpl_online( - request, - prompt_input=request.input, - prompt_embeds=None, + model_config = self.model_config + prompts_seq = prompt_to_seq(request.input) + num_requests = len(prompts_seq) + + parsed_prompts = [ + ( + prompt + if isinstance(prompt, bytes) + else parse_model_prompt(model_config, prompt) + ) + for prompt in prompts_seq + ] + tok_params = request.build_tok_params(model_config) + + params_seq = self._params_to_seq(ctx.pooling_params, num_requests) + seq_lora_requests = self._lora_request_to_seq( + ctx.lora_request, num_requests ) + seq_priority = self._priority_to_seq(ctx.priorities, num_requests) + + requests = [ + EncodeCMPLRenderParams( + prompts=parsed_prompts[i], + tok_params=tok_params, + prompt_extras=ctx.prompt_extras, + skip_mm_cache=False, + params=params_seq[i], + lora_requests=seq_lora_requests[i], + priorities=seq_priority[i], + ) + for i in range(num_requests) + ] + + return requests else: raise ValueError(f"Invalid {self.name} request type") - ctx.engine_inputs = engine_inputs - def post_process_online( self, ctx: PoolingServeContext, @@ -109,7 +180,7 @@ class PoolingIOProcessor: # offline APIs def get_request_factory_offline( - self, ctx: ALLOfflineInputsContext + self, ctx: AnyOfflineInputsContext ) -> tuple[RequestFactory, int]: assert isinstance(ctx, OfflineEncodeInputsContext) @@ -209,84 +280,6 @@ class PoolingIOProcessor: priorities=render_params["priorities"], ) - def _preprocess_cmpl_online( - self, - request: RendererRequest, - prompt_input: str | list[str] | list[int] | list[list[int]] | None, - prompt_embeds: bytes | list[bytes] | None, - ) -> list[EngineInput]: - renderer = self.renderer - model_config = self.model_config - - prompts = list[SingletonPrompt | bytes]() - if prompt_embeds is not None: # embeds take higher priority - prompts.extend(prompt_to_seq(prompt_embeds)) - if prompt_input is not None: - prompts.extend(prompt_to_seq(prompt_input)) - - parsed_prompts = [ - ( - prompt - if isinstance(prompt, bytes) - else parse_model_prompt(model_config, prompt) - ) - for prompt in prompts - ] - tok_params = request.build_tok_params(model_config) - - return renderer.render_cmpl( - parsed_prompts, - tok_params, - prompt_extras={ - k: v - for k in ("mm_processor_kwargs", "cache_salt") - if (v := getattr(request, k, None)) is not None - }, - ) - - def _preprocess_chat_online( - self, - request: RendererChatRequest, - messages: list[ChatCompletionMessageParam], - default_template: str | None, - default_template_content_format: ChatTemplateContentFormatOption, - default_template_kwargs: dict[str, Any] | None, - tool_dicts: list[dict[str, Any]] | None = None, - tool_parser: type[ToolParser] | None = None, - ) -> tuple[list[ConversationMessage], list[EngineInput]]: - renderer = self.renderer - - default_template_kwargs = merge_kwargs( - default_template_kwargs, - dict( - tools=tool_dicts, - tokenize=is_mistral_tokenizer(renderer.tokenizer), - ), - ) - - mm_config = self.model_config.multimodal_config - - tok_params = request.build_tok_params(self.model_config) - chat_params = request.build_chat_params( - default_template, default_template_content_format - ).with_defaults( - default_template_kwargs, - default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None), - ) - - (conversation,), (engine_input,) = renderer.render_chat( - [messages], - chat_params, - tok_params, - prompt_extras={ - k: v - for k in ("mm_processor_kwargs", "cache_salt") - if (v := getattr(request, k, None)) is not None - }, - ) - - return conversation, [engine_input] - def _validate_chat_template( self, request_chat_template: str | None, @@ -341,10 +334,13 @@ class PoolingIOProcessor: def _priority_to_seq( self, - priority: Sequence[int] | None, + priority: int | Sequence[int] | None, num_requests: int, ) -> Sequence[int]: if priority is not None: + if isinstance(priority, int): + return [priority] * num_requests + if len(priority) != num_requests: raise ValueError( f"The lengths of prompts ({num_requests}) " diff --git a/vllm/entrypoints/pooling/base/serving.py b/vllm/entrypoints/pooling/base/serving.py index 79f93a1480..36b1c22e61 100644 --- a/vllm/entrypoints/pooling/base/serving.py +++ b/vllm/entrypoints/pooling/base/serving.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project - +import asyncio from abc import ABC, abstractmethod from collections.abc import AsyncGenerator, Mapping from concurrent.futures import Executor @@ -63,9 +63,6 @@ class PoolingBaseServing(ABC, BaseServing): # Shared thread pool executor for preprocessing and postprocessing. self._executor: Executor = self.renderer._executor - self._preprocessing_async = make_async( - self._preprocessing, executor=self._executor - ) self._postprocessing_async = make_async( self._postprocessing, executor=self._executor ) @@ -77,7 +74,7 @@ class PoolingBaseServing(ABC, BaseServing): ) -> Response: io_processor = self.get_io_processor(request) ctx = await self._init_ctx(io_processor, request, raw_request) - await self._preprocessing_async(io_processor, ctx) + await self._preprocessing(io_processor, ctx) await self._prepare_generators(ctx) await self._collect_batch(ctx) return await self._postprocessing_async(io_processor, ctx) @@ -86,11 +83,17 @@ class PoolingBaseServing(ABC, BaseServing): def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor: raise NotImplementedError - @torch.inference_mode() - def _preprocessing( + async def _preprocessing( self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext ): - return io_processor.pre_process_online(ctx) + requests = io_processor.get_request_factory_online(ctx) + + if len(requests) == 0: + raise ValueError("You must pass at least one prompt") + + ctx.engine_inputs = await asyncio.gather( + *[io_processor.render_async(request) for request in requests] + ) @torch.inference_mode() def _postprocessing( @@ -110,16 +113,26 @@ class PoolingBaseServing(ABC, BaseServing): await self._check_model(request) pooling_params = io_processor.create_pooling_params(request) + lora_request = self._maybe_get_adapters(request) + priorities = getattr(request, "priority", 0) + prompt_extras = { + k: v + for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs") + if (v := getattr(request, k, None)) is not None + } + ctx = PoolingServeContext( request=request, raw_request=raw_request, model_name=model_name, pooling_params=pooling_params, request_id=request_id, + lora_request=lora_request, + priorities=priorities, + prompt_extras=prompt_extras, ) self._validate_request(ctx) - ctx.lora_request = self._maybe_get_adapters(ctx.request) return ctx async def _prepare_generators( @@ -127,7 +140,7 @@ class PoolingBaseServing(ABC, BaseServing): ctx: PoolingServeContext, ): if ctx.engine_inputs is None: - raise ValueError("Engine prompts not available") + raise ValueError("Engine inputs not available") generators: list[AsyncGenerator[PoolingRequestOutput, None]] = [] @@ -139,12 +152,7 @@ class PoolingBaseServing(ABC, BaseServing): assert ctx.pooling_params is not None pooling_params = ctx.pooling_params - - if isinstance(pooling_params, list): - for params in pooling_params: - params.verify(self.model_config) - else: - pooling_params.verify(self.model_config) + pooling_params.verify(self.model_config) for i, engine_input in enumerate(ctx.engine_inputs): prompt_request_id = ( @@ -153,26 +161,20 @@ class PoolingBaseServing(ABC, BaseServing): else ctx.prompt_request_ids[i] ) - params = ( - pooling_params[i] - if isinstance(pooling_params, list) - else pooling_params - ) - self._log_inputs( prompt_request_id, - engine_input, - params=params, + engine_input["prompts"], + params=engine_input["params"], lora_request=ctx.lora_request, ) generator = self.engine_client.encode( - engine_input, - params, - prompt_request_id, - lora_request=ctx.lora_request, + request_id=prompt_request_id, + prompt=engine_input["prompts"], + pooling_params=engine_input["params"], + lora_request=engine_input["lora_requests"], + priority=engine_input["priorities"], trace_headers=trace_headers, - priority=getattr(ctx.request, "priority", 0), ) generators.append(generator) diff --git a/vllm/entrypoints/pooling/classify/api_router.py b/vllm/entrypoints/pooling/classify/api_router.py index 9e016a72e8..fc86f49d6f 100644 --- a/vllm/entrypoints/pooling/classify/api_router.py +++ b/vllm/entrypoints/pooling/classify/api_router.py @@ -16,8 +16,11 @@ from .serving import ServingClassification router = APIRouter() -def classify(request: Request) -> ServingClassification | None: - return request.app.state.serving_classification +def classify(request: Request) -> ServingClassification: + handler = getattr(request.app.state, "serving_classification", None) + if handler is None: + raise NotImplementedError("The model does not support Classification API") + return handler @router.post("/classify", dependencies=[Depends(validate_json_request)]) @@ -27,7 +30,4 @@ async def create_classify( request: ClassificationRequest, raw_request: Request ) -> Response: handler = classify(raw_request) - if handler is None: - raise NotImplementedError("The model does not support Classification API") - return await handler(request, raw_request) diff --git a/vllm/entrypoints/pooling/embed/api_router.py b/vllm/entrypoints/pooling/embed/api_router.py index 7ffb5840d5..adee8a9431 100644 --- a/vllm/entrypoints/pooling/embed/api_router.py +++ b/vllm/entrypoints/pooling/embed/api_router.py @@ -18,8 +18,11 @@ from .serving import ServingEmbedding router = APIRouter() -def embedding(request: Request) -> ServingEmbedding | None: - return request.app.state.serving_embedding +def embedding(request: Request) -> ServingEmbedding: + handler = getattr(request.app.state, "serving_embedding", None) + if handler is None: + raise NotImplementedError("The model does not support Embeddings API") + return handler @router.post( @@ -37,9 +40,6 @@ async def create_embedding( raw_request: Request, ): handler = embedding(raw_request) - if handler is None: - raise NotImplementedError("The model does not support Embeddings API") - return await handler(request, raw_request) @@ -58,7 +58,4 @@ async def create_cohere_embedding( raw_request: Request, ): handler = embedding(raw_request) - if handler is None: - raise NotImplementedError("The model does not support Embeddings API") - return await handler(request, raw_request) diff --git a/vllm/entrypoints/pooling/embed/io_processor.py b/vllm/entrypoints/pooling/embed/io_processor.py index 5c2f825a96..57728c8837 100644 --- a/vllm/entrypoints/pooling/embed/io_processor.py +++ b/vllm/entrypoints/pooling/embed/io_processor.py @@ -17,7 +17,7 @@ from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, CustomChatCompletionMessageParam, ) -from vllm.inputs import EngineInput, tokens_input +from vllm.inputs import tokens_input from vllm.logger import init_logger from vllm.outputs import PoolingOutput, PoolingRequestOutput from vllm.renderers import merge_kwargs @@ -28,11 +28,14 @@ from vllm.utils.mistral import is_mistral_tokenizer from ..base.io_processor import PoolingIOProcessor from ..scoring.io_processor import JinaRankingIOProcessorMixin from ..typing import ( - ALLOfflineInputsContext, + AnyOfflineInputsContext, + AnyRenderParam, ChunkedEmbeddingMetadata, + EncodeChatRenderParams, OfflineEncodeInputsContext, PoolingChatLikeRequest, PoolingCompletionLikeRequest, + PoolingEngineInput, PoolingServeContext, RequestFactory, ) @@ -76,9 +79,11 @@ class EmbedIOProcessor(PoolingIOProcessor): list(self.task_instructions.keys()), ) - def pre_process_online(self, ctx: PoolingServeContext): + def get_request_factory_online( + self, ctx: PoolingServeContext + ) -> Sequence[AnyRenderParam]: if isinstance(ctx.request, CohereEmbedRequest): - self._pre_process_cohere_online(ctx) + requests = self._get_request_factory_cohere_online(ctx) elif isinstance( ctx.request, ( @@ -88,12 +93,11 @@ class EmbedIOProcessor(PoolingIOProcessor): EmbeddingBatchChatInputRequest, ), ): - self._pre_process_openai_chat_online(ctx) + requests = self._get_request_factory_chat_input_online(ctx) else: - super().pre_process_online(ctx) + requests = super().get_request_factory_online(ctx) - if self.enable_chunked_processing: - self._pre_process_chunked(ctx) + return requests def post_process_online( self, @@ -114,18 +118,21 @@ class EmbedIOProcessor(PoolingIOProcessor): # PTAL: examples/pooling/embed/openai_embedding_long_text ################################################################# - def _pre_process_chunked(self, ctx: PoolingServeContext) -> None: + def maybe_pre_process_chunked(self, ctx: PoolingServeContext) -> None: + if not self.enable_chunked_processing: + return None + if ctx.engine_inputs is None: raise ValueError("Engine prompts not available") ctx.original_engine_inputs = ctx.engine_inputs request_id = ctx.request_id max_model_len = self.model_config.max_model_len - chunked_engine_inputs: list[EngineInput] = [] + chunked_engine_inputs: list[PoolingEngineInput] = [] prompt_request_ids: list[str] = [] chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] = [] for prompt_idx, engine_input in enumerate(ctx.engine_inputs): - token_ids = engine_input.get("prompt_token_ids", None) + token_ids = engine_input["prompts"].get("prompt_token_ids", None) if token_ids is None: raise NotImplementedError( "Long Text Embedding with Chunked Processing does " @@ -138,7 +145,12 @@ class EmbedIOProcessor(PoolingIOProcessor): chunk_list(prompt_token_ids, max_model_len) ): chunked_engine_inputs.append( - tokens_input(prompt_token_ids=chunk_tokens) + PoolingEngineInput( + prompts=tokens_input(prompt_token_ids=chunk_tokens), + params=engine_input["params"], + lora_requests=engine_input["lora_requests"], + priorities=engine_input["priorities"], + ) ) prompt_request_ids.append( f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}" @@ -234,7 +246,7 @@ class EmbedIOProcessor(PoolingIOProcessor): # Get original prompt token IDs for this prompt original_prompt = original_engine_inputs[prompt_idx] - token_ids = original_prompt.get("prompt_token_ids", None) + token_ids = original_prompt["prompts"].get("prompt_token_ids", None) if token_ids is None: raise NotImplementedError( "Long Text Embedding with Chunked Processing does " @@ -262,6 +274,68 @@ class EmbedIOProcessor(PoolingIOProcessor): return None + ################################################################# + # Chat input Request Preprocessing & Postprocessing + ################################################################# + + def _get_request_factory_chat_input_online( + self, + ctx: PoolingServeContext, + ) -> Sequence[AnyRenderParam]: + request = ctx.request + renderer = self.renderer + + self._validate_chat_template( + request_chat_template=request.chat_template, + chat_template_kwargs=request.chat_template_kwargs, + trust_request_chat_template=self.trust_request_chat_template, + ) + + if isinstance( + request, (EmbeddingBatchChatRequest, EmbeddingBatchChatInputRequest) + ): + all_messages = request.messages + else: + all_messages = [request.messages] + num_requests = len(all_messages) + + default_template_kwargs = merge_kwargs( + self.template_kwargs, + dict( + tools=self.tool_dicts, + tokenize=is_mistral_tokenizer(renderer.tokenizer), + ), + ) + + mm_config = self.model_config.multimodal_config + tok_params = request.build_tok_params(self.model_config) + chat_params = request.build_chat_params( + self.chat_template, self.chat_template_content_format + ).with_defaults( + default_template_kwargs, + default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None), + ) + + params_seq = self._params_to_seq(ctx.pooling_params, num_requests) + seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests) + seq_priority = self._priority_to_seq(ctx.priorities, num_requests) + + requests = [ + EncodeChatRenderParams( + conversations=all_messages[i], + chat_params=chat_params, + tok_params=tok_params, + prompt_extras=ctx.prompt_extras, + skip_mm_cache=False, + params=params_seq[i], + lora_requests=seq_lora_requests[i], + priorities=seq_priority[i], + ) + for i in range(num_requests) + ] + + return requests + ################################################################# # Cohere Request Preprocessing & Postprocessing ################################################################# @@ -378,71 +452,9 @@ class EmbedIOProcessor(PoolingIOProcessor): ) return super().create_pooling_params(request) - def _pre_process_openai_chat_online( - self, - ctx: PoolingServeContext[ - EmbeddingChatRequest - | EmbeddingBatchChatRequest - | EmbeddingChatInputRequest - | EmbeddingBatchChatInputRequest - ], - ) -> None: - request = ctx.request - self._validate_chat_template( - request_chat_template=request.chat_template, - chat_template_kwargs=request.chat_template_kwargs, - trust_request_chat_template=self.trust_request_chat_template, - ) - - if isinstance( - request, (EmbeddingBatchChatRequest, EmbeddingBatchChatInputRequest) - ): - all_messages = request.messages - else: - all_messages = [request.messages] - ctx.engine_inputs = self._batch_render_openai_chat(request, all_messages) - - def _batch_render_openai_chat( - self, - request: ( - EmbeddingChatRequest - | EmbeddingBatchChatRequest - | EmbeddingChatInputRequest - | EmbeddingBatchChatInputRequest - ), - all_messages: Sequence[list[ChatCompletionMessageParam]], - ) -> list[EngineInput]: - renderer = self.renderer - mm_config = self.model_config.multimodal_config - - tok_params = request.build_tok_params(self.model_config) - chat_params = request.build_chat_params( - self.chat_template, - self.chat_template_content_format, - ).with_defaults( - merge_kwargs( - None, - dict( - tools=None, - tokenize=is_mistral_tokenizer(renderer.tokenizer), - ), - ), - default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None), - ) - - _, engine_inputs = renderer.render_chat( - all_messages, - chat_params, - tok_params, - prompt_extras={ - k: v - for k in ("mm_processor_kwargs", "cache_salt") - if (v := getattr(request, k, None)) is not None - }, - ) - return engine_inputs - - def _pre_process_cohere_online(self, ctx: PoolingServeContext) -> None: + def _get_request_factory_cohere_online( + self, ctx: PoolingServeContext + ) -> Sequence[AnyRenderParam]: """Convert a ``CohereEmbedRequest`` into engine prompts. If a model has a chat template the task instruction are rendered @@ -479,13 +491,12 @@ class EmbedIOProcessor(PoolingIOProcessor): task_prefix = self._get_task_instruction_prefix(input_type) if task_prefix is None: - ctx.engine_inputs = self._preprocess_cohere_text_completion( - request, + return self._get_request_factory_cohere_text_completion( + ctx, texts, truncate_prompt_tokens, truncation_side, ) - return all_messages = [ self._mixed_input_to_messages( @@ -497,27 +508,30 @@ class EmbedIOProcessor(PoolingIOProcessor): for text in texts ] if self._has_chat_template(): - ctx.engine_inputs = self._batch_render_chat( - request, + return self._batch_render_chat( + ctx, all_messages, truncate_prompt_tokens, truncation_side, ) else: - ctx.engine_inputs = self._preprocess_cohere_text_completion( - request, + return self._get_request_factory_cohere_text_completion( + ctx, self._apply_task_instruction(texts, input_type), truncate_prompt_tokens, truncation_side, ) - return task_prefix = self._get_task_instruction_prefix(input_type) all_messages = [ self._mixed_input_to_messages(inp, task_prefix=task_prefix) for inp in input ] - ctx.engine_inputs = self._batch_render_chat( - request, all_messages, truncate_prompt_tokens, truncation_side + + return self._batch_render_chat( + ctx, + all_messages, + truncate_prompt_tokens, + truncation_side, ) def _has_chat_template(self) -> bool: @@ -531,13 +545,14 @@ class EmbedIOProcessor(PoolingIOProcessor): is not None ) - def _preprocess_cohere_text_completion( + def _get_request_factory_cohere_text_completion( self, - request: CohereEmbedRequest, + ctx: PoolingServeContext, texts: list[str], truncate_prompt_tokens: int | None, truncation_side: Literal["left", "right"] | None, - ) -> list[EngineInput]: + ) -> Sequence[AnyRenderParam]: + request = ctx.request proxy = EmbeddingCompletionRequest( model=request.model, input=texts, @@ -546,50 +561,32 @@ class EmbedIOProcessor(PoolingIOProcessor): truncate_prompt_tokens=truncate_prompt_tokens, truncation_side=truncation_side, ) - return self._preprocess_cmpl_online( - proxy, prompt_input=proxy.input, prompt_embeds=None - ) + ctx.request = proxy + requests = super().get_request_factory_online(ctx) + ctx.request = request + return requests def _batch_render_chat( self, - request: CohereEmbedRequest, + ctx: PoolingServeContext, all_messages: Sequence[list[ChatCompletionMessageParam]], truncate_prompt_tokens: int | None, truncation_side: Literal["left", "right"] | None, - ) -> list[EngineInput]: + ) -> Sequence[AnyRenderParam]: """Batch-render multiple conversations through the chat template.""" - if not all_messages: - return [] - - proxy = EmbeddingChatRequest( + request = ctx.request + proxy = EmbeddingBatchChatRequest( model=request.model, - messages=list(all_messages[0]), + messages=all_messages, dimensions=request.output_dimension, encoding_format="float", truncate_prompt_tokens=truncate_prompt_tokens, truncation_side=truncation_side, ) - - renderer = self.renderer - mm_config = self.model_config.multimodal_config - - tok_params = proxy.build_tok_params(self.model_config) - chat_params = proxy.build_chat_params( - self.chat_template, - self.chat_template_content_format, - ).with_defaults( - merge_kwargs( - None, - dict( - tools=None, - tokenize=is_mistral_tokenizer(renderer.tokenizer), - ), - ), - default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None), - ) - - _, engine_inputs = renderer.render_chat(all_messages, chat_params, tok_params) - return engine_inputs + ctx.request = proxy + requests = self._get_request_factory_chat_input_online(ctx) + ctx.request = request + return requests def _validate_input_type(self, input_type: str | None) -> None: """Raise if *input_type* is not supported by this model.""" @@ -640,7 +637,9 @@ class TokenEmbedIOProcessor(PoolingIOProcessor): class JinaRankingTokenEmbedIOProcessor( TokenEmbedIOProcessor, JinaRankingIOProcessorMixin ): - def pre_process_online(self, ctx: PoolingServeContext): + def get_request_factory_online( + self, ctx: PoolingServeContext + ) -> Sequence[AnyRenderParam]: request = ctx.request if isinstance(request, PoolingCompletionLikeRequest): prompts = request.input @@ -655,20 +654,15 @@ class JinaRankingTokenEmbedIOProcessor( query=text_prompts[-1], docs=text_prompts[:-1] ) - engine_inputs = self._preprocess_cmpl_online( - request, - prompt_input=prompt_input, - prompt_embeds=None, - ) + request.input = prompt_input + return super().get_request_factory_online(ctx) elif isinstance(request, PoolingChatLikeRequest): raise ValueError("The JinaForRanking does not support chat Request.") else: raise ValueError(f"Invalid {self.name} request type") - ctx.engine_inputs = engine_inputs - def get_request_factory_offline( - self, ctx: ALLOfflineInputsContext + self, ctx: AnyOfflineInputsContext ) -> tuple[RequestFactory, int]: assert isinstance(ctx, OfflineEncodeInputsContext) if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2: diff --git a/vllm/entrypoints/pooling/embed/protocol.py b/vllm/entrypoints/pooling/embed/protocol.py index b2912e544f..96a87fea20 100644 --- a/vllm/entrypoints/pooling/embed/protocol.py +++ b/vllm/entrypoints/pooling/embed/protocol.py @@ -93,9 +93,9 @@ class EmbeddingBatchChatRequest( ``messages`` instead of introducing a separate batch-specific field. """ - messages: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = ( - Field(..., min_length=1) - ) + messages: Sequence[ + Annotated[list[ChatCompletionMessageParam], Field(min_length=1)] + ] = Field(..., min_length=1) def to_pooling_params(self): return PoolingParams( @@ -133,9 +133,9 @@ class EmbeddingChatInputRequest( class EmbeddingBatchChatInputRequest(EmbeddingBatchChatRequest): """OpenAI embeddings request with batched chat conversations in ``input``.""" - input: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = ( - Field(..., min_length=1) - ) + input: Sequence[ + Annotated[list[ChatCompletionMessageParam], Field(min_length=1)] + ] = Field(..., min_length=1) @model_validator(mode="before") @classmethod diff --git a/vllm/entrypoints/pooling/embed/serving.py b/vllm/entrypoints/pooling/embed/serving.py index fd8140982e..a9f6dd310e 100644 --- a/vllm/entrypoints/pooling/embed/serving.py +++ b/vllm/entrypoints/pooling/embed/serving.py @@ -9,6 +9,7 @@ from vllm.logger import init_logger from vllm.outputs import PoolingRequestOutput from vllm.utils.serial_utils import EmbedDType, Endianness +from ..base.io_processor import PoolingIOProcessor from ..base.serving import PoolingServing from ..typing import PoolingServeContext from ..utils import ( @@ -53,6 +54,12 @@ class ServingEmbedding(PoolingServing): def init_io_processor(self, *args, **kwargs) -> EmbedIOProcessor: return EmbedIOProcessor(*args, **kwargs) + async def _preprocessing( + self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext + ): + await super()._preprocessing(io_processor, ctx) + self.io_processor.maybe_pre_process_chunked(ctx) + def _build_response( self, ctx: PoolingServeContext, diff --git a/vllm/entrypoints/pooling/offline.py b/vllm/entrypoints/pooling/offline.py index bb3be812c6..ade3c03984 100644 --- a/vllm/entrypoints/pooling/offline.py +++ b/vllm/entrypoints/pooling/offline.py @@ -25,7 +25,7 @@ from .factories import init_pooling_io_processors from .scoring.io_processor import ScoringIOProcessor from .scoring.typing import ScoreInput from .typing import ( - ALLOfflineInputsContext, + AnyOfflineInputsContext, OfflineEncodeInputsContext, OfflineOutputsContext, OfflinePluginInputsContext, @@ -104,7 +104,7 @@ class PoolingOfflineMixin(OfflineInferenceMixin): io_processor = self.pooling_io_processors[pooling_task] - ctx: ALLOfflineInputsContext + ctx: AnyOfflineInputsContext if isinstance(prompts, dict) and "data" in prompts: ctx = OfflinePluginInputsContext( pooling_task=pooling_task, @@ -399,6 +399,9 @@ class PoolingOfflineMixin(OfflineInferenceMixin): num_requests: int, use_tqdm: bool | Callable[..., tqdm] = True, ): + if num_requests == 0: + raise ValueError("You must pass at least one prompt") + # Keeping max_num_seqs * 2 requests in the core can already saturate the core. # Therefore, keep most requests waiting outside the core. max_requests_in_core = ( diff --git a/vllm/entrypoints/pooling/pooling/api_router.py b/vllm/entrypoints/pooling/pooling/api_router.py index 653a36f699..40f7b9eb28 100644 --- a/vllm/entrypoints/pooling/pooling/api_router.py +++ b/vllm/entrypoints/pooling/pooling/api_router.py @@ -17,8 +17,11 @@ from .serving import ServingPooling router = APIRouter() -def pooling(request: Request) -> ServingPooling | None: - return request.app.state.serving_pooling +def pooling(request: Request) -> ServingPooling: + handler = getattr(request.app.state, "serving_pooling", None) + if handler is None: + raise NotImplementedError("The model does not support Pooling API") + return handler @router.post( @@ -33,7 +36,4 @@ def pooling(request: Request) -> ServingPooling | None: @load_aware_call async def create_pooling(request: PoolingRequest, raw_request: Request): handler = pooling(raw_request) - if handler is None: - raise NotImplementedError("The model does not support Pooling API") - return await handler(request, raw_request) diff --git a/vllm/entrypoints/pooling/pooling/io_processor.py b/vllm/entrypoints/pooling/pooling/io_processor.py index b0c38b63a3..ecc356c477 100644 --- a/vllm/entrypoints/pooling/pooling/io_processor.py +++ b/vllm/entrypoints/pooling/pooling/io_processor.py @@ -10,7 +10,9 @@ from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq from ..base.io_processor import PoolingIOProcessor from ..typing import ( - ALLOfflineInputsContext, + AnyOfflineInputsContext, + AnyRenderParam, + EncodeCMPLRenderParams, OfflineEncodeInputsContext, OfflineOutputsContext, OfflinePluginInputsContext, @@ -49,7 +51,9 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor): ####################################### # online APIs - def pre_process_online(self, ctx: PoolingServeContext): + def get_request_factory_online( + self, ctx: PoolingServeContext + ) -> Sequence[AnyRenderParam]: assert isinstance(ctx.request, IOProcessorRequest) validated_prompt = self.io_processor.parse_data(ctx.request.data) @@ -66,24 +70,33 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor): ) for prompt in prompt_to_seq(raw_prompts) ] + num_requests = len(parsed_prompts) tok_params = ctx.request.build_tok_params(self.model_config) - ctx.engine_inputs = self.renderer.render_cmpl( - parsed_prompts, - tok_params, - prompt_extras={ - k: v - for k in ("mm_processor_kwargs", "cache_salt") - if (v := getattr(ctx.request, k, None)) is not None - }, - ) - pooling_params = self.io_processor.merge_pooling_params() if pooling_params.task is None: pooling_params.task = "plugin" ctx.pooling_params = pooling_params + params_seq = self._params_to_seq(ctx.pooling_params, num_requests) + seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests) + seq_priority = self._priority_to_seq(ctx.priorities, num_requests) + + requests = [ + EncodeCMPLRenderParams( + prompts=parsed_prompts[i], + tok_params=tok_params, + prompt_extras=ctx.prompt_extras, + skip_mm_cache=False, + params=params_seq[i], + lora_requests=seq_lora_requests[i], + priorities=seq_priority[i], + ) + for i in range(num_requests) + ] + return requests + def post_process_online( self, ctx: PoolingServeContext, @@ -114,7 +127,7 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor): # offline APIs def get_request_factory_offline( - self, ctx: ALLOfflineInputsContext + self, ctx: AnyOfflineInputsContext ) -> tuple[RequestFactory, int]: assert isinstance(ctx, OfflinePluginInputsContext) assert isinstance(ctx.prompts, dict) and "data" in ctx.prompts diff --git a/vllm/entrypoints/pooling/scoring/api_router.py b/vllm/entrypoints/pooling/scoring/api_router.py index f67b5e912f..a26b3ca6d0 100644 --- a/vllm/entrypoints/pooling/scoring/api_router.py +++ b/vllm/entrypoints/pooling/scoring/api_router.py @@ -20,12 +20,18 @@ router = APIRouter() logger = init_logger(__name__) -def score(request: Request) -> ServingScores | None: - return request.app.state.serving_scores +def score(request: Request) -> ServingScores: + handler = getattr(request.app.state, "serving_scores", None) + if handler is None: + raise NotImplementedError("The model does not support Score API") + return handler -def rerank(request: Request) -> ServingScores | None: - return request.app.state.serving_scores +def rerank(request: Request) -> ServingScores: + handler = getattr(request.app.state, "serving_scores", None) + if handler is None: + raise NotImplementedError("The model does not support Rerank (Score) API") + return handler @router.post( @@ -40,9 +46,6 @@ def rerank(request: Request) -> ServingScores | None: @load_aware_call async def create_score(request: ScoreRequest, raw_request: Request): handler = score(raw_request) - if handler is None: - raise NotImplementedError("The model does not support Score API") - return await handler(request, raw_request) @@ -77,9 +80,6 @@ async def create_score_v1(request: ScoreRequest, raw_request: Request): @load_aware_call async def do_rerank(request: RerankRequest, raw_request: Request): handler = rerank(raw_request) - if handler is None: - raise NotImplementedError("The model does not support Rerank (Score) API") - return await handler(request, raw_request) diff --git a/vllm/entrypoints/pooling/scoring/io_processor.py b/vllm/entrypoints/pooling/scoring/io_processor.py index e8b3efc32e..8ebabca57c 100644 --- a/vllm/entrypoints/pooling/scoring/io_processor.py +++ b/vllm/entrypoints/pooling/scoring/io_processor.py @@ -6,8 +6,7 @@ from typing import Any, TypeAlias, cast import torch.nn.functional as F -from vllm import PoolingParams, PoolingRequestOutput, PromptType, TokensPrompt -from vllm.inputs import EngineInput +from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt from vllm.renderers import TokenizeParams from vllm.renderers.hf import safe_apply_chat_template from vllm.renderers.inputs.preprocess import ( @@ -20,8 +19,11 @@ from vllm.utils.mistral import is_mistral_tokenizer from ...chat_utils import ChatTemplateResolutionError from ..base.io_processor import PoolingIOProcessor +from ..pooling.protocol import PoolingCompletionRequest from ..typing import ( - ALLOfflineInputsContext, + AnyOfflineInputsContext, + AnyPoolingRequest, + AnyRenderParam, EncodeChatRenderParams, EncodeCMPLRenderParams, OfflineEncodeInputsContext, @@ -167,6 +169,19 @@ class ScoringIOProcessor(PoolingIOProcessor): ) return scoring_data + def valid_inputs_online(self, request: AnyPoolingRequest): + if isinstance(request, ScoreRequest): + data_1 = request.data_1 + data_2 = request.data_2 + elif isinstance(request, RerankRequest): + data_1 = request.query + data_2 = request.documents + else: + raise ValueError(f"Invalid {request.__class__.__name__} request type") + + scoring_data = self.valid_inputs(data_1, data_2) + return scoring_data + class BiEncoderIOProcessor(ScoringIOProcessor): name = "bi-encoder" @@ -175,19 +190,11 @@ class BiEncoderIOProcessor(ScoringIOProcessor): ####################################### # online APIs - def pre_process_online(self, ctx: ScoringServeContext): + def get_request_factory_online( + self, ctx: PoolingServeContext + ) -> Sequence[AnyRenderParam]: request = ctx.request - - if isinstance(request, ScoreRequest): - data_1 = request.data_1 - data_2 = request.data_2 - elif isinstance(request, RerankRequest): - data_1 = request.query - data_2 = request.documents - else: - raise ValueError(f"Invalid {self.name} request type") - - scoring_data = self.valid_inputs(data_1, data_2) + scoring_data = self.valid_inputs_online(request) max_tokens_per_query, max_tokens_per_doc = self._get_token_limits( request=request @@ -197,19 +204,37 @@ class BiEncoderIOProcessor(ScoringIOProcessor): scoring_data, max_tokens_per_query, max_tokens_per_doc ) - tok_params = request.build_tok_params(self.model_config) - engine_inputs = self._pre_process( - scoring_data, - tok_params, - prompt_extras={ - k: v - for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs") - if (v := getattr(request, k, None)) is not None - }, + data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config) + data_2 = score_data_to_prompts( + scoring_data.data_2, "document", self.model_config ) + prompts = data_1 + data_2 + ctx.n_queries = len(data_1) - ctx.engine_inputs = engine_inputs - ctx.n_queries = len(scoring_data.data_1) + prompts_seq = prompt_to_seq(prompts) + parsed_prompts = [ + parse_model_prompt(self.model_config, prompt) for prompt in prompts_seq + ] + num_requests = len(parsed_prompts) + + tok_params = request.build_tok_params(self.model_config) + params_seq = self._params_to_seq(ctx.pooling_params, num_requests) + seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests) + seq_priority = self._priority_to_seq(ctx.priorities, num_requests) + + requests = [ + EncodeCMPLRenderParams( + prompts=parsed_prompts[i], + tok_params=tok_params, + prompt_extras=ctx.prompt_extras, + skip_mm_cache=False, + params=params_seq[i], + lora_requests=seq_lora_requests[i], + priorities=seq_priority[i], + ) + for i in range(num_requests) + ] + return requests def post_process_online( self, @@ -226,7 +251,7 @@ class BiEncoderIOProcessor(ScoringIOProcessor): # offline APIs def get_request_factory_offline( - self, ctx: ALLOfflineInputsContext + self, ctx: AnyOfflineInputsContext ) -> tuple[RequestFactory, int]: assert isinstance(ctx, OfflineScoringInputsContext) @@ -267,41 +292,6 @@ class BiEncoderIOProcessor(ScoringIOProcessor): ####################################### # helpers - def _pre_process( - self, - scoring_data: ScoringData, - tok_params: TokenizeParams, - prompt_extras: dict[str, Any] | None = None, - ) -> Sequence[EngineInput]: - data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config) - data_2 = score_data_to_prompts( - scoring_data.data_2, "document", self.model_config - ) - - return self._preprocess_cmpl_offline( - prompts=data_1 + data_2, tok_params=tok_params, prompt_extras=prompt_extras - ) - - def _preprocess_cmpl_offline( - self, - prompts: PromptType | Sequence[PromptType], - tok_params: TokenizeParams, - prompt_extras: dict[str, Any] | None = None, - ) -> Sequence[EngineInput]: - prompts = prompt_to_seq(prompts) - parsed_prompts = [ - ( - prompt - if isinstance(prompt, bytes) - else parse_model_prompt(self.model_config, prompt) - ) - for prompt in prompts - ] - - return self.renderer.render_cmpl( - parsed_prompts, tok_params, prompt_extras=prompt_extras - ) - def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int): emb_data_1 = outputs[:n_queries] emb_data_2 = outputs[n_queries:] @@ -432,49 +422,50 @@ class CrossEncoderIOProcessor(ScoringIOProcessor): ####################################### # online APIs - def pre_process_online(self, ctx: ScoringServeContext): + def get_request_factory_online( + self, ctx: PoolingServeContext + ) -> Sequence[AnyRenderParam]: request = ctx.request + scoring_data = self.valid_inputs_online(request) + data_1 = scoring_data.data_1 + data_2 = scoring_data.data_2 + num_requests = len(data_2) - if isinstance(request, ScoreRequest): - data_1 = request.data_1 - data_2 = request.data_2 - elif isinstance(request, RerankRequest): - data_1 = request.query - data_2 = request.documents - else: - raise ValueError(f"Invalid {self.name} request type") - - scoring_data = self.valid_inputs(data_1, data_2) + if len(data_1) == 1: + data_1 = data_1 * num_requests max_tokens_per_query, max_tokens_per_doc = self._get_token_limits( request=request ) tok_params = request.build_tok_params(self.model_config) - pooling_params = self.create_pooling_params(request) + seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests) + seq_priority = self._priority_to_seq(ctx.priorities, num_requests) - engine_inputs, pooling_params_list = self._pre_process( - scoring_data, - tok_params, - pooling_params, - chat_template=self.chat_template, - max_tokens_per_query=max_tokens_per_query, - max_tokens_per_doc=max_tokens_per_doc, - prompt_extras={ - k: v - for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs") - if (v := getattr(request, k, None)) is not None - }, - ) + requests = [ + ScoringRenderParams( + data_1=data_1[i], + data_2=data_2[i], + chat_template=self.chat_template, + max_tokens_per_query=max_tokens_per_query, + max_tokens_per_doc=max_tokens_per_doc, + tok_params=tok_params, + prompt_extras=ctx.prompt_extras, + skip_mm_cache=False, + params=ctx.pooling_params, + lora_requests=seq_lora_requests[i], + priorities=seq_priority[i], + ) + for i in range(num_requests) + ] - ctx.engine_inputs = engine_inputs - ctx.pooling_params = pooling_params_list + return requests ####################################### # offline APIs def get_request_factory_offline( - self, ctx: ALLOfflineInputsContext + self, ctx: AnyOfflineInputsContext ) -> tuple[RequestFactory, int]: assert isinstance(ctx, OfflineScoringInputsContext) @@ -485,10 +476,12 @@ class CrossEncoderIOProcessor(ScoringIOProcessor): if len(data_1) == 1: data_1 = data_1 * num_requests + max_tokens_per_query, max_tokens_per_doc = self._get_token_limits( + pooling_params=ctx.pooling_params + ) tok_params = self.renderer.default_cmpl_tok_params.with_kwargs( **(ctx.tokenization_kwargs or {}) ) - prompt_extras = ctx.pooling_params.extra_kwargs seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests) @@ -500,6 +493,8 @@ class CrossEncoderIOProcessor(ScoringIOProcessor): data_1=data_1[i], data_2=data_2[i], chat_template=ctx.chat_template, + max_tokens_per_query=max_tokens_per_query, + max_tokens_per_doc=max_tokens_per_doc, tok_params=tok_params, prompt_extras=prompt_extras, skip_mm_cache=False, @@ -531,17 +526,13 @@ class CrossEncoderIOProcessor(ScoringIOProcessor): params = render_params["params"] prompt_extras = render_params["prompt_extras"] - max_tokens_per_query, max_tokens_per_doc = self._get_token_limits( - pooling_params=params - ) - _, engine_prompt = self.get_score_prompt( data_1=render_params["data_1"], data_2=render_params["data_2"], encode_kwargs=tok_params.get_encode_kwargs(), chat_template=render_params["chat_template"], - max_tokens_per_query=max_tokens_per_query, - max_tokens_per_doc=max_tokens_per_doc, + max_tokens_per_query=render_params["max_tokens_per_query"], + max_tokens_per_doc=render_params["max_tokens_per_doc"], chat_template_kwargs=prompt_extras.get("chat_template_kwargs") if prompt_extras else None, @@ -551,29 +542,16 @@ class CrossEncoderIOProcessor(ScoringIOProcessor): if token_type_ids := engine_prompt.pop("token_type_ids", None): params = params.clone() - compressed = compress_token_type_ids(token_type_ids) - params.extra_kwargs = {"compressed_token_type_ids": compressed} + compressed = compress_token_type_ids( + _apply_post_tokenization_to_token_type_ids( + self.tokenizer, tok_params, token_type_ids + ) + ) + params.extra_kwargs = { + **(params.extra_kwargs or {}), + "compressed_token_type_ids": compressed, + } - engine_input = self.renderer.process_for_engine(engine_prompt, arrival_time) - - return PoolingEngineInput( - prompts=engine_input, - params=params, - lora_requests=render_params["lora_requests"], - priorities=render_params["priorities"], - ) - - def _pre_process( - self, - scoring_data: ScoringData, - tok_params: TokenizeParams, - pooling_params: PoolingParams | None, - chat_template: str | None = None, - max_tokens_per_query: int = 0, - max_tokens_per_doc: int = 0, - prompt_extras: dict[str, Any] | None = None, - ) -> tuple[Sequence[EngineInput], list[PoolingParams]]: - arrival_time = time.time() engine_prompt_extras = ( { k: v @@ -584,55 +562,18 @@ class CrossEncoderIOProcessor(ScoringIOProcessor): else None ) - data_1 = scoring_data.data_1 - data_2 = scoring_data.data_2 + if engine_prompt_extras: + target_prompt = extract_target_prompt(self.model_config, engine_prompt) + target_prompt.update(engine_prompt_extras) - if len(data_1) == 1: - data_1 = data_1 * len(data_2) + engine_input = self.renderer.process_for_engine(engine_prompt, arrival_time) - if pooling_params is None: - pooling_params = PoolingParams(task="classify") - - pooling_params_list = list[PoolingParams]() - engine_inputs = list[EngineInput]() - for q, d in zip(data_1, data_2): - _, engine_prompt = self.get_score_prompt( - data_1=q, - data_2=d, - encode_kwargs=tok_params.get_encode_kwargs(), - chat_template=chat_template, - max_tokens_per_query=max_tokens_per_query, - max_tokens_per_doc=max_tokens_per_doc, - chat_template_kwargs=prompt_extras.get("chat_template_kwargs") - if prompt_extras - else None, - ) - - token_type_ids = engine_prompt.pop("token_type_ids", None) - tok_params.apply_post_tokenization(self.tokenizer, engine_prompt) - - if token_type_ids is not None: - params = pooling_params.clone() - compressed = compress_token_type_ids( - _apply_post_tokenization_to_token_type_ids( - self.tokenizer, tok_params, token_type_ids - ) - ) - params.extra_kwargs = { - **(params.extra_kwargs or {}), - "compressed_token_type_ids": compressed, - } - pooling_params_list.append(params) - else: - pooling_params_list.append(pooling_params) - - if engine_prompt_extras: - target_prompt = extract_target_prompt(self.model_config, engine_prompt) - target_prompt.update(engine_prompt_extras) - engine_inputs.append( - self.renderer.process_for_engine(engine_prompt, arrival_time) - ) - return engine_inputs, pooling_params_list + return PoolingEngineInput( + prompts=engine_input, + params=params, + lora_requests=render_params["lora_requests"], + priorities=render_params["priorities"], + ) def get_score_prompt( self, @@ -837,8 +778,55 @@ class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorM name = "jina-reranking-scoring" pooling_task: PoolingTask = "token_embed" + def get_request_factory_online( + self, ctx: PoolingServeContext + ) -> Sequence[AnyRenderParam]: + request = ctx.request + ctx.n_queries = 1 + + prompt_extras = ctx.prompt_extras + scoring_data = self.valid_inputs_online(request) + + max_tokens_per_query, max_tokens_per_doc = self._get_token_limits( + request=request + ) + + if max_tokens_per_query > 0 or max_tokens_per_doc > 0: + scoring_data = self._truncate_scoring_data( + scoring_data, max_tokens_per_query, max_tokens_per_doc + ) + + queries = self.ensure_str(scoring_data.data_1) + docs = self.ensure_str(scoring_data.data_2) + + chat_template_kwargs = ( + prompt_extras.get("chat_template_kwargs") if prompt_extras else None + ) + instruction = ( + chat_template_kwargs.get("instruction") if chat_template_kwargs else None + ) + + if len(queries) == 1: + prompts = [ + self.format_docs_prompts_func( + query=queries[0], docs=docs, instruction=instruction + ) + ] + else: + prompts = [ + self.format_docs_prompts_func( + query=q, docs=[d], instruction=instruction + ) + for q, d in zip(queries, docs) + ] + + ctx.request = PoolingCompletionRequest(task="token_embed", input=prompts) + requests = PoolingIOProcessor.get_request_factory_online(self, ctx) + ctx.request = request + return requests + def get_request_factory_offline( - self, ctx: ALLOfflineInputsContext + self, ctx: AnyOfflineInputsContext ) -> tuple[RequestFactory, int]: assert isinstance(ctx, OfflineScoringInputsContext) @@ -880,39 +868,6 @@ class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorM ), ) - def _pre_process( - self, - scoring_data: ScoringData, - tok_params: TokenizeParams, - prompt_extras: dict[str, Any] | None = None, - ) -> Sequence[EngineInput]: - queries = self.ensure_str(scoring_data.data_1) - docs = self.ensure_str(scoring_data.data_2) - chat_template_kwargs = ( - prompt_extras.get("chat_template_kwargs") if prompt_extras else None - ) - instruction = ( - chat_template_kwargs.get("instruction") if chat_template_kwargs else None - ) - - if len(queries) == 1: - prompts = [ - self.format_docs_prompts_func( - query=queries[0], docs=docs, instruction=instruction - ) - ] - else: - prompts = [ - self.format_docs_prompts_func( - query=q, docs=[d], instruction=instruction - ) - for q, d in zip(queries, docs) - ] - - return self._preprocess_cmpl_offline( - prompts=prompts, tok_params=tok_params, prompt_extras=prompt_extras - ) - def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int): final_res_batch: list[PoolingRequestOutput] = [] diff --git a/vllm/entrypoints/pooling/scoring/serving.py b/vllm/entrypoints/pooling/scoring/serving.py index 5937664d56..34a2887d54 100644 --- a/vllm/entrypoints/pooling/scoring/serving.py +++ b/vllm/entrypoints/pooling/scoring/serving.py @@ -190,7 +190,7 @@ class ServingScores(PoolingServing): async def flash_late_interaction(self, *args, **kwargs) -> Response: ctx = await self._init_ctx(self.io_processor, *args, **kwargs) - await self._preprocessing_async(self.io_processor, ctx) + await self._preprocessing(self.io_processor, ctx) # stage 1: encode queries and cache token embeddings on workers. await self._flash_late_interaction_encode_queries(ctx) @@ -211,7 +211,6 @@ class ServingScores(PoolingServing): query_keys = [f"{ctx.request_id}-query-{i}" for i in range(n_queries)] query_uses = [n_docs if n_queries == 1 else 1] * n_queries - query_pooling_params_list = [] for i in range(n_queries): pooling_params = ctx.pooling_params.clone() pooling_params.late_interaction_params = ( @@ -220,23 +219,21 @@ class ServingScores(PoolingServing): query_uses=query_uses[i], ) ) - query_pooling_params_list.append(pooling_params) + query_engine_inputs[i]["params"] = pooling_params - assert ( - n_queries - == len(query_pooling_params_list) - == len(query_engine_inputs) - == len(query_keys) - ) + assert n_queries == len(query_engine_inputs) == len(query_keys) query_ctx = ScoringServeContext( request=ctx.request, raw_request=ctx.raw_request, model_name=ctx.model_name, request_id=ctx.request_id, - pooling_params=query_pooling_params_list, + pooling_params=ctx.pooling_params, prompt_request_ids=query_keys, engine_inputs=query_engine_inputs, + lora_request=ctx.lora_request, + priorities=ctx.priorities, + prompt_extras=ctx.prompt_extras, ) await self._prepare_generators(query_ctx) @@ -255,30 +252,27 @@ class ServingScores(PoolingServing): query_keys = [f"{ctx.request_id}-query-{i}" for i in range(n_queries)] doc_keys = [f"{ctx.request_id}-doc-{i}" for i in range(n_docs)] - doc_pooling_params_list = [] for i in range(n_docs): query_idx = 0 if n_queries == 1 else i pooling_params = ctx.pooling_params.clone() pooling_params.late_interaction_params = build_late_interaction_doc_params( query_key=query_keys[query_idx] ) - doc_pooling_params_list.append(pooling_params) + doc_engine_inputs[i]["params"] = pooling_params - assert ( - n_docs - == len(doc_pooling_params_list) - == len(doc_engine_inputs) - == len(doc_keys) - ) + assert n_docs == len(doc_engine_inputs) == len(doc_keys) doc_ctx = ScoringServeContext( request=ctx.request, raw_request=ctx.raw_request, model_name=ctx.model_name, request_id=ctx.request_id, - pooling_params=doc_pooling_params_list, + pooling_params=ctx.pooling_params, prompt_request_ids=doc_keys, engine_inputs=doc_engine_inputs, + lora_request=ctx.lora_request, + priorities=ctx.priorities, + prompt_extras=ctx.prompt_extras, ) await self._prepare_generators(doc_ctx) diff --git a/vllm/entrypoints/pooling/typing.py b/vllm/entrypoints/pooling/typing.py index b44c9476e7..288f329d09 100644 --- a/vllm/entrypoints/pooling/typing.py +++ b/vllm/entrypoints/pooling/typing.py @@ -88,10 +88,13 @@ class PoolingServeContext(Generic[PoolingRequestT]): raw_request: Request | None = None model_name: str request_id: str - pooling_params: PoolingParams | list[PoolingParams] + pooling_params: PoolingParams + lora_request: LoRARequest | None + priorities: int | Sequence[int] | None + prompt_extras: dict[str, Any] | None + created_time: int = field(default_factory=lambda: int(time.time())) - lora_request: LoRARequest | None = None - engine_inputs: Sequence[EngineInput] | None = None + engine_inputs: Sequence["PoolingEngineInput"] | None = None prompt_request_ids: list[str] | None = None result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = ( @@ -100,7 +103,7 @@ class PoolingServeContext(Generic[PoolingRequestT]): final_res_batch: list[PoolingRequestOutput] = field(default_factory=list) ## for Long Text Embedding with Chunked Processing - original_engine_inputs: Sequence[EngineInput] | None = None + original_engine_inputs: Sequence["PoolingEngineInput"] | None = None chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] | None = None ## for bi-encoder & late-interaction @@ -118,7 +121,7 @@ class OfflineInputsContext: pooling_task: PoolingTask tokenization_kwargs: dict[str, Any] | None lora_request: Sequence[LoRARequest | None] | None - priorities: Sequence[int] | None + priorities: int | Sequence[int] | None @dataclass @@ -140,7 +143,7 @@ class OfflinePluginInputsContext(OfflineInputsContext): pooling_params: PoolingParams | Sequence[PoolingParams] | None -ALLOfflineInputsContext: TypeAlias = ( +AnyOfflineInputsContext: TypeAlias = ( OfflineEncodeInputsContext | OfflineScoringInputsContext | OfflinePluginInputsContext @@ -178,6 +181,15 @@ class ScoringRenderParams(RenderParams): data_1: ScoreData data_2: ScoreData chat_template: str | None + max_tokens_per_query: int + max_tokens_per_doc: int + + +AnyRenderParam: TypeAlias = ( + EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams +) +RequestGenerator: TypeAlias = Generator[AnyRenderParam] +RequestFactory: TypeAlias = Callable[[], RequestGenerator] class PoolingEngineInput(TypedDict): @@ -185,9 +197,3 @@ class PoolingEngineInput(TypedDict): params: PoolingParams lora_requests: LoRARequest | None priorities: int - - -RequestGenerator: TypeAlias = Generator[ - EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams -] -RequestFactory: TypeAlias = Callable[[], RequestGenerator]