[Frontend] Parallelize preprocessing within the same request for pooling models online serving. (#49153)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-07-22 10:56:23 +00:00
committed by GitHub
parent d6dbdb9b0d
commit 61c9ef986a
17 changed files with 538 additions and 820 deletions
@@ -9,8 +9,6 @@ from pydantic import TypeAdapter, ValidationError
from vllm import PoolingParams
from vllm.entrypoints.pooling.embed.io_processor import EmbedIOProcessor
from vllm.entrypoints.pooling.embed.protocol import (
CohereEmbedContent,
CohereEmbedInput,
CohereEmbedRequest,
EmbeddingBatchChatInputRequest,
EmbeddingBatchChatRequest,
@@ -19,7 +17,10 @@ from vllm.entrypoints.pooling.embed.protocol import (
EmbeddingCompletionRequest,
EmbeddingRequest,
)
from vllm.entrypoints.pooling.typing import PoolingServeContext
from vllm.entrypoints.pooling.typing import (
PoolingEngineInput,
PoolingServeContext,
)
from vllm.outputs import PoolingOutput, PoolingRequestOutput
@@ -410,6 +411,7 @@ class TestChunkedEmbeddingProcessing:
def _make_handler(cls):
handler = object.__new__(EmbedIOProcessor)
handler.model_config = cls._FakeModelConfig()
handler.enable_chunked_processing = True
return handler
@staticmethod
@@ -421,15 +423,29 @@ class TestChunkedEmbeddingProcessing:
}
)
assert isinstance(request, EmbeddingCompletionRequest)
pooling_params = PoolingParams()
return PoolingServeContext(
request=request,
pooling_params=PoolingParams(),
pooling_params=pooling_params,
model_name="test",
request_id="embd-client-prompt-999-chunk-888",
engine_inputs=[
{"prompt_token_ids": [0, 1, 2, 3, 4]},
{"prompt_token_ids": [10, 11]},
PoolingEngineInput(
prompts={"prompt_token_ids": [0, 1, 2, 3, 4]},
params=pooling_params,
lora_requests=None,
priorities=0,
),
PoolingEngineInput(
prompts={"prompt_token_ids": [10, 11]},
params=pooling_params,
lora_requests=None,
priorities=0,
),
],
lora_request=None,
priorities=0,
prompt_extras=None,
)
@staticmethod
@@ -450,7 +466,7 @@ class TestChunkedEmbeddingProcessing:
handler = self._make_handler()
ctx = self._make_context()
handler._pre_process_chunked(ctx)
handler.maybe_pre_process_chunked(ctx)
assert ctx.prompt_request_ids == [
"embd-client-prompt-999-chunk-888-prompt-0-chunk-0",
@@ -488,227 +504,3 @@ class TestChunkedEmbeddingProcessing:
ctx.final_res_batch[1].outputs.data,
torch.tensor([9.0, 9.0]),
)
class TestPreProcessCohereOnline:
"""Unit tests for EmbedIOProcessor._pre_process_cohere_online."""
@staticmethod
def _make_context(**request_kwargs) -> PoolingServeContext[CohereEmbedRequest]:
return PoolingServeContext(
request=CohereEmbedRequest(model="test", **request_kwargs),
pooling_params=PoolingParams(),
model_name="test",
request_id="embd-test",
)
@staticmethod
def _make_handler():
handler = object.__new__(EmbedIOProcessor)
handler._validate_input_type = lambda _input_type: None
return handler
def test_text_only_without_task_prefix_uses_completion_path(self):
handler = self._make_handler()
ctx = self._make_context(texts=["hello"])
calls: list[tuple[str, object]] = []
def preprocess_cmpl_online(request, prompt_input, prompt_embeds):
calls.append(("completion", prompt_input))
return ["completion"]
handler._get_task_instruction_prefix = lambda _input_type: None
handler._has_chat_template = lambda: False
handler._preprocess_cmpl_online = preprocess_cmpl_online
handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail(
"text-only request should not require chat rendering"
)
handler._pre_process_cohere_online(ctx)
assert ctx.engine_inputs == ["completion"]
assert calls == [("completion", ["hello"])]
def test_text_only_falls_back_to_prefixed_completion_without_template(self):
handler = self._make_handler()
ctx = self._make_context(texts=["hello"], input_type="query")
calls: list[tuple[str, object]] = []
def preprocess_cmpl(request, prompt_input, prompt_embeds):
calls.append(("completion", prompt_input))
return ["fallback"]
handler._get_task_instruction_prefix = lambda _input_type: "query: "
handler._has_chat_template = lambda: False
handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail(
"chat rendering should be skipped without a template"
)
handler._preprocess_cmpl_online = preprocess_cmpl
handler._pre_process_cohere_online(ctx)
assert ctx.engine_inputs == ["fallback"]
assert calls == [("completion", ["query: hello"])]
def test_text_only_with_template_uses_chat_path(self):
handler = self._make_handler()
ctx = self._make_context(texts=["hello"], input_type="query")
calls: list[tuple[str, object]] = []
def batch_render_chat(
request,
all_messages,
truncate_prompt_tokens,
truncation_side,
):
calls.append(
(
"chat",
{
"request": request,
"all_messages": all_messages,
"truncate_prompt_tokens": truncate_prompt_tokens,
"truncation_side": truncation_side,
},
)
)
return ["chat"]
handler._get_task_instruction_prefix = lambda _input_type: "query: "
handler._has_chat_template = lambda: True
handler._batch_render_chat = batch_render_chat
handler._preprocess_cmpl_online = lambda *_args, **_kwargs: pytest.fail(
"completion path should be skipped when a template exists"
)
handler._pre_process_cohere_online(ctx)
assert ctx.engine_inputs == ["chat"]
assert calls == [
(
"chat",
{
"request": ctx.request,
"all_messages": [
handler._mixed_input_to_messages(
CohereEmbedInput(
content=[CohereEmbedContent(type="text", text="hello")]
),
task_prefix="query: ",
)
],
"truncate_prompt_tokens": -1,
"truncation_side": None,
},
)
]
class TestPreProcessOpenAIEmbeddingChatOnline:
"""Unit tests for OpenAI embedding chat preprocessing."""
class _FakeModelConfig:
max_model_len = 128
encoder_config: dict[str, object] = {}
pooler_config = None
multimodal_config = None
is_encoder_decoder = False
class _FakeRenderer:
tokenizer = object()
def __init__(self):
self.calls = []
def render_chat(
self,
all_messages,
chat_params,
tok_params,
prompt_extras=None,
):
self.calls.append(
{
"all_messages": all_messages,
"chat_params": chat_params,
"tok_params": tok_params,
"prompt_extras": prompt_extras,
}
)
return all_messages, [
{"prompt_token_ids": [index]} for index, _ in enumerate(all_messages)
]
@classmethod
def _make_handler(cls, renderer):
handler = object.__new__(EmbedIOProcessor)
handler.renderer = renderer
handler.model_config = cls._FakeModelConfig()
handler.chat_template = "template"
handler.chat_template_content_format = "auto"
handler.trust_request_chat_template = False
handler.enable_chunked_processing = False
return handler
@staticmethod
def _make_context(
request: (
EmbeddingChatRequest
| EmbeddingBatchChatRequest
| EmbeddingChatInputRequest
| EmbeddingBatchChatInputRequest
),
) -> PoolingServeContext[
EmbeddingChatRequest
| EmbeddingBatchChatRequest
| EmbeddingChatInputRequest
| EmbeddingBatchChatInputRequest
]:
return PoolingServeContext(
request=request,
pooling_params=PoolingParams(),
model_name="test",
request_id="embd-test",
)
def test_chat_template_kwargs_forwarded_for_batched_input_messages(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [
[{"role": "user", "content": "hello"}],
[{"role": "user", "content": "goodbye"}],
],
"add_generation_prompt": True,
"chat_template_kwargs": {"instruction": "Represent the query: "},
"mm_processor_kwargs": {"max_pixels": 1},
"cache_salt": "salt",
}
)
assert isinstance(request, EmbeddingBatchChatInputRequest)
renderer = self._FakeRenderer()
handler = self._make_handler(renderer)
ctx = self._make_context(request)
handler.pre_process_online(ctx)
assert ctx.engine_inputs == [
{"prompt_token_ids": [0]},
{"prompt_token_ids": [1]},
]
assert len(renderer.calls) == 1
call = renderer.calls[0]
assert call["all_messages"] == request.messages
assert call["prompt_extras"] == {
"mm_processor_kwargs": {"max_pixels": 1},
"cache_salt": "salt",
}
chat_template_kwargs = call["chat_params"].chat_template_kwargs
assert chat_template_kwargs["instruction"] == "Represent the query: "
assert chat_template_kwargs["add_generation_prompt"] is True
assert chat_template_kwargs["continue_final_message"] is False
assert "tools" not in chat_template_kwargs
assert chat_template_kwargs["tokenize"] is False
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
from types import SimpleNamespace
import pytest
import torch
@@ -10,10 +9,7 @@ import torch
from tests.models.utils import softmax
from vllm import LLM, PoolingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
from vllm.entrypoints.pooling.scoring.typing import ScoringData
from vllm.platforms import current_platform
from vllm.renderers import TokenizeParams
MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
PROMPT = "The chef prepared a delicious meal."
@@ -145,45 +141,6 @@ def test_max_tokens_per_doc(llm: LLM):
assert with_limit_tokens < no_limit_tokens
def test_token_type_ids_follow_post_tokenization():
processor = object.__new__(CrossEncoderIOProcessor)
processor.tokenizer = SimpleNamespace(truncation_side="right", pad_token_id=-1)
processor.renderer = SimpleNamespace(process_for_engine=lambda prompt, _: prompt)
processor.model_config = None
processor.get_score_prompt = lambda **_: (
"",
{
"prompt_token_ids": list(range(32)),
"token_type_ids": [0] * 16 + [1] * 16,
},
)
engine_inputs, pooling_params = processor._pre_process(
ScoringData(data_1=["query"], data_2=["document"]),
TokenizeParams(
max_total_tokens=None,
truncate_prompt_tokens=16,
truncation_side="left",
),
PoolingParams(task="classify", extra_kwargs={"cache_salt": "salt"}),
)
assert engine_inputs[0]["prompt_token_ids"] == list(range(16, 32))
assert pooling_params[0].extra_kwargs == {
"cache_salt": "salt",
"compressed_token_type_ids": 0,
}
engine_inputs, pooling_params = processor._pre_process(
ScoringData(data_1=["query"], data_2=["document"]),
TokenizeParams(max_total_tokens=None, pad_prompt_tokens=40),
PoolingParams(task="classify"),
)
assert engine_inputs[0]["prompt_token_ids"] == list(range(32)) + [-1] * 8
assert pooling_params[0].extra_kwargs == {"compressed_token_type_ids": 16}
def test_pooling_params(llm: LLM):
def get_outputs(use_activation):
outputs = llm.score(
@@ -161,7 +161,9 @@ def _make_pooling_serving(lora_name: str) -> _ConcretePoolingServing:
return serving
def _make_pooling_ctx(model_name: str) -> PoolingServeContext:
def _make_pooling_ctx(
model_name: str, serving: PoolingBaseServing
) -> PoolingServeContext:
mock_request = MagicMock()
mock_request.model = model_name
return PoolingServeContext(
@@ -169,6 +171,9 @@ def _make_pooling_ctx(model_name: str) -> PoolingServeContext:
model_name=MODEL_NAME,
request_id="test-id",
pooling_params=PoolingParams(),
lora_request=serving._maybe_get_adapters(mock_request),
priorities=0,
prompt_extras=None,
)
@@ -176,9 +181,7 @@ def test_pooling_maybe_get_adapters_lora_name_sets_lora_request():
"""LoRA adapter name must populate ctx.lora_request without raising."""
lora_name = "bot-embed-lora"
serving = _make_pooling_serving(lora_name)
ctx = _make_pooling_ctx(lora_name)
ctx.lora_request = serving._maybe_get_adapters(ctx.request)
ctx = _make_pooling_ctx(lora_name, serving)
assert ctx.lora_request is not None
assert ctx.lora_request.lora_name == lora_name
@@ -187,7 +190,6 @@ def test_pooling_maybe_get_adapters_lora_name_sets_lora_request():
def test_pooling_maybe_get_adapters_unknown_model_raises():
"""An unrecognised model name must still raise VLLMNotFoundError."""
serving = _make_pooling_serving("some-lora")
ctx = _make_pooling_ctx("unknown-model")
with pytest.raises(VLLMNotFoundError):
serving._maybe_get_adapters(ctx.request)
_make_pooling_ctx("unknown-model", serving)
+96 -100
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from concurrent.futures import Executor
from typing import Any, Final, cast
from vllm import (
@@ -10,21 +11,17 @@ from vllm import (
)
from vllm.config import VllmConfig
from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateConfig,
ChatTemplateContentFormatOption,
ConversationMessage,
)
from vllm.entrypoints.serve.engine.typing import RendererChatRequest, RendererRequest
from vllm.inputs import EngineInput, SingletonPrompt
from vllm.lora.request import LoRARequest
from vllm.renderers import BaseRenderer, merge_kwargs
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
from vllm.tool_parsers import ToolParser
from vllm.utils.async_utils import make_async
from vllm.utils.mistral import is_mistral_tokenizer
from ..typing import (
ALLOfflineInputsContext,
AnyOfflineInputsContext,
AnyRenderParam,
EncodeChatRenderParams,
EncodeCMPLRenderParams,
OfflineEncodeInputsContext,
@@ -66,14 +63,25 @@ class PoolingIOProcessor:
chat_template_config.trust_request_chat_template
)
self.template_kwargs = None
self.tool_dicts = None
# Shared thread pool executor for preprocessing
self._executor: Executor = self.renderer._executor
self.render_async = make_async(self.render, executor=self._executor)
#######################################
# online APIs
def create_pooling_params(self, request):
return request.to_pooling_params()
def pre_process_online(self, ctx: PoolingServeContext):
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
renderer = self.renderer
requests: Sequence[AnyRenderParam]
if isinstance(request, PoolingChatLikeRequest):
self._validate_chat_template(
@@ -81,24 +89,87 @@ class PoolingIOProcessor:
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
_, engine_inputs = self._preprocess_chat_online(
request,
request.messages,
default_template=self.chat_template,
default_template_content_format=self.chat_template_content_format,
default_template_kwargs=None,
num_requests = 1
default_template_kwargs = merge_kwargs(
self.template_kwargs,
dict(
tools=self.tool_dicts,
tokenize=is_mistral_tokenizer(renderer.tokenizer),
),
)
mm_config = self.model_config.multimodal_config
tok_params = request.build_tok_params(self.model_config)
chat_params = request.build_chat_params(
self.chat_template, self.chat_template_content_format
).with_defaults(
default_template_kwargs,
default_media_io_kwargs=(
mm_config.media_io_kwargs if mm_config else None
),
)
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(
ctx.lora_request, num_requests
)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeChatRenderParams(
conversations=request.messages,
chat_params=chat_params,
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
elif isinstance(request, PoolingCompletionLikeRequest):
engine_inputs = self._preprocess_cmpl_online(
request,
prompt_input=request.input,
prompt_embeds=None,
model_config = self.model_config
prompts_seq = prompt_to_seq(request.input)
num_requests = len(prompts_seq)
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(model_config, prompt)
)
for prompt in prompts_seq
]
tok_params = request.build_tok_params(model_config)
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(
ctx.lora_request, num_requests
)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeCMPLRenderParams(
prompts=parsed_prompts[i],
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
else:
raise ValueError(f"Invalid {self.name} request type")
ctx.engine_inputs = engine_inputs
def post_process_online(
self,
ctx: PoolingServeContext,
@@ -109,7 +180,7 @@ class PoolingIOProcessor:
# offline APIs
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineEncodeInputsContext)
@@ -209,84 +280,6 @@ class PoolingIOProcessor:
priorities=render_params["priorities"],
)
def _preprocess_cmpl_online(
self,
request: RendererRequest,
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
prompt_embeds: bytes | list[bytes] | None,
) -> list[EngineInput]:
renderer = self.renderer
model_config = self.model_config
prompts = list[SingletonPrompt | bytes]()
if prompt_embeds is not None: # embeds take higher priority
prompts.extend(prompt_to_seq(prompt_embeds))
if prompt_input is not None:
prompts.extend(prompt_to_seq(prompt_input))
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(model_config, prompt)
)
for prompt in prompts
]
tok_params = request.build_tok_params(model_config)
return renderer.render_cmpl(
parsed_prompts,
tok_params,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(request, k, None)) is not None
},
)
def _preprocess_chat_online(
self,
request: RendererChatRequest,
messages: list[ChatCompletionMessageParam],
default_template: str | None,
default_template_content_format: ChatTemplateContentFormatOption,
default_template_kwargs: dict[str, Any] | None,
tool_dicts: list[dict[str, Any]] | None = None,
tool_parser: type[ToolParser] | None = None,
) -> tuple[list[ConversationMessage], list[EngineInput]]:
renderer = self.renderer
default_template_kwargs = merge_kwargs(
default_template_kwargs,
dict(
tools=tool_dicts,
tokenize=is_mistral_tokenizer(renderer.tokenizer),
),
)
mm_config = self.model_config.multimodal_config
tok_params = request.build_tok_params(self.model_config)
chat_params = request.build_chat_params(
default_template, default_template_content_format
).with_defaults(
default_template_kwargs,
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
)
(conversation,), (engine_input,) = renderer.render_chat(
[messages],
chat_params,
tok_params,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(request, k, None)) is not None
},
)
return conversation, [engine_input]
def _validate_chat_template(
self,
request_chat_template: str | None,
@@ -341,10 +334,13 @@ class PoolingIOProcessor:
def _priority_to_seq(
self,
priority: Sequence[int] | None,
priority: int | Sequence[int] | None,
num_requests: int,
) -> Sequence[int]:
if priority is not None:
if isinstance(priority, int):
return [priority] * num_requests
if len(priority) != num_requests:
raise ValueError(
f"The lengths of prompts ({num_requests}) "
+31 -29
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from abc import ABC, abstractmethod
from collections.abc import AsyncGenerator, Mapping
from concurrent.futures import Executor
@@ -63,9 +63,6 @@ class PoolingBaseServing(ABC, BaseServing):
# Shared thread pool executor for preprocessing and postprocessing.
self._executor: Executor = self.renderer._executor
self._preprocessing_async = make_async(
self._preprocessing, executor=self._executor
)
self._postprocessing_async = make_async(
self._postprocessing, executor=self._executor
)
@@ -77,7 +74,7 @@ class PoolingBaseServing(ABC, BaseServing):
) -> Response:
io_processor = self.get_io_processor(request)
ctx = await self._init_ctx(io_processor, request, raw_request)
await self._preprocessing_async(io_processor, ctx)
await self._preprocessing(io_processor, ctx)
await self._prepare_generators(ctx)
await self._collect_batch(ctx)
return await self._postprocessing_async(io_processor, ctx)
@@ -86,11 +83,17 @@ class PoolingBaseServing(ABC, BaseServing):
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
raise NotImplementedError
@torch.inference_mode()
def _preprocessing(
async def _preprocessing(
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
):
return io_processor.pre_process_online(ctx)
requests = io_processor.get_request_factory_online(ctx)
if len(requests) == 0:
raise ValueError("You must pass at least one prompt")
ctx.engine_inputs = await asyncio.gather(
*[io_processor.render_async(request) for request in requests]
)
@torch.inference_mode()
def _postprocessing(
@@ -110,16 +113,26 @@ class PoolingBaseServing(ABC, BaseServing):
await self._check_model(request)
pooling_params = io_processor.create_pooling_params(request)
lora_request = self._maybe_get_adapters(request)
priorities = getattr(request, "priority", 0)
prompt_extras = {
k: v
for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
if (v := getattr(request, k, None)) is not None
}
ctx = PoolingServeContext(
request=request,
raw_request=raw_request,
model_name=model_name,
pooling_params=pooling_params,
request_id=request_id,
lora_request=lora_request,
priorities=priorities,
prompt_extras=prompt_extras,
)
self._validate_request(ctx)
ctx.lora_request = self._maybe_get_adapters(ctx.request)
return ctx
async def _prepare_generators(
@@ -127,7 +140,7 @@ class PoolingBaseServing(ABC, BaseServing):
ctx: PoolingServeContext,
):
if ctx.engine_inputs is None:
raise ValueError("Engine prompts not available")
raise ValueError("Engine inputs not available")
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
@@ -139,12 +152,7 @@ class PoolingBaseServing(ABC, BaseServing):
assert ctx.pooling_params is not None
pooling_params = ctx.pooling_params
if isinstance(pooling_params, list):
for params in pooling_params:
params.verify(self.model_config)
else:
pooling_params.verify(self.model_config)
pooling_params.verify(self.model_config)
for i, engine_input in enumerate(ctx.engine_inputs):
prompt_request_id = (
@@ -153,26 +161,20 @@ class PoolingBaseServing(ABC, BaseServing):
else ctx.prompt_request_ids[i]
)
params = (
pooling_params[i]
if isinstance(pooling_params, list)
else pooling_params
)
self._log_inputs(
prompt_request_id,
engine_input,
params=params,
engine_input["prompts"],
params=engine_input["params"],
lora_request=ctx.lora_request,
)
generator = self.engine_client.encode(
engine_input,
params,
prompt_request_id,
lora_request=ctx.lora_request,
request_id=prompt_request_id,
prompt=engine_input["prompts"],
pooling_params=engine_input["params"],
lora_request=engine_input["lora_requests"],
priority=engine_input["priorities"],
trace_headers=trace_headers,
priority=getattr(ctx.request, "priority", 0),
)
generators.append(generator)
@@ -16,8 +16,11 @@ from .serving import ServingClassification
router = APIRouter()
def classify(request: Request) -> ServingClassification | None:
return request.app.state.serving_classification
def classify(request: Request) -> ServingClassification:
handler = getattr(request.app.state, "serving_classification", None)
if handler is None:
raise NotImplementedError("The model does not support Classification API")
return handler
@router.post("/classify", dependencies=[Depends(validate_json_request)])
@@ -27,7 +30,4 @@ async def create_classify(
request: ClassificationRequest, raw_request: Request
) -> Response:
handler = classify(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Classification API")
return await handler(request, raw_request)
+5 -8
View File
@@ -18,8 +18,11 @@ from .serving import ServingEmbedding
router = APIRouter()
def embedding(request: Request) -> ServingEmbedding | None:
return request.app.state.serving_embedding
def embedding(request: Request) -> ServingEmbedding:
handler = getattr(request.app.state, "serving_embedding", None)
if handler is None:
raise NotImplementedError("The model does not support Embeddings API")
return handler
@router.post(
@@ -37,9 +40,6 @@ async def create_embedding(
raw_request: Request,
):
handler = embedding(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Embeddings API")
return await handler(request, raw_request)
@@ -58,7 +58,4 @@ async def create_cohere_embedding(
raw_request: Request,
):
handler = embedding(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Embeddings API")
return await handler(request, raw_request)
+125 -131
View File
@@ -17,7 +17,7 @@ from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
CustomChatCompletionMessageParam,
)
from vllm.inputs import EngineInput, tokens_input
from vllm.inputs import tokens_input
from vllm.logger import init_logger
from vllm.outputs import PoolingOutput, PoolingRequestOutput
from vllm.renderers import merge_kwargs
@@ -28,11 +28,14 @@ from vllm.utils.mistral import is_mistral_tokenizer
from ..base.io_processor import PoolingIOProcessor
from ..scoring.io_processor import JinaRankingIOProcessorMixin
from ..typing import (
ALLOfflineInputsContext,
AnyOfflineInputsContext,
AnyRenderParam,
ChunkedEmbeddingMetadata,
EncodeChatRenderParams,
OfflineEncodeInputsContext,
PoolingChatLikeRequest,
PoolingCompletionLikeRequest,
PoolingEngineInput,
PoolingServeContext,
RequestFactory,
)
@@ -76,9 +79,11 @@ class EmbedIOProcessor(PoolingIOProcessor):
list(self.task_instructions.keys()),
)
def pre_process_online(self, ctx: PoolingServeContext):
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
if isinstance(ctx.request, CohereEmbedRequest):
self._pre_process_cohere_online(ctx)
requests = self._get_request_factory_cohere_online(ctx)
elif isinstance(
ctx.request,
(
@@ -88,12 +93,11 @@ class EmbedIOProcessor(PoolingIOProcessor):
EmbeddingBatchChatInputRequest,
),
):
self._pre_process_openai_chat_online(ctx)
requests = self._get_request_factory_chat_input_online(ctx)
else:
super().pre_process_online(ctx)
requests = super().get_request_factory_online(ctx)
if self.enable_chunked_processing:
self._pre_process_chunked(ctx)
return requests
def post_process_online(
self,
@@ -114,18 +118,21 @@ class EmbedIOProcessor(PoolingIOProcessor):
# PTAL: examples/pooling/embed/openai_embedding_long_text
#################################################################
def _pre_process_chunked(self, ctx: PoolingServeContext) -> None:
def maybe_pre_process_chunked(self, ctx: PoolingServeContext) -> None:
if not self.enable_chunked_processing:
return None
if ctx.engine_inputs is None:
raise ValueError("Engine prompts not available")
ctx.original_engine_inputs = ctx.engine_inputs
request_id = ctx.request_id
max_model_len = self.model_config.max_model_len
chunked_engine_inputs: list[EngineInput] = []
chunked_engine_inputs: list[PoolingEngineInput] = []
prompt_request_ids: list[str] = []
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] = []
for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
token_ids = engine_input.get("prompt_token_ids", None)
token_ids = engine_input["prompts"].get("prompt_token_ids", None)
if token_ids is None:
raise NotImplementedError(
"Long Text Embedding with Chunked Processing does "
@@ -138,7 +145,12 @@ class EmbedIOProcessor(PoolingIOProcessor):
chunk_list(prompt_token_ids, max_model_len)
):
chunked_engine_inputs.append(
tokens_input(prompt_token_ids=chunk_tokens)
PoolingEngineInput(
prompts=tokens_input(prompt_token_ids=chunk_tokens),
params=engine_input["params"],
lora_requests=engine_input["lora_requests"],
priorities=engine_input["priorities"],
)
)
prompt_request_ids.append(
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
@@ -234,7 +246,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
# Get original prompt token IDs for this prompt
original_prompt = original_engine_inputs[prompt_idx]
token_ids = original_prompt.get("prompt_token_ids", None)
token_ids = original_prompt["prompts"].get("prompt_token_ids", None)
if token_ids is None:
raise NotImplementedError(
"Long Text Embedding with Chunked Processing does "
@@ -262,6 +274,68 @@ class EmbedIOProcessor(PoolingIOProcessor):
return None
#################################################################
# Chat input Request Preprocessing & Postprocessing
#################################################################
def _get_request_factory_chat_input_online(
self,
ctx: PoolingServeContext,
) -> Sequence[AnyRenderParam]:
request = ctx.request
renderer = self.renderer
self._validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
if isinstance(
request, (EmbeddingBatchChatRequest, EmbeddingBatchChatInputRequest)
):
all_messages = request.messages
else:
all_messages = [request.messages]
num_requests = len(all_messages)
default_template_kwargs = merge_kwargs(
self.template_kwargs,
dict(
tools=self.tool_dicts,
tokenize=is_mistral_tokenizer(renderer.tokenizer),
),
)
mm_config = self.model_config.multimodal_config
tok_params = request.build_tok_params(self.model_config)
chat_params = request.build_chat_params(
self.chat_template, self.chat_template_content_format
).with_defaults(
default_template_kwargs,
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
)
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeChatRenderParams(
conversations=all_messages[i],
chat_params=chat_params,
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
#################################################################
# Cohere Request Preprocessing & Postprocessing
#################################################################
@@ -378,71 +452,9 @@ class EmbedIOProcessor(PoolingIOProcessor):
)
return super().create_pooling_params(request)
def _pre_process_openai_chat_online(
self,
ctx: PoolingServeContext[
EmbeddingChatRequest
| EmbeddingBatchChatRequest
| EmbeddingChatInputRequest
| EmbeddingBatchChatInputRequest
],
) -> None:
request = ctx.request
self._validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
if isinstance(
request, (EmbeddingBatchChatRequest, EmbeddingBatchChatInputRequest)
):
all_messages = request.messages
else:
all_messages = [request.messages]
ctx.engine_inputs = self._batch_render_openai_chat(request, all_messages)
def _batch_render_openai_chat(
self,
request: (
EmbeddingChatRequest
| EmbeddingBatchChatRequest
| EmbeddingChatInputRequest
| EmbeddingBatchChatInputRequest
),
all_messages: Sequence[list[ChatCompletionMessageParam]],
) -> list[EngineInput]:
renderer = self.renderer
mm_config = self.model_config.multimodal_config
tok_params = request.build_tok_params(self.model_config)
chat_params = request.build_chat_params(
self.chat_template,
self.chat_template_content_format,
).with_defaults(
merge_kwargs(
None,
dict(
tools=None,
tokenize=is_mistral_tokenizer(renderer.tokenizer),
),
),
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
)
_, engine_inputs = renderer.render_chat(
all_messages,
chat_params,
tok_params,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(request, k, None)) is not None
},
)
return engine_inputs
def _pre_process_cohere_online(self, ctx: PoolingServeContext) -> None:
def _get_request_factory_cohere_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
"""Convert a ``CohereEmbedRequest`` into engine prompts.
If a model has a chat template the task instruction are rendered
@@ -479,13 +491,12 @@ class EmbedIOProcessor(PoolingIOProcessor):
task_prefix = self._get_task_instruction_prefix(input_type)
if task_prefix is None:
ctx.engine_inputs = self._preprocess_cohere_text_completion(
request,
return self._get_request_factory_cohere_text_completion(
ctx,
texts,
truncate_prompt_tokens,
truncation_side,
)
return
all_messages = [
self._mixed_input_to_messages(
@@ -497,27 +508,30 @@ class EmbedIOProcessor(PoolingIOProcessor):
for text in texts
]
if self._has_chat_template():
ctx.engine_inputs = self._batch_render_chat(
request,
return self._batch_render_chat(
ctx,
all_messages,
truncate_prompt_tokens,
truncation_side,
)
else:
ctx.engine_inputs = self._preprocess_cohere_text_completion(
request,
return self._get_request_factory_cohere_text_completion(
ctx,
self._apply_task_instruction(texts, input_type),
truncate_prompt_tokens,
truncation_side,
)
return
task_prefix = self._get_task_instruction_prefix(input_type)
all_messages = [
self._mixed_input_to_messages(inp, task_prefix=task_prefix) for inp in input
]
ctx.engine_inputs = self._batch_render_chat(
request, all_messages, truncate_prompt_tokens, truncation_side
return self._batch_render_chat(
ctx,
all_messages,
truncate_prompt_tokens,
truncation_side,
)
def _has_chat_template(self) -> bool:
@@ -531,13 +545,14 @@ class EmbedIOProcessor(PoolingIOProcessor):
is not None
)
def _preprocess_cohere_text_completion(
def _get_request_factory_cohere_text_completion(
self,
request: CohereEmbedRequest,
ctx: PoolingServeContext,
texts: list[str],
truncate_prompt_tokens: int | None,
truncation_side: Literal["left", "right"] | None,
) -> list[EngineInput]:
) -> Sequence[AnyRenderParam]:
request = ctx.request
proxy = EmbeddingCompletionRequest(
model=request.model,
input=texts,
@@ -546,50 +561,32 @@ class EmbedIOProcessor(PoolingIOProcessor):
truncate_prompt_tokens=truncate_prompt_tokens,
truncation_side=truncation_side,
)
return self._preprocess_cmpl_online(
proxy, prompt_input=proxy.input, prompt_embeds=None
)
ctx.request = proxy
requests = super().get_request_factory_online(ctx)
ctx.request = request
return requests
def _batch_render_chat(
self,
request: CohereEmbedRequest,
ctx: PoolingServeContext,
all_messages: Sequence[list[ChatCompletionMessageParam]],
truncate_prompt_tokens: int | None,
truncation_side: Literal["left", "right"] | None,
) -> list[EngineInput]:
) -> Sequence[AnyRenderParam]:
"""Batch-render multiple conversations through the chat template."""
if not all_messages:
return []
proxy = EmbeddingChatRequest(
request = ctx.request
proxy = EmbeddingBatchChatRequest(
model=request.model,
messages=list(all_messages[0]),
messages=all_messages,
dimensions=request.output_dimension,
encoding_format="float",
truncate_prompt_tokens=truncate_prompt_tokens,
truncation_side=truncation_side,
)
renderer = self.renderer
mm_config = self.model_config.multimodal_config
tok_params = proxy.build_tok_params(self.model_config)
chat_params = proxy.build_chat_params(
self.chat_template,
self.chat_template_content_format,
).with_defaults(
merge_kwargs(
None,
dict(
tools=None,
tokenize=is_mistral_tokenizer(renderer.tokenizer),
),
),
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
)
_, engine_inputs = renderer.render_chat(all_messages, chat_params, tok_params)
return engine_inputs
ctx.request = proxy
requests = self._get_request_factory_chat_input_online(ctx)
ctx.request = request
return requests
def _validate_input_type(self, input_type: str | None) -> None:
"""Raise if *input_type* is not supported by this model."""
@@ -640,7 +637,9 @@ class TokenEmbedIOProcessor(PoolingIOProcessor):
class JinaRankingTokenEmbedIOProcessor(
TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
):
def pre_process_online(self, ctx: PoolingServeContext):
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
if isinstance(request, PoolingCompletionLikeRequest):
prompts = request.input
@@ -655,20 +654,15 @@ class JinaRankingTokenEmbedIOProcessor(
query=text_prompts[-1], docs=text_prompts[:-1]
)
engine_inputs = self._preprocess_cmpl_online(
request,
prompt_input=prompt_input,
prompt_embeds=None,
)
request.input = prompt_input
return super().get_request_factory_online(ctx)
elif isinstance(request, PoolingChatLikeRequest):
raise ValueError("The JinaForRanking does not support chat Request.")
else:
raise ValueError(f"Invalid {self.name} request type")
ctx.engine_inputs = engine_inputs
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineEncodeInputsContext)
if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2:
+6 -6
View File
@@ -93,9 +93,9 @@ class EmbeddingBatchChatRequest(
``messages`` instead of introducing a separate batch-specific field.
"""
messages: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = (
Field(..., min_length=1)
)
messages: Sequence[
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
] = Field(..., min_length=1)
def to_pooling_params(self):
return PoolingParams(
@@ -133,9 +133,9 @@ class EmbeddingChatInputRequest(
class EmbeddingBatchChatInputRequest(EmbeddingBatchChatRequest):
"""OpenAI embeddings request with batched chat conversations in ``input``."""
input: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = (
Field(..., min_length=1)
)
input: Sequence[
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
] = Field(..., min_length=1)
@model_validator(mode="before")
@classmethod
@@ -9,6 +9,7 @@ from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput
from vllm.utils.serial_utils import EmbedDType, Endianness
from ..base.io_processor import PoolingIOProcessor
from ..base.serving import PoolingServing
from ..typing import PoolingServeContext
from ..utils import (
@@ -53,6 +54,12 @@ class ServingEmbedding(PoolingServing):
def init_io_processor(self, *args, **kwargs) -> EmbedIOProcessor:
return EmbedIOProcessor(*args, **kwargs)
async def _preprocessing(
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
):
await super()._preprocessing(io_processor, ctx)
self.io_processor.maybe_pre_process_chunked(ctx)
def _build_response(
self,
ctx: PoolingServeContext,
+5 -2
View File
@@ -25,7 +25,7 @@ from .factories import init_pooling_io_processors
from .scoring.io_processor import ScoringIOProcessor
from .scoring.typing import ScoreInput
from .typing import (
ALLOfflineInputsContext,
AnyOfflineInputsContext,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflinePluginInputsContext,
@@ -104,7 +104,7 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
io_processor = self.pooling_io_processors[pooling_task]
ctx: ALLOfflineInputsContext
ctx: AnyOfflineInputsContext
if isinstance(prompts, dict) and "data" in prompts:
ctx = OfflinePluginInputsContext(
pooling_task=pooling_task,
@@ -399,6 +399,9 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
num_requests: int,
use_tqdm: bool | Callable[..., tqdm] = True,
):
if num_requests == 0:
raise ValueError("You must pass at least one prompt")
# Keeping max_num_seqs * 2 requests in the core can already saturate the core.
# Therefore, keep most requests waiting outside the core.
max_requests_in_core = (
@@ -17,8 +17,11 @@ from .serving import ServingPooling
router = APIRouter()
def pooling(request: Request) -> ServingPooling | None:
return request.app.state.serving_pooling
def pooling(request: Request) -> ServingPooling:
handler = getattr(request.app.state, "serving_pooling", None)
if handler is None:
raise NotImplementedError("The model does not support Pooling API")
return handler
@router.post(
@@ -33,7 +36,4 @@ def pooling(request: Request) -> ServingPooling | None:
@load_aware_call
async def create_pooling(request: PoolingRequest, raw_request: Request):
handler = pooling(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Pooling API")
return await handler(request, raw_request)
@@ -10,7 +10,9 @@ from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
from ..base.io_processor import PoolingIOProcessor
from ..typing import (
ALLOfflineInputsContext,
AnyOfflineInputsContext,
AnyRenderParam,
EncodeCMPLRenderParams,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflinePluginInputsContext,
@@ -49,7 +51,9 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
#######################################
# online APIs
def pre_process_online(self, ctx: PoolingServeContext):
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
assert isinstance(ctx.request, IOProcessorRequest)
validated_prompt = self.io_processor.parse_data(ctx.request.data)
@@ -66,24 +70,33 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
)
for prompt in prompt_to_seq(raw_prompts)
]
num_requests = len(parsed_prompts)
tok_params = ctx.request.build_tok_params(self.model_config)
ctx.engine_inputs = self.renderer.render_cmpl(
parsed_prompts,
tok_params,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(ctx.request, k, None)) is not None
},
)
pooling_params = self.io_processor.merge_pooling_params()
if pooling_params.task is None:
pooling_params.task = "plugin"
ctx.pooling_params = pooling_params
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeCMPLRenderParams(
prompts=parsed_prompts[i],
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
def post_process_online(
self,
ctx: PoolingServeContext,
@@ -114,7 +127,7 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
# offline APIs
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflinePluginInputsContext)
assert isinstance(ctx.prompts, dict) and "data" in ctx.prompts
+10 -10
View File
@@ -20,12 +20,18 @@ router = APIRouter()
logger = init_logger(__name__)
def score(request: Request) -> ServingScores | None:
return request.app.state.serving_scores
def score(request: Request) -> ServingScores:
handler = getattr(request.app.state, "serving_scores", None)
if handler is None:
raise NotImplementedError("The model does not support Score API")
return handler
def rerank(request: Request) -> ServingScores | None:
return request.app.state.serving_scores
def rerank(request: Request) -> ServingScores:
handler = getattr(request.app.state, "serving_scores", None)
if handler is None:
raise NotImplementedError("The model does not support Rerank (Score) API")
return handler
@router.post(
@@ -40,9 +46,6 @@ def rerank(request: Request) -> ServingScores | None:
@load_aware_call
async def create_score(request: ScoreRequest, raw_request: Request):
handler = score(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Score API")
return await handler(request, raw_request)
@@ -77,9 +80,6 @@ async def create_score_v1(request: ScoreRequest, raw_request: Request):
@load_aware_call
async def do_rerank(request: RerankRequest, raw_request: Request):
handler = rerank(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Rerank (Score) API")
return await handler(request, raw_request)
+155 -200
View File
@@ -6,8 +6,7 @@ from typing import Any, TypeAlias, cast
import torch.nn.functional as F
from vllm import PoolingParams, PoolingRequestOutput, PromptType, TokensPrompt
from vllm.inputs import EngineInput
from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
from vllm.renderers import TokenizeParams
from vllm.renderers.hf import safe_apply_chat_template
from vllm.renderers.inputs.preprocess import (
@@ -20,8 +19,11 @@ from vllm.utils.mistral import is_mistral_tokenizer
from ...chat_utils import ChatTemplateResolutionError
from ..base.io_processor import PoolingIOProcessor
from ..pooling.protocol import PoolingCompletionRequest
from ..typing import (
ALLOfflineInputsContext,
AnyOfflineInputsContext,
AnyPoolingRequest,
AnyRenderParam,
EncodeChatRenderParams,
EncodeCMPLRenderParams,
OfflineEncodeInputsContext,
@@ -167,6 +169,19 @@ class ScoringIOProcessor(PoolingIOProcessor):
)
return scoring_data
def valid_inputs_online(self, request: AnyPoolingRequest):
if isinstance(request, ScoreRequest):
data_1 = request.data_1
data_2 = request.data_2
elif isinstance(request, RerankRequest):
data_1 = request.query
data_2 = request.documents
else:
raise ValueError(f"Invalid {request.__class__.__name__} request type")
scoring_data = self.valid_inputs(data_1, data_2)
return scoring_data
class BiEncoderIOProcessor(ScoringIOProcessor):
name = "bi-encoder"
@@ -175,19 +190,11 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
#######################################
# online APIs
def pre_process_online(self, ctx: ScoringServeContext):
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
if isinstance(request, ScoreRequest):
data_1 = request.data_1
data_2 = request.data_2
elif isinstance(request, RerankRequest):
data_1 = request.query
data_2 = request.documents
else:
raise ValueError(f"Invalid {self.name} request type")
scoring_data = self.valid_inputs(data_1, data_2)
scoring_data = self.valid_inputs_online(request)
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
request=request
@@ -197,19 +204,37 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
scoring_data, max_tokens_per_query, max_tokens_per_doc
)
tok_params = request.build_tok_params(self.model_config)
engine_inputs = self._pre_process(
scoring_data,
tok_params,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
if (v := getattr(request, k, None)) is not None
},
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
data_2 = score_data_to_prompts(
scoring_data.data_2, "document", self.model_config
)
prompts = data_1 + data_2
ctx.n_queries = len(data_1)
ctx.engine_inputs = engine_inputs
ctx.n_queries = len(scoring_data.data_1)
prompts_seq = prompt_to_seq(prompts)
parsed_prompts = [
parse_model_prompt(self.model_config, prompt) for prompt in prompts_seq
]
num_requests = len(parsed_prompts)
tok_params = request.build_tok_params(self.model_config)
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeCMPLRenderParams(
prompts=parsed_prompts[i],
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
def post_process_online(
self,
@@ -226,7 +251,7 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
# offline APIs
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
@@ -267,41 +292,6 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
#######################################
# helpers
def _pre_process(
self,
scoring_data: ScoringData,
tok_params: TokenizeParams,
prompt_extras: dict[str, Any] | None = None,
) -> Sequence[EngineInput]:
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
data_2 = score_data_to_prompts(
scoring_data.data_2, "document", self.model_config
)
return self._preprocess_cmpl_offline(
prompts=data_1 + data_2, tok_params=tok_params, prompt_extras=prompt_extras
)
def _preprocess_cmpl_offline(
self,
prompts: PromptType | Sequence[PromptType],
tok_params: TokenizeParams,
prompt_extras: dict[str, Any] | None = None,
) -> Sequence[EngineInput]:
prompts = prompt_to_seq(prompts)
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(self.model_config, prompt)
)
for prompt in prompts
]
return self.renderer.render_cmpl(
parsed_prompts, tok_params, prompt_extras=prompt_extras
)
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
emb_data_1 = outputs[:n_queries]
emb_data_2 = outputs[n_queries:]
@@ -432,49 +422,50 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
#######################################
# online APIs
def pre_process_online(self, ctx: ScoringServeContext):
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
scoring_data = self.valid_inputs_online(request)
data_1 = scoring_data.data_1
data_2 = scoring_data.data_2
num_requests = len(data_2)
if isinstance(request, ScoreRequest):
data_1 = request.data_1
data_2 = request.data_2
elif isinstance(request, RerankRequest):
data_1 = request.query
data_2 = request.documents
else:
raise ValueError(f"Invalid {self.name} request type")
scoring_data = self.valid_inputs(data_1, data_2)
if len(data_1) == 1:
data_1 = data_1 * num_requests
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
request=request
)
tok_params = request.build_tok_params(self.model_config)
pooling_params = self.create_pooling_params(request)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
engine_inputs, pooling_params_list = self._pre_process(
scoring_data,
tok_params,
pooling_params,
chat_template=self.chat_template,
max_tokens_per_query=max_tokens_per_query,
max_tokens_per_doc=max_tokens_per_doc,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
if (v := getattr(request, k, None)) is not None
},
)
requests = [
ScoringRenderParams(
data_1=data_1[i],
data_2=data_2[i],
chat_template=self.chat_template,
max_tokens_per_query=max_tokens_per_query,
max_tokens_per_doc=max_tokens_per_doc,
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=ctx.pooling_params,
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
ctx.engine_inputs = engine_inputs
ctx.pooling_params = pooling_params_list
return requests
#######################################
# offline APIs
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
@@ -485,10 +476,12 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
if len(data_1) == 1:
data_1 = data_1 * num_requests
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
pooling_params=ctx.pooling_params
)
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
**(ctx.tokenization_kwargs or {})
)
prompt_extras = ctx.pooling_params.extra_kwargs
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
@@ -500,6 +493,8 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
data_1=data_1[i],
data_2=data_2[i],
chat_template=ctx.chat_template,
max_tokens_per_query=max_tokens_per_query,
max_tokens_per_doc=max_tokens_per_doc,
tok_params=tok_params,
prompt_extras=prompt_extras,
skip_mm_cache=False,
@@ -531,17 +526,13 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
params = render_params["params"]
prompt_extras = render_params["prompt_extras"]
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
pooling_params=params
)
_, engine_prompt = self.get_score_prompt(
data_1=render_params["data_1"],
data_2=render_params["data_2"],
encode_kwargs=tok_params.get_encode_kwargs(),
chat_template=render_params["chat_template"],
max_tokens_per_query=max_tokens_per_query,
max_tokens_per_doc=max_tokens_per_doc,
max_tokens_per_query=render_params["max_tokens_per_query"],
max_tokens_per_doc=render_params["max_tokens_per_doc"],
chat_template_kwargs=prompt_extras.get("chat_template_kwargs")
if prompt_extras
else None,
@@ -551,29 +542,16 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
if token_type_ids := engine_prompt.pop("token_type_ids", None):
params = params.clone()
compressed = compress_token_type_ids(token_type_ids)
params.extra_kwargs = {"compressed_token_type_ids": compressed}
compressed = compress_token_type_ids(
_apply_post_tokenization_to_token_type_ids(
self.tokenizer, tok_params, token_type_ids
)
)
params.extra_kwargs = {
**(params.extra_kwargs or {}),
"compressed_token_type_ids": compressed,
}
engine_input = self.renderer.process_for_engine(engine_prompt, arrival_time)
return PoolingEngineInput(
prompts=engine_input,
params=params,
lora_requests=render_params["lora_requests"],
priorities=render_params["priorities"],
)
def _pre_process(
self,
scoring_data: ScoringData,
tok_params: TokenizeParams,
pooling_params: PoolingParams | None,
chat_template: str | None = None,
max_tokens_per_query: int = 0,
max_tokens_per_doc: int = 0,
prompt_extras: dict[str, Any] | None = None,
) -> tuple[Sequence[EngineInput], list[PoolingParams]]:
arrival_time = time.time()
engine_prompt_extras = (
{
k: v
@@ -584,55 +562,18 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
else None
)
data_1 = scoring_data.data_1
data_2 = scoring_data.data_2
if engine_prompt_extras:
target_prompt = extract_target_prompt(self.model_config, engine_prompt)
target_prompt.update(engine_prompt_extras)
if len(data_1) == 1:
data_1 = data_1 * len(data_2)
engine_input = self.renderer.process_for_engine(engine_prompt, arrival_time)
if pooling_params is None:
pooling_params = PoolingParams(task="classify")
pooling_params_list = list[PoolingParams]()
engine_inputs = list[EngineInput]()
for q, d in zip(data_1, data_2):
_, engine_prompt = self.get_score_prompt(
data_1=q,
data_2=d,
encode_kwargs=tok_params.get_encode_kwargs(),
chat_template=chat_template,
max_tokens_per_query=max_tokens_per_query,
max_tokens_per_doc=max_tokens_per_doc,
chat_template_kwargs=prompt_extras.get("chat_template_kwargs")
if prompt_extras
else None,
)
token_type_ids = engine_prompt.pop("token_type_ids", None)
tok_params.apply_post_tokenization(self.tokenizer, engine_prompt)
if token_type_ids is not None:
params = pooling_params.clone()
compressed = compress_token_type_ids(
_apply_post_tokenization_to_token_type_ids(
self.tokenizer, tok_params, token_type_ids
)
)
params.extra_kwargs = {
**(params.extra_kwargs or {}),
"compressed_token_type_ids": compressed,
}
pooling_params_list.append(params)
else:
pooling_params_list.append(pooling_params)
if engine_prompt_extras:
target_prompt = extract_target_prompt(self.model_config, engine_prompt)
target_prompt.update(engine_prompt_extras)
engine_inputs.append(
self.renderer.process_for_engine(engine_prompt, arrival_time)
)
return engine_inputs, pooling_params_list
return PoolingEngineInput(
prompts=engine_input,
params=params,
lora_requests=render_params["lora_requests"],
priorities=render_params["priorities"],
)
def get_score_prompt(
self,
@@ -837,8 +778,55 @@ class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorM
name = "jina-reranking-scoring"
pooling_task: PoolingTask = "token_embed"
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
ctx.n_queries = 1
prompt_extras = ctx.prompt_extras
scoring_data = self.valid_inputs_online(request)
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
request=request
)
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
scoring_data = self._truncate_scoring_data(
scoring_data, max_tokens_per_query, max_tokens_per_doc
)
queries = self.ensure_str(scoring_data.data_1)
docs = self.ensure_str(scoring_data.data_2)
chat_template_kwargs = (
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
)
instruction = (
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
)
if len(queries) == 1:
prompts = [
self.format_docs_prompts_func(
query=queries[0], docs=docs, instruction=instruction
)
]
else:
prompts = [
self.format_docs_prompts_func(
query=q, docs=[d], instruction=instruction
)
for q, d in zip(queries, docs)
]
ctx.request = PoolingCompletionRequest(task="token_embed", input=prompts)
requests = PoolingIOProcessor.get_request_factory_online(self, ctx)
ctx.request = request
return requests
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
@@ -880,39 +868,6 @@ class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorM
),
)
def _pre_process(
self,
scoring_data: ScoringData,
tok_params: TokenizeParams,
prompt_extras: dict[str, Any] | None = None,
) -> Sequence[EngineInput]:
queries = self.ensure_str(scoring_data.data_1)
docs = self.ensure_str(scoring_data.data_2)
chat_template_kwargs = (
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
)
instruction = (
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
)
if len(queries) == 1:
prompts = [
self.format_docs_prompts_func(
query=queries[0], docs=docs, instruction=instruction
)
]
else:
prompts = [
self.format_docs_prompts_func(
query=q, docs=[d], instruction=instruction
)
for q, d in zip(queries, docs)
]
return self._preprocess_cmpl_offline(
prompts=prompts, tok_params=tok_params, prompt_extras=prompt_extras
)
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
final_res_batch: list[PoolingRequestOutput] = []
+13 -19
View File
@@ -190,7 +190,7 @@ class ServingScores(PoolingServing):
async def flash_late_interaction(self, *args, **kwargs) -> Response:
ctx = await self._init_ctx(self.io_processor, *args, **kwargs)
await self._preprocessing_async(self.io_processor, ctx)
await self._preprocessing(self.io_processor, ctx)
# stage 1: encode queries and cache token embeddings on workers.
await self._flash_late_interaction_encode_queries(ctx)
@@ -211,7 +211,6 @@ class ServingScores(PoolingServing):
query_keys = [f"{ctx.request_id}-query-{i}" for i in range(n_queries)]
query_uses = [n_docs if n_queries == 1 else 1] * n_queries
query_pooling_params_list = []
for i in range(n_queries):
pooling_params = ctx.pooling_params.clone()
pooling_params.late_interaction_params = (
@@ -220,23 +219,21 @@ class ServingScores(PoolingServing):
query_uses=query_uses[i],
)
)
query_pooling_params_list.append(pooling_params)
query_engine_inputs[i]["params"] = pooling_params
assert (
n_queries
== len(query_pooling_params_list)
== len(query_engine_inputs)
== len(query_keys)
)
assert n_queries == len(query_engine_inputs) == len(query_keys)
query_ctx = ScoringServeContext(
request=ctx.request,
raw_request=ctx.raw_request,
model_name=ctx.model_name,
request_id=ctx.request_id,
pooling_params=query_pooling_params_list,
pooling_params=ctx.pooling_params,
prompt_request_ids=query_keys,
engine_inputs=query_engine_inputs,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
prompt_extras=ctx.prompt_extras,
)
await self._prepare_generators(query_ctx)
@@ -255,30 +252,27 @@ class ServingScores(PoolingServing):
query_keys = [f"{ctx.request_id}-query-{i}" for i in range(n_queries)]
doc_keys = [f"{ctx.request_id}-doc-{i}" for i in range(n_docs)]
doc_pooling_params_list = []
for i in range(n_docs):
query_idx = 0 if n_queries == 1 else i
pooling_params = ctx.pooling_params.clone()
pooling_params.late_interaction_params = build_late_interaction_doc_params(
query_key=query_keys[query_idx]
)
doc_pooling_params_list.append(pooling_params)
doc_engine_inputs[i]["params"] = pooling_params
assert (
n_docs
== len(doc_pooling_params_list)
== len(doc_engine_inputs)
== len(doc_keys)
)
assert n_docs == len(doc_engine_inputs) == len(doc_keys)
doc_ctx = ScoringServeContext(
request=ctx.request,
raw_request=ctx.raw_request,
model_name=ctx.model_name,
request_id=ctx.request_id,
pooling_params=doc_pooling_params_list,
pooling_params=ctx.pooling_params,
prompt_request_ids=doc_keys,
engine_inputs=doc_engine_inputs,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
prompt_extras=ctx.prompt_extras,
)
await self._prepare_generators(doc_ctx)
+18 -12
View File
@@ -88,10 +88,13 @@ class PoolingServeContext(Generic[PoolingRequestT]):
raw_request: Request | None = None
model_name: str
request_id: str
pooling_params: PoolingParams | list[PoolingParams]
pooling_params: PoolingParams
lora_request: LoRARequest | None
priorities: int | Sequence[int] | None
prompt_extras: dict[str, Any] | None
created_time: int = field(default_factory=lambda: int(time.time()))
lora_request: LoRARequest | None = None
engine_inputs: Sequence[EngineInput] | None = None
engine_inputs: Sequence["PoolingEngineInput"] | None = None
prompt_request_ids: list[str] | None = None
result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (
@@ -100,7 +103,7 @@ class PoolingServeContext(Generic[PoolingRequestT]):
final_res_batch: list[PoolingRequestOutput] = field(default_factory=list)
## for Long Text Embedding with Chunked Processing
original_engine_inputs: Sequence[EngineInput] | None = None
original_engine_inputs: Sequence["PoolingEngineInput"] | None = None
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] | None = None
## for bi-encoder & late-interaction
@@ -118,7 +121,7 @@ class OfflineInputsContext:
pooling_task: PoolingTask
tokenization_kwargs: dict[str, Any] | None
lora_request: Sequence[LoRARequest | None] | None
priorities: Sequence[int] | None
priorities: int | Sequence[int] | None
@dataclass
@@ -140,7 +143,7 @@ class OfflinePluginInputsContext(OfflineInputsContext):
pooling_params: PoolingParams | Sequence[PoolingParams] | None
ALLOfflineInputsContext: TypeAlias = (
AnyOfflineInputsContext: TypeAlias = (
OfflineEncodeInputsContext
| OfflineScoringInputsContext
| OfflinePluginInputsContext
@@ -178,6 +181,15 @@ class ScoringRenderParams(RenderParams):
data_1: ScoreData
data_2: ScoreData
chat_template: str | None
max_tokens_per_query: int
max_tokens_per_doc: int
AnyRenderParam: TypeAlias = (
EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams
)
RequestGenerator: TypeAlias = Generator[AnyRenderParam]
RequestFactory: TypeAlias = Callable[[], RequestGenerator]
class PoolingEngineInput(TypedDict):
@@ -185,9 +197,3 @@ class PoolingEngineInput(TypedDict):
params: PoolingParams
lora_requests: LoRARequest | None
priorities: int
RequestGenerator: TypeAlias = Generator[
EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams
]
RequestFactory: TypeAlias = Callable[[], RequestGenerator]