[Frontend] Parallelize preprocessing within the same request for pooling models online serving. (#49153)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -9,8 +9,6 @@ from pydantic import TypeAdapter, ValidationError
|
||||
from vllm import PoolingParams
|
||||
from vllm.entrypoints.pooling.embed.io_processor import EmbedIOProcessor
|
||||
from vllm.entrypoints.pooling.embed.protocol import (
|
||||
CohereEmbedContent,
|
||||
CohereEmbedInput,
|
||||
CohereEmbedRequest,
|
||||
EmbeddingBatchChatInputRequest,
|
||||
EmbeddingBatchChatRequest,
|
||||
@@ -19,7 +17,10 @@ from vllm.entrypoints.pooling.embed.protocol import (
|
||||
EmbeddingCompletionRequest,
|
||||
EmbeddingRequest,
|
||||
)
|
||||
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
||||
from vllm.entrypoints.pooling.typing import (
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
)
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||
|
||||
|
||||
@@ -410,6 +411,7 @@ class TestChunkedEmbeddingProcessing:
|
||||
def _make_handler(cls):
|
||||
handler = object.__new__(EmbedIOProcessor)
|
||||
handler.model_config = cls._FakeModelConfig()
|
||||
handler.enable_chunked_processing = True
|
||||
return handler
|
||||
|
||||
@staticmethod
|
||||
@@ -421,15 +423,29 @@ class TestChunkedEmbeddingProcessing:
|
||||
}
|
||||
)
|
||||
assert isinstance(request, EmbeddingCompletionRequest)
|
||||
pooling_params = PoolingParams()
|
||||
return PoolingServeContext(
|
||||
request=request,
|
||||
pooling_params=PoolingParams(),
|
||||
pooling_params=pooling_params,
|
||||
model_name="test",
|
||||
request_id="embd-client-prompt-999-chunk-888",
|
||||
engine_inputs=[
|
||||
{"prompt_token_ids": [0, 1, 2, 3, 4]},
|
||||
{"prompt_token_ids": [10, 11]},
|
||||
PoolingEngineInput(
|
||||
prompts={"prompt_token_ids": [0, 1, 2, 3, 4]},
|
||||
params=pooling_params,
|
||||
lora_requests=None,
|
||||
priorities=0,
|
||||
),
|
||||
PoolingEngineInput(
|
||||
prompts={"prompt_token_ids": [10, 11]},
|
||||
params=pooling_params,
|
||||
lora_requests=None,
|
||||
priorities=0,
|
||||
),
|
||||
],
|
||||
lora_request=None,
|
||||
priorities=0,
|
||||
prompt_extras=None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -450,7 +466,7 @@ class TestChunkedEmbeddingProcessing:
|
||||
handler = self._make_handler()
|
||||
ctx = self._make_context()
|
||||
|
||||
handler._pre_process_chunked(ctx)
|
||||
handler.maybe_pre_process_chunked(ctx)
|
||||
|
||||
assert ctx.prompt_request_ids == [
|
||||
"embd-client-prompt-999-chunk-888-prompt-0-chunk-0",
|
||||
@@ -488,227 +504,3 @@ class TestChunkedEmbeddingProcessing:
|
||||
ctx.final_res_batch[1].outputs.data,
|
||||
torch.tensor([9.0, 9.0]),
|
||||
)
|
||||
|
||||
|
||||
class TestPreProcessCohereOnline:
|
||||
"""Unit tests for EmbedIOProcessor._pre_process_cohere_online."""
|
||||
|
||||
@staticmethod
|
||||
def _make_context(**request_kwargs) -> PoolingServeContext[CohereEmbedRequest]:
|
||||
return PoolingServeContext(
|
||||
request=CohereEmbedRequest(model="test", **request_kwargs),
|
||||
pooling_params=PoolingParams(),
|
||||
model_name="test",
|
||||
request_id="embd-test",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _make_handler():
|
||||
handler = object.__new__(EmbedIOProcessor)
|
||||
handler._validate_input_type = lambda _input_type: None
|
||||
return handler
|
||||
|
||||
def test_text_only_without_task_prefix_uses_completion_path(self):
|
||||
handler = self._make_handler()
|
||||
ctx = self._make_context(texts=["hello"])
|
||||
calls: list[tuple[str, object]] = []
|
||||
|
||||
def preprocess_cmpl_online(request, prompt_input, prompt_embeds):
|
||||
calls.append(("completion", prompt_input))
|
||||
return ["completion"]
|
||||
|
||||
handler._get_task_instruction_prefix = lambda _input_type: None
|
||||
handler._has_chat_template = lambda: False
|
||||
handler._preprocess_cmpl_online = preprocess_cmpl_online
|
||||
handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail(
|
||||
"text-only request should not require chat rendering"
|
||||
)
|
||||
|
||||
handler._pre_process_cohere_online(ctx)
|
||||
|
||||
assert ctx.engine_inputs == ["completion"]
|
||||
assert calls == [("completion", ["hello"])]
|
||||
|
||||
def test_text_only_falls_back_to_prefixed_completion_without_template(self):
|
||||
handler = self._make_handler()
|
||||
ctx = self._make_context(texts=["hello"], input_type="query")
|
||||
calls: list[tuple[str, object]] = []
|
||||
|
||||
def preprocess_cmpl(request, prompt_input, prompt_embeds):
|
||||
calls.append(("completion", prompt_input))
|
||||
return ["fallback"]
|
||||
|
||||
handler._get_task_instruction_prefix = lambda _input_type: "query: "
|
||||
handler._has_chat_template = lambda: False
|
||||
handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail(
|
||||
"chat rendering should be skipped without a template"
|
||||
)
|
||||
handler._preprocess_cmpl_online = preprocess_cmpl
|
||||
|
||||
handler._pre_process_cohere_online(ctx)
|
||||
|
||||
assert ctx.engine_inputs == ["fallback"]
|
||||
assert calls == [("completion", ["query: hello"])]
|
||||
|
||||
def test_text_only_with_template_uses_chat_path(self):
|
||||
handler = self._make_handler()
|
||||
ctx = self._make_context(texts=["hello"], input_type="query")
|
||||
calls: list[tuple[str, object]] = []
|
||||
|
||||
def batch_render_chat(
|
||||
request,
|
||||
all_messages,
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
):
|
||||
calls.append(
|
||||
(
|
||||
"chat",
|
||||
{
|
||||
"request": request,
|
||||
"all_messages": all_messages,
|
||||
"truncate_prompt_tokens": truncate_prompt_tokens,
|
||||
"truncation_side": truncation_side,
|
||||
},
|
||||
)
|
||||
)
|
||||
return ["chat"]
|
||||
|
||||
handler._get_task_instruction_prefix = lambda _input_type: "query: "
|
||||
handler._has_chat_template = lambda: True
|
||||
handler._batch_render_chat = batch_render_chat
|
||||
handler._preprocess_cmpl_online = lambda *_args, **_kwargs: pytest.fail(
|
||||
"completion path should be skipped when a template exists"
|
||||
)
|
||||
|
||||
handler._pre_process_cohere_online(ctx)
|
||||
|
||||
assert ctx.engine_inputs == ["chat"]
|
||||
assert calls == [
|
||||
(
|
||||
"chat",
|
||||
{
|
||||
"request": ctx.request,
|
||||
"all_messages": [
|
||||
handler._mixed_input_to_messages(
|
||||
CohereEmbedInput(
|
||||
content=[CohereEmbedContent(type="text", text="hello")]
|
||||
),
|
||||
task_prefix="query: ",
|
||||
)
|
||||
],
|
||||
"truncate_prompt_tokens": -1,
|
||||
"truncation_side": None,
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
class TestPreProcessOpenAIEmbeddingChatOnline:
|
||||
"""Unit tests for OpenAI embedding chat preprocessing."""
|
||||
|
||||
class _FakeModelConfig:
|
||||
max_model_len = 128
|
||||
encoder_config: dict[str, object] = {}
|
||||
pooler_config = None
|
||||
multimodal_config = None
|
||||
is_encoder_decoder = False
|
||||
|
||||
class _FakeRenderer:
|
||||
tokenizer = object()
|
||||
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def render_chat(
|
||||
self,
|
||||
all_messages,
|
||||
chat_params,
|
||||
tok_params,
|
||||
prompt_extras=None,
|
||||
):
|
||||
self.calls.append(
|
||||
{
|
||||
"all_messages": all_messages,
|
||||
"chat_params": chat_params,
|
||||
"tok_params": tok_params,
|
||||
"prompt_extras": prompt_extras,
|
||||
}
|
||||
)
|
||||
return all_messages, [
|
||||
{"prompt_token_ids": [index]} for index, _ in enumerate(all_messages)
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def _make_handler(cls, renderer):
|
||||
handler = object.__new__(EmbedIOProcessor)
|
||||
handler.renderer = renderer
|
||||
handler.model_config = cls._FakeModelConfig()
|
||||
handler.chat_template = "template"
|
||||
handler.chat_template_content_format = "auto"
|
||||
handler.trust_request_chat_template = False
|
||||
handler.enable_chunked_processing = False
|
||||
return handler
|
||||
|
||||
@staticmethod
|
||||
def _make_context(
|
||||
request: (
|
||||
EmbeddingChatRequest
|
||||
| EmbeddingBatchChatRequest
|
||||
| EmbeddingChatInputRequest
|
||||
| EmbeddingBatchChatInputRequest
|
||||
),
|
||||
) -> PoolingServeContext[
|
||||
EmbeddingChatRequest
|
||||
| EmbeddingBatchChatRequest
|
||||
| EmbeddingChatInputRequest
|
||||
| EmbeddingBatchChatInputRequest
|
||||
]:
|
||||
return PoolingServeContext(
|
||||
request=request,
|
||||
pooling_params=PoolingParams(),
|
||||
model_name="test",
|
||||
request_id="embd-test",
|
||||
)
|
||||
|
||||
def test_chat_template_kwargs_forwarded_for_batched_input_messages(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [
|
||||
[{"role": "user", "content": "hello"}],
|
||||
[{"role": "user", "content": "goodbye"}],
|
||||
],
|
||||
"add_generation_prompt": True,
|
||||
"chat_template_kwargs": {"instruction": "Represent the query: "},
|
||||
"mm_processor_kwargs": {"max_pixels": 1},
|
||||
"cache_salt": "salt",
|
||||
}
|
||||
)
|
||||
assert isinstance(request, EmbeddingBatchChatInputRequest)
|
||||
|
||||
renderer = self._FakeRenderer()
|
||||
handler = self._make_handler(renderer)
|
||||
ctx = self._make_context(request)
|
||||
|
||||
handler.pre_process_online(ctx)
|
||||
|
||||
assert ctx.engine_inputs == [
|
||||
{"prompt_token_ids": [0]},
|
||||
{"prompt_token_ids": [1]},
|
||||
]
|
||||
assert len(renderer.calls) == 1
|
||||
|
||||
call = renderer.calls[0]
|
||||
assert call["all_messages"] == request.messages
|
||||
assert call["prompt_extras"] == {
|
||||
"mm_processor_kwargs": {"max_pixels": 1},
|
||||
"cache_salt": "salt",
|
||||
}
|
||||
|
||||
chat_template_kwargs = call["chat_params"].chat_template_kwargs
|
||||
assert chat_template_kwargs["instruction"] == "Represent the query: "
|
||||
assert chat_template_kwargs["add_generation_prompt"] is True
|
||||
assert chat_template_kwargs["continue_final_message"] is False
|
||||
assert "tools" not in chat_template_kwargs
|
||||
assert chat_template_kwargs["tokenize"] is False
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import weakref
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -10,10 +9,7 @@ import torch
|
||||
from tests.models.utils import softmax
|
||||
from vllm import LLM, PoolingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoringData
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
|
||||
PROMPT = "The chef prepared a delicious meal."
|
||||
@@ -145,45 +141,6 @@ def test_max_tokens_per_doc(llm: LLM):
|
||||
assert with_limit_tokens < no_limit_tokens
|
||||
|
||||
|
||||
def test_token_type_ids_follow_post_tokenization():
|
||||
processor = object.__new__(CrossEncoderIOProcessor)
|
||||
processor.tokenizer = SimpleNamespace(truncation_side="right", pad_token_id=-1)
|
||||
processor.renderer = SimpleNamespace(process_for_engine=lambda prompt, _: prompt)
|
||||
processor.model_config = None
|
||||
processor.get_score_prompt = lambda **_: (
|
||||
"",
|
||||
{
|
||||
"prompt_token_ids": list(range(32)),
|
||||
"token_type_ids": [0] * 16 + [1] * 16,
|
||||
},
|
||||
)
|
||||
|
||||
engine_inputs, pooling_params = processor._pre_process(
|
||||
ScoringData(data_1=["query"], data_2=["document"]),
|
||||
TokenizeParams(
|
||||
max_total_tokens=None,
|
||||
truncate_prompt_tokens=16,
|
||||
truncation_side="left",
|
||||
),
|
||||
PoolingParams(task="classify", extra_kwargs={"cache_salt": "salt"}),
|
||||
)
|
||||
|
||||
assert engine_inputs[0]["prompt_token_ids"] == list(range(16, 32))
|
||||
assert pooling_params[0].extra_kwargs == {
|
||||
"cache_salt": "salt",
|
||||
"compressed_token_type_ids": 0,
|
||||
}
|
||||
|
||||
engine_inputs, pooling_params = processor._pre_process(
|
||||
ScoringData(data_1=["query"], data_2=["document"]),
|
||||
TokenizeParams(max_total_tokens=None, pad_prompt_tokens=40),
|
||||
PoolingParams(task="classify"),
|
||||
)
|
||||
|
||||
assert engine_inputs[0]["prompt_token_ids"] == list(range(32)) + [-1] * 8
|
||||
assert pooling_params[0].extra_kwargs == {"compressed_token_type_ids": 16}
|
||||
|
||||
|
||||
def test_pooling_params(llm: LLM):
|
||||
def get_outputs(use_activation):
|
||||
outputs = llm.score(
|
||||
|
||||
@@ -161,7 +161,9 @@ def _make_pooling_serving(lora_name: str) -> _ConcretePoolingServing:
|
||||
return serving
|
||||
|
||||
|
||||
def _make_pooling_ctx(model_name: str) -> PoolingServeContext:
|
||||
def _make_pooling_ctx(
|
||||
model_name: str, serving: PoolingBaseServing
|
||||
) -> PoolingServeContext:
|
||||
mock_request = MagicMock()
|
||||
mock_request.model = model_name
|
||||
return PoolingServeContext(
|
||||
@@ -169,6 +171,9 @@ def _make_pooling_ctx(model_name: str) -> PoolingServeContext:
|
||||
model_name=MODEL_NAME,
|
||||
request_id="test-id",
|
||||
pooling_params=PoolingParams(),
|
||||
lora_request=serving._maybe_get_adapters(mock_request),
|
||||
priorities=0,
|
||||
prompt_extras=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -176,9 +181,7 @@ def test_pooling_maybe_get_adapters_lora_name_sets_lora_request():
|
||||
"""LoRA adapter name must populate ctx.lora_request without raising."""
|
||||
lora_name = "bot-embed-lora"
|
||||
serving = _make_pooling_serving(lora_name)
|
||||
ctx = _make_pooling_ctx(lora_name)
|
||||
|
||||
ctx.lora_request = serving._maybe_get_adapters(ctx.request)
|
||||
ctx = _make_pooling_ctx(lora_name, serving)
|
||||
|
||||
assert ctx.lora_request is not None
|
||||
assert ctx.lora_request.lora_name == lora_name
|
||||
@@ -187,7 +190,6 @@ def test_pooling_maybe_get_adapters_lora_name_sets_lora_request():
|
||||
def test_pooling_maybe_get_adapters_unknown_model_raises():
|
||||
"""An unrecognised model name must still raise VLLMNotFoundError."""
|
||||
serving = _make_pooling_serving("some-lora")
|
||||
ctx = _make_pooling_ctx("unknown-model")
|
||||
|
||||
with pytest.raises(VLLMNotFoundError):
|
||||
serving._maybe_get_adapters(ctx.request)
|
||||
_make_pooling_ctx("unknown-model", serving)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from concurrent.futures import Executor
|
||||
from typing import Any, Final, cast
|
||||
|
||||
from vllm import (
|
||||
@@ -10,21 +11,17 @@ from vllm import (
|
||||
)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateConfig,
|
||||
ChatTemplateContentFormatOption,
|
||||
ConversationMessage,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.typing import RendererChatRequest, RendererRequest
|
||||
from vllm.inputs import EngineInput, SingletonPrompt
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
|
||||
from vllm.tool_parsers import ToolParser
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ..typing import (
|
||||
ALLOfflineInputsContext,
|
||||
AnyOfflineInputsContext,
|
||||
AnyRenderParam,
|
||||
EncodeChatRenderParams,
|
||||
EncodeCMPLRenderParams,
|
||||
OfflineEncodeInputsContext,
|
||||
@@ -66,14 +63,25 @@ class PoolingIOProcessor:
|
||||
chat_template_config.trust_request_chat_template
|
||||
)
|
||||
|
||||
self.template_kwargs = None
|
||||
self.tool_dicts = None
|
||||
|
||||
# Shared thread pool executor for preprocessing
|
||||
self._executor: Executor = self.renderer._executor
|
||||
self.render_async = make_async(self.render, executor=self._executor)
|
||||
|
||||
#######################################
|
||||
# online APIs
|
||||
|
||||
def create_pooling_params(self, request):
|
||||
return request.to_pooling_params()
|
||||
|
||||
def pre_process_online(self, ctx: PoolingServeContext):
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
renderer = self.renderer
|
||||
requests: Sequence[AnyRenderParam]
|
||||
|
||||
if isinstance(request, PoolingChatLikeRequest):
|
||||
self._validate_chat_template(
|
||||
@@ -81,24 +89,87 @@ class PoolingIOProcessor:
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=self.trust_request_chat_template,
|
||||
)
|
||||
_, engine_inputs = self._preprocess_chat_online(
|
||||
request,
|
||||
request.messages,
|
||||
default_template=self.chat_template,
|
||||
default_template_content_format=self.chat_template_content_format,
|
||||
default_template_kwargs=None,
|
||||
|
||||
num_requests = 1
|
||||
default_template_kwargs = merge_kwargs(
|
||||
self.template_kwargs,
|
||||
dict(
|
||||
tools=self.tool_dicts,
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
),
|
||||
)
|
||||
|
||||
mm_config = self.model_config.multimodal_config
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
self.chat_template, self.chat_template_content_format
|
||||
).with_defaults(
|
||||
default_template_kwargs,
|
||||
default_media_io_kwargs=(
|
||||
mm_config.media_io_kwargs if mm_config else None
|
||||
),
|
||||
)
|
||||
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(
|
||||
ctx.lora_request, num_requests
|
||||
)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeChatRenderParams(
|
||||
conversations=request.messages,
|
||||
chat_params=chat_params,
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
return requests
|
||||
|
||||
elif isinstance(request, PoolingCompletionLikeRequest):
|
||||
engine_inputs = self._preprocess_cmpl_online(
|
||||
request,
|
||||
prompt_input=request.input,
|
||||
prompt_embeds=None,
|
||||
model_config = self.model_config
|
||||
prompts_seq = prompt_to_seq(request.input)
|
||||
num_requests = len(prompts_seq)
|
||||
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(model_config, prompt)
|
||||
)
|
||||
for prompt in prompts_seq
|
||||
]
|
||||
tok_params = request.build_tok_params(model_config)
|
||||
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(
|
||||
ctx.lora_request, num_requests
|
||||
)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeCMPLRenderParams(
|
||||
prompts=parsed_prompts[i],
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
return requests
|
||||
else:
|
||||
raise ValueError(f"Invalid {self.name} request type")
|
||||
|
||||
ctx.engine_inputs = engine_inputs
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
@@ -109,7 +180,7 @@ class PoolingIOProcessor:
|
||||
# offline APIs
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: ALLOfflineInputsContext
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineEncodeInputsContext)
|
||||
|
||||
@@ -209,84 +280,6 @@ class PoolingIOProcessor:
|
||||
priorities=render_params["priorities"],
|
||||
)
|
||||
|
||||
def _preprocess_cmpl_online(
|
||||
self,
|
||||
request: RendererRequest,
|
||||
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
|
||||
prompt_embeds: bytes | list[bytes] | None,
|
||||
) -> list[EngineInput]:
|
||||
renderer = self.renderer
|
||||
model_config = self.model_config
|
||||
|
||||
prompts = list[SingletonPrompt | bytes]()
|
||||
if prompt_embeds is not None: # embeds take higher priority
|
||||
prompts.extend(prompt_to_seq(prompt_embeds))
|
||||
if prompt_input is not None:
|
||||
prompts.extend(prompt_to_seq(prompt_input))
|
||||
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(model_config, prompt)
|
||||
)
|
||||
for prompt in prompts
|
||||
]
|
||||
tok_params = request.build_tok_params(model_config)
|
||||
|
||||
return renderer.render_cmpl(
|
||||
parsed_prompts,
|
||||
tok_params,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
)
|
||||
|
||||
def _preprocess_chat_online(
|
||||
self,
|
||||
request: RendererChatRequest,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
default_template: str | None,
|
||||
default_template_content_format: ChatTemplateContentFormatOption,
|
||||
default_template_kwargs: dict[str, Any] | None,
|
||||
tool_dicts: list[dict[str, Any]] | None = None,
|
||||
tool_parser: type[ToolParser] | None = None,
|
||||
) -> tuple[list[ConversationMessage], list[EngineInput]]:
|
||||
renderer = self.renderer
|
||||
|
||||
default_template_kwargs = merge_kwargs(
|
||||
default_template_kwargs,
|
||||
dict(
|
||||
tools=tool_dicts,
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
),
|
||||
)
|
||||
|
||||
mm_config = self.model_config.multimodal_config
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
default_template, default_template_content_format
|
||||
).with_defaults(
|
||||
default_template_kwargs,
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
)
|
||||
|
||||
(conversation,), (engine_input,) = renderer.render_chat(
|
||||
[messages],
|
||||
chat_params,
|
||||
tok_params,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
)
|
||||
|
||||
return conversation, [engine_input]
|
||||
|
||||
def _validate_chat_template(
|
||||
self,
|
||||
request_chat_template: str | None,
|
||||
@@ -341,10 +334,13 @@ class PoolingIOProcessor:
|
||||
|
||||
def _priority_to_seq(
|
||||
self,
|
||||
priority: Sequence[int] | None,
|
||||
priority: int | Sequence[int] | None,
|
||||
num_requests: int,
|
||||
) -> Sequence[int]:
|
||||
if priority is not None:
|
||||
if isinstance(priority, int):
|
||||
return [priority] * num_requests
|
||||
|
||||
if len(priority) != num_requests:
|
||||
raise ValueError(
|
||||
f"The lengths of prompts ({num_requests}) "
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from concurrent.futures import Executor
|
||||
@@ -63,9 +63,6 @@ class PoolingBaseServing(ABC, BaseServing):
|
||||
|
||||
# Shared thread pool executor for preprocessing and postprocessing.
|
||||
self._executor: Executor = self.renderer._executor
|
||||
self._preprocessing_async = make_async(
|
||||
self._preprocessing, executor=self._executor
|
||||
)
|
||||
self._postprocessing_async = make_async(
|
||||
self._postprocessing, executor=self._executor
|
||||
)
|
||||
@@ -77,7 +74,7 @@ class PoolingBaseServing(ABC, BaseServing):
|
||||
) -> Response:
|
||||
io_processor = self.get_io_processor(request)
|
||||
ctx = await self._init_ctx(io_processor, request, raw_request)
|
||||
await self._preprocessing_async(io_processor, ctx)
|
||||
await self._preprocessing(io_processor, ctx)
|
||||
await self._prepare_generators(ctx)
|
||||
await self._collect_batch(ctx)
|
||||
return await self._postprocessing_async(io_processor, ctx)
|
||||
@@ -86,11 +83,17 @@ class PoolingBaseServing(ABC, BaseServing):
|
||||
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
|
||||
raise NotImplementedError
|
||||
|
||||
@torch.inference_mode()
|
||||
def _preprocessing(
|
||||
async def _preprocessing(
|
||||
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
|
||||
):
|
||||
return io_processor.pre_process_online(ctx)
|
||||
requests = io_processor.get_request_factory_online(ctx)
|
||||
|
||||
if len(requests) == 0:
|
||||
raise ValueError("You must pass at least one prompt")
|
||||
|
||||
ctx.engine_inputs = await asyncio.gather(
|
||||
*[io_processor.render_async(request) for request in requests]
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def _postprocessing(
|
||||
@@ -110,16 +113,26 @@ class PoolingBaseServing(ABC, BaseServing):
|
||||
await self._check_model(request)
|
||||
|
||||
pooling_params = io_processor.create_pooling_params(request)
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
priorities = getattr(request, "priority", 0)
|
||||
prompt_extras = {
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
}
|
||||
|
||||
ctx = PoolingServeContext(
|
||||
request=request,
|
||||
raw_request=raw_request,
|
||||
model_name=model_name,
|
||||
pooling_params=pooling_params,
|
||||
request_id=request_id,
|
||||
lora_request=lora_request,
|
||||
priorities=priorities,
|
||||
prompt_extras=prompt_extras,
|
||||
)
|
||||
|
||||
self._validate_request(ctx)
|
||||
ctx.lora_request = self._maybe_get_adapters(ctx.request)
|
||||
return ctx
|
||||
|
||||
async def _prepare_generators(
|
||||
@@ -127,7 +140,7 @@ class PoolingBaseServing(ABC, BaseServing):
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
if ctx.engine_inputs is None:
|
||||
raise ValueError("Engine prompts not available")
|
||||
raise ValueError("Engine inputs not available")
|
||||
|
||||
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||
|
||||
@@ -139,12 +152,7 @@ class PoolingBaseServing(ABC, BaseServing):
|
||||
|
||||
assert ctx.pooling_params is not None
|
||||
pooling_params = ctx.pooling_params
|
||||
|
||||
if isinstance(pooling_params, list):
|
||||
for params in pooling_params:
|
||||
params.verify(self.model_config)
|
||||
else:
|
||||
pooling_params.verify(self.model_config)
|
||||
pooling_params.verify(self.model_config)
|
||||
|
||||
for i, engine_input in enumerate(ctx.engine_inputs):
|
||||
prompt_request_id = (
|
||||
@@ -153,26 +161,20 @@ class PoolingBaseServing(ABC, BaseServing):
|
||||
else ctx.prompt_request_ids[i]
|
||||
)
|
||||
|
||||
params = (
|
||||
pooling_params[i]
|
||||
if isinstance(pooling_params, list)
|
||||
else pooling_params
|
||||
)
|
||||
|
||||
self._log_inputs(
|
||||
prompt_request_id,
|
||||
engine_input,
|
||||
params=params,
|
||||
engine_input["prompts"],
|
||||
params=engine_input["params"],
|
||||
lora_request=ctx.lora_request,
|
||||
)
|
||||
|
||||
generator = self.engine_client.encode(
|
||||
engine_input,
|
||||
params,
|
||||
prompt_request_id,
|
||||
lora_request=ctx.lora_request,
|
||||
request_id=prompt_request_id,
|
||||
prompt=engine_input["prompts"],
|
||||
pooling_params=engine_input["params"],
|
||||
lora_request=engine_input["lora_requests"],
|
||||
priority=engine_input["priorities"],
|
||||
trace_headers=trace_headers,
|
||||
priority=getattr(ctx.request, "priority", 0),
|
||||
)
|
||||
|
||||
generators.append(generator)
|
||||
|
||||
@@ -16,8 +16,11 @@ from .serving import ServingClassification
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def classify(request: Request) -> ServingClassification | None:
|
||||
return request.app.state.serving_classification
|
||||
def classify(request: Request) -> ServingClassification:
|
||||
handler = getattr(request.app.state, "serving_classification", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Classification API")
|
||||
return handler
|
||||
|
||||
|
||||
@router.post("/classify", dependencies=[Depends(validate_json_request)])
|
||||
@@ -27,7 +30,4 @@ async def create_classify(
|
||||
request: ClassificationRequest, raw_request: Request
|
||||
) -> Response:
|
||||
handler = classify(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Classification API")
|
||||
|
||||
return await handler(request, raw_request)
|
||||
|
||||
@@ -18,8 +18,11 @@ from .serving import ServingEmbedding
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def embedding(request: Request) -> ServingEmbedding | None:
|
||||
return request.app.state.serving_embedding
|
||||
def embedding(request: Request) -> ServingEmbedding:
|
||||
handler = getattr(request.app.state, "serving_embedding", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Embeddings API")
|
||||
return handler
|
||||
|
||||
|
||||
@router.post(
|
||||
@@ -37,9 +40,6 @@ async def create_embedding(
|
||||
raw_request: Request,
|
||||
):
|
||||
handler = embedding(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Embeddings API")
|
||||
|
||||
return await handler(request, raw_request)
|
||||
|
||||
|
||||
@@ -58,7 +58,4 @@ async def create_cohere_embedding(
|
||||
raw_request: Request,
|
||||
):
|
||||
handler = embedding(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Embeddings API")
|
||||
|
||||
return await handler(request, raw_request)
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
CustomChatCompletionMessageParam,
|
||||
)
|
||||
from vllm.inputs import EngineInput, tokens_input
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||
from vllm.renderers import merge_kwargs
|
||||
@@ -28,11 +28,14 @@ from vllm.utils.mistral import is_mistral_tokenizer
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..scoring.io_processor import JinaRankingIOProcessorMixin
|
||||
from ..typing import (
|
||||
ALLOfflineInputsContext,
|
||||
AnyOfflineInputsContext,
|
||||
AnyRenderParam,
|
||||
ChunkedEmbeddingMetadata,
|
||||
EncodeChatRenderParams,
|
||||
OfflineEncodeInputsContext,
|
||||
PoolingChatLikeRequest,
|
||||
PoolingCompletionLikeRequest,
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
)
|
||||
@@ -76,9 +79,11 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
list(self.task_instructions.keys()),
|
||||
)
|
||||
|
||||
def pre_process_online(self, ctx: PoolingServeContext):
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
if isinstance(ctx.request, CohereEmbedRequest):
|
||||
self._pre_process_cohere_online(ctx)
|
||||
requests = self._get_request_factory_cohere_online(ctx)
|
||||
elif isinstance(
|
||||
ctx.request,
|
||||
(
|
||||
@@ -88,12 +93,11 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
EmbeddingBatchChatInputRequest,
|
||||
),
|
||||
):
|
||||
self._pre_process_openai_chat_online(ctx)
|
||||
requests = self._get_request_factory_chat_input_online(ctx)
|
||||
else:
|
||||
super().pre_process_online(ctx)
|
||||
requests = super().get_request_factory_online(ctx)
|
||||
|
||||
if self.enable_chunked_processing:
|
||||
self._pre_process_chunked(ctx)
|
||||
return requests
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
@@ -114,18 +118,21 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
# PTAL: examples/pooling/embed/openai_embedding_long_text
|
||||
#################################################################
|
||||
|
||||
def _pre_process_chunked(self, ctx: PoolingServeContext) -> None:
|
||||
def maybe_pre_process_chunked(self, ctx: PoolingServeContext) -> None:
|
||||
if not self.enable_chunked_processing:
|
||||
return None
|
||||
|
||||
if ctx.engine_inputs is None:
|
||||
raise ValueError("Engine prompts not available")
|
||||
|
||||
ctx.original_engine_inputs = ctx.engine_inputs
|
||||
request_id = ctx.request_id
|
||||
max_model_len = self.model_config.max_model_len
|
||||
chunked_engine_inputs: list[EngineInput] = []
|
||||
chunked_engine_inputs: list[PoolingEngineInput] = []
|
||||
prompt_request_ids: list[str] = []
|
||||
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] = []
|
||||
for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
|
||||
token_ids = engine_input.get("prompt_token_ids", None)
|
||||
token_ids = engine_input["prompts"].get("prompt_token_ids", None)
|
||||
if token_ids is None:
|
||||
raise NotImplementedError(
|
||||
"Long Text Embedding with Chunked Processing does "
|
||||
@@ -138,7 +145,12 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
chunk_list(prompt_token_ids, max_model_len)
|
||||
):
|
||||
chunked_engine_inputs.append(
|
||||
tokens_input(prompt_token_ids=chunk_tokens)
|
||||
PoolingEngineInput(
|
||||
prompts=tokens_input(prompt_token_ids=chunk_tokens),
|
||||
params=engine_input["params"],
|
||||
lora_requests=engine_input["lora_requests"],
|
||||
priorities=engine_input["priorities"],
|
||||
)
|
||||
)
|
||||
prompt_request_ids.append(
|
||||
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
|
||||
@@ -234,7 +246,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
|
||||
# Get original prompt token IDs for this prompt
|
||||
original_prompt = original_engine_inputs[prompt_idx]
|
||||
token_ids = original_prompt.get("prompt_token_ids", None)
|
||||
token_ids = original_prompt["prompts"].get("prompt_token_ids", None)
|
||||
if token_ids is None:
|
||||
raise NotImplementedError(
|
||||
"Long Text Embedding with Chunked Processing does "
|
||||
@@ -262,6 +274,68 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
|
||||
return None
|
||||
|
||||
#################################################################
|
||||
# Chat input Request Preprocessing & Postprocessing
|
||||
#################################################################
|
||||
|
||||
def _get_request_factory_chat_input_online(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
renderer = self.renderer
|
||||
|
||||
self._validate_chat_template(
|
||||
request_chat_template=request.chat_template,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=self.trust_request_chat_template,
|
||||
)
|
||||
|
||||
if isinstance(
|
||||
request, (EmbeddingBatchChatRequest, EmbeddingBatchChatInputRequest)
|
||||
):
|
||||
all_messages = request.messages
|
||||
else:
|
||||
all_messages = [request.messages]
|
||||
num_requests = len(all_messages)
|
||||
|
||||
default_template_kwargs = merge_kwargs(
|
||||
self.template_kwargs,
|
||||
dict(
|
||||
tools=self.tool_dicts,
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
),
|
||||
)
|
||||
|
||||
mm_config = self.model_config.multimodal_config
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
self.chat_template, self.chat_template_content_format
|
||||
).with_defaults(
|
||||
default_template_kwargs,
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
)
|
||||
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeChatRenderParams(
|
||||
conversations=all_messages[i],
|
||||
chat_params=chat_params,
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
return requests
|
||||
|
||||
#################################################################
|
||||
# Cohere Request Preprocessing & Postprocessing
|
||||
#################################################################
|
||||
@@ -378,71 +452,9 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
)
|
||||
return super().create_pooling_params(request)
|
||||
|
||||
def _pre_process_openai_chat_online(
|
||||
self,
|
||||
ctx: PoolingServeContext[
|
||||
EmbeddingChatRequest
|
||||
| EmbeddingBatchChatRequest
|
||||
| EmbeddingChatInputRequest
|
||||
| EmbeddingBatchChatInputRequest
|
||||
],
|
||||
) -> None:
|
||||
request = ctx.request
|
||||
self._validate_chat_template(
|
||||
request_chat_template=request.chat_template,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=self.trust_request_chat_template,
|
||||
)
|
||||
|
||||
if isinstance(
|
||||
request, (EmbeddingBatchChatRequest, EmbeddingBatchChatInputRequest)
|
||||
):
|
||||
all_messages = request.messages
|
||||
else:
|
||||
all_messages = [request.messages]
|
||||
ctx.engine_inputs = self._batch_render_openai_chat(request, all_messages)
|
||||
|
||||
def _batch_render_openai_chat(
|
||||
self,
|
||||
request: (
|
||||
EmbeddingChatRequest
|
||||
| EmbeddingBatchChatRequest
|
||||
| EmbeddingChatInputRequest
|
||||
| EmbeddingBatchChatInputRequest
|
||||
),
|
||||
all_messages: Sequence[list[ChatCompletionMessageParam]],
|
||||
) -> list[EngineInput]:
|
||||
renderer = self.renderer
|
||||
mm_config = self.model_config.multimodal_config
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
self.chat_template,
|
||||
self.chat_template_content_format,
|
||||
).with_defaults(
|
||||
merge_kwargs(
|
||||
None,
|
||||
dict(
|
||||
tools=None,
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
),
|
||||
),
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
)
|
||||
|
||||
_, engine_inputs = renderer.render_chat(
|
||||
all_messages,
|
||||
chat_params,
|
||||
tok_params,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
)
|
||||
return engine_inputs
|
||||
|
||||
def _pre_process_cohere_online(self, ctx: PoolingServeContext) -> None:
|
||||
def _get_request_factory_cohere_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
"""Convert a ``CohereEmbedRequest`` into engine prompts.
|
||||
|
||||
If a model has a chat template the task instruction are rendered
|
||||
@@ -479,13 +491,12 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
task_prefix = self._get_task_instruction_prefix(input_type)
|
||||
|
||||
if task_prefix is None:
|
||||
ctx.engine_inputs = self._preprocess_cohere_text_completion(
|
||||
request,
|
||||
return self._get_request_factory_cohere_text_completion(
|
||||
ctx,
|
||||
texts,
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
return
|
||||
|
||||
all_messages = [
|
||||
self._mixed_input_to_messages(
|
||||
@@ -497,27 +508,30 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
for text in texts
|
||||
]
|
||||
if self._has_chat_template():
|
||||
ctx.engine_inputs = self._batch_render_chat(
|
||||
request,
|
||||
return self._batch_render_chat(
|
||||
ctx,
|
||||
all_messages,
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
else:
|
||||
ctx.engine_inputs = self._preprocess_cohere_text_completion(
|
||||
request,
|
||||
return self._get_request_factory_cohere_text_completion(
|
||||
ctx,
|
||||
self._apply_task_instruction(texts, input_type),
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
return
|
||||
|
||||
task_prefix = self._get_task_instruction_prefix(input_type)
|
||||
all_messages = [
|
||||
self._mixed_input_to_messages(inp, task_prefix=task_prefix) for inp in input
|
||||
]
|
||||
ctx.engine_inputs = self._batch_render_chat(
|
||||
request, all_messages, truncate_prompt_tokens, truncation_side
|
||||
|
||||
return self._batch_render_chat(
|
||||
ctx,
|
||||
all_messages,
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
|
||||
def _has_chat_template(self) -> bool:
|
||||
@@ -531,13 +545,14 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
is not None
|
||||
)
|
||||
|
||||
def _preprocess_cohere_text_completion(
|
||||
def _get_request_factory_cohere_text_completion(
|
||||
self,
|
||||
request: CohereEmbedRequest,
|
||||
ctx: PoolingServeContext,
|
||||
texts: list[str],
|
||||
truncate_prompt_tokens: int | None,
|
||||
truncation_side: Literal["left", "right"] | None,
|
||||
) -> list[EngineInput]:
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
proxy = EmbeddingCompletionRequest(
|
||||
model=request.model,
|
||||
input=texts,
|
||||
@@ -546,50 +561,32 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
truncation_side=truncation_side,
|
||||
)
|
||||
return self._preprocess_cmpl_online(
|
||||
proxy, prompt_input=proxy.input, prompt_embeds=None
|
||||
)
|
||||
ctx.request = proxy
|
||||
requests = super().get_request_factory_online(ctx)
|
||||
ctx.request = request
|
||||
return requests
|
||||
|
||||
def _batch_render_chat(
|
||||
self,
|
||||
request: CohereEmbedRequest,
|
||||
ctx: PoolingServeContext,
|
||||
all_messages: Sequence[list[ChatCompletionMessageParam]],
|
||||
truncate_prompt_tokens: int | None,
|
||||
truncation_side: Literal["left", "right"] | None,
|
||||
) -> list[EngineInput]:
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
"""Batch-render multiple conversations through the chat template."""
|
||||
if not all_messages:
|
||||
return []
|
||||
|
||||
proxy = EmbeddingChatRequest(
|
||||
request = ctx.request
|
||||
proxy = EmbeddingBatchChatRequest(
|
||||
model=request.model,
|
||||
messages=list(all_messages[0]),
|
||||
messages=all_messages,
|
||||
dimensions=request.output_dimension,
|
||||
encoding_format="float",
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
truncation_side=truncation_side,
|
||||
)
|
||||
|
||||
renderer = self.renderer
|
||||
mm_config = self.model_config.multimodal_config
|
||||
|
||||
tok_params = proxy.build_tok_params(self.model_config)
|
||||
chat_params = proxy.build_chat_params(
|
||||
self.chat_template,
|
||||
self.chat_template_content_format,
|
||||
).with_defaults(
|
||||
merge_kwargs(
|
||||
None,
|
||||
dict(
|
||||
tools=None,
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
),
|
||||
),
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
)
|
||||
|
||||
_, engine_inputs = renderer.render_chat(all_messages, chat_params, tok_params)
|
||||
return engine_inputs
|
||||
ctx.request = proxy
|
||||
requests = self._get_request_factory_chat_input_online(ctx)
|
||||
ctx.request = request
|
||||
return requests
|
||||
|
||||
def _validate_input_type(self, input_type: str | None) -> None:
|
||||
"""Raise if *input_type* is not supported by this model."""
|
||||
@@ -640,7 +637,9 @@ class TokenEmbedIOProcessor(PoolingIOProcessor):
|
||||
class JinaRankingTokenEmbedIOProcessor(
|
||||
TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
|
||||
):
|
||||
def pre_process_online(self, ctx: PoolingServeContext):
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
if isinstance(request, PoolingCompletionLikeRequest):
|
||||
prompts = request.input
|
||||
@@ -655,20 +654,15 @@ class JinaRankingTokenEmbedIOProcessor(
|
||||
query=text_prompts[-1], docs=text_prompts[:-1]
|
||||
)
|
||||
|
||||
engine_inputs = self._preprocess_cmpl_online(
|
||||
request,
|
||||
prompt_input=prompt_input,
|
||||
prompt_embeds=None,
|
||||
)
|
||||
request.input = prompt_input
|
||||
return super().get_request_factory_online(ctx)
|
||||
elif isinstance(request, PoolingChatLikeRequest):
|
||||
raise ValueError("The JinaForRanking does not support chat Request.")
|
||||
else:
|
||||
raise ValueError(f"Invalid {self.name} request type")
|
||||
|
||||
ctx.engine_inputs = engine_inputs
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: ALLOfflineInputsContext
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineEncodeInputsContext)
|
||||
if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2:
|
||||
|
||||
@@ -93,9 +93,9 @@ class EmbeddingBatchChatRequest(
|
||||
``messages`` instead of introducing a separate batch-specific field.
|
||||
"""
|
||||
|
||||
messages: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = (
|
||||
Field(..., min_length=1)
|
||||
)
|
||||
messages: Sequence[
|
||||
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
|
||||
] = Field(..., min_length=1)
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
@@ -133,9 +133,9 @@ class EmbeddingChatInputRequest(
|
||||
class EmbeddingBatchChatInputRequest(EmbeddingBatchChatRequest):
|
||||
"""OpenAI embeddings request with batched chat conversations in ``input``."""
|
||||
|
||||
input: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = (
|
||||
Field(..., min_length=1)
|
||||
)
|
||||
input: Sequence[
|
||||
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
|
||||
] = Field(..., min_length=1)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
|
||||
@@ -9,6 +9,7 @@ from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.utils.serial_utils import EmbedDType, Endianness
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..base.serving import PoolingServing
|
||||
from ..typing import PoolingServeContext
|
||||
from ..utils import (
|
||||
@@ -53,6 +54,12 @@ class ServingEmbedding(PoolingServing):
|
||||
def init_io_processor(self, *args, **kwargs) -> EmbedIOProcessor:
|
||||
return EmbedIOProcessor(*args, **kwargs)
|
||||
|
||||
async def _preprocessing(
|
||||
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
|
||||
):
|
||||
await super()._preprocessing(io_processor, ctx)
|
||||
self.io_processor.maybe_pre_process_chunked(ctx)
|
||||
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
|
||||
@@ -25,7 +25,7 @@ from .factories import init_pooling_io_processors
|
||||
from .scoring.io_processor import ScoringIOProcessor
|
||||
from .scoring.typing import ScoreInput
|
||||
from .typing import (
|
||||
ALLOfflineInputsContext,
|
||||
AnyOfflineInputsContext,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
OfflinePluginInputsContext,
|
||||
@@ -104,7 +104,7 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
|
||||
io_processor = self.pooling_io_processors[pooling_task]
|
||||
|
||||
ctx: ALLOfflineInputsContext
|
||||
ctx: AnyOfflineInputsContext
|
||||
if isinstance(prompts, dict) and "data" in prompts:
|
||||
ctx = OfflinePluginInputsContext(
|
||||
pooling_task=pooling_task,
|
||||
@@ -399,6 +399,9 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
num_requests: int,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
):
|
||||
if num_requests == 0:
|
||||
raise ValueError("You must pass at least one prompt")
|
||||
|
||||
# Keeping max_num_seqs * 2 requests in the core can already saturate the core.
|
||||
# Therefore, keep most requests waiting outside the core.
|
||||
max_requests_in_core = (
|
||||
|
||||
@@ -17,8 +17,11 @@ from .serving import ServingPooling
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def pooling(request: Request) -> ServingPooling | None:
|
||||
return request.app.state.serving_pooling
|
||||
def pooling(request: Request) -> ServingPooling:
|
||||
handler = getattr(request.app.state, "serving_pooling", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Pooling API")
|
||||
return handler
|
||||
|
||||
|
||||
@router.post(
|
||||
@@ -33,7 +36,4 @@ def pooling(request: Request) -> ServingPooling | None:
|
||||
@load_aware_call
|
||||
async def create_pooling(request: PoolingRequest, raw_request: Request):
|
||||
handler = pooling(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Pooling API")
|
||||
|
||||
return await handler(request, raw_request)
|
||||
|
||||
@@ -10,7 +10,9 @@ from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..typing import (
|
||||
ALLOfflineInputsContext,
|
||||
AnyOfflineInputsContext,
|
||||
AnyRenderParam,
|
||||
EncodeCMPLRenderParams,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
OfflinePluginInputsContext,
|
||||
@@ -49,7 +51,9 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
|
||||
#######################################
|
||||
# online APIs
|
||||
|
||||
def pre_process_online(self, ctx: PoolingServeContext):
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
assert isinstance(ctx.request, IOProcessorRequest)
|
||||
|
||||
validated_prompt = self.io_processor.parse_data(ctx.request.data)
|
||||
@@ -66,24 +70,33 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
|
||||
)
|
||||
for prompt in prompt_to_seq(raw_prompts)
|
||||
]
|
||||
num_requests = len(parsed_prompts)
|
||||
|
||||
tok_params = ctx.request.build_tok_params(self.model_config)
|
||||
|
||||
ctx.engine_inputs = self.renderer.render_cmpl(
|
||||
parsed_prompts,
|
||||
tok_params,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(ctx.request, k, None)) is not None
|
||||
},
|
||||
)
|
||||
|
||||
pooling_params = self.io_processor.merge_pooling_params()
|
||||
if pooling_params.task is None:
|
||||
pooling_params.task = "plugin"
|
||||
ctx.pooling_params = pooling_params
|
||||
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeCMPLRenderParams(
|
||||
prompts=parsed_prompts[i],
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
return requests
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
@@ -114,7 +127,7 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
|
||||
# offline APIs
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: ALLOfflineInputsContext
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflinePluginInputsContext)
|
||||
assert isinstance(ctx.prompts, dict) and "data" in ctx.prompts
|
||||
|
||||
@@ -20,12 +20,18 @@ router = APIRouter()
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def score(request: Request) -> ServingScores | None:
|
||||
return request.app.state.serving_scores
|
||||
def score(request: Request) -> ServingScores:
|
||||
handler = getattr(request.app.state, "serving_scores", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Score API")
|
||||
return handler
|
||||
|
||||
|
||||
def rerank(request: Request) -> ServingScores | None:
|
||||
return request.app.state.serving_scores
|
||||
def rerank(request: Request) -> ServingScores:
|
||||
handler = getattr(request.app.state, "serving_scores", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Rerank (Score) API")
|
||||
return handler
|
||||
|
||||
|
||||
@router.post(
|
||||
@@ -40,9 +46,6 @@ def rerank(request: Request) -> ServingScores | None:
|
||||
@load_aware_call
|
||||
async def create_score(request: ScoreRequest, raw_request: Request):
|
||||
handler = score(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Score API")
|
||||
|
||||
return await handler(request, raw_request)
|
||||
|
||||
|
||||
@@ -77,9 +80,6 @@ async def create_score_v1(request: ScoreRequest, raw_request: Request):
|
||||
@load_aware_call
|
||||
async def do_rerank(request: RerankRequest, raw_request: Request):
|
||||
handler = rerank(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Rerank (Score) API")
|
||||
|
||||
return await handler(request, raw_request)
|
||||
|
||||
|
||||
|
||||
@@ -6,8 +6,7 @@ from typing import Any, TypeAlias, cast
|
||||
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import PoolingParams, PoolingRequestOutput, PromptType, TokensPrompt
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.renderers.hf import safe_apply_chat_template
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
@@ -20,8 +19,11 @@ from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ...chat_utils import ChatTemplateResolutionError
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..pooling.protocol import PoolingCompletionRequest
|
||||
from ..typing import (
|
||||
ALLOfflineInputsContext,
|
||||
AnyOfflineInputsContext,
|
||||
AnyPoolingRequest,
|
||||
AnyRenderParam,
|
||||
EncodeChatRenderParams,
|
||||
EncodeCMPLRenderParams,
|
||||
OfflineEncodeInputsContext,
|
||||
@@ -167,6 +169,19 @@ class ScoringIOProcessor(PoolingIOProcessor):
|
||||
)
|
||||
return scoring_data
|
||||
|
||||
def valid_inputs_online(self, request: AnyPoolingRequest):
|
||||
if isinstance(request, ScoreRequest):
|
||||
data_1 = request.data_1
|
||||
data_2 = request.data_2
|
||||
elif isinstance(request, RerankRequest):
|
||||
data_1 = request.query
|
||||
data_2 = request.documents
|
||||
else:
|
||||
raise ValueError(f"Invalid {request.__class__.__name__} request type")
|
||||
|
||||
scoring_data = self.valid_inputs(data_1, data_2)
|
||||
return scoring_data
|
||||
|
||||
|
||||
class BiEncoderIOProcessor(ScoringIOProcessor):
|
||||
name = "bi-encoder"
|
||||
@@ -175,19 +190,11 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
|
||||
#######################################
|
||||
# online APIs
|
||||
|
||||
def pre_process_online(self, ctx: ScoringServeContext):
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
|
||||
if isinstance(request, ScoreRequest):
|
||||
data_1 = request.data_1
|
||||
data_2 = request.data_2
|
||||
elif isinstance(request, RerankRequest):
|
||||
data_1 = request.query
|
||||
data_2 = request.documents
|
||||
else:
|
||||
raise ValueError(f"Invalid {self.name} request type")
|
||||
|
||||
scoring_data = self.valid_inputs(data_1, data_2)
|
||||
scoring_data = self.valid_inputs_online(request)
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
request=request
|
||||
@@ -197,19 +204,37 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
|
||||
scoring_data, max_tokens_per_query, max_tokens_per_doc
|
||||
)
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
engine_inputs = self._pre_process(
|
||||
scoring_data,
|
||||
tok_params,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
|
||||
data_2 = score_data_to_prompts(
|
||||
scoring_data.data_2, "document", self.model_config
|
||||
)
|
||||
prompts = data_1 + data_2
|
||||
ctx.n_queries = len(data_1)
|
||||
|
||||
ctx.engine_inputs = engine_inputs
|
||||
ctx.n_queries = len(scoring_data.data_1)
|
||||
prompts_seq = prompt_to_seq(prompts)
|
||||
parsed_prompts = [
|
||||
parse_model_prompt(self.model_config, prompt) for prompt in prompts_seq
|
||||
]
|
||||
num_requests = len(parsed_prompts)
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeCMPLRenderParams(
|
||||
prompts=parsed_prompts[i],
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
return requests
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
@@ -226,7 +251,7 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
|
||||
# offline APIs
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: ALLOfflineInputsContext
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineScoringInputsContext)
|
||||
|
||||
@@ -267,41 +292,6 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
|
||||
#######################################
|
||||
# helpers
|
||||
|
||||
def _pre_process(
|
||||
self,
|
||||
scoring_data: ScoringData,
|
||||
tok_params: TokenizeParams,
|
||||
prompt_extras: dict[str, Any] | None = None,
|
||||
) -> Sequence[EngineInput]:
|
||||
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
|
||||
data_2 = score_data_to_prompts(
|
||||
scoring_data.data_2, "document", self.model_config
|
||||
)
|
||||
|
||||
return self._preprocess_cmpl_offline(
|
||||
prompts=data_1 + data_2, tok_params=tok_params, prompt_extras=prompt_extras
|
||||
)
|
||||
|
||||
def _preprocess_cmpl_offline(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
tok_params: TokenizeParams,
|
||||
prompt_extras: dict[str, Any] | None = None,
|
||||
) -> Sequence[EngineInput]:
|
||||
prompts = prompt_to_seq(prompts)
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(self.model_config, prompt)
|
||||
)
|
||||
for prompt in prompts
|
||||
]
|
||||
|
||||
return self.renderer.render_cmpl(
|
||||
parsed_prompts, tok_params, prompt_extras=prompt_extras
|
||||
)
|
||||
|
||||
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
|
||||
emb_data_1 = outputs[:n_queries]
|
||||
emb_data_2 = outputs[n_queries:]
|
||||
@@ -432,49 +422,50 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
#######################################
|
||||
# online APIs
|
||||
|
||||
def pre_process_online(self, ctx: ScoringServeContext):
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
scoring_data = self.valid_inputs_online(request)
|
||||
data_1 = scoring_data.data_1
|
||||
data_2 = scoring_data.data_2
|
||||
num_requests = len(data_2)
|
||||
|
||||
if isinstance(request, ScoreRequest):
|
||||
data_1 = request.data_1
|
||||
data_2 = request.data_2
|
||||
elif isinstance(request, RerankRequest):
|
||||
data_1 = request.query
|
||||
data_2 = request.documents
|
||||
else:
|
||||
raise ValueError(f"Invalid {self.name} request type")
|
||||
|
||||
scoring_data = self.valid_inputs(data_1, data_2)
|
||||
if len(data_1) == 1:
|
||||
data_1 = data_1 * num_requests
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
request=request
|
||||
)
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
pooling_params = self.create_pooling_params(request)
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
engine_inputs, pooling_params_list = self._pre_process(
|
||||
scoring_data,
|
||||
tok_params,
|
||||
pooling_params,
|
||||
chat_template=self.chat_template,
|
||||
max_tokens_per_query=max_tokens_per_query,
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
)
|
||||
requests = [
|
||||
ScoringRenderParams(
|
||||
data_1=data_1[i],
|
||||
data_2=data_2[i],
|
||||
chat_template=self.chat_template,
|
||||
max_tokens_per_query=max_tokens_per_query,
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=ctx.pooling_params,
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
ctx.engine_inputs = engine_inputs
|
||||
ctx.pooling_params = pooling_params_list
|
||||
return requests
|
||||
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: ALLOfflineInputsContext
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineScoringInputsContext)
|
||||
|
||||
@@ -485,10 +476,12 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
if len(data_1) == 1:
|
||||
data_1 = data_1 * num_requests
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
pooling_params=ctx.pooling_params
|
||||
)
|
||||
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
|
||||
**(ctx.tokenization_kwargs or {})
|
||||
)
|
||||
|
||||
prompt_extras = ctx.pooling_params.extra_kwargs
|
||||
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
@@ -500,6 +493,8 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
data_1=data_1[i],
|
||||
data_2=data_2[i],
|
||||
chat_template=ctx.chat_template,
|
||||
max_tokens_per_query=max_tokens_per_query,
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
tok_params=tok_params,
|
||||
prompt_extras=prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
@@ -531,17 +526,13 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
params = render_params["params"]
|
||||
prompt_extras = render_params["prompt_extras"]
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
pooling_params=params
|
||||
)
|
||||
|
||||
_, engine_prompt = self.get_score_prompt(
|
||||
data_1=render_params["data_1"],
|
||||
data_2=render_params["data_2"],
|
||||
encode_kwargs=tok_params.get_encode_kwargs(),
|
||||
chat_template=render_params["chat_template"],
|
||||
max_tokens_per_query=max_tokens_per_query,
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
max_tokens_per_query=render_params["max_tokens_per_query"],
|
||||
max_tokens_per_doc=render_params["max_tokens_per_doc"],
|
||||
chat_template_kwargs=prompt_extras.get("chat_template_kwargs")
|
||||
if prompt_extras
|
||||
else None,
|
||||
@@ -551,29 +542,16 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
|
||||
if token_type_ids := engine_prompt.pop("token_type_ids", None):
|
||||
params = params.clone()
|
||||
compressed = compress_token_type_ids(token_type_ids)
|
||||
params.extra_kwargs = {"compressed_token_type_ids": compressed}
|
||||
compressed = compress_token_type_ids(
|
||||
_apply_post_tokenization_to_token_type_ids(
|
||||
self.tokenizer, tok_params, token_type_ids
|
||||
)
|
||||
)
|
||||
params.extra_kwargs = {
|
||||
**(params.extra_kwargs or {}),
|
||||
"compressed_token_type_ids": compressed,
|
||||
}
|
||||
|
||||
engine_input = self.renderer.process_for_engine(engine_prompt, arrival_time)
|
||||
|
||||
return PoolingEngineInput(
|
||||
prompts=engine_input,
|
||||
params=params,
|
||||
lora_requests=render_params["lora_requests"],
|
||||
priorities=render_params["priorities"],
|
||||
)
|
||||
|
||||
def _pre_process(
|
||||
self,
|
||||
scoring_data: ScoringData,
|
||||
tok_params: TokenizeParams,
|
||||
pooling_params: PoolingParams | None,
|
||||
chat_template: str | None = None,
|
||||
max_tokens_per_query: int = 0,
|
||||
max_tokens_per_doc: int = 0,
|
||||
prompt_extras: dict[str, Any] | None = None,
|
||||
) -> tuple[Sequence[EngineInput], list[PoolingParams]]:
|
||||
arrival_time = time.time()
|
||||
engine_prompt_extras = (
|
||||
{
|
||||
k: v
|
||||
@@ -584,55 +562,18 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
else None
|
||||
)
|
||||
|
||||
data_1 = scoring_data.data_1
|
||||
data_2 = scoring_data.data_2
|
||||
if engine_prompt_extras:
|
||||
target_prompt = extract_target_prompt(self.model_config, engine_prompt)
|
||||
target_prompt.update(engine_prompt_extras)
|
||||
|
||||
if len(data_1) == 1:
|
||||
data_1 = data_1 * len(data_2)
|
||||
engine_input = self.renderer.process_for_engine(engine_prompt, arrival_time)
|
||||
|
||||
if pooling_params is None:
|
||||
pooling_params = PoolingParams(task="classify")
|
||||
|
||||
pooling_params_list = list[PoolingParams]()
|
||||
engine_inputs = list[EngineInput]()
|
||||
for q, d in zip(data_1, data_2):
|
||||
_, engine_prompt = self.get_score_prompt(
|
||||
data_1=q,
|
||||
data_2=d,
|
||||
encode_kwargs=tok_params.get_encode_kwargs(),
|
||||
chat_template=chat_template,
|
||||
max_tokens_per_query=max_tokens_per_query,
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
chat_template_kwargs=prompt_extras.get("chat_template_kwargs")
|
||||
if prompt_extras
|
||||
else None,
|
||||
)
|
||||
|
||||
token_type_ids = engine_prompt.pop("token_type_ids", None)
|
||||
tok_params.apply_post_tokenization(self.tokenizer, engine_prompt)
|
||||
|
||||
if token_type_ids is not None:
|
||||
params = pooling_params.clone()
|
||||
compressed = compress_token_type_ids(
|
||||
_apply_post_tokenization_to_token_type_ids(
|
||||
self.tokenizer, tok_params, token_type_ids
|
||||
)
|
||||
)
|
||||
params.extra_kwargs = {
|
||||
**(params.extra_kwargs or {}),
|
||||
"compressed_token_type_ids": compressed,
|
||||
}
|
||||
pooling_params_list.append(params)
|
||||
else:
|
||||
pooling_params_list.append(pooling_params)
|
||||
|
||||
if engine_prompt_extras:
|
||||
target_prompt = extract_target_prompt(self.model_config, engine_prompt)
|
||||
target_prompt.update(engine_prompt_extras)
|
||||
engine_inputs.append(
|
||||
self.renderer.process_for_engine(engine_prompt, arrival_time)
|
||||
)
|
||||
return engine_inputs, pooling_params_list
|
||||
return PoolingEngineInput(
|
||||
prompts=engine_input,
|
||||
params=params,
|
||||
lora_requests=render_params["lora_requests"],
|
||||
priorities=render_params["priorities"],
|
||||
)
|
||||
|
||||
def get_score_prompt(
|
||||
self,
|
||||
@@ -837,8 +778,55 @@ class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorM
|
||||
name = "jina-reranking-scoring"
|
||||
pooling_task: PoolingTask = "token_embed"
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
ctx.n_queries = 1
|
||||
|
||||
prompt_extras = ctx.prompt_extras
|
||||
scoring_data = self.valid_inputs_online(request)
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
request=request
|
||||
)
|
||||
|
||||
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
|
||||
scoring_data = self._truncate_scoring_data(
|
||||
scoring_data, max_tokens_per_query, max_tokens_per_doc
|
||||
)
|
||||
|
||||
queries = self.ensure_str(scoring_data.data_1)
|
||||
docs = self.ensure_str(scoring_data.data_2)
|
||||
|
||||
chat_template_kwargs = (
|
||||
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
|
||||
)
|
||||
instruction = (
|
||||
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
|
||||
)
|
||||
|
||||
if len(queries) == 1:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(
|
||||
query=queries[0], docs=docs, instruction=instruction
|
||||
)
|
||||
]
|
||||
else:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(
|
||||
query=q, docs=[d], instruction=instruction
|
||||
)
|
||||
for q, d in zip(queries, docs)
|
||||
]
|
||||
|
||||
ctx.request = PoolingCompletionRequest(task="token_embed", input=prompts)
|
||||
requests = PoolingIOProcessor.get_request_factory_online(self, ctx)
|
||||
ctx.request = request
|
||||
return requests
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: ALLOfflineInputsContext
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineScoringInputsContext)
|
||||
|
||||
@@ -880,39 +868,6 @@ class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorM
|
||||
),
|
||||
)
|
||||
|
||||
def _pre_process(
|
||||
self,
|
||||
scoring_data: ScoringData,
|
||||
tok_params: TokenizeParams,
|
||||
prompt_extras: dict[str, Any] | None = None,
|
||||
) -> Sequence[EngineInput]:
|
||||
queries = self.ensure_str(scoring_data.data_1)
|
||||
docs = self.ensure_str(scoring_data.data_2)
|
||||
chat_template_kwargs = (
|
||||
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
|
||||
)
|
||||
instruction = (
|
||||
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
|
||||
)
|
||||
|
||||
if len(queries) == 1:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(
|
||||
query=queries[0], docs=docs, instruction=instruction
|
||||
)
|
||||
]
|
||||
else:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(
|
||||
query=q, docs=[d], instruction=instruction
|
||||
)
|
||||
for q, d in zip(queries, docs)
|
||||
]
|
||||
|
||||
return self._preprocess_cmpl_offline(
|
||||
prompts=prompts, tok_params=tok_params, prompt_extras=prompt_extras
|
||||
)
|
||||
|
||||
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
|
||||
|
||||
@@ -190,7 +190,7 @@ class ServingScores(PoolingServing):
|
||||
|
||||
async def flash_late_interaction(self, *args, **kwargs) -> Response:
|
||||
ctx = await self._init_ctx(self.io_processor, *args, **kwargs)
|
||||
await self._preprocessing_async(self.io_processor, ctx)
|
||||
await self._preprocessing(self.io_processor, ctx)
|
||||
|
||||
# stage 1: encode queries and cache token embeddings on workers.
|
||||
await self._flash_late_interaction_encode_queries(ctx)
|
||||
@@ -211,7 +211,6 @@ class ServingScores(PoolingServing):
|
||||
query_keys = [f"{ctx.request_id}-query-{i}" for i in range(n_queries)]
|
||||
query_uses = [n_docs if n_queries == 1 else 1] * n_queries
|
||||
|
||||
query_pooling_params_list = []
|
||||
for i in range(n_queries):
|
||||
pooling_params = ctx.pooling_params.clone()
|
||||
pooling_params.late_interaction_params = (
|
||||
@@ -220,23 +219,21 @@ class ServingScores(PoolingServing):
|
||||
query_uses=query_uses[i],
|
||||
)
|
||||
)
|
||||
query_pooling_params_list.append(pooling_params)
|
||||
query_engine_inputs[i]["params"] = pooling_params
|
||||
|
||||
assert (
|
||||
n_queries
|
||||
== len(query_pooling_params_list)
|
||||
== len(query_engine_inputs)
|
||||
== len(query_keys)
|
||||
)
|
||||
assert n_queries == len(query_engine_inputs) == len(query_keys)
|
||||
|
||||
query_ctx = ScoringServeContext(
|
||||
request=ctx.request,
|
||||
raw_request=ctx.raw_request,
|
||||
model_name=ctx.model_name,
|
||||
request_id=ctx.request_id,
|
||||
pooling_params=query_pooling_params_list,
|
||||
pooling_params=ctx.pooling_params,
|
||||
prompt_request_ids=query_keys,
|
||||
engine_inputs=query_engine_inputs,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
)
|
||||
|
||||
await self._prepare_generators(query_ctx)
|
||||
@@ -255,30 +252,27 @@ class ServingScores(PoolingServing):
|
||||
query_keys = [f"{ctx.request_id}-query-{i}" for i in range(n_queries)]
|
||||
doc_keys = [f"{ctx.request_id}-doc-{i}" for i in range(n_docs)]
|
||||
|
||||
doc_pooling_params_list = []
|
||||
for i in range(n_docs):
|
||||
query_idx = 0 if n_queries == 1 else i
|
||||
pooling_params = ctx.pooling_params.clone()
|
||||
pooling_params.late_interaction_params = build_late_interaction_doc_params(
|
||||
query_key=query_keys[query_idx]
|
||||
)
|
||||
doc_pooling_params_list.append(pooling_params)
|
||||
doc_engine_inputs[i]["params"] = pooling_params
|
||||
|
||||
assert (
|
||||
n_docs
|
||||
== len(doc_pooling_params_list)
|
||||
== len(doc_engine_inputs)
|
||||
== len(doc_keys)
|
||||
)
|
||||
assert n_docs == len(doc_engine_inputs) == len(doc_keys)
|
||||
|
||||
doc_ctx = ScoringServeContext(
|
||||
request=ctx.request,
|
||||
raw_request=ctx.raw_request,
|
||||
model_name=ctx.model_name,
|
||||
request_id=ctx.request_id,
|
||||
pooling_params=doc_pooling_params_list,
|
||||
pooling_params=ctx.pooling_params,
|
||||
prompt_request_ids=doc_keys,
|
||||
engine_inputs=doc_engine_inputs,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
)
|
||||
|
||||
await self._prepare_generators(doc_ctx)
|
||||
|
||||
@@ -88,10 +88,13 @@ class PoolingServeContext(Generic[PoolingRequestT]):
|
||||
raw_request: Request | None = None
|
||||
model_name: str
|
||||
request_id: str
|
||||
pooling_params: PoolingParams | list[PoolingParams]
|
||||
pooling_params: PoolingParams
|
||||
lora_request: LoRARequest | None
|
||||
priorities: int | Sequence[int] | None
|
||||
prompt_extras: dict[str, Any] | None
|
||||
|
||||
created_time: int = field(default_factory=lambda: int(time.time()))
|
||||
lora_request: LoRARequest | None = None
|
||||
engine_inputs: Sequence[EngineInput] | None = None
|
||||
engine_inputs: Sequence["PoolingEngineInput"] | None = None
|
||||
prompt_request_ids: list[str] | None = None
|
||||
|
||||
result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (
|
||||
@@ -100,7 +103,7 @@ class PoolingServeContext(Generic[PoolingRequestT]):
|
||||
final_res_batch: list[PoolingRequestOutput] = field(default_factory=list)
|
||||
|
||||
## for Long Text Embedding with Chunked Processing
|
||||
original_engine_inputs: Sequence[EngineInput] | None = None
|
||||
original_engine_inputs: Sequence["PoolingEngineInput"] | None = None
|
||||
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] | None = None
|
||||
|
||||
## for bi-encoder & late-interaction
|
||||
@@ -118,7 +121,7 @@ class OfflineInputsContext:
|
||||
pooling_task: PoolingTask
|
||||
tokenization_kwargs: dict[str, Any] | None
|
||||
lora_request: Sequence[LoRARequest | None] | None
|
||||
priorities: Sequence[int] | None
|
||||
priorities: int | Sequence[int] | None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -140,7 +143,7 @@ class OfflinePluginInputsContext(OfflineInputsContext):
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None
|
||||
|
||||
|
||||
ALLOfflineInputsContext: TypeAlias = (
|
||||
AnyOfflineInputsContext: TypeAlias = (
|
||||
OfflineEncodeInputsContext
|
||||
| OfflineScoringInputsContext
|
||||
| OfflinePluginInputsContext
|
||||
@@ -178,6 +181,15 @@ class ScoringRenderParams(RenderParams):
|
||||
data_1: ScoreData
|
||||
data_2: ScoreData
|
||||
chat_template: str | None
|
||||
max_tokens_per_query: int
|
||||
max_tokens_per_doc: int
|
||||
|
||||
|
||||
AnyRenderParam: TypeAlias = (
|
||||
EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams
|
||||
)
|
||||
RequestGenerator: TypeAlias = Generator[AnyRenderParam]
|
||||
RequestFactory: TypeAlias = Callable[[], RequestGenerator]
|
||||
|
||||
|
||||
class PoolingEngineInput(TypedDict):
|
||||
@@ -185,9 +197,3 @@ class PoolingEngineInput(TypedDict):
|
||||
params: PoolingParams
|
||||
lora_requests: LoRARequest | None
|
||||
priorities: int
|
||||
|
||||
|
||||
RequestGenerator: TypeAlias = Generator[
|
||||
EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams
|
||||
]
|
||||
RequestFactory: TypeAlias = Callable[[], RequestGenerator]
|
||||
|
||||
Reference in New Issue
Block a user