Added real /v1/embeddings support for messages + chat_template_kw (#45173)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-06-15 09:08:10 +08:00
committed by GitHub
parent c621af1690
commit 2c764c089a
5 changed files with 400 additions and 10 deletions
@@ -3,6 +3,7 @@
"""Unit tests for EmbedIOProcessor."""
import pytest
from pydantic import TypeAdapter
from vllm import PoolingParams
from vllm.entrypoints.pooling.embed.io_processor import EmbedIOProcessor
@@ -10,10 +11,100 @@ from vllm.entrypoints.pooling.embed.protocol import (
CohereEmbedContent,
CohereEmbedInput,
CohereEmbedRequest,
EmbeddingBatchChatInputRequest,
EmbeddingBatchChatRequest,
EmbeddingChatInputRequest,
EmbeddingChatRequest,
EmbeddingCompletionRequest,
EmbeddingRequest,
)
from vllm.entrypoints.pooling.typing import PoolingServeContext
class TestEmbeddingRequestParsing:
"""Unit tests for OpenAI embedding request parsing."""
def test_input_messages_parses_as_chat_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [{"role": "user", "content": "hello"}],
"chat_template_kwargs": {"instruction": "Represent the query: "},
}
)
assert isinstance(request, EmbeddingChatInputRequest)
assert request.input == [{"role": "user", "content": "hello"}]
assert request.messages == [{"role": "user", "content": "hello"}]
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
def test_batched_input_messages_parses_as_batch_chat_input_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [
[{"role": "user", "content": "hello"}],
[{"role": "user", "content": "goodbye"}],
],
"chat_template_kwargs": {"instruction": "Represent the query: "},
}
)
assert isinstance(request, EmbeddingBatchChatInputRequest)
assert request.input == [
[{"role": "user", "content": "hello"}],
[{"role": "user", "content": "goodbye"}],
]
assert request.messages == [
[{"role": "user", "content": "hello"}],
[{"role": "user", "content": "goodbye"}],
]
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
def test_token_ids_still_parse_as_completion_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [[1, 2, 3], [4, 5]],
}
)
assert isinstance(request, EmbeddingCompletionRequest)
assert request.input == [[1, 2, 3], [4, 5]]
def test_messages_still_parses_as_chat_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"messages": [{"role": "user", "content": "hello"}],
"chat_template_kwargs": {"instruction": "Represent the query: "},
}
)
assert isinstance(request, EmbeddingChatRequest)
assert request.messages == [{"role": "user", "content": "hello"}]
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
def test_batched_messages_parses_as_batch_chat_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"messages": [
[{"role": "user", "content": "hello"}],
[{"role": "user", "content": "goodbye"}],
],
"chat_template_kwargs": {"instruction": "Represent the query: "},
}
)
assert isinstance(request, EmbeddingBatchChatRequest)
assert request.messages == [
[{"role": "user", "content": "hello"}],
[{"role": "user", "content": "goodbye"}],
]
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
class TestResolveTruncation:
"""Unit tests for EmbedIOProcessor._resolve_cohere_truncation."""
@@ -324,3 +415,113 @@ class TestPreProcessCohereOnline:
},
)
]
class TestPreProcessOpenAIEmbeddingChatOnline:
"""Unit tests for OpenAI embedding chat preprocessing."""
class _FakeModelConfig:
max_model_len = 128
encoder_config: dict[str, object] = {}
pooler_config = None
multimodal_config = None
is_encoder_decoder = False
class _FakeRenderer:
tokenizer = object()
def __init__(self):
self.calls = []
def render_chat(
self,
all_messages,
chat_params,
tok_params,
prompt_extras=None,
):
self.calls.append(
{
"all_messages": all_messages,
"chat_params": chat_params,
"tok_params": tok_params,
"prompt_extras": prompt_extras,
}
)
return all_messages, [
{"prompt_token_ids": [index]} for index, _ in enumerate(all_messages)
]
@classmethod
def _make_handler(cls, renderer):
handler = object.__new__(EmbedIOProcessor)
handler.renderer = renderer
handler.model_config = cls._FakeModelConfig()
handler.chat_template = "template"
handler.chat_template_content_format = "auto"
handler.trust_request_chat_template = False
handler.enable_chunked_processing = False
return handler
@staticmethod
def _make_context(
request: (
EmbeddingChatRequest
| EmbeddingBatchChatRequest
| EmbeddingChatInputRequest
| EmbeddingBatchChatInputRequest
),
) -> PoolingServeContext[
EmbeddingChatRequest
| EmbeddingBatchChatRequest
| EmbeddingChatInputRequest
| EmbeddingBatchChatInputRequest
]:
return PoolingServeContext(
request=request,
pooling_params=PoolingParams(),
model_name="test",
request_id="embd-test",
)
def test_chat_template_kwargs_forwarded_for_batched_input_messages(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [
[{"role": "user", "content": "hello"}],
[{"role": "user", "content": "goodbye"}],
],
"add_generation_prompt": True,
"chat_template_kwargs": {"instruction": "Represent the query: "},
"mm_processor_kwargs": {"max_pixels": 1},
"cache_salt": "salt",
}
)
assert isinstance(request, EmbeddingBatchChatInputRequest)
renderer = self._FakeRenderer()
handler = self._make_handler(renderer)
ctx = self._make_context(request)
handler.pre_process_online(ctx)
assert ctx.engine_inputs == [
{"prompt_token_ids": [0]},
{"prompt_token_ids": [1]},
]
assert len(renderer.calls) == 1
call = renderer.calls[0]
assert call["all_messages"] == request.messages
assert call["prompt_extras"] == {
"mm_processor_kwargs": {"max_pixels": 1},
"cache_salt": "salt",
}
chat_template_kwargs = call["chat_params"].chat_template_kwargs
assert chat_template_kwargs["instruction"] == "Represent the query: "
assert chat_template_kwargs["add_generation_prompt"] is True
assert chat_template_kwargs["continue_final_message"] is False
assert "tools" not in chat_template_kwargs
assert chat_template_kwargs["tokenize"] is False
+7 -5
View File
@@ -168,11 +168,7 @@ class CompletionRequestMixin(OpenAIBaseModel):
# --8<-- [end:completion-extra-params]
class ChatRequestMixin(OpenAIBaseModel):
# --8<-- [start:chat-params]
messages: list[ChatCompletionMessageParam]
# --8<-- [end:chat-params]
class ChatRequestOptionsMixin(OpenAIBaseModel):
# --8<-- [start:chat-extra-params]
add_generation_prompt: bool = Field(
default=False,
@@ -256,6 +252,12 @@ class ChatRequestMixin(OpenAIBaseModel):
)
class ChatRequestMixin(ChatRequestOptionsMixin):
# --8<-- [start:chat-params]
messages: list[ChatCompletionMessageParam]
# --8<-- [end:chat-params]
class EncodingRequestMixin(OpenAIBaseModel):
# --8<-- [start:encoding-params]
encoding_format: EncodingFormat = "float"
@@ -36,6 +36,9 @@ from .protocol import (
CohereEmbedContent,
CohereEmbedInput,
CohereEmbedRequest,
EmbeddingBatchChatInputRequest,
EmbeddingBatchChatRequest,
EmbeddingChatInputRequest,
EmbeddingChatRequest,
EmbeddingCompletionRequest,
)
@@ -66,6 +69,16 @@ class EmbedIOProcessor(PoolingIOProcessor):
def pre_process_online(self, ctx: PoolingServeContext):
if isinstance(ctx.request, CohereEmbedRequest):
self._pre_process_cohere_online(ctx)
elif isinstance(
ctx.request,
(
EmbeddingChatRequest,
EmbeddingBatchChatRequest,
EmbeddingChatInputRequest,
EmbeddingBatchChatInputRequest,
),
):
self._pre_process_openai_chat_online(ctx)
else:
super().pre_process_online(ctx)
@@ -367,6 +380,70 @@ class EmbedIOProcessor(PoolingIOProcessor):
)
return super().create_pooling_params(request)
def _pre_process_openai_chat_online(
self,
ctx: PoolingServeContext[
EmbeddingChatRequest
| EmbeddingBatchChatRequest
| EmbeddingChatInputRequest
| EmbeddingBatchChatInputRequest
],
) -> None:
request = ctx.request
self._validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
if isinstance(
request, (EmbeddingBatchChatRequest, EmbeddingBatchChatInputRequest)
):
all_messages = request.messages
else:
all_messages = [request.messages]
ctx.engine_inputs = self._batch_render_openai_chat(request, all_messages)
def _batch_render_openai_chat(
self,
request: (
EmbeddingChatRequest
| EmbeddingBatchChatRequest
| EmbeddingChatInputRequest
| EmbeddingBatchChatInputRequest
),
all_messages: Sequence[list[ChatCompletionMessageParam]],
) -> list[EngineInput]:
renderer = self.renderer
mm_config = self.model_config.multimodal_config
tok_params = request.build_tok_params(self.model_config)
chat_params = request.build_chat_params(
self.chat_template,
self.chat_template_content_format,
).with_defaults(
merge_kwargs(
None,
dict(
tools=None,
tokenize=is_mistral_tokenizer(renderer.tokenizer),
),
),
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
)
_, engine_inputs = renderer.render_chat(
all_messages,
chat_params,
tok_params,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(request, k, None)) is not None
},
)
return engine_inputs
def _pre_process_cohere_online(self, ctx: PoolingServeContext) -> None:
"""Convert a ``CohereEmbedRequest`` into engine prompts.
+107 -3
View File
@@ -10,17 +10,19 @@ import builtins
import struct
import time
from collections.abc import Sequence
from typing import Literal, TypeAlias
from typing import Annotated, Any, Literal, TypeAlias
import pybase64 as base64
from pydantic import BaseModel, Field
from pydantic import BaseModel, Field, model_validator
from vllm import PoolingParams
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
from vllm.utils import random_uuid
from ..base.protocol import (
ChatRequestMixin,
ChatRequestOptionsMixin,
CompletionRequestMixin,
EmbeddingTokenizeParamsMixin,
EmbedRequestMixin,
@@ -42,12 +44,34 @@ class EmbeddingCompletionRequest(
)
def _is_chat_message(value: Any) -> bool:
return isinstance(value, dict) and isinstance(value.get("role"), str)
def _is_chat_messages(value: Any) -> bool:
return (
isinstance(value, list)
and bool(value)
and all(_is_chat_message(item) for item in value)
)
def _is_batched_chat_messages(value: Any) -> bool:
return (
isinstance(value, list)
and bool(value)
and all(_is_chat_messages(item) for item in value)
)
class EmbeddingChatRequest(
PoolingBasicRequestMixin,
ChatRequestMixin,
EmbedRequestMixin,
EmbeddingTokenizeParamsMixin,
):
"""OpenAI embeddings request with one top-level chat conversation."""
def to_pooling_params(self):
return PoolingParams(
task="embed",
@@ -56,7 +80,87 @@ class EmbeddingChatRequest(
)
EmbeddingRequest: TypeAlias = EmbeddingCompletionRequest | EmbeddingChatRequest
class EmbeddingBatchChatRequest(
PoolingBasicRequestMixin,
ChatRequestOptionsMixin,
EmbedRequestMixin,
EmbeddingTokenizeParamsMixin,
):
"""OpenAI embeddings request with batched top-level chat conversations.
Mirrors ``BatchChatCompletionRequest`` by keeping batched conversations in
``messages`` instead of introducing a separate batch-specific field.
"""
messages: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = (
Field(..., min_length=1)
)
def to_pooling_params(self):
return PoolingParams(
task="embed",
dimensions=self.dimensions,
use_activation=self.use_activation,
)
class EmbeddingChatInputRequest(
EmbeddingChatRequest,
):
"""OpenAI embeddings request with one chat conversation in ``input``."""
input: list[ChatCompletionMessageParam]
@model_validator(mode="before")
@classmethod
def normalize_input_messages(cls, data):
if not isinstance(data, dict):
return data
if "messages" in data or "input" not in data:
return data
input_data = data["input"]
if not _is_chat_messages(input_data):
return data
normalized = dict(data)
normalized["messages"] = input_data
return normalized
class EmbeddingBatchChatInputRequest(EmbeddingBatchChatRequest):
"""OpenAI embeddings request with batched chat conversations in ``input``."""
input: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = (
Field(..., min_length=1)
)
@model_validator(mode="before")
@classmethod
def normalize_input_messages(cls, data):
if not isinstance(data, dict):
return data
if "messages" in data or "input" not in data:
return data
input_data = data["input"]
if not _is_batched_chat_messages(input_data):
return data
normalized = dict(data)
normalized["messages"] = input_data
return normalized
EmbeddingRequest: TypeAlias = (
EmbeddingCompletionRequest
| EmbeddingChatRequest
| EmbeddingBatchChatRequest
| EmbeddingChatInputRequest
| EmbeddingBatchChatInputRequest
)
# ---------------------------------------------------------------------------
+8 -2
View File
@@ -20,8 +20,10 @@ from .classify.protocol import (
from .embed.protocol import (
CohereEmbedRequest,
EmbeddingBytesResponse,
EmbeddingChatInputRequest,
EmbeddingChatRequest,
EmbeddingCompletionRequest,
EmbeddingRequest,
EmbeddingResponse,
)
from .pooling.protocol import (
@@ -41,11 +43,15 @@ PoolingCompletionLikeRequest: TypeAlias = (
)
PoolingChatLikeRequest: TypeAlias = (
EmbeddingChatRequest | ClassificationChatRequest | PoolingChatRequest
EmbeddingChatRequest
| EmbeddingChatInputRequest
| ClassificationChatRequest
| PoolingChatRequest
)
AnyPoolingRequest: TypeAlias = (
PoolingCompletionLikeRequest
EmbeddingRequest
| PoolingCompletionLikeRequest
| PoolingChatLikeRequest
| IOProcessorRequest
| ScoringRequest