Added real /v1/embeddings support for messages + chat_template_kw (#45173)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
"""Unit tests for EmbedIOProcessor."""
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.entrypoints.pooling.embed.io_processor import EmbedIOProcessor
|
||||
@@ -10,10 +11,100 @@ from vllm.entrypoints.pooling.embed.protocol import (
|
||||
CohereEmbedContent,
|
||||
CohereEmbedInput,
|
||||
CohereEmbedRequest,
|
||||
EmbeddingBatchChatInputRequest,
|
||||
EmbeddingBatchChatRequest,
|
||||
EmbeddingChatInputRequest,
|
||||
EmbeddingChatRequest,
|
||||
EmbeddingCompletionRequest,
|
||||
EmbeddingRequest,
|
||||
)
|
||||
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
||||
|
||||
|
||||
class TestEmbeddingRequestParsing:
|
||||
"""Unit tests for OpenAI embedding request parsing."""
|
||||
|
||||
def test_input_messages_parses_as_chat_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [{"role": "user", "content": "hello"}],
|
||||
"chat_template_kwargs": {"instruction": "Represent the query: "},
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(request, EmbeddingChatInputRequest)
|
||||
assert request.input == [{"role": "user", "content": "hello"}]
|
||||
assert request.messages == [{"role": "user", "content": "hello"}]
|
||||
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
|
||||
|
||||
def test_batched_input_messages_parses_as_batch_chat_input_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [
|
||||
[{"role": "user", "content": "hello"}],
|
||||
[{"role": "user", "content": "goodbye"}],
|
||||
],
|
||||
"chat_template_kwargs": {"instruction": "Represent the query: "},
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(request, EmbeddingBatchChatInputRequest)
|
||||
assert request.input == [
|
||||
[{"role": "user", "content": "hello"}],
|
||||
[{"role": "user", "content": "goodbye"}],
|
||||
]
|
||||
assert request.messages == [
|
||||
[{"role": "user", "content": "hello"}],
|
||||
[{"role": "user", "content": "goodbye"}],
|
||||
]
|
||||
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
|
||||
|
||||
def test_token_ids_still_parse_as_completion_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [[1, 2, 3], [4, 5]],
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(request, EmbeddingCompletionRequest)
|
||||
assert request.input == [[1, 2, 3], [4, 5]]
|
||||
|
||||
def test_messages_still_parses_as_chat_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"chat_template_kwargs": {"instruction": "Represent the query: "},
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(request, EmbeddingChatRequest)
|
||||
assert request.messages == [{"role": "user", "content": "hello"}]
|
||||
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
|
||||
|
||||
def test_batched_messages_parses_as_batch_chat_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"messages": [
|
||||
[{"role": "user", "content": "hello"}],
|
||||
[{"role": "user", "content": "goodbye"}],
|
||||
],
|
||||
"chat_template_kwargs": {"instruction": "Represent the query: "},
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(request, EmbeddingBatchChatRequest)
|
||||
assert request.messages == [
|
||||
[{"role": "user", "content": "hello"}],
|
||||
[{"role": "user", "content": "goodbye"}],
|
||||
]
|
||||
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
|
||||
|
||||
|
||||
class TestResolveTruncation:
|
||||
"""Unit tests for EmbedIOProcessor._resolve_cohere_truncation."""
|
||||
|
||||
@@ -324,3 +415,113 @@ class TestPreProcessCohereOnline:
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
class TestPreProcessOpenAIEmbeddingChatOnline:
|
||||
"""Unit tests for OpenAI embedding chat preprocessing."""
|
||||
|
||||
class _FakeModelConfig:
|
||||
max_model_len = 128
|
||||
encoder_config: dict[str, object] = {}
|
||||
pooler_config = None
|
||||
multimodal_config = None
|
||||
is_encoder_decoder = False
|
||||
|
||||
class _FakeRenderer:
|
||||
tokenizer = object()
|
||||
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def render_chat(
|
||||
self,
|
||||
all_messages,
|
||||
chat_params,
|
||||
tok_params,
|
||||
prompt_extras=None,
|
||||
):
|
||||
self.calls.append(
|
||||
{
|
||||
"all_messages": all_messages,
|
||||
"chat_params": chat_params,
|
||||
"tok_params": tok_params,
|
||||
"prompt_extras": prompt_extras,
|
||||
}
|
||||
)
|
||||
return all_messages, [
|
||||
{"prompt_token_ids": [index]} for index, _ in enumerate(all_messages)
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def _make_handler(cls, renderer):
|
||||
handler = object.__new__(EmbedIOProcessor)
|
||||
handler.renderer = renderer
|
||||
handler.model_config = cls._FakeModelConfig()
|
||||
handler.chat_template = "template"
|
||||
handler.chat_template_content_format = "auto"
|
||||
handler.trust_request_chat_template = False
|
||||
handler.enable_chunked_processing = False
|
||||
return handler
|
||||
|
||||
@staticmethod
|
||||
def _make_context(
|
||||
request: (
|
||||
EmbeddingChatRequest
|
||||
| EmbeddingBatchChatRequest
|
||||
| EmbeddingChatInputRequest
|
||||
| EmbeddingBatchChatInputRequest
|
||||
),
|
||||
) -> PoolingServeContext[
|
||||
EmbeddingChatRequest
|
||||
| EmbeddingBatchChatRequest
|
||||
| EmbeddingChatInputRequest
|
||||
| EmbeddingBatchChatInputRequest
|
||||
]:
|
||||
return PoolingServeContext(
|
||||
request=request,
|
||||
pooling_params=PoolingParams(),
|
||||
model_name="test",
|
||||
request_id="embd-test",
|
||||
)
|
||||
|
||||
def test_chat_template_kwargs_forwarded_for_batched_input_messages(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [
|
||||
[{"role": "user", "content": "hello"}],
|
||||
[{"role": "user", "content": "goodbye"}],
|
||||
],
|
||||
"add_generation_prompt": True,
|
||||
"chat_template_kwargs": {"instruction": "Represent the query: "},
|
||||
"mm_processor_kwargs": {"max_pixels": 1},
|
||||
"cache_salt": "salt",
|
||||
}
|
||||
)
|
||||
assert isinstance(request, EmbeddingBatchChatInputRequest)
|
||||
|
||||
renderer = self._FakeRenderer()
|
||||
handler = self._make_handler(renderer)
|
||||
ctx = self._make_context(request)
|
||||
|
||||
handler.pre_process_online(ctx)
|
||||
|
||||
assert ctx.engine_inputs == [
|
||||
{"prompt_token_ids": [0]},
|
||||
{"prompt_token_ids": [1]},
|
||||
]
|
||||
assert len(renderer.calls) == 1
|
||||
|
||||
call = renderer.calls[0]
|
||||
assert call["all_messages"] == request.messages
|
||||
assert call["prompt_extras"] == {
|
||||
"mm_processor_kwargs": {"max_pixels": 1},
|
||||
"cache_salt": "salt",
|
||||
}
|
||||
|
||||
chat_template_kwargs = call["chat_params"].chat_template_kwargs
|
||||
assert chat_template_kwargs["instruction"] == "Represent the query: "
|
||||
assert chat_template_kwargs["add_generation_prompt"] is True
|
||||
assert chat_template_kwargs["continue_final_message"] is False
|
||||
assert "tools" not in chat_template_kwargs
|
||||
assert chat_template_kwargs["tokenize"] is False
|
||||
|
||||
Reference in New Issue
Block a user