[Frontend] Parallelize preprocessing within the same request for pooling models online serving. (#49153)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-07-22 10:56:23 +00:00
committed by GitHub
parent d6dbdb9b0d
commit 61c9ef986a
17 changed files with 538 additions and 820 deletions
+6 -6
View File
@@ -93,9 +93,9 @@ class EmbeddingBatchChatRequest(
``messages`` instead of introducing a separate batch-specific field.
"""
messages: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = (
Field(..., min_length=1)
)
messages: Sequence[
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
] = Field(..., min_length=1)
def to_pooling_params(self):
return PoolingParams(
@@ -133,9 +133,9 @@ class EmbeddingChatInputRequest(
class EmbeddingBatchChatInputRequest(EmbeddingBatchChatRequest):
"""OpenAI embeddings request with batched chat conversations in ``input``."""
input: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = (
Field(..., min_length=1)
)
input: Sequence[
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
] = Field(..., min_length=1)
@model_validator(mode="before")
@classmethod