[Frontend] Parallelize preprocessing within the same request for pooling models online serving. (#49153)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -93,9 +93,9 @@ class EmbeddingBatchChatRequest(
|
||||
``messages`` instead of introducing a separate batch-specific field.
|
||||
"""
|
||||
|
||||
messages: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = (
|
||||
Field(..., min_length=1)
|
||||
)
|
||||
messages: Sequence[
|
||||
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
|
||||
] = Field(..., min_length=1)
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
@@ -133,9 +133,9 @@ class EmbeddingChatInputRequest(
|
||||
class EmbeddingBatchChatInputRequest(EmbeddingBatchChatRequest):
|
||||
"""OpenAI embeddings request with batched chat conversations in ``input``."""
|
||||
|
||||
input: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = (
|
||||
Field(..., min_length=1)
|
||||
)
|
||||
input: Sequence[
|
||||
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
|
||||
] = Field(..., min_length=1)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
|
||||
Reference in New Issue
Block a user