Add /v1/chat/completions/batch endpoint for batched chat completions (#38011)
Signed-off-by: Matej Rojec <64556640+MatejRojec@users.noreply.github.com>
This commit is contained in:
@@ -787,3 +787,83 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
if data.get("reasoning_effort") == "none":
|
||||
data["include_reasoning"] = False
|
||||
return data
|
||||
|
||||
|
||||
class BatchChatCompletionRequest(OpenAIBaseModel):
|
||||
"""Request model for the /v1/chat/completions/batch endpoint.
|
||||
|
||||
Accepts the same fields as ChatCompletionRequest except that ``messages``
|
||||
is a list of conversations (each conversation is a
|
||||
``list[ChatCompletionMessageParam]``). Each conversation is processed
|
||||
independently and the response contains one choice per conversation,
|
||||
indexed 0, 1, ..., N-1.
|
||||
|
||||
Current limitations compared to the single-conversation endpoint:
|
||||
- Streaming is not supported (``stream`` must be False or omitted).
|
||||
- Tool use is not supported (``tools`` must be omitted).
|
||||
- Beam search is not supported (``use_beam_search`` must be False or omitted).
|
||||
- The ``n`` parameter must be 1 (or omitted).
|
||||
"""
|
||||
|
||||
messages: list[list[ChatCompletionMessageParam]] = Field(..., min_length=1)
|
||||
model: str | None = None
|
||||
|
||||
# Shared sampling / generation fields — mirror ChatCompletionRequest.
|
||||
frequency_penalty: float | None = 0.0
|
||||
logit_bias: dict[str, float] | None = None
|
||||
logprobs: bool | None = False
|
||||
top_logprobs: int | None = 0
|
||||
max_tokens: int | None = None
|
||||
max_completion_tokens: int | None = None
|
||||
n: int | None = 1
|
||||
presence_penalty: float | None = 0.0
|
||||
response_format: Any | None = None
|
||||
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
|
||||
stop: str | list[str] | None = Field(default_factory=list)
|
||||
temperature: float | None = 0.7
|
||||
top_p: float | None = 1.0
|
||||
user: str | None = None
|
||||
|
||||
# vLLM extensions
|
||||
best_of: int | None = None
|
||||
use_beam_search: bool = False
|
||||
top_k: int | None = None
|
||||
min_p: float | None = 0.0
|
||||
repetition_penalty: float | None = 1.0
|
||||
length_penalty: float | None = 1.0
|
||||
early_stopping: bool = False
|
||||
structured_outputs: StructuredOutputsParams | None = None
|
||||
request_id: str | None = None
|
||||
add_generation_prompt: bool = True
|
||||
continue_final_message: bool = False
|
||||
chat_template: str | None = None
|
||||
chat_template_kwargs: dict[str, Any] | None = None
|
||||
include_stop_str_in_output: bool = False
|
||||
guided_decoding_backend: str | None = None
|
||||
echo: bool = False
|
||||
return_token_ids: bool = False
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_batch_mode(cls, data: Any) -> Any:
|
||||
if isinstance(data, BatchChatCompletionRequest):
|
||||
data = data.model_dump(exclude_unset=True)
|
||||
if data.get("use_beam_search"):
|
||||
raise ValueError(
|
||||
"Batch chat completions do not support beam search. "
|
||||
"Please set `use_beam_search` to False."
|
||||
)
|
||||
n = data.get("n", 1)
|
||||
if n is not None and n != 1:
|
||||
raise ValueError(
|
||||
"Batch chat completions do not support `n > 1`. Please set `n` to 1."
|
||||
)
|
||||
return data
|
||||
|
||||
def to_chat_completion_request(
|
||||
self, messages: list[ChatCompletionMessageParam]
|
||||
) -> ChatCompletionRequest:
|
||||
"""Build a single-conversation ChatCompletionRequest from one conversation."""
|
||||
data = self.model_dump(exclude={"messages"}, exclude_none=True)
|
||||
data["messages"] = messages
|
||||
return ChatCompletionRequest.model_validate(data)
|
||||
|
||||
Reference in New Issue
Block a user