Add /v1/chat/completions/batch endpoint for batched chat completions (#38011)

Signed-off-by: Matej Rojec <64556640+MatejRojec@users.noreply.github.com>
This commit is contained in:
Matej Rojec
2026-03-26 12:13:33 +08:00
committed by GitHub
parent e6bf9f15ec
commit 2908094567
8 changed files with 771 additions and 21 deletions
@@ -787,3 +787,83 @@ class ChatCompletionRequest(OpenAIBaseModel):
if data.get("reasoning_effort") == "none":
data["include_reasoning"] = False
return data
class BatchChatCompletionRequest(OpenAIBaseModel):
"""Request model for the /v1/chat/completions/batch endpoint.
Accepts the same fields as ChatCompletionRequest except that ``messages``
is a list of conversations (each conversation is a
``list[ChatCompletionMessageParam]``). Each conversation is processed
independently and the response contains one choice per conversation,
indexed 0, 1, ..., N-1.
Current limitations compared to the single-conversation endpoint:
- Streaming is not supported (``stream`` must be False or omitted).
- Tool use is not supported (``tools`` must be omitted).
- Beam search is not supported (``use_beam_search`` must be False or omitted).
- The ``n`` parameter must be 1 (or omitted).
"""
messages: list[list[ChatCompletionMessageParam]] = Field(..., min_length=1)
model: str | None = None
# Shared sampling / generation fields — mirror ChatCompletionRequest.
frequency_penalty: float | None = 0.0
logit_bias: dict[str, float] | None = None
logprobs: bool | None = False
top_logprobs: int | None = 0
max_tokens: int | None = None
max_completion_tokens: int | None = None
n: int | None = 1
presence_penalty: float | None = 0.0
response_format: Any | None = None
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
stop: str | list[str] | None = Field(default_factory=list)
temperature: float | None = 0.7
top_p: float | None = 1.0
user: str | None = None
# vLLM extensions
best_of: int | None = None
use_beam_search: bool = False
top_k: int | None = None
min_p: float | None = 0.0
repetition_penalty: float | None = 1.0
length_penalty: float | None = 1.0
early_stopping: bool = False
structured_outputs: StructuredOutputsParams | None = None
request_id: str | None = None
add_generation_prompt: bool = True
continue_final_message: bool = False
chat_template: str | None = None
chat_template_kwargs: dict[str, Any] | None = None
include_stop_str_in_output: bool = False
guided_decoding_backend: str | None = None
echo: bool = False
return_token_ids: bool = False
@model_validator(mode="before")
@classmethod
def check_batch_mode(cls, data: Any) -> Any:
if isinstance(data, BatchChatCompletionRequest):
data = data.model_dump(exclude_unset=True)
if data.get("use_beam_search"):
raise ValueError(
"Batch chat completions do not support beam search. "
"Please set `use_beam_search` to False."
)
n = data.get("n", 1)
if n is not None and n != 1:
raise ValueError(
"Batch chat completions do not support `n > 1`. Please set `n` to 1."
)
return data
def to_chat_completion_request(
self, messages: list[ChatCompletionMessageParam]
) -> ChatCompletionRequest:
"""Build a single-conversation ChatCompletionRequest from one conversation."""
data = self.model_dump(exclude={"messages"}, exclude_none=True)
data["messages"] = messages
return ChatCompletionRequest.model_validate(data)