Bound generation inputs before expensive work (#51447)

Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com>
Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Clinton Thomas
2026-08-11 22:03:50 +08:00
committed by GitHub
co-authored by Lucas Bourtoule mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5b184f775a
commit b64a2708b0
11 changed files with 421 additions and 37 deletions
@@ -30,6 +30,7 @@ from vllm.entrypoints.openai.engine.protocol import (
FunctionDefinition,
OpenAIBaseModel,
PerRequestTimingMetrics,
StopParam,
StreamOptions,
ToolCall,
UsageInfo,
@@ -228,7 +229,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
presence_penalty: float | None = 0.0
response_format: AnyResponseFormat | None = None
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
stop: str | list[str] | None = []
stop: StopParam = []
stream: bool | None = False
stream_options: StreamOptions | None = None
temperature: float | None = None
@@ -1077,7 +1078,7 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
presence_penalty: float | None = 0.0
response_format: Any | None = None
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
stop: str | list[str] | None = Field(default_factory=list)
stop: StopParam = Field(default_factory=list)
temperature: float | None = None
top_p: float | None = None
user: str | None = None