Bound generation inputs before expensive work (#51447)

Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com>
Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Clinton Thomas
2026-08-11 22:03:50 +08:00
committed by GitHub
co-authored by Lucas Bourtoule mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5b184f775a
commit b64a2708b0
11 changed files with 421 additions and 37 deletions
@@ -30,6 +30,7 @@ from vllm.entrypoints.openai.engine.protocol import (
FunctionDefinition,
OpenAIBaseModel,
PerRequestTimingMetrics,
StopParam,
StreamOptions,
ToolCall,
UsageInfo,
@@ -228,7 +229,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
presence_penalty: float | None = 0.0
response_format: AnyResponseFormat | None = None
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
stop: str | list[str] | None = []
stop: StopParam = []
stream: bool | None = False
stream_options: StreamOptions | None = None
temperature: float | None = None
@@ -1077,7 +1078,7 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
presence_penalty: float | None = 0.0
response_format: Any | None = None
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
stop: str | list[str] | None = Field(default_factory=list)
stop: StopParam = Field(default_factory=list)
temperature: float | None = None
top_p: float | None = None
user: str | None = None
@@ -14,6 +14,7 @@ from vllm.entrypoints.openai.engine.protocol import (
AnyResponseFormat,
OpenAIBaseModel,
PerRequestTimingMetrics,
StopParam,
StreamOptions,
UsageInfo,
structured_outputs_from_response_format,
@@ -61,7 +62,7 @@ class CompletionRequest(OpenAIBaseModel):
n: int = 1
presence_penalty: float | None = 0.0
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
stop: str | list[str] | None = []
stop: StopParam = []
stream: bool | None = False
stream_options: StreamOptions | None = None
suffix: str | None = None
+6 -1
View File
@@ -6,7 +6,7 @@
import json
import time
from http import HTTPStatus
from typing import Any, ClassVar, Literal, TypeAlias
from typing import Annotated, Any, ClassVar, Literal, TypeAlias
import regex as re
from pydantic import (
@@ -17,6 +17,7 @@ from pydantic import (
model_validator,
)
import vllm.envs as envs
from vllm.config.utils import replace
from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.exceptions import VLLMServerError, VLLMValidationError
@@ -27,6 +28,10 @@ from vllm.utils.import_utils import resolve_obj_by_qualname
logger = init_logger(__name__)
StopParam: TypeAlias = (
str | Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
)
class OpenAIBaseModel(BaseModel):
# OpenAI API does allow extra fields
@@ -60,7 +60,7 @@ from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormatOption,
)
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, StopParam
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
@@ -281,7 +281,7 @@ class ResponsesRequest(OpenAIBaseModel):
repetition_penalty: float | None = None
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
stop: str | list[str] | None = []
stop: StopParam = []
ignore_eos: bool = False
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
default=None,