Bound generation inputs before expensive work (#51447)
Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Bourtoule
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5b184f775a
commit
b64a2708b0
@@ -30,6 +30,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
FunctionDefinition,
|
||||
OpenAIBaseModel,
|
||||
PerRequestTimingMetrics,
|
||||
StopParam,
|
||||
StreamOptions,
|
||||
ToolCall,
|
||||
UsageInfo,
|
||||
@@ -228,7 +229,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
presence_penalty: float | None = 0.0
|
||||
response_format: AnyResponseFormat | None = None
|
||||
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
|
||||
stop: str | list[str] | None = []
|
||||
stop: StopParam = []
|
||||
stream: bool | None = False
|
||||
stream_options: StreamOptions | None = None
|
||||
temperature: float | None = None
|
||||
@@ -1077,7 +1078,7 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
|
||||
presence_penalty: float | None = 0.0
|
||||
response_format: Any | None = None
|
||||
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
|
||||
stop: str | list[str] | None = Field(default_factory=list)
|
||||
stop: StopParam = Field(default_factory=list)
|
||||
temperature: float | None = None
|
||||
top_p: float | None = None
|
||||
user: str | None = None
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
AnyResponseFormat,
|
||||
OpenAIBaseModel,
|
||||
PerRequestTimingMetrics,
|
||||
StopParam,
|
||||
StreamOptions,
|
||||
UsageInfo,
|
||||
structured_outputs_from_response_format,
|
||||
@@ -61,7 +62,7 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
n: int = 1
|
||||
presence_penalty: float | None = 0.0
|
||||
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
|
||||
stop: str | list[str] | None = []
|
||||
stop: StopParam = []
|
||||
stream: bool | None = False
|
||||
stream_options: StreamOptions | None = None
|
||||
suffix: str | None = None
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
import json
|
||||
import time
|
||||
from http import HTTPStatus
|
||||
from typing import Any, ClassVar, Literal, TypeAlias
|
||||
from typing import Annotated, Any, ClassVar, Literal, TypeAlias
|
||||
|
||||
import regex as re
|
||||
from pydantic import (
|
||||
@@ -17,6 +17,7 @@ from pydantic import (
|
||||
model_validator,
|
||||
)
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.utils import replace
|
||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||
from vllm.exceptions import VLLMServerError, VLLMValidationError
|
||||
@@ -27,6 +28,10 @@ from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
StopParam: TypeAlias = (
|
||||
str | Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
|
||||
)
|
||||
|
||||
|
||||
class OpenAIBaseModel(BaseModel):
|
||||
# OpenAI API does allow extra fields
|
||||
|
||||
@@ -60,7 +60,7 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel
|
||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, StopParam
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
@@ -281,7 +281,7 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
|
||||
repetition_penalty: float | None = None
|
||||
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
|
||||
stop: str | list[str] | None = []
|
||||
stop: StopParam = []
|
||||
ignore_eos: bool = False
|
||||
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
|
||||
default=None,
|
||||
|
||||
Reference in New Issue
Block a user