[Bugfix][Frontend] Preserve logprobs when top_logprobs is null (#47838)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Signed-off-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -8,6 +8,7 @@ from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
BeforeValidator,
|
||||
Field,
|
||||
model_serializer,
|
||||
)
|
||||
@@ -26,6 +27,12 @@ StopParam: TypeAlias = (
|
||||
str | Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
|
||||
)
|
||||
|
||||
# `top_logprobs` is nullable in the OpenAI spec; null means the same as omitted.
|
||||
TopLogprobsParam: TypeAlias = Annotated[
|
||||
int,
|
||||
BeforeValidator(lambda v: 0 if v is None else v, json_schema_input_type=int | None),
|
||||
]
|
||||
|
||||
_CACHE_SALT_FORBIDDEN_CHARS = frozenset("@/\\\x00")
|
||||
_MAX_CACHE_SALT_LENGTH = 128
|
||||
|
||||
|
||||
Reference in New Issue
Block a user