[Bugfix][Frontend] Preserve logprobs when top_logprobs is null (#47838)

Signed-off-by: Ting Sun <suntcrick@gmail.com>
Signed-off-by: Flora Feng <4florafeng@gmail.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Ting SUN
2026-10-05 22:43:31 +00:00
committed by GitHub
co-authored by Flora Feng
parent 89a00995c9
commit 4a705c7b54
7 changed files with 46 additions and 12 deletions
@@ -8,6 +8,7 @@ from typing import Annotated, Any, Literal, TypeAlias
from pydantic import (
BaseModel,
BeforeValidator,
Field,
model_serializer,
)
@@ -26,6 +27,12 @@ StopParam: TypeAlias = (
str | Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
)
# `top_logprobs` is nullable in the OpenAI spec; null means the same as omitted.
TopLogprobsParam: TypeAlias = Annotated[
int,
BeforeValidator(lambda v: 0 if v is None else v, json_schema_input_type=int | None),
]
_CACHE_SALT_FORBIDDEN_CHARS = frozenset("@/\\\x00")
_MAX_CACHE_SALT_LENGTH = 128
@@ -247,9 +247,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
for output in final_res.outputs:
self._raise_if_error(output.finish_reason, request_id)
if request.logprobs and (
request.top_logprobs is not None or request.logprob_token_ids
):
if request.logprobs:
assert output.logprobs is not None, "Did not output logprobs"
logprobs = self._create_chat_logprobs(
token_ids=output.token_ids,
@@ -33,6 +33,7 @@ from vllm.entrypoints.generate.base.protocol import (
StopParam,
StreamOptions,
ToolCall,
TopLogprobsParam,
structured_outputs_from_response_format,
validate_cache_salt,
validate_structural_tag_response_format,
@@ -222,7 +223,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
frequency_penalty: float | None = None
logit_bias: dict[str, float] | None = None
logprobs: bool | None = False
top_logprobs: int | None = 0
top_logprobs: TopLogprobsParam = 0
max_tokens: int | None = Field(
default=None,
deprecated="max_tokens is deprecated in favor of "
@@ -1095,7 +1096,7 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
frequency_penalty: float | None = 0.0
logit_bias: dict[str, float] | None = None
logprobs: bool | None = False
top_logprobs: int | None = 0
top_logprobs: TopLogprobsParam = 0
logprob_token_ids: list[int] | None = Field(
default=None,
description=(
@@ -625,9 +625,7 @@ class OpenAIServingChat(GenerateBaseServing):
self._raise_if_error(output.finish_reason, request_id)
if request.logprobs and (
request.top_logprobs is not None or request.logprob_token_ids
):
if request.logprobs:
assert output.logprobs is not None, "Did not output logprobs"
logprobs = self._create_chat_logprobs(
token_ids=output.token_ids,
@@ -970,9 +968,7 @@ class OpenAIServingChat(GenerateBaseServing):
token_ids = output.token_ids
out_logprobs = output.logprobs
if request.logprobs and (
request.top_logprobs is not None or request.logprob_token_ids
):
if request.logprobs:
assert out_logprobs is not None, "Did not output logprobs"
logprobs = self._create_chat_logprobs(
token_ids=token_ids,
@@ -61,6 +61,7 @@ from vllm.entrypoints.chat_utils import (
from vllm.entrypoints.generate.base.protocol import (
PerRequestMetrics,
StopParam,
TopLogprobsParam,
validate_cache_salt,
)
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
@@ -210,7 +211,7 @@ class ResponsesRequest(OpenAIBaseModel):
text: ResponseTextConfig | None = None
tool_choice: ToolChoice = "auto"
tools: list[Tool] = Field(default_factory=list)
top_logprobs: int | None = 0
top_logprobs: TopLogprobsParam = 0
top_p: float | None = None
top_k: int | None = None
truncation: Literal["auto", "disabled"] | None = "disabled"