[Bugfix][Frontend] Preserve logprobs when top_logprobs is null (#47838)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Signed-off-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -16,6 +16,7 @@ from openai import BadRequestError
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
@@ -1012,6 +1013,22 @@ def test_chat_completion_request_n_parameter_default():
|
||||
assert sampling_params.n == 1, f"Expected n=1 (default), got n={sampling_params.n}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("request_cls", "messages"),
|
||||
[
|
||||
(ChatCompletionRequest, [{"role": "user", "content": "Hello"}]),
|
||||
(BatchChatCompletionRequest, [[{"role": "user", "content": "Hello"}]]),
|
||||
],
|
||||
)
|
||||
def test_null_top_logprobs_is_same_as_omitted(request_cls, messages):
|
||||
"""`top_logprobs: null` must not switch off `logprobs: true`."""
|
||||
request = request_cls.model_validate(
|
||||
{"messages": messages, "logprobs": True, "top_logprobs": None}
|
||||
)
|
||||
|
||||
assert request.top_logprobs == 0
|
||||
|
||||
|
||||
def test_chat_completion_request_accepts_model_specific_reasoning_effort():
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
|
||||
@@ -84,6 +84,20 @@ class TestResponsesRequestSamplingParams:
|
||||
assert sampling_params.stop == [] # Empty list
|
||||
assert sampling_params.extra_args == {} # Empty dict
|
||||
|
||||
def test_null_top_logprobs_is_same_as_omitted(self):
|
||||
"""`top_logprobs: null` must not drop the requested output logprobs."""
|
||||
request = ResponsesRequest.model_validate(
|
||||
{
|
||||
"input": "test input",
|
||||
"include": ["message.output_text.logprobs"],
|
||||
"top_logprobs": None,
|
||||
}
|
||||
)
|
||||
|
||||
sampling_params = request.to_sampling_params(default_max_tokens=1000)
|
||||
|
||||
assert sampling_params.logprobs == 0
|
||||
|
||||
def test_seed_bounds_validation(self):
|
||||
"""Test that seed values outside torch.long bounds are rejected."""
|
||||
# Test seed below minimum
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
BeforeValidator,
|
||||
Field,
|
||||
model_serializer,
|
||||
)
|
||||
@@ -26,6 +27,12 @@ StopParam: TypeAlias = (
|
||||
str | Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
|
||||
)
|
||||
|
||||
# `top_logprobs` is nullable in the OpenAI spec; null means the same as omitted.
|
||||
TopLogprobsParam: TypeAlias = Annotated[
|
||||
int,
|
||||
BeforeValidator(lambda v: 0 if v is None else v, json_schema_input_type=int | None),
|
||||
]
|
||||
|
||||
_CACHE_SALT_FORBIDDEN_CHARS = frozenset("@/\\\x00")
|
||||
_MAX_CACHE_SALT_LENGTH = 128
|
||||
|
||||
|
||||
@@ -247,9 +247,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
for output in final_res.outputs:
|
||||
self._raise_if_error(output.finish_reason, request_id)
|
||||
|
||||
if request.logprobs and (
|
||||
request.top_logprobs is not None or request.logprob_token_ids
|
||||
):
|
||||
if request.logprobs:
|
||||
assert output.logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_chat_logprobs(
|
||||
token_ids=output.token_ids,
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
StopParam,
|
||||
StreamOptions,
|
||||
ToolCall,
|
||||
TopLogprobsParam,
|
||||
structured_outputs_from_response_format,
|
||||
validate_cache_salt,
|
||||
validate_structural_tag_response_format,
|
||||
@@ -222,7 +223,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
frequency_penalty: float | None = None
|
||||
logit_bias: dict[str, float] | None = None
|
||||
logprobs: bool | None = False
|
||||
top_logprobs: int | None = 0
|
||||
top_logprobs: TopLogprobsParam = 0
|
||||
max_tokens: int | None = Field(
|
||||
default=None,
|
||||
deprecated="max_tokens is deprecated in favor of "
|
||||
@@ -1095,7 +1096,7 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
|
||||
frequency_penalty: float | None = 0.0
|
||||
logit_bias: dict[str, float] | None = None
|
||||
logprobs: bool | None = False
|
||||
top_logprobs: int | None = 0
|
||||
top_logprobs: TopLogprobsParam = 0
|
||||
logprob_token_ids: list[int] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
|
||||
@@ -625,9 +625,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
|
||||
self._raise_if_error(output.finish_reason, request_id)
|
||||
|
||||
if request.logprobs and (
|
||||
request.top_logprobs is not None or request.logprob_token_ids
|
||||
):
|
||||
if request.logprobs:
|
||||
assert output.logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_chat_logprobs(
|
||||
token_ids=output.token_ids,
|
||||
@@ -970,9 +968,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
token_ids = output.token_ids
|
||||
out_logprobs = output.logprobs
|
||||
|
||||
if request.logprobs and (
|
||||
request.top_logprobs is not None or request.logprob_token_ids
|
||||
):
|
||||
if request.logprobs:
|
||||
assert out_logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_chat_logprobs(
|
||||
token_ids=token_ids,
|
||||
|
||||
@@ -61,6 +61,7 @@ from vllm.entrypoints.chat_utils import (
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
PerRequestMetrics,
|
||||
StopParam,
|
||||
TopLogprobsParam,
|
||||
validate_cache_salt,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
@@ -210,7 +211,7 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
text: ResponseTextConfig | None = None
|
||||
tool_choice: ToolChoice = "auto"
|
||||
tools: list[Tool] = Field(default_factory=list)
|
||||
top_logprobs: int | None = 0
|
||||
top_logprobs: TopLogprobsParam = 0
|
||||
top_p: float | None = None
|
||||
top_k: int | None = None
|
||||
truncation: Literal["auto", "disabled"] | None = "disabled"
|
||||
|
||||
Reference in New Issue
Block a user