[Bugfix][Frontend] Preserve logprobs when top_logprobs is null (#47838)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Signed-off-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -247,9 +247,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
for output in final_res.outputs:
|
||||
self._raise_if_error(output.finish_reason, request_id)
|
||||
|
||||
if request.logprobs and (
|
||||
request.top_logprobs is not None or request.logprob_token_ids
|
||||
):
|
||||
if request.logprobs:
|
||||
assert output.logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_chat_logprobs(
|
||||
token_ids=output.token_ids,
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
StopParam,
|
||||
StreamOptions,
|
||||
ToolCall,
|
||||
TopLogprobsParam,
|
||||
structured_outputs_from_response_format,
|
||||
validate_cache_salt,
|
||||
validate_structural_tag_response_format,
|
||||
@@ -222,7 +223,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
frequency_penalty: float | None = None
|
||||
logit_bias: dict[str, float] | None = None
|
||||
logprobs: bool | None = False
|
||||
top_logprobs: int | None = 0
|
||||
top_logprobs: TopLogprobsParam = 0
|
||||
max_tokens: int | None = Field(
|
||||
default=None,
|
||||
deprecated="max_tokens is deprecated in favor of "
|
||||
@@ -1095,7 +1096,7 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
|
||||
frequency_penalty: float | None = 0.0
|
||||
logit_bias: dict[str, float] | None = None
|
||||
logprobs: bool | None = False
|
||||
top_logprobs: int | None = 0
|
||||
top_logprobs: TopLogprobsParam = 0
|
||||
logprob_token_ids: list[int] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
|
||||
@@ -625,9 +625,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
|
||||
self._raise_if_error(output.finish_reason, request_id)
|
||||
|
||||
if request.logprobs and (
|
||||
request.top_logprobs is not None or request.logprob_token_ids
|
||||
):
|
||||
if request.logprobs:
|
||||
assert output.logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_chat_logprobs(
|
||||
token_ids=output.token_ids,
|
||||
@@ -970,9 +968,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
token_ids = output.token_ids
|
||||
out_logprobs = output.logprobs
|
||||
|
||||
if request.logprobs and (
|
||||
request.top_logprobs is not None or request.logprob_token_ids
|
||||
):
|
||||
if request.logprobs:
|
||||
assert out_logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_chat_logprobs(
|
||||
token_ids=token_ids,
|
||||
|
||||
Reference in New Issue
Block a user