[Frontend] Expose logprob_token_ids on Python OpenAI endpoints (#43463)
Signed-off-by: Lang Zhao <lang.zhao@galileo.ai> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
7738ef35b8
commit
8b8af2caf7
@@ -251,12 +251,15 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
for output in final_res.outputs:
|
||||
self._raise_if_error(output.finish_reason, request_id)
|
||||
|
||||
if request.logprobs and request.top_logprobs is not None:
|
||||
if request.logprobs and (
|
||||
request.top_logprobs is not None or request.logprob_token_ids
|
||||
):
|
||||
assert output.logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_chat_logprobs(
|
||||
token_ids=output.token_ids,
|
||||
top_logprobs=output.logprobs,
|
||||
num_output_top_logprobs=request.top_logprobs,
|
||||
logprob_token_ids=request.logprob_token_ids,
|
||||
tokenizer=tokenizer,
|
||||
return_as_token_id=request.return_tokens_as_token_ids,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user