[Fix] Fix logprobs=0 handling for /inference/v1/generate endpoint (#34010)
Signed-off-by: SumanthRH <sumanthrh99@gmail.com>
This commit is contained in:
@@ -184,7 +184,7 @@ class ServingTokens(OpenAIServing):
|
||||
out_logprobs = output.logprobs
|
||||
|
||||
# This is top_logprobs in completions API
|
||||
if sampling_params.logprobs:
|
||||
if sampling_params.logprobs is not None:
|
||||
assert out_logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_tokens_logprobs(
|
||||
token_ids=token_ids,
|
||||
@@ -284,7 +284,8 @@ class ServingTokens(OpenAIServing):
|
||||
logprob=max(p[1].logprob, -9999.0),
|
||||
)
|
||||
for i, p in enumerate(step_top_logprobs.items())
|
||||
if num_output_top_logprobs and i < num_output_top_logprobs
|
||||
if num_output_top_logprobs is not None
|
||||
and i < max(num_output_top_logprobs, 1)
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user