Fix full logprobs in token-in/token-out responses (#58488)
Signed-off-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -26,3 +26,13 @@ def test_logprobs_zero_emits_sampled_token():
|
||||
num_output_top_logprobs=0,
|
||||
)
|
||||
assert len(result.content[0].top_logprobs) == 1
|
||||
|
||||
|
||||
def test_logprobs_minus_one_emits_all_tokens():
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
num_output_top_logprobs=-1,
|
||||
)
|
||||
assert len(result.content[0].top_logprobs) == 2
|
||||
|
||||
@@ -590,7 +590,10 @@ class ServingTokens(GenerateBaseServing):
|
||||
step_top_logprobs.items()
|
||||
)
|
||||
if num_output_top_logprobs is not None
|
||||
and i < max(num_output_top_logprobs, 1)
|
||||
and (
|
||||
num_output_top_logprobs == -1
|
||||
or i < max(num_output_top_logprobs, 1)
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user