[Frontend] Apply each endpoint's top_logprobs cut in derender (#59518)
Signed-off-by: Yin Li <kevinli.ai.work@gmail.com> Co-authored-by: Yin Li <kevinli.ai.work@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Yin Li
Cyrus Leung
parent
011fa68ca9
commit
3dcfaeb4e8
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, AsyncIterator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
@@ -19,6 +20,7 @@ from vllm.entrypoints.generate.base.serving import (
|
||||
build_per_request_timing_metrics,
|
||||
build_spec_decoding_metrics,
|
||||
clamp_prompt_logprobs,
|
||||
completion_top_logprobs_limit,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
@@ -721,10 +723,12 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
tokenizer,
|
||||
return_as_token_id=should_return_as_token_id,
|
||||
): max(top_lp[1].logprob, -9999.0)
|
||||
for i, top_lp in enumerate(step_top_logprobs.items())
|
||||
if logprob_token_ids
|
||||
or num_output_top_logprobs == -1
|
||||
or num_output_top_logprobs >= i
|
||||
for top_lp in itertools.islice(
|
||||
step_top_logprobs.items(),
|
||||
completion_top_logprobs_limit(
|
||||
num_output_top_logprobs, bool(logprob_token_ids)
|
||||
),
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user