[Frontend] Apply each endpoint's top_logprobs cut in derender (#59518)

Signed-off-by: Yin Li <kevinli.ai.work@gmail.com>
Co-authored-by: Yin Li <kevinli.ai.work@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Yin Li
2026-10-15 15:04:04 +00:00
committed by GitHub
co-authored by Yin Li Cyrus Leung
parent 011fa68ca9
commit 3dcfaeb4e8
12 changed files with 409 additions and 112 deletions
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import itertools
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
@@ -19,6 +20,7 @@ from vllm.entrypoints.generate.base.serving import (
build_per_request_timing_metrics,
build_spec_decoding_metrics,
clamp_prompt_logprobs,
completion_top_logprobs_limit,
format_token_id_placeholder,
)
from vllm.entrypoints.openai.completion.protocol import (
@@ -721,10 +723,12 @@ class OpenAIServingCompletion(GenerateBaseServing):
tokenizer,
return_as_token_id=should_return_as_token_id,
): max(top_lp[1].logprob, -9999.0)
for i, top_lp in enumerate(step_top_logprobs.items())
if logprob_token_ids
or num_output_top_logprobs == -1
or num_output_top_logprobs >= i
for top_lp in itertools.islice(
step_top_logprobs.items(),
completion_top_logprobs_limit(
num_output_top_logprobs, bool(logprob_token_ids)
),
)
}
)