[Frontend] Apply each endpoint's top_logprobs cut in derender (#59518)

Signed-off-by: Yin Li <kevinli.ai.work@gmail.com>
Co-authored-by: Yin Li <kevinli.ai.work@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Yin Li
2026-10-15 15:04:04 +00:00
committed by GitHub
co-authored by Yin Li Cyrus Leung
parent 011fa68ca9
commit 3dcfaeb4e8
12 changed files with 409 additions and 112 deletions
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import itertools
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
@@ -22,6 +23,7 @@ from vllm.entrypoints.generate.base.serving import (
GenerateBaseServing,
build_per_request_timing_metrics,
build_spec_decoding_metrics,
chat_top_logprobs_limit,
clamp_prompt_logprobs,
format_token_id_placeholder,
)
@@ -1231,10 +1233,9 @@ class OpenAIServingChat(GenerateBaseServing):
logprob=max(p[1].logprob, -9999.0),
bytes=list(token.encode("utf-8", errors="replace")),
)
for i, p in enumerate(logprobs.items())
if return_all
or top_logprobs == -1
or (top_logprobs is not None and i < top_logprobs)
for p in itertools.islice(
logprobs.items(), chat_top_logprobs_limit(top_logprobs, return_all)
)
]
def _create_chat_logprobs(