[Frontend] Apply each endpoint's top_logprobs cut in derender (#59518)

Signed-off-by: Yin Li <kevinli.ai.work@gmail.com>
Co-authored-by: Yin Li <kevinli.ai.work@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Yin Li
2026-10-15 15:04:04 +00:00
committed by GitHub
co-authored by Yin Li Cyrus Leung
parent 011fa68ca9
commit 3dcfaeb4e8
12 changed files with 409 additions and 112 deletions
+26
View File
@@ -356,6 +356,32 @@ def decode_token_ids(
return out
def chat_top_logprobs_limit(
top_logprobs: int | None, return_all: bool = False
) -> int | None:
"""How many of a position's logprob candidates `/v1/chat/completions`
returns, in the engine's order (sampled token first): the first
``top_logprobs``, none when it is unset, and all of them (``None``) for
``top_logprobs=-1`` or ``logprob_token_ids`` (``return_all``). Shared with
chat derender so the two cannot drift."""
if return_all or top_logprobs == -1:
return None
return top_logprobs or 0
def completion_top_logprobs_limit(
logprobs: int | None, return_all: bool = False
) -> int | None:
"""How many of a position's logprob candidates `/v1/completions` returns:
the first ``logprobs + 1`` (the OpenAI completions convention), and all of
them (``None``) for ``logprobs=-1``, ``logprob_token_ids`` (``return_all``)
or when ``logprobs`` is unset. Shared with completions derender so the two
cannot drift."""
if return_all or logprobs is None or logprobs == -1:
return None
return max(logprobs + 1, 0)
def clamp_prompt_logprobs(
prompt_logprobs: PromptLogprobs | None,
) -> PromptLogprobs | None:
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import itertools
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
@@ -22,6 +23,7 @@ from vllm.entrypoints.generate.base.serving import (
GenerateBaseServing,
build_per_request_timing_metrics,
build_spec_decoding_metrics,
chat_top_logprobs_limit,
clamp_prompt_logprobs,
format_token_id_placeholder,
)
@@ -1231,10 +1233,9 @@ class OpenAIServingChat(GenerateBaseServing):
logprob=max(p[1].logprob, -9999.0),
bytes=list(token.encode("utf-8", errors="replace")),
)
for i, p in enumerate(logprobs.items())
if return_all
or top_logprobs == -1
or (top_logprobs is not None and i < top_logprobs)
for p in itertools.islice(
logprobs.items(), chat_top_logprobs_limit(top_logprobs, return_all)
)
]
def _create_chat_logprobs(
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import itertools
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
@@ -19,6 +20,7 @@ from vllm.entrypoints.generate.base.serving import (
build_per_request_timing_metrics,
build_spec_decoding_metrics,
clamp_prompt_logprobs,
completion_top_logprobs_limit,
format_token_id_placeholder,
)
from vllm.entrypoints.openai.completion.protocol import (
@@ -721,10 +723,12 @@ class OpenAIServingCompletion(GenerateBaseServing):
tokenizer,
return_as_token_id=should_return_as_token_id,
): max(top_lp[1].logprob, -9999.0)
for i, top_lp in enumerate(step_top_logprobs.items())
if logprob_token_ids
or num_output_top_logprobs == -1
or num_output_top_logprobs >= i
for top_lp in itertools.islice(
step_top_logprobs.items(),
completion_top_logprobs_limit(
num_output_top_logprobs, bool(logprob_token_ids)
),
)
}
)
@@ -424,14 +424,12 @@ class ServingTokens(GenerateBaseServing):
logprobs = self._create_text_logprobs(
token_ids=token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
tokenizer=tokenizer,
)
else:
logprobs = self._create_tokens_logprobs(
token_ids=token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
)
else:
logprobs = None
@@ -599,14 +597,12 @@ class ServingTokens(GenerateBaseServing):
logprobs = self._create_text_logprobs(
token_ids=delta_token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
tokenizer=tokenizer,
)
else:
logprobs = self._create_tokens_logprobs(
token_ids=delta_token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
)
else:
logprobs = None
@@ -728,14 +724,15 @@ class ServingTokens(GenerateBaseServing):
self,
token_ids: GenericSequence[int],
top_logprobs: GenericSequence[dict[int, Logprob] | None],
num_output_top_logprobs: int | None = None,
tokenizer: TokenizerLike | None = None,
) -> ChatCompletionLogProbs:
"""Create OpenAI-style logprobs for ``output_mode="text"``.
With a ``tokenizer`` the tokens are decoded strings and carry ``bytes``;
without one (``--return-tokens-as-token-ids``) they are ``token_id:N``
placeholders.
placeholders. Like tokens mode, ``top_logprobs`` carries every candidate
the engine returned (the sampled token first, then the top k), not cut
to the requested count.
"""
logprobs_content: list[ChatCompletionLogProbsContent] = []
@@ -757,14 +754,7 @@ class ServingTokens(GenerateBaseServing):
bytes=token_bytes,
top_logprobs=[
_top_logprob(top_id, logprob, tokenizer)
for i, (top_id, logprob) in enumerate(
step_top_logprobs.items()
)
if num_output_top_logprobs is not None
and (
num_output_top_logprobs == -1
or i < max(num_output_top_logprobs, 1)
)
for top_id, logprob in step_top_logprobs.items()
],
)
)
@@ -775,10 +765,16 @@ class ServingTokens(GenerateBaseServing):
self,
token_ids: GenericSequence[int],
top_logprobs: GenericSequence[dict[int, Logprob] | None],
num_output_top_logprobs: int | None = None,
) -> GenerateLogProbs:
"""Create generate-shaped logprobs (integer token ids, no tokenizer).
``top_logprobs`` carries every candidate the engine returned for the
position: the sampled token first, then the top k, so k entries when the
sampled token is in the top k and k + 1 when it is not. It is not cut to
the requested count: `/v1/chat/completions` and
`/v1/completions` cut differently, and derender applies each endpoint's
cut the way the coupled server does.
The engine reports rank 0 for a sampled token whose logprob is NaN; that
is not a rank, so it is sent as ``None`` (the logprob is clamped).
"""
@@ -806,14 +802,7 @@ class ServingTokens(GenerateBaseServing):
logprob=_clamp_logprob(top_logprob.logprob),
rank=top_logprob.rank or None,
)
for rank_index, (top_token_id, top_logprob) in enumerate(
step_top_logprobs.items()
)
if num_output_top_logprobs is not None
and (
num_output_top_logprobs == -1
or rank_index < max(num_output_top_logprobs, 1)
)
for top_token_id, top_logprob in step_top_logprobs.items()
],
)
)