[Frontend] Integer token IDs for generate output logprobs (GenerateLogProbs) (#58181)

Signed-off-by: Yin Li <kevinli.ai.work@gmail.com>
Co-authored-by: Yin Li <kevinli.ai.work@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Yin Li
2026-10-06 17:22:47 +00:00
committed by GitHub
co-authored by Yin Li Claude
parent e82b80099a
commit 7b665f7a58
18 changed files with 772 additions and 187 deletions
+26 -24
View File
@@ -32,6 +32,7 @@ from vllm.logger import init_logger
from vllm.logprobs import Logprob, PromptLogprobs
from vllm.lora.request import LoRARequest
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
from vllm.tracing import (
contains_trace_headers,
extract_trace_headers,
@@ -346,32 +347,33 @@ def format_token_id_placeholder(token_id: int) -> str:
return f"token_id:{token_id}"
def resolve_token_id_placeholder(
token: str, tokenizer: TokenizerLike
) -> tuple[str, list[int] | None]:
"""Decode a 'token_id:N' placeholder back to a token string and UTF-8 bytes.
def decode_token_ids(
token_ids: list[int], tokenizer: TokenizerLike
) -> list[tuple[str, list[int] | None]]:
"""Decode token ids individually to their token strings and UTF-8 bytes.
Returns (token, None) unchanged if token is not a placeholder.
This is the inverse of format_token_id_placeholder / _get_decoded_token
when return_as_token_id=True.
Uses the engine's per-token detokenization, which restores the
SentencePiece leading space that `convert_tokens_to_string` drops, so the
strings match the coupled endpoints. Ids are decoded in one batch (callers
pass a position's sampled id together with its top-k ids). An id with no
vocab entry decodes to ("", None).
"""
suffix = token.removeprefix("token_id:")
if suffix == token:
return token, None
try:
token_id = int(suffix)
except ValueError:
return token, None
token_repr = tokenizer.convert_ids_to_tokens([token_id])[0]
if token_repr is None:
logger.warning_once(
"resolve_token_id_placeholder: token_id %d has no vocab entry; "
"substituting empty string",
token_id,
)
return "", None
token_str = tokenizer.convert_tokens_to_string([token_repr])
return token_str, list(token_str.encode("utf-8", errors="replace"))
pieces = tokenizer.convert_ids_to_tokens(token_ids)
known = [tid for tid, piece in zip(token_ids, pieces) if piece is not None]
decoded = iter(convert_ids_list_to_tokens(tokenizer, known))
out: list[tuple[str, list[int] | None]] = []
for tid, piece in zip(token_ids, pieces):
if piece is None:
logger.warning_once(
"decode_token_ids: token_id %d has no vocab entry; "
"substituting empty string",
tid,
)
out.append(("", None))
continue
token_str = next(decoded)
out.append((token_str, list(token_str.encode("utf-8", errors="replace"))))
return out
def clamp_prompt_logprobs(
@@ -360,6 +360,39 @@ class GenerateRequest(BaseModel):
)
class GenerateLogProb(BaseModel):
"""A single (token, logprob) candidate on the generate wire protocol.
Unlike the OpenAI logprob shapes this carries the integer token id: the
generate server has no tokenizer, so decoding to a string belongs in
derender (or the coupled chat/completions path), not here.
"""
token_id: int
logprob: float
rank: int | None = None
class GenerateLogProbsContent(GenerateLogProb):
"""The sampled token at one position, plus its top-k candidates.
``top_logprobs`` is a list, not a dict: JSON turns dict keys into strings
and the order would be implicit. It is in the engine's order: the sampled
token first, then the remaining candidates in rank order.
"""
top_logprobs: list[GenerateLogProb] = []
class GenerateLogProbs(BaseModel):
"""Output logprobs for one choice.
``content`` holds one entry per generated token.
"""
content: list[GenerateLogProbsContent] | None = None
class GenerateChoiceBase(BaseModel):
"""Fields shared by every `output_mode` of a non-streaming choice."""
@@ -388,8 +421,8 @@ class GenerateChoiceBase(BaseModel):
class GenerateTokensChoice(GenerateChoiceBase):
logprobs: ChatCompletionLogProbs | None = None
"""Logprobs whose tokens are `token_id:N` placeholders."""
logprobs: GenerateLogProbs | None = None
"""Logprobs with integer token ids (no tokenizer on the generate server)."""
class GenerateTextChoice(GenerateChoiceBase):
@@ -413,7 +446,7 @@ class GenerateStreamChoiceBase(BaseModel):
class GenerateTokensStreamChoice(GenerateStreamChoiceBase):
logprobs: ChatCompletionLogProbs | None = None
logprobs: GenerateLogProbs | None = None
class GenerateTextStreamChoice(GenerateStreamChoiceBase):
@@ -3,6 +3,7 @@
import asyncio
import math
import time
from collections.abc import AsyncGenerator
from collections.abc import Sequence as GenericSequence
@@ -57,6 +58,9 @@ from .mm_features import (
placeholder_ranges_from_engine_input,
)
from .protocol import (
GenerateLogProb,
GenerateLogProbs,
GenerateLogProbsContent,
GenerateRequest,
GenerateResponse,
GenerateTextChoice,
@@ -72,6 +76,15 @@ from .protocol import (
logger = init_logger(__name__)
def _clamp_logprob(logprob: float) -> float:
"""The OpenAI shapes' floor for a logprob: ``-inf`` and NaN become ``-9999.0``.
``max(nan, -9999.0)`` is NaN in Python, so NaN needs its own check; NaN
would otherwise fail ``JSONResponse`` or reach the client as ``null``.
"""
return -9999.0 if math.isnan(logprob) else max(logprob, -9999.0)
def _logprob_token(
token_id: int, logprob: Logprob | None, tokenizer: TokenizerLike | None
) -> tuple[str, list[int] | None]:
@@ -94,7 +107,7 @@ def _top_logprob(
) -> ChatCompletionLogProb:
token, token_bytes = _logprob_token(token_id, logprob, tokenizer)
return ChatCompletionLogProb(
token=token, logprob=max(logprob.logprob, -9999.0), bytes=token_bytes
token=token, logprob=_clamp_logprob(logprob.logprob), bytes=token_bytes
)
@@ -406,12 +419,20 @@ class ServingTokens(GenerateBaseServing):
# This is top_logprobs in completions API
if sampling_params.logprobs is not None:
assert out_logprobs is not None, "Did not output logprobs"
logprobs = self._create_tokens_logprobs(
token_ids=token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
tokenizer=tokenizer,
)
logprobs: GenerateLogProbs | ChatCompletionLogProbs | None
if text_mode:
logprobs = self._create_text_logprobs(
token_ids=token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
tokenizer=tokenizer,
)
else:
logprobs = self._create_tokens_logprobs(
token_ids=token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
)
else:
logprobs = None
@@ -571,12 +592,20 @@ class ServingTokens(GenerateBaseServing):
if sampling_params.logprobs is not None:
out_logprobs = output.logprobs
assert out_logprobs is not None, "Did not output logprobs"
logprobs = self._create_tokens_logprobs(
token_ids=delta_token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
tokenizer=tokenizer,
)
logprobs: GenerateLogProbs | ChatCompletionLogProbs | None
if text_mode:
logprobs = self._create_text_logprobs(
token_ids=delta_token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
tokenizer=tokenizer,
)
else:
logprobs = self._create_tokens_logprobs(
token_ids=delta_token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
)
else:
logprobs = None
@@ -691,17 +720,18 @@ class ServingTokens(GenerateBaseServing):
return None
return self.renderer.tokenizer
def _create_tokens_logprobs(
def _create_text_logprobs(
self,
token_ids: GenericSequence[int],
top_logprobs: GenericSequence[dict[int, Logprob] | None],
num_output_top_logprobs: int | None = None,
tokenizer: TokenizerLike | None = None,
) -> ChatCompletionLogProbs:
"""Create OpenAI-style logprobs.
"""Create OpenAI-style logprobs for ``output_mode="text"``.
Tokens are ``token_id:N`` placeholders. With a ``tokenizer`` they are
decoded strings and carry ``bytes``.
With a ``tokenizer`` the tokens are decoded strings and carry ``bytes``;
without one (``--return-tokens-as-token-ids``) they are ``token_id:N``
placeholders.
"""
logprobs_content: list[ChatCompletionLogProbsContent] = []
@@ -719,7 +749,7 @@ class ServingTokens(GenerateBaseServing):
logprobs_content.append(
ChatCompletionLogProbsContent(
token=token,
logprob=max(step_token.logprob, -9999.0),
logprob=_clamp_logprob(step_token.logprob),
bytes=token_bytes,
top_logprobs=[
_top_logprob(top_id, logprob, tokenizer)
@@ -736,3 +766,52 @@ class ServingTokens(GenerateBaseServing):
)
return ChatCompletionLogProbs(content=logprobs_content)
def _create_tokens_logprobs(
self,
token_ids: GenericSequence[int],
top_logprobs: GenericSequence[dict[int, Logprob] | None],
num_output_top_logprobs: int | None = None,
) -> GenerateLogProbs:
"""Create generate-shaped logprobs (integer token ids, no tokenizer).
The engine reports rank 0 for a sampled token whose logprob is NaN; that
is not a rank, so it is sent as ``None`` (the logprob is clamped).
"""
logprobs_content: list[GenerateLogProbsContent] = []
for i, token_id in enumerate(token_ids):
step_top_logprobs = top_logprobs[i]
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
# Same sentinel the OpenAI shapes use when the sampled token
# has no entry in the engine's top-k map.
logprobs_content.append(
GenerateLogProbsContent(token_id=token_id, logprob=-9999.0)
)
else:
step_token = step_top_logprobs[token_id]
logprobs_content.append(
GenerateLogProbsContent(
token_id=token_id,
logprob=_clamp_logprob(step_token.logprob),
rank=step_token.rank or None,
top_logprobs=[
GenerateLogProb(
token_id=top_token_id,
logprob=_clamp_logprob(top_logprob.logprob),
rank=top_logprob.rank or None,
)
for rank_index, (top_token_id, top_logprob) in enumerate(
step_top_logprobs.items()
)
if num_output_top_logprobs is not None
and (
num_output_top_logprobs == -1
or rank_index < max(num_output_top_logprobs, 1)
)
],
)
)
return GenerateLogProbs(content=logprobs_content)