[Frontend] Integer token IDs for generate output logprobs (GenerateLogProbs) (#58181)

Signed-off-by: Yin Li <kevinli.ai.work@gmail.com>
Co-authored-by: Yin Li <kevinli.ai.work@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Yin Li
2026-10-06 17:22:47 +00:00
committed by GitHub
co-authored by Yin Li Claude
parent e82b80099a
commit 7b665f7a58
18 changed files with 772 additions and 187 deletions
@@ -360,6 +360,39 @@ class GenerateRequest(BaseModel):
)
class GenerateLogProb(BaseModel):
"""A single (token, logprob) candidate on the generate wire protocol.
Unlike the OpenAI logprob shapes this carries the integer token id: the
generate server has no tokenizer, so decoding to a string belongs in
derender (or the coupled chat/completions path), not here.
"""
token_id: int
logprob: float
rank: int | None = None
class GenerateLogProbsContent(GenerateLogProb):
"""The sampled token at one position, plus its top-k candidates.
``top_logprobs`` is a list, not a dict: JSON turns dict keys into strings
and the order would be implicit. It is in the engine's order: the sampled
token first, then the remaining candidates in rank order.
"""
top_logprobs: list[GenerateLogProb] = []
class GenerateLogProbs(BaseModel):
"""Output logprobs for one choice.
``content`` holds one entry per generated token.
"""
content: list[GenerateLogProbsContent] | None = None
class GenerateChoiceBase(BaseModel):
"""Fields shared by every `output_mode` of a non-streaming choice."""
@@ -388,8 +421,8 @@ class GenerateChoiceBase(BaseModel):
class GenerateTokensChoice(GenerateChoiceBase):
logprobs: ChatCompletionLogProbs | None = None
"""Logprobs whose tokens are `token_id:N` placeholders."""
logprobs: GenerateLogProbs | None = None
"""Logprobs with integer token ids (no tokenizer on the generate server)."""
class GenerateTextChoice(GenerateChoiceBase):
@@ -413,7 +446,7 @@ class GenerateStreamChoiceBase(BaseModel):
class GenerateTokensStreamChoice(GenerateStreamChoiceBase):
logprobs: ChatCompletionLogProbs | None = None
logprobs: GenerateLogProbs | None = None
class GenerateTextStreamChoice(GenerateStreamChoiceBase):
@@ -3,6 +3,7 @@
import asyncio
import math
import time
from collections.abc import AsyncGenerator
from collections.abc import Sequence as GenericSequence
@@ -57,6 +58,9 @@ from .mm_features import (
placeholder_ranges_from_engine_input,
)
from .protocol import (
GenerateLogProb,
GenerateLogProbs,
GenerateLogProbsContent,
GenerateRequest,
GenerateResponse,
GenerateTextChoice,
@@ -72,6 +76,15 @@ from .protocol import (
logger = init_logger(__name__)
def _clamp_logprob(logprob: float) -> float:
"""The OpenAI shapes' floor for a logprob: ``-inf`` and NaN become ``-9999.0``.
``max(nan, -9999.0)`` is NaN in Python, so NaN needs its own check; NaN
would otherwise fail ``JSONResponse`` or reach the client as ``null``.
"""
return -9999.0 if math.isnan(logprob) else max(logprob, -9999.0)
def _logprob_token(
token_id: int, logprob: Logprob | None, tokenizer: TokenizerLike | None
) -> tuple[str, list[int] | None]:
@@ -94,7 +107,7 @@ def _top_logprob(
) -> ChatCompletionLogProb:
token, token_bytes = _logprob_token(token_id, logprob, tokenizer)
return ChatCompletionLogProb(
token=token, logprob=max(logprob.logprob, -9999.0), bytes=token_bytes
token=token, logprob=_clamp_logprob(logprob.logprob), bytes=token_bytes
)
@@ -406,12 +419,20 @@ class ServingTokens(GenerateBaseServing):
# This is top_logprobs in completions API
if sampling_params.logprobs is not None:
assert out_logprobs is not None, "Did not output logprobs"
logprobs = self._create_tokens_logprobs(
token_ids=token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
tokenizer=tokenizer,
)
logprobs: GenerateLogProbs | ChatCompletionLogProbs | None
if text_mode:
logprobs = self._create_text_logprobs(
token_ids=token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
tokenizer=tokenizer,
)
else:
logprobs = self._create_tokens_logprobs(
token_ids=token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
)
else:
logprobs = None
@@ -571,12 +592,20 @@ class ServingTokens(GenerateBaseServing):
if sampling_params.logprobs is not None:
out_logprobs = output.logprobs
assert out_logprobs is not None, "Did not output logprobs"
logprobs = self._create_tokens_logprobs(
token_ids=delta_token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
tokenizer=tokenizer,
)
logprobs: GenerateLogProbs | ChatCompletionLogProbs | None
if text_mode:
logprobs = self._create_text_logprobs(
token_ids=delta_token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
tokenizer=tokenizer,
)
else:
logprobs = self._create_tokens_logprobs(
token_ids=delta_token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
)
else:
logprobs = None
@@ -691,17 +720,18 @@ class ServingTokens(GenerateBaseServing):
return None
return self.renderer.tokenizer
def _create_tokens_logprobs(
def _create_text_logprobs(
self,
token_ids: GenericSequence[int],
top_logprobs: GenericSequence[dict[int, Logprob] | None],
num_output_top_logprobs: int | None = None,
tokenizer: TokenizerLike | None = None,
) -> ChatCompletionLogProbs:
"""Create OpenAI-style logprobs.
"""Create OpenAI-style logprobs for ``output_mode="text"``.
Tokens are ``token_id:N`` placeholders. With a ``tokenizer`` they are
decoded strings and carry ``bytes``.
With a ``tokenizer`` the tokens are decoded strings and carry ``bytes``;
without one (``--return-tokens-as-token-ids``) they are ``token_id:N``
placeholders.
"""
logprobs_content: list[ChatCompletionLogProbsContent] = []
@@ -719,7 +749,7 @@ class ServingTokens(GenerateBaseServing):
logprobs_content.append(
ChatCompletionLogProbsContent(
token=token,
logprob=max(step_token.logprob, -9999.0),
logprob=_clamp_logprob(step_token.logprob),
bytes=token_bytes,
top_logprobs=[
_top_logprob(top_id, logprob, tokenizer)
@@ -736,3 +766,52 @@ class ServingTokens(GenerateBaseServing):
)
return ChatCompletionLogProbs(content=logprobs_content)
def _create_tokens_logprobs(
self,
token_ids: GenericSequence[int],
top_logprobs: GenericSequence[dict[int, Logprob] | None],
num_output_top_logprobs: int | None = None,
) -> GenerateLogProbs:
"""Create generate-shaped logprobs (integer token ids, no tokenizer).
The engine reports rank 0 for a sampled token whose logprob is NaN; that
is not a rank, so it is sent as ``None`` (the logprob is clamped).
"""
logprobs_content: list[GenerateLogProbsContent] = []
for i, token_id in enumerate(token_ids):
step_top_logprobs = top_logprobs[i]
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
# Same sentinel the OpenAI shapes use when the sampled token
# has no entry in the engine's top-k map.
logprobs_content.append(
GenerateLogProbsContent(token_id=token_id, logprob=-9999.0)
)
else:
step_token = step_top_logprobs[token_id]
logprobs_content.append(
GenerateLogProbsContent(
token_id=token_id,
logprob=_clamp_logprob(step_token.logprob),
rank=step_token.rank or None,
top_logprobs=[
GenerateLogProb(
token_id=top_token_id,
logprob=_clamp_logprob(top_logprob.logprob),
rank=top_logprob.rank or None,
)
for rank_index, (top_token_id, top_logprob) in enumerate(
step_top_logprobs.items()
)
if num_output_top_logprobs is not None
and (
num_output_top_logprobs == -1
or rank_index < max(num_output_top_logprobs, 1)
)
],
)
)
return GenerateLogProbs(content=logprobs_content)