[Frontend] Integer token IDs for generate output logprobs (GenerateLogProbs) (#58181)
Signed-off-by: Yin Li <kevinli.ai.work@gmail.com> Co-authored-by: Yin Li <kevinli.ai.work@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -32,6 +32,7 @@ from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob, PromptLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
|
||||
from vllm.tracing import (
|
||||
contains_trace_headers,
|
||||
extract_trace_headers,
|
||||
@@ -346,32 +347,33 @@ def format_token_id_placeholder(token_id: int) -> str:
|
||||
return f"token_id:{token_id}"
|
||||
|
||||
|
||||
def resolve_token_id_placeholder(
|
||||
token: str, tokenizer: TokenizerLike
|
||||
) -> tuple[str, list[int] | None]:
|
||||
"""Decode a 'token_id:N' placeholder back to a token string and UTF-8 bytes.
|
||||
def decode_token_ids(
|
||||
token_ids: list[int], tokenizer: TokenizerLike
|
||||
) -> list[tuple[str, list[int] | None]]:
|
||||
"""Decode token ids individually to their token strings and UTF-8 bytes.
|
||||
|
||||
Returns (token, None) unchanged if token is not a placeholder.
|
||||
This is the inverse of format_token_id_placeholder / _get_decoded_token
|
||||
when return_as_token_id=True.
|
||||
Uses the engine's per-token detokenization, which restores the
|
||||
SentencePiece leading space that `convert_tokens_to_string` drops, so the
|
||||
strings match the coupled endpoints. Ids are decoded in one batch (callers
|
||||
pass a position's sampled id together with its top-k ids). An id with no
|
||||
vocab entry decodes to ("", None).
|
||||
"""
|
||||
suffix = token.removeprefix("token_id:")
|
||||
if suffix == token:
|
||||
return token, None
|
||||
try:
|
||||
token_id = int(suffix)
|
||||
except ValueError:
|
||||
return token, None
|
||||
token_repr = tokenizer.convert_ids_to_tokens([token_id])[0]
|
||||
if token_repr is None:
|
||||
logger.warning_once(
|
||||
"resolve_token_id_placeholder: token_id %d has no vocab entry; "
|
||||
"substituting empty string",
|
||||
token_id,
|
||||
)
|
||||
return "", None
|
||||
token_str = tokenizer.convert_tokens_to_string([token_repr])
|
||||
return token_str, list(token_str.encode("utf-8", errors="replace"))
|
||||
pieces = tokenizer.convert_ids_to_tokens(token_ids)
|
||||
known = [tid for tid, piece in zip(token_ids, pieces) if piece is not None]
|
||||
decoded = iter(convert_ids_list_to_tokens(tokenizer, known))
|
||||
out: list[tuple[str, list[int] | None]] = []
|
||||
for tid, piece in zip(token_ids, pieces):
|
||||
if piece is None:
|
||||
logger.warning_once(
|
||||
"decode_token_ids: token_id %d has no vocab entry; "
|
||||
"substituting empty string",
|
||||
tid,
|
||||
)
|
||||
out.append(("", None))
|
||||
continue
|
||||
token_str = next(decoded)
|
||||
out.append((token_str, list(token_str.encode("utf-8", errors="replace"))))
|
||||
return out
|
||||
|
||||
|
||||
def clamp_prompt_logprobs(
|
||||
|
||||
@@ -360,6 +360,39 @@ class GenerateRequest(BaseModel):
|
||||
)
|
||||
|
||||
|
||||
class GenerateLogProb(BaseModel):
|
||||
"""A single (token, logprob) candidate on the generate wire protocol.
|
||||
|
||||
Unlike the OpenAI logprob shapes this carries the integer token id: the
|
||||
generate server has no tokenizer, so decoding to a string belongs in
|
||||
derender (or the coupled chat/completions path), not here.
|
||||
"""
|
||||
|
||||
token_id: int
|
||||
logprob: float
|
||||
rank: int | None = None
|
||||
|
||||
|
||||
class GenerateLogProbsContent(GenerateLogProb):
|
||||
"""The sampled token at one position, plus its top-k candidates.
|
||||
|
||||
``top_logprobs`` is a list, not a dict: JSON turns dict keys into strings
|
||||
and the order would be implicit. It is in the engine's order: the sampled
|
||||
token first, then the remaining candidates in rank order.
|
||||
"""
|
||||
|
||||
top_logprobs: list[GenerateLogProb] = []
|
||||
|
||||
|
||||
class GenerateLogProbs(BaseModel):
|
||||
"""Output logprobs for one choice.
|
||||
|
||||
``content`` holds one entry per generated token.
|
||||
"""
|
||||
|
||||
content: list[GenerateLogProbsContent] | None = None
|
||||
|
||||
|
||||
class GenerateChoiceBase(BaseModel):
|
||||
"""Fields shared by every `output_mode` of a non-streaming choice."""
|
||||
|
||||
@@ -388,8 +421,8 @@ class GenerateChoiceBase(BaseModel):
|
||||
|
||||
|
||||
class GenerateTokensChoice(GenerateChoiceBase):
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
"""Logprobs whose tokens are `token_id:N` placeholders."""
|
||||
logprobs: GenerateLogProbs | None = None
|
||||
"""Logprobs with integer token ids (no tokenizer on the generate server)."""
|
||||
|
||||
|
||||
class GenerateTextChoice(GenerateChoiceBase):
|
||||
@@ -413,7 +446,7 @@ class GenerateStreamChoiceBase(BaseModel):
|
||||
|
||||
|
||||
class GenerateTokensStreamChoice(GenerateStreamChoiceBase):
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
logprobs: GenerateLogProbs | None = None
|
||||
|
||||
|
||||
class GenerateTextStreamChoice(GenerateStreamChoiceBase):
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
@@ -57,6 +58,9 @@ from .mm_features import (
|
||||
placeholder_ranges_from_engine_input,
|
||||
)
|
||||
from .protocol import (
|
||||
GenerateLogProb,
|
||||
GenerateLogProbs,
|
||||
GenerateLogProbsContent,
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
GenerateTextChoice,
|
||||
@@ -72,6 +76,15 @@ from .protocol import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _clamp_logprob(logprob: float) -> float:
|
||||
"""The OpenAI shapes' floor for a logprob: ``-inf`` and NaN become ``-9999.0``.
|
||||
|
||||
``max(nan, -9999.0)`` is NaN in Python, so NaN needs its own check; NaN
|
||||
would otherwise fail ``JSONResponse`` or reach the client as ``null``.
|
||||
"""
|
||||
return -9999.0 if math.isnan(logprob) else max(logprob, -9999.0)
|
||||
|
||||
|
||||
def _logprob_token(
|
||||
token_id: int, logprob: Logprob | None, tokenizer: TokenizerLike | None
|
||||
) -> tuple[str, list[int] | None]:
|
||||
@@ -94,7 +107,7 @@ def _top_logprob(
|
||||
) -> ChatCompletionLogProb:
|
||||
token, token_bytes = _logprob_token(token_id, logprob, tokenizer)
|
||||
return ChatCompletionLogProb(
|
||||
token=token, logprob=max(logprob.logprob, -9999.0), bytes=token_bytes
|
||||
token=token, logprob=_clamp_logprob(logprob.logprob), bytes=token_bytes
|
||||
)
|
||||
|
||||
|
||||
@@ -406,12 +419,20 @@ class ServingTokens(GenerateBaseServing):
|
||||
# This is top_logprobs in completions API
|
||||
if sampling_params.logprobs is not None:
|
||||
assert out_logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_tokens_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
logprobs: GenerateLogProbs | ChatCompletionLogProbs | None
|
||||
if text_mode:
|
||||
logprobs = self._create_text_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
else:
|
||||
logprobs = self._create_tokens_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
)
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
@@ -571,12 +592,20 @@ class ServingTokens(GenerateBaseServing):
|
||||
if sampling_params.logprobs is not None:
|
||||
out_logprobs = output.logprobs
|
||||
assert out_logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_tokens_logprobs(
|
||||
token_ids=delta_token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
logprobs: GenerateLogProbs | ChatCompletionLogProbs | None
|
||||
if text_mode:
|
||||
logprobs = self._create_text_logprobs(
|
||||
token_ids=delta_token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
else:
|
||||
logprobs = self._create_tokens_logprobs(
|
||||
token_ids=delta_token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
)
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
@@ -691,17 +720,18 @@ class ServingTokens(GenerateBaseServing):
|
||||
return None
|
||||
return self.renderer.tokenizer
|
||||
|
||||
def _create_tokens_logprobs(
|
||||
def _create_text_logprobs(
|
||||
self,
|
||||
token_ids: GenericSequence[int],
|
||||
top_logprobs: GenericSequence[dict[int, Logprob] | None],
|
||||
num_output_top_logprobs: int | None = None,
|
||||
tokenizer: TokenizerLike | None = None,
|
||||
) -> ChatCompletionLogProbs:
|
||||
"""Create OpenAI-style logprobs.
|
||||
"""Create OpenAI-style logprobs for ``output_mode="text"``.
|
||||
|
||||
Tokens are ``token_id:N`` placeholders. With a ``tokenizer`` they are
|
||||
decoded strings and carry ``bytes``.
|
||||
With a ``tokenizer`` the tokens are decoded strings and carry ``bytes``;
|
||||
without one (``--return-tokens-as-token-ids``) they are ``token_id:N``
|
||||
placeholders.
|
||||
"""
|
||||
logprobs_content: list[ChatCompletionLogProbsContent] = []
|
||||
|
||||
@@ -719,7 +749,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
logprobs_content.append(
|
||||
ChatCompletionLogProbsContent(
|
||||
token=token,
|
||||
logprob=max(step_token.logprob, -9999.0),
|
||||
logprob=_clamp_logprob(step_token.logprob),
|
||||
bytes=token_bytes,
|
||||
top_logprobs=[
|
||||
_top_logprob(top_id, logprob, tokenizer)
|
||||
@@ -736,3 +766,52 @@ class ServingTokens(GenerateBaseServing):
|
||||
)
|
||||
|
||||
return ChatCompletionLogProbs(content=logprobs_content)
|
||||
|
||||
def _create_tokens_logprobs(
|
||||
self,
|
||||
token_ids: GenericSequence[int],
|
||||
top_logprobs: GenericSequence[dict[int, Logprob] | None],
|
||||
num_output_top_logprobs: int | None = None,
|
||||
) -> GenerateLogProbs:
|
||||
"""Create generate-shaped logprobs (integer token ids, no tokenizer).
|
||||
|
||||
The engine reports rank 0 for a sampled token whose logprob is NaN; that
|
||||
is not a rank, so it is sent as ``None`` (the logprob is clamped).
|
||||
"""
|
||||
logprobs_content: list[GenerateLogProbsContent] = []
|
||||
|
||||
for i, token_id in enumerate(token_ids):
|
||||
step_top_logprobs = top_logprobs[i]
|
||||
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
|
||||
# Same sentinel the OpenAI shapes use when the sampled token
|
||||
# has no entry in the engine's top-k map.
|
||||
logprobs_content.append(
|
||||
GenerateLogProbsContent(token_id=token_id, logprob=-9999.0)
|
||||
)
|
||||
else:
|
||||
step_token = step_top_logprobs[token_id]
|
||||
|
||||
logprobs_content.append(
|
||||
GenerateLogProbsContent(
|
||||
token_id=token_id,
|
||||
logprob=_clamp_logprob(step_token.logprob),
|
||||
rank=step_token.rank or None,
|
||||
top_logprobs=[
|
||||
GenerateLogProb(
|
||||
token_id=top_token_id,
|
||||
logprob=_clamp_logprob(top_logprob.logprob),
|
||||
rank=top_logprob.rank or None,
|
||||
)
|
||||
for rank_index, (top_token_id, top_logprob) in enumerate(
|
||||
step_top_logprobs.items()
|
||||
)
|
||||
if num_output_top_logprobs is not None
|
||||
and (
|
||||
num_output_top_logprobs == -1
|
||||
or rank_index < max(num_output_top_logprobs, 1)
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
return GenerateLogProbs(content=logprobs_content)
|
||||
|
||||
Reference in New Issue
Block a user