[Frontend] Integer token IDs for generate output logprobs (GenerateLogProbs) (#58181)
Signed-off-by: Yin Li <kevinli.ai.work@gmail.com> Co-authored-by: Yin Li <kevinli.ai.work@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -32,6 +32,7 @@ from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob, PromptLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
|
||||
from vllm.tracing import (
|
||||
contains_trace_headers,
|
||||
extract_trace_headers,
|
||||
@@ -346,32 +347,33 @@ def format_token_id_placeholder(token_id: int) -> str:
|
||||
return f"token_id:{token_id}"
|
||||
|
||||
|
||||
def resolve_token_id_placeholder(
|
||||
token: str, tokenizer: TokenizerLike
|
||||
) -> tuple[str, list[int] | None]:
|
||||
"""Decode a 'token_id:N' placeholder back to a token string and UTF-8 bytes.
|
||||
def decode_token_ids(
|
||||
token_ids: list[int], tokenizer: TokenizerLike
|
||||
) -> list[tuple[str, list[int] | None]]:
|
||||
"""Decode token ids individually to their token strings and UTF-8 bytes.
|
||||
|
||||
Returns (token, None) unchanged if token is not a placeholder.
|
||||
This is the inverse of format_token_id_placeholder / _get_decoded_token
|
||||
when return_as_token_id=True.
|
||||
Uses the engine's per-token detokenization, which restores the
|
||||
SentencePiece leading space that `convert_tokens_to_string` drops, so the
|
||||
strings match the coupled endpoints. Ids are decoded in one batch (callers
|
||||
pass a position's sampled id together with its top-k ids). An id with no
|
||||
vocab entry decodes to ("", None).
|
||||
"""
|
||||
suffix = token.removeprefix("token_id:")
|
||||
if suffix == token:
|
||||
return token, None
|
||||
try:
|
||||
token_id = int(suffix)
|
||||
except ValueError:
|
||||
return token, None
|
||||
token_repr = tokenizer.convert_ids_to_tokens([token_id])[0]
|
||||
if token_repr is None:
|
||||
logger.warning_once(
|
||||
"resolve_token_id_placeholder: token_id %d has no vocab entry; "
|
||||
"substituting empty string",
|
||||
token_id,
|
||||
)
|
||||
return "", None
|
||||
token_str = tokenizer.convert_tokens_to_string([token_repr])
|
||||
return token_str, list(token_str.encode("utf-8", errors="replace"))
|
||||
pieces = tokenizer.convert_ids_to_tokens(token_ids)
|
||||
known = [tid for tid, piece in zip(token_ids, pieces) if piece is not None]
|
||||
decoded = iter(convert_ids_list_to_tokens(tokenizer, known))
|
||||
out: list[tuple[str, list[int] | None]] = []
|
||||
for tid, piece in zip(token_ids, pieces):
|
||||
if piece is None:
|
||||
logger.warning_once(
|
||||
"decode_token_ids: token_id %d has no vocab entry; "
|
||||
"substituting empty string",
|
||||
tid,
|
||||
)
|
||||
out.append(("", None))
|
||||
continue
|
||||
token_str = next(decoded)
|
||||
out.append((token_str, list(token_str.encode("utf-8", errors="replace"))))
|
||||
return out
|
||||
|
||||
|
||||
def clamp_prompt_logprobs(
|
||||
|
||||
Reference in New Issue
Block a user