[Frontend] Integer token IDs for generate output logprobs (GenerateLogProbs) (#58181)

Signed-off-by: Yin Li <kevinli.ai.work@gmail.com>
Co-authored-by: Yin Li <kevinli.ai.work@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Yin Li
2026-10-06 17:22:47 +00:00
committed by GitHub
co-authored by Yin Li Claude
parent e82b80099a
commit 7b665f7a58
18 changed files with 772 additions and 187 deletions
@@ -74,14 +74,13 @@ def _make_generate_response(
}
def _make_logprobs_with_placeholders(token_id: int = 1234) -> dict:
def _make_generate_logprobs(token_id: int = 1234) -> dict:
"""A `GenerateLogProbs` payload: integer token ids, no token/bytes."""
entry = {
"token": f"token_id:{token_id}",
"token_id": token_id,
"logprob": -1.0,
"bytes": None,
"top_logprobs": [
{"token": f"token_id:{token_id + 1}", "logprob": -2.0, "bytes": None}
],
"rank": 1,
"top_logprobs": [{"token_id": token_id + 1, "logprob": -2.0, "rank": 2}],
}
return {"content": [entry]}
@@ -154,7 +153,7 @@ async def test_derender_chat_usage_default(client):
@pytest.mark.asyncio
async def test_derender_chat_logprobs(client):
"""token_id:N placeholders in content.token are resolved to real strings."""
"""Integer token ids are decoded to real strings by derender."""
gen_req = await _render_chat(client)
synthetic_ids = gen_req["token_ids"][:3]
token_id = synthetic_ids[0]
@@ -165,7 +164,7 @@ async def test_derender_chat_logprobs(client):
"model": MODEL_NAME,
"generate_response": _make_generate_response(
synthetic_ids,
logprobs=_make_logprobs_with_placeholders(token_id),
logprobs=_make_generate_logprobs(token_id),
),
},
)
@@ -176,8 +175,8 @@ async def test_derender_chat_logprobs(client):
content = logprobs["content"]
assert content is not None and len(content) == 1
token_str = content[0]["token"]
assert not token_str.startswith("token_id:"), (
f"Placeholder was not resolved: {token_str!r}"
assert token_str and not token_str.startswith("token_id:"), (
f"token id was not decoded: {token_str!r}"
)
@@ -194,7 +193,7 @@ async def test_derender_chat_logprobs_bytes(client):
"model": MODEL_NAME,
"generate_response": _make_generate_response(
synthetic_ids,
logprobs=_make_logprobs_with_placeholders(token_id),
logprobs=_make_generate_logprobs(token_id),
),
},
)
@@ -208,7 +207,7 @@ async def test_derender_chat_logprobs_bytes(client):
@pytest.mark.asyncio
async def test_derender_chat_top_logprobs(client):
"""top_logprobs entries also have their placeholders resolved."""
"""top_logprobs entries are decoded too."""
gen_req = await _render_chat(client)
synthetic_ids = gen_req["token_ids"][:3]
token_id = synthetic_ids[0]
@@ -219,7 +218,7 @@ async def test_derender_chat_top_logprobs(client):
"model": MODEL_NAME,
"generate_response": _make_generate_response(
synthetic_ids,
logprobs=_make_logprobs_with_placeholders(token_id),
logprobs=_make_generate_logprobs(token_id),
),
},
)
@@ -228,7 +227,7 @@ async def test_derender_chat_top_logprobs(client):
top = content[0]["top_logprobs"]
assert len(top) == 1
assert not top[0]["token"].startswith("token_id:"), (
f"top_logprobs placeholder not resolved: {top[0]['token']!r}"
f"top_logprobs token id not decoded: {top[0]['token']!r}"
)
@@ -633,8 +632,8 @@ async def test_derender_completion_empty_generate_responses(client):
@pytest.mark.asyncio
async def test_derender_completion_logprobs(client):
"""token_id:N placeholders in logprobs are resolved; CompletionLogProbs
flat-list structure is returned with non-empty tokens and text_offsets."""
"""Integer token ids are decoded; CompletionLogProbs flat-list structure is
returned with non-empty tokens and text_offsets."""
gr1 = await _render_completion(client, "Hello world")
ids1 = gr1["token_ids"][:3]
token_id = ids1[0]
@@ -647,7 +646,7 @@ async def test_derender_completion_logprobs(client):
_make_completion_generate_response(
ids1,
gr1["request_id"],
logprobs=_make_logprobs_with_placeholders(token_id),
logprobs=_make_generate_logprobs(token_id),
),
],
},
@@ -658,7 +657,7 @@ async def test_derender_completion_logprobs(client):
tokens = logprobs["tokens"]
assert len(tokens) == 1
assert not tokens[0].startswith("token_id:"), (
f"Placeholder was not resolved: {tokens[0]!r}"
f"token id was not decoded: {tokens[0]!r}"
)
assert len(logprobs["token_logprobs"]) == 1
assert isinstance(logprobs["token_logprobs"][0], float)
@@ -794,7 +793,7 @@ async def test_derender_chat_oversized_logprobs_rejected(client):
"""logprobs.content longer than max_model_len returns 400."""
oversized_logprobs: dict = {
"content": [
{"token": "x", "logprob": -1.0, "bytes": None, "top_logprobs": []}
{"token_id": 42, "logprob": -1.0, "rank": 1, "top_logprobs": []}
for _ in range(1_000_000)
]
}
@@ -825,11 +824,11 @@ async def test_derender_chat_oversized_top_logprobs_rejected(client):
oversized_top_logprobs = {
"content": [
{
"token": "x",
"token_id": 42,
"logprob": -1.0,
"bytes": None,
"rank": 1,
"top_logprobs": [
{"token": f"t{i}", "logprob": -float(i), "bytes": None}
{"token_id": i, "logprob": -float(i), "rank": i + 1}
for i in range(25)
],
}
@@ -237,21 +237,47 @@ async def test_parity_reasoning_and_tool_call(client):
assert _tool_sig(disagg["choices"][0])
def _to_generate_logprobs(openai_logprobs: dict) -> dict:
"""Turn a coupled response's `token_id:N` logprobs into `GenerateLogProbs`.
A real GPU-less worker emits integer ids directly; here the coupled server
with `return_tokens_as_token_ids=True` is the only source of the same ids,
so the placeholders are parsed back out to build the generate-shaped
payload that `/derender` now consumes.
"""
def _id(token: str) -> int:
assert token.startswith("token_id:"), token
return int(token.removeprefix("token_id:"))
return {
"content": [
{
"token_id": _id(entry["token"]),
"logprob": entry["logprob"],
"top_logprobs": [
{"token_id": _id(top["token"]), "logprob": top["logprob"]}
for top in entry["top_logprobs"]
],
}
for entry in openai_logprobs["content"]
]
}
@pytest.mark.asyncio
async def test_parity_logprobs(client):
"""token_id:N resolution parity vs. the coupled server's real strings.
"""Integer-id logprob decoding parity vs. the coupled server's strings.
A real disaggregated worker only has token IDs so it emits logprobs
with `token_id:N` placeholders (`return_tokens_as_token_ids=True`
reproduces that shape here). `/derender` must resolve those
placeholders to the same token strings/bytes the coupled server
resolves them to directly.
A real disaggregated worker only has token IDs, so it emits
`GenerateLogProbs` with integer `token_id`s. `/derender` must decode those
to the same token strings/bytes the coupled server produces directly.
"""
messages = [{"role": "user", "content": "What is 2+2?"}]
extra = {"logprobs": True, "top_logprobs": 3}
# What a real GPU less worker would hand to /derender is token IDs plus
# logprobs still in token_id:N placeholder form
# The coupled server with placeholders is used only as a source of the
# token ids a GPU-less worker would have had.
placeholder = await _coupled(
client, messages, return_tokens_as_token_ids=True, **extra
)
@@ -263,7 +289,7 @@ async def test_parity_logprobs(client):
placeholder["usage"]["prompt_tokens"],
ch["finish_reason"],
chat_request,
logprobs=ch["logprobs"],
logprobs=_to_generate_logprobs(ch["logprobs"]),
)
# The coupled server resolving the same greedy generation
@@ -64,6 +64,7 @@ from vllm.renderers.online_derenderer import (
_seed_stream_state,
)
from vllm.tokenizers import get_tokenizer
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
from vllm.utils import random_uuid
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
@@ -212,17 +213,12 @@ def _make_stream_chunk(
)
def _placeholder_logprobs(token_ids: list[int]) -> dict:
"""Per-token logprob entries using token_id:N placeholders, as sent by
the generate worker."""
def _generate_logprobs(token_ids: list[int]) -> dict:
"""Per-token `GenerateLogProbs` entries (integer token ids, no token or
bytes), as sent by the generate worker."""
return {
"content": [
{
"token": f"token_id:{tid}",
"logprob": -0.5,
"bytes": None,
"top_logprobs": [],
}
{"token_id": tid, "logprob": -0.5, "rank": 1, "top_logprobs": []}
for tid in token_ids
]
}
@@ -892,12 +888,12 @@ class TestDerenderCompletionStream:
class TestStreamLogprobs:
"""Streaming derender must carry per-chunk logprobs with placeholders
resolved, matching what the generate streaming path emits."""
"""Streaming derender must carry per-chunk logprobs with token ids
decoded, matching what the generate streaming path emits."""
@pytest.mark.asyncio
async def test_chat_stream_logprobs_resolved_per_chunk(self, derenderer, tokenizer):
"""Each streamed chunk carries logprobs with token_id:N resolved."""
"""Each streamed chunk carries logprobs with its token ids decoded."""
token_ids = tokenizer.encode("hello world")[:6]
mid = len(token_ids) // 2
@@ -906,17 +902,15 @@ class TestStreamLogprobs:
chunk, state = await derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
part, logprobs=_placeholder_logprobs(part)
part, logprobs=_generate_logprobs(part)
),
state=state,
)
logprobs = chunk.choices[0].logprobs
assert logprobs is not None and logprobs.content is not None
assert len(logprobs.content) == len(part)
for entry in logprobs.content:
assert not entry.token.startswith("token_id:"), (
f"placeholder not resolved: {entry.token!r}"
)
assert [entry.token for entry in logprobs.content] == (
convert_ids_list_to_tokens(tokenizer, part)
)
@pytest.mark.asyncio
async def test_chat_stream_logprobs_multibyte_across_chunks(
@@ -936,7 +930,7 @@ class TestStreamLogprobs:
last_chunk, state = await derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
[tid], logprobs=_placeholder_logprobs([tid])
[tid], logprobs=_generate_logprobs([tid])
),
state=state,
)
@@ -959,13 +953,13 @@ class TestStreamLogprobs:
chunk1, state = await derenderer.derender_completion_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
token_ids[:mid], logprobs=_placeholder_logprobs(token_ids[:mid])
token_ids[:mid], logprobs=_generate_logprobs(token_ids[:mid])
),
)
chunk2, _ = await derenderer.derender_completion_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
token_ids[mid:], logprobs=_placeholder_logprobs(token_ids[mid:])
token_ids[mid:], logprobs=_generate_logprobs(token_ids[mid:])
),
state=state,
)
@@ -982,7 +976,7 @@ class TestStreamLogprobs:
_, state = await derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
[tid], logprobs=_placeholder_logprobs([tid])
[tid], logprobs=_generate_logprobs([tid])
),
state=state,
)
@@ -0,0 +1,172 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the generate -> OpenAI logprob conversion in derender.
These exercise `_resolve_logprobs` directly. A stub tokenizer covers the
byte-fallback correction path; a real SentencePiece tokenizer covers the
leading-space restoration.
"""
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateLogProb,
GenerateLogProbs,
GenerateLogProbsContent,
)
from vllm.renderers.online_derenderer import (
_convert_chat_logprobs_to_completion_logprobs,
_resolve_logprobs,
)
from vllm.tokenizers import get_tokenizer
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
# A two-byte character split across two byte-fallback tokens: neither decodes
# to anything printable on its own, but the pair decodes to "é".
FIRST_BYTE = 10
SECOND_BYTE = 11
PLAIN = 12
VOCAB = {FIRST_BYTE: "<0xC3>", SECOND_BYTE: "<0xA9>", PLAIN: "ok"}
class _StubTokenizer:
"""Decodes byte-fallback tokens only when both halves are present."""
def convert_ids_to_tokens(self, ids: list[int]) -> list[str | None]:
return [VOCAB.get(i) for i in ids]
def convert_tokens_to_string(self, tokens: list[str]) -> str:
return self.decode([k for t in tokens for k, v in VOCAB.items() if v == t])
def decode(self, ids: list[int], **kwargs) -> str:
out = ""
i = 0
while i < len(ids):
if ids[i] == FIRST_BYTE and i + 1 < len(ids) and ids[i + 1] == SECOND_BYTE:
out += "é"
i += 2
elif ids[i] in (FIRST_BYTE, SECOND_BYTE):
out += "�"
i += 1
else:
out += VOCAB.get(ids[i], "")
i += 1
return out
def test_token_ids_are_decoded_with_bytes_and_order_preserved():
logprobs = GenerateLogProbs(
content=[
GenerateLogProbsContent(
token_id=PLAIN,
logprob=-0.5,
rank=1,
top_logprobs=[
GenerateLogProb(token_id=PLAIN, logprob=-0.5, rank=1),
GenerateLogProb(token_id=FIRST_BYTE, logprob=-2.0, rank=2),
],
)
]
)
resolved = _resolve_logprobs(logprobs, _StubTokenizer())
assert resolved.content is not None
entry = resolved.content[0]
assert entry.token == "ok"
assert entry.logprob == -0.5
assert entry.bytes == list(b"ok")
# Rank order survives the conversion (the OpenAI shape has no rank field).
assert [t.logprob for t in entry.top_logprobs] == [-0.5, -2.0]
assert entry.top_logprobs[0].token == "ok"
def test_byte_fallback_second_half_uses_preceding_token_as_context():
"""U+FFFD correction needs the preceding sampled ids, which the integer
shape carries directly (no placeholder parsing)."""
logprobs = GenerateLogProbs(
content=[
GenerateLogProbsContent(token_id=FIRST_BYTE, logprob=-0.1),
GenerateLogProbsContent(token_id=SECOND_BYTE, logprob=-0.2),
]
)
resolved = _resolve_logprobs(logprobs, _StubTokenizer())
assert resolved.content is not None
# The first half has no preceding context to repair it with, so it stays
# empty (existing `_correct_decoded_token` behaviour). The second resolves
# to the complete character once the first is used as context.
assert resolved.content[0].token == ""
assert resolved.content[0].bytes == []
assert resolved.content[1].token == "é"
assert resolved.content[1].bytes == list("é".encode())
def test_content_none_is_not_an_error():
resolved = _resolve_logprobs(GenerateLogProbs(), _StubTokenizer())
assert resolved.content is None
def test_sentencepiece_leading_space_matches_engine():
"""`▁true` and `true` are distinct tokens and must stay distinct: the
strings match the engine's per-token detokenization (leading space
restored), so `/v1/completions/derender` top_logprobs keys do not collide.
Regression test for #59241."""
tokenizer = get_tokenizer("hmellor/tiny-random-LlamaForCausalLM")
spaced = tokenizer.convert_tokens_to_ids("▁true")
bare = tokenizer.convert_tokens_to_ids("true")
assert spaced != bare
logprobs = GenerateLogProbs(
content=[
GenerateLogProbsContent(
token_id=spaced,
logprob=-0.5,
rank=1,
top_logprobs=[
GenerateLogProb(token_id=spaced, logprob=-0.5, rank=1),
GenerateLogProb(token_id=bare, logprob=-1.5, rank=2),
],
)
]
)
resolved = _resolve_logprobs(logprobs, tokenizer)
assert resolved.content is not None
entry = resolved.content[0]
assert entry.token == " true"
assert entry.bytes == list(b" true")
assert [t.token for t in entry.top_logprobs] == convert_ids_list_to_tokens(
tokenizer, [spaced, bare]
)
completion = _convert_chat_logprobs_to_completion_logprobs(resolved)
assert completion.top_logprobs == [{" true": -0.5, "true": -1.5}]
def test_unknown_top_id_does_not_shift_the_other_candidates():
"""Ids are decoded in one batch per position; an id with no vocab entry
becomes an empty token without shifting its neighbours."""
unknown = 99
logprobs = GenerateLogProbs(
content=[
GenerateLogProbsContent(
token_id=PLAIN,
logprob=-0.5,
rank=1,
top_logprobs=[
GenerateLogProb(token_id=unknown, logprob=-1.0, rank=1),
GenerateLogProb(token_id=PLAIN, logprob=-0.5, rank=2),
],
)
]
)
resolved = _resolve_logprobs(logprobs, _StubTokenizer())
assert resolved.content is not None
entry = resolved.content[0]
assert entry.token == "ok"
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
("", None),
("ok", list(b"ok")),
]
@@ -1123,7 +1123,8 @@ async def test_stream_with_logprobs():
lp = dc["choices"][0]["logprobs"]
assert lp is not None
assert len(lp["content"]) == 1
assert lp["content"][0]["token"].startswith("token_id:")
assert isinstance(lp["content"][0]["token_id"], int)
assert "token" not in lp["content"][0]
@pytest.mark.asyncio
@@ -183,7 +183,7 @@ async def test_generate_sampling_mask(client):
assert token_id in support
# processed_logprobs: exactly the support carries finite probability.
for top in entry["top_logprobs"]:
in_support = int(top["token"].removeprefix("token_id:")) in support
in_support = top["token_id"] in support
assert in_support == (top["logprob"] > -9999.0)
@@ -1,22 +1,62 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from unittest.mock import MagicMock
import pytest
from pydantic import ValidationError
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateLogProb
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
from vllm.logprobs import Logprob
def test_top_logprobs_alternatives_have_own_token_ids():
"""Each top_logprobs alternative must carry its own token_id placeholder."""
"""Each top_logprobs alternative must carry its own integer token id."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[262],
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2), 428: Logprob(-2.3)}],
num_output_top_logprobs=2,
)
tokens = {e.token for e in result.content[0].top_logprobs}
assert tokens == {"token_id:262", "token_id:257"}, f"got {tokens}"
token_ids = {e.token_id for e in result.content[0].top_logprobs}
assert token_ids == {262, 257}, f"got {token_ids}"
def test_sampled_entry_carries_token_id_and_rank():
"""The sampled token is identified by id, and the engine's rank survives."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[262],
top_logprobs=[
{262: Logprob(-0.1, rank=1), 257: Logprob(-1.2, rank=2)},
],
num_output_top_logprobs=2,
)
entry = result.content[0]
assert entry.token_id == 262
assert entry.logprob == -0.1
assert entry.rank == 1
assert [(t.token_id, t.rank) for t in entry.top_logprobs] == [(262, 1), (257, 2)]
# No tokenizer on the generate server: no string token, no bytes.
assert not hasattr(entry, "token")
assert not hasattr(entry, "bytes")
def test_sampled_token_absent_from_topk_uses_sentinel():
"""A sampled token missing from the engine's map keeps the -9999.0 sentinel."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[5],
top_logprobs=[{7: Logprob(-0.9)}],
num_output_top_logprobs=1,
)
entry = result.content[0]
assert entry.token_id == 5
assert entry.logprob == -9999.0
assert entry.rank is None
assert entry.top_logprobs == []
def test_logprobs_zero_emits_sampled_token():
@@ -40,21 +80,23 @@ def test_logprobs_minus_one_emits_all_tokens():
assert len(result.content[0].top_logprobs) == 2
def test_tokens_level_logprobs_have_no_bytes():
result = ServingTokens._create_tokens_logprobs(
def test_text_logprobs_without_tokenizer_are_placeholders_without_bytes():
"""--return-tokens-as-token-ids keeps `token_id:N` placeholders in text mode."""
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
num_output_top_logprobs=2,
)
entry = result.content[0]
assert entry.token == "token_id:7"
assert entry.bytes is None
assert all(t.bytes is None for t in entry.top_logprobs)
def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
tokenizer = MagicMock()
result = ServingTokens._create_tokens_logprobs(
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7],
top_logprobs=[
@@ -75,7 +117,7 @@ def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
def test_resolved_logprobs_decode_tokens_the_engine_did_not():
tokenizer = MagicMock()
tokenizer.decode.side_effect = lambda ids: f"<{ids[0]}>"
result = ServingTokens._create_tokens_logprobs(
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7, 9],
top_logprobs=[{7: Logprob(-0.9)}, None],
@@ -85,3 +127,59 @@ def test_resolved_logprobs_decode_tokens_the_engine_did_not():
assert [e.token for e in result.content] == ["<7>", "<9>"]
assert result.content[0].top_logprobs[0].token == "<7>"
assert result.content[1].bytes == list(b"<9>")
def test_sampled_token_outside_topk_comes_first():
"""The engine puts the sampled token first, then ranks 1..k. When the
sampled token is outside the top k it takes one of the k slots, so rank k
is left out (same as the OpenAI endpoints)."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[50],
top_logprobs=[
{
50: Logprob(-3.0, rank=5),
10: Logprob(-0.2, rank=1),
20: Logprob(-1.0, rank=2),
}
],
num_output_top_logprobs=2,
)
assert [t.rank for t in result.content[0].top_logprobs] == [5, 1]
def test_logprob_is_required_on_the_wire():
"""A payload missing logprob is rejected rather than read as -9999."""
with pytest.raises(ValidationError):
GenerateLogProb.model_validate({"token_id": 1})
def test_rank_zero_from_a_nan_logprob_is_none():
"""The engine reports rank 0 for a sampled token whose logprob is NaN."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(float("nan"), rank=0), 8: Logprob(-0.2, rank=1)}],
num_output_top_logprobs=1,
)
entry = result.content[0]
assert entry.rank is None
assert entry.top_logprobs[0].rank is None
# Clamped like the Rust frontend (max(nan, x) is nan in Python).
assert entry.logprob == -9999.0
assert entry.top_logprobs[0].logprob == -9999.0
json.dumps(result.model_dump(), allow_nan=False)
def test_text_logprobs_clamp_a_nan_logprob():
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(float("nan"), rank=0, decoded_token="a")}],
num_output_top_logprobs=1,
tokenizer=MagicMock(),
)
entry = result.content[0]
assert entry.logprob == -9999.0
assert entry.top_logprobs[0].logprob == -9999.0
json.dumps(result.model_dump(), allow_nan=False)