[Frontend] Integer token IDs for generate output logprobs (GenerateLogProbs) (#58181)
Signed-off-by: Yin Li <kevinli.ai.work@gmail.com> Co-authored-by: Yin Li <kevinli.ai.work@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -74,14 +74,13 @@ def _make_generate_response(
|
||||
}
|
||||
|
||||
|
||||
def _make_logprobs_with_placeholders(token_id: int = 1234) -> dict:
|
||||
def _make_generate_logprobs(token_id: int = 1234) -> dict:
|
||||
"""A `GenerateLogProbs` payload: integer token ids, no token/bytes."""
|
||||
entry = {
|
||||
"token": f"token_id:{token_id}",
|
||||
"token_id": token_id,
|
||||
"logprob": -1.0,
|
||||
"bytes": None,
|
||||
"top_logprobs": [
|
||||
{"token": f"token_id:{token_id + 1}", "logprob": -2.0, "bytes": None}
|
||||
],
|
||||
"rank": 1,
|
||||
"top_logprobs": [{"token_id": token_id + 1, "logprob": -2.0, "rank": 2}],
|
||||
}
|
||||
return {"content": [entry]}
|
||||
|
||||
@@ -154,7 +153,7 @@ async def test_derender_chat_usage_default(client):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_logprobs(client):
|
||||
"""token_id:N placeholders in content.token are resolved to real strings."""
|
||||
"""Integer token ids are decoded to real strings by derender."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
token_id = synthetic_ids[0]
|
||||
@@ -165,7 +164,7 @@ async def test_derender_chat_logprobs(client):
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
synthetic_ids,
|
||||
logprobs=_make_logprobs_with_placeholders(token_id),
|
||||
logprobs=_make_generate_logprobs(token_id),
|
||||
),
|
||||
},
|
||||
)
|
||||
@@ -176,8 +175,8 @@ async def test_derender_chat_logprobs(client):
|
||||
content = logprobs["content"]
|
||||
assert content is not None and len(content) == 1
|
||||
token_str = content[0]["token"]
|
||||
assert not token_str.startswith("token_id:"), (
|
||||
f"Placeholder was not resolved: {token_str!r}"
|
||||
assert token_str and not token_str.startswith("token_id:"), (
|
||||
f"token id was not decoded: {token_str!r}"
|
||||
)
|
||||
|
||||
|
||||
@@ -194,7 +193,7 @@ async def test_derender_chat_logprobs_bytes(client):
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
synthetic_ids,
|
||||
logprobs=_make_logprobs_with_placeholders(token_id),
|
||||
logprobs=_make_generate_logprobs(token_id),
|
||||
),
|
||||
},
|
||||
)
|
||||
@@ -208,7 +207,7 @@ async def test_derender_chat_logprobs_bytes(client):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_top_logprobs(client):
|
||||
"""top_logprobs entries also have their placeholders resolved."""
|
||||
"""top_logprobs entries are decoded too."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
token_id = synthetic_ids[0]
|
||||
@@ -219,7 +218,7 @@ async def test_derender_chat_top_logprobs(client):
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
synthetic_ids,
|
||||
logprobs=_make_logprobs_with_placeholders(token_id),
|
||||
logprobs=_make_generate_logprobs(token_id),
|
||||
),
|
||||
},
|
||||
)
|
||||
@@ -228,7 +227,7 @@ async def test_derender_chat_top_logprobs(client):
|
||||
top = content[0]["top_logprobs"]
|
||||
assert len(top) == 1
|
||||
assert not top[0]["token"].startswith("token_id:"), (
|
||||
f"top_logprobs placeholder not resolved: {top[0]['token']!r}"
|
||||
f"top_logprobs token id not decoded: {top[0]['token']!r}"
|
||||
)
|
||||
|
||||
|
||||
@@ -633,8 +632,8 @@ async def test_derender_completion_empty_generate_responses(client):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_logprobs(client):
|
||||
"""token_id:N placeholders in logprobs are resolved; CompletionLogProbs
|
||||
flat-list structure is returned with non-empty tokens and text_offsets."""
|
||||
"""Integer token ids are decoded; CompletionLogProbs flat-list structure is
|
||||
returned with non-empty tokens and text_offsets."""
|
||||
gr1 = await _render_completion(client, "Hello world")
|
||||
ids1 = gr1["token_ids"][:3]
|
||||
token_id = ids1[0]
|
||||
@@ -647,7 +646,7 @@ async def test_derender_completion_logprobs(client):
|
||||
_make_completion_generate_response(
|
||||
ids1,
|
||||
gr1["request_id"],
|
||||
logprobs=_make_logprobs_with_placeholders(token_id),
|
||||
logprobs=_make_generate_logprobs(token_id),
|
||||
),
|
||||
],
|
||||
},
|
||||
@@ -658,7 +657,7 @@ async def test_derender_completion_logprobs(client):
|
||||
tokens = logprobs["tokens"]
|
||||
assert len(tokens) == 1
|
||||
assert not tokens[0].startswith("token_id:"), (
|
||||
f"Placeholder was not resolved: {tokens[0]!r}"
|
||||
f"token id was not decoded: {tokens[0]!r}"
|
||||
)
|
||||
assert len(logprobs["token_logprobs"]) == 1
|
||||
assert isinstance(logprobs["token_logprobs"][0], float)
|
||||
@@ -794,7 +793,7 @@ async def test_derender_chat_oversized_logprobs_rejected(client):
|
||||
"""logprobs.content longer than max_model_len returns 400."""
|
||||
oversized_logprobs: dict = {
|
||||
"content": [
|
||||
{"token": "x", "logprob": -1.0, "bytes": None, "top_logprobs": []}
|
||||
{"token_id": 42, "logprob": -1.0, "rank": 1, "top_logprobs": []}
|
||||
for _ in range(1_000_000)
|
||||
]
|
||||
}
|
||||
@@ -825,11 +824,11 @@ async def test_derender_chat_oversized_top_logprobs_rejected(client):
|
||||
oversized_top_logprobs = {
|
||||
"content": [
|
||||
{
|
||||
"token": "x",
|
||||
"token_id": 42,
|
||||
"logprob": -1.0,
|
||||
"bytes": None,
|
||||
"rank": 1,
|
||||
"top_logprobs": [
|
||||
{"token": f"t{i}", "logprob": -float(i), "bytes": None}
|
||||
{"token_id": i, "logprob": -float(i), "rank": i + 1}
|
||||
for i in range(25)
|
||||
],
|
||||
}
|
||||
|
||||
@@ -237,21 +237,47 @@ async def test_parity_reasoning_and_tool_call(client):
|
||||
assert _tool_sig(disagg["choices"][0])
|
||||
|
||||
|
||||
def _to_generate_logprobs(openai_logprobs: dict) -> dict:
|
||||
"""Turn a coupled response's `token_id:N` logprobs into `GenerateLogProbs`.
|
||||
|
||||
A real GPU-less worker emits integer ids directly; here the coupled server
|
||||
with `return_tokens_as_token_ids=True` is the only source of the same ids,
|
||||
so the placeholders are parsed back out to build the generate-shaped
|
||||
payload that `/derender` now consumes.
|
||||
"""
|
||||
|
||||
def _id(token: str) -> int:
|
||||
assert token.startswith("token_id:"), token
|
||||
return int(token.removeprefix("token_id:"))
|
||||
|
||||
return {
|
||||
"content": [
|
||||
{
|
||||
"token_id": _id(entry["token"]),
|
||||
"logprob": entry["logprob"],
|
||||
"top_logprobs": [
|
||||
{"token_id": _id(top["token"]), "logprob": top["logprob"]}
|
||||
for top in entry["top_logprobs"]
|
||||
],
|
||||
}
|
||||
for entry in openai_logprobs["content"]
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parity_logprobs(client):
|
||||
"""token_id:N resolution parity vs. the coupled server's real strings.
|
||||
"""Integer-id logprob decoding parity vs. the coupled server's strings.
|
||||
|
||||
A real disaggregated worker only has token IDs so it emits logprobs
|
||||
with `token_id:N` placeholders (`return_tokens_as_token_ids=True`
|
||||
reproduces that shape here). `/derender` must resolve those
|
||||
placeholders to the same token strings/bytes the coupled server
|
||||
resolves them to directly.
|
||||
A real disaggregated worker only has token IDs, so it emits
|
||||
`GenerateLogProbs` with integer `token_id`s. `/derender` must decode those
|
||||
to the same token strings/bytes the coupled server produces directly.
|
||||
"""
|
||||
messages = [{"role": "user", "content": "What is 2+2?"}]
|
||||
extra = {"logprobs": True, "top_logprobs": 3}
|
||||
|
||||
# What a real GPU less worker would hand to /derender is token IDs plus
|
||||
# logprobs still in token_id:N placeholder form
|
||||
# The coupled server with placeholders is used only as a source of the
|
||||
# token ids a GPU-less worker would have had.
|
||||
placeholder = await _coupled(
|
||||
client, messages, return_tokens_as_token_ids=True, **extra
|
||||
)
|
||||
@@ -263,7 +289,7 @@ async def test_parity_logprobs(client):
|
||||
placeholder["usage"]["prompt_tokens"],
|
||||
ch["finish_reason"],
|
||||
chat_request,
|
||||
logprobs=ch["logprobs"],
|
||||
logprobs=_to_generate_logprobs(ch["logprobs"]),
|
||||
)
|
||||
|
||||
# The coupled server resolving the same greedy generation
|
||||
|
||||
@@ -64,6 +64,7 @@ from vllm.renderers.online_derenderer import (
|
||||
_seed_stream_state,
|
||||
)
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
@@ -212,17 +213,12 @@ def _make_stream_chunk(
|
||||
)
|
||||
|
||||
|
||||
def _placeholder_logprobs(token_ids: list[int]) -> dict:
|
||||
"""Per-token logprob entries using token_id:N placeholders, as sent by
|
||||
the generate worker."""
|
||||
def _generate_logprobs(token_ids: list[int]) -> dict:
|
||||
"""Per-token `GenerateLogProbs` entries (integer token ids, no token or
|
||||
bytes), as sent by the generate worker."""
|
||||
return {
|
||||
"content": [
|
||||
{
|
||||
"token": f"token_id:{tid}",
|
||||
"logprob": -0.5,
|
||||
"bytes": None,
|
||||
"top_logprobs": [],
|
||||
}
|
||||
{"token_id": tid, "logprob": -0.5, "rank": 1, "top_logprobs": []}
|
||||
for tid in token_ids
|
||||
]
|
||||
}
|
||||
@@ -892,12 +888,12 @@ class TestDerenderCompletionStream:
|
||||
|
||||
|
||||
class TestStreamLogprobs:
|
||||
"""Streaming derender must carry per-chunk logprobs with placeholders
|
||||
resolved, matching what the generate streaming path emits."""
|
||||
"""Streaming derender must carry per-chunk logprobs with token ids
|
||||
decoded, matching what the generate streaming path emits."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_stream_logprobs_resolved_per_chunk(self, derenderer, tokenizer):
|
||||
"""Each streamed chunk carries logprobs with token_id:N resolved."""
|
||||
"""Each streamed chunk carries logprobs with its token ids decoded."""
|
||||
token_ids = tokenizer.encode("hello world")[:6]
|
||||
mid = len(token_ids) // 2
|
||||
|
||||
@@ -906,17 +902,15 @@ class TestStreamLogprobs:
|
||||
chunk, state = await derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
part, logprobs=_placeholder_logprobs(part)
|
||||
part, logprobs=_generate_logprobs(part)
|
||||
),
|
||||
state=state,
|
||||
)
|
||||
logprobs = chunk.choices[0].logprobs
|
||||
assert logprobs is not None and logprobs.content is not None
|
||||
assert len(logprobs.content) == len(part)
|
||||
for entry in logprobs.content:
|
||||
assert not entry.token.startswith("token_id:"), (
|
||||
f"placeholder not resolved: {entry.token!r}"
|
||||
)
|
||||
assert [entry.token for entry in logprobs.content] == (
|
||||
convert_ids_list_to_tokens(tokenizer, part)
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_stream_logprobs_multibyte_across_chunks(
|
||||
@@ -936,7 +930,7 @@ class TestStreamLogprobs:
|
||||
last_chunk, state = await derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
[tid], logprobs=_placeholder_logprobs([tid])
|
||||
[tid], logprobs=_generate_logprobs([tid])
|
||||
),
|
||||
state=state,
|
||||
)
|
||||
@@ -959,13 +953,13 @@ class TestStreamLogprobs:
|
||||
chunk1, state = await derenderer.derender_completion_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
token_ids[:mid], logprobs=_placeholder_logprobs(token_ids[:mid])
|
||||
token_ids[:mid], logprobs=_generate_logprobs(token_ids[:mid])
|
||||
),
|
||||
)
|
||||
chunk2, _ = await derenderer.derender_completion_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
token_ids[mid:], logprobs=_placeholder_logprobs(token_ids[mid:])
|
||||
token_ids[mid:], logprobs=_generate_logprobs(token_ids[mid:])
|
||||
),
|
||||
state=state,
|
||||
)
|
||||
@@ -982,7 +976,7 @@ class TestStreamLogprobs:
|
||||
_, state = await derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
[tid], logprobs=_placeholder_logprobs([tid])
|
||||
[tid], logprobs=_generate_logprobs([tid])
|
||||
),
|
||||
state=state,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the generate -> OpenAI logprob conversion in derender.
|
||||
|
||||
These exercise `_resolve_logprobs` directly. A stub tokenizer covers the
|
||||
byte-fallback correction path; a real SentencePiece tokenizer covers the
|
||||
leading-space restoration.
|
||||
"""
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateLogProb,
|
||||
GenerateLogProbs,
|
||||
GenerateLogProbsContent,
|
||||
)
|
||||
from vllm.renderers.online_derenderer import (
|
||||
_convert_chat_logprobs_to_completion_logprobs,
|
||||
_resolve_logprobs,
|
||||
)
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
|
||||
|
||||
# A two-byte character split across two byte-fallback tokens: neither decodes
|
||||
# to anything printable on its own, but the pair decodes to "é".
|
||||
FIRST_BYTE = 10
|
||||
SECOND_BYTE = 11
|
||||
PLAIN = 12
|
||||
|
||||
VOCAB = {FIRST_BYTE: "<0xC3>", SECOND_BYTE: "<0xA9>", PLAIN: "ok"}
|
||||
|
||||
|
||||
class _StubTokenizer:
|
||||
"""Decodes byte-fallback tokens only when both halves are present."""
|
||||
|
||||
def convert_ids_to_tokens(self, ids: list[int]) -> list[str | None]:
|
||||
return [VOCAB.get(i) for i in ids]
|
||||
|
||||
def convert_tokens_to_string(self, tokens: list[str]) -> str:
|
||||
return self.decode([k for t in tokens for k, v in VOCAB.items() if v == t])
|
||||
|
||||
def decode(self, ids: list[int], **kwargs) -> str:
|
||||
out = ""
|
||||
i = 0
|
||||
while i < len(ids):
|
||||
if ids[i] == FIRST_BYTE and i + 1 < len(ids) and ids[i + 1] == SECOND_BYTE:
|
||||
out += "é"
|
||||
i += 2
|
||||
elif ids[i] in (FIRST_BYTE, SECOND_BYTE):
|
||||
out += "�"
|
||||
i += 1
|
||||
else:
|
||||
out += VOCAB.get(ids[i], "")
|
||||
i += 1
|
||||
return out
|
||||
|
||||
|
||||
def test_token_ids_are_decoded_with_bytes_and_order_preserved():
|
||||
logprobs = GenerateLogProbs(
|
||||
content=[
|
||||
GenerateLogProbsContent(
|
||||
token_id=PLAIN,
|
||||
logprob=-0.5,
|
||||
rank=1,
|
||||
top_logprobs=[
|
||||
GenerateLogProb(token_id=PLAIN, logprob=-0.5, rank=1),
|
||||
GenerateLogProb(token_id=FIRST_BYTE, logprob=-2.0, rank=2),
|
||||
],
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
resolved = _resolve_logprobs(logprobs, _StubTokenizer())
|
||||
|
||||
assert resolved.content is not None
|
||||
entry = resolved.content[0]
|
||||
assert entry.token == "ok"
|
||||
assert entry.logprob == -0.5
|
||||
assert entry.bytes == list(b"ok")
|
||||
# Rank order survives the conversion (the OpenAI shape has no rank field).
|
||||
assert [t.logprob for t in entry.top_logprobs] == [-0.5, -2.0]
|
||||
assert entry.top_logprobs[0].token == "ok"
|
||||
|
||||
|
||||
def test_byte_fallback_second_half_uses_preceding_token_as_context():
|
||||
"""U+FFFD correction needs the preceding sampled ids, which the integer
|
||||
shape carries directly (no placeholder parsing)."""
|
||||
logprobs = GenerateLogProbs(
|
||||
content=[
|
||||
GenerateLogProbsContent(token_id=FIRST_BYTE, logprob=-0.1),
|
||||
GenerateLogProbsContent(token_id=SECOND_BYTE, logprob=-0.2),
|
||||
]
|
||||
)
|
||||
|
||||
resolved = _resolve_logprobs(logprobs, _StubTokenizer())
|
||||
|
||||
assert resolved.content is not None
|
||||
# The first half has no preceding context to repair it with, so it stays
|
||||
# empty (existing `_correct_decoded_token` behaviour). The second resolves
|
||||
# to the complete character once the first is used as context.
|
||||
assert resolved.content[0].token == ""
|
||||
assert resolved.content[0].bytes == []
|
||||
assert resolved.content[1].token == "é"
|
||||
assert resolved.content[1].bytes == list("é".encode())
|
||||
|
||||
|
||||
def test_content_none_is_not_an_error():
|
||||
resolved = _resolve_logprobs(GenerateLogProbs(), _StubTokenizer())
|
||||
assert resolved.content is None
|
||||
|
||||
|
||||
def test_sentencepiece_leading_space_matches_engine():
|
||||
"""`▁true` and `true` are distinct tokens and must stay distinct: the
|
||||
strings match the engine's per-token detokenization (leading space
|
||||
restored), so `/v1/completions/derender` top_logprobs keys do not collide.
|
||||
Regression test for #59241."""
|
||||
tokenizer = get_tokenizer("hmellor/tiny-random-LlamaForCausalLM")
|
||||
spaced = tokenizer.convert_tokens_to_ids("▁true")
|
||||
bare = tokenizer.convert_tokens_to_ids("true")
|
||||
assert spaced != bare
|
||||
logprobs = GenerateLogProbs(
|
||||
content=[
|
||||
GenerateLogProbsContent(
|
||||
token_id=spaced,
|
||||
logprob=-0.5,
|
||||
rank=1,
|
||||
top_logprobs=[
|
||||
GenerateLogProb(token_id=spaced, logprob=-0.5, rank=1),
|
||||
GenerateLogProb(token_id=bare, logprob=-1.5, rank=2),
|
||||
],
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
resolved = _resolve_logprobs(logprobs, tokenizer)
|
||||
|
||||
assert resolved.content is not None
|
||||
entry = resolved.content[0]
|
||||
assert entry.token == " true"
|
||||
assert entry.bytes == list(b" true")
|
||||
assert [t.token for t in entry.top_logprobs] == convert_ids_list_to_tokens(
|
||||
tokenizer, [spaced, bare]
|
||||
)
|
||||
completion = _convert_chat_logprobs_to_completion_logprobs(resolved)
|
||||
assert completion.top_logprobs == [{" true": -0.5, "true": -1.5}]
|
||||
|
||||
|
||||
def test_unknown_top_id_does_not_shift_the_other_candidates():
|
||||
"""Ids are decoded in one batch per position; an id with no vocab entry
|
||||
becomes an empty token without shifting its neighbours."""
|
||||
unknown = 99
|
||||
logprobs = GenerateLogProbs(
|
||||
content=[
|
||||
GenerateLogProbsContent(
|
||||
token_id=PLAIN,
|
||||
logprob=-0.5,
|
||||
rank=1,
|
||||
top_logprobs=[
|
||||
GenerateLogProb(token_id=unknown, logprob=-1.0, rank=1),
|
||||
GenerateLogProb(token_id=PLAIN, logprob=-0.5, rank=2),
|
||||
],
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
resolved = _resolve_logprobs(logprobs, _StubTokenizer())
|
||||
|
||||
assert resolved.content is not None
|
||||
entry = resolved.content[0]
|
||||
assert entry.token == "ok"
|
||||
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
|
||||
("", None),
|
||||
("ok", list(b"ok")),
|
||||
]
|
||||
@@ -1123,7 +1123,8 @@ async def test_stream_with_logprobs():
|
||||
lp = dc["choices"][0]["logprobs"]
|
||||
assert lp is not None
|
||||
assert len(lp["content"]) == 1
|
||||
assert lp["content"][0]["token"].startswith("token_id:")
|
||||
assert isinstance(lp["content"][0]["token_id"], int)
|
||||
assert "token" not in lp["content"][0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -183,7 +183,7 @@ async def test_generate_sampling_mask(client):
|
||||
assert token_id in support
|
||||
# processed_logprobs: exactly the support carries finite probability.
|
||||
for top in entry["top_logprobs"]:
|
||||
in_support = int(top["token"].removeprefix("token_id:")) in support
|
||||
in_support = top["token_id"] in support
|
||||
assert in_support == (top["logprob"] > -9999.0)
|
||||
|
||||
|
||||
|
||||
@@ -1,22 +1,62 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateLogProb
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||
from vllm.logprobs import Logprob
|
||||
|
||||
|
||||
def test_top_logprobs_alternatives_have_own_token_ids():
|
||||
"""Each top_logprobs alternative must carry its own token_id placeholder."""
|
||||
"""Each top_logprobs alternative must carry its own integer token id."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[262],
|
||||
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2), 428: Logprob(-2.3)}],
|
||||
num_output_top_logprobs=2,
|
||||
)
|
||||
tokens = {e.token for e in result.content[0].top_logprobs}
|
||||
assert tokens == {"token_id:262", "token_id:257"}, f"got {tokens}"
|
||||
token_ids = {e.token_id for e in result.content[0].top_logprobs}
|
||||
assert token_ids == {262, 257}, f"got {token_ids}"
|
||||
|
||||
|
||||
def test_sampled_entry_carries_token_id_and_rank():
|
||||
"""The sampled token is identified by id, and the engine's rank survives."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[262],
|
||||
top_logprobs=[
|
||||
{262: Logprob(-0.1, rank=1), 257: Logprob(-1.2, rank=2)},
|
||||
],
|
||||
num_output_top_logprobs=2,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token_id == 262
|
||||
assert entry.logprob == -0.1
|
||||
assert entry.rank == 1
|
||||
assert [(t.token_id, t.rank) for t in entry.top_logprobs] == [(262, 1), (257, 2)]
|
||||
# No tokenizer on the generate server: no string token, no bytes.
|
||||
assert not hasattr(entry, "token")
|
||||
assert not hasattr(entry, "bytes")
|
||||
|
||||
|
||||
def test_sampled_token_absent_from_topk_uses_sentinel():
|
||||
"""A sampled token missing from the engine's map keeps the -9999.0 sentinel."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[5],
|
||||
top_logprobs=[{7: Logprob(-0.9)}],
|
||||
num_output_top_logprobs=1,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token_id == 5
|
||||
assert entry.logprob == -9999.0
|
||||
assert entry.rank is None
|
||||
assert entry.top_logprobs == []
|
||||
|
||||
|
||||
def test_logprobs_zero_emits_sampled_token():
|
||||
@@ -40,21 +80,23 @@ def test_logprobs_minus_one_emits_all_tokens():
|
||||
assert len(result.content[0].top_logprobs) == 2
|
||||
|
||||
|
||||
def test_tokens_level_logprobs_have_no_bytes():
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
def test_text_logprobs_without_tokenizer_are_placeholders_without_bytes():
|
||||
"""--return-tokens-as-token-ids keeps `token_id:N` placeholders in text mode."""
|
||||
result = ServingTokens._create_text_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
num_output_top_logprobs=2,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token == "token_id:7"
|
||||
assert entry.bytes is None
|
||||
assert all(t.bytes is None for t in entry.top_logprobs)
|
||||
|
||||
|
||||
def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
|
||||
tokenizer = MagicMock()
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
result = ServingTokens._create_text_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[
|
||||
@@ -75,7 +117,7 @@ def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
|
||||
def test_resolved_logprobs_decode_tokens_the_engine_did_not():
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.decode.side_effect = lambda ids: f"<{ids[0]}>"
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
result = ServingTokens._create_text_logprobs(
|
||||
None,
|
||||
token_ids=[7, 9],
|
||||
top_logprobs=[{7: Logprob(-0.9)}, None],
|
||||
@@ -85,3 +127,59 @@ def test_resolved_logprobs_decode_tokens_the_engine_did_not():
|
||||
assert [e.token for e in result.content] == ["<7>", "<9>"]
|
||||
assert result.content[0].top_logprobs[0].token == "<7>"
|
||||
assert result.content[1].bytes == list(b"<9>")
|
||||
|
||||
|
||||
def test_sampled_token_outside_topk_comes_first():
|
||||
"""The engine puts the sampled token first, then ranks 1..k. When the
|
||||
sampled token is outside the top k it takes one of the k slots, so rank k
|
||||
is left out (same as the OpenAI endpoints)."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[50],
|
||||
top_logprobs=[
|
||||
{
|
||||
50: Logprob(-3.0, rank=5),
|
||||
10: Logprob(-0.2, rank=1),
|
||||
20: Logprob(-1.0, rank=2),
|
||||
}
|
||||
],
|
||||
num_output_top_logprobs=2,
|
||||
)
|
||||
assert [t.rank for t in result.content[0].top_logprobs] == [5, 1]
|
||||
|
||||
|
||||
def test_logprob_is_required_on_the_wire():
|
||||
"""A payload missing logprob is rejected rather than read as -9999."""
|
||||
with pytest.raises(ValidationError):
|
||||
GenerateLogProb.model_validate({"token_id": 1})
|
||||
|
||||
|
||||
def test_rank_zero_from_a_nan_logprob_is_none():
|
||||
"""The engine reports rank 0 for a sampled token whose logprob is NaN."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(float("nan"), rank=0), 8: Logprob(-0.2, rank=1)}],
|
||||
num_output_top_logprobs=1,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.rank is None
|
||||
assert entry.top_logprobs[0].rank is None
|
||||
# Clamped like the Rust frontend (max(nan, x) is nan in Python).
|
||||
assert entry.logprob == -9999.0
|
||||
assert entry.top_logprobs[0].logprob == -9999.0
|
||||
json.dumps(result.model_dump(), allow_nan=False)
|
||||
|
||||
|
||||
def test_text_logprobs_clamp_a_nan_logprob():
|
||||
result = ServingTokens._create_text_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(float("nan"), rank=0, decoded_token="a")}],
|
||||
num_output_top_logprobs=1,
|
||||
tokenizer=MagicMock(),
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.logprob == -9999.0
|
||||
assert entry.top_logprobs[0].logprob == -9999.0
|
||||
json.dumps(result.model_dump(), allow_nan=False)
|
||||
|
||||
Reference in New Issue
Block a user