[Frontend] Expose logprob_token_ids on Python OpenAI endpoints (#43463)

Signed-off-by: Lang Zhao <lang.zhao@galileo.ai>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
langzhao-netizen
2026-07-13 14:40:21 -07:00
committed by GitHub
co-authored by Claude
parent 7738ef35b8
commit 8b8af2caf7
9 changed files with 401 additions and 7 deletions
@@ -187,3 +187,37 @@ async def test_batched_chat_completions_return_tokens_as_token_ids(
content = data["choices"][0]["logprobs"]["content"]
assert content
assert all(entry["token"].startswith("token_id:") for entry in content)
@pytest.mark.asyncio
async def test_batched_chat_completions_logprob_token_ids(
server: RemoteOpenAIServer,
) -> None:
conversations = [[{"role": "user", "content": "Hello"}]]
async with httpx.AsyncClient() as http_client:
response = await http_client.post(
f"{server.url_for('v1/chat/completions/batch')}",
json={
"model": MODEL_NAME,
"messages": conversations,
"max_tokens": 1,
"temperature": 0,
"logprobs": True,
"top_logprobs": 5,
"logprob_token_ids": [100, 1000, 5000],
"return_tokens_as_token_ids": True,
},
timeout=60,
)
assert response.status_code == 200, response.text
content = response.json()["choices"][0]["logprobs"]["content"]
assert content
sampled_token = content[0]["token"]
assert {entry["token"] for entry in content[0]["top_logprobs"]} == {
"token_id:100",
"token_id:1000",
"token_id:5000",
sampled_token,
}
@@ -0,0 +1,208 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end tests for the `logprob_token_ids` field on the OpenAI-compat
chat-completion endpoint.
`logprob_token_ids` lets a caller pin the set of vocab ids whose logprobs
should appear in the response, independent of where those ids would rank in
the natural top-k distribution. This is the primitive that multilabel
scoring postprocessors use to gather logprobs at a fixed small label
vocabulary (e.g. PII detection where each label corresponds to a known
digit-token vocab id).
"""
import math
import pytest
from pydantic import ValidationError
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
@pytest.fixture(scope="module")
def server():
args = [
"--max-model-len",
"1024",
"--max-num-seqs",
"8",
"--enforce-eager",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
def _logprob_entries(resp) -> list:
"""Return the top_logprobs list from the first generated position."""
return resp.choices[0].logprobs.content[0].top_logprobs
def _token_id(token: str) -> int:
assert token.startswith("token_id:"), (
"expected return_tokens_as_token_ids=True to yield the "
f"`token_id:<int>` form, got {token!r}"
)
return int(token.removeprefix("token_id:"))
def _top_logprob_token_ids(resp) -> list[int]:
return [_token_id(entry.token) for entry in _logprob_entries(resp)]
def test_chat_request_decouples_top_k_from_explicit_token_ids():
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Hello"}],
logprobs=True,
top_logprobs=5,
logprob_token_ids=[5000],
)
sampling_params = request.to_sampling_params(
max_tokens=1, default_sampling_params={}
)
assert sampling_params.logprobs is None
assert sampling_params.logprob_token_ids == [5000]
assert sampling_params.num_logprobs == 1
def test_completion_request_decouples_top_k_from_explicit_token_ids():
request = CompletionRequest(
model=MODEL_NAME,
prompt="Hello",
logprobs=5,
logprob_token_ids=[5000],
)
sampling_params = request.to_sampling_params(max_tokens=1)
assert sampling_params.logprobs is None
assert sampling_params.logprob_token_ids == [5000]
assert sampling_params.num_logprobs == 1
def test_completion_rejects_explicit_token_ids_without_generated_tokens():
with pytest.raises(ValidationError, match="no output tokens are generated"):
CompletionRequest(
model=MODEL_NAME,
prompt="Hello",
echo=True,
max_tokens=0,
logprobs=5,
logprob_token_ids=[5000],
)
def test_requests_reject_explicit_token_ids_with_beam_search():
with pytest.raises(ValidationError, match="not supported with beam search"):
ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Hello"}],
logprobs=True,
logprob_token_ids=[5000],
use_beam_search=True,
)
with pytest.raises(ValidationError, match="not supported with beam search"):
CompletionRequest(
model=MODEL_NAME,
prompt="Hello",
logprobs=5,
logprob_token_ids=[5000],
use_beam_search=True,
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("requested_ids", "top_logprobs", "sampled_id"),
[
pytest.param([5000], 5, 42, id="original-issue-shape"),
pytest.param([100, 1000, 5000], 3, 42, id="sampled-outside-set"),
pytest.param([100, 1000, 5000], 3, 5000, id="sampled-inside-set"),
],
)
async def test_logprob_token_ids_returns_requested_ids(
server, requested_ids, top_logprobs, sampled_id
):
async with server.get_async_client() as client:
resp = await client.chat.completions.create(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Hello"}],
max_tokens=1,
temperature=0,
logprobs=True,
top_logprobs=top_logprobs,
extra_body={
"logprob_token_ids": requested_ids,
"allowed_token_ids": [sampled_id],
"return_tokens_as_token_ids": True,
},
)
entries = _logprob_entries(resp)
returned_token_ids = _top_logprob_token_ids(resp)
assert _token_id(resp.choices[0].logprobs.content[0].token) == sampled_id
assert set(returned_token_ids) == {*requested_ids, sampled_id}
assert len(returned_token_ids) == len(set(returned_token_ids))
for e in entries:
assert isinstance(e.logprob, float)
assert not math.isnan(e.logprob)
assert not math.isinf(e.logprob)
@pytest.mark.asyncio
async def test_logprob_token_ids_stream_with_no_top_k(server):
async with server.get_async_client() as client:
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Hello"}],
max_tokens=1,
temperature=0,
logprobs=True,
top_logprobs=None,
stream=True,
extra_body={
"logprob_token_ids": [5000],
"allowed_token_ids": [42],
"return_tokens_as_token_ids": True,
},
)
returned_token_ids: list[int] = []
async for chunk in stream:
if not chunk.choices or chunk.choices[0].logprobs is None:
continue
for content in chunk.choices[0].logprobs.content:
returned_token_ids.extend(
_token_id(entry.token) for entry in content.top_logprobs
)
assert set(returned_token_ids) == {42, 5000}
@pytest.mark.asyncio
async def test_logprob_token_ids_default_behavior_unchanged(server):
"""Without `logprob_token_ids`, the response carries the natural top-k
most-likely tokens. This guards against the new field accidentally
changing the default-path output."""
async with server.get_async_client() as client:
resp = await client.chat.completions.create(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Hello"}],
max_tokens=1,
temperature=0,
logprobs=True,
top_logprobs=5,
)
entries = _logprob_entries(resp)
assert len(entries) == 5
# Default mode emits log_softmax, so all values are <= 0.
for e in entries:
assert e.logprob <= 0.0
@@ -175,6 +175,61 @@ async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
assert 5 <= len(choice.logprobs.top_logprobs[0]) <= 6
@pytest.mark.asyncio
async def test_logprob_token_ids(client: openai.AsyncOpenAI):
completion = await client.completions.create(
model=MODEL_NAME,
prompt="Hello",
max_tokens=1,
temperature=0.0,
logprobs=5,
extra_body={
"logprob_token_ids": [5000],
"allowed_token_ids": [42],
"return_tokens_as_token_ids": True,
},
)
choice = completion.choices[0]
assert choice.logprobs is not None
assert choice.logprobs.tokens == ["token_id:42"]
assert choice.logprobs.top_logprobs is not None
assert set(choice.logprobs.top_logprobs[0]) == {
"token_id:42",
"token_id:5000",
}
@pytest.mark.asyncio
async def test_logprob_token_ids_stream(client: openai.AsyncOpenAI):
stream = await client.completions.create(
model=MODEL_NAME,
prompt="Hello",
max_tokens=1,
temperature=0.0,
logprobs=5,
stream=True,
extra_body={
"logprob_token_ids": [5000],
"allowed_token_ids": [42],
"return_tokens_as_token_ids": True,
},
)
returned_top_logprobs: list[dict[str, float]] = []
async for chunk in stream:
logprobs = chunk.choices[0].logprobs
if logprobs is not None and logprobs.top_logprobs is not None:
returned_top_logprobs.extend(
top_logprobs
for top_logprobs in logprobs.top_logprobs
if top_logprobs is not None
)
assert len(returned_top_logprobs) == 1
assert set(returned_top_logprobs[0]) == {"token_id:42", "token_id:5000"}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",