From 18f658bb3185779ee58999a328246d09886d568b Mon Sep 17 00:00:00 2001 From: David Feng <15175853+fenghourun@users.noreply.github.com> Date: Fri, 3 Jul 2026 06:01:34 -0400 Subject: [PATCH] [Bugfix][Frontend] Fix batch chat endpoint corrupting logprobs when return_token_ids is set (#47384) Signed-off-by: David Feng Co-authored-by: Claude --- .../test_batched_chat_completions.py | 76 +++++++++++++++++++ .../openai/chat_completion/batch_serving.py | 2 +- .../openai/chat_completion/protocol.py | 3 + 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py index c3a8d0b2bd..87a206b566 100644 --- a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py +++ b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py @@ -111,3 +111,79 @@ async def test_batched_chat_completions_with_json_schema( parsed = json.loads(choice["message"]["content"]) assert "answer" in parsed assert parsed["answer"] in ("yes", "no") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_name", + [MODEL_NAME], +) +async def test_batched_chat_completions_logprobs_not_token_id_placeholders( + server: RemoteOpenAIServer, model_name: str +) -> None: + # Regression test: requesting `return_token_ids` alongside logprobs must not + # corrupt the logprob `token` fields into "token_id:{id}" placeholders. That + # placeholder rendering is controlled by `return_tokens_as_token_ids`, which + # this request leaves unset. + conversations = [ + [{"role": "user", "content": "Reply with exactly the word: alpha"}], + ] + + async with httpx.AsyncClient() as http_client: + response = await http_client.post( + f"{server.url_for('v1/chat/completions/batch')}", + json={ + "model": model_name, + "messages": conversations, + "logprobs": True, + "top_logprobs": 1, + "return_token_ids": True, + }, + timeout=60, + ) + + assert response.status_code == 200, response.text + data = response.json() + + content = data["choices"][0]["logprobs"]["content"] + assert content + for entry in content: + assert not entry["token"].startswith("token_id:") + for top in entry["top_logprobs"]: + assert not top["token"].startswith("token_id:") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_name", + [MODEL_NAME], +) +async def test_batched_chat_completions_return_tokens_as_token_ids( + server: RemoteOpenAIServer, model_name: str +) -> None: + # Complementary check: when `return_tokens_as_token_ids` is explicitly set, + # the logprob tokens *should* be rendered as "token_id:{id}" placeholders, + # proving the new field is actually wired through. + conversations = [ + [{"role": "user", "content": "Reply with exactly the word: alpha"}], + ] + + async with httpx.AsyncClient() as http_client: + response = await http_client.post( + f"{server.url_for('v1/chat/completions/batch')}", + json={ + "model": model_name, + "messages": conversations, + "logprobs": True, + "top_logprobs": 1, + "return_tokens_as_token_ids": True, + }, + timeout=60, + ) + + assert response.status_code == 200, response.text + data = response.json() + + content = data["choices"][0]["logprobs"]["content"] + assert content + assert all(entry["token"].startswith("token_id:") for entry in content) diff --git a/vllm/entrypoints/openai/chat_completion/batch_serving.py b/vllm/entrypoints/openai/chat_completion/batch_serving.py index 6acc568f49..c00ab0418f 100644 --- a/vllm/entrypoints/openai/chat_completion/batch_serving.py +++ b/vllm/entrypoints/openai/chat_completion/batch_serving.py @@ -258,7 +258,7 @@ class OpenAIServingChatBatch(OpenAIServingChat): top_logprobs=output.logprobs, num_output_top_logprobs=request.top_logprobs, tokenizer=tokenizer, - return_as_token_id=request.return_token_ids, + return_as_token_id=request.return_tokens_as_token_ids, ) else: logprobs = None diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 5d5014616b..4745947d49 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -1023,6 +1023,9 @@ class BatchChatCompletionRequest(OpenAIBaseModel): include_stop_str_in_output: bool = False guided_decoding_backend: str | None = None echo: bool = False + # None falls back to the server-level --return-tokens-as-token-ids default, + # matching ChatCompletionRequest.return_tokens_as_token_ids. + return_tokens_as_token_ids: bool | None = None return_token_ids: bool = False @model_validator(mode="before")