[Bugfix][Frontend] Fix batch chat endpoint corrupting logprobs when return_token_ids is set (#47384)
Signed-off-by: David Feng <fenghourun@meta.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -111,3 +111,79 @@ async def test_batched_chat_completions_with_json_schema(
|
||||
parsed = json.loads(choice["message"]["content"])
|
||||
assert "answer" in parsed
|
||||
assert parsed["answer"] in ("yes", "no")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_batched_chat_completions_logprobs_not_token_id_placeholders(
|
||||
server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
# Regression test: requesting `return_token_ids` alongside logprobs must not
|
||||
# corrupt the logprob `token` fields into "token_id:{id}" placeholders. That
|
||||
# placeholder rendering is controlled by `return_tokens_as_token_ids`, which
|
||||
# this request leaves unset.
|
||||
conversations = [
|
||||
[{"role": "user", "content": "Reply with exactly the word: alpha"}],
|
||||
]
|
||||
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
response = await http_client.post(
|
||||
f"{server.url_for('v1/chat/completions/batch')}",
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": conversations,
|
||||
"logprobs": True,
|
||||
"top_logprobs": 1,
|
||||
"return_token_ids": True,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
data = response.json()
|
||||
|
||||
content = data["choices"][0]["logprobs"]["content"]
|
||||
assert content
|
||||
for entry in content:
|
||||
assert not entry["token"].startswith("token_id:")
|
||||
for top in entry["top_logprobs"]:
|
||||
assert not top["token"].startswith("token_id:")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_batched_chat_completions_return_tokens_as_token_ids(
|
||||
server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
# Complementary check: when `return_tokens_as_token_ids` is explicitly set,
|
||||
# the logprob tokens *should* be rendered as "token_id:{id}" placeholders,
|
||||
# proving the new field is actually wired through.
|
||||
conversations = [
|
||||
[{"role": "user", "content": "Reply with exactly the word: alpha"}],
|
||||
]
|
||||
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
response = await http_client.post(
|
||||
f"{server.url_for('v1/chat/completions/batch')}",
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": conversations,
|
||||
"logprobs": True,
|
||||
"top_logprobs": 1,
|
||||
"return_tokens_as_token_ids": True,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
data = response.json()
|
||||
|
||||
content = data["choices"][0]["logprobs"]["content"]
|
||||
assert content
|
||||
assert all(entry["token"].startswith("token_id:") for entry in content)
|
||||
|
||||
@@ -258,7 +258,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
top_logprobs=output.logprobs,
|
||||
num_output_top_logprobs=request.top_logprobs,
|
||||
tokenizer=tokenizer,
|
||||
return_as_token_id=request.return_token_ids,
|
||||
return_as_token_id=request.return_tokens_as_token_ids,
|
||||
)
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
@@ -1023,6 +1023,9 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
|
||||
include_stop_str_in_output: bool = False
|
||||
guided_decoding_backend: str | None = None
|
||||
echo: bool = False
|
||||
# None falls back to the server-level --return-tokens-as-token-ids default,
|
||||
# matching ChatCompletionRequest.return_tokens_as_token_ids.
|
||||
return_tokens_as_token_ids: bool | None = None
|
||||
return_token_ids: bool = False
|
||||
|
||||
@model_validator(mode="before")
|
||||
|
||||
Reference in New Issue
Block a user