[Bugfix][Frontend] Only echo the assistant turn in batched chat completions (#52529)

Signed-off-by: Kaif <kaifkohari10@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Kaif Kohari
2026-08-29 09:44:17 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent 5b0e5b69ac
commit 7fbfca2670
2 changed files with 106 additions and 1 deletions
@@ -2,11 +2,20 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json import json
from collections.abc import AsyncGenerator
import httpx import httpx
import pytest import pytest
from tests.utils import RemoteOpenAIServer from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.openai.chat_completion.batch_serving import (
OpenAIServingChatBatch,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
)
from vllm.entrypoints.openai.engine.protocol import RequestResponseMetadata
from vllm.outputs import CompletionOutput, RequestOutput
# any model with a chat template defined in tokenizer_config should work here # any model with a chat template defined in tokenizer_config should work here
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct" MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
@@ -221,3 +230,95 @@ async def test_batched_chat_completions_logprob_token_ids(
"token_id:5000", "token_id:5000",
sampled_token, sampled_token,
} }
def _make_request_output(prompt_idx: int, text: str) -> RequestOutput:
return RequestOutput(
request_id=f"req-{prompt_idx}",
prompt=None,
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text=text,
token_ids=[4, 5],
cumulative_logprob=None,
logprobs=None,
finish_reason="stop",
)
],
finished=True,
)
async def _generator(prompt_idx: int, text: str) -> AsyncGenerator[RequestOutput, None]:
yield _make_request_output(prompt_idx, text)
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_echo_does_not_prepend_user_prompt() -> None:
"""echo must only echo the assistant turn, never the user's prompt.
With `add_generation_prompt` (the default) the response role is `assistant`
while the last conversation message is the user's, so there is nothing to
echo; without the role check the user prompt leaks into the answer.
"""
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
serving.response_role = "assistant"
serving.system_fingerprint = None
conversations = [[{"role": "user", "content": "USER PROMPT"}]]
request = BatchChatCompletionRequest(
model="test-model",
messages=conversations,
echo=True,
)
response = await serving.chat_completion_full_generator_batch(
request=request,
generators=[_generator(0, "ASSISTANT ANSWER")],
request_id="req-echo",
model_name="test-model",
all_conversations=conversations,
tokenizer=None,
request_metadata=RequestResponseMetadata(request_id="req-echo"),
)
assert response.choices[0].message.content == "ASSISTANT ANSWER"
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_echo_prepends_matching_assistant_prefix() -> None:
"""A trailing assistant turn is a real prefix and must still be echoed."""
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
serving.response_role = "assistant"
serving.system_fingerprint = None
conversations = [
[
{"role": "user", "content": "USER PROMPT"},
{"role": "assistant", "content": "PREFIX "},
]
]
request = BatchChatCompletionRequest(
model="test-model",
messages=conversations,
echo=True,
add_generation_prompt=False,
continue_final_message=True,
)
response = await serving.chat_completion_full_generator_batch(
request=request,
generators=[_generator(0, "ASSISTANT ANSWER")],
request_id="req-echo-2",
model_name="test-model",
all_conversations=conversations,
tokenizer=None,
request_metadata=RequestResponseMetadata(request_id="req-echo-2"),
)
assert response.choices[0].message.content == "PREFIX ASSISTANT ANSWER"
@@ -291,7 +291,11 @@ class OpenAIServingChatBatch(OpenAIServingChat):
if request.echo: if request.echo:
conversation = all_conversations[prompt_idx] conversation = all_conversations[prompt_idx]
last_msg_content: str | list[dict[str, str]] = "" last_msg_content: str | list[dict[str, str]] = ""
if conversation and "content" in conversation[-1]: if (
conversation
and "content" in conversation[-1]
and conversation[-1].get("role") == role
):
last_msg_content = conversation[-1]["content"] or "" last_msg_content = conversation[-1]["content"] or ""
if isinstance(last_msg_content, list): if isinstance(last_msg_content, list):
last_msg_content = "\n".join( last_msg_content = "\n".join(