[Frontend] Make finish_reason and Responses item status truthful (#59324)

Signed-off-by: Flora Feng <4florafeng@gmail.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
Flora Feng
2026-10-15 10:12:02 +01:00
committed by GitHub
co-authored by Chauncey
parent 427915bed2
commit 80dd705c1d
14 changed files with 421 additions and 62 deletions
@@ -456,3 +456,27 @@ def test_num_init_messages_offset():
items = ctx.make_response_output_items()
assert len(items) == 1
assert items[0].type == "message"
def test_length_truncation_marks_last_item_incomplete():
"""A turn cut by max_output_tokens ends in an ``incomplete`` item, both
with a parser (tool call) and without one (plain message)."""
request = _make_request(
tool_choice="auto",
tools=[
{
"type": "function",
"name": "get_weather",
"parameters": {"type": "object", "properties": {}},
}
],
)
ctx = _make_context(_ToolCallingParser, request=request, enable_auto_tools=True)
ctx.append_output(_make_request_output(text="calling tool", finish_reason="length"))
assert ctx.response_messages[-1].type == "function_call"
assert ctx.response_messages[-1].status == "incomplete"
ctx = _make_context(None)
ctx.append_output(_make_request_output(text="cut mid sen", finish_reason="length"))
assert ctx.response_messages[-1].type == "message"
assert ctx.response_messages[-1].status == "incomplete"
@@ -991,3 +991,28 @@ class TestConstructInputMessagesInstructionsLeak:
assert len(msgs) == 2
assert msgs[0] == {"role": "system", "content": "be helpful"}
assert msgs[1] == {"role": "user", "content": "hello"}
class TestBuildResponseOutputItemsStatus:
"""Item ``status`` must follow the finish reason of the generation
(#57998): a ``length`` cut marks the item that was cut ``incomplete``."""
_CALLS = [
FunctionCall(name="get_weather", arguments='{"city": "Paris"}'),
FunctionCall(name="get_weather", arguments='{"city": "Tok'),
]
def test_length_marks_truncated_message_incomplete(self):
(item,) = build_response_output_items(
reasoning=None, content="cut mid sen", tool_calls=[], finish_reason="length"
)
assert item.type == "message"
assert item.status == "incomplete"
def test_length_marks_only_last_tool_call_incomplete(self):
message, first, last = build_response_output_items(
reasoning=None, content="hi", tool_calls=self._CALLS, finish_reason="length"
)
assert message.status == "completed"
assert first.status == "completed"
assert last.status == "incomplete"
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from collections import deque
from contextlib import AsyncExitStack, asynccontextmanager
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
@@ -51,6 +53,7 @@ from vllm.entrypoints.openai.responses.context import (
from vllm.entrypoints.openai.responses.protocol import (
ResponseCompletedEvent,
ResponseCreatedEvent,
ResponseIncompleteEvent,
ResponseRawMessageAndToken,
ResponsesRequest,
ResponsesResponse,
@@ -1414,6 +1417,7 @@ def _make_request_output(
text,
token_ids,
metrics: RequestStateStats | None = None,
finish_reason: str | None = None,
):
completion = CompletionOutput(
index=0,
@@ -1421,7 +1425,7 @@ def _make_request_output(
token_ids=token_ids,
cumulative_logprob=0.0,
logprobs=None,
finish_reason=None,
finish_reason=finish_reason,
stop_reason=None,
)
return RequestOutput(
@@ -1441,10 +1445,11 @@ def _make_simple_context_with_output(
token_ids,
response_parser=None,
metrics: RequestStateStats | None = None,
finish_reason: str | None = None,
):
"""Create a SimpleContext with a RequestOutput containing the given text."""
ctx = SimpleContext(response_parser=response_parser)
req_output = _make_request_output(text, token_ids, metrics)
req_output = _make_request_output(text, token_ids, metrics, finish_reason)
ctx.append_output(req_output)
ctx.request_metrics = metrics
return ctx
@@ -1561,6 +1566,66 @@ async def test_responses_streaming_metrics_only_on_completed_event():
assert events[-1].response.metrics.time_to_first_token_ms == pytest.approx(500.0)
async def _stream_truncated_by_length(serving: OpenAIServingResponses):
request = ResponsesRequest(input="hi", tools=[], stream=True, store=False)
context = _make_simple_context_with_output(
"cut mid sen", [10, 20], finish_reason="length"
)
async def one_context():
yield context
return [
event
async for event in serving.responses_stream_generator(
request=request,
sampling_params=SamplingParams(max_tokens=2),
result_generator=one_context(),
context=context,
model_name="test-model",
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="req"),
)
]
@pytest.mark.asyncio
async def test_responses_stream_truncated_by_length_reports_incomplete():
"""A max_output_tokens cut must end the stream with ``response.incomplete``
and mark the cut item ``incomplete`` in both ``output_item.done`` and the
final response (#57998)."""
events = await _stream_truncated_by_length(_make_serving_instance())
item_done = [e for e in events if e.type == "response.output_item.done"]
assert [e.item.status for e in item_done] == ["incomplete"]
final = events[-1]
assert isinstance(final, ResponseIncompleteEvent)
assert final.response.status == "incomplete"
assert final.response.incomplete_details.reason == "max_output_tokens"
assert [item.status for item in final.response.output] == ["incomplete"]
@pytest.mark.asyncio
async def test_background_stream_replay_stops_on_response_incomplete():
"""Replaying a stored background stream must end at ``response.incomplete``
as it does at ``response.completed``, instead of waiting for more events."""
serving = _make_serving_instance()
events = await _stream_truncated_by_length(serving)
assert isinstance(events[-1], ResponseIncompleteEvent)
serving.event_store["resp_bg"] = (deque(events), asyncio.Event())
async def replay(starting_after=None):
return [
event
async for event in serving.responses_background_stream_generator(
"resp_bg", starting_after=starting_after
)
]
assert await asyncio.wait_for(replay(), timeout=5) == events
assert await asyncio.wait_for(replay(starting_after=0), timeout=5) == events[1:]
@pytest.mark.asyncio
async def test_responses_metrics_suppressed_for_multiple_generation_turns():
response = await _make_full_metrics_response(