From 80dd705c1d422d04fcbd65d061dfedbf738ff573 Mon Sep 17 00:00:00 2001 From: Flora Feng <4florafeng@gmail.com> Date: Thu, 15 Oct 2026 05:12:02 -0400 Subject: [PATCH] [Frontend] Make finish_reason and Responses item status truthful (#59324) Signed-off-by: Flora Feng <4florafeng@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Chauncey --- .../chat_completion/test_serving_chat.py | 135 ++++++++++++++++++ .../responses/test_parsable_context_unit.py | 24 ++++ .../openai/responses/test_responses_utils.py | 25 ++++ .../responses/test_serving_responses.py | 69 ++++++++- .../scale_out/derender/test_derender.py | 59 ++++++++ .../derender/test_derender_stream.py | 15 +- .../openai/chat_completion/serving.py | 43 +----- vllm/entrypoints/openai/responses/context.py | 4 +- vllm/entrypoints/openai/responses/protocol.py | 8 ++ vllm/entrypoints/openai/responses/serving.py | 30 +++- .../openai/responses/streaming_events.py | 17 ++- vllm/entrypoints/openai/responses/utils.py | 21 ++- .../serve/utils/tool_calls_utils.py | 22 +++ vllm/renderers/online_derenderer.py | 11 +- 14 files changed, 421 insertions(+), 62 deletions(-) diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index d22f581f67..747b84a938 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -42,6 +42,7 @@ from vllm.entrypoints.openai.models.serving import ( ) from vllm.entrypoints.openai.parser.harmony_utils import get_encoding from vllm.entrypoints.serve.engine.protocol import ErrorResponse +from vllm.entrypoints.serve.utils.tool_calls_utils import resolve_finish_reason from vllm.exceptions import QueueOverflowError, VLLMValidationError from vllm.inputs import TokensPrompt from vllm.logprobs import Logprob @@ -3051,3 +3052,137 @@ def test_chat_kv_transfer_prompt_token_ids_allows_text_only_parts(): kv_transfer_params={"prompt_token_ids": [10, 20, 30]}, ) assert request.kv_transfer_params == {"prompt_token_ids": [10, 20, 30]} + + +_TOOL_FINISH_REASON_TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } +] +_HERMES_TOOL_CALL_TEXT = ( + '\n{"name": "get_weather", "arguments": {"city": "Tokyo"}}\n' +) + + +_NAMED_WEATHER_TOOL_CHOICE = {"type": "function", "function": {"name": "get_weather"}} + + +@pytest.mark.parametrize( + ("tool_choice", "engine_finish_reason", "tool_calls_made", "expected"), + [ + ("auto", "stop", True, "tool_calls"), + ("auto", "length", True, "length"), + ("auto", "stop", False, "stop"), + ("required", "stop", True, "tool_calls"), + ("required", "stop", False, "stop"), + ("required", "length", True, "length"), + (_NAMED_WEATHER_TOOL_CHOICE, "stop", True, "stop"), + (_NAMED_WEATHER_TOOL_CHOICE, "length", True, "length"), + ("none", "stop", False, "stop"), + (None, None, False, "stop"), + ], +) +def test_resolve_finish_reason( + tool_choice, engine_finish_reason, tool_calls_made, expected +): + """``tool_calls`` needs a real call, a natural stop and no forced tool; + ``length`` is never hidden behind a parsed call.""" + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "test"}], + tools=_TOOL_FINISH_REASON_TOOLS if tool_choice else None, + tool_choice=tool_choice, + ) + assert ( + resolve_finish_reason(engine_finish_reason, request, tool_calls_made) + == expected + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"), + [ + ("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"), + ("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"), + ("required", "Hello, how are you?", "stop", "stop"), + (_NAMED_WEATHER_TOOL_CHOICE, _HERMES_TOOL_CALL_TEXT, "stop", "stop"), + ], +) +async def test_non_streaming_finish_reason_matches_tool_calls( + monkeypatch: pytest.MonkeyPatch, + tool_choice, + text: str, + engine_finish_reason: str, + expected_finish_reason: str, +): + """Non-streaming ``finish_reason`` must agree with ``message.tool_calls`` + and never hide a ``length`` truncation (#53255, #58228).""" + serving_chat = _build_serving_chat( + _build_mock_engine(), tool_parser="hermes", enable_auto_tools=True + ) + # Parse required/named calls from the native format regardless + # of VLLM_ENFORCE_STRICT_TOOL_CALLING, so one text serves every case. + monkeypatch.setattr( + serving_chat.parser_cls.tool_parser_cls, "supports_required_and_named", False + ) + + tokenizer = get_tokenizer(MODEL_NAME) + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "test"}], + tools=_TOOL_FINISH_REASON_TOOLS, + tool_choice=tool_choice, + ) + parser = serving_chat.parser_cls( + tokenizer, + request.tools, + chat_template_kwargs=serving_chat._effective_chat_template_kwargs(request), + model_config=serving_chat.model_config, + ) + request_output = RequestOutput( + request_id="test-req", + prompt="test", + prompt_token_ids=[1, 2, 3], + prompt_logprobs=None, + outputs=[ + CompletionOutput( + index=0, + text=text, + token_ids=tokenizer.encode(text, add_special_tokens=False), + cumulative_logprob=0.0, + logprobs=None, + finish_reason=engine_finish_reason, + ) + ], + finished=True, + ) + + response = await serving_chat.chat_completion_full_generator( + request=request, + result_generator=_single_request_output(request_output), + request_id="test-req", + model_name=MODEL_NAME, + conversation=[], + tokenizer=tokenizer, + request_metadata=RequestResponseMetadata( + request_id="test-req", model_name=MODEL_NAME + ), + parser=parser, + ) + + assert isinstance(response, ChatCompletionResponse) + choice = response.choices[0] + assert choice.finish_reason == expected_finish_reason + assert bool(choice.message.tool_calls) == (text != "Hello, how are you?"), ( + "finish_reason must reflect whether tool calls were emitted" + ) diff --git a/tests/entrypoints/openai/responses/test_parsable_context_unit.py b/tests/entrypoints/openai/responses/test_parsable_context_unit.py index f3927c9d71..db11443ad1 100644 --- a/tests/entrypoints/openai/responses/test_parsable_context_unit.py +++ b/tests/entrypoints/openai/responses/test_parsable_context_unit.py @@ -456,3 +456,27 @@ def test_num_init_messages_offset(): items = ctx.make_response_output_items() assert len(items) == 1 assert items[0].type == "message" + + +def test_length_truncation_marks_last_item_incomplete(): + """A turn cut by max_output_tokens ends in an ``incomplete`` item, both + with a parser (tool call) and without one (plain message).""" + request = _make_request( + tool_choice="auto", + tools=[ + { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + } + ], + ) + ctx = _make_context(_ToolCallingParser, request=request, enable_auto_tools=True) + ctx.append_output(_make_request_output(text="calling tool", finish_reason="length")) + assert ctx.response_messages[-1].type == "function_call" + assert ctx.response_messages[-1].status == "incomplete" + + ctx = _make_context(None) + ctx.append_output(_make_request_output(text="cut mid sen", finish_reason="length")) + assert ctx.response_messages[-1].type == "message" + assert ctx.response_messages[-1].status == "incomplete" diff --git a/tests/entrypoints/openai/responses/test_responses_utils.py b/tests/entrypoints/openai/responses/test_responses_utils.py index 6789ddc08b..13da0a1e09 100644 --- a/tests/entrypoints/openai/responses/test_responses_utils.py +++ b/tests/entrypoints/openai/responses/test_responses_utils.py @@ -991,3 +991,28 @@ class TestConstructInputMessagesInstructionsLeak: assert len(msgs) == 2 assert msgs[0] == {"role": "system", "content": "be helpful"} assert msgs[1] == {"role": "user", "content": "hello"} + + +class TestBuildResponseOutputItemsStatus: + """Item ``status`` must follow the finish reason of the generation + (#57998): a ``length`` cut marks the item that was cut ``incomplete``.""" + + _CALLS = [ + FunctionCall(name="get_weather", arguments='{"city": "Paris"}'), + FunctionCall(name="get_weather", arguments='{"city": "Tok'), + ] + + def test_length_marks_truncated_message_incomplete(self): + (item,) = build_response_output_items( + reasoning=None, content="cut mid sen", tool_calls=[], finish_reason="length" + ) + assert item.type == "message" + assert item.status == "incomplete" + + def test_length_marks_only_last_tool_call_incomplete(self): + message, first, last = build_response_output_items( + reasoning=None, content="hi", tool_calls=self._CALLS, finish_reason="length" + ) + assert message.status == "completed" + assert first.status == "completed" + assert last.status == "incomplete" diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index 1622d0e8f8..67b9dd4e4f 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import asyncio +from collections import deque from contextlib import AsyncExitStack, asynccontextmanager from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -51,6 +53,7 @@ from vllm.entrypoints.openai.responses.context import ( from vllm.entrypoints.openai.responses.protocol import ( ResponseCompletedEvent, ResponseCreatedEvent, + ResponseIncompleteEvent, ResponseRawMessageAndToken, ResponsesRequest, ResponsesResponse, @@ -1414,6 +1417,7 @@ def _make_request_output( text, token_ids, metrics: RequestStateStats | None = None, + finish_reason: str | None = None, ): completion = CompletionOutput( index=0, @@ -1421,7 +1425,7 @@ def _make_request_output( token_ids=token_ids, cumulative_logprob=0.0, logprobs=None, - finish_reason=None, + finish_reason=finish_reason, stop_reason=None, ) return RequestOutput( @@ -1441,10 +1445,11 @@ def _make_simple_context_with_output( token_ids, response_parser=None, metrics: RequestStateStats | None = None, + finish_reason: str | None = None, ): """Create a SimpleContext with a RequestOutput containing the given text.""" ctx = SimpleContext(response_parser=response_parser) - req_output = _make_request_output(text, token_ids, metrics) + req_output = _make_request_output(text, token_ids, metrics, finish_reason) ctx.append_output(req_output) ctx.request_metrics = metrics return ctx @@ -1561,6 +1566,66 @@ async def test_responses_streaming_metrics_only_on_completed_event(): assert events[-1].response.metrics.time_to_first_token_ms == pytest.approx(500.0) +async def _stream_truncated_by_length(serving: OpenAIServingResponses): + request = ResponsesRequest(input="hi", tools=[], stream=True, store=False) + context = _make_simple_context_with_output( + "cut mid sen", [10, 20], finish_reason="length" + ) + + async def one_context(): + yield context + + return [ + event + async for event in serving.responses_stream_generator( + request=request, + sampling_params=SamplingParams(max_tokens=2), + result_generator=one_context(), + context=context, + model_name="test-model", + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="req"), + ) + ] + + +@pytest.mark.asyncio +async def test_responses_stream_truncated_by_length_reports_incomplete(): + """A max_output_tokens cut must end the stream with ``response.incomplete`` + and mark the cut item ``incomplete`` in both ``output_item.done`` and the + final response (#57998).""" + events = await _stream_truncated_by_length(_make_serving_instance()) + + item_done = [e for e in events if e.type == "response.output_item.done"] + assert [e.item.status for e in item_done] == ["incomplete"] + final = events[-1] + assert isinstance(final, ResponseIncompleteEvent) + assert final.response.status == "incomplete" + assert final.response.incomplete_details.reason == "max_output_tokens" + assert [item.status for item in final.response.output] == ["incomplete"] + + +@pytest.mark.asyncio +async def test_background_stream_replay_stops_on_response_incomplete(): + """Replaying a stored background stream must end at ``response.incomplete`` + as it does at ``response.completed``, instead of waiting for more events.""" + serving = _make_serving_instance() + events = await _stream_truncated_by_length(serving) + assert isinstance(events[-1], ResponseIncompleteEvent) + serving.event_store["resp_bg"] = (deque(events), asyncio.Event()) + + async def replay(starting_after=None): + return [ + event + async for event in serving.responses_background_stream_generator( + "resp_bg", starting_after=starting_after + ) + ] + + assert await asyncio.wait_for(replay(), timeout=5) == events + assert await asyncio.wait_for(replay(starting_after=0), timeout=5) == events[1:] + + @pytest.mark.asyncio async def test_responses_metrics_suppressed_for_multiple_generation_turns(): response = await _make_full_metrics_response( diff --git a/tests/entrypoints/scale_out/derender/test_derender.py b/tests/entrypoints/scale_out/derender/test_derender.py index 30d64ed39b..771213309f 100644 --- a/tests/entrypoints/scale_out/derender/test_derender.py +++ b/tests/entrypoints/scale_out/derender/test_derender.py @@ -3,11 +3,18 @@ """Tests for the /derender endpoints (postprocessing counterpart to /render).""" +from unittest.mock import MagicMock + import httpx import pytest import pytest_asyncio from tests.utils import RemoteLaunchRenderServer +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + GenerateTokensResponse, +) +from vllm.renderers.online_derenderer import OnlineDerenderer from vllm.tokenizers import get_tokenizer MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM" @@ -1432,3 +1439,55 @@ async def test_e2e_harmony_reasoning(harmony_client, harmony_tokenizer): assert msg["reasoning"] is not None assert reasoning_text in msg["reasoning"] assert answer_text in (msg["content"] or "") + + +_HERMES_TOOL_CALL_TEXT = ( + '\n{"name": "get_weather", "arguments": {"city": "Paris"}}\n' +) + + +@pytest.mark.parametrize( + ("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"), + [ + ("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"), + ("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"), + ("required", "Hello there", "stop", "stop"), + ], +) +def test_derender_chat_finish_reason_matches_chat_completion( + tool_choice, text, engine_finish_reason, expected_finish_reason +): + """/derender applies the same finish_reason rule as the coupled chat + path: ``tool_calls`` only for a real call at a natural stop.""" + tokenizer = get_tokenizer(MODEL_NAME) + renderer = MagicMock() + renderer.get_tokenizer.return_value = tokenizer + model_config = MagicMock() + model_config.model = MODEL_NAME + model_config.hf_config.model_type = "llama" + derenderer = OnlineDerenderer( + model_config, + renderer, + request_logger=None, + chat_template="{{ messages }}", + chat_template_content_format="auto", + enable_auto_tools=True, + tool_parser="hermes", + ) + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "Weather in Paris?"}], + tools=_E2E_TOOLS, + tool_choice=tool_choice, + ) + response = GenerateTokensResponse.model_validate( + _make_generate_response( + tokenizer.encode(text, add_special_tokens=False), + finish_reason=engine_finish_reason, + ) + ) + + (choice,) = derenderer._derender_chat(response, request) + + assert choice.finish_reason == expected_finish_reason + assert bool(choice.message.tool_calls) == (text == _HERMES_TOOL_CALL_TEXT) diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index 55b0789843..a1c4e2a330 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -1289,7 +1289,15 @@ class TestDerenderChatStreamParsed: assert state.last_tool_call_ids == [first_id] @pytest.mark.asyncio - async def test_finish_reason_rewritten_to_tool_calls(self, parsed_derenderer): + @pytest.mark.parametrize( + ("engine_finish_reason", "expected_finish_reason"), + [("stop", "tool_calls"), ("length", "length")], + ) + async def test_finish_reason_rewritten_to_tool_calls( + self, parsed_derenderer, engine_finish_reason, expected_finish_reason + ): + """A streamed tool call is reported as ``tool_calls`` only at a + natural stop; a ``length`` cut must stay visible.""" chat_request = _chat_request( tools=[{"type": "function", "function": {"name": "get_weather"}}], tool_choice="auto", @@ -1297,11 +1305,12 @@ class TestDerenderChatStreamParsed: chunk, _ = await parsed_derenderer.derender_chat_stream( model=MODEL_NAME, generate_chunk=_make_stream_chunk( - [_FakeParser.TOOL_START, _FakeParser.TOOL_ARG], finish_reason="stop" + [_FakeParser.TOOL_START, _FakeParser.TOOL_ARG], + finish_reason=engine_finish_reason, ), chat_request=chat_request, ) - assert chunk.choices[0].finish_reason == "tool_calls" + assert chunk.choices[0].finish_reason == expected_finish_reason @pytest.mark.asyncio async def test_finish_reason_stop_for_named_tool_choice(self, parsed_derenderer): diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 84061a8822..007c961ca8 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -48,6 +48,7 @@ from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_includ from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.entrypoints.serve.utils.tool_calls_utils import ( maybe_filter_parallel_tool_calls, + resolve_finish_reason, ) from vllm.exceptions import GenerationError from vllm.inputs import EngineInput, MultiModalPlaceholders @@ -491,11 +492,6 @@ class OpenAIServingChat(GenerateBaseServing): num_cache_creation_tokens = None tools_streamed = [False] * num_choices - if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam): - tool_choice_function_name = request.tool_choice.function.name - else: - tool_choice_function_name = None - previous_texts = [""] * num_choices try: @@ -764,25 +760,13 @@ class OpenAIServingChat(GenerateBaseServing): # if the model is finished generating else: # Send the finish response for each request.n only once - # In OpenAI's API, when a tool is called, the - # finish_reason is: - # "tool_calls" for "auto" or "required" tool calls, - # and "stop" for named tool calls. - if ( - tools_streamed[i] - and not tool_choice_function_name - and output.finish_reason == "stop" - ): - finish_reason_ = "tool_calls" - else: - finish_reason_ = ( - output.finish_reason if output.finish_reason else "stop" - ) choice_data = ChatCompletionResponseStreamChoice( index=i, delta=delta_message, logprobs=logprobs, - finish_reason=finish_reason_, + finish_reason=resolve_finish_reason( + output.finish_reason, request, tools_streamed[i] + ), stop_reason=output.stop_reason, token_ids=( as_list(output.token_ids) if include_token_ids else None @@ -1013,7 +997,6 @@ class OpenAIServingChat(GenerateBaseServing): tool_calls = [] suppress_metadata = False - auto_tools_called = False is_named_tool_choice = ( request.tool_choice is not None and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam @@ -1056,7 +1039,6 @@ class OpenAIServingChat(GenerateBaseServing): and self.enable_auto_tools and tool_parser_cls ): - auto_tools_called = tool_calls is not None and len(tool_calls) > 0 if tool_calls: message = self._create_chat_message( role=role, @@ -1093,15 +1075,6 @@ class OpenAIServingChat(GenerateBaseServing): # metadata cached on the reasoning parser. message = self._finalize_response_message(message, parser=parser) - # In OpenAI's API, when a tool is called, the finish_reason is: - # "tool_calls" for "auto" or "required" tool calls, - # and "stop" for named tool calls. - is_finish_reason_tool_calls = auto_tools_called or ( - request.tool_choice - and request.tool_choice == "required" - and output.finish_reason == "stop" - ) - routed_experts_b64 = ( numpy2base64(output.routed_experts) if output.routed_experts is not None @@ -1112,11 +1085,9 @@ class OpenAIServingChat(GenerateBaseServing): index=output.index, message=message, logprobs=logprobs, - finish_reason="tool_calls" - if is_finish_reason_tool_calls - else output.finish_reason - if output.finish_reason - else "stop", + finish_reason=resolve_finish_reason( + output.finish_reason, request, bool(message.tool_calls) + ), stop_reason=output.stop_reason, token_ids=( as_list(output.token_ids) diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index 9f5a281163..6aaae51910 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -32,6 +32,7 @@ from vllm.entrypoints.openai.responses.protocol import ( from vllm.entrypoints.openai.responses.utils import ( build_response_output_items, construct_tool_dicts, + response_item_status, ) from vllm.entrypoints.serve.utils.tool_calls_utils import ( maybe_filter_parallel_tool_calls, @@ -380,6 +381,7 @@ class ParsableContext(ConversationContext): tool_calls or [], self.request ), tools=self.request.tools, + finish_reason=completion.finish_reason, ) ) elif completion.text: @@ -387,7 +389,7 @@ class ParsableContext(ConversationContext): ResponseOutputMessage( type="message", id=f"msg_{random_uuid()}", - status="completed", + status=response_item_status(completion.finish_reason), role="assistant", content=[ ResponseOutputText( diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index 200054a4ef..0e94bca51f 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -40,6 +40,9 @@ from openai.types.responses import ( ResponseCompletedEvent as OpenAIResponseCompletedEvent, ) from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreatedEvent +from openai.types.responses import ( + ResponseIncompleteEvent as OpenAIResponseIncompleteEvent, +) from openai.types.responses import ( ResponseInProgressEvent as OpenAIResponseInProgressEvent, ) @@ -927,6 +930,10 @@ class ResponseCompletedEvent(OpenAIResponseCompletedEvent): response: ResponsesResponse # type: ignore[override] +class ResponseIncompleteEvent(OpenAIResponseIncompleteEvent): + response: ResponsesResponse # type: ignore[override] + + class ResponseCreatedEvent(OpenAIResponseCreatedEvent): response: ResponsesResponse # type: ignore[override] @@ -939,6 +946,7 @@ StreamingResponsesResponse: TypeAlias = ( ResponseCreatedEvent | ResponseInProgressEvent | ResponseCompletedEvent + | ResponseIncompleteEvent | ResponseOutputItemAddedEvent | ResponseOutputItemDoneEvent | ResponseContentPartAddedEvent diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 0d46855c94..dd154f968b 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -45,6 +45,7 @@ from vllm.entrypoints.openai.responses.protocol import ( OutputTokensDetails, ResponseCompletedEvent, ResponseCreatedEvent, + ResponseIncompleteEvent, ResponseInProgressEvent, ResponseInputOutputItem, ResponseInputOutputMessage, @@ -66,6 +67,7 @@ from vllm.entrypoints.openai.responses.utils import ( build_response_output_items, extract_function_tool_names, extract_tool_types, + response_item_status, reuse_streamed_item_ids, ) from vllm.entrypoints.serve.engine.protocol import ErrorResponse @@ -1045,6 +1047,7 @@ class OpenAIServingResponses(GenerateBaseServing): tool_calls=maybe_filter_parallel_tool_calls(tool_calls or [], request), logprobs=logprobs, tools=request.tools, + finish_reason=final_output.finish_reason, ) # Fallback when no parser is configured @@ -1062,7 +1065,7 @@ class OpenAIServingResponses(GenerateBaseServing): if final_output.text else [], role="assistant", - status="completed", + status=response_item_status(final_output.finish_reason), type="message", ) ] @@ -1124,7 +1127,10 @@ class OpenAIServingResponses(GenerateBaseServing): while current_index < len(event_deque): event = event_deque[current_index] yield event - if getattr(event, "type", "unknown") == "response.completed": + if getattr(event, "type", "unknown") in ( + "response.completed", + "response.incomplete", + ): return current_index += 1 @@ -1222,6 +1228,7 @@ class OpenAIServingResponses(GenerateBaseServing): top_logprobs=request.top_logprobs, ) + finish_reason: str | None = None async for ctx in result_generator: assert isinstance(ctx, SimpleContext) if ctx.last_output is None or not ctx.last_output.outputs: @@ -1229,6 +1236,7 @@ class OpenAIServingResponses(GenerateBaseServing): output = ctx.last_output.outputs[0] self._raise_if_error(output.finish_reason, request.request_id) + finish_reason = output.finish_reason delta_text = output.text delta_token_ids = as_list(output.token_ids) @@ -1261,7 +1269,7 @@ class OpenAIServingResponses(GenerateBaseServing): for event in processor.emit_delta(dm, output, _get_logprobs): yield _increment_sequence_number_and_return(event) - for event in processor.close_current(): + for event in processor.close_current(response_item_status(finish_reason)): yield _increment_sequence_number_and_return(event) assert isinstance(context, SimpleContext) @@ -1415,10 +1423,20 @@ class OpenAIServingResponses(GenerateBaseServing): request_metadata, created_time=created_time, ) - yield _increment_sequence_number_and_return( - ResponseCompletedEvent( + terminal_event: StreamingResponsesResponse + if ( + isinstance(final_response, ResponsesResponse) + and final_response.status == "incomplete" + ): + terminal_event = ResponseIncompleteEvent( + type="response.incomplete", + sequence_number=-1, + response=final_response, + ) + else: + terminal_event = ResponseCompletedEvent( type="response.completed", sequence_number=-1, response=final_response, ) - ) + yield _increment_sequence_number_and_return(terminal_event) diff --git a/vllm/entrypoints/openai/responses/streaming_events.py b/vllm/entrypoints/openai/responses/streaming_events.py index 49107661a7..b14379cf12 100644 --- a/vllm/entrypoints/openai/responses/streaming_events.py +++ b/vllm/entrypoints/openai/responses/streaming_events.py @@ -18,7 +18,7 @@ import json from collections.abc import Callable from dataclasses import dataclass, field from enum import Enum, auto -from typing import Any, ClassVar, Final, NamedTuple +from typing import Any, ClassVar, Final, Literal, NamedTuple from openai.types.responses import ( ResponseCodeInterpreterCallCodeDeltaEvent, @@ -910,6 +910,7 @@ def emit_simple_content_delta( def emit_simple_content_done( state: SimpleStreamingState, + status: Literal["completed", "incomplete"] = "completed", ) -> list[StreamingResponsesResponse]: part = ResponseOutputText( type="output_text", @@ -944,7 +945,7 @@ def emit_simple_content_done( type="message", role="assistant", content=[part] if state.accumulated_text else [], - status="completed", + status=status, ), ), ] @@ -1002,6 +1003,7 @@ def emit_simple_reasoning_delta( def emit_simple_reasoning_done( state: SimpleStreamingState, + status: Literal["completed", "incomplete"] = "completed", ) -> list[StreamingResponsesResponse]: part = ResponseReasoningTextContent( text=state.accumulated_text, @@ -1031,7 +1033,7 @@ def emit_simple_reasoning_done( item=ResponseReasoningItem( type="reasoning", content=[part], - status="completed", + status=status, id=state.current_item_id, summary=[], ), @@ -1134,6 +1136,7 @@ def emit_simple_tool_call_delta( def emit_simple_tool_call_done( state: SimpleStreamingState, + status: Literal["completed", "incomplete"] = "completed", ) -> list[StreamingResponsesResponse]: events: list[StreamingResponsesResponse] item: ResponseOutputItem @@ -1172,7 +1175,7 @@ def emit_simple_tool_call_done( name=state.tool_call_name, namespace=state.tool_call_namespace, arguments=state.accumulated_text, - status="completed", + status=status, id=state.current_item_id, call_id=state.tool_call_id, ) @@ -1312,12 +1315,14 @@ class SimpleStreamingEventProcessor: and self.state.tool_call_index != tool_call.index ) - def close_current(self) -> list[StreamingResponsesResponse]: + def close_current( + self, status: Literal["completed", "incomplete"] = "completed" + ) -> list[StreamingResponsesResponse]: """Close the current state and emit its 'done' event sequence.""" handlers = self._STATE_HANDLERS.get(self.state.current_state) if handlers is None: return [] - events = handlers.done_fn(self.state) + events = handlers.done_fn(self.state, status) self.output_items.extend( event.item for event in events diff --git a/vllm/entrypoints/openai/responses/utils.py b/vllm/entrypoints/openai/responses/utils.py index d9b71d6849..d3e3d698c4 100644 --- a/vllm/entrypoints/openai/responses/utils.py +++ b/vllm/entrypoints/openai/responses/utils.py @@ -3,7 +3,7 @@ import json from collections.abc import Iterable -from typing import Any +from typing import Any, Literal import partial_json_parser from openai.types.chat import ( @@ -100,16 +100,30 @@ def _item_dict(item: ResponseInputOutputItem) -> dict[str, Any]: return item if isinstance(item, dict) else item.model_dump() +def response_item_status( + finish_reason: str | None, +) -> Literal["completed", "incomplete"]: + """Status of the output item that ended a generation with ``finish_reason``.""" + return "incomplete" if finish_reason == "length" else "completed" + + def build_response_output_items( reasoning: str | None, content: str | None, tool_calls: list[FunctionCall] | None, logprobs: list[Logprob] | None = None, tools: list[Tool] | None = None, + finish_reason: str | None = None, ) -> list[ResponseOutputItem]: + """Build the output items of one generation. + + Only the last item can have been cut short, so ``finish_reason`` decides + that item's status and every earlier item is ``completed``. + """ outputs: list[ResponseOutputItem] = [] tool_call_name_map = build_responses_tool_call_name_map(tools) custom_names = custom_tool_names(tools) + last_item_status = response_item_status(finish_reason) if reasoning: outputs.append( @@ -137,13 +151,14 @@ def build_response_output_items( ) ], role="assistant", - status="completed", + status="completed" if tool_calls else last_item_status, type="message", ) ) if tool_calls: for idx, tool_call in enumerate(tool_calls): + is_last = idx == len(tool_calls) - 1 call_id = tool_call.id or make_tool_call_id( func_name=tool_call.name, idx=idx ) @@ -166,7 +181,7 @@ def build_response_output_items( id=f"fc_{random_uuid()}", call_id=call_id, type="function_call", - status="completed", + status=last_item_status if is_last else "completed", name=call_name.name, namespace=call_name.namespace, arguments=tool_call.arguments, diff --git a/vllm/entrypoints/serve/utils/tool_calls_utils.py b/vllm/entrypoints/serve/utils/tool_calls_utils.py index 1252218127..07ed15b897 100644 --- a/vllm/entrypoints/serve/utils/tool_calls_utils.py +++ b/vllm/entrypoints/serve/utils/tool_calls_utils.py @@ -4,6 +4,7 @@ from typing import TypeVar from vllm.entrypoints.generate.base.protocol import DeltaMessage, FunctionCall from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionNamedToolChoiceParam, ChatCompletionRequest, ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, @@ -41,3 +42,24 @@ def maybe_filter_parallel_tool_calls( tool_call for tool_call in delta.tool_calls if tool_call.index == 0 ] return item + + +def resolve_finish_reason( + engine_finish_reason: str | None, + request: ChatCompletionRequest | None, + tool_calls_made: bool, +) -> str: + """Map the engine finish reason to the OpenAI chat ``finish_reason``. + + ``tool_calls`` is reported only when the choice actually carries tool + calls, the engine stopped on its own (a ``length`` cut stays visible so + clients can tell a truncated call from a complete one) and the caller did + not force a single named tool, for which OpenAI reports ``stop``. + """ + finish_reason = engine_finish_reason or "stop" + forced_named_tool = request is not None and isinstance( + request.tool_choice, ChatCompletionNamedToolChoiceParam + ) + if tool_calls_made and finish_reason == "stop" and not forced_named_tool: + return "tool_calls" + return finish_reason diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index f7e04fd1c7..09ea33857c 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -37,6 +37,7 @@ from vllm.entrypoints.serve.engine.protocol import UsageInfo from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.entrypoints.serve.utils.tool_calls_utils import ( maybe_filter_parallel_tool_calls, + resolve_finish_reason, ) from vllm.logger import init_logger from vllm.parser import Parser, ParserManager @@ -205,7 +206,9 @@ class OnlineDerenderer: index=choice.index, message=message, logprobs=resolved_logprobs, - finish_reason=choice.finish_reason, + finish_reason=resolve_finish_reason( + choice.finish_reason, chat_request, bool(message.tool_calls) + ), stop_reason=choice.stop_reason, token_ids=choice.token_ids if return_token_ids else None, ) @@ -664,11 +667,9 @@ class OnlineDerenderer: finish_reason = choice.finish_reason if finish_reason is not None: - is_named_tool_choice = ( - type(chat_request.tool_choice) is ChatCompletionNamedToolChoiceParam + finish_reason = resolve_finish_reason( + finish_reason, chat_request, tools_streamed ) - if tools_streamed and not is_named_tool_choice: - finish_reason = "tool_calls" stream_choice = ChatCompletionResponseStreamChoice( index=choice.index,