[Frontend] Make finish_reason and Responses item status truthful (#59324)

Signed-off-by: Flora Feng <4florafeng@gmail.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
Flora Feng
2026-10-15 10:12:02 +01:00
committed by GitHub
co-authored by Chauncey
parent 427915bed2
commit 80dd705c1d
14 changed files with 421 additions and 62 deletions
@@ -48,6 +48,7 @@ from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_includ
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.entrypoints.serve.utils.tool_calls_utils import (
maybe_filter_parallel_tool_calls,
resolve_finish_reason,
)
from vllm.exceptions import GenerationError
from vllm.inputs import EngineInput, MultiModalPlaceholders
@@ -491,11 +492,6 @@ class OpenAIServingChat(GenerateBaseServing):
num_cache_creation_tokens = None
tools_streamed = [False] * num_choices
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
tool_choice_function_name = request.tool_choice.function.name
else:
tool_choice_function_name = None
previous_texts = [""] * num_choices
try:
@@ -764,25 +760,13 @@ class OpenAIServingChat(GenerateBaseServing):
# if the model is finished generating
else:
# Send the finish response for each request.n only once
# In OpenAI's API, when a tool is called, the
# finish_reason is:
# "tool_calls" for "auto" or "required" tool calls,
# and "stop" for named tool calls.
if (
tools_streamed[i]
and not tool_choice_function_name
and output.finish_reason == "stop"
):
finish_reason_ = "tool_calls"
else:
finish_reason_ = (
output.finish_reason if output.finish_reason else "stop"
)
choice_data = ChatCompletionResponseStreamChoice(
index=i,
delta=delta_message,
logprobs=logprobs,
finish_reason=finish_reason_,
finish_reason=resolve_finish_reason(
output.finish_reason, request, tools_streamed[i]
),
stop_reason=output.stop_reason,
token_ids=(
as_list(output.token_ids) if include_token_ids else None
@@ -1013,7 +997,6 @@ class OpenAIServingChat(GenerateBaseServing):
tool_calls = []
suppress_metadata = False
auto_tools_called = False
is_named_tool_choice = (
request.tool_choice is not None
and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
@@ -1056,7 +1039,6 @@ class OpenAIServingChat(GenerateBaseServing):
and self.enable_auto_tools
and tool_parser_cls
):
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
if tool_calls:
message = self._create_chat_message(
role=role,
@@ -1093,15 +1075,6 @@ class OpenAIServingChat(GenerateBaseServing):
# metadata cached on the reasoning parser.
message = self._finalize_response_message(message, parser=parser)
# In OpenAI's API, when a tool is called, the finish_reason is:
# "tool_calls" for "auto" or "required" tool calls,
# and "stop" for named tool calls.
is_finish_reason_tool_calls = auto_tools_called or (
request.tool_choice
and request.tool_choice == "required"
and output.finish_reason == "stop"
)
routed_experts_b64 = (
numpy2base64(output.routed_experts)
if output.routed_experts is not None
@@ -1112,11 +1085,9 @@ class OpenAIServingChat(GenerateBaseServing):
index=output.index,
message=message,
logprobs=logprobs,
finish_reason="tool_calls"
if is_finish_reason_tool_calls
else output.finish_reason
if output.finish_reason
else "stop",
finish_reason=resolve_finish_reason(
output.finish_reason, request, bool(message.tool_calls)
),
stop_reason=output.stop_reason,
token_ids=(
as_list(output.token_ids)
+3 -1
View File
@@ -32,6 +32,7 @@ from vllm.entrypoints.openai.responses.protocol import (
from vllm.entrypoints.openai.responses.utils import (
build_response_output_items,
construct_tool_dicts,
response_item_status,
)
from vllm.entrypoints.serve.utils.tool_calls_utils import (
maybe_filter_parallel_tool_calls,
@@ -380,6 +381,7 @@ class ParsableContext(ConversationContext):
tool_calls or [], self.request
),
tools=self.request.tools,
finish_reason=completion.finish_reason,
)
)
elif completion.text:
@@ -387,7 +389,7 @@ class ParsableContext(ConversationContext):
ResponseOutputMessage(
type="message",
id=f"msg_{random_uuid()}",
status="completed",
status=response_item_status(completion.finish_reason),
role="assistant",
content=[
ResponseOutputText(
@@ -40,6 +40,9 @@ from openai.types.responses import (
ResponseCompletedEvent as OpenAIResponseCompletedEvent,
)
from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreatedEvent
from openai.types.responses import (
ResponseIncompleteEvent as OpenAIResponseIncompleteEvent,
)
from openai.types.responses import (
ResponseInProgressEvent as OpenAIResponseInProgressEvent,
)
@@ -927,6 +930,10 @@ class ResponseCompletedEvent(OpenAIResponseCompletedEvent):
response: ResponsesResponse # type: ignore[override]
class ResponseIncompleteEvent(OpenAIResponseIncompleteEvent):
response: ResponsesResponse # type: ignore[override]
class ResponseCreatedEvent(OpenAIResponseCreatedEvent):
response: ResponsesResponse # type: ignore[override]
@@ -939,6 +946,7 @@ StreamingResponsesResponse: TypeAlias = (
ResponseCreatedEvent
| ResponseInProgressEvent
| ResponseCompletedEvent
| ResponseIncompleteEvent
| ResponseOutputItemAddedEvent
| ResponseOutputItemDoneEvent
| ResponseContentPartAddedEvent
+24 -6
View File
@@ -45,6 +45,7 @@ from vllm.entrypoints.openai.responses.protocol import (
OutputTokensDetails,
ResponseCompletedEvent,
ResponseCreatedEvent,
ResponseIncompleteEvent,
ResponseInProgressEvent,
ResponseInputOutputItem,
ResponseInputOutputMessage,
@@ -66,6 +67,7 @@ from vllm.entrypoints.openai.responses.utils import (
build_response_output_items,
extract_function_tool_names,
extract_tool_types,
response_item_status,
reuse_streamed_item_ids,
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
@@ -1045,6 +1047,7 @@ class OpenAIServingResponses(GenerateBaseServing):
tool_calls=maybe_filter_parallel_tool_calls(tool_calls or [], request),
logprobs=logprobs,
tools=request.tools,
finish_reason=final_output.finish_reason,
)
# Fallback when no parser is configured
@@ -1062,7 +1065,7 @@ class OpenAIServingResponses(GenerateBaseServing):
if final_output.text
else [],
role="assistant",
status="completed",
status=response_item_status(final_output.finish_reason),
type="message",
)
]
@@ -1124,7 +1127,10 @@ class OpenAIServingResponses(GenerateBaseServing):
while current_index < len(event_deque):
event = event_deque[current_index]
yield event
if getattr(event, "type", "unknown") == "response.completed":
if getattr(event, "type", "unknown") in (
"response.completed",
"response.incomplete",
):
return
current_index += 1
@@ -1222,6 +1228,7 @@ class OpenAIServingResponses(GenerateBaseServing):
top_logprobs=request.top_logprobs,
)
finish_reason: str | None = None
async for ctx in result_generator:
assert isinstance(ctx, SimpleContext)
if ctx.last_output is None or not ctx.last_output.outputs:
@@ -1229,6 +1236,7 @@ class OpenAIServingResponses(GenerateBaseServing):
output = ctx.last_output.outputs[0]
self._raise_if_error(output.finish_reason, request.request_id)
finish_reason = output.finish_reason
delta_text = output.text
delta_token_ids = as_list(output.token_ids)
@@ -1261,7 +1269,7 @@ class OpenAIServingResponses(GenerateBaseServing):
for event in processor.emit_delta(dm, output, _get_logprobs):
yield _increment_sequence_number_and_return(event)
for event in processor.close_current():
for event in processor.close_current(response_item_status(finish_reason)):
yield _increment_sequence_number_and_return(event)
assert isinstance(context, SimpleContext)
@@ -1415,10 +1423,20 @@ class OpenAIServingResponses(GenerateBaseServing):
request_metadata,
created_time=created_time,
)
yield _increment_sequence_number_and_return(
ResponseCompletedEvent(
terminal_event: StreamingResponsesResponse
if (
isinstance(final_response, ResponsesResponse)
and final_response.status == "incomplete"
):
terminal_event = ResponseIncompleteEvent(
type="response.incomplete",
sequence_number=-1,
response=final_response,
)
else:
terminal_event = ResponseCompletedEvent(
type="response.completed",
sequence_number=-1,
response=final_response,
)
)
yield _increment_sequence_number_and_return(terminal_event)
@@ -18,7 +18,7 @@ import json
from collections.abc import Callable
from dataclasses import dataclass, field
from enum import Enum, auto
from typing import Any, ClassVar, Final, NamedTuple
from typing import Any, ClassVar, Final, Literal, NamedTuple
from openai.types.responses import (
ResponseCodeInterpreterCallCodeDeltaEvent,
@@ -910,6 +910,7 @@ def emit_simple_content_delta(
def emit_simple_content_done(
state: SimpleStreamingState,
status: Literal["completed", "incomplete"] = "completed",
) -> list[StreamingResponsesResponse]:
part = ResponseOutputText(
type="output_text",
@@ -944,7 +945,7 @@ def emit_simple_content_done(
type="message",
role="assistant",
content=[part] if state.accumulated_text else [],
status="completed",
status=status,
),
),
]
@@ -1002,6 +1003,7 @@ def emit_simple_reasoning_delta(
def emit_simple_reasoning_done(
state: SimpleStreamingState,
status: Literal["completed", "incomplete"] = "completed",
) -> list[StreamingResponsesResponse]:
part = ResponseReasoningTextContent(
text=state.accumulated_text,
@@ -1031,7 +1033,7 @@ def emit_simple_reasoning_done(
item=ResponseReasoningItem(
type="reasoning",
content=[part],
status="completed",
status=status,
id=state.current_item_id,
summary=[],
),
@@ -1134,6 +1136,7 @@ def emit_simple_tool_call_delta(
def emit_simple_tool_call_done(
state: SimpleStreamingState,
status: Literal["completed", "incomplete"] = "completed",
) -> list[StreamingResponsesResponse]:
events: list[StreamingResponsesResponse]
item: ResponseOutputItem
@@ -1172,7 +1175,7 @@ def emit_simple_tool_call_done(
name=state.tool_call_name,
namespace=state.tool_call_namespace,
arguments=state.accumulated_text,
status="completed",
status=status,
id=state.current_item_id,
call_id=state.tool_call_id,
)
@@ -1312,12 +1315,14 @@ class SimpleStreamingEventProcessor:
and self.state.tool_call_index != tool_call.index
)
def close_current(self) -> list[StreamingResponsesResponse]:
def close_current(
self, status: Literal["completed", "incomplete"] = "completed"
) -> list[StreamingResponsesResponse]:
"""Close the current state and emit its 'done' event sequence."""
handlers = self._STATE_HANDLERS.get(self.state.current_state)
if handlers is None:
return []
events = handlers.done_fn(self.state)
events = handlers.done_fn(self.state, status)
self.output_items.extend(
event.item
for event in events
+18 -3
View File
@@ -3,7 +3,7 @@
import json
from collections.abc import Iterable
from typing import Any
from typing import Any, Literal
import partial_json_parser
from openai.types.chat import (
@@ -100,16 +100,30 @@ def _item_dict(item: ResponseInputOutputItem) -> dict[str, Any]:
return item if isinstance(item, dict) else item.model_dump()
def response_item_status(
finish_reason: str | None,
) -> Literal["completed", "incomplete"]:
"""Status of the output item that ended a generation with ``finish_reason``."""
return "incomplete" if finish_reason == "length" else "completed"
def build_response_output_items(
reasoning: str | None,
content: str | None,
tool_calls: list[FunctionCall] | None,
logprobs: list[Logprob] | None = None,
tools: list[Tool] | None = None,
finish_reason: str | None = None,
) -> list[ResponseOutputItem]:
"""Build the output items of one generation.
Only the last item can have been cut short, so ``finish_reason`` decides
that item's status and every earlier item is ``completed``.
"""
outputs: list[ResponseOutputItem] = []
tool_call_name_map = build_responses_tool_call_name_map(tools)
custom_names = custom_tool_names(tools)
last_item_status = response_item_status(finish_reason)
if reasoning:
outputs.append(
@@ -137,13 +151,14 @@ def build_response_output_items(
)
],
role="assistant",
status="completed",
status="completed" if tool_calls else last_item_status,
type="message",
)
)
if tool_calls:
for idx, tool_call in enumerate(tool_calls):
is_last = idx == len(tool_calls) - 1
call_id = tool_call.id or make_tool_call_id(
func_name=tool_call.name, idx=idx
)
@@ -166,7 +181,7 @@ def build_response_output_items(
id=f"fc_{random_uuid()}",
call_id=call_id,
type="function_call",
status="completed",
status=last_item_status if is_last else "completed",
name=call_name.name,
namespace=call_name.namespace,
arguments=tool_call.arguments,
@@ -4,6 +4,7 @@ from typing import TypeVar
from vllm.entrypoints.generate.base.protocol import DeltaMessage, FunctionCall
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
ChatCompletionResponseChoice,
ChatCompletionResponseStreamChoice,
@@ -41,3 +42,24 @@ def maybe_filter_parallel_tool_calls(
tool_call for tool_call in delta.tool_calls if tool_call.index == 0
]
return item
def resolve_finish_reason(
engine_finish_reason: str | None,
request: ChatCompletionRequest | None,
tool_calls_made: bool,
) -> str:
"""Map the engine finish reason to the OpenAI chat ``finish_reason``.
``tool_calls`` is reported only when the choice actually carries tool
calls, the engine stopped on its own (a ``length`` cut stays visible so
clients can tell a truncated call from a complete one) and the caller did
not force a single named tool, for which OpenAI reports ``stop``.
"""
finish_reason = engine_finish_reason or "stop"
forced_named_tool = request is not None and isinstance(
request.tool_choice, ChatCompletionNamedToolChoiceParam
)
if tool_calls_made and finish_reason == "stop" and not forced_named_tool:
return "tool_calls"
return finish_reason