[Frontend] Make finish_reason and Responses item status truthful (#59324)
Signed-off-by: Flora Feng <4florafeng@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -48,6 +48,7 @@ from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_includ
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
||||
maybe_filter_parallel_tool_calls,
|
||||
resolve_finish_reason,
|
||||
)
|
||||
from vllm.exceptions import GenerationError
|
||||
from vllm.inputs import EngineInput, MultiModalPlaceholders
|
||||
@@ -491,11 +492,6 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
num_cache_creation_tokens = None
|
||||
tools_streamed = [False] * num_choices
|
||||
|
||||
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
|
||||
tool_choice_function_name = request.tool_choice.function.name
|
||||
else:
|
||||
tool_choice_function_name = None
|
||||
|
||||
previous_texts = [""] * num_choices
|
||||
|
||||
try:
|
||||
@@ -764,25 +760,13 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
# if the model is finished generating
|
||||
else:
|
||||
# Send the finish response for each request.n only once
|
||||
# In OpenAI's API, when a tool is called, the
|
||||
# finish_reason is:
|
||||
# "tool_calls" for "auto" or "required" tool calls,
|
||||
# and "stop" for named tool calls.
|
||||
if (
|
||||
tools_streamed[i]
|
||||
and not tool_choice_function_name
|
||||
and output.finish_reason == "stop"
|
||||
):
|
||||
finish_reason_ = "tool_calls"
|
||||
else:
|
||||
finish_reason_ = (
|
||||
output.finish_reason if output.finish_reason else "stop"
|
||||
)
|
||||
choice_data = ChatCompletionResponseStreamChoice(
|
||||
index=i,
|
||||
delta=delta_message,
|
||||
logprobs=logprobs,
|
||||
finish_reason=finish_reason_,
|
||||
finish_reason=resolve_finish_reason(
|
||||
output.finish_reason, request, tools_streamed[i]
|
||||
),
|
||||
stop_reason=output.stop_reason,
|
||||
token_ids=(
|
||||
as_list(output.token_ids) if include_token_ids else None
|
||||
@@ -1013,7 +997,6 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
tool_calls = []
|
||||
suppress_metadata = False
|
||||
|
||||
auto_tools_called = False
|
||||
is_named_tool_choice = (
|
||||
request.tool_choice is not None
|
||||
and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
|
||||
@@ -1056,7 +1039,6 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
and self.enable_auto_tools
|
||||
and tool_parser_cls
|
||||
):
|
||||
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
|
||||
if tool_calls:
|
||||
message = self._create_chat_message(
|
||||
role=role,
|
||||
@@ -1093,15 +1075,6 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
# metadata cached on the reasoning parser.
|
||||
message = self._finalize_response_message(message, parser=parser)
|
||||
|
||||
# In OpenAI's API, when a tool is called, the finish_reason is:
|
||||
# "tool_calls" for "auto" or "required" tool calls,
|
||||
# and "stop" for named tool calls.
|
||||
is_finish_reason_tool_calls = auto_tools_called or (
|
||||
request.tool_choice
|
||||
and request.tool_choice == "required"
|
||||
and output.finish_reason == "stop"
|
||||
)
|
||||
|
||||
routed_experts_b64 = (
|
||||
numpy2base64(output.routed_experts)
|
||||
if output.routed_experts is not None
|
||||
@@ -1112,11 +1085,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
index=output.index,
|
||||
message=message,
|
||||
logprobs=logprobs,
|
||||
finish_reason="tool_calls"
|
||||
if is_finish_reason_tool_calls
|
||||
else output.finish_reason
|
||||
if output.finish_reason
|
||||
else "stop",
|
||||
finish_reason=resolve_finish_reason(
|
||||
output.finish_reason, request, bool(message.tool_calls)
|
||||
),
|
||||
stop_reason=output.stop_reason,
|
||||
token_ids=(
|
||||
as_list(output.token_ids)
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.entrypoints.openai.responses.protocol import (
|
||||
from vllm.entrypoints.openai.responses.utils import (
|
||||
build_response_output_items,
|
||||
construct_tool_dicts,
|
||||
response_item_status,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
||||
maybe_filter_parallel_tool_calls,
|
||||
@@ -380,6 +381,7 @@ class ParsableContext(ConversationContext):
|
||||
tool_calls or [], self.request
|
||||
),
|
||||
tools=self.request.tools,
|
||||
finish_reason=completion.finish_reason,
|
||||
)
|
||||
)
|
||||
elif completion.text:
|
||||
@@ -387,7 +389,7 @@ class ParsableContext(ConversationContext):
|
||||
ResponseOutputMessage(
|
||||
type="message",
|
||||
id=f"msg_{random_uuid()}",
|
||||
status="completed",
|
||||
status=response_item_status(completion.finish_reason),
|
||||
role="assistant",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
|
||||
@@ -40,6 +40,9 @@ from openai.types.responses import (
|
||||
ResponseCompletedEvent as OpenAIResponseCompletedEvent,
|
||||
)
|
||||
from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreatedEvent
|
||||
from openai.types.responses import (
|
||||
ResponseIncompleteEvent as OpenAIResponseIncompleteEvent,
|
||||
)
|
||||
from openai.types.responses import (
|
||||
ResponseInProgressEvent as OpenAIResponseInProgressEvent,
|
||||
)
|
||||
@@ -927,6 +930,10 @@ class ResponseCompletedEvent(OpenAIResponseCompletedEvent):
|
||||
response: ResponsesResponse # type: ignore[override]
|
||||
|
||||
|
||||
class ResponseIncompleteEvent(OpenAIResponseIncompleteEvent):
|
||||
response: ResponsesResponse # type: ignore[override]
|
||||
|
||||
|
||||
class ResponseCreatedEvent(OpenAIResponseCreatedEvent):
|
||||
response: ResponsesResponse # type: ignore[override]
|
||||
|
||||
@@ -939,6 +946,7 @@ StreamingResponsesResponse: TypeAlias = (
|
||||
ResponseCreatedEvent
|
||||
| ResponseInProgressEvent
|
||||
| ResponseCompletedEvent
|
||||
| ResponseIncompleteEvent
|
||||
| ResponseOutputItemAddedEvent
|
||||
| ResponseOutputItemDoneEvent
|
||||
| ResponseContentPartAddedEvent
|
||||
|
||||
@@ -45,6 +45,7 @@ from vllm.entrypoints.openai.responses.protocol import (
|
||||
OutputTokensDetails,
|
||||
ResponseCompletedEvent,
|
||||
ResponseCreatedEvent,
|
||||
ResponseIncompleteEvent,
|
||||
ResponseInProgressEvent,
|
||||
ResponseInputOutputItem,
|
||||
ResponseInputOutputMessage,
|
||||
@@ -66,6 +67,7 @@ from vllm.entrypoints.openai.responses.utils import (
|
||||
build_response_output_items,
|
||||
extract_function_tool_names,
|
||||
extract_tool_types,
|
||||
response_item_status,
|
||||
reuse_streamed_item_ids,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
@@ -1045,6 +1047,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
tool_calls=maybe_filter_parallel_tool_calls(tool_calls or [], request),
|
||||
logprobs=logprobs,
|
||||
tools=request.tools,
|
||||
finish_reason=final_output.finish_reason,
|
||||
)
|
||||
|
||||
# Fallback when no parser is configured
|
||||
@@ -1062,7 +1065,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
if final_output.text
|
||||
else [],
|
||||
role="assistant",
|
||||
status="completed",
|
||||
status=response_item_status(final_output.finish_reason),
|
||||
type="message",
|
||||
)
|
||||
]
|
||||
@@ -1124,7 +1127,10 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
while current_index < len(event_deque):
|
||||
event = event_deque[current_index]
|
||||
yield event
|
||||
if getattr(event, "type", "unknown") == "response.completed":
|
||||
if getattr(event, "type", "unknown") in (
|
||||
"response.completed",
|
||||
"response.incomplete",
|
||||
):
|
||||
return
|
||||
current_index += 1
|
||||
|
||||
@@ -1222,6 +1228,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
top_logprobs=request.top_logprobs,
|
||||
)
|
||||
|
||||
finish_reason: str | None = None
|
||||
async for ctx in result_generator:
|
||||
assert isinstance(ctx, SimpleContext)
|
||||
if ctx.last_output is None or not ctx.last_output.outputs:
|
||||
@@ -1229,6 +1236,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
|
||||
output = ctx.last_output.outputs[0]
|
||||
self._raise_if_error(output.finish_reason, request.request_id)
|
||||
finish_reason = output.finish_reason
|
||||
delta_text = output.text
|
||||
delta_token_ids = as_list(output.token_ids)
|
||||
|
||||
@@ -1261,7 +1269,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
for event in processor.emit_delta(dm, output, _get_logprobs):
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
|
||||
for event in processor.close_current():
|
||||
for event in processor.close_current(response_item_status(finish_reason)):
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
|
||||
assert isinstance(context, SimpleContext)
|
||||
@@ -1415,10 +1423,20 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
request_metadata,
|
||||
created_time=created_time,
|
||||
)
|
||||
yield _increment_sequence_number_and_return(
|
||||
ResponseCompletedEvent(
|
||||
terminal_event: StreamingResponsesResponse
|
||||
if (
|
||||
isinstance(final_response, ResponsesResponse)
|
||||
and final_response.status == "incomplete"
|
||||
):
|
||||
terminal_event = ResponseIncompleteEvent(
|
||||
type="response.incomplete",
|
||||
sequence_number=-1,
|
||||
response=final_response,
|
||||
)
|
||||
else:
|
||||
terminal_event = ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
sequence_number=-1,
|
||||
response=final_response,
|
||||
)
|
||||
)
|
||||
yield _increment_sequence_number_and_return(terminal_event)
|
||||
|
||||
@@ -18,7 +18,7 @@ import json
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum, auto
|
||||
from typing import Any, ClassVar, Final, NamedTuple
|
||||
from typing import Any, ClassVar, Final, Literal, NamedTuple
|
||||
|
||||
from openai.types.responses import (
|
||||
ResponseCodeInterpreterCallCodeDeltaEvent,
|
||||
@@ -910,6 +910,7 @@ def emit_simple_content_delta(
|
||||
|
||||
def emit_simple_content_done(
|
||||
state: SimpleStreamingState,
|
||||
status: Literal["completed", "incomplete"] = "completed",
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
part = ResponseOutputText(
|
||||
type="output_text",
|
||||
@@ -944,7 +945,7 @@ def emit_simple_content_done(
|
||||
type="message",
|
||||
role="assistant",
|
||||
content=[part] if state.accumulated_text else [],
|
||||
status="completed",
|
||||
status=status,
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -1002,6 +1003,7 @@ def emit_simple_reasoning_delta(
|
||||
|
||||
def emit_simple_reasoning_done(
|
||||
state: SimpleStreamingState,
|
||||
status: Literal["completed", "incomplete"] = "completed",
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
part = ResponseReasoningTextContent(
|
||||
text=state.accumulated_text,
|
||||
@@ -1031,7 +1033,7 @@ def emit_simple_reasoning_done(
|
||||
item=ResponseReasoningItem(
|
||||
type="reasoning",
|
||||
content=[part],
|
||||
status="completed",
|
||||
status=status,
|
||||
id=state.current_item_id,
|
||||
summary=[],
|
||||
),
|
||||
@@ -1134,6 +1136,7 @@ def emit_simple_tool_call_delta(
|
||||
|
||||
def emit_simple_tool_call_done(
|
||||
state: SimpleStreamingState,
|
||||
status: Literal["completed", "incomplete"] = "completed",
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
events: list[StreamingResponsesResponse]
|
||||
item: ResponseOutputItem
|
||||
@@ -1172,7 +1175,7 @@ def emit_simple_tool_call_done(
|
||||
name=state.tool_call_name,
|
||||
namespace=state.tool_call_namespace,
|
||||
arguments=state.accumulated_text,
|
||||
status="completed",
|
||||
status=status,
|
||||
id=state.current_item_id,
|
||||
call_id=state.tool_call_id,
|
||||
)
|
||||
@@ -1312,12 +1315,14 @@ class SimpleStreamingEventProcessor:
|
||||
and self.state.tool_call_index != tool_call.index
|
||||
)
|
||||
|
||||
def close_current(self) -> list[StreamingResponsesResponse]:
|
||||
def close_current(
|
||||
self, status: Literal["completed", "incomplete"] = "completed"
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
"""Close the current state and emit its 'done' event sequence."""
|
||||
handlers = self._STATE_HANDLERS.get(self.state.current_state)
|
||||
if handlers is None:
|
||||
return []
|
||||
events = handlers.done_fn(self.state)
|
||||
events = handlers.done_fn(self.state, status)
|
||||
self.output_items.extend(
|
||||
event.item
|
||||
for event in events
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
from typing import Any, Literal
|
||||
|
||||
import partial_json_parser
|
||||
from openai.types.chat import (
|
||||
@@ -100,16 +100,30 @@ def _item_dict(item: ResponseInputOutputItem) -> dict[str, Any]:
|
||||
return item if isinstance(item, dict) else item.model_dump()
|
||||
|
||||
|
||||
def response_item_status(
|
||||
finish_reason: str | None,
|
||||
) -> Literal["completed", "incomplete"]:
|
||||
"""Status of the output item that ended a generation with ``finish_reason``."""
|
||||
return "incomplete" if finish_reason == "length" else "completed"
|
||||
|
||||
|
||||
def build_response_output_items(
|
||||
reasoning: str | None,
|
||||
content: str | None,
|
||||
tool_calls: list[FunctionCall] | None,
|
||||
logprobs: list[Logprob] | None = None,
|
||||
tools: list[Tool] | None = None,
|
||||
finish_reason: str | None = None,
|
||||
) -> list[ResponseOutputItem]:
|
||||
"""Build the output items of one generation.
|
||||
|
||||
Only the last item can have been cut short, so ``finish_reason`` decides
|
||||
that item's status and every earlier item is ``completed``.
|
||||
"""
|
||||
outputs: list[ResponseOutputItem] = []
|
||||
tool_call_name_map = build_responses_tool_call_name_map(tools)
|
||||
custom_names = custom_tool_names(tools)
|
||||
last_item_status = response_item_status(finish_reason)
|
||||
|
||||
if reasoning:
|
||||
outputs.append(
|
||||
@@ -137,13 +151,14 @@ def build_response_output_items(
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
status="completed",
|
||||
status="completed" if tool_calls else last_item_status,
|
||||
type="message",
|
||||
)
|
||||
)
|
||||
|
||||
if tool_calls:
|
||||
for idx, tool_call in enumerate(tool_calls):
|
||||
is_last = idx == len(tool_calls) - 1
|
||||
call_id = tool_call.id or make_tool_call_id(
|
||||
func_name=tool_call.name, idx=idx
|
||||
)
|
||||
@@ -166,7 +181,7 @@ def build_response_output_items(
|
||||
id=f"fc_{random_uuid()}",
|
||||
call_id=call_id,
|
||||
type="function_call",
|
||||
status="completed",
|
||||
status=last_item_status if is_last else "completed",
|
||||
name=call_name.name,
|
||||
namespace=call_name.namespace,
|
||||
arguments=tool_call.arguments,
|
||||
|
||||
@@ -4,6 +4,7 @@ from typing import TypeVar
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage, FunctionCall
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponseChoice,
|
||||
ChatCompletionResponseStreamChoice,
|
||||
@@ -41,3 +42,24 @@ def maybe_filter_parallel_tool_calls(
|
||||
tool_call for tool_call in delta.tool_calls if tool_call.index == 0
|
||||
]
|
||||
return item
|
||||
|
||||
|
||||
def resolve_finish_reason(
|
||||
engine_finish_reason: str | None,
|
||||
request: ChatCompletionRequest | None,
|
||||
tool_calls_made: bool,
|
||||
) -> str:
|
||||
"""Map the engine finish reason to the OpenAI chat ``finish_reason``.
|
||||
|
||||
``tool_calls`` is reported only when the choice actually carries tool
|
||||
calls, the engine stopped on its own (a ``length`` cut stays visible so
|
||||
clients can tell a truncated call from a complete one) and the caller did
|
||||
not force a single named tool, for which OpenAI reports ``stop``.
|
||||
"""
|
||||
finish_reason = engine_finish_reason or "stop"
|
||||
forced_named_tool = request is not None and isinstance(
|
||||
request.tool_choice, ChatCompletionNamedToolChoiceParam
|
||||
)
|
||||
if tool_calls_made and finish_reason == "stop" and not forced_named_tool:
|
||||
return "tool_calls"
|
||||
return finish_reason
|
||||
|
||||
Reference in New Issue
Block a user