[Bugfix][Responses API] Reuse streamed item ids in final harmony response (#59859)
Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -12,6 +12,7 @@ from typing import Any, Final, cast
|
||||
from fastapi import Request
|
||||
from openai.types.responses import (
|
||||
ResponseOutputItem,
|
||||
ResponseOutputItemDoneEvent,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputText,
|
||||
ResponseStatus,
|
||||
@@ -68,6 +69,7 @@ from vllm.entrypoints.openai.responses.utils import (
|
||||
build_response_output_items,
|
||||
extract_function_tool_names,
|
||||
extract_tool_types,
|
||||
reuse_streamed_item_ids,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
@@ -781,17 +783,21 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
assert isinstance(context, HarmonyContext)
|
||||
output = []
|
||||
harmony_msgs = context.messages[context.num_init_messages :]
|
||||
if harmony_msgs:
|
||||
fn_names = context.function_tool_names
|
||||
for msg in harmony_msgs[:-1]:
|
||||
output.extend(harmony_to_response_output(msg, fn_names))
|
||||
output.extend(
|
||||
harmony_to_response_output(
|
||||
harmony_msgs[-1],
|
||||
fn_names,
|
||||
incomplete=context.last_append_flush_status,
|
||||
)
|
||||
# Streamed messages are a subsequence of harmony_msgs, in order.
|
||||
streamed = iter(context.streamed_items_by_message)
|
||||
next_streamed = next(streamed, None)
|
||||
for i, msg in enumerate(harmony_msgs):
|
||||
items = harmony_to_response_output(
|
||||
msg,
|
||||
context.function_tool_names,
|
||||
incomplete=(
|
||||
i == len(harmony_msgs) - 1 and context.last_append_flush_status
|
||||
),
|
||||
)
|
||||
if next_streamed is not None and next_streamed[0] is msg:
|
||||
reuse_streamed_item_ids(items, next_streamed[1])
|
||||
next_streamed = next(streamed, None)
|
||||
output.extend(items)
|
||||
|
||||
if request.enable_response_messages:
|
||||
input_messages = context.messages[: context.num_init_messages]
|
||||
@@ -1291,16 +1297,24 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
|
||||
elif completed_message := segment.completed_message:
|
||||
done_items: list[ResponseOutputItem] = []
|
||||
# TODO: Fix browser emitted as MCP calls
|
||||
for event in emit_previous_item_done_events(
|
||||
completed_message, state, ctx.function_tool_names
|
||||
):
|
||||
if isinstance(event, ResponseOutputItemDoneEvent):
|
||||
done_items.append(event.item)
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
|
||||
for event in emit_tool_action_events(
|
||||
completed_message, state, self.tool_server
|
||||
):
|
||||
if isinstance(event, ResponseOutputItemDoneEvent):
|
||||
done_items.append(event.item)
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
ctx.streamed_items_by_message.append(
|
||||
(completed_message, done_items)
|
||||
)
|
||||
state.reset_for_new_item()
|
||||
|
||||
async def responses_stream_generator(
|
||||
|
||||
Reference in New Issue
Block a user