[Bugfix][Responses API] Reuse streamed item ids in final harmony response (#59859)
Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -656,6 +656,11 @@ class HarmonyContext(ConversationContext):
|
||||
|
||||
self.last_append_segments: list[Segment] = []
|
||||
self.last_append_flush_status: bool = False
|
||||
# Each completed message with the items sent for it in
|
||||
# output_item.done events, so the final response can reuse their ids.
|
||||
self.streamed_items_by_message: list[
|
||||
tuple[Message, list[ResponseOutputItem]]
|
||||
] = []
|
||||
|
||||
# Turn tracking - replaces multiple individual tracking variables
|
||||
self.current_turn_metrics = TurnMetrics()
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import Any, Final, cast
|
||||
from fastapi import Request
|
||||
from openai.types.responses import (
|
||||
ResponseOutputItem,
|
||||
ResponseOutputItemDoneEvent,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputText,
|
||||
ResponseStatus,
|
||||
@@ -68,6 +69,7 @@ from vllm.entrypoints.openai.responses.utils import (
|
||||
build_response_output_items,
|
||||
extract_function_tool_names,
|
||||
extract_tool_types,
|
||||
reuse_streamed_item_ids,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
@@ -781,17 +783,21 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
assert isinstance(context, HarmonyContext)
|
||||
output = []
|
||||
harmony_msgs = context.messages[context.num_init_messages :]
|
||||
if harmony_msgs:
|
||||
fn_names = context.function_tool_names
|
||||
for msg in harmony_msgs[:-1]:
|
||||
output.extend(harmony_to_response_output(msg, fn_names))
|
||||
output.extend(
|
||||
harmony_to_response_output(
|
||||
harmony_msgs[-1],
|
||||
fn_names,
|
||||
incomplete=context.last_append_flush_status,
|
||||
)
|
||||
# Streamed messages are a subsequence of harmony_msgs, in order.
|
||||
streamed = iter(context.streamed_items_by_message)
|
||||
next_streamed = next(streamed, None)
|
||||
for i, msg in enumerate(harmony_msgs):
|
||||
items = harmony_to_response_output(
|
||||
msg,
|
||||
context.function_tool_names,
|
||||
incomplete=(
|
||||
i == len(harmony_msgs) - 1 and context.last_append_flush_status
|
||||
),
|
||||
)
|
||||
if next_streamed is not None and next_streamed[0] is msg:
|
||||
reuse_streamed_item_ids(items, next_streamed[1])
|
||||
next_streamed = next(streamed, None)
|
||||
output.extend(items)
|
||||
|
||||
if request.enable_response_messages:
|
||||
input_messages = context.messages[: context.num_init_messages]
|
||||
@@ -1291,16 +1297,24 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
|
||||
elif completed_message := segment.completed_message:
|
||||
done_items: list[ResponseOutputItem] = []
|
||||
# TODO: Fix browser emitted as MCP calls
|
||||
for event in emit_previous_item_done_events(
|
||||
completed_message, state, ctx.function_tool_names
|
||||
):
|
||||
if isinstance(event, ResponseOutputItemDoneEvent):
|
||||
done_items.append(event.item)
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
|
||||
for event in emit_tool_action_events(
|
||||
completed_message, state, self.tool_server
|
||||
):
|
||||
if isinstance(event, ResponseOutputItemDoneEvent):
|
||||
done_items.append(event.item)
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
ctx.streamed_items_by_message.append(
|
||||
(completed_message, done_items)
|
||||
)
|
||||
state.reset_for_new_item()
|
||||
|
||||
async def responses_stream_generator(
|
||||
|
||||
@@ -202,7 +202,7 @@ def emit_reasoning_delta_events(
|
||||
events: list[StreamingResponsesResponse] = []
|
||||
if not state.sent_output_item_added:
|
||||
state.sent_output_item_added = True
|
||||
state.current_item_id = f"msg_{random_uuid()}"
|
||||
state.current_item_id = f"rs_{random_uuid()}"
|
||||
events.append(
|
||||
ResponseOutputItemAddedEvent(
|
||||
type="response.output_item.added",
|
||||
|
||||
@@ -112,6 +112,28 @@ def build_response_output_items(
|
||||
return outputs
|
||||
|
||||
|
||||
def reuse_streamed_item_ids(
|
||||
items: list[ResponseOutputItem],
|
||||
streamed_items: list[ResponseOutputItem],
|
||||
) -> None:
|
||||
"""Give rebuilt output items the ids already streamed for the same message.
|
||||
|
||||
Items are paired by type in order; an item without a streamed counterpart
|
||||
keeps its own id.
|
||||
"""
|
||||
remaining = list(streamed_items)
|
||||
for item in items:
|
||||
for i, streamed in enumerate(remaining):
|
||||
if streamed.type != item.type:
|
||||
continue
|
||||
item.id = streamed.id
|
||||
if isinstance(item, ResponseFunctionToolCall):
|
||||
assert isinstance(streamed, ResponseFunctionToolCall)
|
||||
item.call_id = streamed.call_id
|
||||
del remaining[i]
|
||||
break
|
||||
|
||||
|
||||
def should_continue_final_message(
|
||||
request_input: str | list[ResponseInputOutputItem],
|
||||
) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user