# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Streaming SSE event builders for the Responses API. Pure functions that translate streaming state + delta data into OpenAI Response API SSE events. Used by the streaming event processors in serving.py. The file is organized as: 1. StreamingState dataclass + utility helpers 2. Shared leaf helpers — delta events (take plain strings, no context) 3. Shared leaf helpers — done events (take plain strings, no context) 4. Harmony-specific dispatchers (route ctx/previous_item → leaf helpers) 5. Harmony-specific tool lifecycle helpers """ import json from collections.abc import Callable from dataclasses import dataclass, field from enum import Enum, auto from typing import Any, ClassVar, Final, Literal, NamedTuple from openai.types.responses import ( ResponseCodeInterpreterCallCodeDeltaEvent, ResponseCodeInterpreterCallCodeDoneEvent, ResponseCodeInterpreterCallCompletedEvent, ResponseCodeInterpreterCallInProgressEvent, ResponseCodeInterpreterCallInterpretingEvent, ResponseCodeInterpreterToolCallParam, ResponseContentPartAddedEvent, ResponseContentPartDoneEvent, ResponseCustomToolCall, ResponseCustomToolCallInputDeltaEvent, ResponseCustomToolCallInputDoneEvent, ResponseFunctionCallArgumentsDeltaEvent, ResponseFunctionCallArgumentsDoneEvent, ResponseFunctionToolCall, ResponseFunctionToolCallItem, ResponseFunctionWebSearch, ResponseMcpCallArgumentsDeltaEvent, ResponseMcpCallArgumentsDoneEvent, ResponseMcpCallCompletedEvent, ResponseMcpCallInProgressEvent, ResponseOutputItem, ResponseOutputItemAddedEvent, ResponseOutputItemDoneEvent, ResponseOutputMessage, ResponseOutputText, ResponseReasoningItem, ResponseReasoningTextDeltaEvent, ResponseReasoningTextDoneEvent, ResponseTextDeltaEvent, ResponseTextDoneEvent, ResponseWebSearchCallCompletedEvent, ResponseWebSearchCallInProgressEvent, ResponseWebSearchCallSearchingEvent, response_function_web_search, response_output_text, response_text_delta_event, ) from openai.types.responses.response_output_item import McpCall from openai.types.responses.response_reasoning_item import ( Content as ResponseReasoningTextContent, ) from openai.types.responses.tool import Tool from openai_harmony import Message as HarmonyMessage from vllm.entrypoints.generate.base.protocol import DeltaMessage, DeltaToolCall from vllm.entrypoints.mcp.tool_server import ToolServer from vllm.entrypoints.openai.parser.harmony_utils import ( extract_function_from_recipient, is_function_recipient, ) from vllm.entrypoints.openai.responses.protocol import ( StreamingResponsesResponse, ) from vllm.entrypoints.openai.responses.utils import ( build_responses_tool_call_name_map, custom_tool_names, decode_custom_tool_input, decode_custom_tool_input_prefix, resolve_responses_tool_call_name, ) from vllm.outputs import CompletionOutput from vllm.parser.harmony import Segment from vllm.renderers.chat_utils import make_tool_call_id from vllm.utils import random_uuid TOOL_NAME_TO_MCP_SERVER_LABEL: Final[dict[str, str]] = { "python": "code_interpreter", "container": "container", "browser": "web_search_preview", } def _resolve_mcp_name_label(recipient: str) -> tuple[str, str]: """Resolve MCP tool name and server label from a recipient string. - ``mcp.*`` recipients: strip prefix, use the bare name as both name and server_label. - Everything else: use the recipient as the name and look up the server_label in TOOL_NAME_TO_MCP_SERVER_LABEL. """ if recipient.startswith("mcp."): name = recipient[len("mcp.") :] return name, name return recipient, TOOL_NAME_TO_MCP_SERVER_LABEL.get(recipient, recipient) @dataclass class StreamingState: """Mutable state for streaming event processing.""" current_content_index: int = -1 current_output_index: int = 0 current_item_id: str = "" current_call_id: str = "" sent_output_item_added: bool = False is_first_function_call_delta: bool = False def reset_for_new_item(self) -> None: """Reset state when expecting a new output item.""" self.current_output_index += 1 self.current_content_index = -1 self.sent_output_item_added = False self.is_first_function_call_delta = False self.current_call_id = "" def is_mcp_tool_by_namespace( recipient: str | None, allowed_function_tool_names: frozenset[str] | None = None, ) -> bool: """Determine if a tool call is an MCP tool based on recipient prefix. Inverse of :func:`is_function_recipient` — everything that is not a function call is an MCP tool. """ if recipient is None: return False return not is_function_recipient(recipient, allowed_function_tool_names) # ===================================================================== # Shared leaf helpers — delta events # ===================================================================== def emit_text_delta_events( delta: str, state: StreamingState, ) -> list[StreamingResponsesResponse]: """Emit events for text content delta streaming.""" events: list[StreamingResponsesResponse] = [] if not state.sent_output_item_added: state.sent_output_item_added = True state.current_item_id = f"msg_{random_uuid()}" events.append( ResponseOutputItemAddedEvent( type="response.output_item.added", sequence_number=-1, output_index=state.current_output_index, item=ResponseOutputMessage( id=state.current_item_id, type="message", role="assistant", content=[], status="in_progress", ), ) ) state.current_content_index += 1 events.append( ResponseContentPartAddedEvent( type="response.content_part.added", sequence_number=-1, output_index=state.current_output_index, item_id=state.current_item_id, content_index=state.current_content_index, part=ResponseOutputText( type="output_text", text="", annotations=[], logprobs=[], ), ) ) events.append( ResponseTextDeltaEvent( type="response.output_text.delta", sequence_number=-1, content_index=state.current_content_index, output_index=state.current_output_index, item_id=state.current_item_id, delta=delta, # TODO, use logprobs from ctx.last_request_output logprobs=[], ) ) return events def emit_reasoning_delta_events( delta: str, state: StreamingState, ) -> list[StreamingResponsesResponse]: """Emit events for reasoning text delta streaming.""" events: list[StreamingResponsesResponse] = [] if not state.sent_output_item_added: state.sent_output_item_added = True state.current_item_id = f"rs_{random_uuid()}" events.append( ResponseOutputItemAddedEvent( type="response.output_item.added", sequence_number=-1, output_index=state.current_output_index, item=ResponseReasoningItem( type="reasoning", id=state.current_item_id, summary=[], status="in_progress", ), ) ) state.current_content_index += 1 events.append( ResponseContentPartAddedEvent( type="response.content_part.added", sequence_number=-1, output_index=state.current_output_index, item_id=state.current_item_id, content_index=state.current_content_index, part={"text": "", "type": "reasoning_text"}, ) ) events.append( ResponseReasoningTextDeltaEvent( type="response.reasoning_text.delta", item_id=state.current_item_id, output_index=state.current_output_index, content_index=state.current_content_index, delta=delta, sequence_number=-1, ) ) return events def emit_function_call_delta_events( delta: str, function_name: str, state: StreamingState, ) -> list[StreamingResponsesResponse]: """Emit events for function call argument deltas.""" events: list[StreamingResponsesResponse] = [] if state.is_first_function_call_delta is False: state.is_first_function_call_delta = True state.current_item_id = f"fc_{random_uuid()}" state.current_call_id = f"call_{random_uuid()}" tool_call_item = ResponseFunctionToolCall( name=function_name, type="function_call", id=state.current_item_id, call_id=state.current_call_id, arguments="", status="in_progress", ) events.append( ResponseOutputItemAddedEvent( type="response.output_item.added", sequence_number=-1, output_index=state.current_output_index, item=tool_call_item, ) ) # Always emit the delta (including on first call) events.append( ResponseFunctionCallArgumentsDeltaEvent( item_id=state.current_item_id, delta=delta, output_index=state.current_output_index, sequence_number=-1, type="response.function_call_arguments.delta", ) ) return events def emit_mcp_delta_events( delta: str, state: StreamingState, recipient: str, ) -> list[StreamingResponsesResponse]: """Emit events for MCP tool delta streaming.""" name, server_label = _resolve_mcp_name_label(recipient) events: list[StreamingResponsesResponse] = [] if not state.sent_output_item_added: state.sent_output_item_added = True state.current_item_id = f"mcp_{random_uuid()}" events.append( ResponseOutputItemAddedEvent( type="response.output_item.added", sequence_number=-1, output_index=state.current_output_index, item=McpCall( type="mcp_call", id=state.current_item_id, name=name, arguments="", server_label=server_label, status="in_progress", ), ) ) events.append( ResponseMcpCallInProgressEvent( type="response.mcp_call.in_progress", sequence_number=-1, output_index=state.current_output_index, item_id=state.current_item_id, ) ) events.append( ResponseMcpCallArgumentsDeltaEvent( type="response.mcp_call_arguments.delta", sequence_number=-1, output_index=state.current_output_index, item_id=state.current_item_id, delta=delta, ) ) return events def emit_code_interpreter_delta_events( delta: str, state: StreamingState, ) -> list[StreamingResponsesResponse]: """Emit events for code interpreter delta streaming.""" events: list[StreamingResponsesResponse] = [] if not state.sent_output_item_added: state.sent_output_item_added = True state.current_item_id = f"tool_{random_uuid()}" events.append( ResponseOutputItemAddedEvent( type="response.output_item.added", sequence_number=-1, output_index=state.current_output_index, item=ResponseCodeInterpreterToolCallParam( type="code_interpreter_call", id=state.current_item_id, code=None, container_id="auto", outputs=None, status="in_progress", ), ) ) events.append( ResponseCodeInterpreterCallInProgressEvent( type="response.code_interpreter_call.in_progress", sequence_number=-1, output_index=state.current_output_index, item_id=state.current_item_id, ) ) events.append( ResponseCodeInterpreterCallCodeDeltaEvent( type="response.code_interpreter_call_code.delta", sequence_number=-1, output_index=state.current_output_index, item_id=state.current_item_id, delta=delta, ) ) return events # ===================================================================== # Shared leaf helpers — done events # ===================================================================== def emit_text_output_done_events( text: str, state: StreamingState, ) -> list[StreamingResponsesResponse]: """Emit events when a final text output item completes.""" text_content = ResponseOutputText( type="output_text", text=text, annotations=[], ) events: list[StreamingResponsesResponse] = [] events.append( ResponseTextDoneEvent( type="response.output_text.done", sequence_number=-1, output_index=state.current_output_index, content_index=state.current_content_index, text=text, logprobs=[], item_id=state.current_item_id, ) ) events.append( ResponseContentPartDoneEvent( type="response.content_part.done", sequence_number=-1, item_id=state.current_item_id, output_index=state.current_output_index, content_index=state.current_content_index, part=text_content, ) ) events.append( ResponseOutputItemDoneEvent( type="response.output_item.done", sequence_number=-1, output_index=state.current_output_index, item=ResponseOutputMessage( id=state.current_item_id, type="message", role="assistant", content=[text_content], status="completed", ), ) ) return events def emit_reasoning_done_events( text: str, state: StreamingState, ) -> list[StreamingResponsesResponse]: """Emit events when a reasoning (analysis) item completes.""" content = ResponseReasoningTextContent( text=text, type="reasoning_text", ) reasoning_item = ResponseReasoningItem( type="reasoning", content=[content], status="completed", id=state.current_item_id, summary=[], ) events: list[StreamingResponsesResponse] = [] events.append( ResponseReasoningTextDoneEvent( type="response.reasoning_text.done", item_id=state.current_item_id, sequence_number=-1, output_index=state.current_output_index, content_index=state.current_content_index, text=text, ) ) events.append( ResponseContentPartDoneEvent( type="response.content_part.done", sequence_number=-1, item_id=state.current_item_id, output_index=state.current_output_index, content_index=state.current_content_index, part=content.model_dump(), ) ) events.append( ResponseOutputItemDoneEvent( type="response.output_item.done", sequence_number=-1, output_index=state.current_output_index, item=reasoning_item, ) ) return events def emit_function_call_done_events( function_name: str, arguments: str, state: StreamingState, ) -> list[StreamingResponsesResponse]: """Emit events when a function call completes.""" events: list[StreamingResponsesResponse] = [] events.append( ResponseFunctionCallArgumentsDoneEvent( type="response.function_call_arguments.done", arguments=arguments, name=function_name, item_id=state.current_item_id, output_index=state.current_output_index, sequence_number=-1, ) ) function_call_item = ResponseFunctionToolCall( type="function_call", arguments=arguments, name=function_name, id=state.current_item_id, output_index=state.current_output_index, sequence_number=-1, call_id=state.current_call_id, status="completed", ) events.append( ResponseOutputItemDoneEvent( type="response.output_item.done", sequence_number=-1, output_index=state.current_output_index, item=function_call_item, ) ) return events def emit_mcp_completion_events( recipient: str, arguments: str, state: StreamingState, ) -> list[StreamingResponsesResponse]: """Emit events when an MCP tool call completes.""" name, server_label = _resolve_mcp_name_label(recipient) events: list[StreamingResponsesResponse] = [] events.append( ResponseMcpCallArgumentsDoneEvent( type="response.mcp_call_arguments.done", arguments=arguments, name=name, item_id=state.current_item_id, output_index=state.current_output_index, sequence_number=-1, ) ) events.append( ResponseMcpCallCompletedEvent( type="response.mcp_call.completed", sequence_number=-1, output_index=state.current_output_index, item_id=state.current_item_id, ) ) events.append( ResponseOutputItemDoneEvent( type="response.output_item.done", sequence_number=-1, output_index=state.current_output_index, item=McpCall( type="mcp_call", arguments=arguments, name=name, id=state.current_item_id, server_label=server_label, status="completed", ), ) ) return events # ===================================================================== # Harmony-specific dispatchers # ===================================================================== def emit_content_delta_events( segment: Segment, state: StreamingState, function_tool_names: frozenset[str] | None = None, ) -> list[StreamingResponsesResponse]: """Emit events for content delta streaming based on channel type. This is a Harmony-specific dispatcher that extracts values from the latest append segment and delegates to shared leaf helpers. """ delta = segment.delta if not delta: return [] channel = segment.channel recipient = segment.recipient if channel in ("final", "commentary") and recipient is None: # Preambles (commentary with no recipient) and final messages # are both user-visible text. return emit_text_delta_events(delta, state) elif channel == "analysis" and recipient is None: return emit_reasoning_delta_events(delta, state) elif recipient is not None: fn_names = function_tool_names if is_function_recipient(recipient, fn_names): function_name = extract_function_from_recipient(recipient) return emit_function_call_delta_events(delta, function_name, state) elif recipient == "python": return emit_code_interpreter_delta_events(delta, state) elif recipient.startswith("mcp.") or is_mcp_tool_by_namespace( recipient, fn_names ): return emit_mcp_delta_events(delta, state, recipient) return [] def emit_previous_item_done_events( previous_item: HarmonyMessage, state: StreamingState, function_tool_names: frozenset[str] | None = None, ) -> list[StreamingResponsesResponse]: """Emit done events for the previous item when expecting a new start. This is a Harmony-specific dispatcher that extracts values from the Harmony parser's message object and delegates to shared leaf helpers. """ if not state.sent_output_item_added and not state.is_first_function_call_delta: # Suppress done events for items had no delta and thus had no # added/in-progress lifecycle events. This is a bug. # TODO: Ensure added/in-progress events are emitted for zero-delta items. return [] text = previous_item.content[0].text if previous_item.recipient is not None: # Deal with tool call if is_function_recipient(previous_item.recipient, function_tool_names): function_name = extract_function_from_recipient(previous_item.recipient) return emit_function_call_done_events(function_name, text, state) elif previous_item.recipient == "python": return emit_code_interpreter_completion_events(previous_item, state) elif ( is_mcp_tool_by_namespace(previous_item.recipient, function_tool_names) and state.current_item_id is not None and state.current_item_id.startswith("mcp_") ): return emit_mcp_completion_events(previous_item.recipient, text, state) elif previous_item.channel == "analysis": return emit_reasoning_done_events(text, state) elif previous_item.channel in ("commentary", "final"): # Preambles (commentary with no recipient) and final messages # are both user-visible text. return emit_text_output_done_events(text, state) return [] # ===================================================================== # Harmony-specific tool lifecycle helpers # ===================================================================== def emit_browser_tool_events( previous_item: HarmonyMessage, state: StreamingState, ) -> list[StreamingResponsesResponse]: """Emit events for browser tool calls (web search).""" function_name = previous_item.recipient[len("browser.") :] parsed_args = json.loads(previous_item.content[0].text) action = None if function_name == "search": action = response_function_web_search.ActionSearch( type="search", query=parsed_args["query"], ) elif function_name == "open": action = response_function_web_search.ActionOpenPage( type="open_page", # TODO: translate to url url=f"cursor:{parsed_args.get('cursor', '')}", ) elif function_name == "find": action = response_function_web_search.ActionFind( type="find_in_page", pattern=parsed_args["pattern"], # TODO: translate to url url=f"cursor:{parsed_args.get('cursor', '')}", ) else: raise ValueError(f"Unknown function name: {function_name}") state.current_item_id = f"tool_{random_uuid()}" events: list[StreamingResponsesResponse] = [] events.append( ResponseOutputItemAddedEvent( type="response.output_item.added", sequence_number=-1, output_index=state.current_output_index, item=response_function_web_search.ResponseFunctionWebSearch( # TODO: generate a unique id for web search call type="web_search_call", id=state.current_item_id, action=action, status="in_progress", ), ) ) events.append( ResponseWebSearchCallInProgressEvent( type="response.web_search_call.in_progress", sequence_number=-1, output_index=state.current_output_index, item_id=state.current_item_id, ) ) events.append( ResponseWebSearchCallSearchingEvent( type="response.web_search_call.searching", sequence_number=-1, output_index=state.current_output_index, item_id=state.current_item_id, ) ) # enqueue events.append( ResponseWebSearchCallCompletedEvent( type="response.web_search_call.completed", sequence_number=-1, output_index=state.current_output_index, item_id=state.current_item_id, ) ) events.append( ResponseOutputItemDoneEvent( type="response.output_item.done", sequence_number=-1, output_index=state.current_output_index, item=ResponseFunctionWebSearch( type="web_search_call", id=state.current_item_id, action=action, status="completed", ), ) ) return events def emit_code_interpreter_completion_events( previous_item: HarmonyMessage, state: StreamingState, ) -> list[StreamingResponsesResponse]: """Emit events when code interpreter completes.""" events: list[StreamingResponsesResponse] = [] events.append( ResponseCodeInterpreterCallCodeDoneEvent( type="response.code_interpreter_call_code.done", sequence_number=-1, output_index=state.current_output_index, item_id=state.current_item_id, code=previous_item.content[0].text, ) ) events.append( ResponseCodeInterpreterCallInterpretingEvent( type="response.code_interpreter_call.interpreting", sequence_number=-1, output_index=state.current_output_index, item_id=state.current_item_id, ) ) events.append( ResponseCodeInterpreterCallCompletedEvent( type="response.code_interpreter_call.completed", sequence_number=-1, output_index=state.current_output_index, item_id=state.current_item_id, ) ) events.append( ResponseOutputItemDoneEvent( type="response.output_item.done", sequence_number=-1, output_index=state.current_output_index, item=ResponseCodeInterpreterToolCallParam( type="code_interpreter_call", id=state.current_item_id, code=previous_item.content[0].text, container_id="auto", outputs=[], status="completed", ), ) ) return events def emit_tool_action_events( previous_item: HarmonyMessage, state: StreamingState, tool_server: ToolServer | None, ) -> list[StreamingResponsesResponse]: """Emit events for a completed assistant action turn.""" # Handle browser tool if ( previous_item.author.role == "assistant" and previous_item.recipient is not None and previous_item.recipient.startswith("browser.") and tool_server is not None and tool_server.has_tool("browser") ): return emit_browser_tool_events(previous_item, state) return [] # ===================================================================== # Simple streaming helpers # ===================================================================== class _StateType(Enum): NONE = auto() CONTENT = auto() REASONING = auto() TOOL_CALL = auto() @dataclass class SimpleStreamingState: output_index: int = 0 current_item_id: str = "" content_index: int = 0 accumulated_text: str = "" accumulated_logprobs: list[response_output_text.Logprob] = field( default_factory=list ) tool_call_id: str = "" tool_call_name: str = "" tool_call_namespace: str | None = None tool_call_index: int | None = None tool_call_is_custom: bool = False tool_call_payload: str = "" current_state: _StateType = field(default_factory=lambda: _StateType.NONE) def emit_simple_content_open( state: SimpleStreamingState, ) -> list[StreamingResponsesResponse]: state.current_state = _StateType.CONTENT state.current_item_id = f"msg_{random_uuid()}" state.content_index = 0 state.accumulated_text = "" state.accumulated_logprobs = [] return [ ResponseOutputItemAddedEvent( type="response.output_item.added", sequence_number=-1, output_index=state.output_index, item=ResponseOutputMessage( id=state.current_item_id, type="message", role="assistant", content=[], status="in_progress", ), ), ResponseContentPartAddedEvent( type="response.content_part.added", sequence_number=-1, output_index=state.output_index, item_id=state.current_item_id, content_index=state.content_index, part=ResponseOutputText( type="output_text", text="", annotations=[], logprobs=[], ), ), ] def _to_stream_logprobs( logprobs: list[response_output_text.Logprob], ) -> list[response_text_delta_event.Logprob]: return [ response_text_delta_event.Logprob( token=lp.token, logprob=lp.logprob, top_logprobs=[ response_text_delta_event.LogprobTopLogprob( token=tl.token, logprob=tl.logprob ) for tl in lp.top_logprobs ], ) for lp in logprobs ] def emit_simple_content_delta( state: SimpleStreamingState, delta: str, logprobs: list[response_output_text.Logprob] | None = None, ) -> list[StreamingResponsesResponse]: state.accumulated_text += delta if logprobs: state.accumulated_logprobs.extend(logprobs) return [ ResponseTextDeltaEvent( type="response.output_text.delta", sequence_number=-1, content_index=state.content_index, output_index=state.output_index, item_id=state.current_item_id, delta=delta, logprobs=_to_stream_logprobs(logprobs or []), ) ] def emit_simple_content_done( state: SimpleStreamingState, status: Literal["completed", "incomplete"] = "completed", ) -> list[StreamingResponsesResponse]: part = ResponseOutputText( type="output_text", text=state.accumulated_text, annotations=[], logprobs=state.accumulated_logprobs or None, ) events: list[StreamingResponsesResponse] = [ ResponseTextDoneEvent( type="response.output_text.done", sequence_number=-1, output_index=state.output_index, content_index=state.content_index, text=state.accumulated_text, logprobs=[], item_id=state.current_item_id, ), ResponseContentPartDoneEvent( type="response.content_part.done", sequence_number=-1, item_id=state.current_item_id, output_index=state.output_index, content_index=state.content_index, part=part, ), ResponseOutputItemDoneEvent( type="response.output_item.done", sequence_number=-1, output_index=state.output_index, item=ResponseOutputMessage( id=state.current_item_id, type="message", role="assistant", content=[part] if state.accumulated_text else [], status=status, ), ), ] state.output_index += 1 state.current_state = _StateType.NONE return events def emit_simple_reasoning_open( state: SimpleStreamingState, ) -> list[StreamingResponsesResponse]: state.current_state = _StateType.REASONING state.current_item_id = f"rs_{random_uuid()}" state.content_index = 0 state.accumulated_text = "" return [ ResponseOutputItemAddedEvent( type="response.output_item.added", sequence_number=-1, output_index=state.output_index, item=ResponseReasoningItem( type="reasoning", id=state.current_item_id, summary=[], status="in_progress", ), ), ResponseContentPartAddedEvent( type="response.content_part.added", sequence_number=-1, output_index=state.output_index, item_id=state.current_item_id, content_index=state.content_index, part={"text": "", "type": "reasoning_text"}, ), ] def emit_simple_reasoning_delta( state: SimpleStreamingState, delta: str, ) -> list[StreamingResponsesResponse]: state.accumulated_text += delta return [ ResponseReasoningTextDeltaEvent( type="response.reasoning_text.delta", item_id=state.current_item_id, sequence_number=-1, output_index=state.output_index, content_index=state.content_index, delta=delta, ) ] def emit_simple_reasoning_done( state: SimpleStreamingState, status: Literal["completed", "incomplete"] = "completed", ) -> list[StreamingResponsesResponse]: part = ResponseReasoningTextContent( text=state.accumulated_text, type="reasoning_text", ) events: list[StreamingResponsesResponse] = [ ResponseReasoningTextDoneEvent( type="response.reasoning_text.done", item_id=state.current_item_id, sequence_number=-1, output_index=state.output_index, content_index=state.content_index, text=state.accumulated_text, ), ResponseContentPartDoneEvent( type="response.content_part.done", sequence_number=-1, item_id=state.current_item_id, output_index=state.output_index, content_index=state.content_index, part=part.model_dump(), ), ResponseOutputItemDoneEvent( type="response.output_item.done", sequence_number=-1, output_index=state.output_index, item=ResponseReasoningItem( type="reasoning", content=[part], status=status, id=state.current_item_id, summary=[], ), ), ] state.output_index += 1 state.current_state = _StateType.NONE return events def emit_simple_tool_call_open( state: SimpleStreamingState, name: str, index: int | None, namespace: str | None = None, is_custom: bool = False, call_id: str | None = None, ) -> list[StreamingResponsesResponse]: state.current_state = _StateType.TOOL_CALL state.current_item_id = ( f"ctc_{random_uuid()}" if is_custom else f"fc_{random_uuid()}" ) state.tool_call_id = call_id or make_tool_call_id() state.tool_call_name = name state.tool_call_namespace = namespace state.tool_call_index = index state.tool_call_is_custom = is_custom state.tool_call_payload = "" state.accumulated_text = "" item: ResponseOutputItem if is_custom: item = ResponseCustomToolCall( type="custom_tool_call", id=state.current_item_id, call_id=state.tool_call_id, name=name, input="", ) else: item = ResponseFunctionToolCallItem( type="function_call", id=state.current_item_id, call_id=state.tool_call_id, name=name, namespace=namespace, arguments="", status="in_progress", ) return [ ResponseOutputItemAddedEvent( type="response.output_item.added", sequence_number=-1, output_index=state.output_index, item=item, ), ] def _emit_custom_tool_input_delta( state: SimpleStreamingState, payload: str, ) -> list[StreamingResponsesResponse]: streamed = state.tool_call_payload if payload == streamed: return [] state.tool_call_payload = payload # A payload that is not an extension of what was streamed cannot be # retracted, so the authoritative value is resent in full. delta = payload[len(streamed) :] if payload.startswith(streamed) else payload return [ ResponseCustomToolCallInputDeltaEvent( type="response.custom_tool_call_input.delta", sequence_number=-1, output_index=state.output_index, item_id=state.current_item_id, delta=delta, ) ] def emit_simple_tool_call_delta( state: SimpleStreamingState, delta: str, ) -> list[StreamingResponsesResponse]: state.accumulated_text += delta if state.tool_call_is_custom: return _emit_custom_tool_input_delta( state, decode_custom_tool_input_prefix(state.accumulated_text) ) return [ ResponseFunctionCallArgumentsDeltaEvent( type="response.function_call_arguments.delta", sequence_number=-1, output_index=state.output_index, item_id=state.current_item_id, delta=delta, ) ] def emit_simple_tool_call_done( state: SimpleStreamingState, status: Literal["completed", "incomplete"] = "completed", ) -> list[StreamingResponsesResponse]: events: list[StreamingResponsesResponse] item: ResponseOutputItem if state.tool_call_is_custom: payload = decode_custom_tool_input(state.accumulated_text) events = _emit_custom_tool_input_delta(state, payload) events.append( ResponseCustomToolCallInputDoneEvent( type="response.custom_tool_call_input.done", sequence_number=-1, output_index=state.output_index, item_id=state.current_item_id, input=payload, ) ) item = ResponseCustomToolCall( type="custom_tool_call", id=state.current_item_id, call_id=state.tool_call_id, name=state.tool_call_name, input=payload, ) else: events = [ ResponseFunctionCallArgumentsDoneEvent( type="response.function_call_arguments.done", sequence_number=-1, output_index=state.output_index, item_id=state.current_item_id, arguments=state.accumulated_text, name=state.tool_call_name, ) ] item = ResponseFunctionToolCall( type="function_call", name=state.tool_call_name, namespace=state.tool_call_namespace, arguments=state.accumulated_text, status=status, id=state.current_item_id, call_id=state.tool_call_id, ) events.append( ResponseOutputItemDoneEvent( type="response.output_item.done", sequence_number=-1, output_index=state.output_index, item=item, ) ) state.output_index += 1 state.tool_call_namespace = None state.tool_call_is_custom = False state.current_state = _StateType.NONE return events class _StateHandlers(NamedTuple): """Tuple for each state: open(start), delta(chunk), done(finish).""" open_fn: Callable[..., list[StreamingResponsesResponse]] delta_fn: Callable[..., list[StreamingResponsesResponse]] done_fn: Callable[..., list[StreamingResponsesResponse]] def split_delta(delta: DeltaMessage) -> list[DeltaMessage]: """Decompose a DeltaMessage with multiple fields into atomic deltas. The Responses API emits typed SSE events (one type per event), so a compound DeltaMessage must be split before entering the state machine. Order: reasoning -> content -> tool_calls (grouped by index). """ has_reasoning = delta.reasoning is not None has_content = delta.content is not None has_tools = bool(delta.tool_calls) parts = int(has_reasoning) + int(has_content) + int(has_tools) if parts <= 1 and ( not has_tools or len({tc.index for tc in delta.tool_calls if tc.index is not None}) <= 1 ): return [delta] deltas: list[DeltaMessage] = [] if has_reasoning: deltas.append(DeltaMessage(reasoning=delta.reasoning)) if has_content: deltas.append(DeltaMessage(content=delta.content)) if has_tools: groups: dict[int | None, list[DeltaToolCall]] = {} for tc in delta.tool_calls: groups.setdefault(tc.index, []).append(tc) for tcs in groups.values(): deltas.append(DeltaMessage(tool_calls=tcs)) return deltas or [delta] class SimpleStreamingEventProcessor: """State-machine processor for the simple (non-Harmony) streaming path. Core flow: 1. Resolve the target state from the delta_message (CONTENT / REASONING / TOOL_CALL). 2. If the target state differs from the current one, close_current() then open() the new state. 3. emit_delta() produces the incremental events for the state. State lifecycle: open() -> repeated emit_delta() -> close_current() """ _STATE_HANDLERS: ClassVar[dict[_StateType, _StateHandlers]] = { _StateType.CONTENT: _StateHandlers( emit_simple_content_open, emit_simple_content_delta, emit_simple_content_done, ), _StateType.REASONING: _StateHandlers( emit_simple_reasoning_open, emit_simple_reasoning_delta, emit_simple_reasoning_done, ), _StateType.TOOL_CALL: _StateHandlers( emit_simple_tool_call_open, emit_simple_tool_call_delta, emit_simple_tool_call_done, ), } def __init__( self, state: SimpleStreamingState | None = None, tools: list[Tool] | None = None, ) -> None: self.state = state or SimpleStreamingState() self.tool_call_name_map = build_responses_tool_call_name_map(tools) self.custom_tool_names = custom_tool_names(tools) self.output_items: list[ResponseOutputItem] = [] def resolve_target_state( self, delta_message: DeltaMessage ) -> tuple[_StateType, Any]: """Decide which state the next delta belongs to. Priority: TOOL_CALL > REASONING > CONTENT, fallback to NONE. For TOOL_CALL the first tool_call object is also returned so callers can detect a switch between consecutive tools. """ if ( delta_message.tool_calls and delta_message.tool_calls[0].function is not None ): return _StateType.TOOL_CALL, delta_message.tool_calls[0] if delta_message.reasoning is not None: return _StateType.REASONING, None if delta_message.content: return _StateType.CONTENT, None return _StateType.NONE, None def needs_transition(self, target_state: _StateType, tool_call: Any) -> bool: """Return True when we must close the current state and open a new one. Two cases trigger a transition: 1. The target state differs from the current state (e.g. CONTENT -> TOOL_CALL). 2. We are already in TOOL_CALL but the next tool_call has a different index (multiple consecutive tool calls). """ if self.state.current_state != target_state: return True return ( target_state == _StateType.TOOL_CALL and tool_call is not None and self.state.tool_call_index is not None and tool_call.index is not None and self.state.tool_call_index != tool_call.index ) def close_current( self, status: Literal["completed", "incomplete"] = "completed" ) -> list[StreamingResponsesResponse]: """Close the current state and emit its 'done' event sequence.""" handlers = self._STATE_HANDLERS.get(self.state.current_state) if handlers is None: return [] events = handlers.done_fn(self.state, status) self.output_items.extend( event.item for event in events if isinstance(event, ResponseOutputItemDoneEvent) ) return events def open( self, target_state: _StateType, tool_call: Any = None ) -> list[StreamingResponsesResponse]: """Open a new state and emit its 'added' / 'open' event sequence.""" handlers = self._STATE_HANDLERS[target_state] if target_state == _StateType.TOOL_CALL: assert tool_call is not None call_name = resolve_responses_tool_call_name( tool_call.function.name, tool_call_name_map=self.tool_call_name_map, ) return handlers.open_fn( self.state, call_name.name, tool_call.index, call_name.namespace, call_name.name in self.custom_tool_names, tool_call.id, ) return handlers.open_fn(self.state) def emit_delta( self, delta_message: DeltaMessage, output: CompletionOutput, get_logprobs: Callable[[CompletionOutput], list[response_output_text.Logprob]] | None = None, ) -> list[StreamingResponsesResponse]: """Emit incremental events for the current state from the delta.""" handlers = self._STATE_HANDLERS[self.state.current_state] if self.state.current_state == _StateType.TOOL_CALL: assert delta_message.tool_calls is not None combined_args = "" for tc in delta_message.tool_calls: if tc.function is not None and tc.function.arguments: combined_args += tc.function.arguments if combined_args: return handlers.delta_fn(self.state, combined_args) return [] elif self.state.current_state == _StateType.REASONING: assert delta_message.reasoning is not None return handlers.delta_fn(self.state, delta_message.reasoning) elif self.state.current_state == _StateType.CONTENT: assert delta_message.content is not None logprobs = get_logprobs(output) if get_logprobs else [] return handlers.delta_fn(self.state, delta_message.content, logprobs) return []