[Refactor][GPT-OSS] Harmony Responses API Refactor to use HarmonyParser (#47185)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-06-30 19:23:20 -04:00
committed by GitHub
parent 92c7fac640
commit b1190d03cc
9 changed files with 438 additions and 868 deletions
+37 -120
View File
@@ -18,7 +18,7 @@ from openai.types.responses.response_output_item import McpCall
from openai.types.responses.response_output_message import ResponseOutputMessage
from openai.types.responses.response_output_text import ResponseOutputText
from openai.types.responses.tool import Mcp
from openai_harmony import Author, Message, Role, StreamState, TextContent
from openai_harmony import Author, HarmonyError, Message, Role, TextContent
from vllm import envs
from vllm.entrypoints.chat_utils import (
@@ -29,11 +29,7 @@ from vllm.entrypoints.mcp.tool_server import ToolServer
from vllm.entrypoints.openai.engine.protocol import (
FunctionCall,
)
from vllm.entrypoints.openai.parser.harmony_utils import (
get_encoding,
get_streamable_parser_for_assistant,
render_for_completion,
)
from vllm.entrypoints.openai.parser.harmony_utils import render_for_completion
from vllm.entrypoints.openai.responses.protocol import (
ResponseInputOutputItem,
ResponseRawMessageAndToken,
@@ -597,18 +593,21 @@ class HarmonyContext(ConversationContext):
self,
messages: list,
available_tools: list[str],
function_tool_names: frozenset[str] | None = None,
function_tool_names: frozenset[str],
response_parser: Parser | None = None,
):
from vllm.parser.harmony import HarmonyParser, Segment
assert isinstance(response_parser, HarmonyParser)
self._messages = messages
self.response_parser = response_parser
self.response_parser: HarmonyParser = response_parser
self.finish_reason: str | None = None
self.available_tools = available_tools
self.function_tool_names = function_tool_names
self._tool_sessions: dict[str, ClientSession | Tool] = {}
self.called_tools: set[str] = set()
self.parser = get_streamable_parser_for_assistant()
self.num_init_messages = len(messages)
self.num_prompt_tokens = 0
self.num_output_tokens = 0
@@ -616,44 +615,47 @@ class HarmonyContext(ConversationContext):
self.num_reasoning_tokens = 0
self.num_tool_output_tokens = 0
self.last_append_segments: list[Segment] = []
self.last_append_flush_status: bool | HarmonyError = False
# Turn tracking - replaces multiple individual tracking variables
self.current_turn_metrics = TurnMetrics()
# Track metrics for all turns
self.all_turn_metrics: list[TurnMetrics] = []
self.is_first_turn = True
self.first_tok_of_message = True # For streaming support
self.first_tok_of_message = True
self.kv_transfer_params: dict[str, Any] | None = None
def _update_num_reasoning_tokens(self):
channel = self.parser.current_channel
if channel == "analysis":
self.num_reasoning_tokens += 1
elif channel == "commentary" and self.parser.current_recipient is not None:
# Tool interactions (python/browser/container) are hidden.
# Preambles (recipient=None) are visible user text.
self.num_reasoning_tokens += 1
def append_output(self, output: RequestOutput) -> None:
if self.first_tok_of_message:
self.finish_reason = None
self._update_prefill_token_usage(output)
output_token_ids = output.outputs[0].token_ids
self.parser = get_streamable_parser_for_assistant()
for token_id in output_token_ids:
self.parser.process(token_id)
# Check if the current token is part of reasoning content
self._update_num_reasoning_tokens()
self._update_prefill_token_usage(output)
result = self.response_parser.process_chunk(output_token_ids)
segments = result.segments
self.num_reasoning_tokens += result.reasoning_token_count
self.first_tok_of_message = output.finished
self._update_decode_token_usage(output)
if output.kv_transfer_params is not None:
self.kv_transfer_params = output.kv_transfer_params
# Append current turn to all turn list for next turn's calculations
self.all_turn_metrics.append(self.current_turn_metrics.copy())
self.current_turn_metrics.reset()
# append_output is called only once before tool calling
# in non-streaming case
# so we can append all the parser messages to _messages
output_msgs = self.parser.messages
# The responses finish reason is set in the last message
self.finish_reason = output.outputs[0].finish_reason
self._messages.extend(output_msgs)
if output.finished:
self.finish_reason = output.outputs[0].finish_reason
flushed = self.response_parser.flush()
if flushed is not None:
segments.append(flushed)
self.last_append_flush_status = flushed is not None
self.all_turn_metrics.append(self.current_turn_metrics.copy())
self.current_turn_metrics.reset()
self.last_append_segments = segments
self._messages.extend(
segment.completed_message
for segment in segments
if segment.completed_message is not None
)
def append_tool_output(self, output: list[Message]) -> None:
output_msgs = output
@@ -920,88 +922,3 @@ class HarmonyContext(ConversationContext):
for tool in self.called_tools
)
)
class StreamingHarmonyContext(HarmonyContext):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.last_output = None
self.parser = get_streamable_parser_for_assistant()
self.encoding = get_encoding()
self.last_tok = None
self.first_tok_of_message = True
self.last_content_delta = None
@property
def messages(self) -> list:
return self._messages
def append_output(self, output: RequestOutput) -> None:
# append_output is called for each output token in streaming case,
# so we only want to add the prompt tokens once for each message.
self.last_content_delta = None
if self.first_tok_of_message:
self._update_prefill_token_usage(output)
# Reset self.first_tok_of_message if needed:
# if the current token is the last one of the current message
# (finished=True), then the next token processed will mark the
# beginning of a new message
self.first_tok_of_message = output.finished
last_delta_text = ""
for tok in output.outputs[0].token_ids:
self.parser.process(tok)
last_delta_text += self.parser.last_content_delta or ""
if last_delta_text:
self.last_content_delta = last_delta_text
self._update_decode_token_usage(output)
if output.kv_transfer_params is not None:
self.kv_transfer_params = output.kv_transfer_params
# For streaming, update previous turn when message is complete
if output.finished:
self.all_turn_metrics.append(self.current_turn_metrics.copy())
self.current_turn_metrics.reset()
# Check if the current token is part of reasoning content
self._update_num_reasoning_tokens()
self.last_tok = tok
if len(self._messages) - self.num_init_messages < len(self.parser.messages):
self._messages.extend(
self.parser.messages[len(self._messages) - self.num_init_messages :]
)
def append_tool_output(self, output: list[Message]) -> None:
# Handle the case of tool output in direct message format
assert len(output) == 1, "Tool output should be a single message"
msg = output[0]
# Sometimes the recipient is not set for tool messages,
# so we set it to "assistant"
if msg.author.role == Role.TOOL and msg.recipient is None:
msg.recipient = "assistant"
toks = self.encoding.render(msg)
for tok in toks:
self.parser.process(tok)
self.last_tok = toks[-1]
# TODO: add tool_output messages to self._messages
def is_expecting_start(self) -> bool:
return self.parser.state == StreamState.EXPECT_START
def is_assistant_action_turn(self) -> bool:
return self.last_tok in self.encoding.stop_tokens_for_assistant_actions()
def render_for_completion(self) -> list[int]:
# now this list of tokens as next turn's starting tokens
# `<|start|>assistant`,
# we need to process them in parser.
rendered_tokens = super().render_for_completion()
last_n = -1
to_process = []
while rendered_tokens[last_n] != self.last_tok:
to_process.append(rendered_tokens[last_n])
last_n -= 1
for tok in reversed(to_process):
self.parser.process(tok)
return rendered_tokens
+26 -128
View File
@@ -27,7 +27,7 @@ from openai.types.responses.response_output_item import McpCall
from openai.types.responses.response_reasoning_item import (
Content as ResponseReasoningTextContent,
)
from openai_harmony import Author, Message, Role, StreamableParser, TextContent
from openai_harmony import Author, Message, Role, TextContent
from vllm.entrypoints.openai.parser.harmony_utils import (
BUILTIN_TOOL_TO_MCP_SERVER_LABEL,
@@ -306,7 +306,9 @@ def _parse_browser_tool_call(message: Message, recipient: str) -> ResponseOutput
)
def _parse_function_call(message: Message, recipient: str) -> list[ResponseOutputItem]:
def _parse_function_call(
message: Message, recipient: str, incomplete: bool = False
) -> list[ResponseOutputItem]:
"""Parse function calls into function tool call items."""
function_name = extract_function_from_recipient(recipient)
output_items = []
@@ -318,7 +320,7 @@ def _parse_function_call(message: Message, recipient: str) -> list[ResponseOutpu
type="function_call",
name=function_name,
id=f"fc_{random_id}",
status="completed",
status="incomplete" if incomplete else "completed",
)
output_items.append(response_item)
return output_items
@@ -341,7 +343,9 @@ def _parse_reasoning(message: Message) -> list[ResponseOutputItem]:
return output_items
def _parse_final_message(message: Message) -> ResponseOutputItem:
def _parse_final_message(
message: Message, incomplete: bool = False
) -> ResponseOutputItem:
"""Parse final channel messages into output message items."""
contents = []
for content in message.content:
@@ -356,7 +360,7 @@ def _parse_final_message(message: Message) -> ResponseOutputItem:
id=f"msg_{random_uuid()}",
content=contents,
role=message.author.role,
status="completed",
status="incomplete" if incomplete else "completed",
type="message",
)
@@ -381,7 +385,9 @@ def _parse_mcp_recipient(recipient: str) -> tuple[str, str]:
return server_label, tool_name
def _parse_mcp_call(message: Message, recipient: str) -> list[ResponseOutputItem]:
def _parse_mcp_call(
message: Message, recipient: str, incomplete: bool = False
) -> list[ResponseOutputItem]:
"""Parse MCP calls into MCP call items."""
# Handle built-in tools that need server_label mapping
if recipient in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
@@ -398,7 +404,7 @@ def _parse_mcp_call(message: Message, recipient: str) -> list[ResponseOutputItem
name=tool_name,
server_label=server_label,
id=f"mcp_{random_uuid()}",
status="completed",
status="incomplete" if incomplete else "completed",
)
output_items.append(response_item)
return output_items
@@ -406,6 +412,7 @@ def _parse_mcp_call(message: Message, recipient: str) -> list[ResponseOutputItem
def _parse_message_no_recipient(
message: Message,
incomplete: bool = False,
) -> list[ResponseOutputItem]:
"""Parse a Harmony message with no recipient based on its channel."""
if message.channel == "analysis":
@@ -415,7 +422,7 @@ def _parse_message_no_recipient(
# Per Harmony format, preambles (commentary with no recipient) and
# final channel content are both intended to be shown to end-users.
# See: https://cookbook.openai.com/articles/openai-harmony
return [_parse_final_message(message)]
return [_parse_final_message(message, incomplete=incomplete)]
raise ValueError(f"Unknown channel: {message.channel}")
@@ -427,7 +434,8 @@ def _parse_message_no_recipient(
def harmony_to_response_output(
message: Message,
function_tool_names: frozenset[str] | None = None,
function_tool_names: frozenset[str],
incomplete: bool = False,
) -> list[ResponseOutputItem]:
"""Parse a Harmony message into a list of output response items.
@@ -445,11 +453,14 @@ def harmony_to_response_output(
if recipient is not None:
# Browser tool calls (browser.search, browser.open, browser.find)
if recipient.startswith("browser."):
output_items.append(_parse_browser_tool_call(message, recipient))
if not incomplete:
output_items.append(_parse_browser_tool_call(message, recipient))
# Function calls (with or without "functions." prefix)
elif is_function_recipient(recipient, function_tool_names):
output_items.extend(_parse_function_call(message, recipient))
output_items.extend(
_parse_function_call(message, recipient, incomplete=incomplete)
)
# Built-in MCP tools (python, browser, container)
elif recipient in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
@@ -457,125 +468,12 @@ def harmony_to_response_output(
# All other recipients are MCP calls
else:
output_items.extend(_parse_mcp_call(message, recipient))
output_items.extend(
_parse_mcp_call(message, recipient, incomplete=incomplete)
)
# No recipient - handle based on channel for non-tool messages
else:
output_items.extend(_parse_message_no_recipient(message))
output_items.extend(_parse_message_no_recipient(message, incomplete=incomplete))
return output_items
def parser_state_to_response_output(
parser: StreamableParser,
function_tool_names: frozenset[str] | None = None,
) -> list[ResponseOutputItem]:
"""Extract in-progress response items from incomplete parser state.
Called when the parser has buffered content that hasn't formed a
complete message yet (e.g., generation was cut short).
"""
if not parser.current_content:
return []
if parser.current_role != Role.ASSISTANT:
return []
current_recipient = parser.current_recipient
if current_recipient is not None and current_recipient.startswith("browser."):
return []
if current_recipient:
if is_function_recipient(current_recipient, function_tool_names):
rid = random_uuid()
return [
ResponseFunctionToolCall(
arguments=parser.current_content,
call_id=f"call_{rid}",
type="function_call",
name=extract_function_from_recipient(current_recipient),
id=f"fc_{rid}",
status="in_progress",
)
]
# Built-in MCP tools (python, browser, container)
elif current_recipient in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
return [
ResponseReasoningItem(
id=f"rs_{random_uuid()}",
summary=[],
type="reasoning",
content=[
ResponseReasoningTextContent(
text=parser.current_content, type="reasoning_text"
)
],
status=None,
)
]
# All other recipients are MCP calls
else:
rid = random_uuid()
server_label, tool_name = _parse_mcp_recipient(current_recipient)
return [
McpCall(
arguments=parser.current_content,
type="mcp_call",
name=tool_name,
server_label=server_label,
id=f"mcp_{rid}",
status="in_progress",
)
]
if parser.current_channel == "commentary":
# Per Harmony format, preambles (commentary with no recipient) are
# intended to be shown to end-users, unlike analysis channel content.
output_text = ResponseOutputText(
text=parser.current_content,
annotations=[],
type="output_text",
logprobs=None,
)
return [
ResponseOutputMessage(
id=f"msg_{random_uuid()}",
content=[output_text],
role="assistant",
status="incomplete",
type="message",
)
]
if parser.current_channel == "analysis":
return [
ResponseReasoningItem(
id=f"rs_{random_uuid()}",
summary=[],
type="reasoning",
content=[
ResponseReasoningTextContent(
text=parser.current_content, type="reasoning_text"
)
],
status=None,
)
]
if parser.current_channel == "final":
output_text = ResponseOutputText(
text=parser.current_content,
annotations=[], # TODO
type="output_text",
logprobs=None, # TODO
)
text_item = ResponseOutputMessage(
id=f"msg_{random_uuid()}",
content=[output_text],
role="assistant",
# if the parser still has messages (ie if the generator got cut
# abruptly), this should be incomplete
status="incomplete",
type="message",
)
return [text_item]
return []
+37 -46
View File
@@ -54,12 +54,10 @@ from vllm.entrypoints.openai.responses.context import (
HarmonyContext,
ParsableContext,
SimpleContext,
StreamingHarmonyContext,
)
from vllm.entrypoints.openai.responses.harmony import (
construct_harmony_previous_input_messages,
harmony_to_response_output,
parser_state_to_response_output,
response_input_to_harmony,
)
from vllm.entrypoints.openai.responses.protocol import (
@@ -462,20 +460,12 @@ class OpenAIServingResponses(OpenAIServing):
context: ConversationContext
function_tool_names = extract_function_tool_names(request.tools)
if self.use_harmony:
if request.stream:
context = StreamingHarmonyContext(
messages,
available_tools,
function_tool_names,
response_parser=response_parser,
)
else:
context = HarmonyContext(
messages,
available_tools,
function_tool_names,
response_parser=response_parser,
)
context = HarmonyContext(
messages,
available_tools,
function_tool_names,
response_parser=response_parser,
)
else:
if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT:
# This is a feature in development for parsing
@@ -718,7 +708,7 @@ class OpenAIServingResponses(OpenAIServing):
# Create inputs for the next turn.
# Render the next prompt token ids and update sampling_params.
if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):
if isinstance(context, HarmonyContext):
token_ids = context.render_for_completion()
engine_input = tokens_input(token_ids)
@@ -814,7 +804,20 @@ class OpenAIServingResponses(OpenAIServing):
output_messages: ResponseInputOutputMessage | None = None
if self.use_harmony:
assert isinstance(context, HarmonyContext)
output = self._make_response_output_items_with_harmony(context)
output = []
harmony_msgs = context.messages[context.num_init_messages :]
if harmony_msgs:
fn_names = context.function_tool_names
for msg in harmony_msgs[:-1]:
output.extend(harmony_to_response_output(msg, fn_names))
output.extend(
harmony_to_response_output(
harmony_msgs[-1],
fn_names,
incomplete=context.last_append_flush_status,
)
)
if request.enable_response_messages:
input_messages = context.messages[: context.num_init_messages]
output_messages = context.messages[context.num_init_messages :]
@@ -1092,21 +1095,6 @@ class OpenAIServingResponses(OpenAIServing):
)
]
def _make_response_output_items_with_harmony(
self,
context: HarmonyContext,
) -> list[ResponseOutputItem]:
output_items: list[ResponseOutputItem] = []
num_init_messages = context.num_init_messages
fn_names = context.function_tool_names
for msg in context.messages[num_init_messages:]:
output_items.extend(harmony_to_response_output(msg, fn_names))
# Handle the generation stopped in the middle (if any).
last_items = parser_state_to_response_output(context.parser, fn_names)
if last_items:
output_items.extend(last_items)
return output_items
def _get_harmony_builtin_tool_descriptions(
self, request: ResponsesRequest, tool_types: set[str]
) -> dict[str, str | None]:
@@ -1423,27 +1411,30 @@ class OpenAIServingResponses(OpenAIServing):
state = StreamingState()
async for ctx in result_generator:
assert isinstance(ctx, StreamingHarmonyContext)
assert isinstance(ctx, HarmonyContext)
# finish_reason='error' indicates a retryable error
self._raise_if_error(ctx.finish_reason, request.request_id)
if ctx.is_expecting_start():
if len(ctx.parser.messages) > 0:
previous_item = ctx.parser.messages[-1]
for event in emit_previous_item_done_events(
previous_item, state, ctx.function_tool_names
for segment in ctx.last_append_segments:
if segment.delta:
for event in emit_content_delta_events(
segment, state, ctx.function_tool_names
):
yield _increment_sequence_number_and_return(event)
state.reset_for_new_item()
# Stream the output of a harmony message
for event in emit_content_delta_events(ctx, state):
yield _increment_sequence_number_and_return(event)
elif completed_message := segment.completed_message:
# TODO: Fix browser emitted as MCP calls
for event in emit_previous_item_done_events(
completed_message, state, ctx.function_tool_names
):
yield _increment_sequence_number_and_return(event)
# Stream tool call outputs
for event in emit_tool_action_events(ctx, state, self.tool_server):
yield _increment_sequence_number_and_return(event)
for event in emit_tool_action_events(
completed_message, state, self.tool_server
):
yield _increment_sequence_number_and_return(event)
state.reset_for_new_item()
async def responses_stream_generator(
self,
@@ -66,13 +66,13 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
extract_function_from_recipient,
is_function_recipient,
)
from vllm.entrypoints.openai.responses.context import StreamingHarmonyContext
from vllm.entrypoints.openai.responses.protocol import (
ResponseReasoningPartAddedEvent,
ResponseReasoningPartDoneEvent,
StreamingResponsesResponse,
)
from vllm.outputs import CompletionOutput
from vllm.parser.harmony import Segment
from vllm.utils import random_uuid
TOOL_NAME_TO_MCP_SERVER_LABEL: Final[dict[str, str]] = {
@@ -110,6 +110,7 @@ class StreamingState:
def reset_for_new_item(self) -> None:
"""Reset state when expecting a new output item."""
self.current_output_index += 1
self.current_content_index = -1
self.sent_output_item_added = False
self.is_first_function_call_delta = False
self.current_call_id = ""
@@ -558,20 +559,21 @@ def emit_mcp_completion_events(
def emit_content_delta_events(
ctx: StreamingHarmonyContext,
segment: Segment,
state: StreamingState,
function_tool_names: frozenset[str] | None = None,
) -> list[StreamingResponsesResponse]:
"""Emit events for content delta streaming based on channel type.
This is a Harmony-specific dispatcher that extracts values from the
Harmony context and delegates to shared leaf helpers.
latest append segment and delegates to shared leaf helpers.
"""
delta = ctx.last_content_delta
delta = segment.delta
if not delta:
return []
channel = ctx.parser.current_channel
recipient = ctx.parser.current_recipient
channel = segment.channel
recipient = segment.recipient
if channel in ("final", "commentary") and recipient is None:
# Preambles (commentary with no recipient) and final messages
@@ -580,7 +582,7 @@ def emit_content_delta_events(
elif channel == "analysis" and recipient is None:
return emit_reasoning_delta_events(delta, state)
elif recipient is not None:
fn_names = ctx.function_tool_names
fn_names = function_tool_names
if is_function_recipient(recipient, fn_names):
function_name = extract_function_from_recipient(recipient)
return emit_function_call_delta_events(delta, function_name, state)
@@ -604,6 +606,12 @@ def emit_previous_item_done_events(
This is a Harmony-specific dispatcher that extracts values from the
Harmony parser's message object and delegates to shared leaf helpers.
"""
if not state.sent_output_item_added and not state.is_first_function_call_delta:
# Suppress done events for items had no delta and thus had no
# added/in-progress lifecycle events. This is a bug.
# TODO: Ensure added/in-progress events are emitted for zero-delta items.
return []
text = previous_item.content[0].text
if previous_item.recipient is not None:
# Deal with tool call
@@ -769,47 +777,22 @@ def emit_code_interpreter_completion_events(
def emit_tool_action_events(
ctx: StreamingHarmonyContext,
previous_item: HarmonyMessage,
state: StreamingState,
tool_server: ToolServer | None,
) -> list[StreamingResponsesResponse]:
"""Emit events for tool action turn."""
if not ctx.is_assistant_action_turn() or len(ctx.parser.messages) == 0:
return []
events: list[StreamingResponsesResponse] = []
previous_item = ctx.parser.messages[-1]
"""Emit events for a completed assistant action turn."""
# Handle browser tool
if (
tool_server is not None
and tool_server.has_tool("browser")
previous_item.author.role == "assistant"
and previous_item.recipient is not None
and previous_item.recipient.startswith("browser.")
and tool_server is not None
and tool_server.has_tool("browser")
):
events.extend(emit_browser_tool_events(previous_item, state))
return emit_browser_tool_events(previous_item, state)
# Handle tool completion
if (
tool_server is not None
and previous_item.recipient is not None
and state.current_item_id is not None
and state.sent_output_item_added
):
recipient = previous_item.recipient
fn_names = ctx.function_tool_names
if recipient == "python":
events.extend(emit_code_interpreter_completion_events(previous_item, state))
elif recipient.startswith("mcp.") or is_mcp_tool_by_namespace(
recipient, fn_names
):
events.extend(
emit_mcp_completion_events(
recipient, previous_item.content[0].text, state
)
)
return events
return []
# =====================================================================