[Refactor][GPT-OSS] Harmony Responses API Refactor to use HarmonyParser (#47185)
Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
@@ -18,7 +18,7 @@ from openai.types.responses.response_output_item import McpCall
|
||||
from openai.types.responses.response_output_message import ResponseOutputMessage
|
||||
from openai.types.responses.response_output_text import ResponseOutputText
|
||||
from openai.types.responses.tool import Mcp
|
||||
from openai_harmony import Author, Message, Role, StreamState, TextContent
|
||||
from openai_harmony import Author, HarmonyError, Message, Role, TextContent
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
@@ -29,11 +29,7 @@ from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
FunctionCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
get_encoding,
|
||||
get_streamable_parser_for_assistant,
|
||||
render_for_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import render_for_completion
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponseInputOutputItem,
|
||||
ResponseRawMessageAndToken,
|
||||
@@ -597,18 +593,21 @@ class HarmonyContext(ConversationContext):
|
||||
self,
|
||||
messages: list,
|
||||
available_tools: list[str],
|
||||
function_tool_names: frozenset[str] | None = None,
|
||||
function_tool_names: frozenset[str],
|
||||
response_parser: Parser | None = None,
|
||||
):
|
||||
from vllm.parser.harmony import HarmonyParser, Segment
|
||||
|
||||
assert isinstance(response_parser, HarmonyParser)
|
||||
|
||||
self._messages = messages
|
||||
self.response_parser = response_parser
|
||||
self.response_parser: HarmonyParser = response_parser
|
||||
self.finish_reason: str | None = None
|
||||
self.available_tools = available_tools
|
||||
self.function_tool_names = function_tool_names
|
||||
self._tool_sessions: dict[str, ClientSession | Tool] = {}
|
||||
self.called_tools: set[str] = set()
|
||||
|
||||
self.parser = get_streamable_parser_for_assistant()
|
||||
self.num_init_messages = len(messages)
|
||||
self.num_prompt_tokens = 0
|
||||
self.num_output_tokens = 0
|
||||
@@ -616,44 +615,47 @@ class HarmonyContext(ConversationContext):
|
||||
self.num_reasoning_tokens = 0
|
||||
self.num_tool_output_tokens = 0
|
||||
|
||||
self.last_append_segments: list[Segment] = []
|
||||
self.last_append_flush_status: bool | HarmonyError = False
|
||||
|
||||
# Turn tracking - replaces multiple individual tracking variables
|
||||
self.current_turn_metrics = TurnMetrics()
|
||||
# Track metrics for all turns
|
||||
self.all_turn_metrics: list[TurnMetrics] = []
|
||||
self.is_first_turn = True
|
||||
self.first_tok_of_message = True # For streaming support
|
||||
self.first_tok_of_message = True
|
||||
self.kv_transfer_params: dict[str, Any] | None = None
|
||||
|
||||
def _update_num_reasoning_tokens(self):
|
||||
channel = self.parser.current_channel
|
||||
if channel == "analysis":
|
||||
self.num_reasoning_tokens += 1
|
||||
elif channel == "commentary" and self.parser.current_recipient is not None:
|
||||
# Tool interactions (python/browser/container) are hidden.
|
||||
# Preambles (recipient=None) are visible user text.
|
||||
self.num_reasoning_tokens += 1
|
||||
|
||||
def append_output(self, output: RequestOutput) -> None:
|
||||
if self.first_tok_of_message:
|
||||
self.finish_reason = None
|
||||
self._update_prefill_token_usage(output)
|
||||
|
||||
output_token_ids = output.outputs[0].token_ids
|
||||
self.parser = get_streamable_parser_for_assistant()
|
||||
for token_id in output_token_ids:
|
||||
self.parser.process(token_id)
|
||||
# Check if the current token is part of reasoning content
|
||||
self._update_num_reasoning_tokens()
|
||||
self._update_prefill_token_usage(output)
|
||||
result = self.response_parser.process_chunk(output_token_ids)
|
||||
segments = result.segments
|
||||
self.num_reasoning_tokens += result.reasoning_token_count
|
||||
|
||||
self.first_tok_of_message = output.finished
|
||||
self._update_decode_token_usage(output)
|
||||
if output.kv_transfer_params is not None:
|
||||
self.kv_transfer_params = output.kv_transfer_params
|
||||
# Append current turn to all turn list for next turn's calculations
|
||||
self.all_turn_metrics.append(self.current_turn_metrics.copy())
|
||||
self.current_turn_metrics.reset()
|
||||
# append_output is called only once before tool calling
|
||||
# in non-streaming case
|
||||
# so we can append all the parser messages to _messages
|
||||
output_msgs = self.parser.messages
|
||||
# The responses finish reason is set in the last message
|
||||
self.finish_reason = output.outputs[0].finish_reason
|
||||
self._messages.extend(output_msgs)
|
||||
|
||||
if output.finished:
|
||||
self.finish_reason = output.outputs[0].finish_reason
|
||||
flushed = self.response_parser.flush()
|
||||
if flushed is not None:
|
||||
segments.append(flushed)
|
||||
self.last_append_flush_status = flushed is not None
|
||||
self.all_turn_metrics.append(self.current_turn_metrics.copy())
|
||||
self.current_turn_metrics.reset()
|
||||
|
||||
self.last_append_segments = segments
|
||||
self._messages.extend(
|
||||
segment.completed_message
|
||||
for segment in segments
|
||||
if segment.completed_message is not None
|
||||
)
|
||||
|
||||
def append_tool_output(self, output: list[Message]) -> None:
|
||||
output_msgs = output
|
||||
@@ -920,88 +922,3 @@ class HarmonyContext(ConversationContext):
|
||||
for tool in self.called_tools
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class StreamingHarmonyContext(HarmonyContext):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.last_output = None
|
||||
|
||||
self.parser = get_streamable_parser_for_assistant()
|
||||
self.encoding = get_encoding()
|
||||
self.last_tok = None
|
||||
self.first_tok_of_message = True
|
||||
self.last_content_delta = None
|
||||
|
||||
@property
|
||||
def messages(self) -> list:
|
||||
return self._messages
|
||||
|
||||
def append_output(self, output: RequestOutput) -> None:
|
||||
# append_output is called for each output token in streaming case,
|
||||
# so we only want to add the prompt tokens once for each message.
|
||||
self.last_content_delta = None
|
||||
if self.first_tok_of_message:
|
||||
self._update_prefill_token_usage(output)
|
||||
# Reset self.first_tok_of_message if needed:
|
||||
# if the current token is the last one of the current message
|
||||
# (finished=True), then the next token processed will mark the
|
||||
# beginning of a new message
|
||||
self.first_tok_of_message = output.finished
|
||||
last_delta_text = ""
|
||||
for tok in output.outputs[0].token_ids:
|
||||
self.parser.process(tok)
|
||||
last_delta_text += self.parser.last_content_delta or ""
|
||||
if last_delta_text:
|
||||
self.last_content_delta = last_delta_text
|
||||
self._update_decode_token_usage(output)
|
||||
if output.kv_transfer_params is not None:
|
||||
self.kv_transfer_params = output.kv_transfer_params
|
||||
|
||||
# For streaming, update previous turn when message is complete
|
||||
if output.finished:
|
||||
self.all_turn_metrics.append(self.current_turn_metrics.copy())
|
||||
self.current_turn_metrics.reset()
|
||||
# Check if the current token is part of reasoning content
|
||||
self._update_num_reasoning_tokens()
|
||||
self.last_tok = tok
|
||||
if len(self._messages) - self.num_init_messages < len(self.parser.messages):
|
||||
self._messages.extend(
|
||||
self.parser.messages[len(self._messages) - self.num_init_messages :]
|
||||
)
|
||||
|
||||
def append_tool_output(self, output: list[Message]) -> None:
|
||||
# Handle the case of tool output in direct message format
|
||||
assert len(output) == 1, "Tool output should be a single message"
|
||||
msg = output[0]
|
||||
# Sometimes the recipient is not set for tool messages,
|
||||
# so we set it to "assistant"
|
||||
if msg.author.role == Role.TOOL and msg.recipient is None:
|
||||
msg.recipient = "assistant"
|
||||
toks = self.encoding.render(msg)
|
||||
for tok in toks:
|
||||
self.parser.process(tok)
|
||||
self.last_tok = toks[-1]
|
||||
# TODO: add tool_output messages to self._messages
|
||||
|
||||
def is_expecting_start(self) -> bool:
|
||||
return self.parser.state == StreamState.EXPECT_START
|
||||
|
||||
def is_assistant_action_turn(self) -> bool:
|
||||
return self.last_tok in self.encoding.stop_tokens_for_assistant_actions()
|
||||
|
||||
def render_for_completion(self) -> list[int]:
|
||||
# now this list of tokens as next turn's starting tokens
|
||||
# `<|start|>assistant`,
|
||||
# we need to process them in parser.
|
||||
rendered_tokens = super().render_for_completion()
|
||||
|
||||
last_n = -1
|
||||
to_process = []
|
||||
while rendered_tokens[last_n] != self.last_tok:
|
||||
to_process.append(rendered_tokens[last_n])
|
||||
last_n -= 1
|
||||
for tok in reversed(to_process):
|
||||
self.parser.process(tok)
|
||||
|
||||
return rendered_tokens
|
||||
|
||||
@@ -27,7 +27,7 @@ from openai.types.responses.response_output_item import McpCall
|
||||
from openai.types.responses.response_reasoning_item import (
|
||||
Content as ResponseReasoningTextContent,
|
||||
)
|
||||
from openai_harmony import Author, Message, Role, StreamableParser, TextContent
|
||||
from openai_harmony import Author, Message, Role, TextContent
|
||||
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
BUILTIN_TOOL_TO_MCP_SERVER_LABEL,
|
||||
@@ -306,7 +306,9 @@ def _parse_browser_tool_call(message: Message, recipient: str) -> ResponseOutput
|
||||
)
|
||||
|
||||
|
||||
def _parse_function_call(message: Message, recipient: str) -> list[ResponseOutputItem]:
|
||||
def _parse_function_call(
|
||||
message: Message, recipient: str, incomplete: bool = False
|
||||
) -> list[ResponseOutputItem]:
|
||||
"""Parse function calls into function tool call items."""
|
||||
function_name = extract_function_from_recipient(recipient)
|
||||
output_items = []
|
||||
@@ -318,7 +320,7 @@ def _parse_function_call(message: Message, recipient: str) -> list[ResponseOutpu
|
||||
type="function_call",
|
||||
name=function_name,
|
||||
id=f"fc_{random_id}",
|
||||
status="completed",
|
||||
status="incomplete" if incomplete else "completed",
|
||||
)
|
||||
output_items.append(response_item)
|
||||
return output_items
|
||||
@@ -341,7 +343,9 @@ def _parse_reasoning(message: Message) -> list[ResponseOutputItem]:
|
||||
return output_items
|
||||
|
||||
|
||||
def _parse_final_message(message: Message) -> ResponseOutputItem:
|
||||
def _parse_final_message(
|
||||
message: Message, incomplete: bool = False
|
||||
) -> ResponseOutputItem:
|
||||
"""Parse final channel messages into output message items."""
|
||||
contents = []
|
||||
for content in message.content:
|
||||
@@ -356,7 +360,7 @@ def _parse_final_message(message: Message) -> ResponseOutputItem:
|
||||
id=f"msg_{random_uuid()}",
|
||||
content=contents,
|
||||
role=message.author.role,
|
||||
status="completed",
|
||||
status="incomplete" if incomplete else "completed",
|
||||
type="message",
|
||||
)
|
||||
|
||||
@@ -381,7 +385,9 @@ def _parse_mcp_recipient(recipient: str) -> tuple[str, str]:
|
||||
return server_label, tool_name
|
||||
|
||||
|
||||
def _parse_mcp_call(message: Message, recipient: str) -> list[ResponseOutputItem]:
|
||||
def _parse_mcp_call(
|
||||
message: Message, recipient: str, incomplete: bool = False
|
||||
) -> list[ResponseOutputItem]:
|
||||
"""Parse MCP calls into MCP call items."""
|
||||
# Handle built-in tools that need server_label mapping
|
||||
if recipient in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
|
||||
@@ -398,7 +404,7 @@ def _parse_mcp_call(message: Message, recipient: str) -> list[ResponseOutputItem
|
||||
name=tool_name,
|
||||
server_label=server_label,
|
||||
id=f"mcp_{random_uuid()}",
|
||||
status="completed",
|
||||
status="incomplete" if incomplete else "completed",
|
||||
)
|
||||
output_items.append(response_item)
|
||||
return output_items
|
||||
@@ -406,6 +412,7 @@ def _parse_mcp_call(message: Message, recipient: str) -> list[ResponseOutputItem
|
||||
|
||||
def _parse_message_no_recipient(
|
||||
message: Message,
|
||||
incomplete: bool = False,
|
||||
) -> list[ResponseOutputItem]:
|
||||
"""Parse a Harmony message with no recipient based on its channel."""
|
||||
if message.channel == "analysis":
|
||||
@@ -415,7 +422,7 @@ def _parse_message_no_recipient(
|
||||
# Per Harmony format, preambles (commentary with no recipient) and
|
||||
# final channel content are both intended to be shown to end-users.
|
||||
# See: https://cookbook.openai.com/articles/openai-harmony
|
||||
return [_parse_final_message(message)]
|
||||
return [_parse_final_message(message, incomplete=incomplete)]
|
||||
|
||||
raise ValueError(f"Unknown channel: {message.channel}")
|
||||
|
||||
@@ -427,7 +434,8 @@ def _parse_message_no_recipient(
|
||||
|
||||
def harmony_to_response_output(
|
||||
message: Message,
|
||||
function_tool_names: frozenset[str] | None = None,
|
||||
function_tool_names: frozenset[str],
|
||||
incomplete: bool = False,
|
||||
) -> list[ResponseOutputItem]:
|
||||
"""Parse a Harmony message into a list of output response items.
|
||||
|
||||
@@ -445,11 +453,14 @@ def harmony_to_response_output(
|
||||
if recipient is not None:
|
||||
# Browser tool calls (browser.search, browser.open, browser.find)
|
||||
if recipient.startswith("browser."):
|
||||
output_items.append(_parse_browser_tool_call(message, recipient))
|
||||
if not incomplete:
|
||||
output_items.append(_parse_browser_tool_call(message, recipient))
|
||||
|
||||
# Function calls (with or without "functions." prefix)
|
||||
elif is_function_recipient(recipient, function_tool_names):
|
||||
output_items.extend(_parse_function_call(message, recipient))
|
||||
output_items.extend(
|
||||
_parse_function_call(message, recipient, incomplete=incomplete)
|
||||
)
|
||||
|
||||
# Built-in MCP tools (python, browser, container)
|
||||
elif recipient in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
|
||||
@@ -457,125 +468,12 @@ def harmony_to_response_output(
|
||||
|
||||
# All other recipients are MCP calls
|
||||
else:
|
||||
output_items.extend(_parse_mcp_call(message, recipient))
|
||||
output_items.extend(
|
||||
_parse_mcp_call(message, recipient, incomplete=incomplete)
|
||||
)
|
||||
|
||||
# No recipient - handle based on channel for non-tool messages
|
||||
else:
|
||||
output_items.extend(_parse_message_no_recipient(message))
|
||||
output_items.extend(_parse_message_no_recipient(message, incomplete=incomplete))
|
||||
|
||||
return output_items
|
||||
|
||||
|
||||
def parser_state_to_response_output(
|
||||
parser: StreamableParser,
|
||||
function_tool_names: frozenset[str] | None = None,
|
||||
) -> list[ResponseOutputItem]:
|
||||
"""Extract in-progress response items from incomplete parser state.
|
||||
|
||||
Called when the parser has buffered content that hasn't formed a
|
||||
complete message yet (e.g., generation was cut short).
|
||||
"""
|
||||
if not parser.current_content:
|
||||
return []
|
||||
if parser.current_role != Role.ASSISTANT:
|
||||
return []
|
||||
current_recipient = parser.current_recipient
|
||||
if current_recipient is not None and current_recipient.startswith("browser."):
|
||||
return []
|
||||
|
||||
if current_recipient:
|
||||
if is_function_recipient(current_recipient, function_tool_names):
|
||||
rid = random_uuid()
|
||||
return [
|
||||
ResponseFunctionToolCall(
|
||||
arguments=parser.current_content,
|
||||
call_id=f"call_{rid}",
|
||||
type="function_call",
|
||||
name=extract_function_from_recipient(current_recipient),
|
||||
id=f"fc_{rid}",
|
||||
status="in_progress",
|
||||
)
|
||||
]
|
||||
# Built-in MCP tools (python, browser, container)
|
||||
elif current_recipient in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
|
||||
return [
|
||||
ResponseReasoningItem(
|
||||
id=f"rs_{random_uuid()}",
|
||||
summary=[],
|
||||
type="reasoning",
|
||||
content=[
|
||||
ResponseReasoningTextContent(
|
||||
text=parser.current_content, type="reasoning_text"
|
||||
)
|
||||
],
|
||||
status=None,
|
||||
)
|
||||
]
|
||||
# All other recipients are MCP calls
|
||||
else:
|
||||
rid = random_uuid()
|
||||
server_label, tool_name = _parse_mcp_recipient(current_recipient)
|
||||
return [
|
||||
McpCall(
|
||||
arguments=parser.current_content,
|
||||
type="mcp_call",
|
||||
name=tool_name,
|
||||
server_label=server_label,
|
||||
id=f"mcp_{rid}",
|
||||
status="in_progress",
|
||||
)
|
||||
]
|
||||
|
||||
if parser.current_channel == "commentary":
|
||||
# Per Harmony format, preambles (commentary with no recipient) are
|
||||
# intended to be shown to end-users, unlike analysis channel content.
|
||||
output_text = ResponseOutputText(
|
||||
text=parser.current_content,
|
||||
annotations=[],
|
||||
type="output_text",
|
||||
logprobs=None,
|
||||
)
|
||||
return [
|
||||
ResponseOutputMessage(
|
||||
id=f"msg_{random_uuid()}",
|
||||
content=[output_text],
|
||||
role="assistant",
|
||||
status="incomplete",
|
||||
type="message",
|
||||
)
|
||||
]
|
||||
|
||||
if parser.current_channel == "analysis":
|
||||
return [
|
||||
ResponseReasoningItem(
|
||||
id=f"rs_{random_uuid()}",
|
||||
summary=[],
|
||||
type="reasoning",
|
||||
content=[
|
||||
ResponseReasoningTextContent(
|
||||
text=parser.current_content, type="reasoning_text"
|
||||
)
|
||||
],
|
||||
status=None,
|
||||
)
|
||||
]
|
||||
|
||||
if parser.current_channel == "final":
|
||||
output_text = ResponseOutputText(
|
||||
text=parser.current_content,
|
||||
annotations=[], # TODO
|
||||
type="output_text",
|
||||
logprobs=None, # TODO
|
||||
)
|
||||
text_item = ResponseOutputMessage(
|
||||
id=f"msg_{random_uuid()}",
|
||||
content=[output_text],
|
||||
role="assistant",
|
||||
# if the parser still has messages (ie if the generator got cut
|
||||
# abruptly), this should be incomplete
|
||||
status="incomplete",
|
||||
type="message",
|
||||
)
|
||||
return [text_item]
|
||||
|
||||
return []
|
||||
|
||||
@@ -54,12 +54,10 @@ from vllm.entrypoints.openai.responses.context import (
|
||||
HarmonyContext,
|
||||
ParsableContext,
|
||||
SimpleContext,
|
||||
StreamingHarmonyContext,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.harmony import (
|
||||
construct_harmony_previous_input_messages,
|
||||
harmony_to_response_output,
|
||||
parser_state_to_response_output,
|
||||
response_input_to_harmony,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
@@ -462,20 +460,12 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
context: ConversationContext
|
||||
function_tool_names = extract_function_tool_names(request.tools)
|
||||
if self.use_harmony:
|
||||
if request.stream:
|
||||
context = StreamingHarmonyContext(
|
||||
messages,
|
||||
available_tools,
|
||||
function_tool_names,
|
||||
response_parser=response_parser,
|
||||
)
|
||||
else:
|
||||
context = HarmonyContext(
|
||||
messages,
|
||||
available_tools,
|
||||
function_tool_names,
|
||||
response_parser=response_parser,
|
||||
)
|
||||
context = HarmonyContext(
|
||||
messages,
|
||||
available_tools,
|
||||
function_tool_names,
|
||||
response_parser=response_parser,
|
||||
)
|
||||
else:
|
||||
if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT:
|
||||
# This is a feature in development for parsing
|
||||
@@ -718,7 +708,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
|
||||
# Create inputs for the next turn.
|
||||
# Render the next prompt token ids and update sampling_params.
|
||||
if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):
|
||||
if isinstance(context, HarmonyContext):
|
||||
token_ids = context.render_for_completion()
|
||||
engine_input = tokens_input(token_ids)
|
||||
|
||||
@@ -814,7 +804,20 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
output_messages: ResponseInputOutputMessage | None = None
|
||||
if self.use_harmony:
|
||||
assert isinstance(context, HarmonyContext)
|
||||
output = self._make_response_output_items_with_harmony(context)
|
||||
output = []
|
||||
harmony_msgs = context.messages[context.num_init_messages :]
|
||||
if harmony_msgs:
|
||||
fn_names = context.function_tool_names
|
||||
for msg in harmony_msgs[:-1]:
|
||||
output.extend(harmony_to_response_output(msg, fn_names))
|
||||
output.extend(
|
||||
harmony_to_response_output(
|
||||
harmony_msgs[-1],
|
||||
fn_names,
|
||||
incomplete=context.last_append_flush_status,
|
||||
)
|
||||
)
|
||||
|
||||
if request.enable_response_messages:
|
||||
input_messages = context.messages[: context.num_init_messages]
|
||||
output_messages = context.messages[context.num_init_messages :]
|
||||
@@ -1092,21 +1095,6 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
)
|
||||
]
|
||||
|
||||
def _make_response_output_items_with_harmony(
|
||||
self,
|
||||
context: HarmonyContext,
|
||||
) -> list[ResponseOutputItem]:
|
||||
output_items: list[ResponseOutputItem] = []
|
||||
num_init_messages = context.num_init_messages
|
||||
fn_names = context.function_tool_names
|
||||
for msg in context.messages[num_init_messages:]:
|
||||
output_items.extend(harmony_to_response_output(msg, fn_names))
|
||||
# Handle the generation stopped in the middle (if any).
|
||||
last_items = parser_state_to_response_output(context.parser, fn_names)
|
||||
if last_items:
|
||||
output_items.extend(last_items)
|
||||
return output_items
|
||||
|
||||
def _get_harmony_builtin_tool_descriptions(
|
||||
self, request: ResponsesRequest, tool_types: set[str]
|
||||
) -> dict[str, str | None]:
|
||||
@@ -1423,27 +1411,30 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
state = StreamingState()
|
||||
|
||||
async for ctx in result_generator:
|
||||
assert isinstance(ctx, StreamingHarmonyContext)
|
||||
assert isinstance(ctx, HarmonyContext)
|
||||
|
||||
# finish_reason='error' indicates a retryable error
|
||||
self._raise_if_error(ctx.finish_reason, request.request_id)
|
||||
|
||||
if ctx.is_expecting_start():
|
||||
if len(ctx.parser.messages) > 0:
|
||||
previous_item = ctx.parser.messages[-1]
|
||||
for event in emit_previous_item_done_events(
|
||||
previous_item, state, ctx.function_tool_names
|
||||
for segment in ctx.last_append_segments:
|
||||
if segment.delta:
|
||||
for event in emit_content_delta_events(
|
||||
segment, state, ctx.function_tool_names
|
||||
):
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
state.reset_for_new_item()
|
||||
|
||||
# Stream the output of a harmony message
|
||||
for event in emit_content_delta_events(ctx, state):
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
elif completed_message := segment.completed_message:
|
||||
# TODO: Fix browser emitted as MCP calls
|
||||
for event in emit_previous_item_done_events(
|
||||
completed_message, state, ctx.function_tool_names
|
||||
):
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
|
||||
# Stream tool call outputs
|
||||
for event in emit_tool_action_events(ctx, state, self.tool_server):
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
for event in emit_tool_action_events(
|
||||
completed_message, state, self.tool_server
|
||||
):
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
state.reset_for_new_item()
|
||||
|
||||
async def responses_stream_generator(
|
||||
self,
|
||||
|
||||
@@ -66,13 +66,13 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
extract_function_from_recipient,
|
||||
is_function_recipient,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.context import StreamingHarmonyContext
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponseReasoningPartAddedEvent,
|
||||
ResponseReasoningPartDoneEvent,
|
||||
StreamingResponsesResponse,
|
||||
)
|
||||
from vllm.outputs import CompletionOutput
|
||||
from vllm.parser.harmony import Segment
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
TOOL_NAME_TO_MCP_SERVER_LABEL: Final[dict[str, str]] = {
|
||||
@@ -110,6 +110,7 @@ class StreamingState:
|
||||
def reset_for_new_item(self) -> None:
|
||||
"""Reset state when expecting a new output item."""
|
||||
self.current_output_index += 1
|
||||
self.current_content_index = -1
|
||||
self.sent_output_item_added = False
|
||||
self.is_first_function_call_delta = False
|
||||
self.current_call_id = ""
|
||||
@@ -558,20 +559,21 @@ def emit_mcp_completion_events(
|
||||
|
||||
|
||||
def emit_content_delta_events(
|
||||
ctx: StreamingHarmonyContext,
|
||||
segment: Segment,
|
||||
state: StreamingState,
|
||||
function_tool_names: frozenset[str] | None = None,
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
"""Emit events for content delta streaming based on channel type.
|
||||
|
||||
This is a Harmony-specific dispatcher that extracts values from the
|
||||
Harmony context and delegates to shared leaf helpers.
|
||||
latest append segment and delegates to shared leaf helpers.
|
||||
"""
|
||||
delta = ctx.last_content_delta
|
||||
delta = segment.delta
|
||||
if not delta:
|
||||
return []
|
||||
|
||||
channel = ctx.parser.current_channel
|
||||
recipient = ctx.parser.current_recipient
|
||||
channel = segment.channel
|
||||
recipient = segment.recipient
|
||||
|
||||
if channel in ("final", "commentary") and recipient is None:
|
||||
# Preambles (commentary with no recipient) and final messages
|
||||
@@ -580,7 +582,7 @@ def emit_content_delta_events(
|
||||
elif channel == "analysis" and recipient is None:
|
||||
return emit_reasoning_delta_events(delta, state)
|
||||
elif recipient is not None:
|
||||
fn_names = ctx.function_tool_names
|
||||
fn_names = function_tool_names
|
||||
if is_function_recipient(recipient, fn_names):
|
||||
function_name = extract_function_from_recipient(recipient)
|
||||
return emit_function_call_delta_events(delta, function_name, state)
|
||||
@@ -604,6 +606,12 @@ def emit_previous_item_done_events(
|
||||
This is a Harmony-specific dispatcher that extracts values from the
|
||||
Harmony parser's message object and delegates to shared leaf helpers.
|
||||
"""
|
||||
if not state.sent_output_item_added and not state.is_first_function_call_delta:
|
||||
# Suppress done events for items had no delta and thus had no
|
||||
# added/in-progress lifecycle events. This is a bug.
|
||||
# TODO: Ensure added/in-progress events are emitted for zero-delta items.
|
||||
return []
|
||||
|
||||
text = previous_item.content[0].text
|
||||
if previous_item.recipient is not None:
|
||||
# Deal with tool call
|
||||
@@ -769,47 +777,22 @@ def emit_code_interpreter_completion_events(
|
||||
|
||||
|
||||
def emit_tool_action_events(
|
||||
ctx: StreamingHarmonyContext,
|
||||
previous_item: HarmonyMessage,
|
||||
state: StreamingState,
|
||||
tool_server: ToolServer | None,
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
"""Emit events for tool action turn."""
|
||||
if not ctx.is_assistant_action_turn() or len(ctx.parser.messages) == 0:
|
||||
return []
|
||||
|
||||
events: list[StreamingResponsesResponse] = []
|
||||
previous_item = ctx.parser.messages[-1]
|
||||
|
||||
"""Emit events for a completed assistant action turn."""
|
||||
# Handle browser tool
|
||||
if (
|
||||
tool_server is not None
|
||||
and tool_server.has_tool("browser")
|
||||
previous_item.author.role == "assistant"
|
||||
and previous_item.recipient is not None
|
||||
and previous_item.recipient.startswith("browser.")
|
||||
and tool_server is not None
|
||||
and tool_server.has_tool("browser")
|
||||
):
|
||||
events.extend(emit_browser_tool_events(previous_item, state))
|
||||
return emit_browser_tool_events(previous_item, state)
|
||||
|
||||
# Handle tool completion
|
||||
if (
|
||||
tool_server is not None
|
||||
and previous_item.recipient is not None
|
||||
and state.current_item_id is not None
|
||||
and state.sent_output_item_added
|
||||
):
|
||||
recipient = previous_item.recipient
|
||||
fn_names = ctx.function_tool_names
|
||||
if recipient == "python":
|
||||
events.extend(emit_code_interpreter_completion_events(previous_item, state))
|
||||
elif recipient.startswith("mcp.") or is_mcp_tool_by_namespace(
|
||||
recipient, fn_names
|
||||
):
|
||||
events.extend(
|
||||
emit_mcp_completion_events(
|
||||
recipient, previous_item.content[0].text, state
|
||||
)
|
||||
)
|
||||
|
||||
return events
|
||||
return []
|
||||
|
||||
|
||||
# =====================================================================
|
||||
|
||||
Reference in New Issue
Block a user