[Bugfix] Handle real-world gpt-oss tool call output in Harmony parsing (#42454)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
This commit is contained in:
Ben Browning
2026-05-13 17:54:46 +00:00
committed by GitHub
parent b3c69595a6
commit 0f69128a37
12 changed files with 801 additions and 50 deletions
@@ -199,12 +199,107 @@ class TestExtractHarmonyStreamingDelta:
assert delta_message.content == delta_text
assert tools_streamed is False
@pytest.mark.parametrize("channel", ["commentary", "analysis"])
@patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
def test_new_tool_call_without_functions_prefix(
self, mock_make_tool_call_id, channel
):
mock_make_tool_call_id.return_value = "call_bare123"
parser = MockStreamableParser()
token_states = [TokenState(channel=channel, recipient="get_weather", text="")]
delta_message, tools_streamed = extract_harmony_streaming_delta(
harmony_parser=parser,
token_states=token_states,
prev_recipient=None,
include_reasoning=False,
)
assert delta_message is not None
assert len(delta_message.tool_calls) == 1
tool_call = delta_message.tool_calls[0]
assert tool_call.id == "call_bare123"
assert tool_call.type == "function"
assert tool_call.function.name == "get_weather"
assert tool_call.function.arguments == ""
assert tool_call.index == 0
assert tools_streamed is True
@pytest.mark.parametrize("channel", ["commentary", "analysis"])
def test_tool_call_argument_streaming_without_functions_prefix(self, channel):
parser = MockStreamableParser()
args_text = '{"location": "Paris"}'
token_states = [
TokenState(channel=channel, recipient="get_weather", text=args_text)
]
delta_message, tools_streamed = extract_harmony_streaming_delta(
harmony_parser=parser,
token_states=token_states,
prev_recipient="get_weather",
include_reasoning=False,
)
assert delta_message is not None
tool_call = delta_message.tool_calls[0]
assert tool_call.id is None
assert tool_call.function.arguments == args_text
assert tool_call.index == 0
assert tools_streamed is True
def test_tool_call_index_from_previous_messages_without_functions_prefix(self):
messages = [
MockMessage(channel="commentary", recipient="tool1"),
]
parser = MockStreamableParser(messages=messages)
token_states = [
TokenState(channel="commentary", recipient="tool2", text="args")
]
delta_message, _ = extract_harmony_streaming_delta(
harmony_parser=parser,
token_states=token_states,
prev_recipient="tool2",
include_reasoning=False,
)
assert delta_message.tool_calls[0].index == 1
@pytest.mark.parametrize("channel", ["commentary", "analysis"])
@patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
def test_new_tool_call_dotted_function_name(self, mock_make_tool_call_id, channel):
mock_make_tool_call_id.return_value = "call_dotted123"
parser = MockStreamableParser()
token_states = [TokenState(channel=channel, recipient="math.sum", text="")]
delta_message, tools_streamed = extract_harmony_streaming_delta(
harmony_parser=parser,
token_states=token_states,
prev_recipient=None,
include_reasoning=False,
)
assert delta_message is not None
assert len(delta_message.tool_calls) == 1
tool_call = delta_message.tool_calls[0]
assert tool_call.id == "call_dotted123"
assert tool_call.type == "function"
assert tool_call.function.name == "math.sum"
assert tool_call.function.arguments == ""
assert tool_call.index == 0
assert tools_streamed is True
@pytest.mark.parametrize(
"channel,recipient",
[
(None, None),
("unknown_channel", None),
("commentary", "browser.search"),
("commentary", "assistant"),
],
)
def test_returns_none_for_invalid_inputs(self, channel, recipient):
@@ -348,3 +443,92 @@ class TestExtractHarmonyStreamingDelta:
assert tool_c_args.function.arguments == '{"key_c": "val_c"}'
assert delta_message.content == "Thinking... Thinking again..."
class TestToolCallsOnNonStandardChannels:
"""Tool calls are detected by recipient, not channel.
Models sometimes emit tool calls on unexpected channels (e.g. ``comment``
instead of ``commentary``). These tests verify that the streaming delta
extraction is channel-agnostic for tool call detection.
"""
@patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
def test_prefixed_tool_call_on_comment_channel(self, mock_make_tool_call_id):
mock_make_tool_call_id.return_value = "call_comment_chan"
parser = MockStreamableParser()
token_states = [
TokenState(channel="comment", recipient="functions.get_weather", text="")
]
delta_message, tools_streamed = extract_harmony_streaming_delta(
harmony_parser=parser,
token_states=token_states,
prev_recipient=None,
include_reasoning=False,
)
assert delta_message is not None
assert len(delta_message.tool_calls) == 1
assert delta_message.tool_calls[0].function.name == "get_weather"
assert tools_streamed is True
@patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
def test_bare_tool_call_on_comment_channel(self, mock_make_tool_call_id):
mock_make_tool_call_id.return_value = "call_bare_comment"
parser = MockStreamableParser()
token_states = [TokenState(channel="comment", recipient="get_weather", text="")]
delta_message, tools_streamed = extract_harmony_streaming_delta(
harmony_parser=parser,
token_states=token_states,
prev_recipient=None,
include_reasoning=False,
)
assert delta_message is not None
assert len(delta_message.tool_calls) == 1
assert delta_message.tool_calls[0].function.name == "get_weather"
assert tools_streamed is True
def test_tool_call_arguments_on_comment_channel(self):
parser = MockStreamableParser()
args_text = '{"location": "Paris"}'
token_states = [
TokenState(
channel="comment", recipient="functions.get_weather", text=args_text
)
]
delta_message, tools_streamed = extract_harmony_streaming_delta(
harmony_parser=parser,
token_states=token_states,
prev_recipient="functions.get_weather",
include_reasoning=False,
)
assert delta_message is not None
assert delta_message.tool_calls[0].function.arguments == args_text
assert tools_streamed is True
def test_base_index_counts_tool_calls_on_comment_channel(self):
messages = [
MockMessage(channel="comment", recipient="functions.tool1"),
]
parser = MockStreamableParser(messages=messages)
token_states = [
TokenState(channel="commentary", recipient="functions.tool2", text="args")
]
delta_message, _ = extract_harmony_streaming_delta(
harmony_parser=parser,
token_states=token_states,
prev_recipient="functions.tool2",
include_reasoning=False,
)
assert delta_message.tool_calls[0].index == 1
@@ -7,9 +7,11 @@ from openai_harmony import Message, Role
from tests.entrypoints.openai.utils import verify_harmony_messages
from vllm.entrypoints.openai.parser.harmony_utils import (
auto_drop_analysis_messages,
extract_function_from_recipient,
get_encoding,
get_system_message,
has_custom_tools,
is_function_recipient,
parse_chat_input_to_harmony_message,
parse_chat_output,
)
@@ -19,6 +21,182 @@ from vllm.entrypoints.openai.responses.harmony import (
)
class TestIsFunctionRecipient:
@pytest.mark.parametrize(
"recipient",
[
"functions.get_weather",
"functions.search_web",
"functions.math.sum",
],
)
def test_functions_prefix_accepted(self, recipient):
assert is_function_recipient(recipient) is True
@pytest.mark.parametrize(
"recipient",
[
"get_weather",
"search_web",
"calculator",
"my-tool",
],
)
def test_bare_function_name_accepted(self, recipient):
assert is_function_recipient(recipient) is True
@pytest.mark.parametrize(
"recipient",
[
"assistant",
],
)
def test_assistant_rejected(self, recipient):
assert is_function_recipient(recipient) is False
@pytest.mark.parametrize(
"recipient",
[
"math.sum",
"code.run",
"namespace.tool_name",
"my.deeply.nested.tool",
],
)
def test_dotted_function_names_accepted(self, recipient):
assert is_function_recipient(recipient) is True
@pytest.mark.parametrize(
"recipient",
[
"python",
"browser",
"container",
],
)
def test_builtin_tool_names_rejected(self, recipient):
assert is_function_recipient(recipient) is False
@pytest.mark.parametrize(
"recipient",
[
"python.run",
"python.execute",
"browser.search",
"browser.open",
"container.exec",
],
)
def test_builtin_dotted_variants_rejected(self, recipient):
assert is_function_recipient(recipient) is False
@pytest.mark.parametrize(
"recipient",
[
"",
"functions.",
],
)
def test_empty_recipients_rejected(self, recipient):
assert is_function_recipient(recipient) is False
@pytest.mark.parametrize(
"recipient",
[
"<|start|>",
"<|end|>",
"<|channel|>",
],
)
def test_harmony_tokens_rejected(self, recipient):
assert is_function_recipient(recipient) is False
class TestIsFunctionRecipientWithAllowedNames:
"""Tests for is_function_recipient with allowed_function_tool_names."""
def test_prefixed_always_accepted(self):
"""functions. prefix is always accepted regardless of allowed names."""
fn_names = frozenset({"other_tool"})
assert is_function_recipient("functions.get_weather", fn_names) is True
def test_bare_name_accepted_when_in_allowed_names(self):
fn_names = frozenset({"get_weather", "search_web"})
assert is_function_recipient("get_weather", fn_names) is True
assert is_function_recipient("search_web", fn_names) is True
def test_bare_name_rejected_when_not_in_allowed_names(self):
fn_names = frozenset({"get_weather"})
assert is_function_recipient("unknown_tool", fn_names) is False
def test_dotted_name_accepted_when_in_allowed_names(self):
fn_names = frozenset({"math.sum", "namespace.tool_name"})
assert is_function_recipient("math.sum", fn_names) is True
assert is_function_recipient("namespace.tool_name", fn_names) is True
def test_dotted_name_rejected_when_not_in_allowed_names(self):
fn_names = frozenset({"get_weather"})
assert is_function_recipient("custom_server.search", fn_names) is False
def test_empty_allowed_names_rejects_bare_names(self):
"""Empty frozenset means no function tools — bare names are not functions."""
fn_names: frozenset[str] = frozenset()
assert is_function_recipient("get_weather", fn_names) is False
assert is_function_recipient("math.sum", fn_names) is False
def test_builtin_tools_always_rejected(self):
fn_names = frozenset({"python", "browser", "container"})
assert is_function_recipient("python", fn_names) is False
assert is_function_recipient("browser", fn_names) is False
assert is_function_recipient("container", fn_names) is False
def test_builtin_dotted_always_rejected(self):
fn_names = frozenset({"python.run", "browser.search"})
assert is_function_recipient("python.run", fn_names) is False
assert is_function_recipient("browser.search", fn_names) is False
def test_none_allowed_names_uses_heuristic(self):
"""When allowed names is None (Chat Completions), use heuristic."""
assert is_function_recipient("get_weather", None) is True
assert is_function_recipient("math.sum", None) is True
assert is_function_recipient("python", None) is False
class TestExtractFunctionFromRecipient:
@pytest.mark.parametrize(
"recipient,expected",
[
("functions.get_weather", "get_weather"),
("functions.search_web", "search_web"),
("functions.", ""),
],
)
def test_strips_functions_prefix(self, recipient, expected):
assert extract_function_from_recipient(recipient) == expected
@pytest.mark.parametrize(
"recipient",
[
"get_weather",
"calculator",
"my-tool",
],
)
def test_bare_name_returned_as_is(self, recipient):
assert extract_function_from_recipient(recipient) == recipient
@pytest.mark.parametrize(
"recipient,expected",
[
("functions.math.sum", "math.sum"),
("math.sum", "math.sum"),
("namespace.tool_name", "namespace.tool_name"),
],
)
def test_dotted_function_name_extraction(self, recipient, expected):
assert extract_function_from_recipient(recipient) == expected
class TestCommonParseInputToHarmonyMessage:
"""
Tests for scenarios that are common to both Chat Completion
@@ -246,7 +246,8 @@ class TestHarmonyToResponseOutput:
message = message.with_channel("commentary")
message = message.with_recipient("custom_tool")
output_items = harmony_to_response_output(message)
fn_names = frozenset({"other_tool"})
output_items = harmony_to_response_output(message, fn_names)
assert len(output_items) == 1
assert isinstance(output_items[0], McpCall)
@@ -286,13 +287,179 @@ class TestHarmonyToResponseOutput:
assert len(output_items) == 0
class TestHarmonyToResponseOutputWithFunctionToolNames:
"""Tests for bare function name handling with function_tool_names."""
def test_bare_name_creates_function_call_when_in_tool_names(self):
"""Bare function name matching a known tool creates function call."""
message = Message.from_role_and_content(
Role.ASSISTANT, '{"location": "San Francisco"}'
)
message = message.with_channel("commentary")
message = message.with_recipient("get_weather")
fn_names = frozenset({"get_weather"})
output_items = harmony_to_response_output(message, fn_names)
assert len(output_items) == 1
assert isinstance(output_items[0], ResponseFunctionToolCall)
assert output_items[0].type == "function_call"
assert output_items[0].name == "get_weather"
assert output_items[0].arguments == '{"location": "San Francisco"}'
def test_bare_name_creates_mcp_call_when_not_in_tool_names(self):
"""Bare name not matching any known tool creates MCP call."""
message = Message.from_role_and_content(Role.ASSISTANT, '{"arg": "value"}')
message = message.with_channel("commentary")
message = message.with_recipient("custom_tool")
fn_names = frozenset({"get_weather"})
output_items = harmony_to_response_output(message, fn_names)
assert len(output_items) == 1
assert isinstance(output_items[0], McpCall)
assert output_items[0].type == "mcp_call"
def test_dotted_function_name_creates_function_call(self):
"""Dotted function name in tool names creates function call."""
message = Message.from_role_and_content(Role.ASSISTANT, '{"a": 1, "b": 2}')
message = message.with_channel("commentary")
message = message.with_recipient("math.sum")
fn_names = frozenset({"math.sum"})
output_items = harmony_to_response_output(message, fn_names)
assert len(output_items) == 1
assert isinstance(output_items[0], ResponseFunctionToolCall)
assert output_items[0].name == "math.sum"
def test_empty_tool_names_defaults_to_mcp(self):
"""With empty function_tool_names, bare names become MCP calls."""
message = Message.from_role_and_content(Role.ASSISTANT, '{"arg": "value"}')
message = message.with_channel("commentary")
message = message.with_recipient("get_weather")
output_items = harmony_to_response_output(message, frozenset())
assert len(output_items) == 1
assert isinstance(output_items[0], McpCall)
def test_prefixed_name_always_function_call(self):
"""functions. prefix always creates function call even with empty tool names."""
message = Message.from_role_and_content(Role.ASSISTANT, '{"arg": "value"}')
message = message.with_channel("commentary")
message = message.with_recipient("functions.get_weather")
output_items = harmony_to_response_output(message, frozenset())
assert len(output_items) == 1
assert isinstance(output_items[0], ResponseFunctionToolCall)
assert output_items[0].name == "get_weather"
class TestParserStateWithFunctionToolNames:
"""Tests for parser_state_to_response_output with function_tool_names."""
def test_bare_name_creates_function_call(self):
from unittest.mock import Mock
parser = Mock()
parser.current_content = '{"arg": "value"}'
parser.current_role = Role.ASSISTANT
parser.current_channel = "commentary"
parser.current_recipient = "get_weather"
fn_names = frozenset({"get_weather"})
items = parser_state_to_response_output(parser, fn_names)
assert len(items) == 1
assert isinstance(items[0], ResponseFunctionToolCall)
assert items[0].name == "get_weather"
assert items[0].status == "in_progress"
def test_bare_name_creates_mcp_when_not_in_tool_names(self):
from unittest.mock import Mock
parser = Mock()
parser.current_content = '{"arg": "value"}'
parser.current_role = Role.ASSISTANT
parser.current_channel = "commentary"
parser.current_recipient = "unknown_tool"
fn_names = frozenset({"get_weather"})
items = parser_state_to_response_output(parser, fn_names)
assert len(items) == 1
assert isinstance(items[0], McpCall)
assert items[0].name == "unknown_tool"
class TestToolCallsOnNonStandardChannels:
"""Tests verifying tool calls are detected regardless of channel."""
def test_function_call_on_comment_channel(self):
message = Message.from_role_and_content(Role.ASSISTANT, '{"query": "weather"}')
message = message.with_channel("comment")
message = message.with_recipient("functions.get_weather")
output_items = harmony_to_response_output(message)
assert len(output_items) == 1
assert isinstance(output_items[0], ResponseFunctionToolCall)
assert output_items[0].type == "function_call"
assert output_items[0].name == "get_weather"
def test_bare_function_on_comment_channel(self):
message = Message.from_role_and_content(Role.ASSISTANT, '{"query": "weather"}')
message = message.with_channel("comment")
message = message.with_recipient("get_weather")
fn_names = frozenset({"get_weather"})
output_items = harmony_to_response_output(message, fn_names)
assert len(output_items) == 1
assert isinstance(output_items[0], ResponseFunctionToolCall)
assert output_items[0].name == "get_weather"
def test_parser_state_comment_channel_function(self):
from unittest.mock import Mock
parser = Mock()
parser.current_content = '{"arg": "value"}'
parser.current_role = Role.ASSISTANT
parser.current_channel = "comment"
parser.current_recipient = "functions.get_weather"
items = parser_state_to_response_output(parser)
assert len(items) == 1
assert isinstance(items[0], ResponseFunctionToolCall)
assert items[0].name == "get_weather"
def test_parser_state_comment_channel_mcp(self):
from unittest.mock import Mock
parser = Mock()
parser.current_content = '{"arg": "value"}'
parser.current_role = Role.ASSISTANT
parser.current_channel = "comment"
parser.current_recipient = "mcp.server.tool"
fn_names: frozenset[str] = frozenset()
items = parser_state_to_response_output(parser, fn_names)
assert len(items) == 1
assert isinstance(items[0], McpCall)
def test_parse_mcp_call_basic() -> None:
"""Test that MCP calls are parsed with correct type and server_label."""
message = Message.from_role_and_content(Role.ASSISTANT, '{"path": "/tmp"}')
message = message.with_recipient("filesystem")
message = message.with_channel("commentary")
output_items = harmony_to_response_output(message)
fn_names: frozenset[str] = frozenset()
output_items = harmony_to_response_output(message, fn_names)
assert len(output_items) == 1
assert isinstance(output_items[0], McpCall)
@@ -309,7 +476,8 @@ def test_parse_mcp_call_dotted_recipient() -> None:
message = message.with_recipient("repo_browser.list")
message = message.with_channel("commentary")
output_items = harmony_to_response_output(message)
fn_names: frozenset[str] = frozenset()
output_items = harmony_to_response_output(message, fn_names)
assert len(output_items) == 1
assert isinstance(output_items[0], McpCall)
@@ -371,7 +539,8 @@ def test_parser_state_to_response_output_commentary_channel() -> None:
parser_mcp.current_channel = "commentary"
parser_mcp.current_recipient = "filesystem"
mcp_items = parser_state_to_response_output(parser_mcp)
fn_names: frozenset[str] = frozenset()
mcp_items = parser_state_to_response_output(parser_mcp, fn_names)
assert len(mcp_items) == 1
assert isinstance(mcp_items[0], McpCall)
@@ -438,7 +607,8 @@ def test_parser_state_to_response_output_analysis_channel() -> None:
parser_mcp.current_channel = "analysis"
parser_mcp.current_recipient = "database"
mcp_items = parser_state_to_response_output(parser_mcp)
fn_names: frozenset[str] = frozenset()
mcp_items = parser_state_to_response_output(parser_mcp, fn_names)
assert len(mcp_items) == 1
assert isinstance(mcp_items[0], McpCall)