[Refactor] Chat Completions Harmony Refactor, non-streaming path. (#45171)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-06-11 21:18:30 +00:00
committed by GitHub
parent 5a6c7b7ab5
commit f712fd0d7d
16 changed files with 822 additions and 816 deletions
@@ -38,12 +38,12 @@ from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.exceptions import VLLMValidationError
from vllm.inputs import TokensPrompt
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.parser import HarmonyParser
from vllm.renderers.hf import HfRenderer
from vllm.renderers.mistral import MistralRenderer
from vllm.tokenizers import get_tokenizer
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.tool_parsers import ToolParserManager
from vllm.v1.engine.async_llm import AsyncLLM
GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b"
@@ -575,7 +575,13 @@ def _build_serving_render(
)
def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
def _build_serving_chat(
engine: AsyncLLM,
*,
reasoning_parser: str = "",
tool_parser: str | None = None,
enable_auto_tools: bool = False,
) -> OpenAIServingChat:
models = OpenAIServingModels(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
@@ -590,6 +596,9 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
request_logger=None,
reasoning_parser=reasoning_parser,
tool_parser=tool_parser,
enable_auto_tools=enable_auto_tools,
)
return serving_chat
@@ -637,7 +646,7 @@ async def test_serving_chat_returns_correct_model_name():
serving_chat = _build_serving_chat(mock_engine)
messages = [{"role": "user", "content": "what is 1+1?"}]
async def return_model_name(*args):
async def return_model_name(*args, **kwargs):
return args[3]
serving_chat.chat_completion_full_generator = return_model_name
@@ -1210,15 +1219,21 @@ class TestServingChatWithHarmony:
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
mock_engine.model_config.hf_text_config = MockHFConfig(model_type="gpt_oss")
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
return mock_engine
@pytest.fixture()
def serving_chat(self, mock_engine) -> OpenAIServingChat:
chat = _build_serving_chat(mock_engine)
chat.use_harmony = True
chat.tool_parser = ToolParserManager.get_tool_parser("openai")
chat = _build_serving_chat(
mock_engine,
reasoning_parser="openai_gptoss",
tool_parser="openai",
enable_auto_tools=True,
)
assert chat.parser_cls is HarmonyParser
return chat
def mock_request_output_from_req_and_token_ids(
@@ -1277,6 +1292,7 @@ class TestServingChatWithHarmony:
stream: bool = False,
) -> ChatCompletionResponse:
harmony_token_ids = get_encoding().encode(harmony_str, allowed_special="all")
tokenizer = get_tokenizer(GPT_OSS_MODEL_NAME)
async def result_generator():
if stream:
@@ -1304,11 +1320,12 @@ class TestServingChatWithHarmony:
request_id=req.request_id,
model_name=req.model,
conversation=[],
tokenizer=get_tokenizer(req.model),
tokenizer=tokenizer,
request_metadata=RequestResponseMetadata(
request_id=req.request_id,
model_name=req.model,
),
chat_template_kwargs=serving_chat._effective_chat_template_kwargs(req),
)
if stream:
@@ -1316,11 +1333,18 @@ class TestServingChatWithHarmony:
return await result
@pytest.mark.asyncio
async def test_simple_chat(self, serving_chat, stream):
@pytest.mark.parametrize(
"include_reasoning", [True, False], ids=["with_reasoning", "no_reasoning"]
)
async def test_simple_chat(self, serving_chat, stream, include_reasoning):
messages = [{"role": "user", "content": "what is 1+1?"}]
# Test the Harmony messages for the first turn's input
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
req = ChatCompletionRequest(
model=MODEL_NAME,
messages=messages,
include_reasoning=include_reasoning,
)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
@@ -1342,7 +1366,11 @@ class TestServingChatWithHarmony:
response = await self.generate_response_from_harmony_str(
serving_chat, req, response_str, stream=stream
)
verify_chat_response(response, content=final_str, reasoning=reasoning_str)
verify_chat_response(
response,
content=final_str,
reasoning=reasoning_str if include_reasoning else None,
)
# Add the output messages from the first turn as input to the second turn
for choice in response.choices:
@@ -11,12 +11,10 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
auto_drop_analysis_messages,
create_tool_definition,
extract_function_from_recipient,
get_encoding,
get_system_message,
has_custom_tools,
is_function_recipient,
parse_chat_input_to_harmony_message,
parse_chat_output,
)
from vllm.entrypoints.openai.responses.harmony import (
response_input_to_harmony,
@@ -941,110 +939,6 @@ class TestAutoDropAnalysisMessages:
assert cleaned_messages == messages[1:]
class TestParseChatOutput:
def test_parse_chat_output_interrupted_first_message(self) -> None:
harmony_str = "<|channel|>final<|message|>I'm in the middle of answering"
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "I'm in the middle of answering"
def test_parse_chat_output_interrupted_reasoning_first_message(self) -> None:
harmony_str = "<|channel|>analysis<|message|>I'm in the middle of thinking"
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning == "I'm in the middle of thinking"
assert final_content is None
def test_parse_chat_output_complete_reasoning_interrupted_content(self) -> None:
harmony_str = (
"<|channel|>analysis<|message|>I'm thinking.<|end|>"
"<|start|>assistant<|channel|>final"
"<|message|>I'm in the middle of answering"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning == "I'm thinking."
assert final_content == "I'm in the middle of answering"
def test_parse_chat_output_complete_content(self) -> None:
harmony_str = "<|channel|>final<|message|>The answer is 4.<|end|>"
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "The answer is 4."
def test_parse_chat_output_complete_commentary(self) -> None:
harmony_str = (
"<|channel|>commentary<|message|>I need to call some tools.<|end|>"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "I need to call some tools."
def test_parse_chat_output_complete_reasoning(self) -> None:
harmony_str = (
"<|channel|>analysis<|message|>I've thought hard about this.<|end|>"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning == "I've thought hard about this."
assert final_content is None
def test_parse_chat_output_complete_reasoning_and_content(self) -> None:
harmony_str = (
"<|channel|>analysis<|message|>I've thought hard about this.<|end|>"
"<|start|>assistant<|channel|>final<|message|>The answer is 4.<|end|>"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning == "I've thought hard about this."
assert final_content == "The answer is 4."
def test_parse_chat_output_commentary_with_recipient_excluded(self) -> None:
"""Commentary with a recipient (tool call) should not appear in
final_content — those are handled separately by the tool parser.
The first message is a preamble (visible), the second is a tool
call (excluded). Only the preamble should appear in final_content.
"""
harmony_str = (
"<|channel|>commentary"
"<|message|>Let me check the weather.<|end|>"
"<|start|>assistant to=functions.get_weather"
"<|channel|>commentary"
'<|message|>{"location": "SF"}<|end|>'
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "Let me check the weather."
def test_parse_chat_output_interrupted_preamble(self) -> None:
"""Partial/interrupted preamble (commentary without recipient) should
appear in final_content, not reasoning."""
harmony_str = "<|channel|>commentary<|message|>I'll search for that"
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "I'll search for that"
def test_parse_chat_output_preamble_then_final(self) -> None:
"""Preamble followed by a final message should both appear in
final_content, joined by newline."""
harmony_str = (
"<|channel|>commentary"
"<|message|>Let me look that up.<|end|>"
"<|start|>assistant<|channel|>final"
"<|message|>The answer is 42.<|end|>"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "Let me look that up.\nThe answer is 42."
def test_has_custom_tools() -> None:
assert not has_custom_tools(set())
assert not has_custom_tools({"web_search_preview", "code_interpreter", "container"})
+452
View File
@@ -0,0 +1,452 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import Sequence
import pytest
from openai_harmony import (
Conversation,
Message,
RenderConversationConfig,
Role,
)
from transformers import AutoTokenizer
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.engine.protocol import FunctionCall
from vllm.entrypoints.openai.parser.harmony_utils import (
get_encoding,
)
from vllm.parser.harmony import HarmonyParser
from vllm.parser.parser_manager import ParserManager
REASONING_MODEL_NAME = "openai/gpt-oss-20b"
@pytest.fixture(scope="module")
def gpt_oss_tokenizer():
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
@pytest.fixture
def harmony_parser(gpt_oss_tokenizer):
parser_cls = ParserManager.get_parser(
tool_parser_name="openai",
reasoning_parser_name="openai_gptoss",
enable_auto_tools=True,
model_name=REASONING_MODEL_NAME,
is_harmony=True,
)
assert parser_cls is HarmonyParser
return parser_cls(gpt_oss_tokenizer)
@pytest.fixture
def chat_request():
return ChatCompletionRequest(
model="openai/gpt-oss-20b",
messages=[{"role": "user", "content": "Hello"}],
)
def encode_output(harmony_str: str) -> list[int]:
return get_encoding().encode(harmony_str, allowed_special="all")
def assistant(content: str, channel: str) -> Message:
return Message.from_role_and_content(Role.ASSISTANT, content).with_channel(channel)
def tool_call(
recipient: str,
content: str,
channel: str = "commentary",
content_type: str | None = "json",
) -> Message:
message = assistant(content, channel).with_recipient(recipient)
return message if content_type is None else message.with_content_type(content_type)
def get_model_output_tokens(
prompt_messages: Sequence[Message],
response_messages: Sequence[Message],
) -> list[int]:
enc = get_encoding()
# Keep analysis messages when synthesizing model-output-only token sequences
# for parser tests; the default render path drops them after a later final turn.
config = RenderConversationConfig(auto_drop_analysis=False)
prompt_ids = enc.render_conversation_for_completion(
Conversation.from_messages(list(prompt_messages)),
Role.ASSISTANT,
config=config,
)
full_ids = enc.render_conversation_for_completion(
Conversation.from_messages([*prompt_messages, *response_messages]),
Role.ASSISTANT,
config=config,
)
assert full_ids[: len(prompt_ids)] == prompt_ids
return full_ids[len(prompt_ids) :]
def get_text(msg: Message) -> str:
return msg.content[0].text if msg.content else ""
def visible_segments(result) -> list[tuple[str | None, str | None, str]]:
return [
(segment.channel, segment.recipient, segment.delta)
for segment in result.segments
if not segment.is_boundary and segment.delta
]
def tool_call_tuples(tool_calls: list[FunctionCall] | None) -> list[tuple[str, str]]:
return [] if tool_calls is None else [(tc.name, tc.arguments) for tc in tool_calls]
class TestParse:
# Rendered conversation outputs.
def test_reasoning_only(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Why?")]
response = [assistant("This is reasoning", "analysis")]
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert reasoning == "This is reasoning"
assert content is None
assert tool_calls is None
def test_content_only(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Hello")]
response = [assistant("This is a test", "final")]
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert reasoning is None
assert content == "This is a test"
assert tool_calls is None
def test_reasoning_and_content(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "What is 2+2?")]
response = [
assistant("I should think first.", "analysis"),
assistant("The answer is 4.", "final"),
]
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert reasoning == "I should think first."
assert content == "The answer is 4."
assert tool_calls is None
@pytest.mark.parametrize(
"tool_args",
[
'{"location": "Tokyo"}',
'{\n"location": "Tokyo"\n}',
],
)
@pytest.mark.parametrize("tool_channel", ["commentary", "analysis"])
def test_single_tool_call(
self, harmony_parser, chat_request, tool_args, tool_channel
):
prompt = [
Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?")
]
response = [tool_call("functions.get_current_weather", tool_args, tool_channel)]
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert reasoning is None
assert content is None
assert tool_call_tuples(tool_calls) == [
("get_current_weather", json.dumps({"location": "Tokyo"}))
]
def test_multiple_tool_calls_varied_formats(self, harmony_parser, chat_request):
prompt = [
Message.from_role_and_content(
Role.USER, "What is the weather in Tokyo based on where I'm at?"
)
]
response = [
tool_call("functions.get_current_weather", '{"location": "Tokyo"}'),
tool_call("functions.get_user_location", '{"location": "Tokyo"}'),
tool_call(
"functions.no_content_type",
'{"location": "Tokyo"}',
content_type=None,
),
tool_call("functions.not_json_no_content_type", "foo", content_type=None),
tool_call("functions.empty_args", "{}"),
tool_call("functions.no_args", ""),
]
_, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert content is None
assert tool_call_tuples(tool_calls) == [
("get_current_weather", json.dumps({"location": "Tokyo"})),
("get_user_location", json.dumps({"location": "Tokyo"})),
("no_content_type", json.dumps({"location": "Tokyo"})),
("not_json_no_content_type", "foo"),
("empty_args", json.dumps({})),
("no_args", ""),
]
def test_tool_call_bare_recipient(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Weather?")]
response = [tool_call("get_current_weather", '{"location": "Tokyo"}')]
_, _, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert tool_call_tuples(tool_calls) == [
("get_current_weather", json.dumps({"location": "Tokyo"}))
]
def test_multiple_tool_calls_bare_recipients(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Use both tools.")]
response = [
tool_call("get_current_weather", '{"location": "Tokyo"}'),
tool_call("get_user_location", "{}"),
]
_, _, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert tool_call_tuples(tool_calls) == [
("get_current_weather", json.dumps({"location": "Tokyo"})),
("get_user_location", json.dumps({})),
]
def test_assistant_recipient_not_tool(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Hello")]
response = [
tool_call("assistant", "Some tool response", content_type=None),
assistant("Here is the answer", "final"),
]
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert reasoning is None
assert content == "Here is the answer"
assert tool_calls is None
def test_tool_call_dotted_name(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Compute 2+3")]
response = [tool_call("math.sum", '{"a": 2, "b": 3}')]
_, _, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert tool_call_tuples(tool_calls) == [
("math.sum", json.dumps({"a": 2, "b": 3}))
]
def test_tool_calls_with_final_content(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "What is the weather?")]
response = [
assistant("User asked about the weather.", "analysis"),
tool_call("functions.get_current_weather", '{"location": "Tokyo"}'),
assistant("This tool call will get the weather.", "final"),
]
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response),
)
assert reasoning == "User asked about the weather."
assert content == "This tool call will get the weather."
assert tool_call_tuples(tool_calls) == [
("get_current_weather", json.dumps({"location": "Tokyo"}))
]
# Raw/truncated Harmony output streams.
def test_interrupted_first_message(self, harmony_parser, chat_request):
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=encode_output(
"<|channel|>final<|message|>I'm in the middle of answering"
),
)
assert reasoning is None
assert content == "I'm in the middle of answering"
assert tool_calls is None
def test_interrupted_reasoning_first_message(self, harmony_parser, chat_request):
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=encode_output(
"<|channel|>analysis<|message|>I'm in the middle of thinking"
),
)
assert reasoning == "I'm in the middle of thinking"
assert content is None
assert tool_calls is None
def test_truncated_output(self, harmony_parser, chat_request):
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=encode_output(
"<|channel|>analysis<|message|>I'm thinking.<|end|>"
"<|start|>assistant<|channel|>final<|message|>"
"I'm in the middle of answering"
),
)
assert reasoning == "I'm thinking."
assert content == "I'm in the middle of answering"
assert tool_calls is None
@pytest.mark.parametrize(
("harmony_str", "expected_content"),
[
(
"<|channel|>commentary<|message|>I'll search for that",
"I'll search for that",
),
(
"<|channel|>commentary<|message|>Let me look that up.<|end|>"
"<|start|>assistant<|channel|>final<|message|>The answer is 42.<|end|>",
"Let me look that up.\nThe answer is 42.",
),
],
)
def test_commentary_preambles(
self,
harmony_parser,
chat_request,
harmony_str,
expected_content,
):
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=encode_output(harmony_str),
)
assert reasoning is None
assert content == expected_content
assert tool_calls is None
def test_commentary_with_recipient_excluded(self, harmony_parser, chat_request):
reasoning, content, tool_calls = harmony_parser.parse(
"",
chat_request,
model_output_token_ids=encode_output(
"<|channel|>commentary"
"<|message|>Let me check the weather.<|end|>"
"<|start|>assistant to=functions.get_weather"
"<|channel|>commentary"
'<|message|>{"location": "SF"}<|end|>'
),
)
assert reasoning is None
assert content == "Let me check the weather."
assert tool_call_tuples(tool_calls) == [
("get_weather", json.dumps({"location": "SF"}))
]
class TestProcessChunk:
def test_empty(self, harmony_parser):
result = harmony_parser.process_chunk([])
assert result.segments == []
assert result.reasoning_token_count == 0
def test_single_channel(self, harmony_parser):
result = harmony_parser.process_chunk(
encode_output("<|channel|>final<|message|>Hello")
)
assert visible_segments(result) == [("final", None, "Hello")]
def test_cross_channel(self, harmony_parser):
result = harmony_parser.process_chunk(
encode_output(
"<|channel|>analysis<|message|>Think<|end|>"
"<|start|>assistant<|channel|>final<|message|>Answer"
)
)
assert visible_segments(result) == [
("analysis", None, "Think"),
("final", None, "Answer"),
]
def test_boundary_detection(self, harmony_parser):
result = harmony_parser.process_chunk(
encode_output("<|channel|>final<|message|>Done<|end|>")
)
boundary_segments = [
segment for segment in result.segments if segment.is_boundary
]
assert len(boundary_segments) == 1
assert boundary_segments[0].completed_message is not None
assert boundary_segments[0].completed_message.channel == "final"
assert get_text(boundary_segments[0].completed_message) == "Done"
def test_multi_boundary(self, harmony_parser):
result = harmony_parser.process_chunk(
encode_output(
"<|channel|>analysis<|message|>One<|end|>"
"<|start|>assistant<|channel|>final<|message|>Two<|end|>"
)
)
boundary_segments = [
segment for segment in result.segments if segment.is_boundary
]
assert [
get_text(segment.completed_message) for segment in boundary_segments
] == [
"One",
"Two",
]
@@ -1,415 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from openai_harmony import (
Conversation,
DeveloperContent,
HarmonyEncodingName,
Message,
Role,
SystemContent,
load_harmony_encoding,
)
from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
from vllm.tokenizers import get_tokenizer
from vllm.tool_parsers.openai_tool_parser import OpenAIToolParser
MODEL = "gpt2"
@pytest.fixture(scope="module")
def openai_tokenizer():
# The parser does not use the tokenizer, but the constructor requires it.
return get_tokenizer(MODEL)
@pytest.fixture
def openai_tool_parser(openai_tokenizer):
return OpenAIToolParser(openai_tokenizer)
@pytest.fixture(scope="module")
def harmony_encoding():
return load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
def assert_tool_calls(
actual_tool_calls: list[ToolCall],
expected_tool_calls: list[ToolCall],
):
assert len(actual_tool_calls) == len(expected_tool_calls)
for actual_tool_call, expected_tool_call in zip(
actual_tool_calls, expected_tool_calls
):
assert isinstance(actual_tool_call.id, str)
assert len(actual_tool_call.id) > 16 # Default from protocol.py
assert actual_tool_call.type == "function"
assert actual_tool_call.function == expected_tool_call.function
def test_extract_tool_calls_no_tools(openai_tool_parser, harmony_encoding):
convo = Conversation.from_messages(
[
Message.from_role_and_content(
Role.SYSTEM,
SystemContent.new(),
),
Message.from_role_and_content(
Role.DEVELOPER,
DeveloperContent.new().with_instructions("Talk like a pirate!"),
),
Message.from_role_and_content(Role.USER, "Arrr, how be you?"),
Message.from_role_and_content(
Role.ASSISTANT, "This is a test"
).with_channel("final"),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo, Role.ASSISTANT
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert not extracted_info.tools_called
assert extracted_info.tool_calls == []
assert extracted_info.content == "This is a test"
@pytest.mark.parametrize(
"tool_args",
[
'{"location": "Tokyo"}',
'{\n"location": "Tokyo"\n}',
],
)
def test_extract_tool_calls_single_tool(
openai_tool_parser, harmony_encoding, tool_args
):
convo = Conversation.from_messages(
[
Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?"),
Message.from_role_and_content(
Role.ASSISTANT,
'User asks: "What is the weather in Tokyo?" We need to use get_current_weather tool.', # noqa: E501
).with_channel("analysis"),
Message.from_role_and_content(Role.ASSISTANT, tool_args)
.with_channel("commentary")
.with_recipient("functions.get_current_weather")
.with_content_type("json"),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo, Role.ASSISTANT
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert extracted_info.tools_called
expected_tool_calls = [
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps({"location": "Tokyo"}),
)
)
]
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
assert extracted_info.content is None
def test_extract_tool_calls_multiple_tools(
openai_tool_parser,
harmony_encoding,
):
convo = Conversation.from_messages(
[
Message.from_role_and_content(
Role.USER, "What is the weather in Tokyo based on where I'm at?"
),
Message.from_role_and_content(
Role.ASSISTANT,
'User asks: "What is the weather in Tokyo?" based on their location. We need to use get_current_weather tool and get_user_location tool.', # noqa: E501
).with_channel("analysis"),
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
.with_channel("commentary")
.with_recipient("functions.get_current_weather")
.with_content_type("json"),
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
.with_channel("commentary")
.with_recipient("functions.get_user_location")
.with_content_type("json"),
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
.with_channel("commentary")
.with_recipient("functions.no_content_type"),
Message.from_role_and_content(Role.ASSISTANT, "foo")
.with_channel("commentary")
.with_recipient("functions.not_json_no_content_type"),
Message.from_role_and_content(Role.ASSISTANT, "{}")
.with_channel("commentary")
.with_recipient("functions.empty_args")
.with_content_type("json"),
Message.from_role_and_content(Role.ASSISTANT, "")
.with_channel("commentary")
.with_recipient("functions.no_args")
.with_content_type("json"),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo,
Role.ASSISTANT,
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert extracted_info.tools_called
expected_tool_calls = [
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps({"location": "Tokyo"}),
)
),
ToolCall(
function=FunctionCall(
name="get_user_location",
arguments=json.dumps({"location": "Tokyo"}),
)
),
ToolCall(
function=FunctionCall(
name="no_content_type",
arguments=json.dumps({"location": "Tokyo"}),
)
),
ToolCall(
function=FunctionCall(
name="not_json_no_content_type",
arguments="foo",
)
),
ToolCall(
function=FunctionCall(
name="empty_args",
arguments=json.dumps({}),
)
),
ToolCall(
function=FunctionCall(
name="no_args",
arguments="",
)
),
]
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
assert extracted_info.content is None
def test_extract_tool_calls_bare_function_name(
openai_tool_parser,
harmony_encoding,
):
convo = Conversation.from_messages(
[
Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?"),
Message.from_role_and_content(
Role.ASSISTANT,
"We need to use get_current_weather tool.",
).with_channel("analysis"),
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
.with_channel("commentary")
.with_recipient("get_current_weather")
.with_content_type("json"),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo, Role.ASSISTANT
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert extracted_info.tools_called
expected_tool_calls = [
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps({"location": "Tokyo"}),
)
)
]
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
assert extracted_info.content is None
def test_extract_tool_calls_bare_function_name_multiple(
openai_tool_parser,
harmony_encoding,
):
convo = Conversation.from_messages(
[
Message.from_role_and_content(
Role.USER, "What is the weather in Tokyo based on where I'm at?"
),
Message.from_role_and_content(
Role.ASSISTANT,
"We need to use both tools.",
).with_channel("analysis"),
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
.with_channel("commentary")
.with_recipient("get_current_weather")
.with_content_type("json"),
Message.from_role_and_content(Role.ASSISTANT, "{}")
.with_channel("commentary")
.with_recipient("get_user_location")
.with_content_type("json"),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo, Role.ASSISTANT
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert extracted_info.tools_called
expected_tool_calls = [
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps({"location": "Tokyo"}),
)
),
ToolCall(
function=FunctionCall(
name="get_user_location",
arguments=json.dumps({}),
)
),
]
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
def test_extract_tool_calls_assistant_recipient_ignored(
openai_tool_parser,
harmony_encoding,
):
convo = Conversation.from_messages(
[
Message.from_role_and_content(Role.USER, "Hello"),
Message.from_role_and_content(Role.ASSISTANT, "Some tool response")
.with_channel("commentary")
.with_recipient("assistant"),
Message.from_role_and_content(
Role.ASSISTANT, "Here is the answer"
).with_channel("final"),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo, Role.ASSISTANT
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert not extracted_info.tools_called
assert extracted_info.tool_calls == []
assert extracted_info.content == "Here is the answer"
def test_extract_tool_calls_dotted_function_name(
openai_tool_parser,
harmony_encoding,
):
convo = Conversation.from_messages(
[
Message.from_role_and_content(Role.USER, "Compute 2+3"),
Message.from_role_and_content(Role.ASSISTANT, '{"a": 2, "b": 3}')
.with_channel("commentary")
.with_recipient("math.sum")
.with_content_type("json"),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo, Role.ASSISTANT
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert extracted_info.tools_called
expected_tool_calls = [
ToolCall(
function=FunctionCall(
name="math.sum",
arguments=json.dumps({"a": 2, "b": 3}),
)
)
]
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
def test_extract_tool_calls_with_content(
openai_tool_parser,
harmony_encoding,
):
final_content = "This tool call will get the weather."
convo = Conversation.from_messages(
[
Message.from_role_and_content(
Role.USER, "What is the weather in Tokyo based on where I'm at?"
),
Message.from_role_and_content(
Role.ASSISTANT,
'User asks: "What is the weather in Tokyo?" based on their location. We need to use get_current_weather tool and get_user_location tool.', # noqa: E501
).with_channel("analysis"),
Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
.with_channel("commentary")
.with_recipient("functions.get_current_weather")
.with_content_type("json"),
Message.from_role_and_content(Role.ASSISTANT, final_content).with_channel(
"final"
),
]
)
token_ids = harmony_encoding.render_conversation_for_completion(
convo,
Role.ASSISTANT,
)
extracted_info = openai_tool_parser.extract_tool_calls(
"",
request=None,
token_ids=token_ids,
)
assert extracted_info.tools_called
expected_tool_calls = [
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps({"location": "Tokyo"}),
)
),
]
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
assert extracted_info.content == final_content