[Bugfix][GPT-OSS] Fix strict tool-call grammar to accept Harmony renders (#52222)

Signed-off-by: Yifan Zong <yzong@redhat.com>
Co-authored-by: Ben Browning <56071+bbrowning@users.noreply.github.com>
This commit is contained in:
yzong-rh
2026-08-26 18:05:09 -07:00
committed by GitHub
co-authored by Ben Browning
parent d1e5e66ee3
commit 51d1b45182
2 changed files with 111 additions and 81 deletions
+87 -61
View File
@@ -89,8 +89,11 @@ def encode_output(harmony_str: str) -> list[int]:
return get_encoding().encode(harmony_str, allowed_special="all") return get_encoding().encode(harmony_str, allowed_special="all")
def assistant(content: str, channel: str) -> Message: def assistant(content: str, channel: str, content_type: str | None = None) -> Message:
return Message.from_role_and_content(Role.ASSISTANT, content).with_channel(channel) message = Message.from_role_and_content(Role.ASSISTANT, content).with_channel(
channel
)
return message if content_type is None else message.with_content_type(content_type)
def tool_call( def tool_call(
@@ -103,16 +106,14 @@ def tool_call(
return message if content_type is None else message.with_content_type(content_type) return message if content_type is None else message.with_content_type(content_type)
def get_model_output_tokens( def get_model_output_tokens(response_messages: Sequence[Message]) -> list[int]:
prompt_messages: Sequence[Message],
response_messages: Sequence[Message],
) -> list[int]:
enc = get_encoding() enc = get_encoding()
prompt_messages = [Message.from_role_and_content(Role.USER, "x")]
# Keep analysis messages when synthesizing model-output-only token sequences # Keep analysis messages when synthesizing model-output-only token sequences
# for parser tests; the default render path drops them after a later final turn. # for parser tests; the default render path drops them after a later final turn.
config = RenderConversationConfig(auto_drop_analysis=False) config = RenderConversationConfig(auto_drop_analysis=False)
prompt_ids = enc.render_conversation_for_completion( prompt_ids = enc.render_conversation_for_completion(
Conversation.from_messages(list(prompt_messages)), Conversation.from_messages(prompt_messages),
Role.ASSISTANT, Role.ASSISTANT,
config=config, config=config,
) )
@@ -124,6 +125,10 @@ def get_model_output_tokens(
return full_ids[len(prompt_ids) :] return full_ids[len(prompt_ids) :]
def get_model_output_str(response_messages: Sequence[Message]) -> str:
return get_encoding().decode_utf8(get_model_output_tokens(response_messages))
def get_text(msg: Message) -> str: def get_text(msg: Message) -> str:
return msg.content[0].text if msg.content else "" return msg.content[0].text if msg.content else ""
@@ -215,13 +220,12 @@ class TestParse:
# Rendered conversation outputs. # Rendered conversation outputs.
def test_reasoning_only(self, harmony_parser, chat_request): def test_reasoning_only(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Why?")]
response = [assistant("This is reasoning", "analysis")] response = [assistant("This is reasoning", "analysis")]
reasoning, content, tool_calls = harmony_parser.parse( reasoning, content, tool_calls = harmony_parser.parse(
"", "",
chat_request, chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response), model_output_token_ids=get_model_output_tokens(response),
) )
assert reasoning == "This is reasoning" assert reasoning == "This is reasoning"
@@ -229,13 +233,12 @@ class TestParse:
assert tool_calls is None assert tool_calls is None
def test_content_only(self, harmony_parser, chat_request): def test_content_only(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Hello")]
response = [assistant("This is a test", "final")] response = [assistant("This is a test", "final")]
reasoning, content, tool_calls = harmony_parser.parse( reasoning, content, tool_calls = harmony_parser.parse(
"", "",
chat_request, chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response), model_output_token_ids=get_model_output_tokens(response),
) )
assert reasoning is None assert reasoning is None
@@ -243,7 +246,6 @@ class TestParse:
assert tool_calls is None assert tool_calls is None
def test_reasoning_and_content(self, harmony_parser, chat_request): def test_reasoning_and_content(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "What is 2+2?")]
response = [ response = [
assistant("I should think first.", "analysis"), assistant("I should think first.", "analysis"),
assistant("The answer is 4.", "final"), assistant("The answer is 4.", "final"),
@@ -252,7 +254,7 @@ class TestParse:
reasoning, content, tool_calls = harmony_parser.parse( reasoning, content, tool_calls = harmony_parser.parse(
"", "",
chat_request, chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response), model_output_token_ids=get_model_output_tokens(response),
) )
assert reasoning == "I should think first." assert reasoning == "I should think first."
@@ -270,15 +272,12 @@ class TestParse:
def test_single_tool_call( def test_single_tool_call(
self, harmony_parser, chat_request, tool_args, tool_channel self, harmony_parser, chat_request, tool_args, tool_channel
): ):
prompt = [
Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?")
]
response = [tool_call("functions.get_current_weather", tool_args, tool_channel)] response = [tool_call("functions.get_current_weather", tool_args, tool_channel)]
reasoning, content, tool_calls = harmony_parser.parse( reasoning, content, tool_calls = harmony_parser.parse(
"", "",
chat_request, chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response), model_output_token_ids=get_model_output_tokens(response),
) )
assert reasoning is None assert reasoning is None
@@ -288,11 +287,6 @@ class TestParse:
] ]
def test_multiple_tool_calls_varied_formats(self, harmony_parser, chat_request): def test_multiple_tool_calls_varied_formats(self, harmony_parser, chat_request):
prompt = [
Message.from_role_and_content(
Role.USER, "What is the weather in Tokyo based on where I'm at?"
)
]
response = [ response = [
tool_call("functions.get_current_weather", '{"location": "Tokyo"}'), tool_call("functions.get_current_weather", '{"location": "Tokyo"}'),
tool_call("functions.get_user_location", '{"location": "Tokyo"}'), tool_call("functions.get_user_location", '{"location": "Tokyo"}'),
@@ -309,7 +303,7 @@ class TestParse:
_, content, tool_calls = harmony_parser.parse( _, content, tool_calls = harmony_parser.parse(
"", "",
chat_request, chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response), model_output_token_ids=get_model_output_tokens(response),
) )
assert content is None assert content is None
@@ -323,13 +317,12 @@ class TestParse:
] ]
def test_tool_call_bare_recipient(self, harmony_parser, chat_request): def test_tool_call_bare_recipient(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Weather?")]
response = [tool_call("get_current_weather", '{"location": "Tokyo"}')] response = [tool_call("get_current_weather", '{"location": "Tokyo"}')]
_, _, tool_calls = harmony_parser.parse( _, _, tool_calls = harmony_parser.parse(
"", "",
chat_request, chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response), model_output_token_ids=get_model_output_tokens(response),
) )
assert tool_call_tuples(tool_calls) == [ assert tool_call_tuples(tool_calls) == [
@@ -337,7 +330,6 @@ class TestParse:
] ]
def test_multiple_tool_calls_bare_recipients(self, harmony_parser, chat_request): def test_multiple_tool_calls_bare_recipients(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Use both tools.")]
response = [ response = [
tool_call("get_current_weather", '{"location": "Tokyo"}'), tool_call("get_current_weather", '{"location": "Tokyo"}'),
tool_call("get_user_location", "{}"), tool_call("get_user_location", "{}"),
@@ -346,7 +338,7 @@ class TestParse:
_, _, tool_calls = harmony_parser.parse( _, _, tool_calls = harmony_parser.parse(
"", "",
chat_request, chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response), model_output_token_ids=get_model_output_tokens(response),
) )
assert tool_call_tuples(tool_calls) == [ assert tool_call_tuples(tool_calls) == [
@@ -355,7 +347,6 @@ class TestParse:
] ]
def test_assistant_recipient_not_tool(self, harmony_parser, chat_request): def test_assistant_recipient_not_tool(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Hello")]
response = [ response = [
tool_call("assistant", "Some tool response", content_type=None), tool_call("assistant", "Some tool response", content_type=None),
assistant("Here is the answer", "final"), assistant("Here is the answer", "final"),
@@ -364,7 +355,7 @@ class TestParse:
reasoning, content, tool_calls = harmony_parser.parse( reasoning, content, tool_calls = harmony_parser.parse(
"", "",
chat_request, chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response), model_output_token_ids=get_model_output_tokens(response),
) )
assert reasoning is None assert reasoning is None
@@ -372,13 +363,12 @@ class TestParse:
assert tool_calls is None assert tool_calls is None
def test_tool_call_dotted_name(self, harmony_parser, chat_request): def test_tool_call_dotted_name(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "Compute 2+3")]
response = [tool_call("math.sum", '{"a": 2, "b": 3}')] response = [tool_call("math.sum", '{"a": 2, "b": 3}')]
_, _, tool_calls = harmony_parser.parse( _, _, tool_calls = harmony_parser.parse(
"", "",
chat_request, chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response), model_output_token_ids=get_model_output_tokens(response),
) )
assert tool_call_tuples(tool_calls) == [ assert tool_call_tuples(tool_calls) == [
@@ -386,7 +376,6 @@ class TestParse:
] ]
def test_tool_calls_with_final_content(self, harmony_parser, chat_request): def test_tool_calls_with_final_content(self, harmony_parser, chat_request):
prompt = [Message.from_role_and_content(Role.USER, "What is the weather?")]
response = [ response = [
assistant("User asked about the weather.", "analysis"), assistant("User asked about the weather.", "analysis"),
tool_call("functions.get_current_weather", '{"location": "Tokyo"}'), tool_call("functions.get_current_weather", '{"location": "Tokyo"}'),
@@ -396,7 +385,7 @@ class TestParse:
reasoning, content, tool_calls = harmony_parser.parse( reasoning, content, tool_calls = harmony_parser.parse(
"", "",
chat_request, chat_request,
model_output_token_ids=get_model_output_tokens(prompt, response), model_output_token_ids=get_model_output_tokens(response),
) )
assert reasoning == "User asked about the weather." assert reasoning == "User asked about the weather."
@@ -700,12 +689,11 @@ class TestParseDelta:
recipient, recipient,
): ):
parser = HarmonyParser(gpt_oss_tokenizer) parser = HarmonyParser(gpt_oss_tokenizer)
prompt = [Message.from_role_and_content(Role.USER, "Hello")]
response = [tool_call(recipient, "Ignore this", content_type=None)] response = [tool_call(recipient, "Ignore this", content_type=None)]
delta = parser.parse_delta( delta = parser.parse_delta(
delta_text="", delta_text="",
delta_token_ids=get_model_output_tokens(prompt, response), delta_token_ids=get_model_output_tokens(response),
request=chat_request, request=chat_request,
finished=False, finished=False,
) )
@@ -914,35 +902,61 @@ class TestAdjustRequest:
"required": ["answer"], "required": ["answer"],
} }
ANALYSIS = "<|channel|>analysis<|message|>analysis message<|end|><|start|>assistant" ANALYSIS = get_model_output_str([assistant("reasoning", "analysis")])
COMMENTARY = ( COMMENTARY = get_model_output_str([assistant("commentary", "commentary")])
"<|channel|>commentary<|message|>commentary message<|end|><|start|>assistant" TOOL_CALL_1_CHANNEL_FIRST = (
ANALYSIS + f"<|start|>assistant<|channel|>commentary to=functions.{TOOL_1_NAME}"
" <|constrain|>json<|message|>{}<|call|>"
) )
TOOL_CALL_1 = ( TOOL_CALL_1_FUNCTION_FIRST = get_model_output_str(
ANALYSIS + f"<|channel|>commentary to=functions.{TOOL_1_NAME} json<|message|>" [
"{}<|call|>" assistant("reasoning", "analysis"),
tool_call(f"functions.{TOOL_1_NAME}", "{}"),
]
) )
TOOL_CALL_2 = ( TOOL_CALL_2_CHANNEL_FIRST = (
ANALYSIS + f"<|channel|>commentary to=functions.{TOOL_2_NAME} json<|message|>" ANALYSIS + f"<|start|>assistant<|channel|>commentary to=functions.{TOOL_2_NAME}"
'{"city": "Tokyo"}<|call|>' ' <|constrain|>json<|message|>{"city": "Tokyo"}<|call|>'
) )
FINAL_JSON_SCHEMA = ( TOOL_CALL_2_FUNCTION_FIRST = get_model_output_str(
ANALYSIS [
+ '<|channel|>final <|constrain|>json<|message|>{"answer": "Tokyo"}<|end|>' assistant("reasoning", "analysis"),
tool_call(f"functions.{TOOL_2_NAME}", '{"city": "Tokyo"}'),
]
) )
FINAL_JSON_OBJECT = ( FINAL_JSON_SCHEMA = get_model_output_str(
ANALYSIS [
+ '<|channel|>final <|constrain|>json<|message|>{"city": "Tokyo"}<|end|>' assistant("reasoning", "analysis"),
assistant('{"answer": "Tokyo"}', "final", "json"),
]
)
FINAL_JSON_OBJECT = get_model_output_str(
[
assistant("reasoning", "analysis"),
assistant('{"city": "Tokyo"}', "final", "<|constrain|>json"),
]
)
FINAL_TEXT_ONLY = get_model_output_str(
[assistant("reasoning", "analysis"), assistant("any", "final")]
)
FINAL_REGEX = get_model_output_str(
[assistant("reasoning", "analysis"), assistant("regex", "final")]
)
FINAL_CHOICE = get_model_output_str(
[assistant("reasoning", "analysis"), assistant("choice1", "final")]
)
FINAL_GRAMMAR = get_model_output_str(
[assistant("reasoning", "analysis"), assistant("grammar", "final")]
)
FINAL_STRUCTURAL_TAG = get_model_output_str(
[assistant("reasoning", "analysis"), assistant("tag content", "final")]
) )
FINAL_TEXT_ONLY = ANALYSIS + "<|channel|>final<|message|>any<|end|>"
FINAL_REGEX = ANALYSIS + "<|channel|>final<|message|>regex<|end|>"
FINAL_CHOICE = ANALYSIS + "<|channel|>final<|message|>choice1<|end|>"
FINAL_GRAMMAR = ANALYSIS + "<|channel|>final<|message|>grammar<|end|>"
FINAL_STRUCTURAL_TAG = ANALYSIS + "<|channel|>final<|message|>tag content<|end|>"
ADMISSION_SAMPLES = ( ADMISSION_SAMPLES = (
"COMMENTARY", "COMMENTARY",
"TOOL_CALL_1", "TOOL_CALL_1_CHANNEL_FIRST",
"TOOL_CALL_2", "TOOL_CALL_1_FUNCTION_FIRST",
"TOOL_CALL_2_CHANNEL_FIRST",
"TOOL_CALL_2_FUNCTION_FIRST",
"FINAL_JSON_SCHEMA", "FINAL_JSON_SCHEMA",
"FINAL_JSON_OBJECT", "FINAL_JSON_OBJECT",
"FINAL_TEXT_ONLY", "FINAL_TEXT_ONLY",
@@ -1090,8 +1104,10 @@ class TestAdjustRequest:
{"tool_choice": "auto", "strict_tools": True}, {"tool_choice": "auto", "strict_tools": True},
[ [
"COMMENTARY", "COMMENTARY",
"TOOL_CALL_1", "TOOL_CALL_1_CHANNEL_FIRST",
"TOOL_CALL_2", "TOOL_CALL_1_FUNCTION_FIRST",
"TOOL_CALL_2_CHANNEL_FIRST",
"TOOL_CALL_2_FUNCTION_FIRST",
"FINAL_JSON_SCHEMA", "FINAL_JSON_SCHEMA",
"FINAL_JSON_OBJECT", "FINAL_JSON_OBJECT",
"FINAL_TEXT_ONLY", "FINAL_TEXT_ONLY",
@@ -1103,11 +1119,21 @@ class TestAdjustRequest:
), ),
( (
{"tool_choice": "required"}, {"tool_choice": "required"},
["COMMENTARY", "TOOL_CALL_1", "TOOL_CALL_2"], [
"COMMENTARY",
"TOOL_CALL_1_CHANNEL_FIRST",
"TOOL_CALL_1_FUNCTION_FIRST",
"TOOL_CALL_2_CHANNEL_FIRST",
"TOOL_CALL_2_FUNCTION_FIRST",
],
), ),
( (
{"tool_choice": "named"}, {"tool_choice": "named"},
["COMMENTARY", "TOOL_CALL_2"], [
"COMMENTARY",
"TOOL_CALL_2_CHANNEL_FIRST",
"TOOL_CALL_2_FUNCTION_FIRST",
],
), ),
( (
{"response_format_type": "json_schema"}, {"response_format_type": "json_schema"},
+24 -20
View File
@@ -387,6 +387,7 @@ class HarmonyParser(DelegatingParser):
return recipient[:constrain_index].rstrip() or None return recipient[:constrain_index].rstrip() or None
_JSON_CONSTRAINS = [" json", " <|constrain|>json"]
# Harmomy's stop tokens are <|return|>, <|call|>, <|endoftext|> # Harmomy's stop tokens are <|return|>, <|call|>, <|endoftext|>
# They are represented as "" since xgrammar disallows stop tokens while # They are represented as "" since xgrammar disallows stop tokens while
# under constraints, leading to bad or infinite generations. # under constraints, leading to bad or infinite generations.
@@ -398,10 +399,8 @@ _TOOL_CALL_CHANNELS = [
"<|channel|>final", "<|channel|>final",
] ]
_FUNCTION_CALL_BEGINS = [ _FUNCTION_CALL_BEGINS = [
"to=functions.{name} {channel} json<|message|>", " to=functions.{name}{channel}{constrain}<|message|>",
"to=functions.{name} {channel} <|constrain|>json<|message|>", "{channel} to=functions.{name}{constrain}<|message|>",
"{channel} to=functions.{name} json<|message|>",
"{channel} to=functions.{name} <|constrain|>json<|message|>",
] ]
_JSON_CONTENT = JSONSchemaFormat(json_schema={"type": "object"}) _JSON_CONTENT = JSONSchemaFormat(json_schema={"type": "object"})
_ANY_CONTENT = AnyTextFormat() _ANY_CONTENT = AnyTextFormat()
@@ -476,7 +475,9 @@ def get_harmony_structural_tag(
else: else:
tags = [ tags = [
TagFormat( TagFormat(
begin=pattern.format(name=tool.function.name, channel=channel), begin=pattern.format(
name=tool.function.name, channel=channel, constrain=constrain
),
content=JSONSchemaFormat( content=JSONSchemaFormat(
json_schema=get_function_parameters(tool.function) json_schema=get_function_parameters(tool.function)
), ),
@@ -484,23 +485,18 @@ def get_harmony_structural_tag(
) )
for tool in tools for tool in tools
for pattern in _FUNCTION_CALL_BEGINS for pattern in _FUNCTION_CALL_BEGINS
for constrain in _JSON_CONSTRAINS
for channel in _TOOL_CALL_CHANNELS for channel in _TOOL_CALL_CHANNELS
] ]
if tool_choice == "auto": if tool_choice == "auto":
tags.append( tags.extend(
TagFormat( TagFormat(
begin=_FINAL_BEGIN.format(constrain=" <|constrain|>json"), begin=_FINAL_BEGIN.format(constrain=constrain),
content=_ANY_CONTENT,
end=_END_TAG,
)
)
tags.append(
TagFormat(
begin=_FINAL_BEGIN.format(constrain=""),
content=_ANY_CONTENT, content=_ANY_CONTENT,
end=_END_TAG, end=_END_TAG,
) )
for constrain in _JSON_CONSTRAINS + [""]
) )
return _assemble_tag( return _assemble_tag(
@@ -558,14 +554,22 @@ def _adjust_output_format(
return request return request
if isinstance(final_content, JSONSchemaFormat): if isinstance(final_content, JSONSchemaFormat):
begin = _FINAL_BEGIN.format(constrain=" <|constrain|>json") content = OrFormat(
elements=[
TagFormat(
begin=_FINAL_BEGIN.format(constrain=constrain),
content=final_content,
end=_END_TAG,
)
for constrain in _JSON_CONSTRAINS
]
)
else: else:
begin = _FINAL_BEGIN.format(constrain="") content = TagFormat(
begin=_FINAL_BEGIN.format(constrain=""), content=final_content, end=_END_TAG
)
structural_tag = _assemble_tag( structural_tag = _assemble_tag(
allow_analysis=True, allow_analysis=True, allow_commentary=False, content=content
allow_commentary=False,
content=TagFormat(begin=begin, content=final_content, end=_END_TAG),
) )
request.structured_outputs = replace( request.structured_outputs = replace(