[GPT-OSS] Strict tool call and constrained decoding for Harmony (#45560)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-08-01 12:10:26 -04:00
committed by GitHub
parent 127a7cebc9
commit dc818c198d
10 changed files with 632 additions and 511 deletions
@@ -13,7 +13,7 @@ from typing import Any
import pytest import pytest
import pytest_asyncio import pytest_asyncio
import requests import requests
from openai import InternalServerError, NotFoundError, OpenAI from openai import NotFoundError, OpenAI
from openai_harmony import Message from openai_harmony import Message
from tests.utils import RemoteOpenAIServer from tests.utils import RemoteOpenAIServer
@@ -368,8 +368,12 @@ async def test_streaming_types(
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME]) @pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
async def test_function_calling_with_streaming_types( async def test_function_calling_with_streaming_types(
pairs_of_event_types: dict[str, str], client: OpenAI, model_name: str pairs_of_event_types: dict[str, str],
client: OpenAI,
model_name: str,
tool_choice: str,
): ):
"""Streaming event nesting for function-calling responses.""" """Streaming event nesting for function-calling responses."""
@@ -382,6 +386,7 @@ async def test_function_calling_with_streaming_types(
validate_events=_has_function_events, validate_events=_has_function_events,
input=[{"role": "user", "content": "What's the weather like in Paris today?"}], input=[{"role": "user", "content": "What's the weather like in Paris today?"}],
tools=[GET_WEATHER_SCHEMA], tools=[GET_WEATHER_SCHEMA],
tool_choice=tool_choice,
temperature=0.0, temperature=0.0,
) )
@@ -558,7 +563,8 @@ async def test_reasoning_item(client: OpenAI, model_name: str):
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME]) @pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_function_calling(client: OpenAI, model_name: str): @pytest.mark.parametrize("tool_choice", ["auto", "required"])
async def test_function_calling(client: OpenAI, model_name: str, tool_choice: str):
tools = [GET_WEATHER_SCHEMA] tools = [GET_WEATHER_SCHEMA]
response = await retry_for_tool_call( response = await retry_for_tool_call(
@@ -567,8 +573,9 @@ async def test_function_calling(client: OpenAI, model_name: str):
expected_tool_type="function_call", expected_tool_type="function_call",
input="What's the weather like in Paris today?", input="What's the weather like in Paris today?",
tools=tools, tools=tools,
tool_choice=tool_choice,
temperature=0.0, temperature=0.0,
extra_body={"request_id": "test_function_calling_non_resp"}, extra_body={"request_id": f"test_function_calling_non_resp_{tool_choice}"},
) )
assert response.status == "completed" assert response.status == "completed"
assert has_output_type(response, "function_call"), ( assert has_output_type(response, "function_call"), (
@@ -610,7 +617,10 @@ async def test_function_calling(client: OpenAI, model_name: str):
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME]) @pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_function_calling_multi_turn(client: OpenAI, model_name: str): @pytest.mark.parametrize("tool_choice", ["auto", "required"])
async def test_function_calling_multi_turn(
client: OpenAI, model_name: str, tool_choice: str
):
"""Multi-tool, multi-turn function calling with retry at API level.""" """Multi-tool, multi-turn function calling with retry at API level."""
tools = [ tools = [
{ {
@@ -635,6 +645,7 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
expected_tool_type="function_call", expected_tool_type="function_call",
input="Help me plan a trip to a random place. And tell me the weather there.", input="Help me plan a trip to a random place. And tell me the weather there.",
tools=tools, tools=tools,
tool_choice=tool_choice,
temperature=0.0, temperature=0.0,
) )
assert response.status == "completed" assert response.status == "completed"
@@ -659,6 +670,7 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
} }
], ],
tools=tools, tools=tools,
tool_choice=tool_choice,
previous_response_id=response.id, previous_response_id=response.id,
temperature=0.0, temperature=0.0,
) )
@@ -695,20 +707,6 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
) )
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_function_calling_required(client: OpenAI, model_name: str):
tools = [GET_WEATHER_SCHEMA]
with pytest.raises(InternalServerError):
await client.responses.create(
model=model_name,
input="What's the weather like in Paris today?",
tools=tools,
tool_choice="required",
)
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME]) @pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_system_message_with_tools(client: OpenAI, model_name: str): async def test_system_message_with_tools(client: OpenAI, model_name: str):
@@ -726,7 +724,10 @@ async def test_system_message_with_tools(client: OpenAI, model_name: str):
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME]) @pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_function_calling_full_history(client: OpenAI, model_name: str): @pytest.mark.parametrize("tool_choice", ["auto", "required"])
async def test_function_calling_full_history(
client: OpenAI, model_name: str, tool_choice: str
):
tools = [GET_WEATHER_SCHEMA] tools = [GET_WEATHER_SCHEMA]
input_messages = [ input_messages = [
@@ -739,6 +740,7 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str):
expected_tool_type="function_call", expected_tool_type="function_call",
input=input_messages, input=input_messages,
tools=tools, tools=tools,
tool_choice=tool_choice,
temperature=0.0, temperature=0.0,
) )
assert response.status == "completed" assert response.status == "completed"
@@ -772,7 +774,10 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str):
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME]) @pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_function_calling_with_stream(client: OpenAI, model_name: str): @pytest.mark.parametrize("tool_choice", ["auto", "required"])
async def test_function_calling_with_stream(
client: OpenAI, model_name: str, tool_choice: str
):
"""Function calling via streaming, with retry for non-determinism.""" """Function calling via streaming, with retry for non-determinism."""
tools = [GET_WEATHER_SCHEMA] tools = [GET_WEATHER_SCHEMA]
input_list = [ input_list = [
@@ -792,6 +797,7 @@ async def test_function_calling_with_stream(client: OpenAI, model_name: str):
validate_events=_has_function_call, validate_events=_has_function_call,
input=input_list, input=input_list,
tools=tools, tools=tools,
tool_choice=tool_choice,
temperature=0.0, temperature=0.0,
) )
@@ -853,8 +859,9 @@ async def test_function_calling_with_stream(client: OpenAI, model_name: str):
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME]) @pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
async def test_function_calling_no_code_interpreter_events( async def test_function_calling_no_code_interpreter_events(
client: OpenAI, model_name: str client: OpenAI, model_name: str, tool_choice: str
): ):
"""Verify that function calls don't trigger code_interpreter events. """Verify that function calls don't trigger code_interpreter events.
@@ -880,6 +887,7 @@ async def test_function_calling_no_code_interpreter_events(
validate_events=_has_function_call, validate_events=_has_function_call,
input=input_list, input=input_list,
tools=tools, tools=tools,
tool_choice=tool_choice,
temperature=0.0, temperature=0.0,
) )
@@ -1048,8 +1056,9 @@ async def test_output_messages_enabled(client: OpenAI, model_name: str, server):
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME]) @pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
async def test_function_call_with_previous_input_messages( async def test_function_call_with_previous_input_messages(
client: OpenAI, model_name: str client: OpenAI, model_name: str, tool_choice: str
): ):
"""Multi-turn function calling using previous_input_messages.""" """Multi-turn function calling using previous_input_messages."""
tools = [ tools = [
@@ -1074,6 +1083,7 @@ async def test_function_call_with_previous_input_messages(
expected_tool_type="function_call", expected_tool_type="function_call",
input="What is the horoscope for Aquarius today?", input="What is the horoscope for Aquarius today?",
tools=tools, tools=tools,
tool_choice=tool_choice,
temperature=0.0, temperature=0.0,
extra_body={"enable_response_messages": True}, extra_body={"enable_response_messages": True},
max_output_tokens=1000, max_output_tokens=1000,
+337
View File
@@ -3,6 +3,7 @@
import json import json
from collections.abc import Sequence from collections.abc import Sequence
from typing import Any, Literal
import pytest import pytest
from openai_harmony import ( from openai_harmony import (
@@ -12,14 +13,18 @@ from openai_harmony import (
Role, Role,
) )
from transformers import AutoTokenizer from transformers import AutoTokenizer
from xgrammar import Grammar
from xgrammar.testing import _is_grammar_accept_string
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.engine.protocol import FunctionCall from vllm.entrypoints.openai.engine.protocol import FunctionCall
from vllm.entrypoints.openai.parser.harmony_utils import ( from vllm.entrypoints.openai.parser.harmony_utils import (
get_encoding, get_encoding,
) )
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.harmony import HarmonyParser from vllm.parser.harmony import HarmonyParser
from vllm.parser.parser_manager import ParserManager from vllm.parser.parser_manager import ParserManager
from vllm.sampling_params import StructuredOutputsParams
REASONING_MODEL_NAME = "openai/gpt-oss-20b" REASONING_MODEL_NAME = "openai/gpt-oss-20b"
@@ -857,3 +862,335 @@ class TestProcessChunk:
("analysis", "One"), ("analysis", "One"),
("final", "Two"), ("final", "Two"),
] ]
class TestAdjustRequest:
REQUEST_TEXT = "Hello"
TOOL_TYPE = "function"
TOOL_1_NAME = "get_user_location"
TOOL_2_NAME = "get_weather"
TOOLS = [
{
"name": TOOL_1_NAME,
"parameters": {
"type": "object",
"properties": {},
"required": [],
},
},
{
"name": TOOL_2_NAME,
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
]
OUTPUT_SCHEMA = {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"],
}
ANALYSIS = "<|channel|>analysis<|message|>analysis message<|end|><|start|>assistant"
COMMENTARY = (
"<|channel|>commentary<|message|>commentary message<|end|><|start|>assistant"
)
TOOL_CALL_1 = (
ANALYSIS + f"<|channel|>commentary to=functions.{TOOL_1_NAME} json<|message|>"
"{}<|call|>"
)
TOOL_CALL_2 = (
ANALYSIS + f"<|channel|>commentary to=functions.{TOOL_2_NAME} json<|message|>"
'{"city": "Tokyo"}<|call|>'
)
FINAL_JSON_SCHEMA = (
ANALYSIS
+ '<|channel|>final <|constrain|>json<|message|>{"answer": "Tokyo"}<|end|>'
)
FINAL_JSON_OBJECT = (
ANALYSIS
+ '<|channel|>final <|constrain|>json<|message|>{"city": "Tokyo"}<|end|>'
)
FINAL_TEXT_ONLY = ANALYSIS + "<|channel|>final<|message|>any<|end|>"
FINAL_REGEX = ANALYSIS + "<|channel|>final<|message|>regex<|end|>"
FINAL_CHOICE = ANALYSIS + "<|channel|>final<|message|>choice1<|end|>"
FINAL_GRAMMAR = ANALYSIS + "<|channel|>final<|message|>grammar<|end|>"
FINAL_STRUCTURAL_TAG = ANALYSIS + "<|channel|>final<|message|>tag content<|end|>"
ADMISSION_SAMPLES = (
"COMMENTARY",
"TOOL_CALL_1",
"TOOL_CALL_2",
"FINAL_JSON_SCHEMA",
"FINAL_JSON_OBJECT",
"FINAL_TEXT_ONLY",
"FINAL_REGEX",
"FINAL_CHOICE",
"FINAL_GRAMMAR",
"FINAL_STRUCTURAL_TAG",
)
@staticmethod
def _build_request(
request_kind: Literal["chat", "responses"],
tool_choice: str = "none",
strict_tools: bool = False,
response_format_type: str | None = None,
structured_outputs: StructuredOutputsParams | None = None,
) -> ChatCompletionRequest | ResponsesRequest:
data: dict[str, Any] = {
"model": REASONING_MODEL_NAME,
}
if request_kind == "chat":
data["messages"] = [
{
"role": "user",
"content": TestAdjustRequest.REQUEST_TEXT,
}
]
else:
data["input"] = TestAdjustRequest.REQUEST_TEXT
if request_kind == "chat":
data["tools"] = [
{
"type": TestAdjustRequest.TOOL_TYPE,
"function": {"strict": strict_tools, **tool_def},
}
for tool_def in TestAdjustRequest.TOOLS
]
data["tool_choice"] = (
{
"type": TestAdjustRequest.TOOL_TYPE,
"function": {"name": TestAdjustRequest.TOOL_2_NAME},
}
if tool_choice == "named"
else tool_choice
)
else:
data["tools"] = [
{
"type": TestAdjustRequest.TOOL_TYPE,
"strict": strict_tools,
**tool_def,
}
for tool_def in TestAdjustRequest.TOOLS
]
data["tool_choice"] = (
{
"type": TestAdjustRequest.TOOL_TYPE,
"name": TestAdjustRequest.TOOL_2_NAME,
}
if tool_choice == "named"
else tool_choice
)
if response_format_type == "json_schema":
schema_format = {
"name": "answer_format",
"schema": TestAdjustRequest.OUTPUT_SCHEMA,
"strict": True,
}
if request_kind == "chat":
data["response_format"] = {
"type": "json_schema",
"json_schema": schema_format,
}
else:
data["text"] = {
"format": {
"type": "json_schema",
**schema_format,
}
}
elif response_format_type == "json_object":
if request_kind == "chat":
data["response_format"] = {"type": "json_object"}
else:
data["text"] = {"format": {"type": "json_object"}}
if structured_outputs is not None:
data["structured_outputs"] = structured_outputs
if request_kind == "chat":
return ChatCompletionRequest.model_validate(data)
return ResponsesRequest.model_validate(data)
@staticmethod
def _assert_format_cleared(
adjusted_request: ChatCompletionRequest | ResponsesRequest,
) -> None:
if isinstance(adjusted_request, ResponsesRequest):
assert adjusted_request.text is None or adjusted_request.text.format is None
else:
assert adjusted_request.response_format is None
structured_outputs = adjusted_request.structured_outputs
assert structured_outputs is not None
assert structured_outputs.structural_tag is not None
assert structured_outputs.all_non_structural_tag_constraints_none()
@classmethod
def _assert_structured_outputs_admission(
cls,
adjusted_request: ChatCompletionRequest | ResponsesRequest,
expected_admission: Sequence[str],
) -> None:
structured_outputs = adjusted_request.structured_outputs
assert structured_outputs is not None
assert structured_outputs.structural_tag is not None
assert structured_outputs.all_non_structural_tag_constraints_none()
grammar = Grammar.from_structural_tag(structured_outputs.structural_tag)
expected_admission_set = set(expected_admission)
for sample_name in cls.ADMISSION_SAMPLES:
admitted = _is_grammar_accept_string(
grammar,
getattr(cls, sample_name),
require_termination=False,
)
should_admit = sample_name in expected_admission_set
assert admitted is should_admit, (
f"Expected structured_outputs admission for {sample_name} "
f"to be {should_admit}, got {admitted}."
)
@pytest.mark.parametrize("request_kind", ["chat", "responses"])
@pytest.mark.parametrize(
("request_kwargs", "expected_admission"),
[
(
{"tool_choice": "auto", "strict_tools": True},
[
"COMMENTARY",
"TOOL_CALL_1",
"TOOL_CALL_2",
"FINAL_JSON_SCHEMA",
"FINAL_JSON_OBJECT",
"FINAL_TEXT_ONLY",
"FINAL_REGEX",
"FINAL_CHOICE",
"FINAL_GRAMMAR",
"FINAL_STRUCTURAL_TAG",
],
),
(
{"tool_choice": "required"},
["COMMENTARY", "TOOL_CALL_1", "TOOL_CALL_2"],
),
(
{"tool_choice": "named"},
["COMMENTARY", "TOOL_CALL_2"],
),
(
{"response_format_type": "json_schema"},
["FINAL_JSON_SCHEMA"],
),
(
{"response_format_type": "json_object"},
["FINAL_JSON_SCHEMA", "FINAL_JSON_OBJECT"],
),
(
{"structured_outputs": StructuredOutputsParams(json=OUTPUT_SCHEMA)},
["FINAL_JSON_SCHEMA"],
),
(
{"structured_outputs": StructuredOutputsParams(json_object=True)},
["FINAL_JSON_SCHEMA", "FINAL_JSON_OBJECT"],
),
(
{"structured_outputs": StructuredOutputsParams(regex=r"regex")},
["FINAL_REGEX"],
),
(
{
"structured_outputs": StructuredOutputsParams(
choice=["choice1", "choice2"]
)
},
["FINAL_CHOICE"],
),
(
{
"structured_outputs": StructuredOutputsParams(
grammar='root ::= "grammar"'
)
},
["FINAL_GRAMMAR"],
),
(
{
"structured_outputs": StructuredOutputsParams(
structural_tag=json.dumps(
{
"type": "structural_tag",
"format": {
"type": "json_schema",
"json_schema": OUTPUT_SCHEMA,
},
}
)
)
},
["FINAL_JSON_SCHEMA"],
),
(
{
"structured_outputs": StructuredOutputsParams(
structural_tag=json.dumps(
{
"type": "structural_tag",
"structures": [
{
"begin": "<tag>",
"schema": {"type": "object"},
"end": "</tag>",
}
],
"triggers": ["<tag>"],
}
)
)
},
[
# Legacy triggered tags allow free text until a trigger, so
# unconstrained final-channel payloads are also admitted.
"FINAL_TEXT_ONLY",
"FINAL_REGEX",
"FINAL_CHOICE",
"FINAL_GRAMMAR",
"FINAL_STRUCTURAL_TAG",
],
),
],
ids=[
"tool_auto_strict",
"tool_required",
"tool_named",
"response_format_json_schema",
"response_format_json_object",
"structured_outputs_json",
"structured_outputs_json_object",
"structured_outputs_regex",
"structured_outputs_choice",
"structured_outputs_grammar",
"structured_outputs_structural_tag_modern",
"structured_outputs_structural_tag_legacy",
],
)
def test_adjust_request(
self,
harmony_parser,
request_kind,
request_kwargs,
expected_admission,
):
request = self._build_request(request_kind, **request_kwargs)
adjusted_request = harmony_parser.adjust_request(request)
self._assert_format_cleared(adjusted_request)
self._assert_structured_outputs_admission(
adjusted_request,
expected_admission,
)
+5 -344
View File
@@ -1,351 +1,12 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from unittest.mock import Mock from unittest.mock import Mock
import pytest from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
from transformers import AutoTokenizer
from vllm.entrypoints.mcp.tool_server import ToolServer
from vllm.reasoning import ReasoningParser
from vllm.reasoning.gptoss_reasoning_parser import (
GptOssReasoningParser,
from_builtin_tool_to_tag,
no_func_reasoning_tag,
)
REASONING_MODEL_NAME = "openai/gpt-oss-120b"
@pytest.fixture(scope="module") def test_gptoss_reasoning_ended_is_true():
def gpt_oss_tokenizer(): parser = GptOssReasoningParser(Mock())
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) assert parser.is_reasoning_end([]) is True
assert parser.is_reasoning_end_streaming([], []) is True
USER_MESSAGE_START = "<|start|>user<|message|>"
REASONING_SECTION_START = "<|end|><|start|>assistant<|channel|>analysis<|message|>"
END = "<|end|>"
ASSISTANT_START = "<|start|>assistant"
ASSISTANT_CONTENT_START_PREFIX = END + ASSISTANT_START + "<|channel|>final"
ASSISTANT_CONTENT_START_SUFFIX = "<|message|>"
ASSISTANT_CONTENT_START = (
ASSISTANT_CONTENT_START_PREFIX + ASSISTANT_CONTENT_START_SUFFIX
)
BASIC_CONTENT = {
"output": REASONING_SECTION_START
+ "This is reasoning"
+ ASSISTANT_CONTENT_START
+ "This is the rest",
"is_reasoning_end": True,
}
BASIC_REASONING_ONLY = {
"output": REASONING_SECTION_START + "This is reasoning" + "<|end|>",
"is_reasoning_end": False,
}
BASIC_NO_REASONING_NO_ASSISTANT = {
"output": USER_MESSAGE_START + "This is a user message",
"is_reasoning_end": False,
}
# Edge-case where the model omits the assistant tag entirely.
BASIC_NO_REASONING_ASSISTANT = {
"output": USER_MESSAGE_START + "This is a user message<|end|><|channel|>final",
"is_reasoning_end": True,
}
COMPLEX_CONTENT_INCOMPLETE_PREFIX_ONLY = {
"output": REASONING_SECTION_START
+ "This is reasoning"
+ ASSISTANT_CONTENT_START_PREFIX,
"is_reasoning_end": False,
}
COMPLEX_CONTENT_SUFFIX_ONLY = {
"output": REASONING_SECTION_START
+ "This is reasoning"
+ ASSISTANT_CONTENT_START_SUFFIX,
"is_reasoning_end": False,
}
COMPLEX_CONTENT_1_NO_SUFFIX = {
"output": REASONING_SECTION_START
+ "This is reasoning"
+ ASSISTANT_CONTENT_START_PREFIX
+ "<|constrain|> JSON ",
"is_reasoning_end": False,
}
COMPLEX_CONTENT_1 = {
"output": REASONING_SECTION_START
+ "This is reasoning"
+ ASSISTANT_CONTENT_START_PREFIX
+ "<|constrain|> JSON "
+ ASSISTANT_CONTENT_START_SUFFIX,
"is_reasoning_end": True,
}
COMPLEX_CONTENT_1_WITH_CONTENT = {
"output": REASONING_SECTION_START
+ "This is reasoning"
+ ASSISTANT_CONTENT_START_PREFIX
+ "<|constrain|> JSON "
+ ASSISTANT_CONTENT_START_SUFFIX
+ "This is the rest",
"is_reasoning_end": True,
}
COMPLEX_CONTENT_2 = {
"output": REASONING_SECTION_START
+ "This is reasoning"
+ ASSISTANT_CONTENT_START_PREFIX
+ "<|constrain|>ReplyAction "
+ ASSISTANT_CONTENT_START_SUFFIX
+ "This is the rest",
"is_reasoning_end": True,
}
MULTI_TURN_CONTENT = {
"output": USER_MESSAGE_START
+ "1st turn user message"
+ REASONING_SECTION_START
+ "1st turn reasoning"
+ ASSISTANT_CONTENT_START
+ "1st turn response"
+ END
+ USER_MESSAGE_START
+ "2nd turn user message"
+ END
+ ASSISTANT_START,
"is_reasoning_end": False,
}
TEST_CASES = [
BASIC_CONTENT,
BASIC_REASONING_ONLY,
COMPLEX_CONTENT_INCOMPLETE_PREFIX_ONLY,
COMPLEX_CONTENT_SUFFIX_ONLY,
COMPLEX_CONTENT_1_NO_SUFFIX,
COMPLEX_CONTENT_1,
COMPLEX_CONTENT_1_WITH_CONTENT,
COMPLEX_CONTENT_2,
MULTI_TURN_CONTENT,
]
@pytest.mark.parametrize(
"output, is_reasoning_end",
[(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
)
def test_gptoss_is_reasoning_end(
output,
is_reasoning_end,
gpt_oss_tokenizer,
):
output = gpt_oss_tokenizer.tokenize(output)
parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
# Test is_reasoning_end
output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(output)
actual_is_reasoning_end = parser.is_reasoning_end(output_ids)
assert is_reasoning_end == actual_is_reasoning_end
class TestGptOssStructuralTags:
"""Test cases for GptOssReasoningParser structural tag functionality."""
@pytest.fixture
def mock_tokenizer(self):
"""Create a mock tokenizer for testing."""
tokenizer = Mock()
tokenizer.encode = Mock(return_value=[1, 2, 3, 4, 5])
tokenizer.get_vocab = Mock(return_value={"<|end|>": 6})
return tokenizer
@pytest.fixture
def reasoning_parser(self, mock_tokenizer):
"""Create a GptOssReasoningParser instance."""
return GptOssReasoningParser(mock_tokenizer)
def test_prepare_structured_tag_no_tool_server(self, reasoning_parser):
"""Test prepare_structured_tag with no tool server."""
result = reasoning_parser.prepare_structured_tag(None, None)
expected = json.dumps(no_func_reasoning_tag)
assert result == expected
# Verify the structure is correct
parsed = json.loads(result)
assert parsed["type"] == "structural_tag"
assert parsed["format"]["type"] == "triggered_tags"
assert len(parsed["format"]["tags"]) == 1
assert parsed["format"]["tags"][0]["begin"] == "<|channel|>analysis<|message|>"
assert parsed["format"]["triggers"] == ["<|channel|>analysis"]
def test_prepare_structured_tag_with_original_tag(self, reasoning_parser):
"""Test prepare_structured_tag when original_tag is provided."""
original_tag = '{"custom": "tag"}'
result = reasoning_parser.prepare_structured_tag(original_tag, None)
# Should return the original tag unchanged
assert result == original_tag
def test_from_builtin_tool_to_tag(self):
"""Test from_builtin_tool_to_tag function."""
tags = from_builtin_tool_to_tag("python")
assert len(tags) == 2
assert tags[0]["begin"] == "<|channel|>commentary to=python"
assert tags[0]["content"]["type"] == "any_text"
assert tags[0]["end"] == "<|end|>"
assert tags[1]["begin"] == "<|channel|>analysis to=python"
assert tags[1]["content"]["type"] == "any_text"
assert tags[1]["end"] == "<|end|>"
@pytest.mark.parametrize(
"tools",
[
[],
["browser"],
["python"],
["container"],
["browser", "python"],
["browser", "container"],
["python", "container"],
["browser", "python", "container"],
],
)
def test_json_validity_comprehensive(self, reasoning_parser, tools):
"""Test JSON validity across all possible tool combinations."""
tool_server = Mock(spec=ToolServer)
tool_server.has_tool = Mock(side_effect=lambda tool: tool in tools)
result = reasoning_parser.prepare_structured_tag(None, tool_server)
parsed_result = json.loads(result)
assert parsed_result["type"] == "structural_tag"
assert "format" in parsed_result
assert "tags" in parsed_result["format"]
assert "triggers" in parsed_result["format"]
# Tag count should be: 1 (analysis) + 2 * len(tools)
expected_tag_count = 1 + (2 * len(tools))
assert len(parsed_result["format"]["tags"]) == expected_tag_count
# Verify triggers are correctly configured
expected_triggers = ["<|channel|>analysis"]
if tools:
expected_triggers.append("<|channel|>commentary to=")
assert set(parsed_result["format"]["triggers"]) == set(expected_triggers)
def test_no_cross_request_state_pollution(self, reasoning_parser):
"""Test that sequential calls with different tool servers produce
independent results, guarding against shared mutable state
(e.g. missing deepcopy in tag_with_builtin_funcs)."""
tool_server_1 = Mock(spec=ToolServer)
tool_server_1.has_tool = Mock(side_effect=lambda tool: tool == "python")
tool_server_2 = Mock(spec=ToolServer)
tool_server_2.has_tool = Mock(side_effect=lambda tool: tool == "browser")
result_1 = reasoning_parser.prepare_structured_tag(None, tool_server_1)
result_2 = reasoning_parser.prepare_structured_tag(None, tool_server_2)
tags_1 = [tag["begin"] for tag in json.loads(result_1)["format"]["tags"]]
tags_2 = [tag["begin"] for tag in json.loads(result_2)["format"]["tags"]]
assert "<|channel|>commentary to=python" in tags_1
assert "<|channel|>commentary to=browser" not in tags_1
assert "<|channel|>commentary to=browser" in tags_2
assert "<|channel|>commentary to=python" not in tags_2
def test_tag_format_consistency(self, reasoning_parser):
"""Test that all generated tags follow consistent format,
catching malformed tags from from_builtin_tool_to_tag."""
tool_server = Mock(spec=ToolServer)
tool_server.has_tool = Mock(
side_effect=lambda tool: tool in ["python", "browser"]
)
result = reasoning_parser.prepare_structured_tag(None, tool_server)
parsed_result = json.loads(result)
for tag in parsed_result["format"]["tags"]:
assert "begin" in tag
assert "content" in tag
assert "end" in tag
assert tag["content"]["type"] == "any_text"
assert tag["end"] == "<|end|>"
assert tag["begin"].startswith("<|channel|>")
@pytest.mark.parametrize(
"output, is_reasoning_end",
[(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
)
def test_gptoss_is_reasoning_end_streaming(
output,
is_reasoning_end,
gpt_oss_tokenizer,
):
"""Streaming override must agree with is_reasoning_end for all cases."""
tokens = gpt_oss_tokenizer.tokenize(output)
parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens)
delta_ids = output_ids[-1:] if output_ids else []
actual = parser.is_reasoning_end_streaming(output_ids, delta_ids)
assert is_reasoning_end == actual
@pytest.mark.parametrize(
"output, is_reasoning_end",
[(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
)
def test_gptoss_is_reasoning_end_streaming_long_prefix(
output,
is_reasoning_end,
gpt_oss_tokenizer,
):
"""Windowing must produce correct results even with a long prefix."""
tokens = gpt_oss_tokenizer.tokenize(output)
parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens)
# Prepend 10k dummy reasoning tokens to simulate a long generation
long_prefix = [1] * 10_000
padded_ids = long_prefix + list(output_ids)
delta_ids = output_ids[-1:] if output_ids else []
actual = parser.is_reasoning_end_streaming(padded_ids, delta_ids)
assert is_reasoning_end == actual
@pytest.mark.parametrize(
"output, is_reasoning_end",
[(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
)
def test_gptoss_is_reasoning_end_streaming_large_delta(
output,
is_reasoning_end,
gpt_oss_tokenizer,
):
"""Simulate speculative decoding where the entire test sequence arrives
as a single large delta appended after a long prefix. The window must
expand to cover delta_ids so the end pattern is never missed."""
tokens = gpt_oss_tokenizer.tokenize(output)
parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens)
long_prefix = [1] * 10_000
padded_ids = long_prefix + list(output_ids)
# delta_ids = the entire test sequence (as if accepted in one spec step)
delta_ids = list(output_ids)
actual = parser.is_reasoning_end_streaming(padded_ids, delta_ids)
assert is_reasoning_end == actual
def test_gptoss_is_reasoning_end_streaming_signature(gpt_oss_tokenizer):
"""Verify the method is callable with the expected signature."""
parser = GptOssReasoningParser(gpt_oss_tokenizer)
result = parser.is_reasoning_end_streaming([], [])
assert result is False
@@ -33,7 +33,7 @@ from vllm.tool_parsers.structural_tag_registry import (
SUPPORTED_STRUCTURAL_TAG_MODELS, SUPPORTED_STRUCTURAL_TAG_MODELS,
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS, VLLM_BUILTIN_STRUCTURAL_TAG_MODELS,
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS, XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS,
_get_function_parameters, get_function_parameters,
get_model_structural_tag, get_model_structural_tag,
) )
@@ -530,7 +530,7 @@ def test_get_function_parameters_relaxes_function_strict_false():
strict=False, strict=False,
) )
assert _get_function_parameters(function) is True assert get_function_parameters(function) is True
def _k3_tools_with_root_defs() -> list[ChatCompletionToolsParam]: def _k3_tools_with_root_defs() -> list[ChatCompletionToolsParam]:
+8 -5
View File
@@ -749,11 +749,14 @@ class OpenAIServingResponses(GenerateBaseServing):
request: ResponsesRequest, request: ResponsesRequest,
prev_response: ResponsesResponse | None, prev_response: ResponsesResponse | None,
): ):
if request.tool_choice not in ("auto", "none"): if self.parser is not None:
raise NotImplementedError( # HarmonyParser doesn't need chat_template_kwargs
"Only 'auto' or 'none' tool_choice is supported " # TODO: Unify adjust_request() call with non-harmony branch
"in response API with Harmony" self.parser(
) self.renderer.get_tokenizer(),
request.tools,
model_config=self.model_config,
).adjust_request(request=request)
arrival_time = time.time() arrival_time = time.time()
messages = self._construct_input_messages_with_harmony(request, prev_response) messages = self._construct_input_messages_with_harmony(request, prev_response)
+230 -2
View File
@@ -5,14 +5,31 @@ from __future__ import annotations
import json import json
from collections.abc import Sequence from collections.abc import Sequence
from dataclasses import dataclass from dataclasses import dataclass, replace
from enum import Enum, auto from enum import Enum, auto
from typing import TYPE_CHECKING, NamedTuple from typing import TYPE_CHECKING, NamedTuple
from openai_harmony import HarmonyError, Message, Role from openai_harmony import HarmonyError, Message, Role
from xgrammar import StructuralTag
from xgrammar.openai_tool_call_schema import BuiltinToolParam, FunctionToolParam
from xgrammar.structural_tag import (
AnyTextFormat,
ConstStringFormat,
Format,
GrammarFormat,
JSONSchemaFormat,
OptionalFormat,
OrFormat,
RegexFormat,
SequenceFormat,
TagFormat,
TriggeredTagsFormat,
)
from vllm.entrypoints.chat_utils import make_tool_call_id from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.engine.protocol import ( from vllm.entrypoints.openai.engine.protocol import (
DeltaFunctionCall, DeltaFunctionCall,
DeltaMessage, DeltaMessage,
@@ -28,7 +45,13 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.parser.abstract_parser import DelegatingParser from vllm.parser.abstract_parser import DelegatingParser
from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
from vllm.sampling_params import StructuredOutputsParams
from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
from vllm.tool_parsers.structural_tag_registry import (
SimplifiedToolChoice,
get_function_parameters,
register_vllm_structural_tag,
)
if TYPE_CHECKING: if TYPE_CHECKING:
from openai_harmony import Message, StreamableParser from openai_harmony import Message, StreamableParser
@@ -346,6 +369,12 @@ class HarmonyParser(DelegatingParser):
reasoning_token_count=reasoning_token_count, reasoning_token_count=reasoning_token_count,
) )
def adjust_request(
self, request: ChatCompletionRequest | ResponsesRequest
) -> ChatCompletionRequest | ResponsesRequest:
request = _adjust_output_format(request)
return super().adjust_request(request)
@staticmethod @staticmethod
def _normalize_recipient(recipient: str | None) -> str | None: def _normalize_recipient(recipient: str | None) -> str | None:
"""Remove constrained formats misparsed into recipients by older Harmony.""" """Remove constrained formats misparsed into recipients by older Harmony."""
@@ -356,3 +385,202 @@ class HarmonyParser(DelegatingParser):
if constrain_index == -1: if constrain_index == -1:
return recipient return recipient
return recipient[:constrain_index].rstrip() or None return recipient[:constrain_index].rstrip() or None
# Harmomy's stop tokens are <|return|>, <|call|>, <|endoftext|>
# <|return|> is represented as "" since it's the default stop token, which xgrammar
# disallows under constraints, leading to bad or infinite generation.
# StreamableParser doesn't consider <|endoftext|> as a message end, so it's excluded
# TODO: Remove <|call|> once #50595 lands.
_END_TAG = ["<|end|>", "<|call|>", ""]
_FINAL_BEGIN = "<|channel|>final{constrain}<|message|>"
_TOOL_CALL_CHANNELS = [
"<|channel|>commentary",
"<|channel|>analysis",
"<|channel|>final",
]
_FUNCTION_CALL_BEGINS = [
"to=functions.{name} {channel} json<|message|>",
"to=functions.{name} {channel} <|constrain|>json<|message|>",
"{channel} to=functions.{name} json<|message|>",
"{channel} to=functions.{name} <|constrain|>json<|message|>",
]
_JSON_CONTENT = JSONSchemaFormat(json_schema={"type": "object"})
_ANY_CONTENT = AnyTextFormat()
def _assemble_tag(
allow_analysis: bool, allow_commentary: bool, content: Format
) -> StructuralTag:
tags = []
if allow_analysis:
analysis_tag = OptionalFormat(
content=SequenceFormat(
elements=[
TagFormat(
begin="<|channel|>analysis<|message|>",
content=_ANY_CONTENT,
end="<|end|>",
),
ConstStringFormat(value="<|start|>assistant"),
]
)
)
tags.append(analysis_tag)
if allow_commentary:
commentary_tag = OptionalFormat(
content=SequenceFormat(
elements=[
TagFormat(
begin="<|channel|>commentary<|message|>",
content=_ANY_CONTENT,
end="<|end|>",
),
ConstStringFormat(value="<|start|>assistant"),
]
)
)
tags.append(commentary_tag)
tags.append(content)
return StructuralTag(format=SequenceFormat(elements=tags))
@register_vllm_structural_tag("harmony")
def get_harmony_structural_tag(
tools: list[FunctionToolParam],
builtin_tools: list[BuiltinToolParam],
tool_choice: SimplifiedToolChoice,
reasoning: bool,
) -> StructuralTag:
# reasoning always enabled for Harmony
del reasoning
if builtin_tools:
# Fallback for built-in tools
tags = [
TagFormat(
begin="to=",
content=AnyTextFormat(excludes=["<|start|>"]),
end=_END_TAG,
)
]
tags.extend(
TagFormat(
begin=channel + " to=",
content=AnyTextFormat(excludes=["<|start|>", "<|channel|>"]),
end=_END_TAG,
)
for channel in _TOOL_CALL_CHANNELS
)
else:
tags = [
TagFormat(
begin=pattern.format(name=tool.function.name, channel=channel),
content=JSONSchemaFormat(
json_schema=get_function_parameters(tool.function)
),
end=_END_TAG,
)
for tool in tools
for pattern in _FUNCTION_CALL_BEGINS
for channel in _TOOL_CALL_CHANNELS
]
if tool_choice == "auto":
tags.append(
TagFormat(
begin=_FINAL_BEGIN.format(constrain=" <|constrain|>json"),
content=_ANY_CONTENT,
end=_END_TAG,
)
)
tags.append(
TagFormat(
begin=_FINAL_BEGIN.format(constrain=""),
content=_ANY_CONTENT,
end=_END_TAG,
)
)
return _assemble_tag(
allow_analysis=True, allow_commentary=True, content=OrFormat(elements=tags)
)
def _params_to_final_content(params: StructuredOutputsParams) -> Format | None:
"""Map StructuredOutputsParams in a XGrammar Format."""
if params.json_object:
return _JSON_CONTENT
if params.json is not None:
schema = params.json
if isinstance(schema, str):
schema = json.loads(schema)
return JSONSchemaFormat(json_schema=schema)
if params.regex is not None:
return RegexFormat(pattern=params.regex)
if params.choice is not None:
return OrFormat(
elements=[ConstStringFormat(value=choice) for choice in params.choice]
)
if params.grammar is not None:
return GrammarFormat(grammar=params.grammar)
if params.structural_tag is not None:
s_tag = json.loads(params.structural_tag)
if "structures" in s_tag:
# LegacyStructuralTagResponseFormat
return TriggeredTagsFormat(
triggers=s_tag["triggers"],
tags=[
TagFormat(
begin=structure["begin"],
content=JSONSchemaFormat(json_schema=structure["schema"]),
end=structure["end"],
)
for structure in s_tag["structures"]
],
)
# StructuralTagResponseFormat
return StructuralTag.model_validate(s_tag).format
return None
def _adjust_output_format(
request: ChatCompletionRequest | ResponsesRequest,
) -> ChatCompletionRequest | ResponsesRequest:
"""Canonicalize request constraints into a reasoning-aware StructuralTag."""
params = request.extract_structured_outputs()
if params is None:
return request
final_content = _params_to_final_content(params)
if final_content is None:
return request
if isinstance(final_content, JSONSchemaFormat):
begin = _FINAL_BEGIN.format(constrain=" <|constrain|>json")
else:
begin = _FINAL_BEGIN.format(constrain="")
structural_tag = _assemble_tag(
allow_analysis=True,
allow_commentary=False,
content=TagFormat(begin=begin, content=final_content, end=_END_TAG),
)
request.structured_outputs = replace(
params,
json=None,
regex=None,
choice=None,
grammar=None,
json_object=None,
structural_tag=json.dumps(structural_tag.model_dump()),
)
if isinstance(request, ResponsesRequest):
request.text = None
else:
request.response_format = None
return request
+2 -130
View File
@@ -1,65 +1,17 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import Iterable, Sequence from collections.abc import Iterable, Sequence
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from transformers import PreTrainedTokenizerBase from transformers import PreTrainedTokenizerBase
from vllm.entrypoints.mcp.tool_server import ToolServer
from vllm.entrypoints.openai.engine.protocol import DeltaMessage from vllm.entrypoints.openai.engine.protocol import DeltaMessage
from vllm.logger import init_logger
from vllm.reasoning import ReasoningParser from vllm.reasoning import ReasoningParser
if TYPE_CHECKING: if TYPE_CHECKING:
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
logger = init_logger(__name__)
no_func_reasoning_tag = {
"type": "structural_tag",
"format": {
"type": "triggered_tags",
"tags": [
{
"begin": "<|channel|>analysis<|message|>",
"content": {"type": "any_text"},
"end": "<|end|>",
}
],
"triggers": ["<|channel|>analysis"],
"stop_after_first": False,
},
}
def from_builtin_tool_to_tag(tool: str) -> list[dict]:
tag = [
{
"begin": f"<|channel|>commentary to={tool}",
"content": {"type": "any_text"},
"end": "<|end|>",
},
{
"begin": f"<|channel|>analysis to={tool}",
"content": {"type": "any_text"},
"end": "<|end|>",
},
]
return tag
def tag_with_builtin_funcs(no_func_reasoning_tag, builtin_tool_list: list[str]) -> dict:
import copy
new_tag = copy.deepcopy(no_func_reasoning_tag)
new_tag["format"]["triggers"].append("<|channel|>commentary to=")
for tool in builtin_tool_list:
new_tag["format"]["tags"].extend(from_builtin_tool_to_tag(tool))
return new_tag
class GptOssReasoningParser(ReasoningParser): class GptOssReasoningParser(ReasoningParser):
""" """
@@ -71,64 +23,14 @@ class GptOssReasoningParser(ReasoningParser):
def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs): def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs):
super().__init__(tokenizer, *args, **kwargs) super().__init__(tokenizer, *args, **kwargs)
# The model can output some special tokens between "final" and "<|message|>"
# So we need to look for both sequences to determine the end of reasoning.
self.reasoning_end_token_ids_prefix = self.model_tokenizer.encode(
"<|channel|>final"
)
self.reasoning_end_token_ids_suffix = self.model_tokenizer.encode("<|message|>")
# We also need to check for the <|end|> token to avoid false positives from
# previous messages in multi-turn conversations.
self.eom_token_id = self.vocab["<|end|>"]
self.reasoning_max_num_between_tokens = 20
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
end_token_ids_prefix = self.reasoning_end_token_ids_prefix return True
end_token_ids_suffix = self.reasoning_end_token_ids_suffix
assert len(end_token_ids_prefix) > 0, "reasoning_end_token_ids_prefix is empty"
assert len(end_token_ids_suffix) > 0, "reasoning_end_token_ids_suffix is empty"
# Check if the end sequence is present in the input_ids.
# We search from the end of input_ids to find the last match.
for i in range(len(input_ids) - len(end_token_ids_prefix), -1, -1):
if input_ids[i] == self.eom_token_id:
# We looped backwards far enough to find the end of a previous message,
# which means we have searched the entirety of the current message
# and can exit early without searching further back into prior
# messages of the conversation.
return False
if input_ids[i : i + len(end_token_ids_prefix)] == end_token_ids_prefix:
# We have found the prefix, now we look for the suffix after the prefix.
suffix_start = i + len(end_token_ids_prefix)
for j in range(
suffix_start, len(input_ids) - len(end_token_ids_suffix) + 1
):
if j - suffix_start >= self.reasoning_max_num_between_tokens:
break
if (
input_ids[j : j + len(end_token_ids_suffix)]
== end_token_ids_suffix
):
return True
return False
def is_reasoning_end_streaming( def is_reasoning_end_streaming(
self, input_ids: Sequence[int], delta_ids: Iterable[int] self, input_ids: Sequence[int], delta_ids: Iterable[int]
) -> bool: ) -> bool:
# The pattern window covers the end-of-reasoning marker itself. return True
# We add len(delta_ids) so that under speculative decoding (where
# a single step can accept many tokens) the entire accepted chunk
# is always inside the scan region.
delta_ids = tuple(delta_ids)
pattern_len = (
len(self.reasoning_end_token_ids_prefix)
+ self.reasoning_max_num_between_tokens
+ len(self.reasoning_end_token_ids_suffix)
)
window = pattern_len + len(delta_ids)
n = len(input_ids)
if n <= window:
return self.is_reasoning_end(input_ids)
return self.is_reasoning_end(input_ids[n - window :])
def extract_content_ids(self, input_ids: list[int]) -> list[int]: def extract_content_ids(self, input_ids: list[int]) -> list[int]:
raise NotImplementedError( raise NotImplementedError(
@@ -159,33 +61,3 @@ class GptOssReasoningParser(ReasoningParser):
"GptOssReasoningParser only provides boundary detection. " "GptOssReasoningParser only provides boundary detection. "
"Use HarmonyParser for output parsing." "Use HarmonyParser for output parsing."
) )
# This function prepares the structural tag to format reasoning output
def prepare_structured_tag(
self, original_tag: str | None, tool_server: ToolServer | None
) -> str | None:
if original_tag is None:
if tool_server is None:
return json.dumps(no_func_reasoning_tag)
else:
builtin_tool_list: list[str] = []
if tool_server.has_tool("browser"):
builtin_tool_list.append("browser")
if tool_server.has_tool("python"):
builtin_tool_list.append("python")
if tool_server.has_tool("container"):
builtin_tool_list.append("container")
if len(builtin_tool_list) > 0:
logger.info("Builtin_tool_list: %s", builtin_tool_list)
func_tag = json.dumps(
tag_with_builtin_funcs(no_func_reasoning_tag, builtin_tool_list)
)
else:
logger.info("Builtin_tool_list is empty")
func_tag = json.dumps(no_func_reasoning_tag)
return func_tag
else:
# There is potential risk for appending the tag to the original tag
return original_tag
+9
View File
@@ -201,6 +201,15 @@ class OnlineRenderer:
) )
else: else:
# For GPT-OSS. # For GPT-OSS.
if self.parser is not None:
# HarmonyParser doesn't need chat_template_kwargs
# TODO: Unify adjust_request() call with non-harmony branch
self.parser(
self.renderer.get_tokenizer(),
request.tools,
model_config=self.model_config,
).adjust_request(request=request)
should_include_tools = tool_dicts is not None should_include_tools = tool_dicts is not None
conversation, engine_inputs = self._make_request_with_harmony( conversation, engine_inputs = self._make_request_with_harmony(
request, should_include_tools request, should_include_tools
+2
View File
@@ -22,6 +22,8 @@ class GptOssToolParser(ToolParser):
capability declaration via HarmonyParser.tool_parser_cls. capability declaration via HarmonyParser.tool_parser_cls.
""" """
structural_tag_model = "harmony"
def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None): def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None):
super().__init__(tokenizer, tools) super().__init__(tokenizer, tools)
+4 -5
View File
@@ -65,7 +65,6 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
"qwen_3_5", "qwen_3_5",
"qwen_3_coder", "qwen_3_coder",
"qwen_3", "qwen_3",
"harmony",
"deepseek_v3_2", "deepseek_v3_2",
"glm_4_7", "glm_4_7",
"deepseek_v4", "deepseek_v4",
@@ -209,7 +208,7 @@ def _dump_allowed_tool_ref_for_xgrammar(tool_ref: AllowedToolRef) -> AllowedTool
return tool_ref return tool_ref
def _get_function_parameters(function) -> dict[str, Any] | bool: def get_function_parameters(function) -> dict[str, Any] | bool:
if getattr(function, "strict", None) is False: if getattr(function, "strict", None) is False:
return True return True
return function.parameters if function.parameters is not None else True return function.parameters if function.parameters is not None else True
@@ -230,7 +229,7 @@ def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
TagFormat( TagFormat(
begin=begin + tool.function.name + arguments_field_prefix, begin=begin + tool.function.name + arguments_field_prefix,
content=JSONSchemaFormat( content=JSONSchemaFormat(
json_schema=_get_function_parameters(tool.function) json_schema=get_function_parameters(tool.function)
), ),
end=end, end=end,
) )
@@ -279,7 +278,7 @@ def _minimax_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
TagFormat( TagFormat(
begin=f'<invoke name="{tool.function.name}">\n', begin=f'<invoke name="{tool.function.name}">\n',
content=JSONSchemaFormat( content=JSONSchemaFormat(
json_schema=_get_function_parameters(tool.function), json_schema=get_function_parameters(tool.function),
style="minimax_xml", style="minimax_xml",
), ),
end="</invoke>\n", end="</invoke>\n",
@@ -537,7 +536,7 @@ def _k3_arguments_block(parameters: dict[str, Any] | bool) -> Any:
def _k3_call_tag(tool: FunctionToolParam) -> TagFormat: def _k3_call_tag(tool: FunctionToolParam) -> TagFormat:
"""One ``call`` tag: ``<|open|>call tool="N" index="<digits>"<|sep|> args``.""" """One ``call`` tag: ``<|open|>call tool="N" index="<digits>"<|sep|> args``."""
function = tool.function function = tool.function
parameters = _get_function_parameters(function) parameters = get_function_parameters(function)
begin = f'{_K3_OPEN}call tool="{_k3_escape_attr(function.name)}" index="' begin = f'{_K3_OPEN}call tool="{_k3_escape_attr(function.name)}" index="'
return TagFormat( return TagFormat(
begin=begin, begin=begin,