[GPT-OSS] Strict tool call and constrained decoding for Harmony (#45560)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-08-01 12:10:26 -04:00
committed by GitHub
parent 127a7cebc9
commit dc818c198d
10 changed files with 632 additions and 511 deletions
@@ -13,7 +13,7 @@ from typing import Any
import pytest
import pytest_asyncio
import requests
from openai import InternalServerError, NotFoundError, OpenAI
from openai import NotFoundError, OpenAI
from openai_harmony import Message
from tests.utils import RemoteOpenAIServer
@@ -368,8 +368,12 @@ async def test_streaming_types(
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
async def test_function_calling_with_streaming_types(
pairs_of_event_types: dict[str, str], client: OpenAI, model_name: str
pairs_of_event_types: dict[str, str],
client: OpenAI,
model_name: str,
tool_choice: str,
):
"""Streaming event nesting for function-calling responses."""
@@ -382,6 +386,7 @@ async def test_function_calling_with_streaming_types(
validate_events=_has_function_events,
input=[{"role": "user", "content": "What's the weather like in Paris today?"}],
tools=[GET_WEATHER_SCHEMA],
tool_choice=tool_choice,
temperature=0.0,
)
@@ -558,7 +563,8 @@ async def test_reasoning_item(client: OpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_function_calling(client: OpenAI, model_name: str):
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
async def test_function_calling(client: OpenAI, model_name: str, tool_choice: str):
tools = [GET_WEATHER_SCHEMA]
response = await retry_for_tool_call(
@@ -567,8 +573,9 @@ async def test_function_calling(client: OpenAI, model_name: str):
expected_tool_type="function_call",
input="What's the weather like in Paris today?",
tools=tools,
tool_choice=tool_choice,
temperature=0.0,
extra_body={"request_id": "test_function_calling_non_resp"},
extra_body={"request_id": f"test_function_calling_non_resp_{tool_choice}"},
)
assert response.status == "completed"
assert has_output_type(response, "function_call"), (
@@ -610,7 +617,10 @@ async def test_function_calling(client: OpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
async def test_function_calling_multi_turn(
client: OpenAI, model_name: str, tool_choice: str
):
"""Multi-tool, multi-turn function calling with retry at API level."""
tools = [
{
@@ -635,6 +645,7 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
expected_tool_type="function_call",
input="Help me plan a trip to a random place. And tell me the weather there.",
tools=tools,
tool_choice=tool_choice,
temperature=0.0,
)
assert response.status == "completed"
@@ -659,6 +670,7 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
}
],
tools=tools,
tool_choice=tool_choice,
previous_response_id=response.id,
temperature=0.0,
)
@@ -695,20 +707,6 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_function_calling_required(client: OpenAI, model_name: str):
tools = [GET_WEATHER_SCHEMA]
with pytest.raises(InternalServerError):
await client.responses.create(
model=model_name,
input="What's the weather like in Paris today?",
tools=tools,
tool_choice="required",
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_system_message_with_tools(client: OpenAI, model_name: str):
@@ -726,7 +724,10 @@ async def test_system_message_with_tools(client: OpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_function_calling_full_history(client: OpenAI, model_name: str):
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
async def test_function_calling_full_history(
client: OpenAI, model_name: str, tool_choice: str
):
tools = [GET_WEATHER_SCHEMA]
input_messages = [
@@ -739,6 +740,7 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str):
expected_tool_type="function_call",
input=input_messages,
tools=tools,
tool_choice=tool_choice,
temperature=0.0,
)
assert response.status == "completed"
@@ -772,7 +774,10 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_function_calling_with_stream(client: OpenAI, model_name: str):
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
async def test_function_calling_with_stream(
client: OpenAI, model_name: str, tool_choice: str
):
"""Function calling via streaming, with retry for non-determinism."""
tools = [GET_WEATHER_SCHEMA]
input_list = [
@@ -792,6 +797,7 @@ async def test_function_calling_with_stream(client: OpenAI, model_name: str):
validate_events=_has_function_call,
input=input_list,
tools=tools,
tool_choice=tool_choice,
temperature=0.0,
)
@@ -853,8 +859,9 @@ async def test_function_calling_with_stream(client: OpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
async def test_function_calling_no_code_interpreter_events(
client: OpenAI, model_name: str
client: OpenAI, model_name: str, tool_choice: str
):
"""Verify that function calls don't trigger code_interpreter events.
@@ -880,6 +887,7 @@ async def test_function_calling_no_code_interpreter_events(
validate_events=_has_function_call,
input=input_list,
tools=tools,
tool_choice=tool_choice,
temperature=0.0,
)
@@ -1048,8 +1056,9 @@ async def test_output_messages_enabled(client: OpenAI, model_name: str, server):
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
async def test_function_call_with_previous_input_messages(
client: OpenAI, model_name: str
client: OpenAI, model_name: str, tool_choice: str
):
"""Multi-turn function calling using previous_input_messages."""
tools = [
@@ -1074,6 +1083,7 @@ async def test_function_call_with_previous_input_messages(
expected_tool_type="function_call",
input="What is the horoscope for Aquarius today?",
tools=tools,
tool_choice=tool_choice,
temperature=0.0,
extra_body={"enable_response_messages": True},
max_output_tokens=1000,
+337
View File
@@ -3,6 +3,7 @@
import json
from collections.abc import Sequence
from typing import Any, Literal
import pytest
from openai_harmony import (
@@ -12,14 +13,18 @@ from openai_harmony import (
Role,
)
from transformers import AutoTokenizer
from xgrammar import Grammar
from xgrammar.testing import _is_grammar_accept_string
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.engine.protocol import FunctionCall
from vllm.entrypoints.openai.parser.harmony_utils import (
get_encoding,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.harmony import HarmonyParser
from vllm.parser.parser_manager import ParserManager
from vllm.sampling_params import StructuredOutputsParams
REASONING_MODEL_NAME = "openai/gpt-oss-20b"
@@ -857,3 +862,335 @@ class TestProcessChunk:
("analysis", "One"),
("final", "Two"),
]
class TestAdjustRequest:
REQUEST_TEXT = "Hello"
TOOL_TYPE = "function"
TOOL_1_NAME = "get_user_location"
TOOL_2_NAME = "get_weather"
TOOLS = [
{
"name": TOOL_1_NAME,
"parameters": {
"type": "object",
"properties": {},
"required": [],
},
},
{
"name": TOOL_2_NAME,
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
]
OUTPUT_SCHEMA = {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"],
}
ANALYSIS = "<|channel|>analysis<|message|>analysis message<|end|><|start|>assistant"
COMMENTARY = (
"<|channel|>commentary<|message|>commentary message<|end|><|start|>assistant"
)
TOOL_CALL_1 = (
ANALYSIS + f"<|channel|>commentary to=functions.{TOOL_1_NAME} json<|message|>"
"{}<|call|>"
)
TOOL_CALL_2 = (
ANALYSIS + f"<|channel|>commentary to=functions.{TOOL_2_NAME} json<|message|>"
'{"city": "Tokyo"}<|call|>'
)
FINAL_JSON_SCHEMA = (
ANALYSIS
+ '<|channel|>final <|constrain|>json<|message|>{"answer": "Tokyo"}<|end|>'
)
FINAL_JSON_OBJECT = (
ANALYSIS
+ '<|channel|>final <|constrain|>json<|message|>{"city": "Tokyo"}<|end|>'
)
FINAL_TEXT_ONLY = ANALYSIS + "<|channel|>final<|message|>any<|end|>"
FINAL_REGEX = ANALYSIS + "<|channel|>final<|message|>regex<|end|>"
FINAL_CHOICE = ANALYSIS + "<|channel|>final<|message|>choice1<|end|>"
FINAL_GRAMMAR = ANALYSIS + "<|channel|>final<|message|>grammar<|end|>"
FINAL_STRUCTURAL_TAG = ANALYSIS + "<|channel|>final<|message|>tag content<|end|>"
ADMISSION_SAMPLES = (
"COMMENTARY",
"TOOL_CALL_1",
"TOOL_CALL_2",
"FINAL_JSON_SCHEMA",
"FINAL_JSON_OBJECT",
"FINAL_TEXT_ONLY",
"FINAL_REGEX",
"FINAL_CHOICE",
"FINAL_GRAMMAR",
"FINAL_STRUCTURAL_TAG",
)
@staticmethod
def _build_request(
request_kind: Literal["chat", "responses"],
tool_choice: str = "none",
strict_tools: bool = False,
response_format_type: str | None = None,
structured_outputs: StructuredOutputsParams | None = None,
) -> ChatCompletionRequest | ResponsesRequest:
data: dict[str, Any] = {
"model": REASONING_MODEL_NAME,
}
if request_kind == "chat":
data["messages"] = [
{
"role": "user",
"content": TestAdjustRequest.REQUEST_TEXT,
}
]
else:
data["input"] = TestAdjustRequest.REQUEST_TEXT
if request_kind == "chat":
data["tools"] = [
{
"type": TestAdjustRequest.TOOL_TYPE,
"function": {"strict": strict_tools, **tool_def},
}
for tool_def in TestAdjustRequest.TOOLS
]
data["tool_choice"] = (
{
"type": TestAdjustRequest.TOOL_TYPE,
"function": {"name": TestAdjustRequest.TOOL_2_NAME},
}
if tool_choice == "named"
else tool_choice
)
else:
data["tools"] = [
{
"type": TestAdjustRequest.TOOL_TYPE,
"strict": strict_tools,
**tool_def,
}
for tool_def in TestAdjustRequest.TOOLS
]
data["tool_choice"] = (
{
"type": TestAdjustRequest.TOOL_TYPE,
"name": TestAdjustRequest.TOOL_2_NAME,
}
if tool_choice == "named"
else tool_choice
)
if response_format_type == "json_schema":
schema_format = {
"name": "answer_format",
"schema": TestAdjustRequest.OUTPUT_SCHEMA,
"strict": True,
}
if request_kind == "chat":
data["response_format"] = {
"type": "json_schema",
"json_schema": schema_format,
}
else:
data["text"] = {
"format": {
"type": "json_schema",
**schema_format,
}
}
elif response_format_type == "json_object":
if request_kind == "chat":
data["response_format"] = {"type": "json_object"}
else:
data["text"] = {"format": {"type": "json_object"}}
if structured_outputs is not None:
data["structured_outputs"] = structured_outputs
if request_kind == "chat":
return ChatCompletionRequest.model_validate(data)
return ResponsesRequest.model_validate(data)
@staticmethod
def _assert_format_cleared(
adjusted_request: ChatCompletionRequest | ResponsesRequest,
) -> None:
if isinstance(adjusted_request, ResponsesRequest):
assert adjusted_request.text is None or adjusted_request.text.format is None
else:
assert adjusted_request.response_format is None
structured_outputs = adjusted_request.structured_outputs
assert structured_outputs is not None
assert structured_outputs.structural_tag is not None
assert structured_outputs.all_non_structural_tag_constraints_none()
@classmethod
def _assert_structured_outputs_admission(
cls,
adjusted_request: ChatCompletionRequest | ResponsesRequest,
expected_admission: Sequence[str],
) -> None:
structured_outputs = adjusted_request.structured_outputs
assert structured_outputs is not None
assert structured_outputs.structural_tag is not None
assert structured_outputs.all_non_structural_tag_constraints_none()
grammar = Grammar.from_structural_tag(structured_outputs.structural_tag)
expected_admission_set = set(expected_admission)
for sample_name in cls.ADMISSION_SAMPLES:
admitted = _is_grammar_accept_string(
grammar,
getattr(cls, sample_name),
require_termination=False,
)
should_admit = sample_name in expected_admission_set
assert admitted is should_admit, (
f"Expected structured_outputs admission for {sample_name} "
f"to be {should_admit}, got {admitted}."
)
@pytest.mark.parametrize("request_kind", ["chat", "responses"])
@pytest.mark.parametrize(
("request_kwargs", "expected_admission"),
[
(
{"tool_choice": "auto", "strict_tools": True},
[
"COMMENTARY",
"TOOL_CALL_1",
"TOOL_CALL_2",
"FINAL_JSON_SCHEMA",
"FINAL_JSON_OBJECT",
"FINAL_TEXT_ONLY",
"FINAL_REGEX",
"FINAL_CHOICE",
"FINAL_GRAMMAR",
"FINAL_STRUCTURAL_TAG",
],
),
(
{"tool_choice": "required"},
["COMMENTARY", "TOOL_CALL_1", "TOOL_CALL_2"],
),
(
{"tool_choice": "named"},
["COMMENTARY", "TOOL_CALL_2"],
),
(
{"response_format_type": "json_schema"},
["FINAL_JSON_SCHEMA"],
),
(
{"response_format_type": "json_object"},
["FINAL_JSON_SCHEMA", "FINAL_JSON_OBJECT"],
),
(
{"structured_outputs": StructuredOutputsParams(json=OUTPUT_SCHEMA)},
["FINAL_JSON_SCHEMA"],
),
(
{"structured_outputs": StructuredOutputsParams(json_object=True)},
["FINAL_JSON_SCHEMA", "FINAL_JSON_OBJECT"],
),
(
{"structured_outputs": StructuredOutputsParams(regex=r"regex")},
["FINAL_REGEX"],
),
(
{
"structured_outputs": StructuredOutputsParams(
choice=["choice1", "choice2"]
)
},
["FINAL_CHOICE"],
),
(
{
"structured_outputs": StructuredOutputsParams(
grammar='root ::= "grammar"'
)
},
["FINAL_GRAMMAR"],
),
(
{
"structured_outputs": StructuredOutputsParams(
structural_tag=json.dumps(
{
"type": "structural_tag",
"format": {
"type": "json_schema",
"json_schema": OUTPUT_SCHEMA,
},
}
)
)
},
["FINAL_JSON_SCHEMA"],
),
(
{
"structured_outputs": StructuredOutputsParams(
structural_tag=json.dumps(
{
"type": "structural_tag",
"structures": [
{
"begin": "<tag>",
"schema": {"type": "object"},
"end": "</tag>",
}
],
"triggers": ["<tag>"],
}
)
)
},
[
# Legacy triggered tags allow free text until a trigger, so
# unconstrained final-channel payloads are also admitted.
"FINAL_TEXT_ONLY",
"FINAL_REGEX",
"FINAL_CHOICE",
"FINAL_GRAMMAR",
"FINAL_STRUCTURAL_TAG",
],
),
],
ids=[
"tool_auto_strict",
"tool_required",
"tool_named",
"response_format_json_schema",
"response_format_json_object",
"structured_outputs_json",
"structured_outputs_json_object",
"structured_outputs_regex",
"structured_outputs_choice",
"structured_outputs_grammar",
"structured_outputs_structural_tag_modern",
"structured_outputs_structural_tag_legacy",
],
)
def test_adjust_request(
self,
harmony_parser,
request_kind,
request_kwargs,
expected_admission,
):
request = self._build_request(request_kind, **request_kwargs)
adjusted_request = harmony_parser.adjust_request(request)
self._assert_format_cleared(adjusted_request)
self._assert_structured_outputs_admission(
adjusted_request,
expected_admission,
)
+5 -344
View File
@@ -1,351 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from unittest.mock import Mock
import pytest
from transformers import AutoTokenizer
from vllm.entrypoints.mcp.tool_server import ToolServer
from vllm.reasoning import ReasoningParser
from vllm.reasoning.gptoss_reasoning_parser import (
GptOssReasoningParser,
from_builtin_tool_to_tag,
no_func_reasoning_tag,
)
REASONING_MODEL_NAME = "openai/gpt-oss-120b"
from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
@pytest.fixture(scope="module")
def gpt_oss_tokenizer():
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
USER_MESSAGE_START = "<|start|>user<|message|>"
REASONING_SECTION_START = "<|end|><|start|>assistant<|channel|>analysis<|message|>"
END = "<|end|>"
ASSISTANT_START = "<|start|>assistant"
ASSISTANT_CONTENT_START_PREFIX = END + ASSISTANT_START + "<|channel|>final"
ASSISTANT_CONTENT_START_SUFFIX = "<|message|>"
ASSISTANT_CONTENT_START = (
ASSISTANT_CONTENT_START_PREFIX + ASSISTANT_CONTENT_START_SUFFIX
)
BASIC_CONTENT = {
"output": REASONING_SECTION_START
+ "This is reasoning"
+ ASSISTANT_CONTENT_START
+ "This is the rest",
"is_reasoning_end": True,
}
BASIC_REASONING_ONLY = {
"output": REASONING_SECTION_START + "This is reasoning" + "<|end|>",
"is_reasoning_end": False,
}
BASIC_NO_REASONING_NO_ASSISTANT = {
"output": USER_MESSAGE_START + "This is a user message",
"is_reasoning_end": False,
}
# Edge-case where the model omits the assistant tag entirely.
BASIC_NO_REASONING_ASSISTANT = {
"output": USER_MESSAGE_START + "This is a user message<|end|><|channel|>final",
"is_reasoning_end": True,
}
COMPLEX_CONTENT_INCOMPLETE_PREFIX_ONLY = {
"output": REASONING_SECTION_START
+ "This is reasoning"
+ ASSISTANT_CONTENT_START_PREFIX,
"is_reasoning_end": False,
}
COMPLEX_CONTENT_SUFFIX_ONLY = {
"output": REASONING_SECTION_START
+ "This is reasoning"
+ ASSISTANT_CONTENT_START_SUFFIX,
"is_reasoning_end": False,
}
COMPLEX_CONTENT_1_NO_SUFFIX = {
"output": REASONING_SECTION_START
+ "This is reasoning"
+ ASSISTANT_CONTENT_START_PREFIX
+ "<|constrain|> JSON ",
"is_reasoning_end": False,
}
COMPLEX_CONTENT_1 = {
"output": REASONING_SECTION_START
+ "This is reasoning"
+ ASSISTANT_CONTENT_START_PREFIX
+ "<|constrain|> JSON "
+ ASSISTANT_CONTENT_START_SUFFIX,
"is_reasoning_end": True,
}
COMPLEX_CONTENT_1_WITH_CONTENT = {
"output": REASONING_SECTION_START
+ "This is reasoning"
+ ASSISTANT_CONTENT_START_PREFIX
+ "<|constrain|> JSON "
+ ASSISTANT_CONTENT_START_SUFFIX
+ "This is the rest",
"is_reasoning_end": True,
}
COMPLEX_CONTENT_2 = {
"output": REASONING_SECTION_START
+ "This is reasoning"
+ ASSISTANT_CONTENT_START_PREFIX
+ "<|constrain|>ReplyAction "
+ ASSISTANT_CONTENT_START_SUFFIX
+ "This is the rest",
"is_reasoning_end": True,
}
MULTI_TURN_CONTENT = {
"output": USER_MESSAGE_START
+ "1st turn user message"
+ REASONING_SECTION_START
+ "1st turn reasoning"
+ ASSISTANT_CONTENT_START
+ "1st turn response"
+ END
+ USER_MESSAGE_START
+ "2nd turn user message"
+ END
+ ASSISTANT_START,
"is_reasoning_end": False,
}
TEST_CASES = [
BASIC_CONTENT,
BASIC_REASONING_ONLY,
COMPLEX_CONTENT_INCOMPLETE_PREFIX_ONLY,
COMPLEX_CONTENT_SUFFIX_ONLY,
COMPLEX_CONTENT_1_NO_SUFFIX,
COMPLEX_CONTENT_1,
COMPLEX_CONTENT_1_WITH_CONTENT,
COMPLEX_CONTENT_2,
MULTI_TURN_CONTENT,
]
@pytest.mark.parametrize(
"output, is_reasoning_end",
[(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
)
def test_gptoss_is_reasoning_end(
output,
is_reasoning_end,
gpt_oss_tokenizer,
):
output = gpt_oss_tokenizer.tokenize(output)
parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
# Test is_reasoning_end
output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(output)
actual_is_reasoning_end = parser.is_reasoning_end(output_ids)
assert is_reasoning_end == actual_is_reasoning_end
class TestGptOssStructuralTags:
"""Test cases for GptOssReasoningParser structural tag functionality."""
@pytest.fixture
def mock_tokenizer(self):
"""Create a mock tokenizer for testing."""
tokenizer = Mock()
tokenizer.encode = Mock(return_value=[1, 2, 3, 4, 5])
tokenizer.get_vocab = Mock(return_value={"<|end|>": 6})
return tokenizer
@pytest.fixture
def reasoning_parser(self, mock_tokenizer):
"""Create a GptOssReasoningParser instance."""
return GptOssReasoningParser(mock_tokenizer)
def test_prepare_structured_tag_no_tool_server(self, reasoning_parser):
"""Test prepare_structured_tag with no tool server."""
result = reasoning_parser.prepare_structured_tag(None, None)
expected = json.dumps(no_func_reasoning_tag)
assert result == expected
# Verify the structure is correct
parsed = json.loads(result)
assert parsed["type"] == "structural_tag"
assert parsed["format"]["type"] == "triggered_tags"
assert len(parsed["format"]["tags"]) == 1
assert parsed["format"]["tags"][0]["begin"] == "<|channel|>analysis<|message|>"
assert parsed["format"]["triggers"] == ["<|channel|>analysis"]
def test_prepare_structured_tag_with_original_tag(self, reasoning_parser):
"""Test prepare_structured_tag when original_tag is provided."""
original_tag = '{"custom": "tag"}'
result = reasoning_parser.prepare_structured_tag(original_tag, None)
# Should return the original tag unchanged
assert result == original_tag
def test_from_builtin_tool_to_tag(self):
"""Test from_builtin_tool_to_tag function."""
tags = from_builtin_tool_to_tag("python")
assert len(tags) == 2
assert tags[0]["begin"] == "<|channel|>commentary to=python"
assert tags[0]["content"]["type"] == "any_text"
assert tags[0]["end"] == "<|end|>"
assert tags[1]["begin"] == "<|channel|>analysis to=python"
assert tags[1]["content"]["type"] == "any_text"
assert tags[1]["end"] == "<|end|>"
@pytest.mark.parametrize(
"tools",
[
[],
["browser"],
["python"],
["container"],
["browser", "python"],
["browser", "container"],
["python", "container"],
["browser", "python", "container"],
],
)
def test_json_validity_comprehensive(self, reasoning_parser, tools):
"""Test JSON validity across all possible tool combinations."""
tool_server = Mock(spec=ToolServer)
tool_server.has_tool = Mock(side_effect=lambda tool: tool in tools)
result = reasoning_parser.prepare_structured_tag(None, tool_server)
parsed_result = json.loads(result)
assert parsed_result["type"] == "structural_tag"
assert "format" in parsed_result
assert "tags" in parsed_result["format"]
assert "triggers" in parsed_result["format"]
# Tag count should be: 1 (analysis) + 2 * len(tools)
expected_tag_count = 1 + (2 * len(tools))
assert len(parsed_result["format"]["tags"]) == expected_tag_count
# Verify triggers are correctly configured
expected_triggers = ["<|channel|>analysis"]
if tools:
expected_triggers.append("<|channel|>commentary to=")
assert set(parsed_result["format"]["triggers"]) == set(expected_triggers)
def test_no_cross_request_state_pollution(self, reasoning_parser):
"""Test that sequential calls with different tool servers produce
independent results, guarding against shared mutable state
(e.g. missing deepcopy in tag_with_builtin_funcs)."""
tool_server_1 = Mock(spec=ToolServer)
tool_server_1.has_tool = Mock(side_effect=lambda tool: tool == "python")
tool_server_2 = Mock(spec=ToolServer)
tool_server_2.has_tool = Mock(side_effect=lambda tool: tool == "browser")
result_1 = reasoning_parser.prepare_structured_tag(None, tool_server_1)
result_2 = reasoning_parser.prepare_structured_tag(None, tool_server_2)
tags_1 = [tag["begin"] for tag in json.loads(result_1)["format"]["tags"]]
tags_2 = [tag["begin"] for tag in json.loads(result_2)["format"]["tags"]]
assert "<|channel|>commentary to=python" in tags_1
assert "<|channel|>commentary to=browser" not in tags_1
assert "<|channel|>commentary to=browser" in tags_2
assert "<|channel|>commentary to=python" not in tags_2
def test_tag_format_consistency(self, reasoning_parser):
"""Test that all generated tags follow consistent format,
catching malformed tags from from_builtin_tool_to_tag."""
tool_server = Mock(spec=ToolServer)
tool_server.has_tool = Mock(
side_effect=lambda tool: tool in ["python", "browser"]
)
result = reasoning_parser.prepare_structured_tag(None, tool_server)
parsed_result = json.loads(result)
for tag in parsed_result["format"]["tags"]:
assert "begin" in tag
assert "content" in tag
assert "end" in tag
assert tag["content"]["type"] == "any_text"
assert tag["end"] == "<|end|>"
assert tag["begin"].startswith("<|channel|>")
@pytest.mark.parametrize(
"output, is_reasoning_end",
[(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
)
def test_gptoss_is_reasoning_end_streaming(
output,
is_reasoning_end,
gpt_oss_tokenizer,
):
"""Streaming override must agree with is_reasoning_end for all cases."""
tokens = gpt_oss_tokenizer.tokenize(output)
parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens)
delta_ids = output_ids[-1:] if output_ids else []
actual = parser.is_reasoning_end_streaming(output_ids, delta_ids)
assert is_reasoning_end == actual
@pytest.mark.parametrize(
"output, is_reasoning_end",
[(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
)
def test_gptoss_is_reasoning_end_streaming_long_prefix(
output,
is_reasoning_end,
gpt_oss_tokenizer,
):
"""Windowing must produce correct results even with a long prefix."""
tokens = gpt_oss_tokenizer.tokenize(output)
parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens)
# Prepend 10k dummy reasoning tokens to simulate a long generation
long_prefix = [1] * 10_000
padded_ids = long_prefix + list(output_ids)
delta_ids = output_ids[-1:] if output_ids else []
actual = parser.is_reasoning_end_streaming(padded_ids, delta_ids)
assert is_reasoning_end == actual
@pytest.mark.parametrize(
"output, is_reasoning_end",
[(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
)
def test_gptoss_is_reasoning_end_streaming_large_delta(
output,
is_reasoning_end,
gpt_oss_tokenizer,
):
"""Simulate speculative decoding where the entire test sequence arrives
as a single large delta appended after a long prefix. The window must
expand to cover delta_ids so the end pattern is never missed."""
tokens = gpt_oss_tokenizer.tokenize(output)
parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens)
long_prefix = [1] * 10_000
padded_ids = long_prefix + list(output_ids)
# delta_ids = the entire test sequence (as if accepted in one spec step)
delta_ids = list(output_ids)
actual = parser.is_reasoning_end_streaming(padded_ids, delta_ids)
assert is_reasoning_end == actual
def test_gptoss_is_reasoning_end_streaming_signature(gpt_oss_tokenizer):
"""Verify the method is callable with the expected signature."""
parser = GptOssReasoningParser(gpt_oss_tokenizer)
result = parser.is_reasoning_end_streaming([], [])
assert result is False
def test_gptoss_reasoning_ended_is_true():
parser = GptOssReasoningParser(Mock())
assert parser.is_reasoning_end([]) is True
assert parser.is_reasoning_end_streaming([], []) is True
@@ -33,7 +33,7 @@ from vllm.tool_parsers.structural_tag_registry import (
SUPPORTED_STRUCTURAL_TAG_MODELS,
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS,
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS,
_get_function_parameters,
get_function_parameters,
get_model_structural_tag,
)
@@ -530,7 +530,7 @@ def test_get_function_parameters_relaxes_function_strict_false():
strict=False,
)
assert _get_function_parameters(function) is True
assert get_function_parameters(function) is True
def _k3_tools_with_root_defs() -> list[ChatCompletionToolsParam]: