[Frontend] Support strict mode for tool calling (#45003)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Co-authored-by: cjackal <44624812+cjackal@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Chauncey
2026-06-12 07:51:48 +00:00
committed by GitHub
co-authored by cjackal mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent bd59c913bc
commit 2043258dec
29 changed files with 692 additions and 1956 deletions
@@ -250,6 +250,7 @@ async def k2_client(k2_server):
@pytest.mark.asyncio
@pytest.mark.skip(reason="Skipping Kimi K2 tool ID test")
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("stream", [True, False])
@pytest.mark.parametrize("tool_choice", ["required"])
@@ -442,7 +443,7 @@ async def test_named_tool_use(
if delta.role:
assert delta.role == "assistant"
assert delta.content is None or len(delta.content) == 0
if delta.tool_calls:
if delta.tool_calls and delta.tool_calls[0].function.arguments:
output.append(delta.tool_calls[0].function.arguments)
if chunk.choices[0].finish_reason is not None:
finish_reason_count += 1
@@ -390,6 +390,7 @@ def server_with_store(default_server_args):
env_dict={
"VLLM_ENABLE_RESPONSES_API_STORE": "1",
"VLLM_SERVER_DEV_MODE": "1",
"VLLM_ENFORCE_STRICT_TOOL_CALLING": "0",
},
) as remote_server:
yield remote_server
+22 -4
View File
@@ -2,13 +2,31 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import os
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.parser.abstract_parser import DelegatingParser
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
_STRICT_TOOL_CALLING_ENV = "VLLM_ENFORCE_STRICT_TOOL_CALLING"
_STRICT_TOOL_CALLING_ENV_VALUE = os.environ.get(_STRICT_TOOL_CALLING_ENV)
os.environ[_STRICT_TOOL_CALLING_ENV] = "0"
from vllm.entrypoints.openai.chat_completion.protocol import ( # noqa: E402
ChatCompletionRequest,
)
from vllm.parser.abstract_parser import DelegatingParser # noqa: E402
from vllm.reasoning.basic_parsers import ( # noqa: E402
BaseThinkingReasoningParser,
)
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser # noqa: E402
@pytest.fixture(scope="module", autouse=True)
def restore_strict_tool_calling_env():
yield
if _STRICT_TOOL_CALLING_ENV_VALUE is None:
os.environ.pop(_STRICT_TOOL_CALLING_ENV, None)
else:
os.environ[_STRICT_TOOL_CALLING_ENV] = _STRICT_TOOL_CALLING_ENV_VALUE
class ThinkReasoningParser(BaseThinkingReasoningParser):
+13 -190
View File
@@ -3,6 +3,7 @@
import json
from collections.abc import Generator
from unittest.mock import MagicMock
import pytest
from openai.types.responses.function_tool import FunctionTool
@@ -19,15 +20,12 @@ from vllm.entrypoints.openai.engine.protocol import (
FunctionCall,
ToolCall,
)
from vllm.parser.abstract_parser import DelegatingParser
from vllm.tokenizers import TokenizerLike, get_tokenizer
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
from vllm.tool_parsers.qwen3coder_tool_parser import (
Qwen3CoderToolParser,
)
from vllm.tool_parsers.qwen3xml_tool_parser import (
Qwen3XMLToolParser,
StreamingXMLToolCallParser,
)
MODEL = "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
@@ -43,17 +41,8 @@ def qwen3_tool_parser(qwen3_tokenizer, sample_tools):
@pytest.fixture
def qwen3_xml_tool_parser(qwen3_tokenizer, sample_tools):
return Qwen3XMLToolParser(qwen3_tokenizer, tools=sample_tools)
@pytest.fixture(params=["xml"])
def qwen3_tool_parser_parametrized(qwen3_tool_parser, qwen3_xml_tool_parser, request):
"""Parameterized fixture that provides both parser types for testing"""
if request.param == "original":
return qwen3_tool_parser
else:
return qwen3_xml_tool_parser
def qwen3_tool_parser_parametrized(qwen3_tool_parser):
return qwen3_tool_parser
WEATHER_PARAMS = {
@@ -168,47 +157,6 @@ def assert_tool_calls(
)
def test_qwen3xml_deferred_array_parses_json_literals():
parser = StreamingXMLToolCallParser()
parser.set_tools(
[
ChatCompletionToolsParam(
type="function",
function={
"name": "AskUserQuestion",
"parameters": QUESTION_PARAMS,
},
)
]
)
delta = parser.parse_single_streaming_chunks(
"""<tool_call>
<function=AskUserQuestion>
<parameter=questions>
[{"question": "Pick a color", "multiSelect": false, "answer": null}]
</parameter>
</function>
</tool_call>"""
)
arguments = "".join(
tool_call.function.arguments or ""
for tool_call in delta.tool_calls or []
if tool_call.function and tool_call.function.arguments is not None
)
assert json.loads(arguments) == {
"questions": [
{
"question": "Pick a color",
"multiSelect": False,
"answer": None,
}
]
}
def stream_delta_message_generator(
qwen3_tool_parser,
qwen3_tokenizer: TokenizerLike,
@@ -523,7 +471,7 @@ hello world
</function>
</tool_call>"""
parser = Qwen3XMLToolParser(qwen3_tokenizer, tools=tools)
parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools)
request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
extracted_tool_calls = parser.extract_tool_calls(model_output, request=request)
@@ -1146,125 +1094,6 @@ TX
assert parsed_args["state"] == "TX"
def test_extract_tool_calls_complex_type_with_single_quote(
qwen3_tokenizer,
):
"""Test parameter type conversion based on tool schema"""
tools = [
ChatCompletionToolsParam(
type="function",
function={
"name": "test_types",
"parameters": {
"type": "object",
"properties": {
"int_param": {"type": "integer"},
"float_param": {"type": "float"},
"bool_param": {"type": "boolean"},
"str_param": {"type": "string"},
"obj_param": {"type": "object"},
},
},
},
)
]
model_output = """<tool_call>
<function=test_types>
<parameter=obj_param>
{'key': 'value'}
</parameter>
</function>
</tool_call>"""
parser = Qwen3XMLToolParser(qwen3_tokenizer, tools=tools)
request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
extracted_tool_calls = parser.extract_tool_calls(model_output, request=request)
args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)
assert args["obj_param"] == {"key": "value"}
def test_extract_tool_calls_streaming_missing_opening_tag(
qwen3_tool_parser_parametrized, qwen3_tokenizer
):
"""Test streaming with missing opening <tool_call> tag
This tests that the streaming parser correctly handles
tool calls that start directly with <function=...>
"""
model_output = """I'll check the weather for you.
<function=get_current_weather>
<parameter=city>
Dallas
</parameter>
<parameter=state>
TX
</parameter>
<parameter=unit>
fahrenheit
</parameter>
</function>
</tool_call>"""
request = ChatCompletionRequest(model=MODEL, messages=[])
other_content = ""
tool_states = {}
for delta_message in stream_delta_message_generator(
qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output, request
):
if delta_message.content:
other_content += delta_message.content
if delta_message.tool_calls:
for tool_call in delta_message.tool_calls:
idx = tool_call.index
if idx not in tool_states:
tool_states[idx] = {
"id": None,
"name": None,
"arguments": "",
"type": None,
}
if tool_call.id:
tool_states[idx]["id"] = tool_call.id
if tool_call.type:
assert tool_call.type == "function"
tool_states[idx]["type"] = tool_call.type
if tool_call.function:
if tool_call.function.name:
tool_states[idx]["name"] = tool_call.function.name
if tool_call.function.arguments is not None:
tool_states[idx]["arguments"] += tool_call.function.arguments
# Verify content was streamed
assert "I'll check the weather for you." in other_content
# Verify we got the tool call
assert len(tool_states) == 1
assert len(qwen3_tool_parser_parametrized.prev_tool_call_arr) == 1
state = tool_states[0]
assert state["id"] is not None
assert state["type"] == "function"
assert state["name"] == "get_current_weather"
# Verify arguments were parsed correctly despite missing opening tag
assert state["arguments"] is not None
args = json.loads(state["arguments"])
assert args["city"] == "Dallas"
assert args["state"] == "TX"
assert args["unit"] == "fahrenheit"
def test_malformed_xml_no_gt_delimiter(qwen3_tool_parser):
"""Regression: malformed XML without '>' must not crash (PR #36774)."""
model_output = (
@@ -1456,15 +1285,12 @@ def test_get_vllm_registry_structural_tag_returns_structural_tag(
@pytest.mark.parametrize("include_reasoning", [True, False])
def test_adjust_request_auto_uses_vllm_registry_structural_tag(
monkeypatch: pytest.MonkeyPatch,
qwen3_tool_parser: Qwen3CoderToolParser,
sample_tools: list[ChatCompletionToolsParam],
include_reasoning: bool,
) -> None:
monkeypatch.setattr(
"vllm.tool_parsers.abstract_tool_parser.VLLM_ENFORCE_STRICT_TOOL_CALLING",
True,
)
class TestParser(DelegatingParser):
tool_parser_cls = Qwen3CoderToolParser
request_tools = _as_chat_completion_tools(sample_tools)
req = ChatCompletionRequest(
messages=[],
@@ -1473,7 +1299,7 @@ def test_adjust_request_auto_uses_vllm_registry_structural_tag(
tool_choice="auto",
include_reasoning=include_reasoning,
)
out = qwen3_tool_parser.adjust_request(req)
out = TestParser(MagicMock(), tools=sample_tools).adjust_request(req)
assert out.structured_outputs is not None
assert out.structured_outputs.structural_tag is not None
assert isinstance(out.structured_outputs.structural_tag, str)
@@ -1482,14 +1308,11 @@ def test_adjust_request_auto_uses_vllm_registry_structural_tag(
def test_adjust_request_required_prefers_structural_tag(
monkeypatch: pytest.MonkeyPatch,
qwen3_tool_parser: Qwen3CoderToolParser,
sample_tools: list[ChatCompletionToolsParam],
) -> None:
monkeypatch.setattr(
"vllm.tool_parsers.abstract_tool_parser.VLLM_ENFORCE_STRICT_TOOL_CALLING",
True,
)
class TestParser(DelegatingParser):
tool_parser_cls = Qwen3CoderToolParser
request_tools = _as_chat_completion_tools(sample_tools)
req = ChatCompletionRequest(
messages=[],
@@ -1497,6 +1320,6 @@ def test_adjust_request_required_prefers_structural_tag(
tools=request_tools,
tool_choice="required",
)
out = qwen3_tool_parser.adjust_request(req)
out = TestParser(MagicMock(), tools=sample_tools).adjust_request(req)
assert out.structured_outputs is not None
assert out.structured_outputs.structural_tag is not None
@@ -1,72 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.tool_parsers.common_tests import (
ToolParserTestConfig,
ToolParserTests,
)
class TestQwen3xmlToolParser(ToolParserTests):
@pytest.fixture
def test_config(self) -> ToolParserTestConfig:
return ToolParserTestConfig(
parser_name="qwen3_xml",
# Test data
no_tool_calls_output="This is a regular response without any tool calls.",
single_tool_call_output="<tool_call>\n<function=get_weather>\n<parameter=city>Tokyo</parameter>\n</function>\n</tool_call>",
parallel_tool_calls_output="<tool_call>\n<function=get_weather>\n<parameter=city>Tokyo</parameter>\n</function>\n</tool_call><tool_call>\n<function=get_time>\n<parameter=timezone>Asia/Tokyo</parameter>\n</function>\n</tool_call>",
various_data_types_output=(
"<tool_call>\n<function=test_function>\n"
"<parameter=string_field>hello</parameter>\n"
"<parameter=int_field>42</parameter>\n"
"<parameter=float_field>3.14</parameter>\n"
"<parameter=bool_field>true</parameter>\n"
"<parameter=null_field>null</parameter>\n"
'<parameter=array_field>["a", "b", "c"]</parameter>\n'
'<parameter=object_field>{"nested": "value"}</parameter>\n'
"</function>\n</tool_call>"
),
empty_arguments_output="<tool_call>\n<function=refresh>\n</function>\n</tool_call>",
surrounding_text_output=(
"Let me check the weather for you.\n\n"
"<tool_call>\n<function=get_weather>\n"
"<parameter=city>Tokyo</parameter>\n"
"</function>\n</tool_call>\n\n"
"I will get that information."
),
escaped_strings_output=(
"<tool_call>\n<function=test_function>\n"
'<parameter=quoted>He said "hello"</parameter>\n'
"<parameter=path>C:\\Users\\file.txt</parameter>\n"
"<parameter=newline>line1\nline2</parameter>\n"
"</function>\n</tool_call>"
),
malformed_input_outputs=[
"<tool_call><function=func>",
"<tool_call><function=></function></tool_call>",
],
# Expected results
single_tool_call_expected_name="get_weather",
single_tool_call_expected_args={"city": "Tokyo"},
parallel_tool_calls_count=2,
parallel_tool_calls_names=["get_weather", "get_time"],
# xfail markers - Qwen3XML has systematic streaming issues
xfail_streaming={
"test_single_tool_call_simple_args": (
"Qwen3XML streaming has systematic issues"
),
"test_parallel_tool_calls": "Qwen3XML streaming has systematic issues",
"test_various_data_types": "Qwen3XML streaming has systematic issues",
"test_empty_arguments": "Qwen3XML streaming has systematic issues",
"test_surrounding_text": "Qwen3XML streaming has systematic issues",
"test_escaped_strings": "Qwen3XML streaming has systematic issues",
"test_streaming_reconstruction": (
"Qwen3XML streaming reconstruction has known issues"
),
},
supports_typed_arguments=False,
)
@@ -0,0 +1,314 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from xgrammar import StructuralTag
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedFunction,
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
ChatCompletionToolsParam,
)
from vllm.parser.abstract_parser import DelegatingParser
from vllm.tool_parsers.abstract_tool_parser import ToolParser
from vllm.tool_parsers.deepseekv3_tool_parser import DeepSeekV3ToolParser
from vllm.tool_parsers.deepseekv4_tool_parser import DeepSeekV4ToolParser
from vllm.tool_parsers.deepseekv31_tool_parser import DeepSeekV31ToolParser
from vllm.tool_parsers.deepseekv32_tool_parser import DeepSeekV32ToolParser
from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser
from vllm.tool_parsers.qwen3coder_tool_parser import Qwen3CoderToolParser
from vllm.tool_parsers.structural_tag_registry import (
SUPPORTED_STRUCTURAL_TAG_MODELS,
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS,
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS,
_get_function_parameters,
get_model_structural_tag,
)
@pytest.fixture
def sample_tools() -> list[ChatCompletionToolsParam]:
return [
ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
)
]
def test_supported_structural_tag_models_include_vllm_builtins():
assert SUPPORTED_STRUCTURAL_TAG_MODELS == (
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
)
assert "hermes" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
def test_get_model_structural_tag_supports_all_xgrammar_builtins(
model: str,
sample_tools: list[ChatCompletionToolsParam],
):
tag = get_model_structural_tag(
model=model,
tools=sample_tools,
tool_choice="auto",
reasoning=False,
)
assert isinstance(tag, StructuralTag)
def test_get_model_structural_tag_supports_vllm_hermes(
sample_tools: list[ChatCompletionToolsParam],
):
tag = get_model_structural_tag(
model="hermes",
tools=sample_tools,
tool_choice="required",
reasoning=False,
)
assert isinstance(tag, StructuralTag)
assert tag.model_dump() == {
"type": "structural_tag",
"format": {
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": '<tool_call>\n{"name": "get_weather", "arguments": ',
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
"style": "json",
},
"end": "}\n</tool_call>",
},
{
"type": "tag",
"begin": '<tool_call>{"name": "get_weather", "arguments": ',
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
"style": "json",
},
"end": "}</tool_call>",
},
],
"separator": "",
"at_least_one": True,
"stop_after_first": False,
},
}
def test_hermes_required_tool_calls_use_empty_separator():
tools = [
ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"parameters": {"type": "object", "properties": {}},
},
),
ChatCompletionToolsParam(
type="function",
function={
"name": "get_time",
"parameters": {"type": "object", "properties": {}},
},
),
]
tag = get_model_structural_tag(
model="hermes",
tools=tools,
tool_choice="required",
reasoning=False,
)
assert tag is not None
assert tag.format.separator == ""
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
def test_get_model_structural_tag_supports_named_tool_choice(
model: str,
sample_tools: list[ChatCompletionToolsParam],
):
tag = get_model_structural_tag(
model=model,
tools=sample_tools,
tool_choice=ChatCompletionNamedToolChoiceParam(
function=ChatCompletionNamedFunction(name="get_weather")
),
reasoning=False,
)
assert isinstance(tag, StructuralTag)
@pytest.mark.parametrize(
("parser_cls", "model"),
[
(DeepSeekV3ToolParser, "deepseek_r1"),
(DeepSeekV31ToolParser, "deepseek_v3_1"),
(DeepSeekV32ToolParser, "deepseek_v3_2"),
(DeepSeekV4ToolParser, "deepseek_v4"),
(Glm47MoeModelToolParser, "glm_4_7"),
(Hermes2ProToolParser, "hermes"),
(KimiK2ToolParser, "kimi"),
(Llama3JsonToolParser, "llama"),
(MinimaxM2ToolParser, "minimax"),
(Qwen3CoderToolParser, "qwen_3_coder"),
],
)
def test_tool_parsers_declare_matching_xgrammar_builtin_model(parser_cls, model):
assert parser_cls.structural_tag_model == model
assert not parser_cls.supports_required_and_named
def test_tool_parsers_without_structural_tag_support_required_and_named():
class NonStructuralTagToolParser(ToolParser):
pass
assert NonStructuralTagToolParser.structural_tag_model is None
assert NonStructuralTagToolParser.supports_required_and_named
def test_non_structural_tag_parser_uses_schema_constraints(
sample_tools: list[ChatCompletionToolsParam],
):
parser = ToolParser(MagicMock())
request = ChatCompletionRequest(
messages=[],
model="m",
tools=sample_tools,
tool_choice="required",
)
out = parser.adjust_request(request)
assert out.structured_outputs is not None
assert out.structured_outputs.json is not None
assert out.structured_outputs.structural_tag is None
def test_get_structural_tag_disables_reasoning(
monkeypatch: pytest.MonkeyPatch,
sample_tools: list[ChatCompletionToolsParam],
):
captured: list[bool] = []
def fake_get_model_structural_tag(*, reasoning: bool, **kwargs):
captured.append(reasoning)
return None
monkeypatch.setattr(
"vllm.tool_parsers.structural_tag_registry.get_model_structural_tag",
fake_get_model_structural_tag,
)
request = ChatCompletionRequest(
messages=[],
model="m",
tools=sample_tools,
tool_choice="auto",
)
parser = Qwen3CoderToolParser(MagicMock(), tools=sample_tools)
parser.get_structural_tag(request)
assert captured == [False]
def test_unified_parser_get_structural_tag_disables_reasoning(
monkeypatch: pytest.MonkeyPatch,
sample_tools: list[ChatCompletionToolsParam],
):
captured: list[bool] = []
def fake_get_model_structural_tag(*, reasoning: bool, **kwargs):
captured.append(reasoning)
return None
monkeypatch.setattr(
"vllm.tool_parsers.structural_tag_registry.get_model_structural_tag",
fake_get_model_structural_tag,
)
class TestParser(DelegatingParser):
tool_parser_cls = Qwen3CoderToolParser
request = ChatCompletionRequest(
messages=[],
model="m",
tools=sample_tools,
tool_choice="auto",
)
parser = TestParser(MagicMock(), tools=sample_tools)
parser.reasoning_parser = MagicMock(adjust_request=lambda request: request)
parser.adjust_request(request)
assert captured == [False]
def test_xgrammar_function_parameters_are_preserved(
monkeypatch: pytest.MonkeyPatch,
sample_tools: list[ChatCompletionToolsParam],
):
captured: list[list[dict]] = []
def fake_get_xgrammar_model_structural_tag(*, tools: list[dict], **kwargs):
captured.append(tools)
return None
monkeypatch.setattr(
"vllm.tool_parsers.structural_tag_registry.get_xgrammar_model_structural_tag",
fake_get_xgrammar_model_structural_tag,
)
get_model_structural_tag(
model="llama",
tools=sample_tools,
tool_choice="auto",
reasoning=False,
)
assert (
captured[0][0]["function"]["parameters"] == sample_tools[0].function.parameters
)
assert sample_tools[0].function.parameters is not None
def test_get_function_parameters_relaxes_function_strict_false():
function = SimpleNamespace(
parameters={"type": "object", "properties": {}},
strict=False,
)
assert _get_function_parameters(function) is True