[Frontend] Add --tool-strict-level for server-side control for structural tag activation (#56268)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: lazymio <mio@lazym.io>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
lazymio
2026-09-18 04:32:39 +00:00
committed by GitHub
co-authored by Claude Opus 5 Bugen Zhao
parent d12c276853
commit dee37d8911
35 changed files with 742 additions and 61 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from types import SimpleNamespace
from unittest.mock import MagicMock
@@ -33,9 +34,11 @@ from vllm.tool_parsers.structural_tag_registry import (
SUPPORTED_STRUCTURAL_TAG_MODELS,
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS,
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS,
ToolChoice,
get_function_parameters,
get_model_structural_tag,
)
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
@pytest.fixture
@@ -231,11 +234,11 @@ def test_get_model_structural_tag_supports_all_xgrammar_builtins(
def test_get_model_structural_tag_supports_vllm_hermes(
sample_tools: list[ChatCompletionToolsParam],
sample_tools_strict: list[ChatCompletionToolsParam],
):
tag = get_model_structural_tag(
model="hermes",
tools=sample_tools,
tools=sample_tools_strict,
tool_choice="required",
reasoning=False,
)
@@ -273,6 +276,22 @@ def test_get_model_structural_tag_supports_vllm_hermes(
assert content["json_schema"] == expected_schema
def test_hermes_required_without_strict_leaves_arguments_free(
sample_tools: list[ChatCompletionToolsParam],
):
tag = get_model_structural_tag(
model="hermes",
tools=sample_tools,
tool_choice="required",
reasoning=False,
)
assert isinstance(tag, StructuralTag)
tags = tag.model_dump()["format"]["tags"]
assert tags
assert all(tag_dump["content"]["json_schema"] is True for tag_dump in tags)
def test_hermes_required_tool_calls_use_empty_separator():
tools = [
ChatCompletionToolsParam(
@@ -361,10 +380,19 @@ def _k3_tools(*calls: str) -> str:
return f"{_K3_TOOLS_OPEN}{''.join(calls)}{_K3_TOOLS_CLOSE}"
def _k3_grammar(tool_choice, tools=None):
def _as_strict(tools: list[ChatCompletionToolsParam]) -> list[ChatCompletionToolsParam]:
for tool in tools:
tool.function.strict = True
return tools
def _k3_grammar(tool_choice, tools=None, *, strict: bool = True):
"""Build the K3 grammar; tools are strict by default so the typed
argument channel is exercised (an unset ``strict`` leaves arguments free)."""
tools = tools if tools is not None else _k3_tools_by_name()
tag = get_model_structural_tag(
model="kimi_k3",
tools=tools if tools is not None else _k3_tools_by_name(),
tools=_as_strict(tools) if strict else tools,
tool_choice=tool_choice,
reasoning=False,
)
@@ -467,6 +495,18 @@ def test_kimi_k3_required_rejects_invalid(body: str):
assert not _is_grammar_accept_string(_k3_grammar("required"), body)
def test_kimi_k3_required_without_strict_leaves_arguments_free():
# An unset ``strict`` pins the call envelope only: a value the schema
# would reject is accepted, and the declared-required argument may be
# omitted.
grammar = _k3_grammar("required", strict=False)
body = _k3_response() + _k3_tools(
_k3_call("get_weather", _k3_arg("days", "number", "abc"))
)
assert _is_grammar_accept_string(grammar, body)
assert not _is_grammar_accept_string(_k3_grammar("required"), body)
def test_kimi_k3_schema_without_required_accepts_empty_call():
tools = [
ChatCompletionToolsParam(
@@ -829,3 +869,209 @@ def test_kimi_k3_forced_tool_choice_builds_single_mandatory_call():
response_only = _k3_response("no call here")
assert _is_grammar_accept_string(grammar, ok)
assert not _is_grammar_accept_string(grammar, response_only)
def _pins_argument_schema(tag: StructuralTag) -> bool:
return '"json_schema": {' in json.dumps(tag.model_dump(), ensure_ascii=False)
def _dumped(tag: StructuralTag) -> str:
return json.dumps(tag.model_dump(), ensure_ascii=False)
def test_tool_strict_level_auto_is_the_default(
sample_tools: list[ChatCompletionToolsParam],
):
"""Auto + no strict tool gets no tag unless the operator raises the floor."""
assert (
get_model_structural_tag(
model="deepseek_v4",
tools=sample_tools,
tool_choice="auto",
reasoning=False,
)
is None
)
@pytest.mark.parametrize("model", sorted(SUPPORTED_STRUCTURAL_TAG_MODELS))
def test_tool_strict_level_function_lifts_auto_gate(
model: str,
sample_tools: list[ChatCompletionToolsParam],
):
tag = get_model_structural_tag(
model=model,
tools=sample_tools,
tool_choice="auto",
reasoning=False,
strict_level=ToolStrictLevel.FUNCTION,
)
assert tag is not None
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
def test_tool_strict_level_function_pins_envelope_only(
model: str,
sample_tools: list[ChatCompletionToolsParam],
):
"""The request's own tools stay untouched; unset ``strict`` stays free."""
tag = get_model_structural_tag(
model=model,
tools=sample_tools,
tool_choice="auto",
reasoning=False,
strict_level=ToolStrictLevel.FUNCTION,
)
assert tag is not None
assert not _pins_argument_schema(tag)
assert all(tool.function.strict is None for tool in sample_tools)
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
def test_tool_strict_level_parameter_pins_argument_schemas(
model: str,
sample_tools: list[ChatCompletionToolsParam],
):
tag = get_model_structural_tag(
model=model,
tools=sample_tools,
tool_choice="auto",
reasoning=False,
strict_level=ToolStrictLevel.PARAMETER,
)
assert tag is not None
assert _pins_argument_schema(tag)
def test_tool_strict_level_function_keeps_client_strict_tools_strict(
sample_tools_strict: list[ChatCompletionToolsParam],
):
tag = get_model_structural_tag(
model="deepseek_v4",
tools=sample_tools_strict,
tool_choice="auto",
reasoning=False,
strict_level=ToolStrictLevel.FUNCTION,
)
assert tag is not None
assert _pins_argument_schema(tag)
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
def test_absent_strict_is_not_pinned_next_to_a_strict_tool(model: str):
"""A strict tool must not drag a neighbour with unset ``strict`` into
schema enforcement."""
weather = ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"strict": True,
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
)
search = ChatCompletionToolsParam(
type="function",
function={
"name": "search",
"parameters": {
"type": "object",
"properties": {"query": {"type": "string"}},
"required": ["query"],
},
},
)
tag = get_model_structural_tag(
model=model,
tools=[weather, search],
tool_choice="auto",
reasoning=False,
)
assert tag is not None
dumped = _dumped(tag)
assert '"city"' in dumped
assert '"query"' not in dumped
@pytest.mark.parametrize(
"tool_choice",
[
"required",
ChatCompletionNamedToolChoiceParam(
function=ChatCompletionNamedFunction(name="get_weather")
),
],
)
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
def test_forced_tool_choice_pins_schema_only_for_strict_tools(
model: str,
tool_choice: ToolChoice,
sample_tools: list[ChatCompletionToolsParam],
sample_tools_strict: list[ChatCompletionToolsParam],
):
"""Required / named always constrain the call; the argument schema
follows the per-tool ``strict`` (or the parameter level)."""
free = get_model_structural_tag(
model=model,
tools=sample_tools,
tool_choice=tool_choice,
reasoning=False,
strict_level=ToolStrictLevel.FUNCTION,
)
pinned = get_model_structural_tag(
model=model,
tools=sample_tools_strict,
tool_choice=tool_choice,
reasoning=False,
)
floored = get_model_structural_tag(
model=model,
tools=sample_tools,
tool_choice=tool_choice,
reasoning=False,
strict_level=ToolStrictLevel.PARAMETER,
)
assert free is not None and not _pins_argument_schema(free)
assert pinned is not None and _pins_argument_schema(pinned)
assert floored is not None and _pins_argument_schema(floored)
def test_tool_strict_level_parameter_overrides_client_strict_false():
tools = [
ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"parameters": {"type": "object", "properties": {}},
"strict": False,
},
)
]
tag = get_model_structural_tag(
model="deepseek_v4",
tools=tools,
tool_choice="required",
reasoning=False,
strict_level=ToolStrictLevel.PARAMETER,
)
assert tag is not None
assert _pins_argument_schema(tag)
def test_tool_strict_level_from_name():
assert ToolStrictLevel.from_name("AUTO") is ToolStrictLevel.AUTO
assert ToolStrictLevel.from_name("Parameter") is ToolStrictLevel.PARAMETER
with pytest.raises(ValueError, match="expected one of auto, function, parameter"):
ToolStrictLevel.from_name("strict")
with pytest.raises(ValueError, match="expected one of auto, function, parameter"):
ToolStrictLevel.from_name("off")