[Frontend] Add --tool-strict-level for server-side control for structural tag activation (#56268)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: lazymio <mio@lazym.io> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 5
Bugen Zhao
parent
d12c276853
commit
dee37d8911
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
@@ -33,9 +34,11 @@ from vllm.tool_parsers.structural_tag_registry import (
|
||||
SUPPORTED_STRUCTURAL_TAG_MODELS,
|
||||
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS,
|
||||
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS,
|
||||
ToolChoice,
|
||||
get_function_parameters,
|
||||
get_model_structural_tag,
|
||||
)
|
||||
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -231,11 +234,11 @@ def test_get_model_structural_tag_supports_all_xgrammar_builtins(
|
||||
|
||||
|
||||
def test_get_model_structural_tag_supports_vllm_hermes(
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
sample_tools_strict: list[ChatCompletionToolsParam],
|
||||
):
|
||||
tag = get_model_structural_tag(
|
||||
model="hermes",
|
||||
tools=sample_tools,
|
||||
tools=sample_tools_strict,
|
||||
tool_choice="required",
|
||||
reasoning=False,
|
||||
)
|
||||
@@ -273,6 +276,22 @@ def test_get_model_structural_tag_supports_vllm_hermes(
|
||||
assert content["json_schema"] == expected_schema
|
||||
|
||||
|
||||
def test_hermes_required_without_strict_leaves_arguments_free(
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
):
|
||||
tag = get_model_structural_tag(
|
||||
model="hermes",
|
||||
tools=sample_tools,
|
||||
tool_choice="required",
|
||||
reasoning=False,
|
||||
)
|
||||
|
||||
assert isinstance(tag, StructuralTag)
|
||||
tags = tag.model_dump()["format"]["tags"]
|
||||
assert tags
|
||||
assert all(tag_dump["content"]["json_schema"] is True for tag_dump in tags)
|
||||
|
||||
|
||||
def test_hermes_required_tool_calls_use_empty_separator():
|
||||
tools = [
|
||||
ChatCompletionToolsParam(
|
||||
@@ -361,10 +380,19 @@ def _k3_tools(*calls: str) -> str:
|
||||
return f"{_K3_TOOLS_OPEN}{''.join(calls)}{_K3_TOOLS_CLOSE}"
|
||||
|
||||
|
||||
def _k3_grammar(tool_choice, tools=None):
|
||||
def _as_strict(tools: list[ChatCompletionToolsParam]) -> list[ChatCompletionToolsParam]:
|
||||
for tool in tools:
|
||||
tool.function.strict = True
|
||||
return tools
|
||||
|
||||
|
||||
def _k3_grammar(tool_choice, tools=None, *, strict: bool = True):
|
||||
"""Build the K3 grammar; tools are strict by default so the typed
|
||||
argument channel is exercised (an unset ``strict`` leaves arguments free)."""
|
||||
tools = tools if tools is not None else _k3_tools_by_name()
|
||||
tag = get_model_structural_tag(
|
||||
model="kimi_k3",
|
||||
tools=tools if tools is not None else _k3_tools_by_name(),
|
||||
tools=_as_strict(tools) if strict else tools,
|
||||
tool_choice=tool_choice,
|
||||
reasoning=False,
|
||||
)
|
||||
@@ -467,6 +495,18 @@ def test_kimi_k3_required_rejects_invalid(body: str):
|
||||
assert not _is_grammar_accept_string(_k3_grammar("required"), body)
|
||||
|
||||
|
||||
def test_kimi_k3_required_without_strict_leaves_arguments_free():
|
||||
# An unset ``strict`` pins the call envelope only: a value the schema
|
||||
# would reject is accepted, and the declared-required argument may be
|
||||
# omitted.
|
||||
grammar = _k3_grammar("required", strict=False)
|
||||
body = _k3_response() + _k3_tools(
|
||||
_k3_call("get_weather", _k3_arg("days", "number", "abc"))
|
||||
)
|
||||
assert _is_grammar_accept_string(grammar, body)
|
||||
assert not _is_grammar_accept_string(_k3_grammar("required"), body)
|
||||
|
||||
|
||||
def test_kimi_k3_schema_without_required_accepts_empty_call():
|
||||
tools = [
|
||||
ChatCompletionToolsParam(
|
||||
@@ -829,3 +869,209 @@ def test_kimi_k3_forced_tool_choice_builds_single_mandatory_call():
|
||||
response_only = _k3_response("no call here")
|
||||
assert _is_grammar_accept_string(grammar, ok)
|
||||
assert not _is_grammar_accept_string(grammar, response_only)
|
||||
|
||||
|
||||
def _pins_argument_schema(tag: StructuralTag) -> bool:
|
||||
return '"json_schema": {' in json.dumps(tag.model_dump(), ensure_ascii=False)
|
||||
|
||||
|
||||
def _dumped(tag: StructuralTag) -> str:
|
||||
return json.dumps(tag.model_dump(), ensure_ascii=False)
|
||||
|
||||
|
||||
def test_tool_strict_level_auto_is_the_default(
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
):
|
||||
"""Auto + no strict tool gets no tag unless the operator raises the floor."""
|
||||
assert (
|
||||
get_model_structural_tag(
|
||||
model="deepseek_v4",
|
||||
tools=sample_tools,
|
||||
tool_choice="auto",
|
||||
reasoning=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", sorted(SUPPORTED_STRUCTURAL_TAG_MODELS))
|
||||
def test_tool_strict_level_function_lifts_auto_gate(
|
||||
model: str,
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
):
|
||||
tag = get_model_structural_tag(
|
||||
model=model,
|
||||
tools=sample_tools,
|
||||
tool_choice="auto",
|
||||
reasoning=False,
|
||||
strict_level=ToolStrictLevel.FUNCTION,
|
||||
)
|
||||
|
||||
assert tag is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
|
||||
def test_tool_strict_level_function_pins_envelope_only(
|
||||
model: str,
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
):
|
||||
"""The request's own tools stay untouched; unset ``strict`` stays free."""
|
||||
tag = get_model_structural_tag(
|
||||
model=model,
|
||||
tools=sample_tools,
|
||||
tool_choice="auto",
|
||||
reasoning=False,
|
||||
strict_level=ToolStrictLevel.FUNCTION,
|
||||
)
|
||||
|
||||
assert tag is not None
|
||||
assert not _pins_argument_schema(tag)
|
||||
assert all(tool.function.strict is None for tool in sample_tools)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
|
||||
def test_tool_strict_level_parameter_pins_argument_schemas(
|
||||
model: str,
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
):
|
||||
tag = get_model_structural_tag(
|
||||
model=model,
|
||||
tools=sample_tools,
|
||||
tool_choice="auto",
|
||||
reasoning=False,
|
||||
strict_level=ToolStrictLevel.PARAMETER,
|
||||
)
|
||||
|
||||
assert tag is not None
|
||||
assert _pins_argument_schema(tag)
|
||||
|
||||
|
||||
def test_tool_strict_level_function_keeps_client_strict_tools_strict(
|
||||
sample_tools_strict: list[ChatCompletionToolsParam],
|
||||
):
|
||||
tag = get_model_structural_tag(
|
||||
model="deepseek_v4",
|
||||
tools=sample_tools_strict,
|
||||
tool_choice="auto",
|
||||
reasoning=False,
|
||||
strict_level=ToolStrictLevel.FUNCTION,
|
||||
)
|
||||
|
||||
assert tag is not None
|
||||
assert _pins_argument_schema(tag)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
|
||||
def test_absent_strict_is_not_pinned_next_to_a_strict_tool(model: str):
|
||||
"""A strict tool must not drag a neighbour with unset ``strict`` into
|
||||
schema enforcement."""
|
||||
weather = ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
)
|
||||
search = ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "search",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"required": ["query"],
|
||||
},
|
||||
},
|
||||
)
|
||||
tag = get_model_structural_tag(
|
||||
model=model,
|
||||
tools=[weather, search],
|
||||
tool_choice="auto",
|
||||
reasoning=False,
|
||||
)
|
||||
|
||||
assert tag is not None
|
||||
dumped = _dumped(tag)
|
||||
assert '"city"' in dumped
|
||||
assert '"query"' not in dumped
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool_choice",
|
||||
[
|
||||
"required",
|
||||
ChatCompletionNamedToolChoiceParam(
|
||||
function=ChatCompletionNamedFunction(name="get_weather")
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
|
||||
def test_forced_tool_choice_pins_schema_only_for_strict_tools(
|
||||
model: str,
|
||||
tool_choice: ToolChoice,
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
sample_tools_strict: list[ChatCompletionToolsParam],
|
||||
):
|
||||
"""Required / named always constrain the call; the argument schema
|
||||
follows the per-tool ``strict`` (or the parameter level)."""
|
||||
free = get_model_structural_tag(
|
||||
model=model,
|
||||
tools=sample_tools,
|
||||
tool_choice=tool_choice,
|
||||
reasoning=False,
|
||||
strict_level=ToolStrictLevel.FUNCTION,
|
||||
)
|
||||
pinned = get_model_structural_tag(
|
||||
model=model,
|
||||
tools=sample_tools_strict,
|
||||
tool_choice=tool_choice,
|
||||
reasoning=False,
|
||||
)
|
||||
floored = get_model_structural_tag(
|
||||
model=model,
|
||||
tools=sample_tools,
|
||||
tool_choice=tool_choice,
|
||||
reasoning=False,
|
||||
strict_level=ToolStrictLevel.PARAMETER,
|
||||
)
|
||||
|
||||
assert free is not None and not _pins_argument_schema(free)
|
||||
assert pinned is not None and _pins_argument_schema(pinned)
|
||||
assert floored is not None and _pins_argument_schema(floored)
|
||||
|
||||
|
||||
def test_tool_strict_level_parameter_overrides_client_strict_false():
|
||||
tools = [
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
"strict": False,
|
||||
},
|
||||
)
|
||||
]
|
||||
tag = get_model_structural_tag(
|
||||
model="deepseek_v4",
|
||||
tools=tools,
|
||||
tool_choice="required",
|
||||
reasoning=False,
|
||||
strict_level=ToolStrictLevel.PARAMETER,
|
||||
)
|
||||
|
||||
assert tag is not None
|
||||
assert _pins_argument_schema(tag)
|
||||
|
||||
|
||||
def test_tool_strict_level_from_name():
|
||||
assert ToolStrictLevel.from_name("AUTO") is ToolStrictLevel.AUTO
|
||||
assert ToolStrictLevel.from_name("Parameter") is ToolStrictLevel.PARAMETER
|
||||
with pytest.raises(ValueError, match="expected one of auto, function, parameter"):
|
||||
ToolStrictLevel.from_name("strict")
|
||||
with pytest.raises(ValueError, match="expected one of auto, function, parameter"):
|
||||
ToolStrictLevel.from_name("off")
|
||||
|
||||
Reference in New Issue
Block a user