[Bugfix][Frontend] Build tool-call grammars from the tools the prompt renders (#59879)
Signed-off-by: Rongjie Jiang <j1870831863@outlook.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -21,6 +21,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.openai.responses.utils import construct_tool_dicts
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.parser.plamo3 import (
|
||||
BEGIN_TOOL_ARGUMENTS,
|
||||
@@ -203,6 +205,131 @@ def test_get_model_structural_tag_supports_all_xgrammar_builtins(
|
||||
assert isinstance(tag, StructuralTag)
|
||||
|
||||
|
||||
_MCP_TOOL = {"type": "mcp", "server_label": "docs", "server_url": "http://mcp"}
|
||||
|
||||
|
||||
def _namespace_tool(strict: bool | None = None) -> dict[str, Any]:
|
||||
lookup = {
|
||||
"type": "function",
|
||||
"name": "lookup",
|
||||
"parameters": {"type": "object", "properties": {"id": {"type": "string"}}},
|
||||
}
|
||||
if strict is not None:
|
||||
lookup["strict"] = strict
|
||||
return {"type": "namespace", "name": "crm", "description": "", "tools": [lookup]}
|
||||
|
||||
|
||||
def _hermes_call(name: str) -> str:
|
||||
return (
|
||||
f'<tool_call>\n{{"name": "{name}", "arguments": {{"id": "1"}}}}\n</tool_call>'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", sorted(SUPPORTED_STRUCTURAL_TAG_MODELS))
|
||||
def test_no_structural_tag_without_function_tools(model: str):
|
||||
"""Builtin tools never reach the prompt, so there is nothing to constrain."""
|
||||
request = ResponsesRequest.model_validate(
|
||||
{"input": "hi", "tools": [_MCP_TOOL, {"type": "web_search"}]}
|
||||
)
|
||||
|
||||
tag = get_model_structural_tag(
|
||||
model=model,
|
||||
tools=request.tools,
|
||||
tool_choice="auto",
|
||||
reasoning=False,
|
||||
strict_level=ToolStrictLevel.FUNCTION,
|
||||
)
|
||||
|
||||
assert tag is None
|
||||
with pytest.raises(VLLMValidationError, match="no function tool"):
|
||||
get_model_structural_tag(
|
||||
model=model,
|
||||
tools=request.tools,
|
||||
tool_choice="required",
|
||||
reasoning=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
|
||||
def test_structural_tag_allows_exactly_the_prompt_tools(tool_choice: str):
|
||||
"""The grammar lets the model call every tool its prompt lists, and no other."""
|
||||
request = ResponsesRequest.model_validate(
|
||||
{
|
||||
"input": "hi",
|
||||
"tools": [
|
||||
{"type": "function", "name": "get_weather", "parameters": {}},
|
||||
_namespace_tool(),
|
||||
_MCP_TOOL,
|
||||
],
|
||||
}
|
||||
)
|
||||
prompt_names = [
|
||||
tool["function"]["name"]
|
||||
for tool in construct_tool_dicts(request.tools, tool_choice)
|
||||
]
|
||||
|
||||
tag = get_model_structural_tag(
|
||||
model="hermes",
|
||||
tools=request.tools,
|
||||
tool_choice=tool_choice,
|
||||
reasoning=False,
|
||||
strict_level=ToolStrictLevel.FUNCTION,
|
||||
)
|
||||
|
||||
assert prompt_names == ["get_weather", "crm__lookup"]
|
||||
assert tag is not None
|
||||
grammar = Grammar.from_structural_tag(json.dumps(tag.model_dump()))
|
||||
for name in prompt_names:
|
||||
assert _is_grammar_accept_string(grammar, _hermes_call(name))
|
||||
assert not _is_grammar_accept_string(grammar, _hermes_call("docs"))
|
||||
|
||||
|
||||
def test_strict_namespace_function_enables_auto_structural_tag():
|
||||
request = ResponsesRequest.model_validate(
|
||||
{"input": "hi", "tools": [_namespace_tool(strict=True)]}
|
||||
)
|
||||
|
||||
tag = get_model_structural_tag(
|
||||
model="hermes", tools=request.tools, tool_choice="auto", reasoning=False
|
||||
)
|
||||
|
||||
assert tag is not None
|
||||
grammar = Grammar.from_structural_tag(json.dumps(tag.model_dump()))
|
||||
assert _is_grammar_accept_string(grammar, _hermes_call("crm__lookup"))
|
||||
|
||||
|
||||
def test_harmony_structural_tag_keeps_builtin_tools():
|
||||
"""Harmony can call builtin tools, so they still shape its grammar."""
|
||||
request = ResponsesRequest.model_validate(
|
||||
{"input": "hi", "tools": [{"type": "web_search"}]}
|
||||
)
|
||||
|
||||
tag = get_model_structural_tag(
|
||||
model="harmony",
|
||||
tools=request.tools,
|
||||
tool_choice="required",
|
||||
reasoning=False,
|
||||
)
|
||||
|
||||
assert tag is not None
|
||||
|
||||
|
||||
def test_glm47_default_level_allows_builtin_only_tools(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""GLM-4.7's default FUNCTION level must not reject builtin-only tools."""
|
||||
monkeypatch.setattr(envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", True)
|
||||
|
||||
class TestParser(DelegatingParser):
|
||||
tool_parser_cls = Glm47MoeModelToolParser
|
||||
|
||||
request = ResponsesRequest.model_validate({"input": "hi", "tools": [_MCP_TOOL]})
|
||||
parser = TestParser(MagicMock(), tools=None)
|
||||
parser._reasoning_parser = MagicMock(adjust_request=lambda request: request)
|
||||
|
||||
assert parser.adjust_request(request).structured_outputs is None
|
||||
|
||||
|
||||
def test_get_model_structural_tag_supports_vllm_hermes(
|
||||
sample_tools_strict: list[ChatCompletionToolsParam],
|
||||
):
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
|
||||
from vllm.tool_parsers.utils import (
|
||||
find_tool_properties,
|
||||
@@ -403,6 +404,11 @@ class TestNonFunctionToolsSkipped:
|
||||
assert len(any_of) == 1
|
||||
assert any_of[0]["properties"]["name"]["enum"] == ["get_weather"]
|
||||
|
||||
def test_get_json_schema_rejects_only_non_function_tools(self):
|
||||
# An empty anyOf would compile to a grammar no output can satisfy.
|
||||
with pytest.raises(VLLMValidationError, match="no function tool"):
|
||||
get_json_schema_from_tools(tools=[WEB_SEARCH_TOOL], tool_choice="required")
|
||||
|
||||
|
||||
class TestParallelToolCallsConstraint:
|
||||
"""`parallel_tool_calls=false` must be enforced by the decoding grammar.
|
||||
|
||||
@@ -50,6 +50,7 @@ from vllm.tool_parsers.streaming import (
|
||||
extract_required_tool_call_streaming,
|
||||
)
|
||||
from vllm.tool_parsers.structural_tag_registry import (
|
||||
get_structural_tag_tools,
|
||||
limit_to_single_tool_call,
|
||||
resolve_tool_strictness,
|
||||
)
|
||||
@@ -520,7 +521,11 @@ class DelegatingParser(Parser):
|
||||
resolved_tools = None
|
||||
if tool_parser.structural_tag_model is not None:
|
||||
resolved_tools = resolve_tool_strictness(
|
||||
request.tools,
|
||||
get_structural_tag_tools(
|
||||
tool_parser.structural_tag_model,
|
||||
request.tools,
|
||||
request.tool_choice,
|
||||
),
|
||||
request.tool_choice,
|
||||
strict_level,
|
||||
)
|
||||
|
||||
@@ -472,7 +472,7 @@ def _assemble_tag(
|
||||
return StructuralTag(format=SequenceFormat(elements=tags))
|
||||
|
||||
|
||||
@register_vllm_structural_tag("harmony")
|
||||
@register_vllm_structural_tag("harmony", builtin_tools=True)
|
||||
def get_harmony_structural_tag(
|
||||
tools: list[FunctionToolParam],
|
||||
builtin_tools: list[BuiltinToolParam],
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
|
||||
from vllm.tool_parsers.utils import get_function_tools, require_function_tools
|
||||
|
||||
ToolChoice: TypeAlias = (
|
||||
Literal["none", "auto", "required"]
|
||||
@@ -85,20 +86,60 @@ SUPPORTED_STRUCTURAL_TAG_MODELS = (
|
||||
)
|
||||
|
||||
_VLLM_STRUCTURAL_TAG_REGISTRY: dict[str, StructuralTagBuilder] = {}
|
||||
_BUILTIN_TOOL_STRUCTURAL_TAG_MODELS: set[str] = set()
|
||||
|
||||
|
||||
def register_vllm_structural_tag(
|
||||
model: str,
|
||||
*,
|
||||
builtin_tools: bool = False,
|
||||
) -> Callable[[StructuralTagBuilder], StructuralTagBuilder]:
|
||||
"""Register a vLLM-owned structural tag builder."""
|
||||
"""Register a vLLM-owned structural tag builder.
|
||||
|
||||
Args:
|
||||
model: Structural tag model key.
|
||||
builtin_tools: Whether the model can call non-function tools, so its
|
||||
builder receives every request tool instead of only the function
|
||||
tools from ``get_function_tools``.
|
||||
|
||||
"""
|
||||
|
||||
def decorator(func: StructuralTagBuilder) -> StructuralTagBuilder:
|
||||
_VLLM_STRUCTURAL_TAG_REGISTRY[model] = func
|
||||
if builtin_tools:
|
||||
_BUILTIN_TOOL_STRUCTURAL_TAG_MODELS.add(model)
|
||||
return func
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def get_structural_tag_tools(
|
||||
model: str,
|
||||
tools: Sequence[ChatCompletionToolsParam | ResponsesTool],
|
||||
tool_choice: ToolChoice,
|
||||
) -> Sequence[ChatCompletionToolsParam | ResponsesTool]:
|
||||
"""Return the tools a structural tag for ``model`` lets the model call.
|
||||
|
||||
Most models are only shown function tools, so their grammar must cover
|
||||
exactly those: a tool missing from the prompt cannot be called, and one
|
||||
missing from the grammar is shown but blocked.
|
||||
|
||||
Raises:
|
||||
VLLMValidationError: ``tool_choice`` requires a call, but the model
|
||||
has no tool it can call.
|
||||
|
||||
"""
|
||||
if model in _BUILTIN_TOOL_STRUCTURAL_TAG_MODELS:
|
||||
return tools
|
||||
if (
|
||||
tool_choice is None
|
||||
or tool_choice == "auto"
|
||||
or getattr(tool_choice, "mode", None) == "auto"
|
||||
):
|
||||
return get_function_tools(tools)
|
||||
return require_function_tools(tools)
|
||||
|
||||
|
||||
def _tool_is_strict(tool: ChatCompletionToolsParam | ResponsesTool) -> bool:
|
||||
if isinstance(tool, FunctionTool):
|
||||
return tool.strict is True
|
||||
@@ -164,6 +205,9 @@ def get_model_structural_tag(
|
||||
if not tools or tool_choice == "none":
|
||||
return None
|
||||
|
||||
tools = get_structural_tag_tools(model, tools, tool_choice)
|
||||
if not tools:
|
||||
return None
|
||||
tools = resolve_tool_strictness(tools, tool_choice, strict_level)
|
||||
if tools is None:
|
||||
return None
|
||||
|
||||
+62
-40
@@ -6,6 +6,7 @@ import json
|
||||
import keyword as _python_keyword
|
||||
import math
|
||||
import warnings
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from json import JSONDecodeError, JSONDecoder
|
||||
from typing import Any, TypeAlias
|
||||
@@ -29,6 +30,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
|
||||
Tool: TypeAlias = ChatCompletionToolsParam | ResponsesTool
|
||||
@@ -179,43 +181,74 @@ def flat_namespace_tool_name(namespace: str, name: str) -> str:
|
||||
return f"{namespace}{_NAMESPACE_TOOL_SEPARATOR}{name}"
|
||||
|
||||
|
||||
def get_function_tools(
|
||||
tools: Sequence[Tool],
|
||||
) -> list[ChatCompletionToolsParam | FunctionTool]:
|
||||
"""Return the function tools the model is shown, in request order.
|
||||
|
||||
This is the single source for both the chat template's tool list and the
|
||||
tools a tool-call grammar constrains, so the two cannot drift apart.
|
||||
Namespace functions are flattened to ``<namespace>__<name>``; other
|
||||
Responses tool types are never rendered for the model and are dropped.
|
||||
"""
|
||||
function_tools: list[ChatCompletionToolsParam | FunctionTool] = []
|
||||
for tool in tools:
|
||||
if isinstance(tool, NamespaceTool):
|
||||
function_tools.extend(
|
||||
FunctionTool.model_validate(
|
||||
{
|
||||
**namespaced_tool.model_dump(by_alias=True),
|
||||
"name": flat_namespace_tool_name(
|
||||
tool.name, namespaced_tool.name
|
||||
),
|
||||
}
|
||||
)
|
||||
for namespaced_tool in tool.tools
|
||||
if namespaced_tool.type == "function"
|
||||
)
|
||||
elif isinstance(tool, (FunctionTool, ChatCompletionToolsParam)):
|
||||
function_tools.append(tool)
|
||||
return function_tools
|
||||
|
||||
|
||||
def require_function_tools(
|
||||
tools: Sequence[Tool],
|
||||
) -> list[ChatCompletionToolsParam | FunctionTool]:
|
||||
"""Return ``get_function_tools(tools)``, rejecting a request that has none.
|
||||
|
||||
Raises:
|
||||
VLLMValidationError: ``tools`` has no tool the model can call, so a
|
||||
tool choice that requires a call can never be satisfied.
|
||||
|
||||
"""
|
||||
function_tools = get_function_tools(tools)
|
||||
if not function_tools:
|
||||
raise VLLMValidationError(
|
||||
"tool_choice requires a function call, but tools contains no "
|
||||
"function tool the model can call.",
|
||||
parameter="tool_choice",
|
||||
)
|
||||
return function_tools
|
||||
|
||||
|
||||
def iter_response_function_tool_info(
|
||||
tool: ResponsesTool,
|
||||
) -> list[tuple[str, dict[str, Any] | None]]:
|
||||
if isinstance(tool, FunctionTool):
|
||||
return [(tool.name, tool.parameters)]
|
||||
if not isinstance(tool, NamespaceTool):
|
||||
return []
|
||||
|
||||
namespace = tool.name
|
||||
return [
|
||||
(
|
||||
flat_namespace_tool_name(namespace, namespaced_tool.name),
|
||||
namespaced_tool.parameters,
|
||||
)
|
||||
for namespaced_tool in tool.tools
|
||||
if namespaced_tool.type == "function"
|
||||
(function_tool.name, function_tool.parameters)
|
||||
for function_tool in get_function_tools([tool])
|
||||
if isinstance(function_tool, FunctionTool)
|
||||
]
|
||||
|
||||
|
||||
def iter_response_function_tool_dicts(
|
||||
tools: list[ResponsesTool],
|
||||
) -> list[dict[str, Any]]:
|
||||
function_tools: list[dict[str, Any]] = []
|
||||
for tool in tools:
|
||||
if isinstance(tool, NamespaceTool):
|
||||
namespace = tool.name
|
||||
for namespaced_tool in tool.tools:
|
||||
if namespaced_tool.type != "function":
|
||||
continue
|
||||
tool_dict = namespaced_tool.model_dump()
|
||||
tool_dict["name"] = flat_namespace_tool_name(
|
||||
namespace, namespaced_tool.name
|
||||
)
|
||||
function_tools.append(tool_dict)
|
||||
elif isinstance(tool, FunctionTool):
|
||||
function_tools.append(tool.model_dump())
|
||||
return function_tools
|
||||
return [
|
||||
tool.model_dump()
|
||||
for tool in get_function_tools(tools)
|
||||
if isinstance(tool, FunctionTool)
|
||||
]
|
||||
|
||||
|
||||
def build_responses_tool_call_name_map(
|
||||
@@ -353,19 +386,8 @@ def _get_json_schema_from_tools(
|
||||
tools: list[Tool],
|
||||
parallel_tool_calls: bool | None = None,
|
||||
) -> dict:
|
||||
fn_tool_schemas: list[dict[str, Any]] = []
|
||||
fn_tools: list[Tool] = []
|
||||
for tool in tools:
|
||||
if isinstance(tool, (FunctionTool, NamespaceTool)):
|
||||
fn_tool_schemas.extend(
|
||||
_get_tool_schema_from_name_and_params(name, params)
|
||||
for name, params in iter_response_function_tool_info(tool)
|
||||
)
|
||||
if isinstance(tool, FunctionTool):
|
||||
fn_tools.append(tool)
|
||||
elif _is_function_tool(tool):
|
||||
fn_tool_schemas.append(_get_tool_schema_from_tool(tool))
|
||||
fn_tools.append(tool)
|
||||
fn_tools: list[Tool] = list(require_function_tools(tools))
|
||||
fn_tool_schemas = [_get_tool_schema_from_tool(tool) for tool in fn_tools]
|
||||
json_schema: dict[str, Any] = {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
|
||||
Reference in New Issue
Block a user