[Frontend] Add --tool-strict-level for server-side control for structural tag activation (#56268)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: lazymio <mio@lazym.io> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 5
Bugen Zhao
parent
d12c276853
commit
dee37d8911
@@ -112,6 +112,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
reasoning_parser: str = "",
|
||||
enable_auto_tools: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
@@ -127,6 +128,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
||||
reasoning_parser=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
tool_strict_level=tool_strict_level,
|
||||
tool_parser=tool_parser,
|
||||
enable_prompt_tokens_details=enable_prompt_tokens_details,
|
||||
enable_force_include_usage=enable_force_include_usage,
|
||||
|
||||
@@ -249,6 +249,7 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
reasoning_parser: str = "",
|
||||
enable_auto_tools: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
@@ -265,6 +266,7 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
||||
reasoning_parser=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
tool_strict_level=tool_strict_level,
|
||||
tool_parser=tool_parser,
|
||||
enable_prompt_tokens_details=enable_prompt_tokens_details,
|
||||
enable_force_include_usage=enable_force_include_usage,
|
||||
|
||||
@@ -112,6 +112,7 @@ async def init_generate_state(
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
tool_server=state.tool_server,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
@@ -137,6 +138,7 @@ async def init_generate_state(
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
@@ -178,6 +180,7 @@ async def init_generate_state(
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
@@ -198,6 +201,7 @@ async def init_generate_state(
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
|
||||
@@ -95,6 +95,7 @@ async def init_app_state(
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
@@ -111,6 +112,7 @@ async def init_app_state(
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.entrypoints.chat_utils import (
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from .utils.constants import (
|
||||
@@ -116,6 +117,13 @@ class BaseFrontendArgs:
|
||||
"""Special the tool parser plugin write to parse the model-generated tool
|
||||
into OpenAI API format, the name register in this plugin can be used in
|
||||
`--tool-call-parser`."""
|
||||
tool_strict_level: ToolStrictLevelName = "auto"
|
||||
"""Server-side floor for structural-tag based tool calling, applied on top
|
||||
of the per-tool `strict` field. `auto` follows the request's tool choice
|
||||
and per-tool strictness; `function` constrains the
|
||||
tool-call envelope (markup and function name) for every request with
|
||||
tools; `parameter` additionally pins argument schemas, as if every tool
|
||||
were `strict: true`."""
|
||||
tool_server: str | None = None
|
||||
"""Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
|
||||
Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
|
||||
|
||||
@@ -60,6 +60,7 @@ async def init_render_app_state(
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
@@ -76,6 +77,7 @@ async def init_render_app_state(
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
|
||||
@@ -132,6 +132,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
enable_auto_tools: bool = False,
|
||||
exclude_tools_when_tool_choice_none: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
enable_log_outputs: bool = False,
|
||||
@@ -161,6 +162,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
tool_parser_name=tool_parser,
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
tool_strict_level=tool_strict_level,
|
||||
model_name=self.model_config.model,
|
||||
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
|
||||
)
|
||||
|
||||
@@ -109,6 +109,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
reasoning_parser: str = "",
|
||||
enable_auto_tools: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
tool_server: ToolServer | None = None,
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
@@ -135,6 +136,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
tool_parser_name=tool_parser,
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
tool_strict_level=tool_strict_level,
|
||||
model_name=self.model_config.model,
|
||||
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
|
||||
)
|
||||
|
||||
@@ -36,6 +36,7 @@ from vllm.tool_parsers.streaming import (
|
||||
extract_named_tool_call_streaming,
|
||||
extract_required_tool_call_streaming,
|
||||
)
|
||||
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -95,6 +96,8 @@ class Parser:
|
||||
# Subclasses should override these if they use specific parser classes
|
||||
reasoning_parser_cls: type[ReasoningParser] | None = None
|
||||
tool_parser_cls: type[ToolParser] | None = None
|
||||
# Server-side floor for tool-call structural tags (--tool-strict-level).
|
||||
tool_strict_level: ToolStrictLevel = ToolStrictLevel.AUTO
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -415,6 +418,7 @@ class DelegatingParser(Parser):
|
||||
structure_tag = self._tool_parser.get_structural_tag(
|
||||
request,
|
||||
reasoning=False,
|
||||
strict_level=self.tool_strict_level,
|
||||
)
|
||||
if structure_tag is None:
|
||||
return request
|
||||
|
||||
@@ -6,6 +6,7 @@ from __future__ import annotations
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
@@ -77,6 +78,7 @@ class ParserManager:
|
||||
enable_auto_tools: bool = False,
|
||||
model_name: str | None = None,
|
||||
is_harmony: bool = False,
|
||||
tool_strict_level: str = "auto",
|
||||
) -> type[Parser] | None:
|
||||
"""Get a Parser that handles both reasoning and tool parsing.
|
||||
|
||||
@@ -89,6 +91,8 @@ class ParserManager:
|
||||
model_name: The model name for parser-specific warnings.
|
||||
is_harmony: Whether the selected model uses the Harmony format.
|
||||
If True, HarmonyParser is always returned.
|
||||
tool_strict_level: Server-side floor for tool-call structural
|
||||
tags (``--tool-strict-level``).
|
||||
|
||||
Returns:
|
||||
A Parser class, or None if neither parser is specified.
|
||||
@@ -105,11 +109,14 @@ class ParserManager:
|
||||
if reasoning_parser_cls is None and tool_parser_cls is None:
|
||||
return None
|
||||
|
||||
strict_level = ToolStrictLevel.from_name(tool_strict_level)
|
||||
|
||||
if is_harmony:
|
||||
from vllm.parser.harmony import HarmonyParser
|
||||
|
||||
HarmonyParser.reasoning_parser_cls = reasoning_parser_cls
|
||||
HarmonyParser.tool_parser_cls = tool_parser_cls
|
||||
HarmonyParser.tool_strict_level = strict_level
|
||||
return HarmonyParser
|
||||
|
||||
if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
|
||||
@@ -121,6 +128,7 @@ class ParserManager:
|
||||
class _KimiK3Parser(KimiK3Parser):
|
||||
reasoning_parser_cls = r_cls
|
||||
tool_parser_cls = t_cls
|
||||
tool_strict_level = strict_level
|
||||
|
||||
return _KimiK3Parser
|
||||
|
||||
@@ -136,6 +144,7 @@ class ParserManager:
|
||||
class _CohereCommandParser(CohereCommandParser):
|
||||
reasoning_parser_cls = r_cls
|
||||
tool_parser_cls = t_cls
|
||||
tool_strict_level = strict_level
|
||||
|
||||
return _CohereCommandParser
|
||||
|
||||
@@ -147,5 +156,6 @@ class ParserManager:
|
||||
class _Parser(DelegatingParser):
|
||||
reasoning_parser_cls = r_cls
|
||||
tool_parser_cls = t_cls
|
||||
tool_strict_level = strict_level
|
||||
|
||||
return _Parser
|
||||
|
||||
@@ -54,6 +54,7 @@ class OnlineDerenderer:
|
||||
exclude_tools_when_tool_choice_none: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
reasoning_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
log_error_stack: bool = False,
|
||||
) -> None:
|
||||
@@ -68,6 +69,7 @@ class OnlineDerenderer:
|
||||
tool_parser_name=tool_parser,
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
tool_strict_level=tool_strict_level,
|
||||
model_name=model_config.model,
|
||||
is_harmony=self.use_harmony,
|
||||
)
|
||||
|
||||
@@ -127,6 +127,7 @@ class OnlineRenderer:
|
||||
exclude_tools_when_tool_choice_none: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
reasoning_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
log_error_stack: bool = False,
|
||||
) -> None:
|
||||
@@ -141,6 +142,7 @@ class OnlineRenderer:
|
||||
tool_parser_name=tool_parser,
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
tool_strict_level=tool_strict_level,
|
||||
model_name=model_config.model,
|
||||
is_harmony=self.use_harmony,
|
||||
)
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm.sampling_params import (
|
||||
StructuredOutputsParams,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
|
||||
from vllm.tool_parsers.utils import Tool, get_json_schema_from_tools
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils.import_utils import import_plugin
|
||||
@@ -168,6 +169,7 @@ class ToolParser:
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
*,
|
||||
reasoning: bool = False,
|
||||
strict_level: ToolStrictLevel = ToolStrictLevel.AUTO,
|
||||
):
|
||||
if self.structural_tag_model is None:
|
||||
return None
|
||||
@@ -180,6 +182,7 @@ class ToolParser:
|
||||
tools=request.tools,
|
||||
tool_choice=request.tool_choice,
|
||||
reasoning=reasoning,
|
||||
strict_level=strict_level,
|
||||
)
|
||||
|
||||
def extract_tool_calls(
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import ToolParser
|
||||
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
|
||||
from vllm.tool_parsers.utils import partial_tag_overlap
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -994,6 +995,7 @@ class HYV4ToolParser(ToolParser):
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
*,
|
||||
reasoning: bool = False,
|
||||
strict_level: ToolStrictLevel = ToolStrictLevel.AUTO,
|
||||
) -> StructuralTag | None:
|
||||
"""Build a structural tag matching HYV4's tool tokens.
|
||||
|
||||
@@ -1009,6 +1011,7 @@ class HYV4ToolParser(ToolParser):
|
||||
Args:
|
||||
request: The request being adjusted.
|
||||
reasoning: Whether the grammar also covers the reasoning phase.
|
||||
strict_level: Server-side floor from ``--tool-strict-level``.
|
||||
|
||||
Returns:
|
||||
The structural tag, or None when structural tagging does not apply.
|
||||
@@ -1041,6 +1044,7 @@ class HYV4ToolParser(ToolParser):
|
||||
tool_choice=request.tool_choice,
|
||||
reasoning=reasoning,
|
||||
token_suffix=self._extractor.token_suffix,
|
||||
strict_level=strict_level,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
|
||||
|
||||
ToolChoice: TypeAlias = (
|
||||
Literal["none", "auto", "required"]
|
||||
@@ -94,15 +95,57 @@ def register_vllm_structural_tag(
|
||||
return decorator
|
||||
|
||||
|
||||
def _tool_is_strict(tool: ChatCompletionToolsParam | ResponsesTool) -> bool:
|
||||
if isinstance(tool, FunctionTool):
|
||||
return tool.strict is True
|
||||
if isinstance(tool, ChatCompletionToolsParam):
|
||||
return tool.function.strict is True
|
||||
return False
|
||||
|
||||
|
||||
def _any_tool_strict(
|
||||
tools: Sequence[ChatCompletionToolsParam | ResponsesTool],
|
||||
) -> bool:
|
||||
for tool in tools:
|
||||
if isinstance(tool, FunctionTool) and tool.strict is True:
|
||||
return True
|
||||
if isinstance(tool, ChatCompletionToolsParam) and tool.function.strict is True:
|
||||
return True
|
||||
return False
|
||||
return any(_tool_is_strict(tool) for tool in tools)
|
||||
|
||||
|
||||
def _with_tool_strict(
|
||||
tool: ChatCompletionToolsParam | ResponsesTool,
|
||||
strict: bool,
|
||||
) -> ChatCompletionToolsParam | ResponsesTool:
|
||||
"""Return a copy of ``tool`` with ``strict`` set, leaving the request's alone."""
|
||||
if isinstance(tool, FunctionTool):
|
||||
return tool.model_copy(update={"strict": strict})
|
||||
if isinstance(tool, ChatCompletionToolsParam):
|
||||
return tool.model_copy(
|
||||
update={"function": tool.function.model_copy(update={"strict": strict})}
|
||||
)
|
||||
return tool
|
||||
|
||||
|
||||
def _resolve_tool_strictness(
|
||||
tools: Sequence[ChatCompletionToolsParam | ResponsesTool],
|
||||
tool_choice: ToolChoice,
|
||||
strict_level: ToolStrictLevel,
|
||||
) -> Sequence[ChatCompletionToolsParam | ResponsesTool] | None:
|
||||
"""Decide whether a structural tag applies and pin each tool's ``strict``.
|
||||
|
||||
A tool without an explicit ``strict`` is treated as non-strict: its call
|
||||
envelope is still constrained, but its arguments stay free unless the
|
||||
server level is PARAMETER. ``None`` means no structural tag.
|
||||
"""
|
||||
if (
|
||||
tool_choice == "auto"
|
||||
and strict_level == ToolStrictLevel.AUTO
|
||||
and not _any_tool_strict(tools)
|
||||
):
|
||||
return None
|
||||
return [
|
||||
_with_tool_strict(
|
||||
tool, strict_level >= ToolStrictLevel.PARAMETER or _tool_is_strict(tool)
|
||||
)
|
||||
for tool in tools
|
||||
]
|
||||
|
||||
|
||||
def get_model_structural_tag(
|
||||
@@ -111,12 +154,14 @@ def get_model_structural_tag(
|
||||
tool_choice: ToolChoice,
|
||||
reasoning: bool,
|
||||
token_suffix: str = "",
|
||||
strict_level: ToolStrictLevel = ToolStrictLevel.AUTO,
|
||||
) -> StructuralTag | None:
|
||||
"""Build a structural tag with xgrammar's builtin model templates."""
|
||||
if not tools or tool_choice == "none":
|
||||
return None
|
||||
|
||||
if tool_choice == "auto" and not _any_tool_strict(tools):
|
||||
tools = _resolve_tool_strictness(tools, tool_choice, strict_level)
|
||||
if tools is None:
|
||||
return None
|
||||
|
||||
dumped_tools = [_dump_tool_for_xgrammar(tool) for tool in tools]
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import enum
|
||||
from typing import Literal
|
||||
|
||||
ToolStrictLevelName = Literal["auto", "function", "parameter"]
|
||||
|
||||
|
||||
class ToolStrictLevel(enum.IntEnum):
|
||||
"""Server-side floor for tool-call structural tags (``--tool-strict-level``).
|
||||
|
||||
AUTO: derive call constraints from ``tool_choice`` and argument-schema
|
||||
enforcement from each tool's explicit ``strict`` field.
|
||||
FUNCTION: constrain the tool-call envelope for every request with tools.
|
||||
PARAMETER: additionally pin argument schemas, as if every tool were
|
||||
``strict: true``.
|
||||
"""
|
||||
|
||||
AUTO = 0
|
||||
FUNCTION = 1
|
||||
PARAMETER = 2
|
||||
|
||||
@classmethod
|
||||
def from_name(cls, name: str) -> "ToolStrictLevel":
|
||||
try:
|
||||
return cls[name.strip().upper()]
|
||||
except KeyError:
|
||||
expected = ", ".join(level.name.lower() for level in cls)
|
||||
raise ValueError(
|
||||
f"Unknown tool strict level {name!r}; expected one of {expected}."
|
||||
) from None
|
||||
Reference in New Issue
Block a user