[Frontend] Add --tool-strict-level for server-side control for structural tag activation (#56268)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: lazymio <mio@lazym.io>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
lazymio
2026-09-18 04:32:39 +00:00
committed by GitHub
co-authored by Claude Opus 5 Bugen Zhao
parent d12c276853
commit dee37d8911
35 changed files with 742 additions and 61 deletions
+2
View File
@@ -112,6 +112,7 @@ class AnthropicServingMessages(OpenAIServingChat):
reasoning_parser: str = "",
enable_auto_tools: bool = False,
tool_parser: str | None = None,
tool_strict_level: str = "auto",
enable_prompt_tokens_details: bool = False,
enable_force_include_usage: bool = False,
default_chat_template_kwargs: dict[str, Any] | None = None,
@@ -127,6 +128,7 @@ class AnthropicServingMessages(OpenAIServingChat):
return_tokens_as_token_ids=return_tokens_as_token_ids,
reasoning_parser=reasoning_parser,
enable_auto_tools=enable_auto_tools,
tool_strict_level=tool_strict_level,
tool_parser=tool_parser,
enable_prompt_tokens_details=enable_prompt_tokens_details,
enable_force_include_usage=enable_force_include_usage,
+2
View File
@@ -249,6 +249,7 @@ class CohereServingChatV2(OpenAIServingChat):
reasoning_parser: str = "",
enable_auto_tools: bool = False,
tool_parser: str | None = None,
tool_strict_level: str = "auto",
enable_prompt_tokens_details: bool = False,
enable_force_include_usage: bool = False,
default_chat_template_kwargs: dict[str, Any] | None = None,
@@ -265,6 +266,7 @@ class CohereServingChatV2(OpenAIServingChat):
return_tokens_as_token_ids=return_tokens_as_token_ids,
reasoning_parser=reasoning_parser,
enable_auto_tools=enable_auto_tools,
tool_strict_level=tool_strict_level,
tool_parser=tool_parser,
enable_prompt_tokens_details=enable_prompt_tokens_details,
enable_force_include_usage=enable_force_include_usage,
+4
View File
@@ -112,6 +112,7 @@ async def init_generate_state(
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
tool_server=state.tool_server,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
@@ -137,6 +138,7 @@ async def init_generate_state(
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
@@ -178,6 +180,7 @@ async def init_generate_state(
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
@@ -198,6 +201,7 @@ async def init_generate_state(
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
@@ -95,6 +95,7 @@ async def init_app_state(
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
@@ -111,6 +112,7 @@ async def init_app_state(
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
+8
View File
@@ -20,6 +20,7 @@ from vllm.entrypoints.chat_utils import (
)
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
from vllm.tool_parsers import ToolParserManager
from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName
from vllm.utils.argparse_utils import FlexibleArgumentParser
from .utils.constants import (
@@ -116,6 +117,13 @@ class BaseFrontendArgs:
"""Special the tool parser plugin write to parse the model-generated tool
into OpenAI API format, the name register in this plugin can be used in
`--tool-call-parser`."""
tool_strict_level: ToolStrictLevelName = "auto"
"""Server-side floor for structural-tag based tool calling, applied on top
of the per-tool `strict` field. `auto` follows the request's tool choice
and per-tool strictness; `function` constrains the
tool-call envelope (markup and function name) for every request with
tools; `parameter` additionally pins argument schemas, as if every tool
were `strict: true`."""
tool_server: str | None = None
"""Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
@@ -60,6 +60,7 @@ async def init_render_app_state(
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=args.reasoning_parser,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
@@ -76,6 +77,7 @@ async def init_render_app_state(
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=args.reasoning_parser,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
@@ -132,6 +132,7 @@ class OpenAIServingChat(GenerateBaseServing):
enable_auto_tools: bool = False,
exclude_tools_when_tool_choice_none: bool = False,
tool_parser: str | None = None,
tool_strict_level: str = "auto",
enable_prompt_tokens_details: bool = False,
enable_force_include_usage: bool = False,
enable_log_outputs: bool = False,
@@ -161,6 +162,7 @@ class OpenAIServingChat(GenerateBaseServing):
tool_parser_name=tool_parser,
reasoning_parser_name=reasoning_parser,
enable_auto_tools=enable_auto_tools,
tool_strict_level=tool_strict_level,
model_name=self.model_config.model,
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
)
@@ -109,6 +109,7 @@ class OpenAIServingResponses(GenerateBaseServing):
reasoning_parser: str = "",
enable_auto_tools: bool = False,
tool_parser: str | None = None,
tool_strict_level: str = "auto",
tool_server: ToolServer | None = None,
enable_prompt_tokens_details: bool = False,
enable_force_include_usage: bool = False,
@@ -135,6 +136,7 @@ class OpenAIServingResponses(GenerateBaseServing):
tool_parser_name=tool_parser,
reasoning_parser_name=reasoning_parser,
enable_auto_tools=enable_auto_tools,
tool_strict_level=tool_strict_level,
model_name=self.model_config.model,
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
)