[Frontend] Add --tool-strict-level for server-side control for structural tag activation (#56268)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: lazymio <mio@lazym.io> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 5
Bugen Zhao
parent
d12c276853
commit
dee37d8911
@@ -112,6 +112,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
reasoning_parser: str = "",
|
||||
enable_auto_tools: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
@@ -127,6 +128,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
||||
reasoning_parser=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
tool_strict_level=tool_strict_level,
|
||||
tool_parser=tool_parser,
|
||||
enable_prompt_tokens_details=enable_prompt_tokens_details,
|
||||
enable_force_include_usage=enable_force_include_usage,
|
||||
|
||||
@@ -249,6 +249,7 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
reasoning_parser: str = "",
|
||||
enable_auto_tools: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
@@ -265,6 +266,7 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
||||
reasoning_parser=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
tool_strict_level=tool_strict_level,
|
||||
tool_parser=tool_parser,
|
||||
enable_prompt_tokens_details=enable_prompt_tokens_details,
|
||||
enable_force_include_usage=enable_force_include_usage,
|
||||
|
||||
@@ -112,6 +112,7 @@ async def init_generate_state(
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
tool_server=state.tool_server,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
@@ -137,6 +138,7 @@ async def init_generate_state(
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
@@ -178,6 +180,7 @@ async def init_generate_state(
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
@@ -198,6 +201,7 @@ async def init_generate_state(
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
|
||||
@@ -95,6 +95,7 @@ async def init_app_state(
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
@@ -111,6 +112,7 @@ async def init_app_state(
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.entrypoints.chat_utils import (
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from .utils.constants import (
|
||||
@@ -116,6 +117,13 @@ class BaseFrontendArgs:
|
||||
"""Special the tool parser plugin write to parse the model-generated tool
|
||||
into OpenAI API format, the name register in this plugin can be used in
|
||||
`--tool-call-parser`."""
|
||||
tool_strict_level: ToolStrictLevelName = "auto"
|
||||
"""Server-side floor for structural-tag based tool calling, applied on top
|
||||
of the per-tool `strict` field. `auto` follows the request's tool choice
|
||||
and per-tool strictness; `function` constrains the
|
||||
tool-call envelope (markup and function name) for every request with
|
||||
tools; `parameter` additionally pins argument schemas, as if every tool
|
||||
were `strict: true`."""
|
||||
tool_server: str | None = None
|
||||
"""Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
|
||||
Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
|
||||
|
||||
@@ -60,6 +60,7 @@ async def init_render_app_state(
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
@@ -76,6 +77,7 @@ async def init_render_app_state(
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
|
||||
@@ -132,6 +132,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
enable_auto_tools: bool = False,
|
||||
exclude_tools_when_tool_choice_none: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
enable_log_outputs: bool = False,
|
||||
@@ -161,6 +162,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
tool_parser_name=tool_parser,
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
tool_strict_level=tool_strict_level,
|
||||
model_name=self.model_config.model,
|
||||
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
|
||||
)
|
||||
|
||||
@@ -109,6 +109,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
reasoning_parser: str = "",
|
||||
enable_auto_tools: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
tool_server: ToolServer | None = None,
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
@@ -135,6 +136,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
tool_parser_name=tool_parser,
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
tool_strict_level=tool_strict_level,
|
||||
model_name=self.model_config.model,
|
||||
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user