[Frontend] Let an unset --tool-strict-level fall back to the tool parser's default (#61328)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
91ed2859c7
commit
8fa5ae844a
@@ -160,7 +160,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
reasoning_parser: str = "",
|
||||
enable_auto_tools: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
tool_strict_level: str | None = None,
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
enable_log_outputs: bool = False,
|
||||
|
||||
@@ -251,7 +251,7 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
reasoning_parser: str = "",
|
||||
enable_auto_tools: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
tool_strict_level: str | None = None,
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
enable_log_outputs: bool = False,
|
||||
|
||||
@@ -128,13 +128,14 @@ class BaseFrontendArgs:
|
||||
"""Special the tool parser plugin write to parse the model-generated tool
|
||||
into OpenAI API format, the name register in this plugin can be used in
|
||||
`--tool-call-parser`."""
|
||||
tool_strict_level: ToolStrictLevelName = "auto"
|
||||
tool_strict_level: ToolStrictLevelName | None = None
|
||||
"""Server-side floor for structural-tag based tool calling, applied on top
|
||||
of the per-tool `strict` field. `auto` follows the request's tool choice
|
||||
and per-tool strictness; `function` constrains the
|
||||
tool-call envelope (markup and function name) for every request with
|
||||
tools; `parameter` additionally pins argument schemas, as if every tool
|
||||
were `strict: true`."""
|
||||
were `strict: true`. When unset, the tool parser's default level applies,
|
||||
which is `auto` for most parsers."""
|
||||
tool_server: str | None = None
|
||||
"""Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
|
||||
Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
|
||||
|
||||
@@ -132,7 +132,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
enable_auto_tools: bool = False,
|
||||
exclude_tools_when_tool_choice_none: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
tool_strict_level: str | None = None,
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
enable_log_outputs: bool = False,
|
||||
|
||||
@@ -113,7 +113,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
reasoning_parser: str = "",
|
||||
enable_auto_tools: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
tool_strict_level: str | None = None,
|
||||
tool_server: ToolServer | None = None,
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
|
||||
@@ -118,8 +118,9 @@ class Parser:
|
||||
# Subclasses should override these if they use specific parser classes
|
||||
reasoning_parser_cls: type[ReasoningParser] | None = None
|
||||
tool_parser_cls: type[ToolParser] | None = None
|
||||
# Server-side floor for tool-call structural tags (--tool-strict-level).
|
||||
tool_strict_level: ToolStrictLevel = ToolStrictLevel.AUTO
|
||||
# Server-side floor for tool-call structural tags (--tool-strict-level), or
|
||||
# None for the tool parser's default level.
|
||||
tool_strict_level: ToolStrictLevel | None = None
|
||||
always_adjust_request: bool = False
|
||||
|
||||
def __init__(
|
||||
@@ -519,10 +520,7 @@ class DelegatingParser(Parser):
|
||||
is_auto = request.tool_choice == "auto"
|
||||
single_call = request.parallel_tool_calls is False
|
||||
strict_level = self.tool_strict_level
|
||||
if (
|
||||
strict_level == ToolStrictLevel.AUTO
|
||||
and tool_parser.default_tool_strict_level is not None
|
||||
):
|
||||
if strict_level is None:
|
||||
strict_level = tool_parser.default_tool_strict_level
|
||||
if single_call and not (is_auto and structured_outputs is not None):
|
||||
# Limiting the call count needs the call envelope in the grammar.
|
||||
|
||||
@@ -104,7 +104,7 @@ class ParserManager:
|
||||
enable_auto_tools: bool = False,
|
||||
model_name: str | None = None,
|
||||
is_harmony: bool = False,
|
||||
tool_strict_level: str = "auto",
|
||||
tool_strict_level: str | None = None,
|
||||
tokenizer: TokenizerLike | None = None,
|
||||
) -> type[Parser] | None:
|
||||
"""Get a Parser that handles both reasoning and tool parsing.
|
||||
@@ -119,7 +119,8 @@ class ParserManager:
|
||||
is_harmony: Whether the selected model uses the Harmony format.
|
||||
If True, HarmonyParser is always returned.
|
||||
tool_strict_level: Server-side floor for tool-call structural
|
||||
tags (``--tool-strict-level``).
|
||||
tags (``--tool-strict-level``), or ``None`` for the tool
|
||||
parser's default level.
|
||||
tokenizer: Tokenizer the composed parser is validated against
|
||||
at startup (see :meth:`validate_parser_tokenizer`). ``None``
|
||||
skips the check (e.g. ``skip_tokenizer_init``).
|
||||
@@ -155,7 +156,7 @@ class ParserManager:
|
||||
enable_auto_tools: bool,
|
||||
model_name: str | None,
|
||||
is_harmony: bool,
|
||||
tool_strict_level: str,
|
||||
tool_strict_level: str | None,
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> type[Parser] | None:
|
||||
if not tool_parser_name and not reasoning_parser_name:
|
||||
@@ -169,7 +170,11 @@ class ParserManager:
|
||||
if reasoning_parser_cls is None and tool_parser_cls is None:
|
||||
return None
|
||||
|
||||
strict_level = ToolStrictLevel.from_name(tool_strict_level)
|
||||
strict_level = (
|
||||
None
|
||||
if tool_strict_level is None
|
||||
else ToolStrictLevel.from_name(tool_strict_level)
|
||||
)
|
||||
|
||||
if is_harmony:
|
||||
from vllm.parser.harmony import HarmonyParser
|
||||
|
||||
@@ -69,7 +69,7 @@ class OnlineDerenderer:
|
||||
exclude_tools_when_tool_choice_none: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
reasoning_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
tool_strict_level: str | None = None,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
log_error_stack: bool = False,
|
||||
) -> None:
|
||||
|
||||
@@ -165,7 +165,7 @@ class OnlineRenderer:
|
||||
exclude_tools_when_tool_choice_none: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
reasoning_parser: str | None = None,
|
||||
tool_strict_level: str = "auto",
|
||||
tool_strict_level: str | None = None,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
log_error_stack: bool = False,
|
||||
) -> None:
|
||||
|
||||
@@ -60,9 +60,9 @@ class ToolParser:
|
||||
# their parsed tool-call syntax matches a builtin xgrammar format.
|
||||
structural_tag_model: str | None = None
|
||||
# Strict level applied when the operator leaves ``--tool-strict-level``
|
||||
# at auto. Operator flags override it, and
|
||||
# unset. Operator flags override it, and
|
||||
# ``VLLM_ENFORCE_STRICT_TOOL_CALLING=0`` still disables structural tags.
|
||||
default_tool_strict_level: ToolStrictLevel | None = None
|
||||
default_tool_strict_level: ToolStrictLevel = ToolStrictLevel.AUTO
|
||||
engine_based_streaming: bool = False
|
||||
|
||||
def __init_subclass__(cls, **kwargs: Any) -> None:
|
||||
|
||||
Reference in New Issue
Block a user