[Frontend] Add --tool-strict-level for server-side control for structural tag activation (#56268)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: lazymio <mio@lazym.io> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 5
Bugen Zhao
parent
d12c276853
commit
dee37d8911
@@ -36,6 +36,7 @@ from vllm.tool_parsers.streaming import (
|
||||
extract_named_tool_call_streaming,
|
||||
extract_required_tool_call_streaming,
|
||||
)
|
||||
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -95,6 +96,8 @@ class Parser:
|
||||
# Subclasses should override these if they use specific parser classes
|
||||
reasoning_parser_cls: type[ReasoningParser] | None = None
|
||||
tool_parser_cls: type[ToolParser] | None = None
|
||||
# Server-side floor for tool-call structural tags (--tool-strict-level).
|
||||
tool_strict_level: ToolStrictLevel = ToolStrictLevel.AUTO
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -415,6 +418,7 @@ class DelegatingParser(Parser):
|
||||
structure_tag = self._tool_parser.get_structural_tag(
|
||||
request,
|
||||
reasoning=False,
|
||||
strict_level=self.tool_strict_level,
|
||||
)
|
||||
if structure_tag is None:
|
||||
return request
|
||||
|
||||
@@ -6,6 +6,7 @@ from __future__ import annotations
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
@@ -77,6 +78,7 @@ class ParserManager:
|
||||
enable_auto_tools: bool = False,
|
||||
model_name: str | None = None,
|
||||
is_harmony: bool = False,
|
||||
tool_strict_level: str = "auto",
|
||||
) -> type[Parser] | None:
|
||||
"""Get a Parser that handles both reasoning and tool parsing.
|
||||
|
||||
@@ -89,6 +91,8 @@ class ParserManager:
|
||||
model_name: The model name for parser-specific warnings.
|
||||
is_harmony: Whether the selected model uses the Harmony format.
|
||||
If True, HarmonyParser is always returned.
|
||||
tool_strict_level: Server-side floor for tool-call structural
|
||||
tags (``--tool-strict-level``).
|
||||
|
||||
Returns:
|
||||
A Parser class, or None if neither parser is specified.
|
||||
@@ -105,11 +109,14 @@ class ParserManager:
|
||||
if reasoning_parser_cls is None and tool_parser_cls is None:
|
||||
return None
|
||||
|
||||
strict_level = ToolStrictLevel.from_name(tool_strict_level)
|
||||
|
||||
if is_harmony:
|
||||
from vllm.parser.harmony import HarmonyParser
|
||||
|
||||
HarmonyParser.reasoning_parser_cls = reasoning_parser_cls
|
||||
HarmonyParser.tool_parser_cls = tool_parser_cls
|
||||
HarmonyParser.tool_strict_level = strict_level
|
||||
return HarmonyParser
|
||||
|
||||
if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
|
||||
@@ -121,6 +128,7 @@ class ParserManager:
|
||||
class _KimiK3Parser(KimiK3Parser):
|
||||
reasoning_parser_cls = r_cls
|
||||
tool_parser_cls = t_cls
|
||||
tool_strict_level = strict_level
|
||||
|
||||
return _KimiK3Parser
|
||||
|
||||
@@ -136,6 +144,7 @@ class ParserManager:
|
||||
class _CohereCommandParser(CohereCommandParser):
|
||||
reasoning_parser_cls = r_cls
|
||||
tool_parser_cls = t_cls
|
||||
tool_strict_level = strict_level
|
||||
|
||||
return _CohereCommandParser
|
||||
|
||||
@@ -147,5 +156,6 @@ class ParserManager:
|
||||
class _Parser(DelegatingParser):
|
||||
reasoning_parser_cls = r_cls
|
||||
tool_parser_cls = t_cls
|
||||
tool_strict_level = strict_level
|
||||
|
||||
return _Parser
|
||||
|
||||
Reference in New Issue
Block a user