[Frontend] Add --tool-strict-level for server-side control for structural tag activation (#56268)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: lazymio <mio@lazym.io> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 5
Bugen Zhao
parent
d12c276853
commit
dee37d8911
@@ -24,6 +24,7 @@ use serde_with::{DefaultOnNull, OneOrMany, serde_as};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use uuid::Uuid;
|
||||
use vllm_chat::GenerationConfigMode;
|
||||
use vllm_chat::ToolStrictLevel;
|
||||
use vllm_chat::multimodal::MmLimitPerPrompt;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_managed_engine::ManagedEngineConfig;
|
||||
@@ -134,6 +135,10 @@ pub struct RenderArgs {
|
||||
/// `none` to disable parsing.
|
||||
#[arg(long, default_value_t)]
|
||||
reasoning_parser: ParserSelection,
|
||||
/// Server-side floor for structural-tag based tool calling: `auto`,
|
||||
/// `function`, or `parameter`.
|
||||
#[arg(long, default_value_t)]
|
||||
tool_strict_level: ToolStrictLevel,
|
||||
/// Select the native chat renderer implementation.
|
||||
#[arg(long = "tokenizer-mode", default_value_t)]
|
||||
renderer: RendererSelection,
|
||||
@@ -171,6 +176,7 @@ impl RenderArgs {
|
||||
port: self.port,
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
reasoning_parser: self.reasoning_parser,
|
||||
tool_strict_level: self.tool_strict_level,
|
||||
renderer: self.renderer,
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs.unwrap_or_default(),
|
||||
@@ -241,6 +247,14 @@ pub struct SharedRuntimeArgs {
|
||||
#[arg(long, default_value_t)]
|
||||
#[serde(default = "default_py_bootstrap_parser_selection")]
|
||||
pub reasoning_parser: ParserSelection,
|
||||
/// Server-side floor for structural-tag based tool calling, applied on
|
||||
/// top of the per-tool `strict` field: `auto` follows the request's
|
||||
/// tool choice and per-tool strictness,
|
||||
/// `function` constrains the tool-call envelope for every request with
|
||||
/// tools, `parameter` additionally pins argument schemas.
|
||||
#[arg(long, default_value_t)]
|
||||
#[serde(default)]
|
||||
pub tool_strict_level: ToolStrictLevel,
|
||||
/// Select the chat renderer implementation.
|
||||
#[arg(long = "tokenizer-mode", default_value_t)]
|
||||
#[serde(default, rename = "tokenizer_mode")]
|
||||
@@ -517,6 +531,7 @@ impl SharedRuntimeArgs {
|
||||
listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
reasoning_parser: self.reasoning_parser,
|
||||
tool_strict_level: self.tool_strict_level,
|
||||
renderer: self.renderer,
|
||||
language_model_only: self.language_model_only,
|
||||
chat_template: self.chat_template,
|
||||
@@ -574,6 +589,7 @@ impl SharedRuntimeArgs {
|
||||
listener_mode,
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
reasoning_parser: self.reasoning_parser,
|
||||
tool_strict_level: self.tool_strict_level,
|
||||
renderer: self.renderer,
|
||||
language_model_only: self.language_model_only,
|
||||
chat_template: self.chat_template,
|
||||
|
||||
@@ -184,6 +184,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_strict_level: Auto,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
@@ -1023,6 +1024,7 @@ fn frontend_args_accept_json() {
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: None,
|
||||
reasoning_parser: None,
|
||||
tool_strict_level: Auto,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
@@ -1693,6 +1695,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_strict_level: Auto,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
@@ -1850,6 +1853,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
},
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_strict_level: Auto,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
@@ -1942,6 +1946,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
},
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_strict_level: Auto,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
@@ -2055,6 +2060,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
},
|
||||
tool_call_parser: None,
|
||||
reasoning_parser: None,
|
||||
tool_strict_level: Auto,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
|
||||
Reference in New Issue
Block a user