[Frontend] Let an unset --tool-strict-level fall back to the tool parser's default (#61328)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
91ed2859c7
commit
8fa5ae844a
@@ -380,7 +380,7 @@ mod tests {
|
||||
let error = match backend.new_chat_output_processor(
|
||||
&mut request,
|
||||
NewChatOutputProcessorOptions {
|
||||
tool_strict_level: crate::ToolStrictLevel::Auto,
|
||||
tool_strict_level: None,
|
||||
tool_call_parser: &ParserSelection::Explicit("json".to_string()),
|
||||
reasoning_parser: &ParserSelection::Auto,
|
||||
},
|
||||
|
||||
@@ -21,7 +21,9 @@ pub mod hf;
|
||||
pub struct NewChatOutputProcessorOptions<'a> {
|
||||
pub tool_call_parser: &'a ParserSelection,
|
||||
pub reasoning_parser: &'a ParserSelection,
|
||||
pub tool_strict_level: ToolStrictLevel,
|
||||
/// Server-side floor for tool-call structural tags, or `None` for the
|
||||
/// tool parser's default level.
|
||||
pub tool_strict_level: Option<ToolStrictLevel>,
|
||||
}
|
||||
|
||||
/// Minimal prompt-processing backend needed by `vllm-chat`.
|
||||
|
||||
@@ -79,8 +79,9 @@ pub struct ChatRequestProcessor {
|
||||
tool_call_parser: ParserSelection,
|
||||
/// Reasoning parser selection used when preparing generation requests.
|
||||
reasoning_parser: ParserSelection,
|
||||
/// Server-side floor for tool-call structural tags.
|
||||
tool_strict_level: ToolStrictLevel,
|
||||
/// Server-side floor for tool-call structural tags, or `None` for the
|
||||
/// tool parser's default level.
|
||||
tool_strict_level: Option<ToolStrictLevel>,
|
||||
}
|
||||
|
||||
impl ChatRequestProcessor {
|
||||
@@ -93,7 +94,7 @@ impl ChatRequestProcessor {
|
||||
max_model_len: Some(max_model_len as usize),
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
tool_strict_level: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,7 +106,7 @@ impl ChatRequestProcessor {
|
||||
max_model_len: None,
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
tool_strict_level: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -120,8 +121,9 @@ impl ChatRequestProcessor {
|
||||
self
|
||||
}
|
||||
|
||||
/// Configure the server-side floor for tool-call structural tags.
|
||||
pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self {
|
||||
/// Configure the server-side floor for tool-call structural tags, or
|
||||
/// `None` for the tool parser's default level.
|
||||
pub fn with_tool_strict_level(mut self, tool_strict_level: Option<ToolStrictLevel>) -> Self {
|
||||
self.tool_strict_level = tool_strict_level;
|
||||
self
|
||||
}
|
||||
@@ -295,8 +297,9 @@ impl ChatLlm {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the server-side floor for tool-call structural tags.
|
||||
pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self {
|
||||
/// Set the server-side floor for tool-call structural tags, or `None` for
|
||||
/// the tool parser's default level.
|
||||
pub fn with_tool_strict_level(mut self, tool_strict_level: Option<ToolStrictLevel>) -> Self {
|
||||
self.processor.tool_strict_level = tool_strict_level;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -70,7 +70,7 @@ impl DefaultChatOutputProcessor {
|
||||
tokenizer: DynTokenizer,
|
||||
tool_call_parser: &ParserSelection,
|
||||
reasoning_parser: &ParserSelection,
|
||||
tool_strict_level: ToolStrictLevel,
|
||||
tool_strict_level: Option<ToolStrictLevel>,
|
||||
) -> ChatResult<Self> {
|
||||
Self::with_response_template(
|
||||
request,
|
||||
@@ -92,7 +92,7 @@ impl DefaultChatOutputProcessor {
|
||||
response_template: &std::result::Result<Arc<ResponseTemplate>, HfTemplateError>,
|
||||
tool_call_parser: &ParserSelection,
|
||||
reasoning_parser: &ParserSelection,
|
||||
tool_strict_level: ToolStrictLevel,
|
||||
tool_strict_level: Option<ToolStrictLevel>,
|
||||
) -> ChatResult<Self> {
|
||||
let tool_name = tool_call_parser.resolve_tool_name(model_id);
|
||||
let reasoning_name = reasoning_parser.resolve_reasoning_name(model_id);
|
||||
@@ -135,6 +135,8 @@ impl DefaultChatOutputProcessor {
|
||||
if parser.preserve_special_tokens() {
|
||||
request.decode_options.skip_special_tokens = false;
|
||||
}
|
||||
let tool_strict_level =
|
||||
tool_strict_level.unwrap_or_else(|| parser.default_tool_strict_level());
|
||||
|
||||
Ok(Self {
|
||||
parser,
|
||||
@@ -292,6 +294,7 @@ mod tests {
|
||||
|
||||
use thiserror_ext::AsReport as _;
|
||||
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
|
||||
use vllm_parser::output_grammar::BuiltOutputGrammar;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use xgrammar_structural_tag::format::Format;
|
||||
|
||||
@@ -309,36 +312,45 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
/// The output grammar for one `search` tool with `strict` under `parser`.
|
||||
fn search_grammar(
|
||||
parser: &str,
|
||||
level: Option<ToolStrictLevel>,
|
||||
strict: Option<bool>,
|
||||
choice: ChatToolChoice,
|
||||
) -> Option<BuiltOutputGrammar> {
|
||||
let tools = vec![ChatTool {
|
||||
name: "search".to_string(),
|
||||
description: None,
|
||||
parameters: serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": { "query": { "type": "string" } },
|
||||
"required": ["query"]
|
||||
}),
|
||||
strict,
|
||||
defer_loading: None,
|
||||
}];
|
||||
let mut request = ChatRequest {
|
||||
tool_context: ResolvedToolContext::new(&[], tools, Some(choice), true).unwrap(),
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
let mut processor = DefaultChatOutputProcessor::new(
|
||||
&mut request,
|
||||
"other-model",
|
||||
tokenizer(),
|
||||
&ParserSelection::Explicit(parser.to_string()),
|
||||
&ParserSelection::None,
|
||||
level,
|
||||
)
|
||||
.unwrap();
|
||||
processor.initialize(&[]).unwrap();
|
||||
processor.build_output_grammar().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_grammar_preserves_tool_strict_level() {
|
||||
let build = |level, strict, choice| {
|
||||
let tools = vec![ChatTool {
|
||||
name: "search".to_string(),
|
||||
description: None,
|
||||
parameters: serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": { "query": { "type": "string" } },
|
||||
"required": ["query"]
|
||||
}),
|
||||
strict,
|
||||
defer_loading: None,
|
||||
}];
|
||||
let mut request = ChatRequest {
|
||||
tool_context: ResolvedToolContext::new(&[], tools, Some(choice), true).unwrap(),
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
let mut processor = DefaultChatOutputProcessor::new(
|
||||
&mut request,
|
||||
"other-model",
|
||||
tokenizer(),
|
||||
&ParserSelection::Explicit("qwen3_coder".to_string()),
|
||||
&ParserSelection::None,
|
||||
level,
|
||||
)
|
||||
.unwrap();
|
||||
processor.initialize(&[]).unwrap();
|
||||
processor.build_output_grammar().unwrap()
|
||||
};
|
||||
let build =
|
||||
|level, strict, choice| search_grammar("qwen3_coder", Some(level), strict, choice);
|
||||
|
||||
assert!(build(ToolStrictLevel::Auto, None, ChatToolChoice::Auto).is_none());
|
||||
let envelope = build(ToolStrictLevel::Function, None, ChatToolChoice::Auto).unwrap();
|
||||
@@ -349,6 +361,19 @@ mod tests {
|
||||
assert!(build(ToolStrictLevel::Parameter, None, ChatToolChoice::None).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unset_tool_strict_level_falls_back_to_the_parser_default() {
|
||||
let build = |parser, level| search_grammar(parser, level, None, ChatToolChoice::Auto);
|
||||
|
||||
// GLM-4.7 constrains auto tool calls unless the level is set explicitly.
|
||||
let function = build("glm47", Some(ToolStrictLevel::Function));
|
||||
assert_eq!(build("glm47", None), function);
|
||||
assert!(function.is_some());
|
||||
assert!(build("glm47", Some(ToolStrictLevel::Auto)).is_none());
|
||||
// Other parsers default to `auto`.
|
||||
assert!(build("qwen3_coder", None).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answer_constraint_keeps_optional_tool_calls() {
|
||||
let tools = vec![ChatTool {
|
||||
@@ -377,7 +402,7 @@ mod tests {
|
||||
tokenizer(),
|
||||
&ParserSelection::Explicit("qwen3_coder".to_string()),
|
||||
&ParserSelection::None,
|
||||
ToolStrictLevel::Auto,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
processor.initialize(&[]).unwrap();
|
||||
@@ -420,7 +445,7 @@ mod tests {
|
||||
tokenizer(),
|
||||
&ParserSelection::Explicit("qwen3_coder".to_string()),
|
||||
&ParserSelection::None,
|
||||
ToolStrictLevel::Auto,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
processor.initialize(&[]).unwrap();
|
||||
@@ -444,7 +469,7 @@ mod tests {
|
||||
tokenizer(),
|
||||
&selection,
|
||||
&selection,
|
||||
ToolStrictLevel::Auto,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
@@ -459,7 +484,7 @@ mod tests {
|
||||
tokenizer(),
|
||||
&ParserSelection::Auto,
|
||||
&ParserSelection::Auto,
|
||||
ToolStrictLevel::Auto,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
@@ -477,7 +502,7 @@ mod tests {
|
||||
tokenizer(),
|
||||
tool,
|
||||
reasoning,
|
||||
ToolStrictLevel::Auto,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
@@ -492,7 +517,7 @@ mod tests {
|
||||
tokenizer(),
|
||||
&ParserSelection::Auto,
|
||||
&ParserSelection::Explicit("gemma4".to_string()),
|
||||
ToolStrictLevel::Auto,
|
||||
None,
|
||||
) {
|
||||
Ok(_) => panic!("expected mixed Gemma4 parser selection to fail"),
|
||||
Err(error) => error,
|
||||
@@ -539,7 +564,7 @@ mod tests {
|
||||
template,
|
||||
tool,
|
||||
reasoning,
|
||||
ToolStrictLevel::Auto,
|
||||
None,
|
||||
)?;
|
||||
Ok(!request.decode_options.skip_special_tokens)
|
||||
}
|
||||
|
||||
@@ -958,7 +958,7 @@ async fn parse_completion(
|
||||
NewChatOutputProcessorOptions {
|
||||
// Pin every tool schema so that builder-backed parsers build an
|
||||
// output grammar for replay. The strict level does not affect parsing.
|
||||
tool_strict_level: ToolStrictLevel::Parameter,
|
||||
tool_strict_level: Some(ToolStrictLevel::Parameter),
|
||||
tool_call_parser: &case.tool_call_parser,
|
||||
reasoning_parser: &case.reasoning_parser,
|
||||
},
|
||||
|
||||
@@ -137,9 +137,10 @@ pub struct RenderArgs {
|
||||
#[arg(long, default_value_t)]
|
||||
reasoning_parser: ParserSelection,
|
||||
/// Server-side floor for structural-tag based tool calling: `auto`,
|
||||
/// `function`, or `parameter`.
|
||||
#[arg(long, default_value_t)]
|
||||
tool_strict_level: ToolStrictLevel,
|
||||
/// `function`, or `parameter`. When unset, the tool parser's default
|
||||
/// level applies, else `auto`.
|
||||
#[arg(long)]
|
||||
tool_strict_level: Option<ToolStrictLevel>,
|
||||
/// Select the native chat renderer implementation.
|
||||
#[arg(long = "tokenizer-mode", default_value_t)]
|
||||
renderer: RendererSelection,
|
||||
@@ -252,10 +253,11 @@ pub struct SharedRuntimeArgs {
|
||||
/// top of the per-tool `strict` field: `auto` follows the request's
|
||||
/// tool choice and per-tool strictness,
|
||||
/// `function` constrains the tool-call envelope for every request with
|
||||
/// tools, `parameter` additionally pins argument schemas.
|
||||
#[arg(long, default_value_t)]
|
||||
/// tools, `parameter` additionally pins argument schemas. When unset, the
|
||||
/// tool parser's default level applies, else `auto`.
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub tool_strict_level: ToolStrictLevel,
|
||||
pub tool_strict_level: Option<ToolStrictLevel>,
|
||||
/// Select the chat renderer implementation.
|
||||
#[arg(long = "tokenizer-mode", default_value_t)]
|
||||
#[serde(default, rename = "tokenizer_mode")]
|
||||
|
||||
@@ -185,7 +185,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_strict_level: Auto,
|
||||
tool_strict_level: None,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
@@ -1088,7 +1088,7 @@ fn frontend_args_accept_json() {
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: None,
|
||||
reasoning_parser: None,
|
||||
tool_strict_level: Auto,
|
||||
tool_strict_level: None,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
@@ -1760,7 +1760,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_strict_level: Auto,
|
||||
tool_strict_level: None,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
@@ -1919,7 +1919,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
grpc_listener_mode: None,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_strict_level: Auto,
|
||||
tool_strict_level: None,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
@@ -2014,7 +2014,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
grpc_listener_mode: None,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_strict_level: Auto,
|
||||
tool_strict_level: None,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
@@ -2153,7 +2153,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
grpc_listener_mode: None,
|
||||
tool_call_parser: None,
|
||||
reasoning_parser: None,
|
||||
tool_strict_level: Auto,
|
||||
tool_strict_level: None,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use super::{GlmXmlToolParser, Separator};
|
||||
use crate::output_grammar::ToolStrictLevel;
|
||||
use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutput};
|
||||
|
||||
/// Tool parser for GLM-4.7 MoE XML-style tool calls.
|
||||
@@ -25,6 +26,11 @@ impl ToolParser for Glm47MoeToolParser {
|
||||
Ok(Box::new(Self::new(tools)))
|
||||
}
|
||||
|
||||
// Constrain auto tool calls by default, like the Python `glm47` parser.
|
||||
fn default_tool_strict_level(&self) -> ToolStrictLevel {
|
||||
ToolStrictLevel::Function
|
||||
}
|
||||
|
||||
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
|
||||
Some(xgrammar_structural_tag::Model::Glm47.builder())
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@ pub use xgrammar_structural_tag::builders::StructuralTagBuilder;
|
||||
use xgrammar_structural_tag::format::Format;
|
||||
use xgrammar_structural_tag::tool::{AllowedToolsMode, ToolChoiceValue};
|
||||
|
||||
use crate::output_grammar::{self, OutputGrammarContext};
|
||||
use crate::output_grammar::{self, OutputGrammarContext, ToolStrictLevel};
|
||||
use crate::utils;
|
||||
|
||||
/// One function-style tool made available to the model.
|
||||
@@ -199,6 +199,12 @@ pub trait ToolParser: Send {
|
||||
false
|
||||
}
|
||||
|
||||
/// Return the tool strict level applied when the operator leaves
|
||||
/// `--tool-strict-level` unset.
|
||||
fn default_tool_strict_level(&self) -> ToolStrictLevel {
|
||||
ToolStrictLevel::Auto
|
||||
}
|
||||
|
||||
/// Build the language of everything the model may emit after reasoning
|
||||
/// ends for the context's tool choice: visible text plus tool calls. `None`
|
||||
/// means no tool grammar applies.
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer};
|
||||
|
||||
use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput};
|
||||
use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext};
|
||||
use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext, ToolStrictLevel};
|
||||
use crate::reasoning::ReasoningParser;
|
||||
use crate::tool::{Tool, ToolParser, ToolParserOutput};
|
||||
|
||||
@@ -83,6 +83,12 @@ impl UnifiedParser for CombinedParser {
|
||||
|| self.tool.as_ref().is_some_and(|parser| parser.preserve_special_tokens())
|
||||
}
|
||||
|
||||
fn default_tool_strict_level(&self) -> ToolStrictLevel {
|
||||
self.tool.as_ref().map_or(ToolStrictLevel::Auto, |parser| {
|
||||
parser.default_tool_strict_level()
|
||||
})
|
||||
}
|
||||
|
||||
fn build_output_grammar(
|
||||
&self,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
|
||||
@@ -20,7 +20,7 @@ use thiserror::Error;
|
||||
use thiserror_ext::Macro;
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer};
|
||||
|
||||
use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext};
|
||||
use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext, ToolStrictLevel};
|
||||
use crate::reasoning::ReasoningError;
|
||||
use crate::tool::{Tool, ToolCallDelta, ToolParserError, ToolParserEvent, ToolParserOutput};
|
||||
use crate::utils::SpecialToken;
|
||||
@@ -242,6 +242,12 @@ pub trait UnifiedParser: Send {
|
||||
false
|
||||
}
|
||||
|
||||
/// Return the tool strict level applied when the operator leaves
|
||||
/// `--tool-strict-level` unset.
|
||||
fn default_tool_strict_level(&self) -> ToolStrictLevel {
|
||||
ToolStrictLevel::Auto
|
||||
}
|
||||
|
||||
/// Build the request output grammar after prompt-based initialization.
|
||||
fn build_output_grammar(
|
||||
&self,
|
||||
|
||||
@@ -17,7 +17,6 @@ use clap::Parser;
|
||||
use futures::StreamExt as _;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_chat::ToolStrictLevel;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_server::{
|
||||
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
|
||||
@@ -73,7 +72,7 @@ async fn main() -> Result<()> {
|
||||
grpc_listener_mode: None,
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
tool_strict_level: None,
|
||||
renderer: RendererSelection::Auto,
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
|
||||
@@ -238,8 +238,9 @@ pub struct Config {
|
||||
pub tool_call_parser: ParserSelection,
|
||||
/// Reasoning parser selection.
|
||||
pub reasoning_parser: ParserSelection,
|
||||
/// Server-side floor for tool-call structural tags.
|
||||
pub tool_strict_level: ToolStrictLevel,
|
||||
/// Server-side floor for tool-call structural tags, or `None` for the
|
||||
/// tool parser's default level.
|
||||
pub tool_strict_level: Option<ToolStrictLevel>,
|
||||
/// Chat renderer selection.
|
||||
pub renderer: RendererSelection,
|
||||
/// Disable frontend-side multimodal preprocessing and render the model as
|
||||
|
||||
@@ -32,7 +32,7 @@ pub struct RenderConfig {
|
||||
pub port: u16,
|
||||
pub tool_call_parser: ParserSelection,
|
||||
pub reasoning_parser: ParserSelection,
|
||||
pub tool_strict_level: ToolStrictLevel,
|
||||
pub tool_strict_level: Option<ToolStrictLevel>,
|
||||
pub renderer: RendererSelection,
|
||||
pub chat_template: Option<String>,
|
||||
pub default_chat_template_kwargs: HashMap<String, Value>,
|
||||
@@ -157,7 +157,7 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
let error = serve_render(
|
||||
RenderConfig {
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
tool_strict_level: None,
|
||||
model: "test-model".to_string(),
|
||||
revision: None,
|
||||
hf_overrides: Default::default(),
|
||||
|
||||
Reference in New Issue
Block a user