diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index 3081047dce..dd56db0d57 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -146,7 +146,7 @@ Most OpenAI-compatible clients and agent frameworks never set `strict` on their | Value | Behavior | | --- | --- | -| `auto` (default) | Follow the request's tool choice and per-tool strictness. Required/named choices activate structural tags; `tool_choice="auto"` activates them when at least one tool sets `strict: true`. | +| `auto` | Follow the request's tool choice and per-tool strictness. Required/named choices activate structural tags; `tool_choice="auto"` activates them when at least one tool sets `strict: true`. | | `function` | Constrain the tool-call envelope (markup and the function name) for every request with tools, leaving argument contents free unless the client marked the tool `strict: true`. | | `parameter` | Additionally pin argument schemas for every tool, as if every tool had `strict: true`. | @@ -154,6 +154,8 @@ Most OpenAI-compatible clients and agent frameworks never set `strict` on their vllm serve ... --tool-strict-level function ``` +When the option is unset, the tool parser's default level applies: `function` for `glm47`, `auto` for the other parsers. Passing `--tool-strict-level auto` overrides a parser default. + The floor never relaxes a constraint the request would already receive: tools the client marked `strict: true` keep their schemas at every level. With `tool_choice="auto"`, the grammar does not force a tool call; a plain text response stays valid. `VLLM_ENFORCE_STRICT_TOOL_CALLING=false` disables structural tags entirely and takes precedence over this option. ## Automatic Function Calling diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index 174115693a..54b31b165d 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -380,7 +380,7 @@ mod tests { let error = match backend.new_chat_output_processor( &mut request, NewChatOutputProcessorOptions { - tool_strict_level: crate::ToolStrictLevel::Auto, + tool_strict_level: None, tool_call_parser: &ParserSelection::Explicit("json".to_string()), reasoning_parser: &ParserSelection::Auto, }, diff --git a/rust/src/chat/src/backend/mod.rs b/rust/src/chat/src/backend/mod.rs index a0343e53f3..76abd7ec55 100644 --- a/rust/src/chat/src/backend/mod.rs +++ b/rust/src/chat/src/backend/mod.rs @@ -21,7 +21,9 @@ pub mod hf; pub struct NewChatOutputProcessorOptions<'a> { pub tool_call_parser: &'a ParserSelection, pub reasoning_parser: &'a ParserSelection, - pub tool_strict_level: ToolStrictLevel, + /// Server-side floor for tool-call structural tags, or `None` for the + /// tool parser's default level. + pub tool_strict_level: Option, } /// Minimal prompt-processing backend needed by `vllm-chat`. diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 0b7e89914c..1ffb95fe37 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -79,8 +79,9 @@ pub struct ChatRequestProcessor { tool_call_parser: ParserSelection, /// Reasoning parser selection used when preparing generation requests. reasoning_parser: ParserSelection, - /// Server-side floor for tool-call structural tags. - tool_strict_level: ToolStrictLevel, + /// Server-side floor for tool-call structural tags, or `None` for the + /// tool parser's default level. + tool_strict_level: Option, } impl ChatRequestProcessor { @@ -93,7 +94,7 @@ impl ChatRequestProcessor { max_model_len: Some(max_model_len as usize), tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, - tool_strict_level: ToolStrictLevel::Auto, + tool_strict_level: None, } } @@ -105,7 +106,7 @@ impl ChatRequestProcessor { max_model_len: None, tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, - tool_strict_level: ToolStrictLevel::Auto, + tool_strict_level: None, } } @@ -120,8 +121,9 @@ impl ChatRequestProcessor { self } - /// Configure the server-side floor for tool-call structural tags. - pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self { + /// Configure the server-side floor for tool-call structural tags, or + /// `None` for the tool parser's default level. + pub fn with_tool_strict_level(mut self, tool_strict_level: Option) -> Self { self.tool_strict_level = tool_strict_level; self } @@ -295,8 +297,9 @@ impl ChatLlm { self } - /// Set the server-side floor for tool-call structural tags. - pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self { + /// Set the server-side floor for tool-call structural tags, or `None` for + /// the tool parser's default level. + pub fn with_tool_strict_level(mut self, tool_strict_level: Option) -> Self { self.processor.tool_strict_level = tool_strict_level; self } diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index e95ebd8b0d..67a5a11eae 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -70,7 +70,7 @@ impl DefaultChatOutputProcessor { tokenizer: DynTokenizer, tool_call_parser: &ParserSelection, reasoning_parser: &ParserSelection, - tool_strict_level: ToolStrictLevel, + tool_strict_level: Option, ) -> ChatResult { Self::with_response_template( request, @@ -92,7 +92,7 @@ impl DefaultChatOutputProcessor { response_template: &std::result::Result, HfTemplateError>, tool_call_parser: &ParserSelection, reasoning_parser: &ParserSelection, - tool_strict_level: ToolStrictLevel, + tool_strict_level: Option, ) -> ChatResult { let tool_name = tool_call_parser.resolve_tool_name(model_id); let reasoning_name = reasoning_parser.resolve_reasoning_name(model_id); @@ -135,6 +135,8 @@ impl DefaultChatOutputProcessor { if parser.preserve_special_tokens() { request.decode_options.skip_special_tokens = false; } + let tool_strict_level = + tool_strict_level.unwrap_or_else(|| parser.default_tool_strict_level()); Ok(Self { parser, @@ -292,6 +294,7 @@ mod tests { use thiserror_ext::AsReport as _; use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams; + use vllm_parser::output_grammar::BuiltOutputGrammar; use vllm_tokenizer::test_utils::TestTokenizer; use xgrammar_structural_tag::format::Format; @@ -309,36 +312,45 @@ mod tests { ) } + /// The output grammar for one `search` tool with `strict` under `parser`. + fn search_grammar( + parser: &str, + level: Option, + strict: Option, + choice: ChatToolChoice, + ) -> Option { + let tools = vec![ChatTool { + name: "search".to_string(), + description: None, + parameters: serde_json::json!({ + "type": "object", + "properties": { "query": { "type": "string" } }, + "required": ["query"] + }), + strict, + defer_loading: None, + }]; + let mut request = ChatRequest { + tool_context: ResolvedToolContext::new(&[], tools, Some(choice), true).unwrap(), + ..ChatRequest::for_test() + }; + let mut processor = DefaultChatOutputProcessor::new( + &mut request, + "other-model", + tokenizer(), + &ParserSelection::Explicit(parser.to_string()), + &ParserSelection::None, + level, + ) + .unwrap(); + processor.initialize(&[]).unwrap(); + processor.build_output_grammar().unwrap() + } + #[test] fn output_grammar_preserves_tool_strict_level() { - let build = |level, strict, choice| { - let tools = vec![ChatTool { - name: "search".to_string(), - description: None, - parameters: serde_json::json!({ - "type": "object", - "properties": { "query": { "type": "string" } }, - "required": ["query"] - }), - strict, - defer_loading: None, - }]; - let mut request = ChatRequest { - tool_context: ResolvedToolContext::new(&[], tools, Some(choice), true).unwrap(), - ..ChatRequest::for_test() - }; - let mut processor = DefaultChatOutputProcessor::new( - &mut request, - "other-model", - tokenizer(), - &ParserSelection::Explicit("qwen3_coder".to_string()), - &ParserSelection::None, - level, - ) - .unwrap(); - processor.initialize(&[]).unwrap(); - processor.build_output_grammar().unwrap() - }; + let build = + |level, strict, choice| search_grammar("qwen3_coder", Some(level), strict, choice); assert!(build(ToolStrictLevel::Auto, None, ChatToolChoice::Auto).is_none()); let envelope = build(ToolStrictLevel::Function, None, ChatToolChoice::Auto).unwrap(); @@ -349,6 +361,19 @@ mod tests { assert!(build(ToolStrictLevel::Parameter, None, ChatToolChoice::None).is_none()); } + #[test] + fn unset_tool_strict_level_falls_back_to_the_parser_default() { + let build = |parser, level| search_grammar(parser, level, None, ChatToolChoice::Auto); + + // GLM-4.7 constrains auto tool calls unless the level is set explicitly. + let function = build("glm47", Some(ToolStrictLevel::Function)); + assert_eq!(build("glm47", None), function); + assert!(function.is_some()); + assert!(build("glm47", Some(ToolStrictLevel::Auto)).is_none()); + // Other parsers default to `auto`. + assert!(build("qwen3_coder", None).is_none()); + } + #[test] fn answer_constraint_keeps_optional_tool_calls() { let tools = vec![ChatTool { @@ -377,7 +402,7 @@ mod tests { tokenizer(), &ParserSelection::Explicit("qwen3_coder".to_string()), &ParserSelection::None, - ToolStrictLevel::Auto, + None, ) .unwrap(); processor.initialize(&[]).unwrap(); @@ -420,7 +445,7 @@ mod tests { tokenizer(), &ParserSelection::Explicit("qwen3_coder".to_string()), &ParserSelection::None, - ToolStrictLevel::Auto, + None, ) .unwrap(); processor.initialize(&[]).unwrap(); @@ -444,7 +469,7 @@ mod tests { tokenizer(), &selection, &selection, - ToolStrictLevel::Auto, + None, ) .unwrap(); } @@ -459,7 +484,7 @@ mod tests { tokenizer(), &ParserSelection::Auto, &ParserSelection::Auto, - ToolStrictLevel::Auto, + None, ) .unwrap(); } @@ -477,7 +502,7 @@ mod tests { tokenizer(), tool, reasoning, - ToolStrictLevel::Auto, + None, ) .unwrap(); } @@ -492,7 +517,7 @@ mod tests { tokenizer(), &ParserSelection::Auto, &ParserSelection::Explicit("gemma4".to_string()), - ToolStrictLevel::Auto, + None, ) { Ok(_) => panic!("expected mixed Gemma4 parser selection to fail"), Err(error) => error, @@ -539,7 +564,7 @@ mod tests { template, tool, reasoning, - ToolStrictLevel::Auto, + None, )?; Ok(!request.decode_options.skip_special_tokens) } diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index d2891ed027..ec4202be74 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -958,7 +958,7 @@ async fn parse_completion( NewChatOutputProcessorOptions { // Pin every tool schema so that builder-backed parsers build an // output grammar for replay. The strict level does not affect parsing. - tool_strict_level: ToolStrictLevel::Parameter, + tool_strict_level: Some(ToolStrictLevel::Parameter), tool_call_parser: &case.tool_call_parser, reasoning_parser: &case.reasoning_parser, }, diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index 66b48f13d3..e6b329cfb4 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -137,9 +137,10 @@ pub struct RenderArgs { #[arg(long, default_value_t)] reasoning_parser: ParserSelection, /// Server-side floor for structural-tag based tool calling: `auto`, - /// `function`, or `parameter`. - #[arg(long, default_value_t)] - tool_strict_level: ToolStrictLevel, + /// `function`, or `parameter`. When unset, the tool parser's default + /// level applies, else `auto`. + #[arg(long)] + tool_strict_level: Option, /// Select the native chat renderer implementation. #[arg(long = "tokenizer-mode", default_value_t)] renderer: RendererSelection, @@ -252,10 +253,11 @@ pub struct SharedRuntimeArgs { /// top of the per-tool `strict` field: `auto` follows the request's /// tool choice and per-tool strictness, /// `function` constrains the tool-call envelope for every request with - /// tools, `parameter` additionally pins argument schemas. - #[arg(long, default_value_t)] + /// tools, `parameter` additionally pins argument schemas. When unset, the + /// tool parser's default level applies, else `auto`. + #[arg(long)] #[serde(default)] - pub tool_strict_level: ToolStrictLevel, + pub tool_strict_level: Option, /// Select the chat renderer implementation. #[arg(long = "tokenizer-mode", default_value_t)] #[serde(default, rename = "tokenizer_mode")] diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 8f4318ac16..047635207e 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -185,7 +185,7 @@ fn serve_args_forward_python_flags_with_separator() { engine_ready_timeout_secs: 600, tool_call_parser: Auto, reasoning_parser: Auto, - tool_strict_level: Auto, + tool_strict_level: None, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1088,7 +1088,7 @@ fn frontend_args_accept_json() { engine_ready_timeout_secs: 600, tool_call_parser: None, reasoning_parser: None, - tool_strict_level: Auto, + tool_strict_level: None, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1760,7 +1760,7 @@ fn serve_args_accept_handshake_aliases() { engine_ready_timeout_secs: 600, tool_call_parser: Auto, reasoning_parser: Auto, - tool_strict_level: Auto, + tool_strict_level: None, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1919,7 +1919,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { grpc_listener_mode: None, tool_call_parser: Auto, reasoning_parser: Auto, - tool_strict_level: Auto, + tool_strict_level: None, renderer: Auto, language_model_only: false, chat_template: None, @@ -2014,7 +2014,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { grpc_listener_mode: None, tool_call_parser: Auto, reasoning_parser: Auto, - tool_strict_level: Auto, + tool_strict_level: None, renderer: Auto, language_model_only: false, chat_template: None, @@ -2153,7 +2153,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present grpc_listener_mode: None, tool_call_parser: None, reasoning_parser: None, - tool_strict_level: Auto, + tool_strict_level: None, renderer: Auto, language_model_only: false, chat_template: None, diff --git a/rust/src/parser/src/tool/glm_xml/glm47_moe.rs b/rust/src/parser/src/tool/glm_xml/glm47_moe.rs index 06477b4049..15f04692c7 100644 --- a/rust/src/parser/src/tool/glm_xml/glm47_moe.rs +++ b/rust/src/parser/src/tool/glm_xml/glm47_moe.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright contributors to the vLLM project use super::{GlmXmlToolParser, Separator}; +use crate::output_grammar::ToolStrictLevel; use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutput}; /// Tool parser for GLM-4.7 MoE XML-style tool calls. @@ -25,6 +26,11 @@ impl ToolParser for Glm47MoeToolParser { Ok(Box::new(Self::new(tools))) } + // Constrain auto tool calls by default, like the Python `glm47` parser. + fn default_tool_strict_level(&self) -> ToolStrictLevel { + ToolStrictLevel::Function + } + fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> { Some(xgrammar_structural_tag::Model::Glm47.builder()) } diff --git a/rust/src/parser/src/tool/mod.rs b/rust/src/parser/src/tool/mod.rs index 69ecee1bff..d8637bad1f 100644 --- a/rust/src/parser/src/tool/mod.rs +++ b/rust/src/parser/src/tool/mod.rs @@ -46,7 +46,7 @@ pub use xgrammar_structural_tag::builders::StructuralTagBuilder; use xgrammar_structural_tag::format::Format; use xgrammar_structural_tag::tool::{AllowedToolsMode, ToolChoiceValue}; -use crate::output_grammar::{self, OutputGrammarContext}; +use crate::output_grammar::{self, OutputGrammarContext, ToolStrictLevel}; use crate::utils; /// One function-style tool made available to the model. @@ -199,6 +199,12 @@ pub trait ToolParser: Send { false } + /// Return the tool strict level applied when the operator leaves + /// `--tool-strict-level` unset. + fn default_tool_strict_level(&self) -> ToolStrictLevel { + ToolStrictLevel::Auto + } + /// Build the language of everything the model may emit after reasoning /// ends for the context's tool choice: visible text plus tool calls. `None` /// means no tool grammar applies. diff --git a/rust/src/parser/src/unified/combined.rs b/rust/src/parser/src/unified/combined.rs index aaab431256..f4dfd0e49e 100644 --- a/rust/src/parser/src/unified/combined.rs +++ b/rust/src/parser/src/unified/combined.rs @@ -6,7 +6,7 @@ use vllm_tokenizer::{DecodedText, DynTokenizer}; use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput}; -use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext}; +use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext, ToolStrictLevel}; use crate::reasoning::ReasoningParser; use crate::tool::{Tool, ToolParser, ToolParserOutput}; @@ -83,6 +83,12 @@ impl UnifiedParser for CombinedParser { || self.tool.as_ref().is_some_and(|parser| parser.preserve_special_tokens()) } + fn default_tool_strict_level(&self) -> ToolStrictLevel { + self.tool.as_ref().map_or(ToolStrictLevel::Auto, |parser| { + parser.default_tool_strict_level() + }) + } + fn build_output_grammar( &self, ctx: &OutputGrammarContext<'_>, diff --git a/rust/src/parser/src/unified/mod.rs b/rust/src/parser/src/unified/mod.rs index 493b7b3f1b..74bef8e1f4 100644 --- a/rust/src/parser/src/unified/mod.rs +++ b/rust/src/parser/src/unified/mod.rs @@ -20,7 +20,7 @@ use thiserror::Error; use thiserror_ext::Macro; use vllm_tokenizer::{DecodedText, DynTokenizer}; -use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext}; +use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext, ToolStrictLevel}; use crate::reasoning::ReasoningError; use crate::tool::{Tool, ToolCallDelta, ToolParserError, ToolParserEvent, ToolParserOutput}; use crate::utils::SpecialToken; @@ -242,6 +242,12 @@ pub trait UnifiedParser: Send { false } + /// Return the tool strict level applied when the operator leaves + /// `--tool-strict-level` unset. + fn default_tool_strict_level(&self) -> ToolStrictLevel { + ToolStrictLevel::Auto + } + /// Build the request output grammar after prompt-based initialization. fn build_output_grammar( &self, diff --git a/rust/src/server/examples/external_engine_openai_qwen.rs b/rust/src/server/examples/external_engine_openai_qwen.rs index a391f56cae..c3b6625d25 100644 --- a/rust/src/server/examples/external_engine_openai_qwen.rs +++ b/rust/src/server/examples/external_engine_openai_qwen.rs @@ -17,7 +17,6 @@ use clap::Parser; use futures::StreamExt as _; use tokio_util::sync::CancellationToken; use tracing_subscriber::EnvFilter; -use vllm_chat::ToolStrictLevel; use vllm_engine_core_client::TransportMode; use vllm_server::{ ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig, @@ -73,7 +72,7 @@ async fn main() -> Result<()> { grpc_listener_mode: None, tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, - tool_strict_level: ToolStrictLevel::Auto, + tool_strict_level: None, renderer: RendererSelection::Auto, language_model_only: false, chat_template: None, diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index d70933fb44..b8fbc09828 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -238,8 +238,9 @@ pub struct Config { pub tool_call_parser: ParserSelection, /// Reasoning parser selection. pub reasoning_parser: ParserSelection, - /// Server-side floor for tool-call structural tags. - pub tool_strict_level: ToolStrictLevel, + /// Server-side floor for tool-call structural tags, or `None` for the + /// tool parser's default level. + pub tool_strict_level: Option, /// Chat renderer selection. pub renderer: RendererSelection, /// Disable frontend-side multimodal preprocessing and render the model as diff --git a/rust/src/server/src/render.rs b/rust/src/server/src/render.rs index 50485e26a4..1fe50d93b2 100644 --- a/rust/src/server/src/render.rs +++ b/rust/src/server/src/render.rs @@ -32,7 +32,7 @@ pub struct RenderConfig { pub port: u16, pub tool_call_parser: ParserSelection, pub reasoning_parser: ParserSelection, - pub tool_strict_level: ToolStrictLevel, + pub tool_strict_level: Option, pub renderer: RendererSelection, pub chat_template: Option, pub default_chat_template_kwargs: HashMap, @@ -157,7 +157,7 @@ mod tests { shutdown.cancel(); let error = serve_render( RenderConfig { - tool_strict_level: ToolStrictLevel::Auto, + tool_strict_level: None, model: "test-model".to_string(), revision: None, hf_overrides: Default::default(), diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 52d0d82469..73a08b3797 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -24,6 +24,7 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.entrypoints.openai.responses.utils import construct_tool_dicts from vllm.exceptions import VLLMValidationError from vllm.parser.abstract_parser import DelegatingParser +from vllm.parser.parser_manager import ParserManager from vllm.parser.plamo3 import ( BEGIN_TOOL_ARGUMENTS, BEGIN_TOOL_NAME, @@ -1991,6 +1992,34 @@ def test_glm47_operator_level_overrides_parser_default( assert content["json_schema"] == sample_tools[0].function.parameters +@pytest.mark.parametrize( + ("tool_strict_level", "constrained"), [(None, True), ("auto", False)] +) +def test_glm47_default_level_applies_only_when_the_level_is_unset( + monkeypatch: pytest.MonkeyPatch, + sample_tools: list[ChatCompletionToolsParam], + tool_strict_level: str | None, + constrained: bool, +): + monkeypatch.setattr(envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", True) + parser_cls = ParserManager.get_parser( + tool_parser_name="glm47", + enable_auto_tools=True, + tool_strict_level=tool_strict_level, + ) + assert parser_cls is not None + request = ChatCompletionRequest( + messages=[], + model="m", + tools=[tool.model_dump(exclude_none=True) for tool in sample_tools], + tool_choice="auto", + ) + + out = parser_cls(MagicMock(), tools=sample_tools).adjust_request(request) + + assert (out.structured_outputs is not None) is constrained + + def test_glm47_required_with_non_strict_tools_attaches_shallow_tag( monkeypatch: pytest.MonkeyPatch, sample_tools: list[ChatCompletionToolsParam], diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 22319f1c70..4b7f2b95a5 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -160,7 +160,7 @@ class AnthropicServingMessages(OpenAIServingChat): reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, - tool_strict_level: str = "auto", + tool_strict_level: str | None = None, enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, enable_log_outputs: bool = False, diff --git a/vllm/entrypoints/cohere/serving.py b/vllm/entrypoints/cohere/serving.py index 45eb369e78..d4bf81d8c2 100644 --- a/vllm/entrypoints/cohere/serving.py +++ b/vllm/entrypoints/cohere/serving.py @@ -251,7 +251,7 @@ class CohereServingChatV2(OpenAIServingChat): reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, - tool_strict_level: str = "auto", + tool_strict_level: str | None = None, enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, enable_log_outputs: bool = False, diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index e2c2625e94..25bfabc974 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -128,13 +128,14 @@ class BaseFrontendArgs: """Special the tool parser plugin write to parse the model-generated tool into OpenAI API format, the name register in this plugin can be used in `--tool-call-parser`.""" - tool_strict_level: ToolStrictLevelName = "auto" + tool_strict_level: ToolStrictLevelName | None = None """Server-side floor for structural-tag based tool calling, applied on top of the per-tool `strict` field. `auto` follows the request's tool choice and per-tool strictness; `function` constrains the tool-call envelope (markup and function name) for every request with tools; `parameter` additionally pins argument schemas, as if every tool - were `strict: true`.""" + were `strict: true`. When unset, the tool parser's default level applies, + which is `auto` for most parsers.""" tool_server: str | None = None """Comma-separated list of host:port pairs (IPv4, IPv6, or hostname). Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index b291ec6e8a..84061a8822 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -132,7 +132,7 @@ class OpenAIServingChat(GenerateBaseServing): enable_auto_tools: bool = False, exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, - tool_strict_level: str = "auto", + tool_strict_level: str | None = None, enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, enable_log_outputs: bool = False, diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 78a3f1015f..0d46855c94 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -113,7 +113,7 @@ class OpenAIServingResponses(GenerateBaseServing): reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, - tool_strict_level: str = "auto", + tool_strict_level: str | None = None, tool_server: ToolServer | None = None, enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 12a9c23698..fbddc45783 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -118,8 +118,9 @@ class Parser: # Subclasses should override these if they use specific parser classes reasoning_parser_cls: type[ReasoningParser] | None = None tool_parser_cls: type[ToolParser] | None = None - # Server-side floor for tool-call structural tags (--tool-strict-level). - tool_strict_level: ToolStrictLevel = ToolStrictLevel.AUTO + # Server-side floor for tool-call structural tags (--tool-strict-level), or + # None for the tool parser's default level. + tool_strict_level: ToolStrictLevel | None = None always_adjust_request: bool = False def __init__( @@ -519,10 +520,7 @@ class DelegatingParser(Parser): is_auto = request.tool_choice == "auto" single_call = request.parallel_tool_calls is False strict_level = self.tool_strict_level - if ( - strict_level == ToolStrictLevel.AUTO - and tool_parser.default_tool_strict_level is not None - ): + if strict_level is None: strict_level = tool_parser.default_tool_strict_level if single_call and not (is_auto and structured_outputs is not None): # Limiting the call count needs the call envelope in the grammar. diff --git a/vllm/parser/parser_manager.py b/vllm/parser/parser_manager.py index 6766d15e91..d6fdbce19c 100644 --- a/vllm/parser/parser_manager.py +++ b/vllm/parser/parser_manager.py @@ -104,7 +104,7 @@ class ParserManager: enable_auto_tools: bool = False, model_name: str | None = None, is_harmony: bool = False, - tool_strict_level: str = "auto", + tool_strict_level: str | None = None, tokenizer: TokenizerLike | None = None, ) -> type[Parser] | None: """Get a Parser that handles both reasoning and tool parsing. @@ -119,7 +119,8 @@ class ParserManager: is_harmony: Whether the selected model uses the Harmony format. If True, HarmonyParser is always returned. tool_strict_level: Server-side floor for tool-call structural - tags (``--tool-strict-level``). + tags (``--tool-strict-level``), or ``None`` for the tool + parser's default level. tokenizer: Tokenizer the composed parser is validated against at startup (see :meth:`validate_parser_tokenizer`). ``None`` skips the check (e.g. ``skip_tokenizer_init``). @@ -155,7 +156,7 @@ class ParserManager: enable_auto_tools: bool, model_name: str | None, is_harmony: bool, - tool_strict_level: str, + tool_strict_level: str | None, tokenizer: TokenizerLike | None, ) -> type[Parser] | None: if not tool_parser_name and not reasoning_parser_name: @@ -169,7 +170,11 @@ class ParserManager: if reasoning_parser_cls is None and tool_parser_cls is None: return None - strict_level = ToolStrictLevel.from_name(tool_strict_level) + strict_level = ( + None + if tool_strict_level is None + else ToolStrictLevel.from_name(tool_strict_level) + ) if is_harmony: from vllm.parser.harmony import HarmonyParser diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index 345e5443ea..f7e04fd1c7 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -69,7 +69,7 @@ class OnlineDerenderer: exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, reasoning_parser: str | None = None, - tool_strict_level: str = "auto", + tool_strict_level: str | None = None, default_chat_template_kwargs: dict[str, Any] | None = None, log_error_stack: bool = False, ) -> None: diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index b4e3fc6c46..babd0ff6e5 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -165,7 +165,7 @@ class OnlineRenderer: exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, reasoning_parser: str | None = None, - tool_strict_level: str = "auto", + tool_strict_level: str | None = None, default_chat_template_kwargs: dict[str, Any] | None = None, log_error_stack: bool = False, ) -> None: diff --git a/vllm/tool_parsers/abstract_tool_parser.py b/vllm/tool_parsers/abstract_tool_parser.py index 3d3f6c1f09..642673507b 100644 --- a/vllm/tool_parsers/abstract_tool_parser.py +++ b/vllm/tool_parsers/abstract_tool_parser.py @@ -60,9 +60,9 @@ class ToolParser: # their parsed tool-call syntax matches a builtin xgrammar format. structural_tag_model: str | None = None # Strict level applied when the operator leaves ``--tool-strict-level`` - # at auto. Operator flags override it, and + # unset. Operator flags override it, and # ``VLLM_ENFORCE_STRICT_TOOL_CALLING=0`` still disables structural tags. - default_tool_strict_level: ToolStrictLevel | None = None + default_tool_strict_level: ToolStrictLevel = ToolStrictLevel.AUTO engine_based_streaming: bool = False def __init_subclass__(cls, **kwargs: Any) -> None: