From c58532c86b8ab821fc1d819636480b7588bce7b5 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Fri, 18 Sep 2026 19:44:05 +0800 Subject: [PATCH] [Frontend] Upgrade XGrammar to 0.2.7 and Rust structural tags to 0.3.0 (#57272) Co-authored-by: Codex Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao --- requirements/common.txt | 2 +- requirements/test/cpu.txt | 4 +- requirements/test/cuda.txt | 2 +- requirements/test/rocm.txt | 2 +- requirements/test/xpu.txt | 2 +- rust/Cargo.lock | 4 +- rust/Cargo.toml | 2 +- .../src/tool/deepseek_dsml/deepseek_v41.rs | 4 +- .../deepseek_v41/structural_tag.rs | 108 ----------------- .../tool/deepseek_dsml/deepseek_v41/tests.rs | 10 +- rust/src/parser/src/tool/hy/structural_tag.rs | 12 +- .../src/unified/kimi_k3/structural_tag.rs | 25 ++-- .../test_structural_tag_registry.py | 71 +++-------- .../deepseekv41_engine_tool_parser.py | 2 +- vllm/tool_parsers/structural_tag_registry.py | 110 +----------------- 15 files changed, 57 insertions(+), 303 deletions(-) delete mode 100644 rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs diff --git a/requirements/common.txt b/requirements/common.txt index ff691ebe47..b55f680b76 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -26,7 +26,7 @@ lm-format-enforcer == 0.11.3 llguidance >= 1.7.0, < 1.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "s390x" outlines_core == 0.2.14 lark == 1.2.2 -xgrammar >= 0.2.1, < 1.0.0; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le" +xgrammar == 0.2.7; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le" typing_extensions >= 4.10 filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317 partial-json-parser # used for parsing partial JSON outputs diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index 996caab96a..c1d4913953 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -43,7 +43,7 @@ anyio==4.14.1 # sse-starlette # starlette # watchfiles -apache-tvm-ffi==0.1.9 +apache-tvm-ffi==0.1.14.post0 # via xgrammar arctic-inference==0.1.1 # via -r requirements/test/cuda.in @@ -1298,7 +1298,7 @@ word2number==1.1 # via lm-eval wrapt==1.17.2 # via smart-open -xgrammar==0.2.3 +xgrammar==0.2.7 # via -r requirements/test/../common.txt xxhash==3.5.0 # via diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index ea94558000..3207d395aa 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -1409,7 +1409,7 @@ word2number==1.1 # via lm-eval wrapt==1.17.2 # via smart-open -xgrammar==0.2.3 +xgrammar==0.2.7 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 631c3952fd..c961b6294a 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -1360,7 +1360,7 @@ word2number==1.1 # via lm-eval wrapt==2.1.2 # via smart-open -xgrammar==0.2.1 +xgrammar==0.2.7 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index 4e4fd628b6..d9572389eb 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -1197,7 +1197,7 @@ werkzeug==3.1.8 # via schemathesis word2number==1.1 # via lm-eval -xgrammar==0.2.3 +xgrammar==0.2.7 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/rust/Cargo.lock b/rust/Cargo.lock index dee3f30789..b4917a303b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -6596,9 +6596,9 @@ checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "xgrammar-structural-tag" -version = "0.2.0+xgrammar.0.2.4.dd729e7" +version = "0.3.0+xgrammar.0.2.7.82505d0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d24c842efc3c24e9756aa426d530cbdac0980e49af223cb384e276e981ca0a" +checksum = "3864005575c2ab76057f789898f59fb9fa1378d07c5ceabc24764e012d0e90bb" dependencies = [ "auto_impl", "serde", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index dde065e790..ea8b813619 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -153,7 +153,7 @@ vllm-text = { path = "src/text" } vllm-tokenizer = { path = "src/tokenizer" } vllm-tracing = { path = "src/tracing" } winnow = { version = "1.0.2", features = ["simd"] } -xgrammar-structural-tag = "0.2.0" +xgrammar-structural-tag = "0.3.0" zeromq = { version = "0.6.0", default-features = false, features = [ "tokio-runtime", "all-transport", diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs index 44658b1d44..be68b6d2e5 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs @@ -4,8 +4,6 @@ use super::{DeepSeekDsmlToolParser, DsmlTokens}; use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutput}; -mod structural_tag; - /// Tool parser for DeepSeek V4.1's spaced DSML tags. /// /// Arguments are emitted only after a full `invoke` block is parsed. @@ -27,7 +25,7 @@ impl ToolParser for DeepSeekV41ToolParser { } fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> { - Some(&structural_tag::DeepSeekV41StructuralTagBuilder) + Some(xgrammar_structural_tag::Model::DeepSeekV41.builder()) } fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs deleted file mode 100644 index d5fb6118fb..0000000000 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -//! Visible-text grammar for DeepSeek V4.1 tool calls. -//! -//! Adapted from xgrammar-structural-tag 0.2.0's DeepSeek V4 builder, with -//! V4.1's spaced DSML tags and schema-unconstrained parameter values. - -use serde_json::Value; -use xgrammar_structural_tag::Result; -use xgrammar_structural_tag::builders::{ - StructuralTagBuilder, StructuralTagContext, StructuralTagOptions, -}; -use xgrammar_structural_tag::format::{ - Format, JsonSchemaFormat, StructuralTag, TagFormat, TriggeredTagsFormat, -}; -use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam}; - -use super::DsmlTokens; - -const TOKENS: DsmlTokens = DsmlTokens::V41; - -pub(super) struct DeepSeekV41StructuralTagBuilder; - -impl StructuralTagBuilder for DeepSeekV41StructuralTagBuilder { - fn build(&self, ctx: StructuralTagContext<'_>) -> Result { - let tags = ctx - .function_tools - .iter() - .map(|tool| call_tag(tool, ctx.options)) - .collect::>(); - let excludes = if ctx.options.exclude_special_tokens { - &["", ""][..] - } else { - &[] - }; - let format = match ctx.tool_choice { - BuilderToolChoice::Auto if tags.is_empty() => Format::any_text_excluding(excludes), - BuilderToolChoice::Auto => Format::TriggeredTags( - TriggeredTagsFormat::new( - &[TOKENS.tool_calls_start], - vec![TagFormat::new( - format!("{}\n", TOKENS.tool_calls_start), - Format::tags_with_separator(tags, "", true, false), - TOKENS.tool_calls_end, - )], - ) - .with_excludes(excludes), - ), - BuilderToolChoice::Forced | BuilderToolChoice::Required => Format::sequence(vec![ - Format::const_string(format!("\n\n{}\n", TOKENS.tool_calls_start)), - Format::tags_with_separator( - tags, - "", - true, - ctx.tool_choice == BuilderToolChoice::Forced, - ), - Format::const_string(TOKENS.tool_calls_end), - ]), - }; - // Reasoning boundaries remain owned by the engine reasoner; serving - // calls this visible-text builder with reasoning=false. - Ok(StructuralTag::new(format)) - } -} - -fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat { - // TODO: enforce tool.parameters and strict schema semantics once V4.1 - // argument conversion is available. This builder intentionally ignores - // parameters and strict: it constrains tool names and DSML/value syntax - // only. Parameter names, presence, uniqueness and value schemas remain - // unconstrained, including for strict=true. Serving still uses strict to - // decide whether auto tool choice activates a grammar. - TagFormat::new( - format!("{} name=\"{}\">\n", TOKENS.invoke_start, tool.function.name), - Format::star(parameter(options)), - format!("{}\n", TOKENS.invoke_end), - ) -} - -fn parameter(options: StructuralTagOptions) -> Format { - Format::tag( - format!("{} name=\"", TOKENS.parameter_start), - Format::sequence(vec![ - Format::regex(r#"[^"]+"#), - Format::const_string("\" string=\""), - Format::or(vec![ - Format::sequence(vec![ - Format::const_string("true\">"), - Format::any_text_excluding(&[ - TOKENS.parameter_end, - TOKENS.invoke_end, - TOKENS.tool_calls_end, - ]), - ]), - Format::sequence(vec![ - Format::const_string("false\">"), - Format::JsonSchema( - JsonSchemaFormat::new(Value::Bool(true)) - .with_any_order(options.any_order) - .with_max_whitespace_cnt(options.max_whitespace_cnt), - ), - ]), - ]), - ]), - format!("{}\n", TOKENS.parameter_end), - ) -} diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs index 5a6758f5c9..208d88ea33 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs @@ -102,7 +102,7 @@ fn v4_tags_remain_text_in_v41_dialect() { } #[test] -fn structural_tag_ignores_parameter_schemas_and_strict() { +fn structural_tag_respects_parameter_schemas_and_strict() { let parser = DeepSeekV41ToolParser::create(&tools()).unwrap(); let builder = parser.structural_tag_builder().unwrap(); let options = StructuralTagOptions::default().with_reasoning(false); @@ -128,7 +128,13 @@ fn structural_tag_ignores_parameter_schemas_and_strict() { ] { let mut function = FunctionDefinition::new("lookup").with_parameters(parameters); function.strict = strict; - assert_eq!(build(function), expected); + let tag = build(function); + if strict == Some(false) { + assert_eq!(tag, expected); + } else { + assert_ne!(tag, expected); + } + assert!(tag.to_json_string().unwrap().contains("deepseek_v4_1_xml")); } } } diff --git a/rust/src/parser/src/tool/hy/structural_tag.rs b/rust/src/parser/src/tool/hy/structural_tag.rs index cce98d5b09..e883acbafd 100644 --- a/rust/src/parser/src/tool/hy/structural_tag.rs +++ b/rust/src/parser/src/tool/hy/structural_tag.rs @@ -219,7 +219,7 @@ mod tests { ToolChoice::required(), ); - expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tags_with_separator","tags":[{"begin":"get_weather\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"days"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}}]},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); + expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tags_with_separator","tags":[{"type":"tag","begin":"get_weather\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"days"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}}]},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); } #[test] @@ -238,7 +238,7 @@ mod tests { ToolChoice::required(), ); - expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"tags_with_separator","tags":[{"begin":"get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","",""]},{"type":"const_string","value":""}]}]},"end":""}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":""}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); + expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"tags_with_separator","tags":[{"type":"tag","begin":"get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""}]}]},"end":""}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":""}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); } #[test] @@ -259,7 +259,7 @@ mod tests { ); let value = serde_json::to_value(tag).unwrap(); - expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"limit"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq( + expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"limit"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq( &serde_json::to_string( &value["format"]["elements"][1]["tags"][0]["content"], ) @@ -276,8 +276,8 @@ mod tests { let auto = build("", &tools, ToolChoice::auto()).to_json_string().unwrap(); let forced = build("", &tools, ToolChoice::function("lookup")).to_json_string().unwrap(); - expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":[""],"tags":[{"begin":"\n","content":{"type":"tags_with_separator","tags":[{"begin":"search\n","content":{"type":"any_text","excludes":[]},"end":""},{"begin":"lookup\n","content":{"type":"any_text","excludes":[]},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto); - expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tag","begin":"lookup\n","content":{"type":"any_text","excludes":[]},"end":""},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&forced); + expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":[""],"tags":[{"type":"tag","begin":"\n","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"search\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":""},{"type":"tag","begin":"lookup\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto); + expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tag","begin":"lookup\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":""},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&forced); } #[test] @@ -314,7 +314,7 @@ mod tests { ); let value = serde_json::to_value(tag).unwrap(); - expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"tenant"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq( + expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"tenant"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq( &serde_json::to_string( &value["format"]["elements"][1]["tags"][0]["content"], ) diff --git a/rust/src/parser/src/unified/kimi_k3/structural_tag.rs b/rust/src/parser/src/unified/kimi_k3/structural_tag.rs index 0a86a274e1..fc30f455e9 100644 --- a/rust/src/parser/src/unified/kimi_k3/structural_tag.rs +++ b/rust/src/parser/src/unified/kimi_k3/structural_tag.rs @@ -6,14 +6,14 @@ use serde_json::{Map, Value}; use xgrammar_structural_tag::Result; use xgrammar_structural_tag::builders::{ - StructuralTagBuilder, StructuralTagContext, StructuralTagOptions, + ReasoningMode, StructuralTagBuilder, StructuralTagContext, StructuralTagOptions, }; use xgrammar_structural_tag::format::{Format, JsonSchemaFormat, StructuralTag, TagFormat}; use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam, function_parameters}; use super::{ ARG_CLOSE, CALL_CLOSE, CLOSE, END_OF_MSG, JSON_CLOSE, JSON_OPEN, MESSAGE_CLOSE, OPEN, - RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, TOOLS_CLOSE, TOOLS_OPEN, + RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, THINK_OPEN, TOOLS_CLOSE, TOOLS_OPEN, }; pub(super) static KIMI_K3_STRUCTURAL_TAG_BUILDER: KimiK3StructuralTagBuilder = @@ -50,14 +50,23 @@ impl StructuralTagBuilder for KimiK3StructuralTagBuilder { } } -fn response_prefix(reasoning: bool) -> Vec { +fn response_prefix(reasoning: ReasoningMode) -> Vec { let mut elements = Vec::new(); - if reasoning { - elements.push(Format::tag( - "", + if reasoning != ReasoningMode::Disabled { + let prefix = Format::tag( + if reasoning == ReasoningMode::Auto { + THINK_OPEN + } else { + "" + }, Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG]), THINK_CLOSE, - )); + ); + elements.push(if reasoning == ReasoningMode::Auto { + Format::optional(prefix) + } else { + prefix + }); elements.push(Format::const_string(RESPONSE_OPEN)); } else { elements.push(Format::optional(Format::const_string(RESPONSE_OPEN))); @@ -386,7 +395,7 @@ mod tests { ) .unwrap(); - expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"]},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap()); + expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"],"max_tokens":null,"max_chars":null},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap()); } #[test] diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index c216b398cc..3221d1fb2c 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -6,7 +6,7 @@ from types import SimpleNamespace from unittest.mock import MagicMock import pytest -from xgrammar import Grammar, StructuralTag, builtin_structural_tag +from xgrammar import Grammar, StructuralTag from xgrammar.testing import _is_grammar_accept_string from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -23,6 +23,7 @@ from vllm.tool_parsers.deepseekv31_tool_parser import DeepSeekV31ToolParser from vllm.tool_parsers.deepseekv32_engine_tool_parser import ( DeepSeekV32EngineToolParser, ) +from vllm.tool_parsers.deepseekv41_engine_tool_parser import DeepSeekV41EngineToolParser from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser @@ -83,51 +84,9 @@ def test_supported_structural_tag_models_include_vllm_builtins(): assert "hermes" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS -@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"]) -def test_deepseek_v41_older_xgrammar_keeps_format_constraints( - choice, - sample_tools_strict, - monkeypatch, -): - monkeypatch.setattr( - builtin_structural_tag, "get_deepseek_v4_1_structural_tag", None, raising=False - ) - tool_choice = ( - ChatCompletionNamedToolChoiceParam( - function=ChatCompletionNamedFunction(name=choice) - ) - if choice == "get_weather" - else choice - ) - tag = get_model_structural_tag( - "deepseek_v41", sample_tools_strict, tool_choice, reasoning=False - ) - grammar = Grammar.from_structural_tag(tag) - begin = '\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n' - end = "\n" - parameter = ( - '<|DSML| parameter name="undeclared" string="false">' - '{"nested": [true, null, 1.5]}\n' - ) - # Required city is omitted; extra and repeated fields remain unconstrained. - assert _is_grammar_accept_string(grammar, begin + end) - assert _is_grammar_accept_string(grammar, begin + parameter * 2 + end) - assert not _is_grammar_accept_string( - grammar, - begin + parameter.replace('{"nested": [true, null, 1.5]}', "invalid") + end, - ) - assert not _is_grammar_accept_string( - grammar, (begin + end).replace('name="get_weather"', 'name="unknown"') - ) - assert not _is_grammar_accept_string( - grammar, begin + parameter.replace("| parameter", "|parameter") + end - ) - assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto") - - def test_deepseek_v41_named_choice_emits_one_call(sample_tools): tag = get_model_structural_tag( - "deepseek_v41", + "deepseek_v4_1", sample_tools, ChatCompletionNamedToolChoiceParam( function=ChatCompletionNamedFunction(name="get_weather") @@ -147,19 +106,14 @@ def test_deepseek_v41_named_choice_emits_one_call(sample_tools): grammar, "\n\n<|DSML| calls>\n" + call * 2 + "" ) assert ( - get_model_structural_tag("deepseek_v41", sample_tools, "auto", reasoning=False) + get_model_structural_tag("deepseek_v4_1", sample_tools, "auto", reasoning=False) is None ) -@pytest.mark.skipif( - not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"), - reason="Requires XGrammar's DeepSeek V4.1 builtin", -) @pytest.mark.parametrize("choice", ["auto", "required", "get_weather"]) -@pytest.mark.parametrize("reasoning", [False, True]) def test_deepseek_v41_constrains_parameters_after_reasoning( - sample_tools_strict, choice, reasoning + sample_tools_strict, choice ): """Strict calls must enforce the schema without requiring another think close.""" tool_choice = ( @@ -171,7 +125,7 @@ def test_deepseek_v41_constrains_parameters_after_reasoning( ) sample_tools_strict[0].function.parameters["additionalProperties"] = False tag = get_model_structural_tag( - "deepseek_v41", sample_tools_strict, tool_choice, reasoning=reasoning + "deepseek_v4_1", sample_tools_strict, tool_choice, reasoning=False ) grammar = Grammar.from_structural_tag(tag) begin = '\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n' @@ -195,14 +149,10 @@ def test_deepseek_v41_constrains_parameters_after_reasoning( assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto") -@pytest.mark.skipif( - not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"), - reason="Requires XGrammar's DeepSeek V4.1 builtin", -) def test_deepseek_v41_non_strict_parallel_calls_keep_typed_dsml(sample_tools): sample_tools[0].function.strict = False tag = get_model_structural_tag( - "deepseek_v41", sample_tools, "required", reasoning=False + "deepseek_v4_1", sample_tools, "required", reasoning=False ) grammar = Grammar.from_structural_tag(tag) call = ( @@ -571,6 +521,7 @@ def test_get_model_structural_tag_supports_named_tool_choice( (DeepSeekV31ToolParser, "deepseek_v3_1"), (DeepSeekV32EngineToolParser, "deepseek_v3_2"), (DeepSeekV4EngineToolParser, "deepseek_v4"), + (DeepSeekV41EngineToolParser, "deepseek_v4_1"), (Glm47MoeModelToolParser, "glm_4_7"), (Hermes2ProToolParser, "hermes"), (KimiK2ToolParser, "kimi"), @@ -638,7 +589,11 @@ def test_get_structural_tag_disables_reasoning( assert captured == [False] +@pytest.mark.parametrize( + "parser_cls", [Qwen3EngineToolParser, DeepSeekV41EngineToolParser] +) def test_unified_parser_get_structural_tag_disables_reasoning( + parser_cls, monkeypatch: pytest.MonkeyPatch, sample_tools_strict: list[ChatCompletionToolsParam], ): @@ -654,7 +609,7 @@ def test_unified_parser_get_structural_tag_disables_reasoning( ) class TestParser(DelegatingParser): - tool_parser_cls = Qwen3EngineToolParser + tool_parser_cls = parser_cls request = ChatCompletionRequest( messages=[], diff --git a/vllm/tool_parsers/deepseekv41_engine_tool_parser.py b/vllm/tool_parsers/deepseekv41_engine_tool_parser.py index a4c699aeab..a94883d06a 100644 --- a/vllm/tool_parsers/deepseekv41_engine_tool_parser.py +++ b/vllm/tool_parsers/deepseekv41_engine_tool_parser.py @@ -5,4 +5,4 @@ from vllm.parser.engine.registered_adapters import DeepSeekV41ParserToolAdapter class DeepSeekV41EngineToolParser(DeepSeekV41ParserToolAdapter): # type: ignore[valid-type, misc] - structural_tag_model = "deepseek_v41" + structural_tag_model = "deepseek_v4_1" diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index 5d65b186fe..ba86395b7a 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -10,7 +10,6 @@ from openai.types.responses.tool import Tool as ResponsesTool from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed from openai.types.responses.tool_choice_function import ToolChoiceFunction from xgrammar import StructuralTag, normalize_tool_choice -from xgrammar import builtin_structural_tag as xgrammar_builtin_structural_tag from xgrammar import get_model_structural_tag as get_xgrammar_model_structural_tag from xgrammar.openai_tool_call_schema import ( BuiltinToolParam, @@ -71,11 +70,10 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset( "deepseek_v3_2", "glm_4_7", "deepseek_v4", + "deepseek_v4_1", } ) -VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset( - {"deepseek_v41", "hermes", "hy_v4", "kimi_k3"} -) +VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "hy_v4", "kimi_k3"}) SUPPORTED_STRUCTURAL_TAG_MODELS = ( XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS ) @@ -270,110 +268,6 @@ def get_function_parameters(function) -> dict[str, Any] | bool: return function.parameters if function.parameters is not None else True -_V41_CALLS_START = "<|DSML| calls>" -_V41_CALLS_END = "" -_V41_INVOKE_END = "" -_V41_PARAMETER_END = "" - - -@register_vllm_structural_tag("deepseek_v41") -def get_deepseek_v41_structural_tag( - tools: list[FunctionToolParam], - builtin_tools: list[BuiltinToolParam], - tool_choice: SimplifiedToolChoice, - reasoning: bool, - token_suffix: str = "", -) -> StructuralTag: - # Serving enables this visible-text grammar after the reasoning boundary. - builder = getattr( - xgrammar_builtin_structural_tag, "get_deepseek_v4_1_structural_tag", None - ) - if builder is not None: - return builder( - tools=tools, - builtin_tools=builtin_tools, - tool_choice=tool_choice, - reasoning="disabled", - ) - - del builtin_tools, reasoning, token_suffix - - # Compatibility with xgrammar releases without the V4.1 builtin. This constrains - # tool names and DSML/value syntax only. Parameter names, presence, uniqueness - # and value schemas remain unconstrained, including for strict=true. The request - # layer still uses strict to decide whether auto tool choice activates a grammar. - parameter = TagFormat( - begin='<|DSML| parameter name="', - content=SequenceFormat( - elements=[ - RegexFormat(pattern=r'[^"]+'), - ConstStringFormat(value='" string="'), - OrFormat( - elements=[ - SequenceFormat( - elements=[ - ConstStringFormat(value='true">'), - AnyTextFormat( - excludes=[ - _V41_PARAMETER_END, - _V41_INVOKE_END, - _V41_CALLS_END, - ] - ), - ] - ), - SequenceFormat( - elements=[ - ConstStringFormat(value='false">'), - JSONSchemaFormat(json_schema=True), - ] - ), - ] - ), - ] - ), - end=f"{_V41_PARAMETER_END}\n", - ) - calls = TagsWithSeparatorFormat( - tags=[ - TagFormat( - begin=f'<|DSML| invoke name="{tool.function.name}">\n', - content=StarFormat(content=parameter), - end=f"{_V41_INVOKE_END}\n", - ) - for tool in tools - ], - separator="", - at_least_one=True, - stop_after_first=tool_choice == "forced", - ) - if tool_choice == "auto": - return StructuralTag( - format=TriggeredTagsFormat( - triggers=[_V41_CALLS_START], - tags=[ - TagFormat( - begin=f"{_V41_CALLS_START}\n", - content=calls, - end=_V41_CALLS_END, - ) - ], - excludes=["", ""], - ) - if tools - else AnyTextFormat(excludes=["", ""]) - ) - return StructuralTag( - format=SequenceFormat( - elements=[ - ConstStringFormat(value=f"\n\n{_V41_CALLS_START}\n"), - calls, - ConstStringFormat(value=_V41_CALLS_END), - ] - ) - ) - - def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]: arguments_field_prefix = '", "arguments": ' formats = [