[Frontend] Upgrade XGrammar to 0.2.7 and Rust structural tags to 0.3.0 (#57272)

Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-18 11:44:05 +00:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent cc09352e5c
commit c58532c86b
15 changed files with 57 additions and 303 deletions
+1 -1
View File
@@ -26,7 +26,7 @@ lm-format-enforcer == 0.11.3
llguidance >= 1.7.0, < 1.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "s390x"
outlines_core == 0.2.14
lark == 1.2.2
xgrammar >= 0.2.1, < 1.0.0; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le"
xgrammar == 0.2.7; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le"
typing_extensions >= 4.10
filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
partial-json-parser # used for parsing partial JSON outputs
+2 -2
View File
@@ -43,7 +43,7 @@ anyio==4.14.1
# sse-starlette
# starlette
# watchfiles
apache-tvm-ffi==0.1.9
apache-tvm-ffi==0.1.14.post0
# via xgrammar
arctic-inference==0.1.1
# via -r requirements/test/cuda.in
@@ -1298,7 +1298,7 @@ word2number==1.1
# via lm-eval
wrapt==1.17.2
# via smart-open
xgrammar==0.2.3
xgrammar==0.2.7
# via -r requirements/test/../common.txt
xxhash==3.5.0
# via
+1 -1
View File
@@ -1409,7 +1409,7 @@ word2number==1.1
# via lm-eval
wrapt==1.17.2
# via smart-open
xgrammar==0.2.3
xgrammar==0.2.7
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
+1 -1
View File
@@ -1360,7 +1360,7 @@ word2number==1.1
# via lm-eval
wrapt==2.1.2
# via smart-open
xgrammar==0.2.1
xgrammar==0.2.7
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
+1 -1
View File
@@ -1197,7 +1197,7 @@ werkzeug==3.1.8
# via schemathesis
word2number==1.1
# via lm-eval
xgrammar==0.2.3
xgrammar==0.2.7
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
+2 -2
View File
@@ -6596,9 +6596,9 @@ checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
[[package]]
name = "xgrammar-structural-tag"
version = "0.2.0+xgrammar.0.2.4.dd729e7"
version = "0.3.0+xgrammar.0.2.7.82505d0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4d24c842efc3c24e9756aa426d530cbdac0980e49af223cb384e276e981ca0a"
checksum = "3864005575c2ab76057f789898f59fb9fa1378d07c5ceabc24764e012d0e90bb"
dependencies = [
"auto_impl",
"serde",
+1 -1
View File
@@ -153,7 +153,7 @@ vllm-text = { path = "src/text" }
vllm-tokenizer = { path = "src/tokenizer" }
vllm-tracing = { path = "src/tracing" }
winnow = { version = "1.0.2", features = ["simd"] }
xgrammar-structural-tag = "0.2.0"
xgrammar-structural-tag = "0.3.0"
zeromq = { version = "0.6.0", default-features = false, features = [
"tokio-runtime",
"all-transport",
@@ -4,8 +4,6 @@
use super::{DeepSeekDsmlToolParser, DsmlTokens};
use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutput};
mod structural_tag;
/// Tool parser for DeepSeek V4.1's spaced DSML tags.
///
/// Arguments are emitted only after a full `invoke` block is parsed.
@@ -27,7 +25,7 @@ impl ToolParser for DeepSeekV41ToolParser {
}
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
Some(&structural_tag::DeepSeekV41StructuralTagBuilder)
Some(xgrammar_structural_tag::Model::DeepSeekV41.builder())
}
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
@@ -1,108 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Visible-text grammar for DeepSeek V4.1 tool calls.
//!
//! Adapted from xgrammar-structural-tag 0.2.0's DeepSeek V4 builder, with
//! V4.1's spaced DSML tags and schema-unconstrained parameter values.
use serde_json::Value;
use xgrammar_structural_tag::Result;
use xgrammar_structural_tag::builders::{
StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
};
use xgrammar_structural_tag::format::{
Format, JsonSchemaFormat, StructuralTag, TagFormat, TriggeredTagsFormat,
};
use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam};
use super::DsmlTokens;
const TOKENS: DsmlTokens = DsmlTokens::V41;
pub(super) struct DeepSeekV41StructuralTagBuilder;
impl StructuralTagBuilder for DeepSeekV41StructuralTagBuilder {
fn build(&self, ctx: StructuralTagContext<'_>) -> Result<StructuralTag> {
let tags = ctx
.function_tools
.iter()
.map(|tool| call_tag(tool, ctx.options))
.collect::<Vec<_>>();
let excludes = if ctx.options.exclude_special_tokens {
&["<think>", "</think>"][..]
} else {
&[]
};
let format = match ctx.tool_choice {
BuilderToolChoice::Auto if tags.is_empty() => Format::any_text_excluding(excludes),
BuilderToolChoice::Auto => Format::TriggeredTags(
TriggeredTagsFormat::new(
&[TOKENS.tool_calls_start],
vec![TagFormat::new(
format!("{}\n", TOKENS.tool_calls_start),
Format::tags_with_separator(tags, "", true, false),
TOKENS.tool_calls_end,
)],
)
.with_excludes(excludes),
),
BuilderToolChoice::Forced | BuilderToolChoice::Required => Format::sequence(vec![
Format::const_string(format!("\n\n{}\n", TOKENS.tool_calls_start)),
Format::tags_with_separator(
tags,
"",
true,
ctx.tool_choice == BuilderToolChoice::Forced,
),
Format::const_string(TOKENS.tool_calls_end),
]),
};
// Reasoning boundaries remain owned by the engine reasoner; serving
// calls this visible-text builder with reasoning=false.
Ok(StructuralTag::new(format))
}
}
fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat {
// TODO: enforce tool.parameters and strict schema semantics once V4.1
// argument conversion is available. This builder intentionally ignores
// parameters and strict: it constrains tool names and DSML/value syntax
// only. Parameter names, presence, uniqueness and value schemas remain
// unconstrained, including for strict=true. Serving still uses strict to
// decide whether auto tool choice activates a grammar.
TagFormat::new(
format!("{} name=\"{}\">\n", TOKENS.invoke_start, tool.function.name),
Format::star(parameter(options)),
format!("{}\n", TOKENS.invoke_end),
)
}
fn parameter(options: StructuralTagOptions) -> Format {
Format::tag(
format!("{} name=\"", TOKENS.parameter_start),
Format::sequence(vec![
Format::regex(r#"[^"]+"#),
Format::const_string("\" string=\""),
Format::or(vec![
Format::sequence(vec![
Format::const_string("true\">"),
Format::any_text_excluding(&[
TOKENS.parameter_end,
TOKENS.invoke_end,
TOKENS.tool_calls_end,
]),
]),
Format::sequence(vec![
Format::const_string("false\">"),
Format::JsonSchema(
JsonSchemaFormat::new(Value::Bool(true))
.with_any_order(options.any_order)
.with_max_whitespace_cnt(options.max_whitespace_cnt),
),
]),
]),
]),
format!("{}\n", TOKENS.parameter_end),
)
}
@@ -102,7 +102,7 @@ fn v4_tags_remain_text_in_v41_dialect() {
}
#[test]
fn structural_tag_ignores_parameter_schemas_and_strict() {
fn structural_tag_respects_parameter_schemas_and_strict() {
let parser = DeepSeekV41ToolParser::create(&tools()).unwrap();
let builder = parser.structural_tag_builder().unwrap();
let options = StructuralTagOptions::default().with_reasoning(false);
@@ -128,7 +128,13 @@ fn structural_tag_ignores_parameter_schemas_and_strict() {
] {
let mut function = FunctionDefinition::new("lookup").with_parameters(parameters);
function.strict = strict;
assert_eq!(build(function), expected);
let tag = build(function);
if strict == Some(false) {
assert_eq!(tag, expected);
} else {
assert_ne!(tag, expected);
}
assert!(tag.to_json_string().unwrap().contains("deepseek_v4_1_xml"));
}
}
}
@@ -219,7 +219,7 @@ mod tests {
ToolChoice::required(),
);
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>\n"},{"type":"tags_with_separator","tags":[{"begin":"<tool_call:opensource>get_weather<tool_sep:opensource>\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"days"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]}}]},"end":"</tool_call:opensource>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>\n"},{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<tool_call:opensource>get_weather<tool_sep:opensource>\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"days"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]}}]},"end":"</tool_call:opensource>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
}
#[test]
@@ -238,7 +238,7 @@ mod tests {
ToolChoice::required(),
);
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>"},{"type":"tags_with_separator","tags":[{"begin":"<tool_call:opensource>get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"}]}]},"end":"</tool_call:opensource>"}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>"},{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<tool_call:opensource>get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value:opensource>"}]}]},"end":"</tool_call:opensource>"}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
}
#[test]
@@ -259,7 +259,7 @@ mod tests {
);
let value = serde_json::to_value(tag).unwrap();
expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"limit"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq(
expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"limit"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq(
&serde_json::to_string(
&value["format"]["elements"][1]["tags"][0]["content"],
)
@@ -276,8 +276,8 @@ mod tests {
let auto = build("", &tools, ToolChoice::auto()).to_json_string().unwrap();
let forced = build("", &tools, ToolChoice::function("lookup")).to_json_string().unwrap();
expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":["<tool_calls>"],"tags":[{"begin":"<tool_calls>\n","content":{"type":"tags_with_separator","tags":[{"begin":"<tool_call>search<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"},{"begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n</tool_calls>"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto);
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls>\n"},{"type":"tag","begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"},{"type":"const_string","value":"\n</tool_calls>"}]}}"#]].assert_eq(&forced);
expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":["<tool_calls>"],"tags":[{"type":"tag","begin":"<tool_calls>\n","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<tool_call>search<tool_sep>\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":"</tool_call>"},{"type":"tag","begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":"</tool_call>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n</tool_calls>"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto);
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls>\n"},{"type":"tag","begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":"</tool_call>"},{"type":"const_string","value":"\n</tool_calls>"}]}}"#]].assert_eq(&forced);
}
#[test]
@@ -314,7 +314,7 @@ mod tests {
);
let value = serde_json::to_value(tag).unwrap();
expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"tenant"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq(
expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"tenant"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq(
&serde_json::to_string(
&value["format"]["elements"][1]["tags"][0]["content"],
)
@@ -6,14 +6,14 @@
use serde_json::{Map, Value};
use xgrammar_structural_tag::Result;
use xgrammar_structural_tag::builders::{
StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
ReasoningMode, StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
};
use xgrammar_structural_tag::format::{Format, JsonSchemaFormat, StructuralTag, TagFormat};
use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam, function_parameters};
use super::{
ARG_CLOSE, CALL_CLOSE, CLOSE, END_OF_MSG, JSON_CLOSE, JSON_OPEN, MESSAGE_CLOSE, OPEN,
RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, TOOLS_CLOSE, TOOLS_OPEN,
RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, THINK_OPEN, TOOLS_CLOSE, TOOLS_OPEN,
};
pub(super) static KIMI_K3_STRUCTURAL_TAG_BUILDER: KimiK3StructuralTagBuilder =
@@ -50,14 +50,23 @@ impl StructuralTagBuilder for KimiK3StructuralTagBuilder {
}
}
fn response_prefix(reasoning: bool) -> Vec<Format> {
fn response_prefix(reasoning: ReasoningMode) -> Vec<Format> {
let mut elements = Vec::new();
if reasoning {
elements.push(Format::tag(
"",
if reasoning != ReasoningMode::Disabled {
let prefix = Format::tag(
if reasoning == ReasoningMode::Auto {
THINK_OPEN
} else {
""
},
Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG]),
THINK_CLOSE,
));
);
elements.push(if reasoning == ReasoningMode::Auto {
Format::optional(prefix)
} else {
prefix
});
elements.push(Format::const_string(RESPONSE_OPEN));
} else {
elements.push(Format::optional(Format::const_string(RESPONSE_OPEN)));
@@ -386,7 +395,7 @@ mod tests {
)
.unwrap();
expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"]},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap());
expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"],"max_tokens":null,"max_chars":null},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap());
}
#[test]
@@ -6,7 +6,7 @@ from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from xgrammar import Grammar, StructuralTag, builtin_structural_tag
from xgrammar import Grammar, StructuralTag
from xgrammar.testing import _is_grammar_accept_string
from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -23,6 +23,7 @@ from vllm.tool_parsers.deepseekv31_tool_parser import DeepSeekV31ToolParser
from vllm.tool_parsers.deepseekv32_engine_tool_parser import (
DeepSeekV32EngineToolParser,
)
from vllm.tool_parsers.deepseekv41_engine_tool_parser import DeepSeekV41EngineToolParser
from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
@@ -83,51 +84,9 @@ def test_supported_structural_tag_models_include_vllm_builtins():
assert "hermes" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"])
def test_deepseek_v41_older_xgrammar_keeps_format_constraints(
choice,
sample_tools_strict,
monkeypatch,
):
monkeypatch.setattr(
builtin_structural_tag, "get_deepseek_v4_1_structural_tag", None, raising=False
)
tool_choice = (
ChatCompletionNamedToolChoiceParam(
function=ChatCompletionNamedFunction(name=choice)
)
if choice == "get_weather"
else choice
)
tag = get_model_structural_tag(
"deepseek_v41", sample_tools_strict, tool_choice, reasoning=False
)
grammar = Grammar.from_structural_tag(tag)
begin = '\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n'
end = "</|DSML| invoke>\n</|DSML| calls>"
parameter = (
'<|DSML| parameter name="undeclared" string="false">'
'{"nested": [true, null, 1.5]}</|DSML| parameter>\n'
)
# Required city is omitted; extra and repeated fields remain unconstrained.
assert _is_grammar_accept_string(grammar, begin + end)
assert _is_grammar_accept_string(grammar, begin + parameter * 2 + end)
assert not _is_grammar_accept_string(
grammar,
begin + parameter.replace('{"nested": [true, null, 1.5]}', "invalid") + end,
)
assert not _is_grammar_accept_string(
grammar, (begin + end).replace('name="get_weather"', 'name="unknown"')
)
assert not _is_grammar_accept_string(
grammar, begin + parameter.replace("| parameter", "|parameter") + end
)
assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto")
def test_deepseek_v41_named_choice_emits_one_call(sample_tools):
tag = get_model_structural_tag(
"deepseek_v41",
"deepseek_v4_1",
sample_tools,
ChatCompletionNamedToolChoiceParam(
function=ChatCompletionNamedFunction(name="get_weather")
@@ -147,19 +106,14 @@ def test_deepseek_v41_named_choice_emits_one_call(sample_tools):
grammar, "\n\n<|DSML| calls>\n" + call * 2 + "</|DSML| calls>"
)
assert (
get_model_structural_tag("deepseek_v41", sample_tools, "auto", reasoning=False)
get_model_structural_tag("deepseek_v4_1", sample_tools, "auto", reasoning=False)
is None
)
@pytest.mark.skipif(
not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"),
reason="Requires XGrammar's DeepSeek V4.1 builtin",
)
@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"])
@pytest.mark.parametrize("reasoning", [False, True])
def test_deepseek_v41_constrains_parameters_after_reasoning(
sample_tools_strict, choice, reasoning
sample_tools_strict, choice
):
"""Strict calls must enforce the schema without requiring another think close."""
tool_choice = (
@@ -171,7 +125,7 @@ def test_deepseek_v41_constrains_parameters_after_reasoning(
)
sample_tools_strict[0].function.parameters["additionalProperties"] = False
tag = get_model_structural_tag(
"deepseek_v41", sample_tools_strict, tool_choice, reasoning=reasoning
"deepseek_v4_1", sample_tools_strict, tool_choice, reasoning=False
)
grammar = Grammar.from_structural_tag(tag)
begin = '\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n'
@@ -195,14 +149,10 @@ def test_deepseek_v41_constrains_parameters_after_reasoning(
assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto")
@pytest.mark.skipif(
not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"),
reason="Requires XGrammar's DeepSeek V4.1 builtin",
)
def test_deepseek_v41_non_strict_parallel_calls_keep_typed_dsml(sample_tools):
sample_tools[0].function.strict = False
tag = get_model_structural_tag(
"deepseek_v41", sample_tools, "required", reasoning=False
"deepseek_v4_1", sample_tools, "required", reasoning=False
)
grammar = Grammar.from_structural_tag(tag)
call = (
@@ -571,6 +521,7 @@ def test_get_model_structural_tag_supports_named_tool_choice(
(DeepSeekV31ToolParser, "deepseek_v3_1"),
(DeepSeekV32EngineToolParser, "deepseek_v3_2"),
(DeepSeekV4EngineToolParser, "deepseek_v4"),
(DeepSeekV41EngineToolParser, "deepseek_v4_1"),
(Glm47MoeModelToolParser, "glm_4_7"),
(Hermes2ProToolParser, "hermes"),
(KimiK2ToolParser, "kimi"),
@@ -638,7 +589,11 @@ def test_get_structural_tag_disables_reasoning(
assert captured == [False]
@pytest.mark.parametrize(
"parser_cls", [Qwen3EngineToolParser, DeepSeekV41EngineToolParser]
)
def test_unified_parser_get_structural_tag_disables_reasoning(
parser_cls,
monkeypatch: pytest.MonkeyPatch,
sample_tools_strict: list[ChatCompletionToolsParam],
):
@@ -654,7 +609,7 @@ def test_unified_parser_get_structural_tag_disables_reasoning(
)
class TestParser(DelegatingParser):
tool_parser_cls = Qwen3EngineToolParser
tool_parser_cls = parser_cls
request = ChatCompletionRequest(
messages=[],
@@ -5,4 +5,4 @@ from vllm.parser.engine.registered_adapters import DeepSeekV41ParserToolAdapter
class DeepSeekV41EngineToolParser(DeepSeekV41ParserToolAdapter): # type: ignore[valid-type, misc]
structural_tag_model = "deepseek_v41"
structural_tag_model = "deepseek_v4_1"
+2 -108
View File
@@ -10,7 +10,6 @@ from openai.types.responses.tool import Tool as ResponsesTool
from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed
from openai.types.responses.tool_choice_function import ToolChoiceFunction
from xgrammar import StructuralTag, normalize_tool_choice
from xgrammar import builtin_structural_tag as xgrammar_builtin_structural_tag
from xgrammar import get_model_structural_tag as get_xgrammar_model_structural_tag
from xgrammar.openai_tool_call_schema import (
BuiltinToolParam,
@@ -71,11 +70,10 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
"deepseek_v3_2",
"glm_4_7",
"deepseek_v4",
"deepseek_v4_1",
}
)
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
{"deepseek_v41", "hermes", "hy_v4", "kimi_k3"}
)
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "hy_v4", "kimi_k3"})
SUPPORTED_STRUCTURAL_TAG_MODELS = (
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
)
@@ -270,110 +268,6 @@ def get_function_parameters(function) -> dict[str, Any] | bool:
return function.parameters if function.parameters is not None else True
_V41_CALLS_START = "<|DSML| calls>"
_V41_CALLS_END = "</|DSML| calls>"
_V41_INVOKE_END = "</|DSML| invoke>"
_V41_PARAMETER_END = "</|DSML| parameter>"
@register_vllm_structural_tag("deepseek_v41")
def get_deepseek_v41_structural_tag(
tools: list[FunctionToolParam],
builtin_tools: list[BuiltinToolParam],
tool_choice: SimplifiedToolChoice,
reasoning: bool,
token_suffix: str = "",
) -> StructuralTag:
# Serving enables this visible-text grammar after the reasoning boundary.
builder = getattr(
xgrammar_builtin_structural_tag, "get_deepseek_v4_1_structural_tag", None
)
if builder is not None:
return builder(
tools=tools,
builtin_tools=builtin_tools,
tool_choice=tool_choice,
reasoning="disabled",
)
del builtin_tools, reasoning, token_suffix
# Compatibility with xgrammar releases without the V4.1 builtin. This constrains
# tool names and DSML/value syntax only. Parameter names, presence, uniqueness
# and value schemas remain unconstrained, including for strict=true. The request
# layer still uses strict to decide whether auto tool choice activates a grammar.
parameter = TagFormat(
begin='<|DSML| parameter name="',
content=SequenceFormat(
elements=[
RegexFormat(pattern=r'[^"]+'),
ConstStringFormat(value='" string="'),
OrFormat(
elements=[
SequenceFormat(
elements=[
ConstStringFormat(value='true">'),
AnyTextFormat(
excludes=[
_V41_PARAMETER_END,
_V41_INVOKE_END,
_V41_CALLS_END,
]
),
]
),
SequenceFormat(
elements=[
ConstStringFormat(value='false">'),
JSONSchemaFormat(json_schema=True),
]
),
]
),
]
),
end=f"{_V41_PARAMETER_END}\n",
)
calls = TagsWithSeparatorFormat(
tags=[
TagFormat(
begin=f'<|DSML| invoke name="{tool.function.name}">\n',
content=StarFormat(content=parameter),
end=f"{_V41_INVOKE_END}\n",
)
for tool in tools
],
separator="",
at_least_one=True,
stop_after_first=tool_choice == "forced",
)
if tool_choice == "auto":
return StructuralTag(
format=TriggeredTagsFormat(
triggers=[_V41_CALLS_START],
tags=[
TagFormat(
begin=f"{_V41_CALLS_START}\n",
content=calls,
end=_V41_CALLS_END,
)
],
excludes=["<think>", "</think>"],
)
if tools
else AnyTextFormat(excludes=["<think>", "</think>"])
)
return StructuralTag(
format=SequenceFormat(
elements=[
ConstStringFormat(value=f"\n\n{_V41_CALLS_START}\n"),
calls,
ConstStringFormat(value=_V41_CALLS_END),
]
)
)
def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
arguments_field_prefix = '", "arguments": '
formats = [