[Frontend] Upgrade XGrammar to 0.2.7 and Rust structural tags to 0.3.0 (#57272)
Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
cc09352e5c
commit
c58532c86b
@@ -26,7 +26,7 @@ lm-format-enforcer == 0.11.3
|
||||
llguidance >= 1.7.0, < 1.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "s390x"
|
||||
outlines_core == 0.2.14
|
||||
lark == 1.2.2
|
||||
xgrammar >= 0.2.1, < 1.0.0; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le"
|
||||
xgrammar == 0.2.7; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le"
|
||||
typing_extensions >= 4.10
|
||||
filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
|
||||
partial-json-parser # used for parsing partial JSON outputs
|
||||
|
||||
@@ -43,7 +43,7 @@ anyio==4.14.1
|
||||
# sse-starlette
|
||||
# starlette
|
||||
# watchfiles
|
||||
apache-tvm-ffi==0.1.9
|
||||
apache-tvm-ffi==0.1.14.post0
|
||||
# via xgrammar
|
||||
arctic-inference==0.1.1
|
||||
# via -r requirements/test/cuda.in
|
||||
@@ -1298,7 +1298,7 @@ word2number==1.1
|
||||
# via lm-eval
|
||||
wrapt==1.17.2
|
||||
# via smart-open
|
||||
xgrammar==0.2.3
|
||||
xgrammar==0.2.7
|
||||
# via -r requirements/test/../common.txt
|
||||
xxhash==3.5.0
|
||||
# via
|
||||
|
||||
@@ -1409,7 +1409,7 @@ word2number==1.1
|
||||
# via lm-eval
|
||||
wrapt==1.17.2
|
||||
# via smart-open
|
||||
xgrammar==0.2.3
|
||||
xgrammar==0.2.7
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
|
||||
@@ -1360,7 +1360,7 @@ word2number==1.1
|
||||
# via lm-eval
|
||||
wrapt==2.1.2
|
||||
# via smart-open
|
||||
xgrammar==0.2.1
|
||||
xgrammar==0.2.7
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
|
||||
@@ -1197,7 +1197,7 @@ werkzeug==3.1.8
|
||||
# via schemathesis
|
||||
word2number==1.1
|
||||
# via lm-eval
|
||||
xgrammar==0.2.3
|
||||
xgrammar==0.2.7
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
|
||||
Generated
+2
-2
@@ -6596,9 +6596,9 @@ checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
|
||||
|
||||
[[package]]
|
||||
name = "xgrammar-structural-tag"
|
||||
version = "0.2.0+xgrammar.0.2.4.dd729e7"
|
||||
version = "0.3.0+xgrammar.0.2.7.82505d0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4d24c842efc3c24e9756aa426d530cbdac0980e49af223cb384e276e981ca0a"
|
||||
checksum = "3864005575c2ab76057f789898f59fb9fa1378d07c5ceabc24764e012d0e90bb"
|
||||
dependencies = [
|
||||
"auto_impl",
|
||||
"serde",
|
||||
|
||||
+1
-1
@@ -153,7 +153,7 @@ vllm-text = { path = "src/text" }
|
||||
vllm-tokenizer = { path = "src/tokenizer" }
|
||||
vllm-tracing = { path = "src/tracing" }
|
||||
winnow = { version = "1.0.2", features = ["simd"] }
|
||||
xgrammar-structural-tag = "0.2.0"
|
||||
xgrammar-structural-tag = "0.3.0"
|
||||
zeromq = { version = "0.6.0", default-features = false, features = [
|
||||
"tokio-runtime",
|
||||
"all-transport",
|
||||
|
||||
@@ -4,8 +4,6 @@
|
||||
use super::{DeepSeekDsmlToolParser, DsmlTokens};
|
||||
use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutput};
|
||||
|
||||
mod structural_tag;
|
||||
|
||||
/// Tool parser for DeepSeek V4.1's spaced DSML tags.
|
||||
///
|
||||
/// Arguments are emitted only after a full `invoke` block is parsed.
|
||||
@@ -27,7 +25,7 @@ impl ToolParser for DeepSeekV41ToolParser {
|
||||
}
|
||||
|
||||
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
|
||||
Some(&structural_tag::DeepSeekV41StructuralTagBuilder)
|
||||
Some(xgrammar_structural_tag::Model::DeepSeekV41.builder())
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
||||
|
||||
@@ -1,108 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Visible-text grammar for DeepSeek V4.1 tool calls.
|
||||
//!
|
||||
//! Adapted from xgrammar-structural-tag 0.2.0's DeepSeek V4 builder, with
|
||||
//! V4.1's spaced DSML tags and schema-unconstrained parameter values.
|
||||
|
||||
use serde_json::Value;
|
||||
use xgrammar_structural_tag::Result;
|
||||
use xgrammar_structural_tag::builders::{
|
||||
StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
|
||||
};
|
||||
use xgrammar_structural_tag::format::{
|
||||
Format, JsonSchemaFormat, StructuralTag, TagFormat, TriggeredTagsFormat,
|
||||
};
|
||||
use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam};
|
||||
|
||||
use super::DsmlTokens;
|
||||
|
||||
const TOKENS: DsmlTokens = DsmlTokens::V41;
|
||||
|
||||
pub(super) struct DeepSeekV41StructuralTagBuilder;
|
||||
|
||||
impl StructuralTagBuilder for DeepSeekV41StructuralTagBuilder {
|
||||
fn build(&self, ctx: StructuralTagContext<'_>) -> Result<StructuralTag> {
|
||||
let tags = ctx
|
||||
.function_tools
|
||||
.iter()
|
||||
.map(|tool| call_tag(tool, ctx.options))
|
||||
.collect::<Vec<_>>();
|
||||
let excludes = if ctx.options.exclude_special_tokens {
|
||||
&["<think>", "</think>"][..]
|
||||
} else {
|
||||
&[]
|
||||
};
|
||||
let format = match ctx.tool_choice {
|
||||
BuilderToolChoice::Auto if tags.is_empty() => Format::any_text_excluding(excludes),
|
||||
BuilderToolChoice::Auto => Format::TriggeredTags(
|
||||
TriggeredTagsFormat::new(
|
||||
&[TOKENS.tool_calls_start],
|
||||
vec![TagFormat::new(
|
||||
format!("{}\n", TOKENS.tool_calls_start),
|
||||
Format::tags_with_separator(tags, "", true, false),
|
||||
TOKENS.tool_calls_end,
|
||||
)],
|
||||
)
|
||||
.with_excludes(excludes),
|
||||
),
|
||||
BuilderToolChoice::Forced | BuilderToolChoice::Required => Format::sequence(vec![
|
||||
Format::const_string(format!("\n\n{}\n", TOKENS.tool_calls_start)),
|
||||
Format::tags_with_separator(
|
||||
tags,
|
||||
"",
|
||||
true,
|
||||
ctx.tool_choice == BuilderToolChoice::Forced,
|
||||
),
|
||||
Format::const_string(TOKENS.tool_calls_end),
|
||||
]),
|
||||
};
|
||||
// Reasoning boundaries remain owned by the engine reasoner; serving
|
||||
// calls this visible-text builder with reasoning=false.
|
||||
Ok(StructuralTag::new(format))
|
||||
}
|
||||
}
|
||||
|
||||
fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat {
|
||||
// TODO: enforce tool.parameters and strict schema semantics once V4.1
|
||||
// argument conversion is available. This builder intentionally ignores
|
||||
// parameters and strict: it constrains tool names and DSML/value syntax
|
||||
// only. Parameter names, presence, uniqueness and value schemas remain
|
||||
// unconstrained, including for strict=true. Serving still uses strict to
|
||||
// decide whether auto tool choice activates a grammar.
|
||||
TagFormat::new(
|
||||
format!("{} name=\"{}\">\n", TOKENS.invoke_start, tool.function.name),
|
||||
Format::star(parameter(options)),
|
||||
format!("{}\n", TOKENS.invoke_end),
|
||||
)
|
||||
}
|
||||
|
||||
fn parameter(options: StructuralTagOptions) -> Format {
|
||||
Format::tag(
|
||||
format!("{} name=\"", TOKENS.parameter_start),
|
||||
Format::sequence(vec![
|
||||
Format::regex(r#"[^"]+"#),
|
||||
Format::const_string("\" string=\""),
|
||||
Format::or(vec![
|
||||
Format::sequence(vec![
|
||||
Format::const_string("true\">"),
|
||||
Format::any_text_excluding(&[
|
||||
TOKENS.parameter_end,
|
||||
TOKENS.invoke_end,
|
||||
TOKENS.tool_calls_end,
|
||||
]),
|
||||
]),
|
||||
Format::sequence(vec![
|
||||
Format::const_string("false\">"),
|
||||
Format::JsonSchema(
|
||||
JsonSchemaFormat::new(Value::Bool(true))
|
||||
.with_any_order(options.any_order)
|
||||
.with_max_whitespace_cnt(options.max_whitespace_cnt),
|
||||
),
|
||||
]),
|
||||
]),
|
||||
]),
|
||||
format!("{}\n", TOKENS.parameter_end),
|
||||
)
|
||||
}
|
||||
@@ -102,7 +102,7 @@ fn v4_tags_remain_text_in_v41_dialect() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn structural_tag_ignores_parameter_schemas_and_strict() {
|
||||
fn structural_tag_respects_parameter_schemas_and_strict() {
|
||||
let parser = DeepSeekV41ToolParser::create(&tools()).unwrap();
|
||||
let builder = parser.structural_tag_builder().unwrap();
|
||||
let options = StructuralTagOptions::default().with_reasoning(false);
|
||||
@@ -128,7 +128,13 @@ fn structural_tag_ignores_parameter_schemas_and_strict() {
|
||||
] {
|
||||
let mut function = FunctionDefinition::new("lookup").with_parameters(parameters);
|
||||
function.strict = strict;
|
||||
assert_eq!(build(function), expected);
|
||||
let tag = build(function);
|
||||
if strict == Some(false) {
|
||||
assert_eq!(tag, expected);
|
||||
} else {
|
||||
assert_ne!(tag, expected);
|
||||
}
|
||||
assert!(tag.to_json_string().unwrap().contains("deepseek_v4_1_xml"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -219,7 +219,7 @@ mod tests {
|
||||
ToolChoice::required(),
|
||||
);
|
||||
|
||||
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>\n"},{"type":"tags_with_separator","tags":[{"begin":"<tool_call:opensource>get_weather<tool_sep:opensource>\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"days"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]}}]},"end":"</tool_call:opensource>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
|
||||
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>\n"},{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<tool_call:opensource>get_weather<tool_sep:opensource>\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"days"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]}}]},"end":"</tool_call:opensource>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -238,7 +238,7 @@ mod tests {
|
||||
ToolChoice::required(),
|
||||
);
|
||||
|
||||
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>"},{"type":"tags_with_separator","tags":[{"begin":"<tool_call:opensource>get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"}]}]},"end":"</tool_call:opensource>"}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
|
||||
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>"},{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<tool_call:opensource>get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value:opensource>"}]}]},"end":"</tool_call:opensource>"}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -259,7 +259,7 @@ mod tests {
|
||||
);
|
||||
let value = serde_json::to_value(tag).unwrap();
|
||||
|
||||
expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"limit"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq(
|
||||
expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"limit"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq(
|
||||
&serde_json::to_string(
|
||||
&value["format"]["elements"][1]["tags"][0]["content"],
|
||||
)
|
||||
@@ -276,8 +276,8 @@ mod tests {
|
||||
let auto = build("", &tools, ToolChoice::auto()).to_json_string().unwrap();
|
||||
let forced = build("", &tools, ToolChoice::function("lookup")).to_json_string().unwrap();
|
||||
|
||||
expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":["<tool_calls>"],"tags":[{"begin":"<tool_calls>\n","content":{"type":"tags_with_separator","tags":[{"begin":"<tool_call>search<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"},{"begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n</tool_calls>"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto);
|
||||
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls>\n"},{"type":"tag","begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"},{"type":"const_string","value":"\n</tool_calls>"}]}}"#]].assert_eq(&forced);
|
||||
expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":["<tool_calls>"],"tags":[{"type":"tag","begin":"<tool_calls>\n","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<tool_call>search<tool_sep>\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":"</tool_call>"},{"type":"tag","begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":"</tool_call>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n</tool_calls>"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto);
|
||||
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls>\n"},{"type":"tag","begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":"</tool_call>"},{"type":"const_string","value":"\n</tool_calls>"}]}}"#]].assert_eq(&forced);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -314,7 +314,7 @@ mod tests {
|
||||
);
|
||||
let value = serde_json::to_value(tag).unwrap();
|
||||
|
||||
expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"tenant"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq(
|
||||
expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"tenant"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq(
|
||||
&serde_json::to_string(
|
||||
&value["format"]["elements"][1]["tags"][0]["content"],
|
||||
)
|
||||
|
||||
@@ -6,14 +6,14 @@
|
||||
use serde_json::{Map, Value};
|
||||
use xgrammar_structural_tag::Result;
|
||||
use xgrammar_structural_tag::builders::{
|
||||
StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
|
||||
ReasoningMode, StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
|
||||
};
|
||||
use xgrammar_structural_tag::format::{Format, JsonSchemaFormat, StructuralTag, TagFormat};
|
||||
use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam, function_parameters};
|
||||
|
||||
use super::{
|
||||
ARG_CLOSE, CALL_CLOSE, CLOSE, END_OF_MSG, JSON_CLOSE, JSON_OPEN, MESSAGE_CLOSE, OPEN,
|
||||
RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, TOOLS_CLOSE, TOOLS_OPEN,
|
||||
RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, THINK_OPEN, TOOLS_CLOSE, TOOLS_OPEN,
|
||||
};
|
||||
|
||||
pub(super) static KIMI_K3_STRUCTURAL_TAG_BUILDER: KimiK3StructuralTagBuilder =
|
||||
@@ -50,14 +50,23 @@ impl StructuralTagBuilder for KimiK3StructuralTagBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
fn response_prefix(reasoning: bool) -> Vec<Format> {
|
||||
fn response_prefix(reasoning: ReasoningMode) -> Vec<Format> {
|
||||
let mut elements = Vec::new();
|
||||
if reasoning {
|
||||
elements.push(Format::tag(
|
||||
"",
|
||||
if reasoning != ReasoningMode::Disabled {
|
||||
let prefix = Format::tag(
|
||||
if reasoning == ReasoningMode::Auto {
|
||||
THINK_OPEN
|
||||
} else {
|
||||
""
|
||||
},
|
||||
Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG]),
|
||||
THINK_CLOSE,
|
||||
));
|
||||
);
|
||||
elements.push(if reasoning == ReasoningMode::Auto {
|
||||
Format::optional(prefix)
|
||||
} else {
|
||||
prefix
|
||||
});
|
||||
elements.push(Format::const_string(RESPONSE_OPEN));
|
||||
} else {
|
||||
elements.push(Format::optional(Format::const_string(RESPONSE_OPEN)));
|
||||
@@ -386,7 +395,7 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"]},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap());
|
||||
expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"],"max_tokens":null,"max_chars":null},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -6,7 +6,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from xgrammar import Grammar, StructuralTag, builtin_structural_tag
|
||||
from xgrammar import Grammar, StructuralTag
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
@@ -23,6 +23,7 @@ from vllm.tool_parsers.deepseekv31_tool_parser import DeepSeekV31ToolParser
|
||||
from vllm.tool_parsers.deepseekv32_engine_tool_parser import (
|
||||
DeepSeekV32EngineToolParser,
|
||||
)
|
||||
from vllm.tool_parsers.deepseekv41_engine_tool_parser import DeepSeekV41EngineToolParser
|
||||
from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
|
||||
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
|
||||
from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
|
||||
@@ -83,51 +84,9 @@ def test_supported_structural_tag_models_include_vllm_builtins():
|
||||
assert "hermes" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"])
|
||||
def test_deepseek_v41_older_xgrammar_keeps_format_constraints(
|
||||
choice,
|
||||
sample_tools_strict,
|
||||
monkeypatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
builtin_structural_tag, "get_deepseek_v4_1_structural_tag", None, raising=False
|
||||
)
|
||||
tool_choice = (
|
||||
ChatCompletionNamedToolChoiceParam(
|
||||
function=ChatCompletionNamedFunction(name=choice)
|
||||
)
|
||||
if choice == "get_weather"
|
||||
else choice
|
||||
)
|
||||
tag = get_model_structural_tag(
|
||||
"deepseek_v41", sample_tools_strict, tool_choice, reasoning=False
|
||||
)
|
||||
grammar = Grammar.from_structural_tag(tag)
|
||||
begin = '\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n'
|
||||
end = "</|DSML| invoke>\n</|DSML| calls>"
|
||||
parameter = (
|
||||
'<|DSML| parameter name="undeclared" string="false">'
|
||||
'{"nested": [true, null, 1.5]}</|DSML| parameter>\n'
|
||||
)
|
||||
# Required city is omitted; extra and repeated fields remain unconstrained.
|
||||
assert _is_grammar_accept_string(grammar, begin + end)
|
||||
assert _is_grammar_accept_string(grammar, begin + parameter * 2 + end)
|
||||
assert not _is_grammar_accept_string(
|
||||
grammar,
|
||||
begin + parameter.replace('{"nested": [true, null, 1.5]}', "invalid") + end,
|
||||
)
|
||||
assert not _is_grammar_accept_string(
|
||||
grammar, (begin + end).replace('name="get_weather"', 'name="unknown"')
|
||||
)
|
||||
assert not _is_grammar_accept_string(
|
||||
grammar, begin + parameter.replace("| parameter", "|parameter") + end
|
||||
)
|
||||
assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto")
|
||||
|
||||
|
||||
def test_deepseek_v41_named_choice_emits_one_call(sample_tools):
|
||||
tag = get_model_structural_tag(
|
||||
"deepseek_v41",
|
||||
"deepseek_v4_1",
|
||||
sample_tools,
|
||||
ChatCompletionNamedToolChoiceParam(
|
||||
function=ChatCompletionNamedFunction(name="get_weather")
|
||||
@@ -147,19 +106,14 @@ def test_deepseek_v41_named_choice_emits_one_call(sample_tools):
|
||||
grammar, "\n\n<|DSML| calls>\n" + call * 2 + "</|DSML| calls>"
|
||||
)
|
||||
assert (
|
||||
get_model_structural_tag("deepseek_v41", sample_tools, "auto", reasoning=False)
|
||||
get_model_structural_tag("deepseek_v4_1", sample_tools, "auto", reasoning=False)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"),
|
||||
reason="Requires XGrammar's DeepSeek V4.1 builtin",
|
||||
)
|
||||
@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"])
|
||||
@pytest.mark.parametrize("reasoning", [False, True])
|
||||
def test_deepseek_v41_constrains_parameters_after_reasoning(
|
||||
sample_tools_strict, choice, reasoning
|
||||
sample_tools_strict, choice
|
||||
):
|
||||
"""Strict calls must enforce the schema without requiring another think close."""
|
||||
tool_choice = (
|
||||
@@ -171,7 +125,7 @@ def test_deepseek_v41_constrains_parameters_after_reasoning(
|
||||
)
|
||||
sample_tools_strict[0].function.parameters["additionalProperties"] = False
|
||||
tag = get_model_structural_tag(
|
||||
"deepseek_v41", sample_tools_strict, tool_choice, reasoning=reasoning
|
||||
"deepseek_v4_1", sample_tools_strict, tool_choice, reasoning=False
|
||||
)
|
||||
grammar = Grammar.from_structural_tag(tag)
|
||||
begin = '\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n'
|
||||
@@ -195,14 +149,10 @@ def test_deepseek_v41_constrains_parameters_after_reasoning(
|
||||
assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"),
|
||||
reason="Requires XGrammar's DeepSeek V4.1 builtin",
|
||||
)
|
||||
def test_deepseek_v41_non_strict_parallel_calls_keep_typed_dsml(sample_tools):
|
||||
sample_tools[0].function.strict = False
|
||||
tag = get_model_structural_tag(
|
||||
"deepseek_v41", sample_tools, "required", reasoning=False
|
||||
"deepseek_v4_1", sample_tools, "required", reasoning=False
|
||||
)
|
||||
grammar = Grammar.from_structural_tag(tag)
|
||||
call = (
|
||||
@@ -571,6 +521,7 @@ def test_get_model_structural_tag_supports_named_tool_choice(
|
||||
(DeepSeekV31ToolParser, "deepseek_v3_1"),
|
||||
(DeepSeekV32EngineToolParser, "deepseek_v3_2"),
|
||||
(DeepSeekV4EngineToolParser, "deepseek_v4"),
|
||||
(DeepSeekV41EngineToolParser, "deepseek_v4_1"),
|
||||
(Glm47MoeModelToolParser, "glm_4_7"),
|
||||
(Hermes2ProToolParser, "hermes"),
|
||||
(KimiK2ToolParser, "kimi"),
|
||||
@@ -638,7 +589,11 @@ def test_get_structural_tag_disables_reasoning(
|
||||
assert captured == [False]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls", [Qwen3EngineToolParser, DeepSeekV41EngineToolParser]
|
||||
)
|
||||
def test_unified_parser_get_structural_tag_disables_reasoning(
|
||||
parser_cls,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sample_tools_strict: list[ChatCompletionToolsParam],
|
||||
):
|
||||
@@ -654,7 +609,7 @@ def test_unified_parser_get_structural_tag_disables_reasoning(
|
||||
)
|
||||
|
||||
class TestParser(DelegatingParser):
|
||||
tool_parser_cls = Qwen3EngineToolParser
|
||||
tool_parser_cls = parser_cls
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
messages=[],
|
||||
|
||||
@@ -5,4 +5,4 @@ from vllm.parser.engine.registered_adapters import DeepSeekV41ParserToolAdapter
|
||||
|
||||
|
||||
class DeepSeekV41EngineToolParser(DeepSeekV41ParserToolAdapter): # type: ignore[valid-type, misc]
|
||||
structural_tag_model = "deepseek_v41"
|
||||
structural_tag_model = "deepseek_v4_1"
|
||||
|
||||
@@ -10,7 +10,6 @@ from openai.types.responses.tool import Tool as ResponsesTool
|
||||
from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed
|
||||
from openai.types.responses.tool_choice_function import ToolChoiceFunction
|
||||
from xgrammar import StructuralTag, normalize_tool_choice
|
||||
from xgrammar import builtin_structural_tag as xgrammar_builtin_structural_tag
|
||||
from xgrammar import get_model_structural_tag as get_xgrammar_model_structural_tag
|
||||
from xgrammar.openai_tool_call_schema import (
|
||||
BuiltinToolParam,
|
||||
@@ -71,11 +70,10 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
|
||||
"deepseek_v3_2",
|
||||
"glm_4_7",
|
||||
"deepseek_v4",
|
||||
"deepseek_v4_1",
|
||||
}
|
||||
)
|
||||
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
|
||||
{"deepseek_v41", "hermes", "hy_v4", "kimi_k3"}
|
||||
)
|
||||
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "hy_v4", "kimi_k3"})
|
||||
SUPPORTED_STRUCTURAL_TAG_MODELS = (
|
||||
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
|
||||
)
|
||||
@@ -270,110 +268,6 @@ def get_function_parameters(function) -> dict[str, Any] | bool:
|
||||
return function.parameters if function.parameters is not None else True
|
||||
|
||||
|
||||
_V41_CALLS_START = "<|DSML| calls>"
|
||||
_V41_CALLS_END = "</|DSML| calls>"
|
||||
_V41_INVOKE_END = "</|DSML| invoke>"
|
||||
_V41_PARAMETER_END = "</|DSML| parameter>"
|
||||
|
||||
|
||||
@register_vllm_structural_tag("deepseek_v41")
|
||||
def get_deepseek_v41_structural_tag(
|
||||
tools: list[FunctionToolParam],
|
||||
builtin_tools: list[BuiltinToolParam],
|
||||
tool_choice: SimplifiedToolChoice,
|
||||
reasoning: bool,
|
||||
token_suffix: str = "",
|
||||
) -> StructuralTag:
|
||||
# Serving enables this visible-text grammar after the reasoning boundary.
|
||||
builder = getattr(
|
||||
xgrammar_builtin_structural_tag, "get_deepseek_v4_1_structural_tag", None
|
||||
)
|
||||
if builder is not None:
|
||||
return builder(
|
||||
tools=tools,
|
||||
builtin_tools=builtin_tools,
|
||||
tool_choice=tool_choice,
|
||||
reasoning="disabled",
|
||||
)
|
||||
|
||||
del builtin_tools, reasoning, token_suffix
|
||||
|
||||
# Compatibility with xgrammar releases without the V4.1 builtin. This constrains
|
||||
# tool names and DSML/value syntax only. Parameter names, presence, uniqueness
|
||||
# and value schemas remain unconstrained, including for strict=true. The request
|
||||
# layer still uses strict to decide whether auto tool choice activates a grammar.
|
||||
parameter = TagFormat(
|
||||
begin='<|DSML| parameter name="',
|
||||
content=SequenceFormat(
|
||||
elements=[
|
||||
RegexFormat(pattern=r'[^"]+'),
|
||||
ConstStringFormat(value='" string="'),
|
||||
OrFormat(
|
||||
elements=[
|
||||
SequenceFormat(
|
||||
elements=[
|
||||
ConstStringFormat(value='true">'),
|
||||
AnyTextFormat(
|
||||
excludes=[
|
||||
_V41_PARAMETER_END,
|
||||
_V41_INVOKE_END,
|
||||
_V41_CALLS_END,
|
||||
]
|
||||
),
|
||||
]
|
||||
),
|
||||
SequenceFormat(
|
||||
elements=[
|
||||
ConstStringFormat(value='false">'),
|
||||
JSONSchemaFormat(json_schema=True),
|
||||
]
|
||||
),
|
||||
]
|
||||
),
|
||||
]
|
||||
),
|
||||
end=f"{_V41_PARAMETER_END}\n",
|
||||
)
|
||||
calls = TagsWithSeparatorFormat(
|
||||
tags=[
|
||||
TagFormat(
|
||||
begin=f'<|DSML| invoke name="{tool.function.name}">\n',
|
||||
content=StarFormat(content=parameter),
|
||||
end=f"{_V41_INVOKE_END}\n",
|
||||
)
|
||||
for tool in tools
|
||||
],
|
||||
separator="",
|
||||
at_least_one=True,
|
||||
stop_after_first=tool_choice == "forced",
|
||||
)
|
||||
if tool_choice == "auto":
|
||||
return StructuralTag(
|
||||
format=TriggeredTagsFormat(
|
||||
triggers=[_V41_CALLS_START],
|
||||
tags=[
|
||||
TagFormat(
|
||||
begin=f"{_V41_CALLS_START}\n",
|
||||
content=calls,
|
||||
end=_V41_CALLS_END,
|
||||
)
|
||||
],
|
||||
excludes=["<think>", "</think>"],
|
||||
)
|
||||
if tools
|
||||
else AnyTextFormat(excludes=["<think>", "</think>"])
|
||||
)
|
||||
return StructuralTag(
|
||||
format=SequenceFormat(
|
||||
elements=[
|
||||
ConstStringFormat(value=f"\n\n{_V41_CALLS_START}\n"),
|
||||
calls,
|
||||
ConstStringFormat(value=_V41_CALLS_END),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
|
||||
arguments_field_prefix = '", "arguments": '
|
||||
formats = [
|
||||
|
||||
Reference in New Issue
Block a user