[Rust Frontend] Integrate xgrammar-structural-tag for strict and required tool calling (#46057)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-24 10:46:49 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 6af0559ddb
commit 4ed8eaafb0
23 changed files with 466 additions and 81 deletions
+26
View File
@@ -316,6 +316,17 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "auto_impl"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "autocfg"
version = "1.5.0"
@@ -5726,6 +5737,7 @@ dependencies = [
"vllm-text",
"vllm-tokenizer",
"vllm-tool-parser",
"xgrammar-structural-tag",
"zeromq",
]
@@ -5981,6 +5993,7 @@ dependencies = [
"thiserror-ext",
"tool-parser",
"winnow",
"xgrammar-structural-tag",
]
[[package]]
@@ -6582,6 +6595,19 @@ version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
[[package]]
name = "xgrammar-structural-tag"
version = "0.1.0+xgrammar.0.2.2.4d145cc"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2436dea2393d55a3b188588aa300c5a8afe8f45a77da52c611fb4498a6c876e6"
dependencies = [
"auto_impl",
"serde",
"serde_json",
"strum",
"thiserror 2.0.18",
]
[[package]]
name = "y4m"
version = "0.8.0"
+1
View File
@@ -123,6 +123,7 @@ vllm-text = { path = "src/text" }
vllm-tokenizer = { path = "src/tokenizer" }
vllm-tool-parser = { path = "src/tool-parser" }
winnow = "1.0.2"
xgrammar-structural-tag = "0.1.0"
zeromq = { version = "0.6.0", default-features = false, features = [
"tokio-runtime",
"all-transport",
+1
View File
@@ -34,6 +34,7 @@ vllm-reasoning-parser.workspace = true
vllm-text.workspace = true
vllm-tokenizer.workspace = true
vllm-tool-parser.workspace = true
xgrammar-structural-tag.workspace = true
[dev-dependencies]
anyhow.workspace = true
+2
View File
@@ -64,6 +64,8 @@ pub enum Error {
StreamClosedBeforeTerminalOutput { request_id: String },
#[error("tool call stream state is inconsistent: {message}")]
ToolCallStreamInvariant { message: String },
#[error("failed to build structural tag: {message}")]
StructuralTag { message: String },
#[error(transparent)]
Text(#[from] vllm_text::Error),
#[error(transparent)]
+6 -3
View File
@@ -1,6 +1,7 @@
//! Default output processing pipeline.
mod reasoning;
mod structural_tag;
mod tool;
use std::sync::Once;
@@ -11,6 +12,7 @@ use trait_set::trait_set;
use vllm_text::tokenizer::DynTokenizer;
use self::reasoning::reasoning_event_stream;
use self::structural_tag::apply_structural_tag_constraint;
use self::tool::tool_event_stream;
use super::structured::structured_chat_event_stream;
use crate::error::Result;
@@ -21,7 +23,7 @@ use crate::output::{
use crate::parser::ParserSelection;
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
use crate::parser::tool::{ToolParser, ToolParserFactory};
use crate::request::{ChatRequest, ChatToolChoice};
use crate::request::ChatRequest;
use crate::{Error, Result as ChatResult};
trait_set! {
@@ -54,8 +56,7 @@ impl DefaultChatOutputProcessor {
tool_call_parser: &ParserSelection,
reasoning_parser: &ParserSelection,
) -> ChatResult<Self> {
let tool_parsing_enabled =
matches!(request.tool_choice, ChatToolChoice::Auto) && !request.tools.is_empty();
let tool_parsing_enabled = request.tool_parsing_enabled();
let tool_parser = if tool_parsing_enabled {
Some(Self::resolve_tool_parser(
request,
@@ -115,6 +116,8 @@ impl DefaultChatOutputProcessor {
request.decode_options.skip_special_tokens = false;
}
apply_structural_tag_constraint(request, parser.as_ref())?;
TOOL_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using tool parser"));
Ok(parser)
}
@@ -0,0 +1,257 @@
//! Applies xgrammar structural-tag constraints for strict tool calling.
use thiserror_ext::AsReport;
use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
use xgrammar_structural_tag::{
FunctionDefinition, FunctionToolParam, ToolChoice as StructuralTagToolChoice, ToolParam,
build_structural_tag,
};
use crate::parser::tool::ToolParser;
use crate::request::{ChatRequest, ChatToolChoice};
use crate::{Error, Result as ChatResult};
/// Apply structural tag constraints to the request based on the tool parser's structural tag
/// support and the request's tool choice.
pub(super) fn apply_structural_tag_constraint(
request: &mut ChatRequest,
parser: &dyn ToolParser,
) -> ChatResult<()> {
let Some(model) = parser.structural_tag_model() else {
return Ok(());
};
let Some(tool_choice) = structural_tag_tool_choice(request) else {
return Ok(());
};
let tools = request
.tools
.iter()
.map(|tool| {
ToolParam::Function(FunctionToolParam::new(FunctionDefinition {
name: tool.name.clone(),
description: tool.description.clone(),
parameters: Some(tool.parameters.clone()),
strict: tool.strict,
}))
})
.collect::<Vec<_>>();
let structural_tag = build_structural_tag(model, &tools, tool_choice, false)
.and_then(|tag| tag.to_json_string())
.map_err(|error| Error::StructuralTag {
message: error.to_report_string(),
})?;
// Overwrite any existing structured output settings with the structural tag constraint.
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
structural_tag: Some(structural_tag),
backend: StructuredOutputBackend::Xgrammar,
..Default::default()
});
Ok(())
}
/// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request.
///
/// Returns `None` if no structural tag constraints should be applied.
fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagToolChoice> {
if request.tools.is_empty() {
return None;
}
match &request.tool_choice {
// For `Auto`, only apply the structural tag if there's at least one strict tool.
ChatToolChoice::Auto if request.tools.iter().any(|tool| tool.strict == Some(true)) => {
Some(StructuralTagToolChoice::auto())
}
ChatToolChoice::Auto | ChatToolChoice::None => None,
ChatToolChoice::Required => Some(StructuralTagToolChoice::required()),
ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())),
}
}
#[cfg(test)]
mod tests {
use serde_json::{Value, json};
use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
use vllm_tool_parser::{Qwen3CoderToolParser, Tool};
use super::*;
fn chat_tool(name: &str, strict: Option<bool>) -> Tool {
Tool {
name: name.to_string(),
description: None,
parameters: json!({
"type": "object",
"properties": {
"query": { "type": "string" }
},
"required": ["query"]
}),
strict,
}
}
fn qwen3_coder_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
Qwen3CoderToolParser::create(tools).expect("Qwen3 Coder parser should build")
}
fn request(tool_choice: ChatToolChoice, tools: Vec<Tool>) -> ChatRequest {
ChatRequest {
tool_choice,
tools,
..ChatRequest::for_test()
}
}
fn structural_tag_value(request: &ChatRequest) -> Value {
let params = request
.sampling_params
.structured_outputs
.as_ref()
.expect("structured outputs should be set");
assert_eq!(params.backend, StructuredOutputBackend::Xgrammar);
serde_json::from_str(
params.structural_tag.as_deref().expect("structural_tag should be set"),
)
.expect("structural_tag should be valid JSON")
}
fn structured_outputs(request: &ChatRequest) -> &StructuredOutputsParams {
request
.sampling_params
.structured_outputs
.as_ref()
.expect("structured outputs should be set")
}
#[test]
fn auto_strict_tool_choice_builds_structural_tag() {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag should build");
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn auto_non_strict_tool_choice_skips_structural_tag() {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]);
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag decision should succeed");
assert!(request.sampling_params.structured_outputs.is_none());
}
#[test]
fn auto_strict_tool_choice_overwrites_existing_json_guidance() {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
json: Some(json!({"type": "object"})),
backend: StructuredOutputBackend::Xgrammar,
..Default::default()
});
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag should build");
let params = structured_outputs(&request);
assert!(params.json.is_none());
assert!(params.structural_tag.is_some());
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn required_tool_choice_builds_structural_tag_without_strict_tools() {
let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag should build");
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn required_tool_choice_overwrites_existing_json_object_guidance() {
let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
json_object: Some(true),
backend: StructuredOutputBackend::Xgrammar,
..Default::default()
});
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag should build");
let params = structured_outputs(&request);
assert!(params.json_object.is_none());
assert!(params.structural_tag.is_some());
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn named_tool_choice_builds_structural_tag_for_named_tool_only() {
let mut request = request(
ChatToolChoice::Function {
name: "lookup".to_string(),
},
vec![chat_tool("search", None), chat_tool("lookup", None)],
);
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag should build");
let tag = structural_tag_value(&request).to_string();
assert!(tag.contains("lookup"));
assert!(!tag.contains("search"));
}
#[test]
fn none_tool_choice_skips_structural_tag() {
let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag decision should succeed");
assert!(request.sampling_params.structured_outputs.is_none());
}
#[test]
fn none_tool_choice_preserves_existing_json_object_guidance() {
let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
json_object: Some(true),
backend: StructuredOutputBackend::Xgrammar,
..Default::default()
});
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag decision should succeed");
let params = structured_outputs(&request);
assert_eq!(params.json_object, Some(true));
assert!(params.structural_tag.is_none());
}
}
+7 -3
View File
@@ -382,12 +382,16 @@ impl ChatOptions {
}
/// Tool-choice semantics supported by `vllm-chat`.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ChatToolChoice {
Auto,
#[default]
None,
Auto,
Required,
Function {
name: String,
},
}
/// One chat request ready to be rendered into a prompt and lowered into a
@@ -486,7 +490,7 @@ impl ChatRequest {
/// Return true if this request should enable tool parsing based on the tool
/// choice and tool list.
pub(crate) fn tool_parsing_enabled(&self) -> bool {
matches!(self.tool_choice, ChatToolChoice::Auto) && !self.tools.is_empty()
!matches!(self.tool_choice, ChatToolChoice::None) && !self.tools.is_empty()
}
/// Return the request-level thinking toggle when explicitly requested.
@@ -361,6 +361,13 @@ fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result<ChatToolChoic
match tool_choice {
None | Some(ToolChoice::Value(ToolChoiceValue::Auto)) => Ok(ChatToolChoice::Auto),
Some(ToolChoice::Value(ToolChoiceValue::None)) => Ok(ChatToolChoice::None),
Some(ToolChoice::Value(ToolChoiceValue::Required)) => Ok(ChatToolChoice::Required),
Some(ToolChoice::Function {
tool_type,
function,
}) if tool_type == "function" => Ok(ChatToolChoice::Function {
name: function.name.clone(),
}),
_ => bail_invalid_request!("tool_choice={:?} is not supported yet.", tool_choice),
}
}
@@ -962,6 +969,74 @@ mod tests {
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::None);
}
#[test]
fn prepare_chat_request_lowers_required_tool_choice() {
let request = ChatCompletionRequest {
tools: Some(vec![Tool {
tool_type: "function".to_string(),
function: Function {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),
parameters: json!({
"type": "object",
"properties": {"city": {"type": "string"}},
}),
strict: None,
},
}]),
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)),
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Required);
}
#[test]
fn prepare_chat_request_lowers_named_function_tool_choice() {
let request = ChatCompletionRequest {
tools: Some(vec![Tool {
tool_type: "function".to_string(),
function: Function {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),
parameters: json!({
"type": "object",
"properties": {"city": {"type": "string"}},
}),
strict: None,
},
}]),
tool_choice: Some(ToolChoice::Function {
tool_type: "function".to_string(),
function: crate::routes::openai::utils::types::FunctionChoice {
name: "get_weather".to_string(),
},
}),
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.tool_choice,
ChatToolChoice::Function {
name: "get_weather".to_string(),
}
);
}
#[test]
fn prepare_chat_request_lowers_logprobs_fields() {
let request = ChatCompletionRequest {
@@ -1,6 +1,6 @@
use super::types::ChatCompletionRequest;
use crate::error::{ApiError, bail_invalid_request};
use crate::routes::openai::utils::types::{ChatMessage, Tool, ToolChoice, ToolChoiceValue};
use crate::routes::openai::utils::types::{ChatMessage, Tool};
/// Enforce the minimal compatibility contract for the Rust OpenAI server.
pub(super) fn validate_request_compat(
@@ -58,30 +58,6 @@ pub(super) fn validate_request_compat(
}
}
if let Some(tool_choice) = &request.tool_choice {
match tool_choice {
ToolChoice::Value(ToolChoiceValue::Auto | ToolChoiceValue::None) => {}
ToolChoice::Value(ToolChoiceValue::Required) => {
bail_invalid_request!(
param = "tool_choice",
"tool_choice=required is not supported yet."
);
}
ToolChoice::Function { .. } => {
bail_invalid_request!(
param = "tool_choice",
"Named function tool_choice is not supported yet."
);
}
ToolChoice::AllowedTools { .. } => {
bail_invalid_request!(
param = "tool_choice",
"allowed_tools tool_choice is not supported yet."
);
}
}
}
if request.use_beam_search {
bail_invalid_request!(
param = "use_beam_search",
@@ -159,8 +135,7 @@ mod tests {
use crate::routes::openai::chat_completions::types::ChatCompletionRequest;
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
use crate::routes::openai::utils::types::{
ChatMessage, Function, FunctionChoice, MessageContent, StringOrArray, Tool, ToolChoice,
ToolChoiceValue, ToolReference,
ChatMessage, Function, MessageContent, StringOrArray, Tool, ToolChoice, ToolChoiceValue,
};
fn served(names: &[&str]) -> Vec<String> {
@@ -357,38 +332,4 @@ mod tests {
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
.expect("tool_choice=none is ok");
}
#[test]
fn validate_request_compat_rejects_required_and_named_tool_choices() {
let required = ChatCompletionRequest {
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)),
..base_request()
};
assert!(validate_request_compat(&required, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
let named = ChatCompletionRequest {
tool_choice: Some(ToolChoice::Function {
tool_type: "function".to_string(),
function: FunctionChoice {
name: "tool".to_string(),
},
}),
..base_request()
};
assert!(validate_request_compat(&named, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
let allowed_tools = ChatCompletionRequest {
tool_choice: Some(ToolChoice::AllowedTools {
tool_type: "allowed_tools".to_string(),
mode: "auto".to_string(),
tools: vec![ToolReference::Function {
name: "tool".to_string(),
}],
}),
..base_request()
};
assert!(
validate_request_compat(&allowed_tools, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()
);
}
}
+1
View File
@@ -14,6 +14,7 @@ serde_json.workspace = true
thiserror.workspace = true
thiserror-ext.workspace = true
winnow.workspace = true
xgrammar-structural-tag.workspace = true
[dev-dependencies]
criterion.workspace = true
@@ -1,5 +1,5 @@
use super::{DeepSeekDsmlToolParser, DsmlTokens};
use crate::{Result, Tool, ToolParser, ToolParserOutput};
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
/// Tool parser for DeepSeek V3.2 models.
///
@@ -44,6 +44,10 @@ impl ToolParser for DeepSeekV32ToolParser {
true
}
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
Some(StructuralTagModel::DeepSeekV32)
}
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
self.0.parse_into(chunk, output)
}
@@ -1,5 +1,5 @@
use super::{DeepSeekDsmlToolParser, DsmlTokens};
use crate::{Result, Tool, ToolParser, ToolParserOutput};
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
/// Tool parser for DeepSeek V4 models.
///
@@ -47,6 +47,10 @@ impl ToolParser for DeepSeekV4ToolParser {
true
}
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
Some(StructuralTagModel::DeepSeekV4)
}
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
self.0.parse_into(chunk, output)
}
@@ -65,8 +69,8 @@ mod tests {
use serde_json::{Value, json};
use super::DeepSeekV4ToolParser;
use crate::ToolParserTestExt as _;
use crate::test_utils::{collect_stream, test_tools};
use crate::{StructuralTagModel, ToolParser, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
let params = params
@@ -83,6 +87,16 @@ mod tests {
)
}
#[test]
fn deepseek_v4_exposes_structural_tag_model() {
let parser = DeepSeekV4ToolParser::new(&test_tools());
assert_eq!(
parser.structural_tag_model(),
Some(StructuralTagModel::DeepSeekV4)
);
}
#[test]
fn deepseek_v4_parse_complete_reuses_dsml_parser_with_tool_calls_token() {
let mut parser = DeepSeekV4ToolParser::new(&test_tools());
@@ -1,5 +1,5 @@
use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser};
use crate::{Result, Tool, ToolParser, ToolParserOutput};
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
/// Tool parser for DeepSeek V3 JSON-fenced tool calls.
///
@@ -32,6 +32,10 @@ impl ToolParser for DeepSeekV3ToolParser {
Ok(Box::new(Self::new(tools)))
}
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
Some(StructuralTagModel::DeepSeekR1)
}
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
self.0.parse_into(chunk, output)
}
@@ -1,5 +1,5 @@
use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser};
use crate::{Result, Tool, ToolParser, ToolParserOutput};
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
/// Tool parser for DeepSeek V3.1 raw JSON tool calls.
///
@@ -28,6 +28,10 @@ impl ToolParser for DeepSeekV31ToolParser {
Ok(Box::new(Self::new(tools)))
}
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
Some(StructuralTagModel::DeepSeekV31)
}
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
self.0.parse_into(chunk, output)
}
@@ -1,5 +1,5 @@
use super::{GlmXmlToolParser, Separator};
use crate::{Result, Tool, ToolParser, ToolParserOutput};
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
/// Tool parser for GLM-4.7 MoE XML-style tool calls.
///
@@ -22,6 +22,10 @@ impl ToolParser for Glm47MoeToolParser {
Ok(Box::new(Self::new(tools)))
}
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
Some(StructuralTagModel::Glm47)
}
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
self.0.parse_into(chunk, output)
}
+5 -1
View File
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
use super::parameters::ToolSchemas;
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::Tool;
use crate::{StructuralTagModel, Tool};
const TOOL_CALLS_START: &str = "<tool_calls>";
const TOOL_CALLS_END: &str = "</tool_calls>";
@@ -113,6 +113,10 @@ impl ToolParser for HyV3ToolParser {
Ok(Box::new(Self::new(tools)))
}
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
Some(StructuralTagModel::HyV3)
}
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
self.buffer.push_str(chunk);
+5 -1
View File
@@ -1,5 +1,5 @@
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
use crate::{Result, Tool, ToolParser, ToolParserOutput};
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
const HERMES_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Hermes",
@@ -45,6 +45,10 @@ impl ToolParser for HermesToolParser {
Ok(Box::new(Self::new(tools)))
}
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
Some(StructuralTagModel::Hermes)
}
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
self.inner.parse_into(chunk, output)
}
+5 -1
View File
@@ -9,7 +9,7 @@ use super::{
argument_delta_event, tool_call_header_event,
};
use crate::utils::{JsonObjectScanState, parse_buffered_event};
use crate::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::{Result, StructuralTagModel, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
#[derive(Debug, Clone, PartialEq, Eq)]
enum LlamaJsonMode {
@@ -133,6 +133,10 @@ impl ToolParser for Llama3JsonToolParser {
Ok(Box::new(Self::new(tools)))
}
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
Some(StructuralTagModel::Llama)
}
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
self.buffer.push_str(chunk);
+5 -1
View File
@@ -1,5 +1,5 @@
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
use crate::{Result, Tool, ToolParser, ToolParserOutput};
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
const QWEN_XML_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Qwen XML",
@@ -47,6 +47,10 @@ impl ToolParser for Qwen3XmlToolParser {
Ok(Box::new(Self::new(tools)))
}
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
Some(StructuralTagModel::Qwen3)
}
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
self.inner.parse_into(chunk, output)
}
+5 -1
View File
@@ -8,7 +8,7 @@ use winnow::token::{literal, rest, take_until, take_while};
use super::utils::{JsonObjectScanState, parse_buffered_event, safe_text_len, take_json_object};
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::Tool;
use crate::{StructuralTagModel, Tool};
const TOOL_CALLS_START: &str = "<|tool_calls_section_begin|>";
const TOOL_CALLS_END: &str = "<|tool_calls_section_end|>";
@@ -147,6 +147,10 @@ impl ToolParser for KimiK2ToolParser {
true
}
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
Some(StructuralTagModel::Kimi)
}
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
self.call_ids.get(&tool_index).map(String::as_str)
}
+6
View File
@@ -35,6 +35,7 @@ pub use minimax_m3::MinimaxM3ToolParser;
pub use qwen_coder::Qwen3CoderToolParser;
use serde::{Deserialize, Serialize};
use serde_json::Value;
pub use xgrammar_structural_tag::Model as StructuralTagModel;
/// One function-style tool made available to the model.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -123,6 +124,11 @@ pub trait ToolParser: Send {
false
}
/// Return the xgrammar structural-tag model used for strict tool calling.
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
None
}
/// Return the parser-provided ID for a tool call by index, if the model
/// emitted one.
fn tool_call_id(&self, _tool_index: usize) -> Option<&str> {
+5 -1
View File
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
use super::parameters::ToolSchemas;
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::Tool;
use crate::{StructuralTagModel, Tool};
const TOOL_CALL_START: &str = "<minimax:tool_call>";
const TOOL_CALL_END: &str = "</minimax:tool_call>";
@@ -112,6 +112,10 @@ impl ToolParser for MinimaxM2ToolParser {
Ok(Box::new(Self::new(tools)))
}
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
Some(StructuralTagModel::Minimax)
}
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
self.buffer.push_str(chunk);
+16 -2
View File
@@ -6,7 +6,7 @@ use winnow::token::{literal, take_until};
use super::parameters::ToolSchemas;
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
use super::{Result, StructuralTagModel, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::Tool;
const TOOL_CALL_START: &str = "<tool_call>";
@@ -113,6 +113,10 @@ impl ToolParser for Qwen3CoderToolParser {
Ok(Box::new(Self::new(tools)))
}
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
Some(StructuralTagModel::Qwen3Coder)
}
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
self.buffer.push_str(chunk);
@@ -236,7 +240,7 @@ mod tests {
use serde_json::{Value, json};
use thiserror_ext::AsReport;
use super::{Qwen3CoderToolParser, ToolParser};
use super::{Qwen3CoderToolParser, StructuralTagModel, ToolParser};
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::{ToolParserOutput, ToolParserTestExt as _};
@@ -249,6 +253,16 @@ mod tests {
format!("<tool_call>\n<function={function_name}>\n{params}\n</function>\n</tool_call>")
}
#[test]
fn qwen_coder_exposes_structural_tag_model() {
let parser = Qwen3CoderToolParser::new(&test_tools());
assert_eq!(
parser.structural_tag_model(),
Some(StructuralTagModel::Qwen3Coder)
);
}
#[test]
fn qwen_coder_parse_complete_without_tool_call_keeps_text() {
let mut parser = Qwen3CoderToolParser::new(&test_tools());