[Rust Frontend] Integrate xgrammar-structural-tag for strict and required tool calling (#46057)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-24 10:46:49 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 6af0559ddb
commit 4ed8eaafb0
23 changed files with 466 additions and 81 deletions
+1
View File
@@ -34,6 +34,7 @@ vllm-reasoning-parser.workspace = true
vllm-text.workspace = true
vllm-tokenizer.workspace = true
vllm-tool-parser.workspace = true
xgrammar-structural-tag.workspace = true
[dev-dependencies]
anyhow.workspace = true
+2
View File
@@ -64,6 +64,8 @@ pub enum Error {
StreamClosedBeforeTerminalOutput { request_id: String },
#[error("tool call stream state is inconsistent: {message}")]
ToolCallStreamInvariant { message: String },
#[error("failed to build structural tag: {message}")]
StructuralTag { message: String },
#[error(transparent)]
Text(#[from] vllm_text::Error),
#[error(transparent)]
+6 -3
View File
@@ -1,6 +1,7 @@
//! Default output processing pipeline.
mod reasoning;
mod structural_tag;
mod tool;
use std::sync::Once;
@@ -11,6 +12,7 @@ use trait_set::trait_set;
use vllm_text::tokenizer::DynTokenizer;
use self::reasoning::reasoning_event_stream;
use self::structural_tag::apply_structural_tag_constraint;
use self::tool::tool_event_stream;
use super::structured::structured_chat_event_stream;
use crate::error::Result;
@@ -21,7 +23,7 @@ use crate::output::{
use crate::parser::ParserSelection;
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
use crate::parser::tool::{ToolParser, ToolParserFactory};
use crate::request::{ChatRequest, ChatToolChoice};
use crate::request::ChatRequest;
use crate::{Error, Result as ChatResult};
trait_set! {
@@ -54,8 +56,7 @@ impl DefaultChatOutputProcessor {
tool_call_parser: &ParserSelection,
reasoning_parser: &ParserSelection,
) -> ChatResult<Self> {
let tool_parsing_enabled =
matches!(request.tool_choice, ChatToolChoice::Auto) && !request.tools.is_empty();
let tool_parsing_enabled = request.tool_parsing_enabled();
let tool_parser = if tool_parsing_enabled {
Some(Self::resolve_tool_parser(
request,
@@ -115,6 +116,8 @@ impl DefaultChatOutputProcessor {
request.decode_options.skip_special_tokens = false;
}
apply_structural_tag_constraint(request, parser.as_ref())?;
TOOL_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using tool parser"));
Ok(parser)
}
@@ -0,0 +1,257 @@
//! Applies xgrammar structural-tag constraints for strict tool calling.
use thiserror_ext::AsReport;
use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
use xgrammar_structural_tag::{
FunctionDefinition, FunctionToolParam, ToolChoice as StructuralTagToolChoice, ToolParam,
build_structural_tag,
};
use crate::parser::tool::ToolParser;
use crate::request::{ChatRequest, ChatToolChoice};
use crate::{Error, Result as ChatResult};
/// Apply structural tag constraints to the request based on the tool parser's structural tag
/// support and the request's tool choice.
pub(super) fn apply_structural_tag_constraint(
request: &mut ChatRequest,
parser: &dyn ToolParser,
) -> ChatResult<()> {
let Some(model) = parser.structural_tag_model() else {
return Ok(());
};
let Some(tool_choice) = structural_tag_tool_choice(request) else {
return Ok(());
};
let tools = request
.tools
.iter()
.map(|tool| {
ToolParam::Function(FunctionToolParam::new(FunctionDefinition {
name: tool.name.clone(),
description: tool.description.clone(),
parameters: Some(tool.parameters.clone()),
strict: tool.strict,
}))
})
.collect::<Vec<_>>();
let structural_tag = build_structural_tag(model, &tools, tool_choice, false)
.and_then(|tag| tag.to_json_string())
.map_err(|error| Error::StructuralTag {
message: error.to_report_string(),
})?;
// Overwrite any existing structured output settings with the structural tag constraint.
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
structural_tag: Some(structural_tag),
backend: StructuredOutputBackend::Xgrammar,
..Default::default()
});
Ok(())
}
/// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request.
///
/// Returns `None` if no structural tag constraints should be applied.
fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagToolChoice> {
if request.tools.is_empty() {
return None;
}
match &request.tool_choice {
// For `Auto`, only apply the structural tag if there's at least one strict tool.
ChatToolChoice::Auto if request.tools.iter().any(|tool| tool.strict == Some(true)) => {
Some(StructuralTagToolChoice::auto())
}
ChatToolChoice::Auto | ChatToolChoice::None => None,
ChatToolChoice::Required => Some(StructuralTagToolChoice::required()),
ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())),
}
}
#[cfg(test)]
mod tests {
use serde_json::{Value, json};
use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
use vllm_tool_parser::{Qwen3CoderToolParser, Tool};
use super::*;
fn chat_tool(name: &str, strict: Option<bool>) -> Tool {
Tool {
name: name.to_string(),
description: None,
parameters: json!({
"type": "object",
"properties": {
"query": { "type": "string" }
},
"required": ["query"]
}),
strict,
}
}
fn qwen3_coder_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
Qwen3CoderToolParser::create(tools).expect("Qwen3 Coder parser should build")
}
fn request(tool_choice: ChatToolChoice, tools: Vec<Tool>) -> ChatRequest {
ChatRequest {
tool_choice,
tools,
..ChatRequest::for_test()
}
}
fn structural_tag_value(request: &ChatRequest) -> Value {
let params = request
.sampling_params
.structured_outputs
.as_ref()
.expect("structured outputs should be set");
assert_eq!(params.backend, StructuredOutputBackend::Xgrammar);
serde_json::from_str(
params.structural_tag.as_deref().expect("structural_tag should be set"),
)
.expect("structural_tag should be valid JSON")
}
fn structured_outputs(request: &ChatRequest) -> &StructuredOutputsParams {
request
.sampling_params
.structured_outputs
.as_ref()
.expect("structured outputs should be set")
}
#[test]
fn auto_strict_tool_choice_builds_structural_tag() {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag should build");
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn auto_non_strict_tool_choice_skips_structural_tag() {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]);
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag decision should succeed");
assert!(request.sampling_params.structured_outputs.is_none());
}
#[test]
fn auto_strict_tool_choice_overwrites_existing_json_guidance() {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
json: Some(json!({"type": "object"})),
backend: StructuredOutputBackend::Xgrammar,
..Default::default()
});
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag should build");
let params = structured_outputs(&request);
assert!(params.json.is_none());
assert!(params.structural_tag.is_some());
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn required_tool_choice_builds_structural_tag_without_strict_tools() {
let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag should build");
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn required_tool_choice_overwrites_existing_json_object_guidance() {
let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
json_object: Some(true),
backend: StructuredOutputBackend::Xgrammar,
..Default::default()
});
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag should build");
let params = structured_outputs(&request);
assert!(params.json_object.is_none());
assert!(params.structural_tag.is_some());
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn named_tool_choice_builds_structural_tag_for_named_tool_only() {
let mut request = request(
ChatToolChoice::Function {
name: "lookup".to_string(),
},
vec![chat_tool("search", None), chat_tool("lookup", None)],
);
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag should build");
let tag = structural_tag_value(&request).to_string();
assert!(tag.contains("lookup"));
assert!(!tag.contains("search"));
}
#[test]
fn none_tool_choice_skips_structural_tag() {
let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag decision should succeed");
assert!(request.sampling_params.structured_outputs.is_none());
}
#[test]
fn none_tool_choice_preserves_existing_json_object_guidance() {
let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
json_object: Some(true),
backend: StructuredOutputBackend::Xgrammar,
..Default::default()
});
let parser = qwen3_coder_parser(&request.tools);
apply_structural_tag_constraint(&mut request, parser.as_ref())
.expect("structural tag decision should succeed");
let params = structured_outputs(&request);
assert_eq!(params.json_object, Some(true));
assert!(params.structural_tag.is_none());
}
}
+7 -3
View File
@@ -382,12 +382,16 @@ impl ChatOptions {
}
/// Tool-choice semantics supported by `vllm-chat`.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ChatToolChoice {
Auto,
#[default]
None,
Auto,
Required,
Function {
name: String,
},
}
/// One chat request ready to be rendered into a prompt and lowered into a
@@ -486,7 +490,7 @@ impl ChatRequest {
/// Return true if this request should enable tool parsing based on the tool
/// choice and tool list.
pub(crate) fn tool_parsing_enabled(&self) -> bool {
matches!(self.tool_choice, ChatToolChoice::Auto) && !self.tools.is_empty()
!matches!(self.tool_choice, ChatToolChoice::None) && !self.tools.is_empty()
}
/// Return the request-level thinking toggle when explicitly requested.