[Rust Frontend] Integrate xgrammar-structural-tag for strict and required tool calling (#46057)
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
6af0559ddb
commit
4ed8eaafb0
@@ -34,6 +34,7 @@ vllm-reasoning-parser.workspace = true
|
||||
vllm-text.workspace = true
|
||||
vllm-tokenizer.workspace = true
|
||||
vllm-tool-parser.workspace = true
|
||||
xgrammar-structural-tag.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
anyhow.workspace = true
|
||||
|
||||
@@ -64,6 +64,8 @@ pub enum Error {
|
||||
StreamClosedBeforeTerminalOutput { request_id: String },
|
||||
#[error("tool call stream state is inconsistent: {message}")]
|
||||
ToolCallStreamInvariant { message: String },
|
||||
#[error("failed to build structural tag: {message}")]
|
||||
StructuralTag { message: String },
|
||||
#[error(transparent)]
|
||||
Text(#[from] vllm_text::Error),
|
||||
#[error(transparent)]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! Default output processing pipeline.
|
||||
|
||||
mod reasoning;
|
||||
mod structural_tag;
|
||||
mod tool;
|
||||
|
||||
use std::sync::Once;
|
||||
@@ -11,6 +12,7 @@ use trait_set::trait_set;
|
||||
use vllm_text::tokenizer::DynTokenizer;
|
||||
|
||||
use self::reasoning::reasoning_event_stream;
|
||||
use self::structural_tag::apply_structural_tag_constraint;
|
||||
use self::tool::tool_event_stream;
|
||||
use super::structured::structured_chat_event_stream;
|
||||
use crate::error::Result;
|
||||
@@ -21,7 +23,7 @@ use crate::output::{
|
||||
use crate::parser::ParserSelection;
|
||||
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
|
||||
use crate::parser::tool::{ToolParser, ToolParserFactory};
|
||||
use crate::request::{ChatRequest, ChatToolChoice};
|
||||
use crate::request::ChatRequest;
|
||||
use crate::{Error, Result as ChatResult};
|
||||
|
||||
trait_set! {
|
||||
@@ -54,8 +56,7 @@ impl DefaultChatOutputProcessor {
|
||||
tool_call_parser: &ParserSelection,
|
||||
reasoning_parser: &ParserSelection,
|
||||
) -> ChatResult<Self> {
|
||||
let tool_parsing_enabled =
|
||||
matches!(request.tool_choice, ChatToolChoice::Auto) && !request.tools.is_empty();
|
||||
let tool_parsing_enabled = request.tool_parsing_enabled();
|
||||
let tool_parser = if tool_parsing_enabled {
|
||||
Some(Self::resolve_tool_parser(
|
||||
request,
|
||||
@@ -115,6 +116,8 @@ impl DefaultChatOutputProcessor {
|
||||
request.decode_options.skip_special_tokens = false;
|
||||
}
|
||||
|
||||
apply_structural_tag_constraint(request, parser.as_ref())?;
|
||||
|
||||
TOOL_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using tool parser"));
|
||||
Ok(parser)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
//! Applies xgrammar structural-tag constraints for strict tool calling.
|
||||
|
||||
use thiserror_ext::AsReport;
|
||||
use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
|
||||
use xgrammar_structural_tag::{
|
||||
FunctionDefinition, FunctionToolParam, ToolChoice as StructuralTagToolChoice, ToolParam,
|
||||
build_structural_tag,
|
||||
};
|
||||
|
||||
use crate::parser::tool::ToolParser;
|
||||
use crate::request::{ChatRequest, ChatToolChoice};
|
||||
use crate::{Error, Result as ChatResult};
|
||||
|
||||
/// Apply structural tag constraints to the request based on the tool parser's structural tag
|
||||
/// support and the request's tool choice.
|
||||
pub(super) fn apply_structural_tag_constraint(
|
||||
request: &mut ChatRequest,
|
||||
parser: &dyn ToolParser,
|
||||
) -> ChatResult<()> {
|
||||
let Some(model) = parser.structural_tag_model() else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(tool_choice) = structural_tag_tool_choice(request) else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let tools = request
|
||||
.tools
|
||||
.iter()
|
||||
.map(|tool| {
|
||||
ToolParam::Function(FunctionToolParam::new(FunctionDefinition {
|
||||
name: tool.name.clone(),
|
||||
description: tool.description.clone(),
|
||||
parameters: Some(tool.parameters.clone()),
|
||||
strict: tool.strict,
|
||||
}))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let structural_tag = build_structural_tag(model, &tools, tool_choice, false)
|
||||
.and_then(|tag| tag.to_json_string())
|
||||
.map_err(|error| Error::StructuralTag {
|
||||
message: error.to_report_string(),
|
||||
})?;
|
||||
|
||||
// Overwrite any existing structured output settings with the structural tag constraint.
|
||||
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
|
||||
structural_tag: Some(structural_tag),
|
||||
backend: StructuredOutputBackend::Xgrammar,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request.
|
||||
///
|
||||
/// Returns `None` if no structural tag constraints should be applied.
|
||||
fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagToolChoice> {
|
||||
if request.tools.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
match &request.tool_choice {
|
||||
// For `Auto`, only apply the structural tag if there's at least one strict tool.
|
||||
ChatToolChoice::Auto if request.tools.iter().any(|tool| tool.strict == Some(true)) => {
|
||||
Some(StructuralTagToolChoice::auto())
|
||||
}
|
||||
ChatToolChoice::Auto | ChatToolChoice::None => None,
|
||||
|
||||
ChatToolChoice::Required => Some(StructuralTagToolChoice::required()),
|
||||
ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::{Value, json};
|
||||
use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
|
||||
use vllm_tool_parser::{Qwen3CoderToolParser, Tool};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn chat_tool(name: &str, strict: Option<bool>) -> Tool {
|
||||
Tool {
|
||||
name: name.to_string(),
|
||||
description: None,
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": { "type": "string" }
|
||||
},
|
||||
"required": ["query"]
|
||||
}),
|
||||
strict,
|
||||
}
|
||||
}
|
||||
|
||||
fn qwen3_coder_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
|
||||
Qwen3CoderToolParser::create(tools).expect("Qwen3 Coder parser should build")
|
||||
}
|
||||
|
||||
fn request(tool_choice: ChatToolChoice, tools: Vec<Tool>) -> ChatRequest {
|
||||
ChatRequest {
|
||||
tool_choice,
|
||||
tools,
|
||||
..ChatRequest::for_test()
|
||||
}
|
||||
}
|
||||
|
||||
fn structural_tag_value(request: &ChatRequest) -> Value {
|
||||
let params = request
|
||||
.sampling_params
|
||||
.structured_outputs
|
||||
.as_ref()
|
||||
.expect("structured outputs should be set");
|
||||
assert_eq!(params.backend, StructuredOutputBackend::Xgrammar);
|
||||
serde_json::from_str(
|
||||
params.structural_tag.as_deref().expect("structural_tag should be set"),
|
||||
)
|
||||
.expect("structural_tag should be valid JSON")
|
||||
}
|
||||
|
||||
fn structured_outputs(request: &ChatRequest) -> &StructuredOutputsParams {
|
||||
request
|
||||
.sampling_params
|
||||
.structured_outputs
|
||||
.as_ref()
|
||||
.expect("structured outputs should be set")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_strict_tool_choice_builds_structural_tag() {
|
||||
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request);
|
||||
assert_eq!(tag["type"], "structural_tag");
|
||||
assert!(tag.to_string().contains("search"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_non_strict_tool_choice_skips_structural_tag() {
|
||||
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]);
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
.expect("structural tag decision should succeed");
|
||||
|
||||
assert!(request.sampling_params.structured_outputs.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_strict_tool_choice_overwrites_existing_json_guidance() {
|
||||
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
|
||||
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
|
||||
json: Some(json!({"type": "object"})),
|
||||
backend: StructuredOutputBackend::Xgrammar,
|
||||
..Default::default()
|
||||
});
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
.expect("structural tag should build");
|
||||
|
||||
let params = structured_outputs(&request);
|
||||
assert!(params.json.is_none());
|
||||
assert!(params.structural_tag.is_some());
|
||||
let tag = structural_tag_value(&request);
|
||||
assert_eq!(tag["type"], "structural_tag");
|
||||
assert!(tag.to_string().contains("search"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn required_tool_choice_builds_structural_tag_without_strict_tools() {
|
||||
let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request);
|
||||
assert_eq!(tag["type"], "structural_tag");
|
||||
assert!(tag.to_string().contains("search"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn required_tool_choice_overwrites_existing_json_object_guidance() {
|
||||
let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
|
||||
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
|
||||
json_object: Some(true),
|
||||
backend: StructuredOutputBackend::Xgrammar,
|
||||
..Default::default()
|
||||
});
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
.expect("structural tag should build");
|
||||
|
||||
let params = structured_outputs(&request);
|
||||
assert!(params.json_object.is_none());
|
||||
assert!(params.structural_tag.is_some());
|
||||
let tag = structural_tag_value(&request);
|
||||
assert_eq!(tag["type"], "structural_tag");
|
||||
assert!(tag.to_string().contains("search"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn named_tool_choice_builds_structural_tag_for_named_tool_only() {
|
||||
let mut request = request(
|
||||
ChatToolChoice::Function {
|
||||
name: "lookup".to_string(),
|
||||
},
|
||||
vec![chat_tool("search", None), chat_tool("lookup", None)],
|
||||
);
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request).to_string();
|
||||
assert!(tag.contains("lookup"));
|
||||
assert!(!tag.contains("search"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn none_tool_choice_skips_structural_tag() {
|
||||
let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
.expect("structural tag decision should succeed");
|
||||
|
||||
assert!(request.sampling_params.structured_outputs.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn none_tool_choice_preserves_existing_json_object_guidance() {
|
||||
let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
|
||||
request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
|
||||
json_object: Some(true),
|
||||
backend: StructuredOutputBackend::Xgrammar,
|
||||
..Default::default()
|
||||
});
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
.expect("structural tag decision should succeed");
|
||||
|
||||
let params = structured_outputs(&request);
|
||||
assert_eq!(params.json_object, Some(true));
|
||||
assert!(params.structural_tag.is_none());
|
||||
}
|
||||
}
|
||||
@@ -382,12 +382,16 @@ impl ChatOptions {
|
||||
}
|
||||
|
||||
/// Tool-choice semantics supported by `vllm-chat`.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ChatToolChoice {
|
||||
Auto,
|
||||
#[default]
|
||||
None,
|
||||
Auto,
|
||||
Required,
|
||||
Function {
|
||||
name: String,
|
||||
},
|
||||
}
|
||||
|
||||
/// One chat request ready to be rendered into a prompt and lowered into a
|
||||
@@ -486,7 +490,7 @@ impl ChatRequest {
|
||||
/// Return true if this request should enable tool parsing based on the tool
|
||||
/// choice and tool list.
|
||||
pub(crate) fn tool_parsing_enabled(&self) -> bool {
|
||||
matches!(self.tool_choice, ChatToolChoice::Auto) && !self.tools.is_empty()
|
||||
!matches!(self.tool_choice, ChatToolChoice::None) && !self.tools.is_empty()
|
||||
}
|
||||
|
||||
/// Return the request-level thinking toggle when explicitly requested.
|
||||
|
||||
@@ -361,6 +361,13 @@ fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result<ChatToolChoic
|
||||
match tool_choice {
|
||||
None | Some(ToolChoice::Value(ToolChoiceValue::Auto)) => Ok(ChatToolChoice::Auto),
|
||||
Some(ToolChoice::Value(ToolChoiceValue::None)) => Ok(ChatToolChoice::None),
|
||||
Some(ToolChoice::Value(ToolChoiceValue::Required)) => Ok(ChatToolChoice::Required),
|
||||
Some(ToolChoice::Function {
|
||||
tool_type,
|
||||
function,
|
||||
}) if tool_type == "function" => Ok(ChatToolChoice::Function {
|
||||
name: function.name.clone(),
|
||||
}),
|
||||
_ => bail_invalid_request!("tool_choice={:?} is not supported yet.", tool_choice),
|
||||
}
|
||||
}
|
||||
@@ -962,6 +969,74 @@ mod tests {
|
||||
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_lowers_required_tool_choice() {
|
||||
let request = ChatCompletionRequest {
|
||||
tools: Some(vec![Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: Function {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}),
|
||||
strict: None,
|
||||
},
|
||||
}]),
|
||||
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)),
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Required);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_lowers_named_function_tool_choice() {
|
||||
let request = ChatCompletionRequest {
|
||||
tools: Some(vec![Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: Function {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}),
|
||||
strict: None,
|
||||
},
|
||||
}]),
|
||||
tool_choice: Some(ToolChoice::Function {
|
||||
tool_type: "function".to_string(),
|
||||
function: crate::routes::openai::utils::types::FunctionChoice {
|
||||
name: "get_weather".to_string(),
|
||||
},
|
||||
}),
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(
|
||||
prepared.chat_request.tool_choice,
|
||||
ChatToolChoice::Function {
|
||||
name: "get_weather".to_string(),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_lowers_logprobs_fields() {
|
||||
let request = ChatCompletionRequest {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use super::types::ChatCompletionRequest;
|
||||
use crate::error::{ApiError, bail_invalid_request};
|
||||
use crate::routes::openai::utils::types::{ChatMessage, Tool, ToolChoice, ToolChoiceValue};
|
||||
use crate::routes::openai::utils::types::{ChatMessage, Tool};
|
||||
|
||||
/// Enforce the minimal compatibility contract for the Rust OpenAI server.
|
||||
pub(super) fn validate_request_compat(
|
||||
@@ -58,30 +58,6 @@ pub(super) fn validate_request_compat(
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(tool_choice) = &request.tool_choice {
|
||||
match tool_choice {
|
||||
ToolChoice::Value(ToolChoiceValue::Auto | ToolChoiceValue::None) => {}
|
||||
ToolChoice::Value(ToolChoiceValue::Required) => {
|
||||
bail_invalid_request!(
|
||||
param = "tool_choice",
|
||||
"tool_choice=required is not supported yet."
|
||||
);
|
||||
}
|
||||
ToolChoice::Function { .. } => {
|
||||
bail_invalid_request!(
|
||||
param = "tool_choice",
|
||||
"Named function tool_choice is not supported yet."
|
||||
);
|
||||
}
|
||||
ToolChoice::AllowedTools { .. } => {
|
||||
bail_invalid_request!(
|
||||
param = "tool_choice",
|
||||
"allowed_tools tool_choice is not supported yet."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if request.use_beam_search {
|
||||
bail_invalid_request!(
|
||||
param = "use_beam_search",
|
||||
@@ -159,8 +135,7 @@ mod tests {
|
||||
use crate::routes::openai::chat_completions::types::ChatCompletionRequest;
|
||||
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatMessage, Function, FunctionChoice, MessageContent, StringOrArray, Tool, ToolChoice,
|
||||
ToolChoiceValue, ToolReference,
|
||||
ChatMessage, Function, MessageContent, StringOrArray, Tool, ToolChoice, ToolChoiceValue,
|
||||
};
|
||||
|
||||
fn served(names: &[&str]) -> Vec<String> {
|
||||
@@ -357,38 +332,4 @@ mod tests {
|
||||
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
|
||||
.expect("tool_choice=none is ok");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_required_and_named_tool_choices() {
|
||||
let required = ChatCompletionRequest {
|
||||
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)),
|
||||
..base_request()
|
||||
};
|
||||
assert!(validate_request_compat(&required, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
|
||||
let named = ChatCompletionRequest {
|
||||
tool_choice: Some(ToolChoice::Function {
|
||||
tool_type: "function".to_string(),
|
||||
function: FunctionChoice {
|
||||
name: "tool".to_string(),
|
||||
},
|
||||
}),
|
||||
..base_request()
|
||||
};
|
||||
assert!(validate_request_compat(&named, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
|
||||
let allowed_tools = ChatCompletionRequest {
|
||||
tool_choice: Some(ToolChoice::AllowedTools {
|
||||
tool_type: "allowed_tools".to_string(),
|
||||
mode: "auto".to_string(),
|
||||
tools: vec![ToolReference::Function {
|
||||
name: "tool".to_string(),
|
||||
}],
|
||||
}),
|
||||
..base_request()
|
||||
};
|
||||
assert!(
|
||||
validate_request_compat(&allowed_tools, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
thiserror-ext.workspace = true
|
||||
winnow.workspace = true
|
||||
xgrammar-structural-tag.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
criterion.workspace = true
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use super::{DeepSeekDsmlToolParser, DsmlTokens};
|
||||
use crate::{Result, Tool, ToolParser, ToolParserOutput};
|
||||
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||
|
||||
/// Tool parser for DeepSeek V3.2 models.
|
||||
///
|
||||
@@ -44,6 +44,10 @@ impl ToolParser for DeepSeekV32ToolParser {
|
||||
true
|
||||
}
|
||||
|
||||
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||
Some(StructuralTagModel::DeepSeekV32)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
||||
self.0.parse_into(chunk, output)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use super::{DeepSeekDsmlToolParser, DsmlTokens};
|
||||
use crate::{Result, Tool, ToolParser, ToolParserOutput};
|
||||
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||
|
||||
/// Tool parser for DeepSeek V4 models.
|
||||
///
|
||||
@@ -47,6 +47,10 @@ impl ToolParser for DeepSeekV4ToolParser {
|
||||
true
|
||||
}
|
||||
|
||||
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||
Some(StructuralTagModel::DeepSeekV4)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
||||
self.0.parse_into(chunk, output)
|
||||
}
|
||||
@@ -65,8 +69,8 @@ mod tests {
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::DeepSeekV4ToolParser;
|
||||
use crate::ToolParserTestExt as _;
|
||||
use crate::test_utils::{collect_stream, test_tools};
|
||||
use crate::{StructuralTagModel, ToolParser, ToolParserTestExt as _};
|
||||
|
||||
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
|
||||
let params = params
|
||||
@@ -83,6 +87,16 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deepseek_v4_exposes_structural_tag_model() {
|
||||
let parser = DeepSeekV4ToolParser::new(&test_tools());
|
||||
|
||||
assert_eq!(
|
||||
parser.structural_tag_model(),
|
||||
Some(StructuralTagModel::DeepSeekV4)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deepseek_v4_parse_complete_reuses_dsml_parser_with_tool_calls_token() {
|
||||
let mut parser = DeepSeekV4ToolParser::new(&test_tools());
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser};
|
||||
use crate::{Result, Tool, ToolParser, ToolParserOutput};
|
||||
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||
|
||||
/// Tool parser for DeepSeek V3 JSON-fenced tool calls.
|
||||
///
|
||||
@@ -32,6 +32,10 @@ impl ToolParser for DeepSeekV3ToolParser {
|
||||
Ok(Box::new(Self::new(tools)))
|
||||
}
|
||||
|
||||
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||
Some(StructuralTagModel::DeepSeekR1)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
||||
self.0.parse_into(chunk, output)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser};
|
||||
use crate::{Result, Tool, ToolParser, ToolParserOutput};
|
||||
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||
|
||||
/// Tool parser for DeepSeek V3.1 raw JSON tool calls.
|
||||
///
|
||||
@@ -28,6 +28,10 @@ impl ToolParser for DeepSeekV31ToolParser {
|
||||
Ok(Box::new(Self::new(tools)))
|
||||
}
|
||||
|
||||
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||
Some(StructuralTagModel::DeepSeekV31)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
||||
self.0.parse_into(chunk, output)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use super::{GlmXmlToolParser, Separator};
|
||||
use crate::{Result, Tool, ToolParser, ToolParserOutput};
|
||||
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||
|
||||
/// Tool parser for GLM-4.7 MoE XML-style tool calls.
|
||||
///
|
||||
@@ -22,6 +22,10 @@ impl ToolParser for Glm47MoeToolParser {
|
||||
Ok(Box::new(Self::new(tools)))
|
||||
}
|
||||
|
||||
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||
Some(StructuralTagModel::Glm47)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
||||
self.0.parse_into(chunk, output)
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
|
||||
use super::parameters::ToolSchemas;
|
||||
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
||||
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||
use crate::Tool;
|
||||
use crate::{StructuralTagModel, Tool};
|
||||
|
||||
const TOOL_CALLS_START: &str = "<tool_calls>";
|
||||
const TOOL_CALLS_END: &str = "</tool_calls>";
|
||||
@@ -113,6 +113,10 @@ impl ToolParser for HyV3ToolParser {
|
||||
Ok(Box::new(Self::new(tools)))
|
||||
}
|
||||
|
||||
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||
Some(StructuralTagModel::HyV3)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
||||
self.buffer.push_str(chunk);
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
|
||||
use crate::{Result, Tool, ToolParser, ToolParserOutput};
|
||||
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||
|
||||
const HERMES_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
||||
parser_name: "Hermes",
|
||||
@@ -45,6 +45,10 @@ impl ToolParser for HermesToolParser {
|
||||
Ok(Box::new(Self::new(tools)))
|
||||
}
|
||||
|
||||
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||
Some(StructuralTagModel::Hermes)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
||||
self.inner.parse_into(chunk, output)
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ use super::{
|
||||
argument_delta_event, tool_call_header_event,
|
||||
};
|
||||
use crate::utils::{JsonObjectScanState, parse_buffered_event};
|
||||
use crate::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||
use crate::{Result, StructuralTagModel, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
enum LlamaJsonMode {
|
||||
@@ -133,6 +133,10 @@ impl ToolParser for Llama3JsonToolParser {
|
||||
Ok(Box::new(Self::new(tools)))
|
||||
}
|
||||
|
||||
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||
Some(StructuralTagModel::Llama)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
||||
self.buffer.push_str(chunk);
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
|
||||
use crate::{Result, Tool, ToolParser, ToolParserOutput};
|
||||
use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
|
||||
|
||||
const QWEN_XML_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
||||
parser_name: "Qwen XML",
|
||||
@@ -47,6 +47,10 @@ impl ToolParser for Qwen3XmlToolParser {
|
||||
Ok(Box::new(Self::new(tools)))
|
||||
}
|
||||
|
||||
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||
Some(StructuralTagModel::Qwen3)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
||||
self.inner.parse_into(chunk, output)
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ use winnow::token::{literal, rest, take_until, take_while};
|
||||
|
||||
use super::utils::{JsonObjectScanState, parse_buffered_event, safe_text_len, take_json_object};
|
||||
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||
use crate::Tool;
|
||||
use crate::{StructuralTagModel, Tool};
|
||||
|
||||
const TOOL_CALLS_START: &str = "<|tool_calls_section_begin|>";
|
||||
const TOOL_CALLS_END: &str = "<|tool_calls_section_end|>";
|
||||
@@ -147,6 +147,10 @@ impl ToolParser for KimiK2ToolParser {
|
||||
true
|
||||
}
|
||||
|
||||
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||
Some(StructuralTagModel::Kimi)
|
||||
}
|
||||
|
||||
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
|
||||
self.call_ids.get(&tool_index).map(String::as_str)
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@ pub use minimax_m3::MinimaxM3ToolParser;
|
||||
pub use qwen_coder::Qwen3CoderToolParser;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
pub use xgrammar_structural_tag::Model as StructuralTagModel;
|
||||
|
||||
/// One function-style tool made available to the model.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -123,6 +124,11 @@ pub trait ToolParser: Send {
|
||||
false
|
||||
}
|
||||
|
||||
/// Return the xgrammar structural-tag model used for strict tool calling.
|
||||
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Return the parser-provided ID for a tool call by index, if the model
|
||||
/// emitted one.
|
||||
fn tool_call_id(&self, _tool_index: usize) -> Option<&str> {
|
||||
|
||||
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
|
||||
use super::parameters::ToolSchemas;
|
||||
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
||||
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||
use crate::Tool;
|
||||
use crate::{StructuralTagModel, Tool};
|
||||
|
||||
const TOOL_CALL_START: &str = "<minimax:tool_call>";
|
||||
const TOOL_CALL_END: &str = "</minimax:tool_call>";
|
||||
@@ -112,6 +112,10 @@ impl ToolParser for MinimaxM2ToolParser {
|
||||
Ok(Box::new(Self::new(tools)))
|
||||
}
|
||||
|
||||
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||
Some(StructuralTagModel::Minimax)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
||||
self.buffer.push_str(chunk);
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ use winnow::token::{literal, take_until};
|
||||
|
||||
use super::parameters::ToolSchemas;
|
||||
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
||||
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||
use super::{Result, StructuralTagModel, ToolCallDelta, ToolParser, ToolParserOutput};
|
||||
use crate::Tool;
|
||||
|
||||
const TOOL_CALL_START: &str = "<tool_call>";
|
||||
@@ -113,6 +113,10 @@ impl ToolParser for Qwen3CoderToolParser {
|
||||
Ok(Box::new(Self::new(tools)))
|
||||
}
|
||||
|
||||
fn structural_tag_model(&self) -> Option<StructuralTagModel> {
|
||||
Some(StructuralTagModel::Qwen3Coder)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
||||
self.buffer.push_str(chunk);
|
||||
|
||||
@@ -236,7 +240,7 @@ mod tests {
|
||||
use serde_json::{Value, json};
|
||||
use thiserror_ext::AsReport;
|
||||
|
||||
use super::{Qwen3CoderToolParser, ToolParser};
|
||||
use super::{Qwen3CoderToolParser, StructuralTagModel, ToolParser};
|
||||
use crate::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||
use crate::{ToolParserOutput, ToolParserTestExt as _};
|
||||
|
||||
@@ -249,6 +253,16 @@ mod tests {
|
||||
format!("<tool_call>\n<function={function_name}>\n{params}\n</function>\n</tool_call>")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen_coder_exposes_structural_tag_model() {
|
||||
let parser = Qwen3CoderToolParser::new(&test_tools());
|
||||
|
||||
assert_eq!(
|
||||
parser.structural_tag_model(),
|
||||
Some(StructuralTagModel::Qwen3Coder)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen_coder_parse_complete_without_tool_call_keeps_text() {
|
||||
let mut parser = Qwen3CoderToolParser::new(&test_tools());
|
||||
|
||||
Reference in New Issue
Block a user