[Rust Frontend] Introduce parser-owned output grammar interfaces (#55269)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
f84325c48c
commit
d50723df04
@@ -11,6 +11,7 @@ test-util = []
|
||||
easy-ext.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_with.workspace = true
|
||||
thiserror.workspace = true
|
||||
thiserror-ext.workspace = true
|
||||
vllm-tokenizer.workspace = true
|
||||
|
||||
@@ -3,9 +3,7 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use vllm_parser::tool::{
|
||||
Result, StructuralTagBuilder, Tool, ToolParser, ToolParserError, ToolParserOutput,
|
||||
};
|
||||
use vllm_parser::tool::{Result, Tool, ToolParser, ToolParserError, ToolParserOutput};
|
||||
use vllm_parser::unified::{
|
||||
UnifiedParser, UnifiedParserError, UnifiedParserEvent, UnifiedParserOutput,
|
||||
};
|
||||
@@ -89,10 +87,6 @@ impl<T: UnifiedParser> ToolParser for UnifiedToolParserAdapter<T> {
|
||||
self.inner.preserve_special_tokens()
|
||||
}
|
||||
|
||||
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
|
||||
self.inner.structural_tag_builder()
|
||||
}
|
||||
|
||||
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
|
||||
self.inner.tool_call_id(tool_index)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
//! Streaming parsers for chat completions.
|
||||
|
||||
pub mod output_grammar;
|
||||
pub mod reasoning;
|
||||
pub mod tool;
|
||||
pub mod unified;
|
||||
|
||||
@@ -0,0 +1,362 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Output grammar construction shared by reasoning and tool parsers.
|
||||
|
||||
use std::{fmt, str::FromStr};
|
||||
|
||||
use serde_with::{DeserializeFromStr, SerializeDisplay};
|
||||
use thiserror::Error;
|
||||
use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagOptions};
|
||||
use xgrammar_structural_tag::format::Format;
|
||||
use xgrammar_structural_tag::tool::ToolChoiceValue;
|
||||
use xgrammar_structural_tag::{
|
||||
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
|
||||
};
|
||||
|
||||
use crate::tool::Tool;
|
||||
|
||||
/// Result alias for output grammar construction.
|
||||
pub type Result<T> = std::result::Result<T, OutputGrammarError>;
|
||||
|
||||
/// How much of the generated stream an output grammar covers.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum GrammarCoverage {
|
||||
/// The grammar covers reasoning and advances from the first generated token.
|
||||
FromTokenZero,
|
||||
/// The grammar covers only output emitted after reasoning ends.
|
||||
FinalOutputOnly,
|
||||
}
|
||||
|
||||
/// One parser-built output grammar and its generated-stream coverage.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct BuiltOutputGrammar {
|
||||
/// Typed XGrammar structural-tag format.
|
||||
pub format: Format,
|
||||
/// Portion of the generated stream covered by `format`.
|
||||
pub coverage: GrammarCoverage,
|
||||
}
|
||||
|
||||
impl BuiltOutputGrammar {
|
||||
/// A grammar that covers reasoning and everything after it.
|
||||
pub fn from_token_zero(format: Format) -> Self {
|
||||
Self {
|
||||
format,
|
||||
coverage: GrammarCoverage::FromTokenZero,
|
||||
}
|
||||
}
|
||||
|
||||
/// A grammar that covers only the output emitted after reasoning ends.
|
||||
pub fn final_output_only(format: Format) -> Self {
|
||||
Self {
|
||||
format,
|
||||
coverage: GrammarCoverage::FinalOutputOnly,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Server-side floor for tool-call structural tags (`--tool-strict-level`).
|
||||
#[derive(
|
||||
Debug,
|
||||
Clone,
|
||||
Copy,
|
||||
PartialEq,
|
||||
Eq,
|
||||
PartialOrd,
|
||||
Ord,
|
||||
Default,
|
||||
DeserializeFromStr,
|
||||
SerializeDisplay,
|
||||
)]
|
||||
pub enum ToolStrictLevel {
|
||||
/// Derive tool constraints from the request's tool choice and per-tool strictness.
|
||||
#[default]
|
||||
Auto,
|
||||
/// Constrain the tool-call envelope for every request with tools.
|
||||
Function,
|
||||
/// Additionally pin argument schemas, as if every tool were `strict: true`.
|
||||
Parameter,
|
||||
}
|
||||
|
||||
impl ToolStrictLevel {
|
||||
pub const AUTO_LITERAL: &str = "auto";
|
||||
pub const FUNCTION_LITERAL: &str = "function";
|
||||
pub const PARAMETER_LITERAL: &str = "parameter";
|
||||
}
|
||||
|
||||
impl FromStr for ToolStrictLevel {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(value: &str) -> std::result::Result<Self, Self::Err> {
|
||||
if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) {
|
||||
Ok(Self::Auto)
|
||||
} else if value.eq_ignore_ascii_case(Self::FUNCTION_LITERAL) {
|
||||
Ok(Self::Function)
|
||||
} else if value.eq_ignore_ascii_case(Self::PARAMETER_LITERAL) {
|
||||
Ok(Self::Parameter)
|
||||
} else {
|
||||
Err(format!(
|
||||
"unknown tool strict level {value:?}; expected one of {}, {}, {}",
|
||||
Self::AUTO_LITERAL,
|
||||
Self::FUNCTION_LITERAL,
|
||||
Self::PARAMETER_LITERAL
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ToolStrictLevel {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(match self {
|
||||
Self::Auto => Self::AUTO_LITERAL,
|
||||
Self::Function => Self::FUNCTION_LITERAL,
|
||||
Self::Parameter => Self::PARAMETER_LITERAL,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Request facts available while an initialized parser builds an output grammar.
|
||||
pub struct OutputGrammarContext<'a> {
|
||||
/// Effective tools available for this generation.
|
||||
pub tools: &'a [Tool],
|
||||
/// Effective tool-choice policy.
|
||||
pub tool_choice: &'a ToolChoice,
|
||||
/// Server-side floor for tool-call constraints.
|
||||
pub tool_strict_level: ToolStrictLevel,
|
||||
/// Whether the request permits more than one tool call.
|
||||
pub parallel_tool_calls: bool,
|
||||
}
|
||||
|
||||
/// Errors produced while building an output grammar.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum OutputGrammarError {
|
||||
/// A model-specific structural-tag builder rejected its inputs.
|
||||
#[error("failed to build output grammar")]
|
||||
Build(#[from] xgrammar_structural_tag::Error),
|
||||
}
|
||||
|
||||
/// Build the visible (post-reasoning) language from a crate structural-tag
|
||||
/// builder, or `None` when the request does not ask for strict tool calling.
|
||||
///
|
||||
/// This is the pre-existing strict-tool-calling grammar: the builder runs with
|
||||
/// `reasoning = false`, so the result describes only what follows the reasoning
|
||||
/// phase. It backs the default `ToolParser::build_visible_format` and the
|
||||
/// native unified parsers that have not yet grown their own builder.
|
||||
pub(crate) fn visible_format_from_builder(
|
||||
builder: Option<&dyn StructuralTagBuilder>,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> Result<Option<Format>> {
|
||||
let Some(builder) = builder else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !tool_grammar_applies(ctx) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let structural_tag = build_structural_tag(
|
||||
builder,
|
||||
&tool_params(ctx.tools, ctx.tool_strict_level),
|
||||
ctx.tool_choice.clone(),
|
||||
StructuralTagOptions::default()
|
||||
.with_reasoning(false)
|
||||
.with_parallel_tool_calls(ctx.parallel_tool_calls),
|
||||
)?;
|
||||
|
||||
Ok(Some(structural_tag.format))
|
||||
}
|
||||
|
||||
fn tool_params(tools: &[Tool], strict_level: ToolStrictLevel) -> Vec<ToolParam> {
|
||||
tools
|
||||
.iter()
|
||||
.map(|tool| {
|
||||
ToolParam::Function(FunctionToolParam::new(FunctionDefinition {
|
||||
name: tool.name.clone(),
|
||||
description: tool.description.clone(),
|
||||
parameters: Some(tool.parameters.clone()),
|
||||
strict: Some(
|
||||
strict_level >= ToolStrictLevel::Parameter || tool.strict == Some(true),
|
||||
),
|
||||
}))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Whether the request asks for a tool grammar at all.
|
||||
fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool {
|
||||
if ctx.tools.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
match ctx.tool_choice {
|
||||
ToolChoice::Value(ToolChoiceValue::None) => false,
|
||||
ToolChoice::Value(ToolChoiceValue::Auto) => {
|
||||
ctx.tool_strict_level >= ToolStrictLevel::Function
|
||||
|| ctx.tools.iter().any(|tool| tool.strict == Some(true))
|
||||
}
|
||||
ToolChoice::Value(ToolChoiceValue::Required)
|
||||
| ToolChoice::NamedFunction(_)
|
||||
| ToolChoice::AllowedTools(_)
|
||||
| ToolChoice::FlatAllowedTools(_)
|
||||
| ToolChoice::Builtin(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
use xgrammar_structural_tag::builders::Qwen3Builder;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn tool(name: &str, strict: Option<bool>) -> Tool {
|
||||
Tool {
|
||||
name: name.to_string(),
|
||||
description: None,
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": { "query": { "type": "string" } },
|
||||
"required": ["query"]
|
||||
}),
|
||||
strict,
|
||||
}
|
||||
}
|
||||
|
||||
fn build(tools: &[Tool], tool_choice: &ToolChoice) -> Option<Format> {
|
||||
visible_format_from_builder(
|
||||
Some(&Qwen3Builder),
|
||||
&OutputGrammarContext {
|
||||
tools,
|
||||
tool_choice,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
parallel_tool_calls: true,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_grammar_requires_tools_and_a_constraining_choice() {
|
||||
let non_strict = [tool("search", None)];
|
||||
assert!(build(&non_strict, &ToolChoice::none()).is_none());
|
||||
assert!(build(&non_strict, &ToolChoice::auto()).is_none());
|
||||
assert!(build(&non_strict, &ToolChoice::required()).is_some());
|
||||
assert!(build(&non_strict, &ToolChoice::function("search")).is_some());
|
||||
|
||||
// `auto` only constrains once at least one tool is strict.
|
||||
let strict = [tool("search", Some(true)), tool("lookup", None)];
|
||||
assert!(build(&strict, &ToolChoice::auto()).is_some());
|
||||
}
|
||||
|
||||
fn build_at_level(tool_choice: ToolChoice, tools: Vec<Tool>, level: ToolStrictLevel) -> Format {
|
||||
visible_format_from_builder(
|
||||
Some(&Qwen3Builder),
|
||||
&OutputGrammarContext {
|
||||
tools: &tools,
|
||||
tool_choice: &tool_choice,
|
||||
tool_strict_level: level,
|
||||
parallel_tool_calls: true,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn tag_string(format: &Format) -> String {
|
||||
serde_json::to_string(format).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn function_level_constrains_auto_without_strict_tools() {
|
||||
let request = build_at_level(
|
||||
ToolChoice::auto(),
|
||||
vec![tool("search", None)],
|
||||
ToolStrictLevel::Function,
|
||||
);
|
||||
|
||||
assert!(tag_string(&request).contains("search"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_strict_is_non_strict() {
|
||||
let unset = build_at_level(
|
||||
ToolChoice::required(),
|
||||
vec![tool("search", None)],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
let non_strict = build_at_level(
|
||||
ToolChoice::required(),
|
||||
vec![tool("search", Some(false))],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
let strict = build_at_level(
|
||||
ToolChoice::required(),
|
||||
vec![tool("search", Some(true))],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
|
||||
assert_eq!(tag_string(&unset), tag_string(&non_strict));
|
||||
assert_ne!(tag_string(&unset), tag_string(&strict));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_tool_does_not_pin_its_neighbour() {
|
||||
let mixed = build_at_level(
|
||||
ToolChoice::auto(),
|
||||
vec![tool("weather", Some(true)), tool("search", None)],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
let explicit = build_at_level(
|
||||
ToolChoice::auto(),
|
||||
vec![tool("weather", Some(true)), tool("search", Some(false))],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
|
||||
assert_eq!(tag_string(&mixed), tag_string(&explicit));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parameter_level_pins_argument_schemas() {
|
||||
let pinned = build_at_level(
|
||||
ToolChoice::auto(),
|
||||
vec![tool("search", None)],
|
||||
ToolStrictLevel::Parameter,
|
||||
);
|
||||
let strict = build_at_level(
|
||||
ToolChoice::auto(),
|
||||
vec![tool("search", Some(true))],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
let envelope_only = build_at_level(
|
||||
ToolChoice::auto(),
|
||||
vec![tool("search", None)],
|
||||
ToolStrictLevel::Function,
|
||||
);
|
||||
|
||||
assert_eq!(tag_string(&pinned), tag_string(&strict));
|
||||
assert_ne!(tag_string(&pinned), tag_string(&envelope_only));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_strict_level_parses_case_insensitively() {
|
||||
assert_eq!("AUTO".parse::<ToolStrictLevel>(), Ok(ToolStrictLevel::Auto));
|
||||
assert_eq!(
|
||||
"PARAMETER".parse::<ToolStrictLevel>(),
|
||||
Ok(ToolStrictLevel::Parameter)
|
||||
);
|
||||
assert_eq!(
|
||||
"function".parse::<ToolStrictLevel>(),
|
||||
Ok(ToolStrictLevel::Function)
|
||||
);
|
||||
assert_eq!(ToolStrictLevel::default(), ToolStrictLevel::Auto);
|
||||
assert_eq!(
|
||||
serde_json::to_value(ToolStrictLevel::Auto).unwrap(),
|
||||
json!("auto")
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<ToolStrictLevel>(json!("auto")).unwrap(),
|
||||
ToolStrictLevel::Auto
|
||||
);
|
||||
assert!("off".parse::<ToolStrictLevel>().is_err());
|
||||
assert!("strict".parse::<ToolStrictLevel>().is_err());
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,9 @@ mod step3p5;
|
||||
|
||||
use thiserror::Error;
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer};
|
||||
use xgrammar_structural_tag::format::Format;
|
||||
|
||||
use crate::output_grammar::{self, OutputGrammarContext};
|
||||
|
||||
pub use self::cohere_cmd::CohereCmdReasoningParser;
|
||||
pub use self::deepseek_r1::DeepSeekR1ReasoningParser;
|
||||
@@ -137,6 +140,15 @@ pub trait ReasoningParser: Send {
|
||||
false
|
||||
}
|
||||
|
||||
/// Prepend the reasoning phase implied by the initialized parser state.
|
||||
fn wrap_visible_format(
|
||||
&self,
|
||||
_ctx: &OutputGrammarContext<'_>,
|
||||
_visible: &Format,
|
||||
) -> output_grammar::Result<Option<Format>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Feed one decoded text delta into the parser.
|
||||
fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta>;
|
||||
|
||||
|
||||
@@ -39,7 +39,9 @@ pub use seed_oss::SeedOssToolParser;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
pub use xgrammar_structural_tag::builders::StructuralTagBuilder;
|
||||
use xgrammar_structural_tag::format::Format;
|
||||
|
||||
use crate::output_grammar::{self, OutputGrammarContext};
|
||||
use crate::utils;
|
||||
|
||||
/// One function-style tool made available to the model.
|
||||
@@ -189,7 +191,23 @@ pub trait ToolParser: Send {
|
||||
false
|
||||
}
|
||||
|
||||
/// Build the language of everything the model may emit after reasoning
|
||||
/// ends: visible text plus tool calls. `None` means no tool grammar applies.
|
||||
///
|
||||
/// The default forwards to [`Self::structural_tag_builder`] with reasoning
|
||||
/// disabled, which is exactly today's strict-tool-calling grammar.
|
||||
fn build_visible_format(
|
||||
&self,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<Format>> {
|
||||
output_grammar::visible_format_from_builder(self.structural_tag_builder(), ctx)
|
||||
}
|
||||
|
||||
/// Return the xgrammar structural-tag builder used for strict tool calling.
|
||||
///
|
||||
/// To be deprecated: this only exists to back the default
|
||||
/// [`Self::build_visible_format`]. New parsers should override that method
|
||||
/// directly instead of exposing a builder.
|
||||
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
|
||||
None
|
||||
}
|
||||
|
||||
@@ -6,8 +6,9 @@
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer};
|
||||
|
||||
use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput};
|
||||
use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext};
|
||||
use crate::reasoning::ReasoningParser;
|
||||
use crate::tool::{StructuralTagBuilder, Tool, ToolParser, ToolParserOutput};
|
||||
use crate::tool::{Tool, ToolParser, ToolParserOutput};
|
||||
|
||||
/// Unified parser that composes existing reasoning and tool parsers.
|
||||
pub struct CombinedParser {
|
||||
@@ -79,8 +80,29 @@ impl UnifiedParser for CombinedParser {
|
||||
|| self.tool.as_ref().is_some_and(|parser| parser.preserve_special_tokens())
|
||||
}
|
||||
|
||||
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
|
||||
self.tool.as_ref().and_then(|parser| parser.structural_tag_builder())
|
||||
fn build_output_grammar(
|
||||
&self,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
|
||||
let Some(tool) = self.tool.as_ref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(visible) = tool.build_visible_format(ctx)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
// The reasoning parser may wrap that language with the reasoning phase
|
||||
// implied by its initialized (prompt-derived) state. If it does, the
|
||||
// grammar describes the stream from the first generated token and the
|
||||
// engine can skip its own reasoning gate; if it declines, the grammar
|
||||
// covers only the final output and the engine gate stays in charge.
|
||||
let wrapped = match self.reasoning.as_ref() {
|
||||
Some(reasoning) => reasoning.wrap_visible_format(ctx, &visible)?,
|
||||
None => None,
|
||||
};
|
||||
Ok(Some(match wrapped {
|
||||
Some(full) => BuiltOutputGrammar::from_token_zero(full),
|
||||
None => BuiltOutputGrammar::final_output_only(visible),
|
||||
}))
|
||||
}
|
||||
|
||||
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
|
||||
@@ -332,7 +354,6 @@ mod tests {
|
||||
fn combined_parser_emits_tool_calls_from_visible_content() {
|
||||
let tool = Qwen3XmlToolParser::create(&test_tools()).unwrap();
|
||||
let mut parser = CombinedParser::new(None, Some(tool));
|
||||
assert!(parser.structural_tag_builder().is_some());
|
||||
|
||||
let output = collect(
|
||||
&mut parser,
|
||||
|
||||
@@ -4,8 +4,9 @@
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer};
|
||||
|
||||
use super::detect_hy_token_suffix;
|
||||
use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext};
|
||||
use crate::reasoning::HyReasoningParser;
|
||||
use crate::tool::{HyDialect, HyToolMarkers, HyToolParser, StructuralTagBuilder, Tool};
|
||||
use crate::tool::{HyDialect, HyToolMarkers, HyToolParser, Tool};
|
||||
use crate::unified::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id};
|
||||
|
||||
/// Unified reasoning and tool parser for HY3 output.
|
||||
@@ -46,8 +47,11 @@ impl UnifiedParser for HyV3UnifiedParser {
|
||||
self.inner.preserve_special_tokens()
|
||||
}
|
||||
|
||||
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
|
||||
self.inner.structural_tag_builder()
|
||||
fn build_output_grammar(
|
||||
&self,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
|
||||
self.inner.build_output_grammar(ctx)
|
||||
}
|
||||
|
||||
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
|
||||
@@ -73,12 +77,10 @@ mod tests {
|
||||
|
||||
use serde_json::json;
|
||||
use vllm_tokenizer::{DecodedText, Tokenizer, test_utils::TestTokenizer};
|
||||
use xgrammar_structural_tag::builders::StructuralTagOptions;
|
||||
use xgrammar_structural_tag::{
|
||||
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
|
||||
};
|
||||
use xgrammar_structural_tag::{StructuralTag, ToolChoice};
|
||||
|
||||
use super::{HyV3UnifiedParser, UnifiedParser};
|
||||
use crate::output_grammar::{GrammarCoverage, OutputGrammarContext};
|
||||
use crate::tool::Tool;
|
||||
use crate::unified::{UnifiedParserEvent, UnifiedParserOutput};
|
||||
|
||||
@@ -151,24 +153,19 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn structural_tag_uses_tokenizer_detected_suffix() {
|
||||
let parser = HyV3UnifiedParser::new(&tools(), Arc::new(tokenizer())).unwrap();
|
||||
let structural_tools = [ToolParam::Function(FunctionToolParam::new(
|
||||
FunctionDefinition::new("get_weather").with_parameters(json!({
|
||||
"type": "object",
|
||||
"properties": { "city": { "type": "string" } },
|
||||
"required": ["city"]
|
||||
})),
|
||||
))];
|
||||
|
||||
let tag = build_structural_tag(
|
||||
parser.structural_tag_builder().unwrap(),
|
||||
&structural_tools,
|
||||
ToolChoice::required(),
|
||||
StructuralTagOptions::default().with_reasoning(false),
|
||||
)
|
||||
.unwrap()
|
||||
.to_json_string()
|
||||
.unwrap();
|
||||
let tools = tools();
|
||||
let parser = HyV3UnifiedParser::new(&tools, Arc::new(tokenizer())).unwrap();
|
||||
let tag = parser
|
||||
.build_output_grammar(&OutputGrammarContext {
|
||||
tools: &tools,
|
||||
tool_strict_level: Default::default(),
|
||||
parallel_tool_calls: true,
|
||||
tool_choice: &ToolChoice::required(),
|
||||
})
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(tag.coverage, GrammarCoverage::FinalOutputOnly);
|
||||
let tag = StructuralTag::new(tag.format).to_json_string().unwrap();
|
||||
|
||||
assert!(tag.contains("<tool_calls:opensource>"));
|
||||
assert!(tag.contains("<arg_value:opensource>"));
|
||||
|
||||
@@ -4,8 +4,9 @@
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer};
|
||||
|
||||
use super::detect_hy_token_suffix;
|
||||
use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext};
|
||||
use crate::reasoning::HyReasoningParser;
|
||||
use crate::tool::{HyDialect, HyToolMarkers, HyToolParser, StructuralTagBuilder, Tool};
|
||||
use crate::tool::{HyDialect, HyToolMarkers, HyToolParser, Tool};
|
||||
use crate::unified::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id};
|
||||
|
||||
/// Unified reasoning and tool parser for HY4 output.
|
||||
@@ -46,8 +47,11 @@ impl UnifiedParser for HyV4UnifiedParser {
|
||||
self.inner.preserve_special_tokens()
|
||||
}
|
||||
|
||||
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
|
||||
self.inner.structural_tag_builder()
|
||||
fn build_output_grammar(
|
||||
&self,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
|
||||
self.inner.build_output_grammar(ctx)
|
||||
}
|
||||
|
||||
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
|
||||
@@ -73,12 +77,10 @@ mod tests {
|
||||
|
||||
use serde_json::json;
|
||||
use vllm_tokenizer::{DecodedText, Tokenizer, test_utils::TestTokenizer};
|
||||
use xgrammar_structural_tag::builders::StructuralTagOptions;
|
||||
use xgrammar_structural_tag::{
|
||||
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
|
||||
};
|
||||
use xgrammar_structural_tag::{StructuralTag, ToolChoice};
|
||||
|
||||
use super::{HyV4UnifiedParser, UnifiedParser};
|
||||
use crate::output_grammar::{GrammarCoverage, OutputGrammarContext};
|
||||
use crate::tool::Tool;
|
||||
use crate::unified::{UnifiedParserEvent, UnifiedParserOutput};
|
||||
|
||||
@@ -164,24 +166,19 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn structural_tag_uses_compact_hy4_skeleton() {
|
||||
let parser = HyV4UnifiedParser::new(&tools(), Arc::new(tokenizer())).unwrap();
|
||||
let structural_tools = [ToolParam::Function(FunctionToolParam::new(
|
||||
FunctionDefinition::new("get_weather").with_parameters(json!({
|
||||
"type": "object",
|
||||
"properties": { "city": { "type": "string" } },
|
||||
"required": ["city"]
|
||||
})),
|
||||
))];
|
||||
|
||||
let tag = build_structural_tag(
|
||||
parser.structural_tag_builder().unwrap(),
|
||||
&structural_tools,
|
||||
ToolChoice::required(),
|
||||
StructuralTagOptions::default().with_reasoning(false),
|
||||
)
|
||||
.unwrap()
|
||||
.to_json_string()
|
||||
.unwrap();
|
||||
let tools = tools();
|
||||
let parser = HyV4UnifiedParser::new(&tools, Arc::new(tokenizer())).unwrap();
|
||||
let tag = parser
|
||||
.build_output_grammar(&OutputGrammarContext {
|
||||
tools: &tools,
|
||||
tool_strict_level: Default::default(),
|
||||
parallel_tool_calls: true,
|
||||
tool_choice: &ToolChoice::required(),
|
||||
})
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(tag.coverage, GrammarCoverage::FinalOutputOnly);
|
||||
let tag = StructuralTag::new(tag.format).to_json_string().unwrap();
|
||||
|
||||
assert!(tag.contains("<tool_calls:opensource>"));
|
||||
assert!(tag.contains("<tool_call:opensource>get_weather"));
|
||||
|
||||
@@ -51,7 +51,10 @@ use winnow::token::{literal, rest, take_till, take_until, take_while};
|
||||
|
||||
use self::structural_tag::KIMI_K3_STRUCTURAL_TAG_BUILDER;
|
||||
use super::{Result, UnifiedParser, UnifiedParserOutput, token_id};
|
||||
use crate::tool::{StructuralTagBuilder, Tool, ToolCallDelta};
|
||||
use crate::output_grammar::{
|
||||
self, BuiltOutputGrammar, OutputGrammarContext, visible_format_from_builder,
|
||||
};
|
||||
use crate::tool::{Tool, ToolCallDelta};
|
||||
use crate::unified::parsing_failed;
|
||||
use crate::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker};
|
||||
|
||||
@@ -275,8 +278,14 @@ impl UnifiedParser for KimiK3UnifiedParser {
|
||||
true
|
||||
}
|
||||
|
||||
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
|
||||
Some(&KIMI_K3_STRUCTURAL_TAG_BUILDER)
|
||||
fn build_output_grammar(
|
||||
&self,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
|
||||
Ok(
|
||||
visible_format_from_builder(Some(&KIMI_K3_STRUCTURAL_TAG_BUILDER), ctx)?
|
||||
.map(BuiltOutputGrammar::final_output_only),
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, delta: DecodedText, output: &mut UnifiedParserOutput) -> Result<()> {
|
||||
|
||||
@@ -18,10 +18,9 @@ use thiserror::Error;
|
||||
use thiserror_ext::Macro;
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer};
|
||||
|
||||
use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext};
|
||||
use crate::reasoning::ReasoningError;
|
||||
use crate::tool::{
|
||||
StructuralTagBuilder, Tool, ToolCallDelta, ToolParserError, ToolParserEvent, ToolParserOutput,
|
||||
};
|
||||
use crate::tool::{Tool, ToolCallDelta, ToolParserError, ToolParserEvent, ToolParserOutput};
|
||||
|
||||
/// Result alias for unified parser operations.
|
||||
pub type Result<T> = std::result::Result<T, UnifiedParserError>;
|
||||
@@ -240,9 +239,12 @@ pub trait UnifiedParser: Send {
|
||||
false
|
||||
}
|
||||
|
||||
/// Return the xgrammar structural-tag builder used for strict tool calling.
|
||||
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
|
||||
None
|
||||
/// Build the request output grammar after prompt-based initialization.
|
||||
fn build_output_grammar(
|
||||
&self,
|
||||
_ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Return the parser-provided ID for a tool call by index, if the model emitted one.
|
||||
|
||||
Reference in New Issue
Block a user