[Frontend] Add the Beam reasoning and tool parsers (#62052)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com>
Co-authored-by: neilkg <33635204+neilkg@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Woosuk Kwon
2026-10-16 14:08:29 -07:00
committed by GitHub
co-authored by Bugen Zhao Yoni Gozlan neilkg Cursor Claude Opus 5.5
parent 9731f30a03
commit 153a17166e
27 changed files with 4464 additions and 92 deletions
+2 -2
View File
@@ -434,7 +434,7 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hf, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_json, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: beam, deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hf, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_json, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
}
#[test]
@@ -445,6 +445,6 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hf, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: beam, cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hf, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
}
}
+29 -12
View File
@@ -23,7 +23,8 @@ use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEvent
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
use crate::parser::tool::{ToolParser, ToolParserFactory};
use crate::parser::unified::{
HfTemplateError, HfUnifiedParser, ResponseTemplate, UnifiedParserFactory, names,
BeamUnifiedParser, HfTemplateError, HfUnifiedParser, ResponseTemplate, UnifiedParserFactory,
names,
};
use crate::parser::{ParserSelection, ToolStrictLevel};
use crate::request::{ChatRequest, ChatTool};
@@ -206,17 +207,19 @@ impl DefaultChatOutputProcessor {
});
}
// `hf` is built from the model's template, never by its registry constructor.
let parser = if parser_name == names::HF {
let template =
response_template.as_ref().map_err(|error| Error::ParserInitialization {
kind: "unified",
name: parser_name.to_string(),
error: Box::new(error.clone()),
})?;
Box::new(HfUnifiedParser::new(template.clone(), tools, tokenizer))
} else {
factory.create(parser_name, tools, tokenizer)?
// Parsers driven by the model's template are built from it, never by
// their registry constructors.
let template = || {
response_template.clone().map_err(|error| Error::ParserInitialization {
kind: "unified",
name: parser_name.to_string(),
error: Box::new(error),
})
};
let parser: Box<dyn UnifiedParser> = match parser_name {
names::HF => Box::new(HfUnifiedParser::new(template()?, tools, tokenizer)),
names::BEAM => Box::new(BeamUnifiedParser::new(template()?, tools, tokenizer)),
_ => factory.create(parser_name, tools, tokenizer)?,
};
UNIFIED_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using unified parser"));
@@ -589,6 +592,20 @@ mod tests {
.assert_eq(&error(invalid_template()));
}
#[test]
fn explicit_beam_uses_the_model_response_template() {
let beam = ParserSelection::Explicit("beam".to_string());
crate::parser::validate_parser_overrides(&beam, &beam).unwrap();
assert!(keeps_special_tokens("other-model", &loaded_template(), &beam, &beam).unwrap());
let error =
keeps_special_tokens("other-model", &Err(HfTemplateError::Missing), &beam, &beam)
.unwrap_err()
.to_report_string();
expect_test::expect!["failed to initialize unified parser `beam`: the model's tokenizer_config.json provides no response_template"]
.assert_eq(&error);
}
#[test]
fn auto_never_selects_hf() {
let auto = ParserSelection::Auto;
+6 -2
View File
@@ -6,8 +6,8 @@
use std::sync::LazyLock;
pub use vllm_parser::unified::{
Gemma4UnifiedParser, HfTemplateError, HfUnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser,
InklingUnifiedParser, KimiK3UnifiedParser, ResponseTemplate, UnifiedParser,
BeamUnifiedParser, Gemma4UnifiedParser, HfTemplateError, HfUnifiedParser, HyV3UnifiedParser,
HyV4UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser, ResponseTemplate, UnifiedParser,
};
use vllm_tokenizer::DynTokenizer;
@@ -26,6 +26,9 @@ pub mod names {
/// name resolution only: it is built from the model's template, never by its
/// registry constructor.
pub const HF: &str = "hf";
/// Beam checkpoints: [`HF`] with Beam's tool-call grammar, built from the
/// model's template the same way.
pub const BEAM: &str = "beam";
}
/// Constructor signature for one registered unified parser implementation.
@@ -55,6 +58,7 @@ impl UnifiedParserFactory {
factory.register_parser::<KimiK3UnifiedParser>(names::KIMI_K3);
// Opt-in only: no model patterns map to it.
factory.register_parser::<HfUnifiedParser>(names::HF);
factory.register_parser::<BeamUnifiedParser>(names::BEAM);
factory
.register_pattern("gemma-4", names::GEMMA4)
+3 -2
View File
@@ -207,7 +207,8 @@ pub(crate) fn full_format_from_builder_for_test(
)
}
fn tool_params(tools: &[Tool], strict_level: ToolStrictLevel) -> Vec<ToolParam> {
/// Tools as builder inputs, strict when the tool or the server floor asks for it.
pub(crate) fn tool_params(tools: &[Tool], strict_level: ToolStrictLevel) -> Vec<ToolParam> {
tools
.iter()
.map(|tool| {
@@ -240,7 +241,7 @@ pub(crate) fn normalize_tool_choice(
}
/// Whether the request asks for a tool grammar at all.
fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool {
pub(crate) fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool {
if ctx.tools.is_empty() {
return false;
}
+77 -20
View File
@@ -26,7 +26,7 @@
//! - `allOf` with more than one schema accepts any value.
use serde_json::{Value, json};
use xgrammar_structural_tag::format::{Format, JsonSchemaFormat};
use xgrammar_structural_tag::format::{EndBoundary, Format, JsonSchemaFormat, TokenFormat};
pub use crate::schema::JsonType;
use crate::schema::{self, OptionSource, SchemaOption, SchemaRoot};
@@ -60,7 +60,13 @@ impl ParameterKey<'_> {
/// `prefix KEY suffix content end`: a tag for a declared key, or a
/// sequence matching [`Self::FREE_PATTERN`] for a free key.
pub fn tag(&self, prefix: &str, suffix: &str, content: Format, end: &str) -> Format {
pub fn tag(
&self,
prefix: &str,
suffix: &str,
content: Format,
end: impl Into<EndBoundary>,
) -> Format {
match self {
Self::Declared(key) => Format::tag(format!("{prefix}{key}{suffix}"), content, end),
Self::Free => Format::sequence(vec![
@@ -68,7 +74,13 @@ impl ParameterKey<'_> {
Format::regex(Self::FREE_PATTERN),
Format::const_string(suffix),
content,
Format::const_string(end),
match end.into() {
EndBoundary::Text(end) => Format::const_string(end),
EndBoundary::Texts(ends) => {
one_of(ends.into_iter().map(Format::const_string).collect())
}
EndBoundary::Token(end) => Format::Token(TokenFormat { token: end.token }),
},
]),
}
}
@@ -183,6 +195,36 @@ impl ArgumentContext<'_> {
/// Build the grammar of one call's arguments under `schema`, the tool's
/// parameters schema, rendering each parameter with `syntax`.
pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &ArgumentOptions) -> Format {
parameter_list(
options,
syntax.separator(),
parameters(schema, syntax, options),
)
}
/// The parameters of one call, each rendered by an [`ArgumentSyntax`], for a
/// syntax that lists them itself rather than through [`arguments`].
pub struct Parameters {
/// The declared parameters, in declared order.
pub declared: Vec<Parameter>,
/// Any one additional parameter, if the schema allows them.
pub additional: Option<Format>,
}
/// One declared parameter.
pub struct Parameter {
pub key: String,
pub format: Format,
pub required: bool,
}
/// The parameters of one call under `schema`, the tool's parameters schema,
/// rendering each parameter with `syntax`.
pub fn parameters(
schema: &Value,
syntax: &dyn ArgumentSyntax,
options: &ArgumentOptions,
) -> Parameters {
let cx = ArgumentContext {
options,
root: SchemaRoot::new(schema),
@@ -190,8 +232,7 @@ pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &Argument
let parameter =
|key: ParameterKey<'_>, schema: &Value| syntax.parameter(key, &cx.value_options(schema));
let schema = cx.root.resolved();
let (properties, additional) = match schema {
match cx.root.resolved() {
Value::Object(schema) => {
let additional = match schema.get("additionalProperties") {
Some(Value::Bool(false)) => None,
@@ -206,22 +247,33 @@ pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &Argument
.and_then(Value::as_array)
.map(|required| required.iter().filter_map(Value::as_str).collect::<Vec<_>>())
.unwrap_or_default();
let properties = schema
let declared = schema
.get("properties")
.and_then(Value::as_object)
.into_iter()
.flatten()
.filter_map(|(key, schema)| {
let parameter = parameter(ParameterKey::Declared(key), schema)?;
Some((parameter, required.contains(&key.as_str())))
Some(Parameter {
key: key.clone(),
format: parameter(ParameterKey::Declared(key), schema)?,
required: required.contains(&key.as_str()),
})
})
.collect();
(properties, additional)
Parameters {
declared,
additional,
}
}
Value::Bool(false) => (vec![], None),
_ => (vec![], parameter(ParameterKey::Free, &ANY_SCHEMA)),
};
parameter_list(options, syntax.separator(), properties, additional)
Value::Bool(false) => Parameters {
declared: vec![],
additional: None,
},
_ => Parameters {
declared: vec![],
additional: parameter(ParameterKey::Free, &ANY_SCHEMA),
},
}
}
/// Parameters in declared order, required ones mandatory, followed by any
@@ -230,8 +282,10 @@ pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &Argument
fn parameter_list(
options: &ArgumentOptions,
separator: Option<Format>,
properties: Vec<(Format, bool)>,
additional: Option<Format>,
Parameters {
declared,
additional,
}: Parameters,
) -> Format {
let separated = |parameter: Format| match &separator {
Some(separator) => Format::sequence(vec![parameter, separator.clone()]),
@@ -239,10 +293,10 @@ fn parameter_list(
};
let mut elements = Vec::from_iter(separator.clone());
if options.any_order {
let any_required = properties.iter().any(|(_, required)| *required);
let parameters = properties
let any_required = declared.iter().any(|parameter| parameter.required);
let parameters = declared
.into_iter()
.map(|(parameter, _)| parameter)
.map(|parameter| parameter.format)
.chain(additional)
.collect::<Vec<_>>();
if !parameters.is_empty() {
@@ -254,8 +308,11 @@ fn parameter_list(
});
}
} else {
for (parameter, required) in properties {
let parameter = separated(parameter);
for Parameter {
format, required, ..
} in declared
{
let parameter = separated(format);
elements.push(if required {
parameter
} else {
+543
View File
@@ -0,0 +1,543 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Unified parser for Beam checkpoints.
//!
//! Beam output is parsed by [`HfUnifiedParser`] with the checkpoint's
//! `response_template`; this parser adds Beam's output grammar.
//! A Beam message after `<|start|>assistant` is:
//!
//! ```text
//! <|channel|>analysis<|message|>REASONING<|end|><|start|>assistant
//! ( <|channel|>final<|message|>ANSWER<|return|>
//! | CALL (<|end|><|start|>assistant CALL)* <|call|> )
//! ```
//!
//! where reasoning may end at any closer of the template's reasoning field,
//! and `CALL` is
//! ` to=NAME<|channel|>analysis <|constrain|>xml<|message|>` followed by
//! `<rfl:parameter name="KEY">\nVALUE\n</rfl:parameter>` parameters separated
//! by newlines. Where the template looks ahead for the next parameter, a value
//! may also end on a newline right before it.
//!
//! The grammar enforces the schema of every declared tool's arguments, in
//! declared or alphabetical order, since Beam drifts from schemas it does not
//! enforce. When the model may answer instead, calls may also name any other
//! tool with any arguments, which the parser passes on so a harness can tell
//! the model the tool does not exist. `ANSWER` follows the request's structured
//! output, if any.
//!
//! The chat template rejects a history turn without reasoning, so a message the
//! model opens holds non-empty reasoning: an empty one would come back as null
//! reasoning and fail the next request.
//!
//! Unless `VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT` turns it off, the grammar
//! applies to every message, including one the request does not constrain.
use std::collections::BTreeMap;
use std::sync::{Arc, LazyLock};
use serde_json::Value;
use vllm_tokenizer::{DecodedText, DynTokenizer};
use xgrammar_structural_tag::ToolChoice;
use xgrammar_structural_tag::format::{EndBoundary, Format};
use xgrammar_structural_tag::tool::{
BuilderToolChoice, NormalizedToolChoice, ToolChoiceValue, normalize_tool_choice,
};
use super::{
HfPosition, HfUnifiedParser, ResponseTemplate, Result, UnifiedParser, UnifiedParserError,
UnifiedParserOutput,
};
use crate::output_grammar::arguments::{
self, ArgumentOptions, ArgumentSyntax, JsonType, ParameterKey, Parameters, ValueOption, one_of,
};
use crate::output_grammar::{
self, BuiltOutputGrammar, OutputGrammarContext, tool_grammar_applies, tool_params,
};
use crate::tool::Tool;
const ANALYSIS_OPEN: &str = "<|channel|>analysis<|message|>";
const FINAL_OPEN: &str = "<|channel|>final<|message|>";
const END: &str = "<|end|>";
/// Closes one message and opens the next assistant message.
const NEXT_MESSAGE: &str = "<|end|><|start|>assistant";
const RETURN: &str = "<|return|>";
const CALL: &str = "<|call|>";
/// Follows ` to=NAME` in a call.
const CALL_HEADER_END: &str = "<|channel|>analysis <|constrain|>xml<|message|>";
const PARAMETER_OPEN: &str = "<rfl:parameter name=\"";
const PARAMETER_NAME_CLOSE: &str = "\">\n";
const PARAMETER_CLOSE: &str = "\n</rfl:parameter>";
/// Ends a value where the template looks ahead for the next parameter.
const NEXT_PARAMETER: &str = "\n<rfl:parameter name=\"";
/// Ends of a lenient value that another parameter follows: the closer and
/// separator before the next opener, or the opener right after the value.
const NEXT_PARAMETER_ENDS: [&str; 3] = [
"\n</rfl:parameter><rfl:parameter name=\"",
"\n</rfl:parameter>\n<rfl:parameter name=\"",
NEXT_PARAMETER,
];
/// Text that ends the call in the parser, even inside a JSON value.
const CALL_TERMINATORS: [&str; 2] = [END, CALL];
/// Whether every Beam message follows the grammar, even when the request
/// constrains neither tool calls nor the answer. Read from
/// `VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT` like the Python frontend: on unless
/// set to something other than `true` or `1`.
static ALWAYS_CONSTRAIN_OUTPUT: LazyLock<bool> = LazyLock::new(|| {
std::env::var("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT").map_or(true, |value| {
matches!(value.to_lowercase().as_str(), "true" | "1")
})
});
/// Beam parser: [`HfUnifiedParser`] with Beam's output grammar.
pub struct BeamUnifiedParser {
inner: HfUnifiedParser,
/// Whether a value may end where the next parameter opens.
lenient: bool,
/// What ends the reasoning and opens the next message.
reasoning_ends: Vec<String>,
}
impl BeamUnifiedParser {
/// Create a parser for one request from the checkpoint's `response_template`.
pub fn new(template: Arc<ResponseTemplate>, tools: &[Tool], tokenizer: DynTokenizer) -> Self {
Self {
lenient: template.argument_lookahead() == Some(PARAMETER_OPEN),
reasoning_ends: match template.reasoning_closers() {
[] => vec![NEXT_MESSAGE.to_string()],
closers => closers.to_vec(),
},
inner: HfUnifiedParser::new(template, tools, tokenizer),
}
}
}
impl UnifiedParser for BeamUnifiedParser {
fn create(_tools: &[Tool], _tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>> {
Err(UnifiedParserError::NoNamedConstructor {
parser: "beam",
built_from: "the checkpoint's response_template",
})
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.inner.initialize(prompt_token_ids)
}
fn preserve_special_tokens(&self) -> bool {
self.inner.preserve_special_tokens()
}
fn build_output_grammar(
&self,
ctx: &OutputGrammarContext<'_>,
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
build_output_grammar(
self.inner.position(),
ctx,
*ALWAYS_CONSTRAIN_OUTPUT,
self.lenient,
&self.reasoning_ends,
)
}
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
self.inner.tool_call_id(tool_index)
}
fn parse_into(&mut self, delta: DecodedText, output: &mut UnifiedParserOutput) -> Result<()> {
self.inner.parse_into(delta, output)
}
fn finish(&mut self) -> Result<UnifiedParserOutput> {
self.inner.finish()
}
fn reset(&mut self) -> String {
self.inner.reset()
}
}
/// The grammar of a Beam message from the first generated token, or `None`
/// when the parser is neither at the start of the message nor inside its
/// reasoning, or when the request constrains nothing and `always_constrain` is
/// off. Values may end where the next parameter opens if `lenient`, and
/// reasoning ends at any of `reasoning_ends`.
fn build_output_grammar(
position: HfPosition,
ctx: &OutputGrammarContext<'_>,
always_constrain: bool,
lenient: bool,
reasoning_ends: &[String],
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
let forced = !ctx.tools.is_empty()
&& matches!(
ctx.tool_choice,
ToolChoice::Value(ToolChoiceValue::Required) | ToolChoice::NamedFunction(_)
);
let answer_format = ctx.answer.filter(|_| !forced);
// The prompt may still list the tools, so only the grammar keeps the model
// from calling them.
let tools_forbidden = !ctx.tools.is_empty()
&& matches!(ctx.tool_choice, ToolChoice::Value(ToolChoiceValue::None));
if !(always_constrain
|| tool_grammar_applies(ctx)
|| tools_forbidden
|| answer_format.is_some())
{
return Ok(None);
}
let text = Format::any_text();
let reasoning_end = EndBoundary::from(reasoning_ends.to_vec());
let reasoning = match position {
HfPosition::ResponseStart => Format::tag(
ANALYSIS_OPEN,
// At least one character, which cannot start Beam's closers.
Format::sequence(vec![Format::regex("[^<]"), text]),
reasoning_end,
),
HfPosition::Reasoning => Format::tag("", text, reasoning_end),
// Inside an answer the prompt opened, only the always-on grammar
// applies, finishing a free answer. A tool choice that forces a call
// cannot be met there, and the request's own constraint applies as is.
HfPosition::Content => {
return Ok(
(always_constrain && !forced && answer_format.is_none()).then(|| {
BuiltOutputGrammar::from_token_zero(Format::sequence(answer_rest(None)))
}),
);
}
HfPosition::Other => return Ok(None),
};
let NormalizedToolChoice {
function_tools,
choice,
..
} = normalize_tool_choice(
&tool_params(ctx.tools, ctx.tool_strict_level),
ctx.tool_choice.clone(),
)?;
if function_tools.is_empty() {
return Ok(match choice {
// `none`, or no tools to call: the model can only answer.
BuilderToolChoice::Auto => {
Some(BuiltOutputGrammar::from_token_zero(Format::sequence(vec![
reasoning,
answer(answer_format),
])))
}
BuilderToolChoice::Required | BuilderToolChoice::Forced => None,
});
}
let mut calls = function_tools
.iter()
.map(|tool| {
let header =
Format::const_string(format!(" to={}{CALL_HEADER_END}", tool.function.name));
let schema = tool.function.parameters.clone().unwrap_or(Value::Bool(true));
Format::sequence(vec![header, arguments(&schema, lenient)])
})
.collect::<Vec<_>>();
if matches!(choice, BuilderToolChoice::Auto) {
let names = ctx.tools.iter().map(|tool| tool.name.as_str()).collect::<Vec<_>>();
let header = Format::sequence(vec![
Format::const_string(" to="),
Format::regex(other_names(&names)),
Format::const_string(CALL_HEADER_END),
]);
calls.push(Format::sequence(vec![
header,
arguments(&Value::Bool(true), lenient),
]));
}
let call = one_of(calls);
let mut calls = vec![call.clone()];
if ctx.parallel_tool_calls && !choice.is_forced() {
calls.push(Format::star(Format::sequence(vec![
Format::const_string(NEXT_MESSAGE),
call,
])));
}
// The last call ends with the `<|call|>` stop token.
calls.push(Format::optional(Format::const_string(CALL)));
let calls = Format::sequence(calls);
let output = match choice {
BuilderToolChoice::Auto => Format::or(vec![calls, answer(answer_format)]),
BuilderToolChoice::Required | BuilderToolChoice::Forced => calls,
};
Ok(Some(BuiltOutputGrammar::from_token_zero(Format::sequence(
vec![reasoning, output],
))))
}
/// A final answer, ending with the `<|return|>` stop token.
fn answer(format: Option<&Format>) -> Format {
Format::sequence([vec![Format::const_string(FINAL_OPEN)], answer_rest(format)].concat())
}
/// The rest of an open final answer, holding `format` or else free text.
fn answer_rest(format: Option<&Format>) -> Vec<Format> {
vec![
// Without `<|end|>`, no unconstrained message can follow the answer.
format.map_or_else(|| Format::any_text_excluding(&[END, RETURN]), Format::clone),
Format::optional(Format::const_string(RETURN)),
]
}
/// The arguments under `schema`: its declared parameters in declared or
/// alphabetical order, required ones mandatory, then any additional ones. If
/// `lenient`, each value but the last may end by opening the next parameter.
fn arguments(schema: &Value, lenient: bool) -> Format {
let options = ArgumentOptions {
excludes: CALL_TERMINATORS.map(str::to_string).to_vec(),
..ArgumentOptions::default()
};
let render = |syntax: BeamArguments| arguments::parameters(schema, &syntax, &options);
if lenient {
lenient_arguments(render(BeamArguments::Joined), render(BeamArguments::Last))
} else {
closed_arguments(render(BeamArguments::Closed))
}
}
/// Arguments whose values all end with a closer.
fn closed_arguments(
Parameters {
declared,
additional,
}: Parameters,
) -> Format {
let additional = additional.map(Format::star);
let lists = orders(&declared, |parameter| &parameter.key)
.into_iter()
.map(|order| {
let listed = order
.into_iter()
.map(|parameter| listed(parameter.format.clone(), parameter.required));
listed.chain(additional.clone()).collect::<Vec<_>>()
})
.filter(|list| !list.is_empty())
.map(Format::sequence)
.collect::<Vec<_>>();
if lists.is_empty() {
Format::const_string("")
} else {
one_of(lists)
}
}
/// A declared parameter as one another parameter follows, `joined`, and as the
/// last one.
struct LenientParameter {
key: String,
joined: Format,
last: Format,
required: bool,
}
impl LenientParameter {
fn joined(&self) -> Format {
listed(self.joined.clone(), self.required)
}
}
/// Arguments whose values end by opening the next parameter, but for the last
/// one, from the parameters as [`BeamArguments::Joined`] and as
/// [`BeamArguments::Last`].
fn lenient_arguments(joined: Parameters, last: Parameters) -> Format {
// Both forms share their terminators, so they keep the same parameters.
let declared = joined
.declared
.into_iter()
.zip(last.declared)
.map(|(joined, last)| LenientParameter {
key: joined.key,
joined: joined.format,
last: last.format,
required: joined.required,
})
.collect::<Vec<_>>();
let additional = joined.additional.zip(last.additional);
let mut lists = vec![];
for order in orders(&declared, |parameter| &parameter.key) {
// The last parameter is a declared one that follows every required
// one, or an additional one.
if !order.is_empty() {
let first = order.iter().rposition(|parameter| parameter.required).unwrap_or(0);
let mut list =
order[..first].iter().map(|parameter| parameter.joined()).collect::<Vec<_>>();
list.push(last_present(&order[first..]));
lists.push(Format::sequence(list));
}
if let Some((joined, last)) = &additional {
let mut list = order.iter().map(|parameter| parameter.joined()).collect::<Vec<_>>();
list.extend([Format::star(joined.clone()), last.clone()]);
lists.push(Format::sequence(list));
}
}
if lists.is_empty() {
return Format::const_string("");
}
// Each parameter starts past the opener, as the value before writes it.
let arguments = Format::sequence(vec![
separator(),
Format::const_string(PARAMETER_OPEN),
one_of(lists),
]);
if declared.iter().any(|parameter| parameter.required) {
arguments
} else {
Format::optional(arguments)
}
}
/// `parameters`, at least one, in order and the last present one in its last
/// form. Only the first may be required.
fn last_present(parameters: &[&LenientParameter]) -> Format {
if let [only] = parameters {
return only.last.clone();
}
// The last one is in the first half, or after all of it. Halving keeps the
// grammar O(n log n) in size and O(log n) deep.
let (head, tail) = parameters.split_at(parameters.len() / 2);
let mut after_head = head.iter().map(|parameter| parameter.joined()).collect::<Vec<_>>();
after_head.push(last_present(tail));
Format::or(vec![last_present(head), Format::sequence(after_head)])
}
/// `parameters` in declared order, and in alphabetical order by `key` if that
/// differs, since Beam often writes them sorted.
fn orders<P>(parameters: &[P], key: impl Fn(&P) -> &str) -> Vec<Vec<&P>> {
let declared = parameters.iter().collect::<Vec<_>>();
let mut sorted = declared.clone();
sorted.sort_by_key(|parameter| key(parameter));
if sorted
.iter()
.map(|parameter| key(parameter))
.eq(declared.iter().map(|parameter| key(parameter)))
{
vec![declared]
} else {
vec![declared, sorted]
}
}
/// `parameter` as listed: optional unless `required`.
fn listed(parameter: Format, required: bool) -> Format {
if required {
parameter
} else {
Format::optional(parameter)
}
}
fn separator() -> Format {
Format::optional(Format::const_string("\n"))
}
/// A regex for the tool names the call header can spell (no whitespace or
/// `<`) besides `names`.
fn other_names(names: &[&str]) -> String {
#[derive(Default)]
struct Node {
children: BTreeMap<char, Node>,
end: bool,
}
fn escape(char: char, meta: &str) -> String {
if meta.contains(char) {
format!("\\{char}")
} else {
char.to_string()
}
}
/// A name character other than `chars`.
fn other<'a>(chars: impl Iterator<Item = &'a char>) -> String {
let chars = chars.map(|&char| escape(char, r"\]^-[")).collect::<String>();
format!(r"[^\s<{chars}]")
}
/// The rest of a name past `node` that ends on none of `names`.
fn rest(node: &Node, first: bool) -> String {
let any = other([].iter());
let mut branches = vec![format!("{}{any}*", other(node.children.keys()))];
branches.extend(node.children.iter().map(|(&char, child)| {
format!("{}{}", escape(char, r".^$*+?{}[]\|()"), rest(child, false))
}));
let optional = if first || node.end { "" } else { "?" };
format!("(?:{}){optional}", branches.join("|"))
}
let mut root = Node::default();
for name in names {
let mut node = &mut root;
for char in name.chars() {
node = node.children.entry(char).or_default();
}
node.end = true;
}
rest(&root, true)
}
/// How one parameter is written: `<rfl:parameter name="KEY">\nVALUE` followed
/// by its end.
///
/// The parser reads every value as text and converts it by the tool schema, so
/// string options are raw text and every other option is JSON.
enum BeamArguments {
/// The whole parameter, ending with `\n</rfl:parameter>`, after an optional
/// newline.
Closed,
/// The parameter past its opener, where the template looks ahead for the
/// next one: it ends by opening the next parameter, also right after the
/// value's newline, so raw text holds no such opener.
Joined,
/// The last parameter past its opener under such a template, ending with
/// `\n</rfl:parameter>`.
Last,
}
impl ArgumentSyntax for BeamArguments {
fn separator(&self) -> Option<Format> {
None
}
fn parameter(&self, key: ParameterKey<'_>, options: &[ValueOption<'_>]) -> Option<Format> {
let terminators: &[&str] = match self {
Self::Closed => &["</rfl:parameter>"],
Self::Joined | Self::Last => &["</rfl:parameter>", NEXT_PARAMETER],
};
let values = options
.iter()
.filter_map(|option| match option.ty {
JsonType::String => option.raw_string(terminators),
_ => Some(option.json()),
})
.collect::<Vec<_>>();
(!values.is_empty()).then(|| {
let value = one_of(values);
match self {
Self::Closed => Format::sequence(vec![
separator(),
key.tag(PARAMETER_OPEN, PARAMETER_NAME_CLOSE, value, PARAMETER_CLOSE),
]),
Self::Joined => key.tag(
"",
PARAMETER_NAME_CLOSE,
value,
NEXT_PARAMETER_ENDS.to_vec(),
),
Self::Last => key.tag("", PARAMETER_NAME_CLOSE, value, PARAMETER_CLOSE),
}
})
}
}
#[cfg(test)]
mod tests;
+897
View File
@@ -0,0 +1,897 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::sync::Arc;
use serde_json::{Value, json};
use vllm_tokenizer::test_utils::TestTokenizer;
use vllm_tokenizer::{DecodedText, Tokenizer as _};
use xgrammar_structural_tag::ToolChoice;
use xgrammar_structural_tag::format::Format;
use super::{BeamUnifiedParser, HfPosition, build_output_grammar, other_names};
use crate::output_grammar::test_utils::outline;
use crate::output_grammar::{GrammarCoverage, OutputGrammarContext, ToolStrictLevel};
use crate::tool::Tool;
use crate::unified::{ResponseTemplate, UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
/// The `tag_pattern` of the checkpoint of [`template`].
const TAG_PATTERN: &str =
r#"<rfl:parameter name="(?P<key>[^"]+)">\n(?P<value>.*?)\n</rfl:parameter>"#;
/// [`TAG_PATTERN`], but a value may also end where the next parameter opens.
const LENIENT_TAG_PATTERN: &str = r#"<rfl:parameter name="(?P<key>[^"]+)">\n(?P<value>.*?)\n(?:</rfl:parameter>|(?=<rfl:parameter name="))"#;
/// The Beam-1.0 `response_template`, with `tag_pattern`.
fn template(tag_pattern: &str) -> Arc<ResponseTemplate> {
template_with(tag_pattern, json!("<|end|><|start|>assistant"))
}
/// [`template`] with `thinking_close` as the reasoning closer.
fn template_with(tag_pattern: &str, thinking_close: Value) -> Arc<ResponseTemplate> {
let template = json!({
"defaults": {"role": "assistant"},
"start_anchor_pattern": r"<\|start\|>assistant",
"fields": {
"thinking": {
"open": "<|channel|>analysis<|message|>",
"close": thinking_close,
"content": "text"
},
"content": {
"open": "<|channel|>final<|message|>",
"close": ["<|return|>", "<|end|>", "<|call|>"],
"content": "text"
},
"tool_calls": {
"open_pattern": r" to=(?P<name>[^\s<]+)<\|channel\|>analysis <\|constrain\|>xml<\|message\|>",
"close": ["<|end|><|start|>assistant", "<|call|>"],
"repeats": true,
"content": "xml-inline",
"content_args": {
"tag_pattern": tag_pattern,
"value_parser": {"name": "text", "args": {"strip": false}}
},
"transform": {
"type": "function",
"function": {"name": "{name}", "arguments": "{content}"}
}
}
}
});
Arc::new(ResponseTemplate::from_json(&template).unwrap())
}
fn tools() -> Vec<Tool> {
vec![
Tool {
name: "get_weather".to_string(),
description: None,
parameters: json!({
"type": "object",
"properties": {
"location": {"type": "string"},
"days": {"type": "integer"},
"unit": {"enum": ["c", "f"]}
},
"required": ["location"],
"additionalProperties": false
}),
strict: Some(true),
defer_loading: None,
},
Tool {
name: "get_time".to_string(),
description: None,
parameters: json!({
"type": "object",
"properties": {"zone": {"type": "string"}}
}),
strict: None,
defer_loading: None,
},
]
}
/// A parser initialized from `prompt`, the rendered conversation.
fn parser(prompt: &str) -> BeamUnifiedParser {
parser_with(TAG_PATTERN, prompt)
}
fn parser_with(tag_pattern: &str, prompt: &str) -> BeamUnifiedParser {
let tokenizer = Arc::new(TestTokenizer::new());
let prompt_ids = tokenizer.encode(prompt, false).unwrap();
let mut parser = BeamUnifiedParser::new(template(tag_pattern), &tools(), tokenizer);
parser.initialize(&prompt_ids).unwrap();
parser
}
const PROMPT: &str = "<|start|>user<|message|>Weather in Paris?<|end|><|start|>assistant";
fn grammar(position: HfPosition, tool_choice: ToolChoice, parallel: bool) -> Option<String> {
grammar_with(position, &tools(), tool_choice, parallel, false, false)
}
fn grammar_with(
position: HfPosition,
tools: &[Tool],
tool_choice: ToolChoice,
parallel: bool,
always_constrain: bool,
lenient: bool,
) -> Option<String> {
let ctx = OutputGrammarContext {
tools,
tool_choice: &tool_choice,
tool_strict_level: ToolStrictLevel::Auto,
parallel_tool_calls: parallel,
answer: None,
};
grammar_in(position, &ctx, always_constrain, lenient)
}
fn grammar_in(
position: HfPosition,
ctx: &OutputGrammarContext<'_>,
always_constrain: bool,
lenient: bool,
) -> Option<String> {
let grammar = build_output_grammar(
position,
ctx,
always_constrain,
lenient,
template(TAG_PATTERN).reasoning_closers(),
)
.unwrap()?;
assert_eq!(grammar.coverage, GrammarCoverage::FromTokenZero);
Some(outline(&grammar.format))
}
fn constrained_grammar(
position: HfPosition,
tool_choice: ToolChoice,
answer: Format,
always_constrain: bool,
) -> Option<String> {
let tools = tools();
let ctx = OutputGrammarContext {
tools: &tools,
tool_choice: &tool_choice,
tool_strict_level: ToolStrictLevel::Auto,
parallel_tool_calls: true,
answer: Some(&answer),
};
grammar_in(position, &ctx, always_constrain, false)
}
#[test]
fn position_follows_the_prompt_and_then_the_output() {
let position = |suffix: &str| parser(&format!("{PROMPT}{suffix}")).inner.position();
assert_eq!(position(""), HfPosition::ResponseStart);
assert_eq!(
position("<|channel|>analysis<|message|>Let me"),
HfPosition::Reasoning
);
// The reasoning closer ends with the anchor, so the next message starts.
assert_eq!(
position("<|channel|>analysis<|message|>Done.<|end|><|start|>assistant"),
HfPosition::ResponseStart
);
// A partial closer stays buffered.
assert_eq!(
position("<|channel|>analysis<|message|>Done.<|end|>"),
HfPosition::Other
);
assert_eq!(
position("<|channel|>final<|message|>It is"),
HfPosition::Content
);
// A partial boundary stays buffered.
assert_eq!(position("<|channel|>ana"), HfPosition::Other);
// Without the anchor, the parser parses the output from the start.
assert_eq!(
parser("no anchor").inner.position(),
HfPosition::ResponseStart
);
let mut parser = parser(PROMPT);
parser
.parse_into(
DecodedText::unattributed("<|channel|>analysis<|message|>Hmm"),
&mut UnifiedParserOutput::default(),
)
.unwrap();
assert_eq!(parser.inner.position(), HfPosition::Reasoning);
}
#[test]
fn required_calls_any_tool_after_reasoning() {
expect_test::expect![[r#"
sequence
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
sequence
/[^<]/
text
sequence
or
sequence
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
or
sequence
sequence
optional `\n`
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
or
`c`
`f`
sequence
optional
sequence
optional `\n`
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
sequence
optional `\n`
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
or
`c`
`f`
sequence
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
sequence
optional
sequence
optional `\n`
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
star
sequence
`<|end|><|start|>assistant`
or
sequence
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
or
sequence
sequence
optional `\n`
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
or
`c`
`f`
sequence
optional
sequence
optional `\n`
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
sequence
optional `\n`
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
or
`c`
`f`
sequence
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
sequence
optional
sequence
optional `\n`
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
optional `<|call|>`
"#]].assert_eq(
&grammar(HfPosition::ResponseStart, ToolChoice::required(), true).unwrap(),
);
}
#[test]
fn auto_with_a_strict_tool_also_allows_an_answer() {
expect_test::expect![[r#"
sequence
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
sequence
/[^<]/
text
or
sequence
or
sequence
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
or
sequence
sequence
optional `\n`
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
or
`c`
`f`
sequence
optional
sequence
optional `\n`
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
sequence
optional `\n`
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
or
`c`
`f`
sequence
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
sequence
optional
sequence
optional `\n`
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
sequence
sequence
` to=`
/(?:[^\s<g][^\s<]*|g(?:[^\s<e][^\s<]*|e(?:[^\s<t][^\s<]*|t(?:[^\s<_][^\s<]*|_(?:[^\s<tw][^\s<]*|t(?:[^\s<i][^\s<]*|i(?:[^\s<m][^\s<]*|m(?:[^\s<e][^\s<]*|e(?:[^\s<][^\s<]*))?)?)?|w(?:[^\s<e][^\s<]*|e(?:[^\s<a][^\s<]*|a(?:[^\s<t][^\s<]*|t(?:[^\s<h][^\s<]*|h(?:[^\s<e][^\s<]*|e(?:[^\s<r][^\s<]*|r(?:[^\s<][^\s<]*))?)?)?)?)?)?)?)?)?)?)/
`<|channel|>analysis <|constrain|>xml<|message|>`
sequence
star
sequence
optional `\n`
sequence
`<rfl:parameter name="`
/[a-zA-Z_][a-zA-Z0-9_]*/
`">\n`
or
text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`]
json(number) excluding [`<|end|>`, `<|call|>`]
json(boolean) excluding [`<|end|>`, `<|call|>`]
json(null) excluding [`<|end|>`, `<|call|>`]
json({ ... }) excluding [`<|end|>`, `<|call|>`]
json(any[]) excluding [`<|end|>`, `<|call|>`]
`\n</rfl:parameter>`
star
sequence
`<|end|><|start|>assistant`
or
sequence
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
or
sequence
sequence
optional `\n`
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
or
`c`
`f`
sequence
optional
sequence
optional `\n`
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
sequence
optional `\n`
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
or
`c`
`f`
sequence
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
sequence
optional
sequence
optional `\n`
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
sequence
sequence
` to=`
/(?:[^\s<g][^\s<]*|g(?:[^\s<e][^\s<]*|e(?:[^\s<t][^\s<]*|t(?:[^\s<_][^\s<]*|_(?:[^\s<tw][^\s<]*|t(?:[^\s<i][^\s<]*|i(?:[^\s<m][^\s<]*|m(?:[^\s<e][^\s<]*|e(?:[^\s<][^\s<]*))?)?)?|w(?:[^\s<e][^\s<]*|e(?:[^\s<a][^\s<]*|a(?:[^\s<t][^\s<]*|t(?:[^\s<h][^\s<]*|h(?:[^\s<e][^\s<]*|e(?:[^\s<r][^\s<]*|r(?:[^\s<][^\s<]*))?)?)?)?)?)?)?)?)?)?)/
`<|channel|>analysis <|constrain|>xml<|message|>`
sequence
star
sequence
optional `\n`
sequence
`<rfl:parameter name="`
/[a-zA-Z_][a-zA-Z0-9_]*/
`">\n`
or
text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`]
json(number) excluding [`<|end|>`, `<|call|>`]
json(boolean) excluding [`<|end|>`, `<|call|>`]
json(null) excluding [`<|end|>`, `<|call|>`]
json({ ... }) excluding [`<|end|>`, `<|call|>`]
json(any[]) excluding [`<|end|>`, `<|call|>`]
`\n</rfl:parameter>`
optional `<|call|>`
sequence
`<|channel|>final<|message|>`
text excluding [`<|end|>`, `<|return|>`]
optional `<|return|>`
"#]]
.assert_eq(&grammar(HfPosition::ResponseStart, ToolChoice::auto(), true).unwrap());
}
#[test]
fn named_choice_calls_that_tool_once_after_prompted_reasoning() {
expect_test::expect![[r#"
sequence
tag `` text `<|end|><|start|>assistant`
sequence
sequence
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
or
sequence
sequence
optional `\n`
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
or
`c`
`f`
sequence
optional
sequence
optional `\n`
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
sequence
optional `\n`
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
optional
sequence
optional `\n`
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
or
`c`
`f`
optional `<|call|>`
"#]].assert_eq(
&grammar(HfPosition::Reasoning, ToolChoice::function("get_weather"), true).unwrap(),
);
}
#[test]
fn no_grammar_without_a_constraining_request_or_known_position() {
let non_strict = vec![tools().pop().unwrap()];
for tools in [&[][..], &non_strict] {
assert!(
grammar_with(
HfPosition::ResponseStart,
tools,
ToolChoice::auto(),
true,
false,
false
)
.is_none()
);
}
assert!(
grammar_with(
HfPosition::Other,
&tools(),
ToolChoice::required(),
true,
true,
false
)
.is_none()
);
}
#[test]
fn none_with_tools_only_answers() {
expect_test::expect![[r#"
sequence
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
sequence
/[^<]/
text
sequence
`<|channel|>final<|message|>`
text excluding [`<|end|>`, `<|return|>`]
optional `<|return|>`
"#]]
.assert_eq(&grammar(HfPosition::ResponseStart, ToolChoice::none(), true).unwrap());
}
#[test]
fn always_constrain_answers_without_tools() {
expect_test::expect![[r#"
sequence
tag `` text `<|end|><|start|>assistant`
sequence
`<|channel|>final<|message|>`
text excluding [`<|end|>`, `<|return|>`]
optional `<|return|>`
"#]]
.assert_eq(
&grammar_with(
HfPosition::Reasoning,
&[],
ToolChoice::auto(),
true,
true,
false,
)
.unwrap(),
);
}
#[test]
fn always_constrain_allows_non_strict_calls_or_an_answer() {
let non_strict = vec![tools().pop().unwrap()];
expect_test::expect![[r#"
sequence
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
sequence
/[^<]/
text
or
sequence
or
sequence
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
sequence
optional
sequence
optional `\n`
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
sequence
sequence
` to=`
/(?:[^\s<g][^\s<]*|g(?:[^\s<e][^\s<]*|e(?:[^\s<t][^\s<]*|t(?:[^\s<_][^\s<]*|_(?:[^\s<t][^\s<]*|t(?:[^\s<i][^\s<]*|i(?:[^\s<m][^\s<]*|m(?:[^\s<e][^\s<]*|e(?:[^\s<][^\s<]*))?)?)?)?)?)?)?)/
`<|channel|>analysis <|constrain|>xml<|message|>`
sequence
star
sequence
optional `\n`
sequence
`<rfl:parameter name="`
/[a-zA-Z_][a-zA-Z0-9_]*/
`">\n`
or
text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`]
json(number) excluding [`<|end|>`, `<|call|>`]
json(boolean) excluding [`<|end|>`, `<|call|>`]
json(null) excluding [`<|end|>`, `<|call|>`]
json({ ... }) excluding [`<|end|>`, `<|call|>`]
json(any[]) excluding [`<|end|>`, `<|call|>`]
`\n</rfl:parameter>`
optional `<|call|>`
sequence
`<|channel|>final<|message|>`
text excluding [`<|end|>`, `<|return|>`]
optional `<|return|>`
"#]]
.assert_eq(
&grammar_with(
HfPosition::ResponseStart,
&non_strict,
ToolChoice::auto(),
false,
true,
false,
)
.unwrap(),
);
}
#[test]
fn always_constrain_finishes_an_open_answer() {
expect_test::expect![[r#"
sequence
text excluding [`<|end|>`, `<|return|>`]
optional `<|return|>`
"#]]
.assert_eq(
&grammar_with(
HfPosition::Content,
&tools(),
ToolChoice::auto(),
true,
true,
false,
)
.unwrap(),
);
assert!(
grammar_with(
HfPosition::Content,
&tools(),
ToolChoice::auto(),
true,
false,
false
)
.is_none()
);
assert!(
grammar_with(
HfPosition::Content,
&tools(),
ToolChoice::required(),
true,
true,
false
)
.is_none()
);
}
#[test]
fn answer_holds_the_requested_format() {
let json = || Format::json_schema(json!({"type": "object"}));
let answer = constrained_grammar(HfPosition::ResponseStart, ToolChoice::none(), json(), false);
expect_test::expect![[r#"
sequence
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
sequence
/[^<]/
text
sequence
`<|channel|>final<|message|>`
json(object)
optional `<|return|>`
"#]]
.assert_eq(&answer.unwrap());
// Under `auto`, the answer that may replace calls holds it too.
let auto =
constrained_grammar(HfPosition::ResponseStart, ToolChoice::auto(), json(), false).unwrap();
assert!(auto.ends_with(
"`<|channel|>final<|message|>`\n json(object)\n optional `<|return|>`\n"
));
// A forced call leaves no answer to constrain.
assert_eq!(
constrained_grammar(
HfPosition::ResponseStart,
ToolChoice::required(),
json(),
false
),
grammar(HfPosition::ResponseStart, ToolChoice::required(), true)
);
}
#[test]
fn request_constraint_applies_alone_in_an_open_answer() {
let json = Format::json_schema(json!({"type": "object"}));
assert!(constrained_grammar(HfPosition::Content, ToolChoice::auto(), json, true).is_none());
}
/// Parse `output` after the prompt under `tag_pattern`, returning the calls as
/// `(name, arguments)`.
fn calls(tag_pattern: &str, output: &str) -> Vec<(String, Value)> {
let mut parser = parser_with(tag_pattern, PROMPT);
let mut events = UnifiedParserOutput::default();
parser.parse_into(DecodedText::unattributed(output), &mut events).unwrap();
events.append(parser.finish().unwrap());
let mut calls: Vec<(String, String)> = Vec::new();
for event in events.events {
if let UnifiedParserEvent::ToolCall(delta) = event {
if let Some(name) = delta.name {
calls.push((name, String::new()));
}
calls.last_mut().unwrap().1.push_str(&delta.arguments);
}
}
calls
.into_iter()
.map(|(name, arguments)| (name, serde_json::from_str(&arguments).unwrap()))
.collect()
}
/// Output in the grammar's canonical spelling parses back to the values the
/// grammar admitted.
#[test]
fn parser_reads_back_the_grammar_spelling() {
let parameter = |key: &str, value: &str| {
format!("<rfl:parameter name=\"{key}\">\n{value}\n</rfl:parameter>")
};
let call = |name: &str, parameters: &[String]| {
format!(
" to={name}<|channel|>analysis <|constrain|>xml<|message|>{}",
parameters.join("\n")
)
};
let output = format!(
"<|channel|>analysis<|message|>Two tools.<|end|><|start|>assistant{}<|end|><|start|>assistant{}",
call(
"get_weather",
&[
parameter("location", "Paris"),
parameter("days", "2"),
parameter("unit", "c")
]
),
call("get_time", &[parameter("zone", "multi\nline")]),
);
for tag_pattern in [TAG_PATTERN, LENIENT_TAG_PATTERN] {
assert_eq!(
calls(tag_pattern, &output),
vec![
(
"get_weather".to_string(),
json!({"location": "Paris", "days": 2, "unit": "c"})
),
("get_time".to_string(), json!({"zone": "multi\nline"})),
]
);
}
}
#[test]
fn values_end_before_the_next_parameter_where_the_template_looks_ahead() {
let output = " to=get_weather<|channel|>analysis <|constrain|>xml<|message|>\
<rfl:parameter name=\"days\">\n2\n\
<rfl:parameter name=\"location\">\nParis\n</rfl:parameter>";
assert!(parser_with(LENIENT_TAG_PATTERN, PROMPT).lenient);
assert_eq!(
calls(LENIENT_TAG_PATTERN, output),
vec![(
"get_weather".to_string(),
json!({"days": 2, "location": "Paris"})
)]
);
assert!(!parser(PROMPT).lenient);
}
#[test]
fn reasoning_ends_at_every_closer_of_the_template() {
let closers = [
"<|end|><|start|>assistant",
"<|constrain|><|start|>assistant",
"<|start|>assistant",
];
let tools = tools();
let tokenizer = Arc::new(TestTokenizer::new());
let mut parser = BeamUnifiedParser::new(
template_with(TAG_PATTERN, json!(closers)),
&tools,
tokenizer.clone(),
);
parser.initialize(&tokenizer.encode(PROMPT, false).unwrap()).unwrap();
let grammar = parser
.build_output_grammar(&OutputGrammarContext {
tools: &tools,
tool_choice: &ToolChoice::required(),
tool_strict_level: ToolStrictLevel::Auto,
parallel_tool_calls: false,
answer: None,
})
.unwrap()
.unwrap();
assert_eq!(
outline(&grammar.format).lines().nth(1).unwrap(),
" tag `<|channel|>analysis<|message|>` .. `<|constrain|><|start|>assistant` | \
`<|end|><|start|>assistant` | `<|start|>assistant`"
);
}
#[test]
fn lenient_arguments_come_in_declared_or_sorted_order() {
expect_test::expect![[r#"
sequence
tag `` text `<|end|><|start|>assistant`
sequence
sequence
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
sequence
optional `\n`
`<rfl:parameter name="`
or
sequence
or
tag `location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`, `\n<rfl:parameter name="`] `\n</rfl:parameter>`
sequence
tag `location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`, `\n<rfl:parameter name="`] `\n</rfl:parameter><rfl:parameter name="` | `\n</rfl:parameter>\n<rfl:parameter name="` | `\n<rfl:parameter name="`
or
tag `days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
sequence
optional tag `days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter><rfl:parameter name="` | `\n</rfl:parameter>\n<rfl:parameter name="` | `\n<rfl:parameter name="`
tag `unit">\n` .. `\n</rfl:parameter>`
or
`c`
`f`
sequence
optional tag `days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter><rfl:parameter name="` | `\n</rfl:parameter>\n<rfl:parameter name="` | `\n<rfl:parameter name="`
or
tag `location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`, `\n<rfl:parameter name="`] `\n</rfl:parameter>`
sequence
tag `location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`, `\n<rfl:parameter name="`] `\n</rfl:parameter><rfl:parameter name="` | `\n</rfl:parameter>\n<rfl:parameter name="` | `\n<rfl:parameter name="`
tag `unit">\n` .. `\n</rfl:parameter>`
or
`c`
`f`
optional `<|call|>`
"#]].assert_eq(
&grammar_with(
HfPosition::Reasoning,
&tools(),
ToolChoice::function("get_weather"),
true,
false,
true,
)
.unwrap(),
);
}
#[test]
fn other_names_match_every_name_but_the_given_ones() {
let regex = regex_automata::meta::Regex::new(&format!(
"^(?:{})$",
other_names(&["get", "get_time", "a.b"])
))
.unwrap();
for name in [
"g",
"ge",
"gets",
"get_",
"get_t",
"get_times",
"x",
"a",
"a.",
"axb",
"a.bc",
] {
assert!(regex.is_match(name), "{name}");
}
for name in ["get", "get_time", "a.b", "", "get time", "get<"] {
assert!(!regex.is_match(name), "{name}");
}
}
+77 -2
View File
@@ -6,8 +6,12 @@
//! Port of `content_parsers.py`. Each parser takes a chunk of captured text and
//! parses it into a single key in the output message dictionary.
use std::borrow::Cow;
use regex_automata::meta::Regex;
use regex_automata::util::iter::Searcher;
use regex_automata::util::syntax;
use regex_automata::{Input, Match};
use serde::Deserialize;
use serde_json::{Map, Value};
use thiserror_ext::AsReport as _;
@@ -22,6 +26,10 @@ use crate::unified::{self, parsing_failed};
const LAX_OPEN: char = '\u{1}';
const LAX_CLOSE: char = '\u{2}';
/// Name of the group that stands for the lookahead ending an `xml-inline` tag
/// pattern.
const AHEAD_GROUP: &str = "lookahead";
/// A configured content parser (`content` plus `content_args` of one field).
#[derive(Debug, Clone)]
pub(super) enum ContentParser {
@@ -66,6 +74,9 @@ pub(super) struct XmlInlineArgs {
key_group: usize,
/// Index of the `value` group, if the pattern declares one.
value_group: Option<usize>,
/// Index of the group standing for the lookahead that ends the pattern,
/// and the lookahead's own pattern, if any.
lookahead: Option<(usize, String)>,
value_parser: Option<Box<ContentParser>>,
/// Collect duplicate keys into a list.
merge_duplicates: bool,
@@ -143,7 +154,8 @@ impl ContentParser {
}
ContentKind::XmlInline => {
let spec: XmlInlineSpec = deserialize(&args, &context)?;
let tag_pattern = compile_regex(scope, &spec.tag_pattern)?;
let (pattern, lookahead) = final_lookahead(&spec.tag_pattern);
let tag_pattern = compile_regex(scope, &pattern)?;
let group_info = tag_pattern.group_info();
let Some(key_group) = group_info.to_index(Default::default(), "key") else {
return Err(invalid!(
@@ -152,10 +164,14 @@ impl ContentParser {
));
};
let value_group = group_info.to_index(Default::default(), "value");
let lookahead = group_info
.to_index(Default::default(), AHEAD_GROUP)
.zip(lookahead.map(str::to_string));
Self::XmlInline(XmlInlineArgs {
tag_pattern,
key_group,
value_group,
lookahead,
value_parser: ValueParserSpec::compile(scope, spec.value_parser)?,
merge_duplicates: spec.merge_duplicates,
})
@@ -193,6 +209,32 @@ impl ContentParser {
Self::KvLines(args) => args.parse(text),
}
}
/// The pattern of the lookahead that ends an `xml-inline` tag pattern, if
/// any.
pub fn lookahead(&self) -> Option<&str> {
match self {
Self::XmlInline(args) => args.lookahead.as_ref().map(|(_, ahead)| ahead.as_str()),
_ => None,
}
}
}
/// `pattern` with a lookahead that ends it matched as the group
/// [`AHEAD_GROUP`] instead, as the Rust regex engine has no lookahead, and
/// the lookahead's own pattern. The engine rejects any other lookahead.
fn final_lookahead(pattern: &str) -> (Cow<'_, str>, Option<&str>) {
if let Some((head, rest)) = pattern.split_once("(?=") {
// Only closing parentheses may follow a lookahead that ends the pattern.
let ahead = rest.trim_end_matches(')');
if ahead.len() < rest.len() && !ahead.contains(['(', ')']) {
return (
format!("{head}(?P<{AHEAD_GROUP}>{rest}").into(),
Some(ahead),
);
}
}
(pattern.into(), None)
}
impl ValueParserSpec {
@@ -258,7 +300,21 @@ impl XmlInlineArgs {
/// Parse shallow XML-ish tags into a dict.
fn parse(&self, text: &str) -> unified::Result<Value> {
let mut out = Map::new();
for captures in self.tag_pattern.captures_iter(text) {
let mut captures = self.tag_pattern.create_captures();
let mut tags = Searcher::new(Input::new(text));
while tags
.advance(|input| {
self.tag_pattern.search_captures(input, &mut captures);
// The lookahead's text is not part of the tag.
let ahead =
self.lookahead.as_ref().and_then(|(group, _)| captures.get_group(*group));
Ok(captures.get_match().map(|tag| match ahead {
Some(ahead) => Match::new(tag.pattern(), tag.start()..ahead.start),
None => tag,
}))
})
.is_some()
{
let key =
captures.get_group(self.key_group).map(|span| &text[span.range()]).ok_or_else(
|| parsing_failed!("xml-inline: named group 'key' did not participate"),
@@ -490,6 +546,25 @@ mod tests {
);
}
#[test]
fn xml_inline_tags_may_end_where_the_pattern_looks_ahead() {
let xml = parser(
"xml-inline",
json!({
"tag_pattern": r#"<arg name="(?P<key>\w+)">(?P<value>.*?)\n(?:</arg>|(?=<arg name="))"#,
}),
);
let value = xml
.parse("<arg name=\"a\">1\n<arg name=\"b\">x <arg name=\"c\"> y\n</arg>")
.unwrap();
assert_eq!(value, json!({"a": "1", "b": "x <arg name=\"c\"> y"}));
assert_eq!(xml.lookahead(), Some("<arg name=\""));
// Any other lookahead is unsupported.
let args = json!({"tag_pattern": r"(?P<key>\w+)(?=:)(?P<value>.*)"});
assert!(ContentParser::new("test", kind("xml-inline"), args.as_object().unwrap()).is_err());
}
#[test]
fn kv_lines_split_and_strip() {
let kv = parser("kv-lines", json!({"value_parser": {"name": "int"}}));
+1 -1
View File
@@ -25,7 +25,7 @@ mod transform;
#[cfg(test)]
mod tests;
pub use parser::HfUnifiedParser;
pub use parser::{HfPosition, HfUnifiedParser};
pub use template::ResponseTemplate;
use thiserror::Error;
use thiserror_ext::Macro;
+44
View File
@@ -59,6 +59,20 @@ pub struct HfUnifiedParser {
emitted_tool_count: usize,
}
/// Where the parser is in the response; see [`HfUnifiedParser::position`].
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HfPosition {
/// Nothing parsed since the start anchor, or since the start of the stream
/// when there is none.
ResponseStart,
/// Inside an open reasoning field.
Reasoning,
/// Inside an open content field.
Content,
/// Anywhere else.
Other,
}
/// Parser mode: the region currently receiving text.
enum HfMode {
/// No implicit field is declared: text outside regions is discarded.
@@ -156,6 +170,36 @@ impl HfUnifiedParser {
parser
}
/// Where the parser is in the response. Between
/// [`UnifiedParser::initialize`] and the first generated text, this is where
/// the prompt leaves the response.
///
/// Text outside every region, which a template without an implicit field
/// discards, leaves no state behind and so does not move the parser from
/// [`HfPosition::ResponseStart`].
pub fn position(&self) -> HfPosition {
if !self.buffer.text.is_empty() {
return HfPosition::Other;
}
match &self.mode {
HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) if occurrence.opened => {
match &self.template.regions[occurrence.region].kind {
RegionKind::Text(TextRegion {
role: TextRole::Reasoning,
..
}) => HfPosition::Reasoning,
RegionKind::Text(TextRegion {
role: TextRole::Content,
..
}) => HfPosition::Content,
_ => HfPosition::Other,
}
}
_ if self.closed.contains(&true) || self.emitted_tool_count > 0 => HfPosition::Other,
_ => HfPosition::ResponseStart,
}
}
/// Clear all per-stream state.
fn reset_state(&mut self) {
self.buffer.clear();
@@ -191,6 +191,31 @@ impl ResponseTemplate {
})
}
/// The pattern of the lookahead that ends the `xml-inline` tag pattern of
/// tool-call arguments, such as for the opener of the next argument.
pub fn argument_lookahead(&self) -> Option<&str> {
self.regions.iter().find_map(|region| match &region.kind {
RegionKind::ToolCalls(calls) => calls.content.lookahead(),
_ => None,
})
}
/// The literal closers of the reasoning field, longest first, or none if it
/// has no literal close.
pub fn reasoning_closers(&self) -> &[String] {
self.regions
.iter()
.find_map(|region| match (&region.kind, &region.close) {
(RegionKind::Text(text), Some(Boundary::Literals(closers)))
if text.role == TextRole::Reasoning =>
{
Some(closers.as_slice())
}
_ => None,
})
.unwrap_or_default()
}
/// Return the start and end offsets of the last start-anchor match in
/// `prompt`.
pub(super) fn last_start_anchor(&self, prompt: &str) -> Option<(usize, usize)> {
+3 -1
View File
@@ -3,6 +3,7 @@
//! Unified parser interface for reasoning and tool-call deltas.
mod beam;
mod combined;
mod gemma4;
mod hf;
@@ -10,9 +11,10 @@ mod hy;
mod inkling;
mod kimi_k3;
pub use beam::BeamUnifiedParser;
pub use combined::CombinedParser;
pub use gemma4::Gemma4UnifiedParser;
pub use hf::{HfTemplateError, HfUnifiedParser, ResponseTemplate};
pub use hf::{HfPosition, HfTemplateError, HfUnifiedParser, ResponseTemplate};
pub use hy::{HyV3UnifiedParser, HyV4UnifiedParser};
pub use inkling::InklingUnifiedParser;
pub use kimi_k3::KimiK3UnifiedParser;