[Frontend] Add the Beam reasoning and tool parsers (#62052)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com> Co-authored-by: neilkg <33635204+neilkg@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Bugen Zhao
Yoni Gozlan
neilkg
Cursor
Claude Opus 5.5
parent
9731f30a03
commit
153a17166e
@@ -434,7 +434,7 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hf, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_json, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: beam, deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hf, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_json, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -445,6 +445,6 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hf, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: beam, cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hf, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,8 @@ use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEvent
|
||||
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
|
||||
use crate::parser::tool::{ToolParser, ToolParserFactory};
|
||||
use crate::parser::unified::{
|
||||
HfTemplateError, HfUnifiedParser, ResponseTemplate, UnifiedParserFactory, names,
|
||||
BeamUnifiedParser, HfTemplateError, HfUnifiedParser, ResponseTemplate, UnifiedParserFactory,
|
||||
names,
|
||||
};
|
||||
use crate::parser::{ParserSelection, ToolStrictLevel};
|
||||
use crate::request::{ChatRequest, ChatTool};
|
||||
@@ -206,17 +207,19 @@ impl DefaultChatOutputProcessor {
|
||||
});
|
||||
}
|
||||
|
||||
// `hf` is built from the model's template, never by its registry constructor.
|
||||
let parser = if parser_name == names::HF {
|
||||
let template =
|
||||
response_template.as_ref().map_err(|error| Error::ParserInitialization {
|
||||
kind: "unified",
|
||||
name: parser_name.to_string(),
|
||||
error: Box::new(error.clone()),
|
||||
})?;
|
||||
Box::new(HfUnifiedParser::new(template.clone(), tools, tokenizer))
|
||||
} else {
|
||||
factory.create(parser_name, tools, tokenizer)?
|
||||
// Parsers driven by the model's template are built from it, never by
|
||||
// their registry constructors.
|
||||
let template = || {
|
||||
response_template.clone().map_err(|error| Error::ParserInitialization {
|
||||
kind: "unified",
|
||||
name: parser_name.to_string(),
|
||||
error: Box::new(error),
|
||||
})
|
||||
};
|
||||
let parser: Box<dyn UnifiedParser> = match parser_name {
|
||||
names::HF => Box::new(HfUnifiedParser::new(template()?, tools, tokenizer)),
|
||||
names::BEAM => Box::new(BeamUnifiedParser::new(template()?, tools, tokenizer)),
|
||||
_ => factory.create(parser_name, tools, tokenizer)?,
|
||||
};
|
||||
|
||||
UNIFIED_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using unified parser"));
|
||||
@@ -589,6 +592,20 @@ mod tests {
|
||||
.assert_eq(&error(invalid_template()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_beam_uses_the_model_response_template() {
|
||||
let beam = ParserSelection::Explicit("beam".to_string());
|
||||
crate::parser::validate_parser_overrides(&beam, &beam).unwrap();
|
||||
assert!(keeps_special_tokens("other-model", &loaded_template(), &beam, &beam).unwrap());
|
||||
|
||||
let error =
|
||||
keeps_special_tokens("other-model", &Err(HfTemplateError::Missing), &beam, &beam)
|
||||
.unwrap_err()
|
||||
.to_report_string();
|
||||
expect_test::expect!["failed to initialize unified parser `beam`: the model's tokenizer_config.json provides no response_template"]
|
||||
.assert_eq(&error);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_never_selects_hf() {
|
||||
let auto = ParserSelection::Auto;
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub use vllm_parser::unified::{
|
||||
Gemma4UnifiedParser, HfTemplateError, HfUnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser,
|
||||
InklingUnifiedParser, KimiK3UnifiedParser, ResponseTemplate, UnifiedParser,
|
||||
BeamUnifiedParser, Gemma4UnifiedParser, HfTemplateError, HfUnifiedParser, HyV3UnifiedParser,
|
||||
HyV4UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser, ResponseTemplate, UnifiedParser,
|
||||
};
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
@@ -26,6 +26,9 @@ pub mod names {
|
||||
/// name resolution only: it is built from the model's template, never by its
|
||||
/// registry constructor.
|
||||
pub const HF: &str = "hf";
|
||||
/// Beam checkpoints: [`HF`] with Beam's tool-call grammar, built from the
|
||||
/// model's template the same way.
|
||||
pub const BEAM: &str = "beam";
|
||||
}
|
||||
|
||||
/// Constructor signature for one registered unified parser implementation.
|
||||
@@ -55,6 +58,7 @@ impl UnifiedParserFactory {
|
||||
factory.register_parser::<KimiK3UnifiedParser>(names::KIMI_K3);
|
||||
// Opt-in only: no model patterns map to it.
|
||||
factory.register_parser::<HfUnifiedParser>(names::HF);
|
||||
factory.register_parser::<BeamUnifiedParser>(names::BEAM);
|
||||
|
||||
factory
|
||||
.register_pattern("gemma-4", names::GEMMA4)
|
||||
|
||||
@@ -207,7 +207,8 @@ pub(crate) fn full_format_from_builder_for_test(
|
||||
)
|
||||
}
|
||||
|
||||
fn tool_params(tools: &[Tool], strict_level: ToolStrictLevel) -> Vec<ToolParam> {
|
||||
/// Tools as builder inputs, strict when the tool or the server floor asks for it.
|
||||
pub(crate) fn tool_params(tools: &[Tool], strict_level: ToolStrictLevel) -> Vec<ToolParam> {
|
||||
tools
|
||||
.iter()
|
||||
.map(|tool| {
|
||||
@@ -240,7 +241,7 @@ pub(crate) fn normalize_tool_choice(
|
||||
}
|
||||
|
||||
/// Whether the request asks for a tool grammar at all.
|
||||
fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool {
|
||||
pub(crate) fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool {
|
||||
if ctx.tools.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
//! - `allOf` with more than one schema accepts any value.
|
||||
|
||||
use serde_json::{Value, json};
|
||||
use xgrammar_structural_tag::format::{Format, JsonSchemaFormat};
|
||||
use xgrammar_structural_tag::format::{EndBoundary, Format, JsonSchemaFormat, TokenFormat};
|
||||
|
||||
pub use crate::schema::JsonType;
|
||||
use crate::schema::{self, OptionSource, SchemaOption, SchemaRoot};
|
||||
@@ -60,7 +60,13 @@ impl ParameterKey<'_> {
|
||||
|
||||
/// `prefix KEY suffix content end`: a tag for a declared key, or a
|
||||
/// sequence matching [`Self::FREE_PATTERN`] for a free key.
|
||||
pub fn tag(&self, prefix: &str, suffix: &str, content: Format, end: &str) -> Format {
|
||||
pub fn tag(
|
||||
&self,
|
||||
prefix: &str,
|
||||
suffix: &str,
|
||||
content: Format,
|
||||
end: impl Into<EndBoundary>,
|
||||
) -> Format {
|
||||
match self {
|
||||
Self::Declared(key) => Format::tag(format!("{prefix}{key}{suffix}"), content, end),
|
||||
Self::Free => Format::sequence(vec![
|
||||
@@ -68,7 +74,13 @@ impl ParameterKey<'_> {
|
||||
Format::regex(Self::FREE_PATTERN),
|
||||
Format::const_string(suffix),
|
||||
content,
|
||||
Format::const_string(end),
|
||||
match end.into() {
|
||||
EndBoundary::Text(end) => Format::const_string(end),
|
||||
EndBoundary::Texts(ends) => {
|
||||
one_of(ends.into_iter().map(Format::const_string).collect())
|
||||
}
|
||||
EndBoundary::Token(end) => Format::Token(TokenFormat { token: end.token }),
|
||||
},
|
||||
]),
|
||||
}
|
||||
}
|
||||
@@ -183,6 +195,36 @@ impl ArgumentContext<'_> {
|
||||
/// Build the grammar of one call's arguments under `schema`, the tool's
|
||||
/// parameters schema, rendering each parameter with `syntax`.
|
||||
pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &ArgumentOptions) -> Format {
|
||||
parameter_list(
|
||||
options,
|
||||
syntax.separator(),
|
||||
parameters(schema, syntax, options),
|
||||
)
|
||||
}
|
||||
|
||||
/// The parameters of one call, each rendered by an [`ArgumentSyntax`], for a
|
||||
/// syntax that lists them itself rather than through [`arguments`].
|
||||
pub struct Parameters {
|
||||
/// The declared parameters, in declared order.
|
||||
pub declared: Vec<Parameter>,
|
||||
/// Any one additional parameter, if the schema allows them.
|
||||
pub additional: Option<Format>,
|
||||
}
|
||||
|
||||
/// One declared parameter.
|
||||
pub struct Parameter {
|
||||
pub key: String,
|
||||
pub format: Format,
|
||||
pub required: bool,
|
||||
}
|
||||
|
||||
/// The parameters of one call under `schema`, the tool's parameters schema,
|
||||
/// rendering each parameter with `syntax`.
|
||||
pub fn parameters(
|
||||
schema: &Value,
|
||||
syntax: &dyn ArgumentSyntax,
|
||||
options: &ArgumentOptions,
|
||||
) -> Parameters {
|
||||
let cx = ArgumentContext {
|
||||
options,
|
||||
root: SchemaRoot::new(schema),
|
||||
@@ -190,8 +232,7 @@ pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &Argument
|
||||
let parameter =
|
||||
|key: ParameterKey<'_>, schema: &Value| syntax.parameter(key, &cx.value_options(schema));
|
||||
|
||||
let schema = cx.root.resolved();
|
||||
let (properties, additional) = match schema {
|
||||
match cx.root.resolved() {
|
||||
Value::Object(schema) => {
|
||||
let additional = match schema.get("additionalProperties") {
|
||||
Some(Value::Bool(false)) => None,
|
||||
@@ -206,22 +247,33 @@ pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &Argument
|
||||
.and_then(Value::as_array)
|
||||
.map(|required| required.iter().filter_map(Value::as_str).collect::<Vec<_>>())
|
||||
.unwrap_or_default();
|
||||
let properties = schema
|
||||
let declared = schema
|
||||
.get("properties")
|
||||
.and_then(Value::as_object)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(|(key, schema)| {
|
||||
let parameter = parameter(ParameterKey::Declared(key), schema)?;
|
||||
Some((parameter, required.contains(&key.as_str())))
|
||||
Some(Parameter {
|
||||
key: key.clone(),
|
||||
format: parameter(ParameterKey::Declared(key), schema)?,
|
||||
required: required.contains(&key.as_str()),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
(properties, additional)
|
||||
Parameters {
|
||||
declared,
|
||||
additional,
|
||||
}
|
||||
}
|
||||
Value::Bool(false) => (vec![], None),
|
||||
_ => (vec![], parameter(ParameterKey::Free, &ANY_SCHEMA)),
|
||||
};
|
||||
parameter_list(options, syntax.separator(), properties, additional)
|
||||
Value::Bool(false) => Parameters {
|
||||
declared: vec![],
|
||||
additional: None,
|
||||
},
|
||||
_ => Parameters {
|
||||
declared: vec![],
|
||||
additional: parameter(ParameterKey::Free, &ANY_SCHEMA),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Parameters in declared order, required ones mandatory, followed by any
|
||||
@@ -230,8 +282,10 @@ pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &Argument
|
||||
fn parameter_list(
|
||||
options: &ArgumentOptions,
|
||||
separator: Option<Format>,
|
||||
properties: Vec<(Format, bool)>,
|
||||
additional: Option<Format>,
|
||||
Parameters {
|
||||
declared,
|
||||
additional,
|
||||
}: Parameters,
|
||||
) -> Format {
|
||||
let separated = |parameter: Format| match &separator {
|
||||
Some(separator) => Format::sequence(vec![parameter, separator.clone()]),
|
||||
@@ -239,10 +293,10 @@ fn parameter_list(
|
||||
};
|
||||
let mut elements = Vec::from_iter(separator.clone());
|
||||
if options.any_order {
|
||||
let any_required = properties.iter().any(|(_, required)| *required);
|
||||
let parameters = properties
|
||||
let any_required = declared.iter().any(|parameter| parameter.required);
|
||||
let parameters = declared
|
||||
.into_iter()
|
||||
.map(|(parameter, _)| parameter)
|
||||
.map(|parameter| parameter.format)
|
||||
.chain(additional)
|
||||
.collect::<Vec<_>>();
|
||||
if !parameters.is_empty() {
|
||||
@@ -254,8 +308,11 @@ fn parameter_list(
|
||||
});
|
||||
}
|
||||
} else {
|
||||
for (parameter, required) in properties {
|
||||
let parameter = separated(parameter);
|
||||
for Parameter {
|
||||
format, required, ..
|
||||
} in declared
|
||||
{
|
||||
let parameter = separated(format);
|
||||
elements.push(if required {
|
||||
parameter
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,543 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Unified parser for Beam checkpoints.
|
||||
//!
|
||||
//! Beam output is parsed by [`HfUnifiedParser`] with the checkpoint's
|
||||
//! `response_template`; this parser adds Beam's output grammar.
|
||||
//! A Beam message after `<|start|>assistant` is:
|
||||
//!
|
||||
//! ```text
|
||||
//! <|channel|>analysis<|message|>REASONING<|end|><|start|>assistant
|
||||
//! ( <|channel|>final<|message|>ANSWER<|return|>
|
||||
//! | CALL (<|end|><|start|>assistant CALL)* <|call|> )
|
||||
//! ```
|
||||
//!
|
||||
//! where reasoning may end at any closer of the template's reasoning field,
|
||||
//! and `CALL` is
|
||||
//! ` to=NAME<|channel|>analysis <|constrain|>xml<|message|>` followed by
|
||||
//! `<rfl:parameter name="KEY">\nVALUE\n</rfl:parameter>` parameters separated
|
||||
//! by newlines. Where the template looks ahead for the next parameter, a value
|
||||
//! may also end on a newline right before it.
|
||||
//!
|
||||
//! The grammar enforces the schema of every declared tool's arguments, in
|
||||
//! declared or alphabetical order, since Beam drifts from schemas it does not
|
||||
//! enforce. When the model may answer instead, calls may also name any other
|
||||
//! tool with any arguments, which the parser passes on so a harness can tell
|
||||
//! the model the tool does not exist. `ANSWER` follows the request's structured
|
||||
//! output, if any.
|
||||
//!
|
||||
//! The chat template rejects a history turn without reasoning, so a message the
|
||||
//! model opens holds non-empty reasoning: an empty one would come back as null
|
||||
//! reasoning and fail the next request.
|
||||
//!
|
||||
//! Unless `VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT` turns it off, the grammar
|
||||
//! applies to every message, including one the request does not constrain.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
|
||||
use serde_json::Value;
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer};
|
||||
use xgrammar_structural_tag::ToolChoice;
|
||||
use xgrammar_structural_tag::format::{EndBoundary, Format};
|
||||
use xgrammar_structural_tag::tool::{
|
||||
BuilderToolChoice, NormalizedToolChoice, ToolChoiceValue, normalize_tool_choice,
|
||||
};
|
||||
|
||||
use super::{
|
||||
HfPosition, HfUnifiedParser, ResponseTemplate, Result, UnifiedParser, UnifiedParserError,
|
||||
UnifiedParserOutput,
|
||||
};
|
||||
use crate::output_grammar::arguments::{
|
||||
self, ArgumentOptions, ArgumentSyntax, JsonType, ParameterKey, Parameters, ValueOption, one_of,
|
||||
};
|
||||
use crate::output_grammar::{
|
||||
self, BuiltOutputGrammar, OutputGrammarContext, tool_grammar_applies, tool_params,
|
||||
};
|
||||
use crate::tool::Tool;
|
||||
|
||||
const ANALYSIS_OPEN: &str = "<|channel|>analysis<|message|>";
|
||||
const FINAL_OPEN: &str = "<|channel|>final<|message|>";
|
||||
const END: &str = "<|end|>";
|
||||
/// Closes one message and opens the next assistant message.
|
||||
const NEXT_MESSAGE: &str = "<|end|><|start|>assistant";
|
||||
const RETURN: &str = "<|return|>";
|
||||
const CALL: &str = "<|call|>";
|
||||
/// Follows ` to=NAME` in a call.
|
||||
const CALL_HEADER_END: &str = "<|channel|>analysis <|constrain|>xml<|message|>";
|
||||
const PARAMETER_OPEN: &str = "<rfl:parameter name=\"";
|
||||
const PARAMETER_NAME_CLOSE: &str = "\">\n";
|
||||
const PARAMETER_CLOSE: &str = "\n</rfl:parameter>";
|
||||
/// Ends a value where the template looks ahead for the next parameter.
|
||||
const NEXT_PARAMETER: &str = "\n<rfl:parameter name=\"";
|
||||
/// Ends of a lenient value that another parameter follows: the closer and
|
||||
/// separator before the next opener, or the opener right after the value.
|
||||
const NEXT_PARAMETER_ENDS: [&str; 3] = [
|
||||
"\n</rfl:parameter><rfl:parameter name=\"",
|
||||
"\n</rfl:parameter>\n<rfl:parameter name=\"",
|
||||
NEXT_PARAMETER,
|
||||
];
|
||||
/// Text that ends the call in the parser, even inside a JSON value.
|
||||
const CALL_TERMINATORS: [&str; 2] = [END, CALL];
|
||||
|
||||
/// Whether every Beam message follows the grammar, even when the request
|
||||
/// constrains neither tool calls nor the answer. Read from
|
||||
/// `VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT` like the Python frontend: on unless
|
||||
/// set to something other than `true` or `1`.
|
||||
static ALWAYS_CONSTRAIN_OUTPUT: LazyLock<bool> = LazyLock::new(|| {
|
||||
std::env::var("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT").map_or(true, |value| {
|
||||
matches!(value.to_lowercase().as_str(), "true" | "1")
|
||||
})
|
||||
});
|
||||
|
||||
/// Beam parser: [`HfUnifiedParser`] with Beam's output grammar.
|
||||
pub struct BeamUnifiedParser {
|
||||
inner: HfUnifiedParser,
|
||||
/// Whether a value may end where the next parameter opens.
|
||||
lenient: bool,
|
||||
/// What ends the reasoning and opens the next message.
|
||||
reasoning_ends: Vec<String>,
|
||||
}
|
||||
|
||||
impl BeamUnifiedParser {
|
||||
/// Create a parser for one request from the checkpoint's `response_template`.
|
||||
pub fn new(template: Arc<ResponseTemplate>, tools: &[Tool], tokenizer: DynTokenizer) -> Self {
|
||||
Self {
|
||||
lenient: template.argument_lookahead() == Some(PARAMETER_OPEN),
|
||||
reasoning_ends: match template.reasoning_closers() {
|
||||
[] => vec![NEXT_MESSAGE.to_string()],
|
||||
closers => closers.to_vec(),
|
||||
},
|
||||
inner: HfUnifiedParser::new(template, tools, tokenizer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl UnifiedParser for BeamUnifiedParser {
|
||||
fn create(_tools: &[Tool], _tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>> {
|
||||
Err(UnifiedParserError::NoNamedConstructor {
|
||||
parser: "beam",
|
||||
built_from: "the checkpoint's response_template",
|
||||
})
|
||||
}
|
||||
|
||||
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
|
||||
self.inner.initialize(prompt_token_ids)
|
||||
}
|
||||
|
||||
fn preserve_special_tokens(&self) -> bool {
|
||||
self.inner.preserve_special_tokens()
|
||||
}
|
||||
|
||||
fn build_output_grammar(
|
||||
&self,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
|
||||
build_output_grammar(
|
||||
self.inner.position(),
|
||||
ctx,
|
||||
*ALWAYS_CONSTRAIN_OUTPUT,
|
||||
self.lenient,
|
||||
&self.reasoning_ends,
|
||||
)
|
||||
}
|
||||
|
||||
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
|
||||
self.inner.tool_call_id(tool_index)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, delta: DecodedText, output: &mut UnifiedParserOutput) -> Result<()> {
|
||||
self.inner.parse_into(delta, output)
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Result<UnifiedParserOutput> {
|
||||
self.inner.finish()
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> String {
|
||||
self.inner.reset()
|
||||
}
|
||||
}
|
||||
|
||||
/// The grammar of a Beam message from the first generated token, or `None`
|
||||
/// when the parser is neither at the start of the message nor inside its
|
||||
/// reasoning, or when the request constrains nothing and `always_constrain` is
|
||||
/// off. Values may end where the next parameter opens if `lenient`, and
|
||||
/// reasoning ends at any of `reasoning_ends`.
|
||||
fn build_output_grammar(
|
||||
position: HfPosition,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
always_constrain: bool,
|
||||
lenient: bool,
|
||||
reasoning_ends: &[String],
|
||||
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
|
||||
let forced = !ctx.tools.is_empty()
|
||||
&& matches!(
|
||||
ctx.tool_choice,
|
||||
ToolChoice::Value(ToolChoiceValue::Required) | ToolChoice::NamedFunction(_)
|
||||
);
|
||||
let answer_format = ctx.answer.filter(|_| !forced);
|
||||
// The prompt may still list the tools, so only the grammar keeps the model
|
||||
// from calling them.
|
||||
let tools_forbidden = !ctx.tools.is_empty()
|
||||
&& matches!(ctx.tool_choice, ToolChoice::Value(ToolChoiceValue::None));
|
||||
if !(always_constrain
|
||||
|| tool_grammar_applies(ctx)
|
||||
|| tools_forbidden
|
||||
|| answer_format.is_some())
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let text = Format::any_text();
|
||||
let reasoning_end = EndBoundary::from(reasoning_ends.to_vec());
|
||||
let reasoning = match position {
|
||||
HfPosition::ResponseStart => Format::tag(
|
||||
ANALYSIS_OPEN,
|
||||
// At least one character, which cannot start Beam's closers.
|
||||
Format::sequence(vec![Format::regex("[^<]"), text]),
|
||||
reasoning_end,
|
||||
),
|
||||
HfPosition::Reasoning => Format::tag("", text, reasoning_end),
|
||||
// Inside an answer the prompt opened, only the always-on grammar
|
||||
// applies, finishing a free answer. A tool choice that forces a call
|
||||
// cannot be met there, and the request's own constraint applies as is.
|
||||
HfPosition::Content => {
|
||||
return Ok(
|
||||
(always_constrain && !forced && answer_format.is_none()).then(|| {
|
||||
BuiltOutputGrammar::from_token_zero(Format::sequence(answer_rest(None)))
|
||||
}),
|
||||
);
|
||||
}
|
||||
HfPosition::Other => return Ok(None),
|
||||
};
|
||||
let NormalizedToolChoice {
|
||||
function_tools,
|
||||
choice,
|
||||
..
|
||||
} = normalize_tool_choice(
|
||||
&tool_params(ctx.tools, ctx.tool_strict_level),
|
||||
ctx.tool_choice.clone(),
|
||||
)?;
|
||||
if function_tools.is_empty() {
|
||||
return Ok(match choice {
|
||||
// `none`, or no tools to call: the model can only answer.
|
||||
BuilderToolChoice::Auto => {
|
||||
Some(BuiltOutputGrammar::from_token_zero(Format::sequence(vec![
|
||||
reasoning,
|
||||
answer(answer_format),
|
||||
])))
|
||||
}
|
||||
BuilderToolChoice::Required | BuilderToolChoice::Forced => None,
|
||||
});
|
||||
}
|
||||
|
||||
let mut calls = function_tools
|
||||
.iter()
|
||||
.map(|tool| {
|
||||
let header =
|
||||
Format::const_string(format!(" to={}{CALL_HEADER_END}", tool.function.name));
|
||||
let schema = tool.function.parameters.clone().unwrap_or(Value::Bool(true));
|
||||
Format::sequence(vec![header, arguments(&schema, lenient)])
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if matches!(choice, BuilderToolChoice::Auto) {
|
||||
let names = ctx.tools.iter().map(|tool| tool.name.as_str()).collect::<Vec<_>>();
|
||||
let header = Format::sequence(vec![
|
||||
Format::const_string(" to="),
|
||||
Format::regex(other_names(&names)),
|
||||
Format::const_string(CALL_HEADER_END),
|
||||
]);
|
||||
calls.push(Format::sequence(vec![
|
||||
header,
|
||||
arguments(&Value::Bool(true), lenient),
|
||||
]));
|
||||
}
|
||||
let call = one_of(calls);
|
||||
let mut calls = vec![call.clone()];
|
||||
if ctx.parallel_tool_calls && !choice.is_forced() {
|
||||
calls.push(Format::star(Format::sequence(vec![
|
||||
Format::const_string(NEXT_MESSAGE),
|
||||
call,
|
||||
])));
|
||||
}
|
||||
// The last call ends with the `<|call|>` stop token.
|
||||
calls.push(Format::optional(Format::const_string(CALL)));
|
||||
let calls = Format::sequence(calls);
|
||||
let output = match choice {
|
||||
BuilderToolChoice::Auto => Format::or(vec![calls, answer(answer_format)]),
|
||||
BuilderToolChoice::Required | BuilderToolChoice::Forced => calls,
|
||||
};
|
||||
Ok(Some(BuiltOutputGrammar::from_token_zero(Format::sequence(
|
||||
vec![reasoning, output],
|
||||
))))
|
||||
}
|
||||
|
||||
/// A final answer, ending with the `<|return|>` stop token.
|
||||
fn answer(format: Option<&Format>) -> Format {
|
||||
Format::sequence([vec![Format::const_string(FINAL_OPEN)], answer_rest(format)].concat())
|
||||
}
|
||||
|
||||
/// The rest of an open final answer, holding `format` or else free text.
|
||||
fn answer_rest(format: Option<&Format>) -> Vec<Format> {
|
||||
vec![
|
||||
// Without `<|end|>`, no unconstrained message can follow the answer.
|
||||
format.map_or_else(|| Format::any_text_excluding(&[END, RETURN]), Format::clone),
|
||||
Format::optional(Format::const_string(RETURN)),
|
||||
]
|
||||
}
|
||||
|
||||
/// The arguments under `schema`: its declared parameters in declared or
|
||||
/// alphabetical order, required ones mandatory, then any additional ones. If
|
||||
/// `lenient`, each value but the last may end by opening the next parameter.
|
||||
fn arguments(schema: &Value, lenient: bool) -> Format {
|
||||
let options = ArgumentOptions {
|
||||
excludes: CALL_TERMINATORS.map(str::to_string).to_vec(),
|
||||
..ArgumentOptions::default()
|
||||
};
|
||||
let render = |syntax: BeamArguments| arguments::parameters(schema, &syntax, &options);
|
||||
if lenient {
|
||||
lenient_arguments(render(BeamArguments::Joined), render(BeamArguments::Last))
|
||||
} else {
|
||||
closed_arguments(render(BeamArguments::Closed))
|
||||
}
|
||||
}
|
||||
|
||||
/// Arguments whose values all end with a closer.
|
||||
fn closed_arguments(
|
||||
Parameters {
|
||||
declared,
|
||||
additional,
|
||||
}: Parameters,
|
||||
) -> Format {
|
||||
let additional = additional.map(Format::star);
|
||||
let lists = orders(&declared, |parameter| ¶meter.key)
|
||||
.into_iter()
|
||||
.map(|order| {
|
||||
let listed = order
|
||||
.into_iter()
|
||||
.map(|parameter| listed(parameter.format.clone(), parameter.required));
|
||||
listed.chain(additional.clone()).collect::<Vec<_>>()
|
||||
})
|
||||
.filter(|list| !list.is_empty())
|
||||
.map(Format::sequence)
|
||||
.collect::<Vec<_>>();
|
||||
if lists.is_empty() {
|
||||
Format::const_string("")
|
||||
} else {
|
||||
one_of(lists)
|
||||
}
|
||||
}
|
||||
|
||||
/// A declared parameter as one another parameter follows, `joined`, and as the
|
||||
/// last one.
|
||||
struct LenientParameter {
|
||||
key: String,
|
||||
joined: Format,
|
||||
last: Format,
|
||||
required: bool,
|
||||
}
|
||||
|
||||
impl LenientParameter {
|
||||
fn joined(&self) -> Format {
|
||||
listed(self.joined.clone(), self.required)
|
||||
}
|
||||
}
|
||||
|
||||
/// Arguments whose values end by opening the next parameter, but for the last
|
||||
/// one, from the parameters as [`BeamArguments::Joined`] and as
|
||||
/// [`BeamArguments::Last`].
|
||||
fn lenient_arguments(joined: Parameters, last: Parameters) -> Format {
|
||||
// Both forms share their terminators, so they keep the same parameters.
|
||||
let declared = joined
|
||||
.declared
|
||||
.into_iter()
|
||||
.zip(last.declared)
|
||||
.map(|(joined, last)| LenientParameter {
|
||||
key: joined.key,
|
||||
joined: joined.format,
|
||||
last: last.format,
|
||||
required: joined.required,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let additional = joined.additional.zip(last.additional);
|
||||
|
||||
let mut lists = vec![];
|
||||
for order in orders(&declared, |parameter| ¶meter.key) {
|
||||
// The last parameter is a declared one that follows every required
|
||||
// one, or an additional one.
|
||||
if !order.is_empty() {
|
||||
let first = order.iter().rposition(|parameter| parameter.required).unwrap_or(0);
|
||||
let mut list =
|
||||
order[..first].iter().map(|parameter| parameter.joined()).collect::<Vec<_>>();
|
||||
list.push(last_present(&order[first..]));
|
||||
lists.push(Format::sequence(list));
|
||||
}
|
||||
if let Some((joined, last)) = &additional {
|
||||
let mut list = order.iter().map(|parameter| parameter.joined()).collect::<Vec<_>>();
|
||||
list.extend([Format::star(joined.clone()), last.clone()]);
|
||||
lists.push(Format::sequence(list));
|
||||
}
|
||||
}
|
||||
if lists.is_empty() {
|
||||
return Format::const_string("");
|
||||
}
|
||||
|
||||
// Each parameter starts past the opener, as the value before writes it.
|
||||
let arguments = Format::sequence(vec![
|
||||
separator(),
|
||||
Format::const_string(PARAMETER_OPEN),
|
||||
one_of(lists),
|
||||
]);
|
||||
if declared.iter().any(|parameter| parameter.required) {
|
||||
arguments
|
||||
} else {
|
||||
Format::optional(arguments)
|
||||
}
|
||||
}
|
||||
|
||||
/// `parameters`, at least one, in order and the last present one in its last
|
||||
/// form. Only the first may be required.
|
||||
fn last_present(parameters: &[&LenientParameter]) -> Format {
|
||||
if let [only] = parameters {
|
||||
return only.last.clone();
|
||||
}
|
||||
// The last one is in the first half, or after all of it. Halving keeps the
|
||||
// grammar O(n log n) in size and O(log n) deep.
|
||||
let (head, tail) = parameters.split_at(parameters.len() / 2);
|
||||
let mut after_head = head.iter().map(|parameter| parameter.joined()).collect::<Vec<_>>();
|
||||
after_head.push(last_present(tail));
|
||||
Format::or(vec![last_present(head), Format::sequence(after_head)])
|
||||
}
|
||||
|
||||
/// `parameters` in declared order, and in alphabetical order by `key` if that
|
||||
/// differs, since Beam often writes them sorted.
|
||||
fn orders<P>(parameters: &[P], key: impl Fn(&P) -> &str) -> Vec<Vec<&P>> {
|
||||
let declared = parameters.iter().collect::<Vec<_>>();
|
||||
let mut sorted = declared.clone();
|
||||
sorted.sort_by_key(|parameter| key(parameter));
|
||||
if sorted
|
||||
.iter()
|
||||
.map(|parameter| key(parameter))
|
||||
.eq(declared.iter().map(|parameter| key(parameter)))
|
||||
{
|
||||
vec![declared]
|
||||
} else {
|
||||
vec![declared, sorted]
|
||||
}
|
||||
}
|
||||
|
||||
/// `parameter` as listed: optional unless `required`.
|
||||
fn listed(parameter: Format, required: bool) -> Format {
|
||||
if required {
|
||||
parameter
|
||||
} else {
|
||||
Format::optional(parameter)
|
||||
}
|
||||
}
|
||||
|
||||
fn separator() -> Format {
|
||||
Format::optional(Format::const_string("\n"))
|
||||
}
|
||||
|
||||
/// A regex for the tool names the call header can spell (no whitespace or
|
||||
/// `<`) besides `names`.
|
||||
fn other_names(names: &[&str]) -> String {
|
||||
#[derive(Default)]
|
||||
struct Node {
|
||||
children: BTreeMap<char, Node>,
|
||||
end: bool,
|
||||
}
|
||||
|
||||
fn escape(char: char, meta: &str) -> String {
|
||||
if meta.contains(char) {
|
||||
format!("\\{char}")
|
||||
} else {
|
||||
char.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// A name character other than `chars`.
|
||||
fn other<'a>(chars: impl Iterator<Item = &'a char>) -> String {
|
||||
let chars = chars.map(|&char| escape(char, r"\]^-[")).collect::<String>();
|
||||
format!(r"[^\s<{chars}]")
|
||||
}
|
||||
|
||||
/// The rest of a name past `node` that ends on none of `names`.
|
||||
fn rest(node: &Node, first: bool) -> String {
|
||||
let any = other([].iter());
|
||||
let mut branches = vec![format!("{}{any}*", other(node.children.keys()))];
|
||||
branches.extend(node.children.iter().map(|(&char, child)| {
|
||||
format!("{}{}", escape(char, r".^$*+?{}[]\|()"), rest(child, false))
|
||||
}));
|
||||
let optional = if first || node.end { "" } else { "?" };
|
||||
format!("(?:{}){optional}", branches.join("|"))
|
||||
}
|
||||
|
||||
let mut root = Node::default();
|
||||
for name in names {
|
||||
let mut node = &mut root;
|
||||
for char in name.chars() {
|
||||
node = node.children.entry(char).or_default();
|
||||
}
|
||||
node.end = true;
|
||||
}
|
||||
rest(&root, true)
|
||||
}
|
||||
|
||||
/// How one parameter is written: `<rfl:parameter name="KEY">\nVALUE` followed
|
||||
/// by its end.
|
||||
///
|
||||
/// The parser reads every value as text and converts it by the tool schema, so
|
||||
/// string options are raw text and every other option is JSON.
|
||||
enum BeamArguments {
|
||||
/// The whole parameter, ending with `\n</rfl:parameter>`, after an optional
|
||||
/// newline.
|
||||
Closed,
|
||||
/// The parameter past its opener, where the template looks ahead for the
|
||||
/// next one: it ends by opening the next parameter, also right after the
|
||||
/// value's newline, so raw text holds no such opener.
|
||||
Joined,
|
||||
/// The last parameter past its opener under such a template, ending with
|
||||
/// `\n</rfl:parameter>`.
|
||||
Last,
|
||||
}
|
||||
|
||||
impl ArgumentSyntax for BeamArguments {
|
||||
fn separator(&self) -> Option<Format> {
|
||||
None
|
||||
}
|
||||
|
||||
fn parameter(&self, key: ParameterKey<'_>, options: &[ValueOption<'_>]) -> Option<Format> {
|
||||
let terminators: &[&str] = match self {
|
||||
Self::Closed => &["</rfl:parameter>"],
|
||||
Self::Joined | Self::Last => &["</rfl:parameter>", NEXT_PARAMETER],
|
||||
};
|
||||
let values = options
|
||||
.iter()
|
||||
.filter_map(|option| match option.ty {
|
||||
JsonType::String => option.raw_string(terminators),
|
||||
_ => Some(option.json()),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
(!values.is_empty()).then(|| {
|
||||
let value = one_of(values);
|
||||
match self {
|
||||
Self::Closed => Format::sequence(vec![
|
||||
separator(),
|
||||
key.tag(PARAMETER_OPEN, PARAMETER_NAME_CLOSE, value, PARAMETER_CLOSE),
|
||||
]),
|
||||
Self::Joined => key.tag(
|
||||
"",
|
||||
PARAMETER_NAME_CLOSE,
|
||||
value,
|
||||
NEXT_PARAMETER_ENDS.to_vec(),
|
||||
),
|
||||
Self::Last => key.tag("", PARAMETER_NAME_CLOSE, value, PARAMETER_CLOSE),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,897 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::{Value, json};
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use vllm_tokenizer::{DecodedText, Tokenizer as _};
|
||||
use xgrammar_structural_tag::ToolChoice;
|
||||
use xgrammar_structural_tag::format::Format;
|
||||
|
||||
use super::{BeamUnifiedParser, HfPosition, build_output_grammar, other_names};
|
||||
use crate::output_grammar::test_utils::outline;
|
||||
use crate::output_grammar::{GrammarCoverage, OutputGrammarContext, ToolStrictLevel};
|
||||
use crate::tool::Tool;
|
||||
use crate::unified::{ResponseTemplate, UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
|
||||
|
||||
/// The `tag_pattern` of the checkpoint of [`template`].
|
||||
const TAG_PATTERN: &str =
|
||||
r#"<rfl:parameter name="(?P<key>[^"]+)">\n(?P<value>.*?)\n</rfl:parameter>"#;
|
||||
/// [`TAG_PATTERN`], but a value may also end where the next parameter opens.
|
||||
const LENIENT_TAG_PATTERN: &str = r#"<rfl:parameter name="(?P<key>[^"]+)">\n(?P<value>.*?)\n(?:</rfl:parameter>|(?=<rfl:parameter name="))"#;
|
||||
|
||||
/// The Beam-1.0 `response_template`, with `tag_pattern`.
|
||||
fn template(tag_pattern: &str) -> Arc<ResponseTemplate> {
|
||||
template_with(tag_pattern, json!("<|end|><|start|>assistant"))
|
||||
}
|
||||
|
||||
/// [`template`] with `thinking_close` as the reasoning closer.
|
||||
fn template_with(tag_pattern: &str, thinking_close: Value) -> Arc<ResponseTemplate> {
|
||||
let template = json!({
|
||||
"defaults": {"role": "assistant"},
|
||||
"start_anchor_pattern": r"<\|start\|>assistant",
|
||||
"fields": {
|
||||
"thinking": {
|
||||
"open": "<|channel|>analysis<|message|>",
|
||||
"close": thinking_close,
|
||||
"content": "text"
|
||||
},
|
||||
"content": {
|
||||
"open": "<|channel|>final<|message|>",
|
||||
"close": ["<|return|>", "<|end|>", "<|call|>"],
|
||||
"content": "text"
|
||||
},
|
||||
"tool_calls": {
|
||||
"open_pattern": r" to=(?P<name>[^\s<]+)<\|channel\|>analysis <\|constrain\|>xml<\|message\|>",
|
||||
"close": ["<|end|><|start|>assistant", "<|call|>"],
|
||||
"repeats": true,
|
||||
"content": "xml-inline",
|
||||
"content_args": {
|
||||
"tag_pattern": tag_pattern,
|
||||
"value_parser": {"name": "text", "args": {"strip": false}}
|
||||
},
|
||||
"transform": {
|
||||
"type": "function",
|
||||
"function": {"name": "{name}", "arguments": "{content}"}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
Arc::new(ResponseTemplate::from_json(&template).unwrap())
|
||||
}
|
||||
|
||||
fn tools() -> Vec<Tool> {
|
||||
vec![
|
||||
Tool {
|
||||
name: "get_weather".to_string(),
|
||||
description: None,
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"},
|
||||
"days": {"type": "integer"},
|
||||
"unit": {"enum": ["c", "f"]}
|
||||
},
|
||||
"required": ["location"],
|
||||
"additionalProperties": false
|
||||
}),
|
||||
strict: Some(true),
|
||||
defer_loading: None,
|
||||
},
|
||||
Tool {
|
||||
name: "get_time".to_string(),
|
||||
description: None,
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {"zone": {"type": "string"}}
|
||||
}),
|
||||
strict: None,
|
||||
defer_loading: None,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/// A parser initialized from `prompt`, the rendered conversation.
|
||||
fn parser(prompt: &str) -> BeamUnifiedParser {
|
||||
parser_with(TAG_PATTERN, prompt)
|
||||
}
|
||||
|
||||
fn parser_with(tag_pattern: &str, prompt: &str) -> BeamUnifiedParser {
|
||||
let tokenizer = Arc::new(TestTokenizer::new());
|
||||
let prompt_ids = tokenizer.encode(prompt, false).unwrap();
|
||||
let mut parser = BeamUnifiedParser::new(template(tag_pattern), &tools(), tokenizer);
|
||||
parser.initialize(&prompt_ids).unwrap();
|
||||
parser
|
||||
}
|
||||
|
||||
const PROMPT: &str = "<|start|>user<|message|>Weather in Paris?<|end|><|start|>assistant";
|
||||
|
||||
fn grammar(position: HfPosition, tool_choice: ToolChoice, parallel: bool) -> Option<String> {
|
||||
grammar_with(position, &tools(), tool_choice, parallel, false, false)
|
||||
}
|
||||
|
||||
fn grammar_with(
|
||||
position: HfPosition,
|
||||
tools: &[Tool],
|
||||
tool_choice: ToolChoice,
|
||||
parallel: bool,
|
||||
always_constrain: bool,
|
||||
lenient: bool,
|
||||
) -> Option<String> {
|
||||
let ctx = OutputGrammarContext {
|
||||
tools,
|
||||
tool_choice: &tool_choice,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
parallel_tool_calls: parallel,
|
||||
answer: None,
|
||||
};
|
||||
grammar_in(position, &ctx, always_constrain, lenient)
|
||||
}
|
||||
|
||||
fn grammar_in(
|
||||
position: HfPosition,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
always_constrain: bool,
|
||||
lenient: bool,
|
||||
) -> Option<String> {
|
||||
let grammar = build_output_grammar(
|
||||
position,
|
||||
ctx,
|
||||
always_constrain,
|
||||
lenient,
|
||||
template(TAG_PATTERN).reasoning_closers(),
|
||||
)
|
||||
.unwrap()?;
|
||||
assert_eq!(grammar.coverage, GrammarCoverage::FromTokenZero);
|
||||
Some(outline(&grammar.format))
|
||||
}
|
||||
|
||||
fn constrained_grammar(
|
||||
position: HfPosition,
|
||||
tool_choice: ToolChoice,
|
||||
answer: Format,
|
||||
always_constrain: bool,
|
||||
) -> Option<String> {
|
||||
let tools = tools();
|
||||
let ctx = OutputGrammarContext {
|
||||
tools: &tools,
|
||||
tool_choice: &tool_choice,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
parallel_tool_calls: true,
|
||||
answer: Some(&answer),
|
||||
};
|
||||
grammar_in(position, &ctx, always_constrain, false)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn position_follows_the_prompt_and_then_the_output() {
|
||||
let position = |suffix: &str| parser(&format!("{PROMPT}{suffix}")).inner.position();
|
||||
|
||||
assert_eq!(position(""), HfPosition::ResponseStart);
|
||||
assert_eq!(
|
||||
position("<|channel|>analysis<|message|>Let me"),
|
||||
HfPosition::Reasoning
|
||||
);
|
||||
// The reasoning closer ends with the anchor, so the next message starts.
|
||||
assert_eq!(
|
||||
position("<|channel|>analysis<|message|>Done.<|end|><|start|>assistant"),
|
||||
HfPosition::ResponseStart
|
||||
);
|
||||
// A partial closer stays buffered.
|
||||
assert_eq!(
|
||||
position("<|channel|>analysis<|message|>Done.<|end|>"),
|
||||
HfPosition::Other
|
||||
);
|
||||
assert_eq!(
|
||||
position("<|channel|>final<|message|>It is"),
|
||||
HfPosition::Content
|
||||
);
|
||||
// A partial boundary stays buffered.
|
||||
assert_eq!(position("<|channel|>ana"), HfPosition::Other);
|
||||
// Without the anchor, the parser parses the output from the start.
|
||||
assert_eq!(
|
||||
parser("no anchor").inner.position(),
|
||||
HfPosition::ResponseStart
|
||||
);
|
||||
|
||||
let mut parser = parser(PROMPT);
|
||||
parser
|
||||
.parse_into(
|
||||
DecodedText::unattributed("<|channel|>analysis<|message|>Hmm"),
|
||||
&mut UnifiedParserOutput::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(parser.inner.position(), HfPosition::Reasoning);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn required_calls_any_tool_after_reasoning() {
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
|
||||
sequence
|
||||
/[^<]/
|
||||
text
|
||||
sequence
|
||||
or
|
||||
sequence
|
||||
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
or
|
||||
sequence
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
star
|
||||
sequence
|
||||
`<|end|><|start|>assistant`
|
||||
or
|
||||
sequence
|
||||
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
or
|
||||
sequence
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional `<|call|>`
|
||||
"#]].assert_eq(
|
||||
&grammar(HfPosition::ResponseStart, ToolChoice::required(), true).unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_with_a_strict_tool_also_allows_an_answer() {
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
|
||||
sequence
|
||||
/[^<]/
|
||||
text
|
||||
or
|
||||
sequence
|
||||
or
|
||||
sequence
|
||||
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
or
|
||||
sequence
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
sequence
|
||||
` to=`
|
||||
/(?:[^\s<g][^\s<]*|g(?:[^\s<e][^\s<]*|e(?:[^\s<t][^\s<]*|t(?:[^\s<_][^\s<]*|_(?:[^\s<tw][^\s<]*|t(?:[^\s<i][^\s<]*|i(?:[^\s<m][^\s<]*|m(?:[^\s<e][^\s<]*|e(?:[^\s<][^\s<]*))?)?)?|w(?:[^\s<e][^\s<]*|e(?:[^\s<a][^\s<]*|a(?:[^\s<t][^\s<]*|t(?:[^\s<h][^\s<]*|h(?:[^\s<e][^\s<]*|e(?:[^\s<r][^\s<]*|r(?:[^\s<][^\s<]*))?)?)?)?)?)?)?)?)?)?)/
|
||||
`<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
star
|
||||
sequence
|
||||
optional `\n`
|
||||
sequence
|
||||
`<rfl:parameter name="`
|
||||
/[a-zA-Z_][a-zA-Z0-9_]*/
|
||||
`">\n`
|
||||
or
|
||||
text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`]
|
||||
json(number) excluding [`<|end|>`, `<|call|>`]
|
||||
json(boolean) excluding [`<|end|>`, `<|call|>`]
|
||||
json(null) excluding [`<|end|>`, `<|call|>`]
|
||||
json({ ... }) excluding [`<|end|>`, `<|call|>`]
|
||||
json(any[]) excluding [`<|end|>`, `<|call|>`]
|
||||
`\n</rfl:parameter>`
|
||||
star
|
||||
sequence
|
||||
`<|end|><|start|>assistant`
|
||||
or
|
||||
sequence
|
||||
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
or
|
||||
sequence
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
sequence
|
||||
` to=`
|
||||
/(?:[^\s<g][^\s<]*|g(?:[^\s<e][^\s<]*|e(?:[^\s<t][^\s<]*|t(?:[^\s<_][^\s<]*|_(?:[^\s<tw][^\s<]*|t(?:[^\s<i][^\s<]*|i(?:[^\s<m][^\s<]*|m(?:[^\s<e][^\s<]*|e(?:[^\s<][^\s<]*))?)?)?|w(?:[^\s<e][^\s<]*|e(?:[^\s<a][^\s<]*|a(?:[^\s<t][^\s<]*|t(?:[^\s<h][^\s<]*|h(?:[^\s<e][^\s<]*|e(?:[^\s<r][^\s<]*|r(?:[^\s<][^\s<]*))?)?)?)?)?)?)?)?)?)?)/
|
||||
`<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
star
|
||||
sequence
|
||||
optional `\n`
|
||||
sequence
|
||||
`<rfl:parameter name="`
|
||||
/[a-zA-Z_][a-zA-Z0-9_]*/
|
||||
`">\n`
|
||||
or
|
||||
text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`]
|
||||
json(number) excluding [`<|end|>`, `<|call|>`]
|
||||
json(boolean) excluding [`<|end|>`, `<|call|>`]
|
||||
json(null) excluding [`<|end|>`, `<|call|>`]
|
||||
json({ ... }) excluding [`<|end|>`, `<|call|>`]
|
||||
json(any[]) excluding [`<|end|>`, `<|call|>`]
|
||||
`\n</rfl:parameter>`
|
||||
optional `<|call|>`
|
||||
sequence
|
||||
`<|channel|>final<|message|>`
|
||||
text excluding [`<|end|>`, `<|return|>`]
|
||||
optional `<|return|>`
|
||||
"#]]
|
||||
.assert_eq(&grammar(HfPosition::ResponseStart, ToolChoice::auto(), true).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn named_choice_calls_that_tool_once_after_prompted_reasoning() {
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `` text `<|end|><|start|>assistant`
|
||||
sequence
|
||||
sequence
|
||||
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
or
|
||||
sequence
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
optional `<|call|>`
|
||||
"#]].assert_eq(
|
||||
&grammar(HfPosition::Reasoning, ToolChoice::function("get_weather"), true).unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_grammar_without_a_constraining_request_or_known_position() {
|
||||
let non_strict = vec![tools().pop().unwrap()];
|
||||
for tools in [&[][..], &non_strict] {
|
||||
assert!(
|
||||
grammar_with(
|
||||
HfPosition::ResponseStart,
|
||||
tools,
|
||||
ToolChoice::auto(),
|
||||
true,
|
||||
false,
|
||||
false
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
grammar_with(
|
||||
HfPosition::Other,
|
||||
&tools(),
|
||||
ToolChoice::required(),
|
||||
true,
|
||||
true,
|
||||
false
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn none_with_tools_only_answers() {
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
|
||||
sequence
|
||||
/[^<]/
|
||||
text
|
||||
sequence
|
||||
`<|channel|>final<|message|>`
|
||||
text excluding [`<|end|>`, `<|return|>`]
|
||||
optional `<|return|>`
|
||||
"#]]
|
||||
.assert_eq(&grammar(HfPosition::ResponseStart, ToolChoice::none(), true).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn always_constrain_answers_without_tools() {
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `` text `<|end|><|start|>assistant`
|
||||
sequence
|
||||
`<|channel|>final<|message|>`
|
||||
text excluding [`<|end|>`, `<|return|>`]
|
||||
optional `<|return|>`
|
||||
"#]]
|
||||
.assert_eq(
|
||||
&grammar_with(
|
||||
HfPosition::Reasoning,
|
||||
&[],
|
||||
ToolChoice::auto(),
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn always_constrain_allows_non_strict_calls_or_an_answer() {
|
||||
let non_strict = vec![tools().pop().unwrap()];
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
|
||||
sequence
|
||||
/[^<]/
|
||||
text
|
||||
or
|
||||
sequence
|
||||
or
|
||||
sequence
|
||||
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
sequence
|
||||
` to=`
|
||||
/(?:[^\s<g][^\s<]*|g(?:[^\s<e][^\s<]*|e(?:[^\s<t][^\s<]*|t(?:[^\s<_][^\s<]*|_(?:[^\s<t][^\s<]*|t(?:[^\s<i][^\s<]*|i(?:[^\s<m][^\s<]*|m(?:[^\s<e][^\s<]*|e(?:[^\s<][^\s<]*))?)?)?)?)?)?)?)/
|
||||
`<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
star
|
||||
sequence
|
||||
optional `\n`
|
||||
sequence
|
||||
`<rfl:parameter name="`
|
||||
/[a-zA-Z_][a-zA-Z0-9_]*/
|
||||
`">\n`
|
||||
or
|
||||
text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`]
|
||||
json(number) excluding [`<|end|>`, `<|call|>`]
|
||||
json(boolean) excluding [`<|end|>`, `<|call|>`]
|
||||
json(null) excluding [`<|end|>`, `<|call|>`]
|
||||
json({ ... }) excluding [`<|end|>`, `<|call|>`]
|
||||
json(any[]) excluding [`<|end|>`, `<|call|>`]
|
||||
`\n</rfl:parameter>`
|
||||
optional `<|call|>`
|
||||
sequence
|
||||
`<|channel|>final<|message|>`
|
||||
text excluding [`<|end|>`, `<|return|>`]
|
||||
optional `<|return|>`
|
||||
"#]]
|
||||
.assert_eq(
|
||||
&grammar_with(
|
||||
HfPosition::ResponseStart,
|
||||
&non_strict,
|
||||
ToolChoice::auto(),
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn always_constrain_finishes_an_open_answer() {
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
text excluding [`<|end|>`, `<|return|>`]
|
||||
optional `<|return|>`
|
||||
"#]]
|
||||
.assert_eq(
|
||||
&grammar_with(
|
||||
HfPosition::Content,
|
||||
&tools(),
|
||||
ToolChoice::auto(),
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
assert!(
|
||||
grammar_with(
|
||||
HfPosition::Content,
|
||||
&tools(),
|
||||
ToolChoice::auto(),
|
||||
true,
|
||||
false,
|
||||
false
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
grammar_with(
|
||||
HfPosition::Content,
|
||||
&tools(),
|
||||
ToolChoice::required(),
|
||||
true,
|
||||
true,
|
||||
false
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answer_holds_the_requested_format() {
|
||||
let json = || Format::json_schema(json!({"type": "object"}));
|
||||
let answer = constrained_grammar(HfPosition::ResponseStart, ToolChoice::none(), json(), false);
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
|
||||
sequence
|
||||
/[^<]/
|
||||
text
|
||||
sequence
|
||||
`<|channel|>final<|message|>`
|
||||
json(object)
|
||||
optional `<|return|>`
|
||||
"#]]
|
||||
.assert_eq(&answer.unwrap());
|
||||
|
||||
// Under `auto`, the answer that may replace calls holds it too.
|
||||
let auto =
|
||||
constrained_grammar(HfPosition::ResponseStart, ToolChoice::auto(), json(), false).unwrap();
|
||||
assert!(auto.ends_with(
|
||||
"`<|channel|>final<|message|>`\n json(object)\n optional `<|return|>`\n"
|
||||
));
|
||||
|
||||
// A forced call leaves no answer to constrain.
|
||||
assert_eq!(
|
||||
constrained_grammar(
|
||||
HfPosition::ResponseStart,
|
||||
ToolChoice::required(),
|
||||
json(),
|
||||
false
|
||||
),
|
||||
grammar(HfPosition::ResponseStart, ToolChoice::required(), true)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_constraint_applies_alone_in_an_open_answer() {
|
||||
let json = Format::json_schema(json!({"type": "object"}));
|
||||
assert!(constrained_grammar(HfPosition::Content, ToolChoice::auto(), json, true).is_none());
|
||||
}
|
||||
|
||||
/// Parse `output` after the prompt under `tag_pattern`, returning the calls as
|
||||
/// `(name, arguments)`.
|
||||
fn calls(tag_pattern: &str, output: &str) -> Vec<(String, Value)> {
|
||||
let mut parser = parser_with(tag_pattern, PROMPT);
|
||||
let mut events = UnifiedParserOutput::default();
|
||||
parser.parse_into(DecodedText::unattributed(output), &mut events).unwrap();
|
||||
events.append(parser.finish().unwrap());
|
||||
let mut calls: Vec<(String, String)> = Vec::new();
|
||||
for event in events.events {
|
||||
if let UnifiedParserEvent::ToolCall(delta) = event {
|
||||
if let Some(name) = delta.name {
|
||||
calls.push((name, String::new()));
|
||||
}
|
||||
calls.last_mut().unwrap().1.push_str(&delta.arguments);
|
||||
}
|
||||
}
|
||||
calls
|
||||
.into_iter()
|
||||
.map(|(name, arguments)| (name, serde_json::from_str(&arguments).unwrap()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Output in the grammar's canonical spelling parses back to the values the
|
||||
/// grammar admitted.
|
||||
#[test]
|
||||
fn parser_reads_back_the_grammar_spelling() {
|
||||
let parameter = |key: &str, value: &str| {
|
||||
format!("<rfl:parameter name=\"{key}\">\n{value}\n</rfl:parameter>")
|
||||
};
|
||||
let call = |name: &str, parameters: &[String]| {
|
||||
format!(
|
||||
" to={name}<|channel|>analysis <|constrain|>xml<|message|>{}",
|
||||
parameters.join("\n")
|
||||
)
|
||||
};
|
||||
let output = format!(
|
||||
"<|channel|>analysis<|message|>Two tools.<|end|><|start|>assistant{}<|end|><|start|>assistant{}",
|
||||
call(
|
||||
"get_weather",
|
||||
&[
|
||||
parameter("location", "Paris"),
|
||||
parameter("days", "2"),
|
||||
parameter("unit", "c")
|
||||
]
|
||||
),
|
||||
call("get_time", &[parameter("zone", "multi\nline")]),
|
||||
);
|
||||
|
||||
for tag_pattern in [TAG_PATTERN, LENIENT_TAG_PATTERN] {
|
||||
assert_eq!(
|
||||
calls(tag_pattern, &output),
|
||||
vec![
|
||||
(
|
||||
"get_weather".to_string(),
|
||||
json!({"location": "Paris", "days": 2, "unit": "c"})
|
||||
),
|
||||
("get_time".to_string(), json!({"zone": "multi\nline"})),
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn values_end_before_the_next_parameter_where_the_template_looks_ahead() {
|
||||
let output = " to=get_weather<|channel|>analysis <|constrain|>xml<|message|>\
|
||||
<rfl:parameter name=\"days\">\n2\n\
|
||||
<rfl:parameter name=\"location\">\nParis\n</rfl:parameter>";
|
||||
|
||||
assert!(parser_with(LENIENT_TAG_PATTERN, PROMPT).lenient);
|
||||
assert_eq!(
|
||||
calls(LENIENT_TAG_PATTERN, output),
|
||||
vec![(
|
||||
"get_weather".to_string(),
|
||||
json!({"days": 2, "location": "Paris"})
|
||||
)]
|
||||
);
|
||||
assert!(!parser(PROMPT).lenient);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reasoning_ends_at_every_closer_of_the_template() {
|
||||
let closers = [
|
||||
"<|end|><|start|>assistant",
|
||||
"<|constrain|><|start|>assistant",
|
||||
"<|start|>assistant",
|
||||
];
|
||||
let tools = tools();
|
||||
let tokenizer = Arc::new(TestTokenizer::new());
|
||||
let mut parser = BeamUnifiedParser::new(
|
||||
template_with(TAG_PATTERN, json!(closers)),
|
||||
&tools,
|
||||
tokenizer.clone(),
|
||||
);
|
||||
parser.initialize(&tokenizer.encode(PROMPT, false).unwrap()).unwrap();
|
||||
let grammar = parser
|
||||
.build_output_grammar(&OutputGrammarContext {
|
||||
tools: &tools,
|
||||
tool_choice: &ToolChoice::required(),
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
parallel_tool_calls: false,
|
||||
answer: None,
|
||||
})
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
outline(&grammar.format).lines().nth(1).unwrap(),
|
||||
" tag `<|channel|>analysis<|message|>` .. `<|constrain|><|start|>assistant` | \
|
||||
`<|end|><|start|>assistant` | `<|start|>assistant`"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lenient_arguments_come_in_declared_or_sorted_order() {
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `` text `<|end|><|start|>assistant`
|
||||
sequence
|
||||
sequence
|
||||
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
optional `\n`
|
||||
`<rfl:parameter name="`
|
||||
or
|
||||
sequence
|
||||
or
|
||||
tag `location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`, `\n<rfl:parameter name="`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
tag `location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`, `\n<rfl:parameter name="`] `\n</rfl:parameter><rfl:parameter name="` | `\n</rfl:parameter>\n<rfl:parameter name="` | `\n<rfl:parameter name="`
|
||||
or
|
||||
tag `days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
optional tag `days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter><rfl:parameter name="` | `\n</rfl:parameter>\n<rfl:parameter name="` | `\n<rfl:parameter name="`
|
||||
tag `unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
optional tag `days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter><rfl:parameter name="` | `\n</rfl:parameter>\n<rfl:parameter name="` | `\n<rfl:parameter name="`
|
||||
or
|
||||
tag `location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`, `\n<rfl:parameter name="`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
tag `location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`, `\n<rfl:parameter name="`] `\n</rfl:parameter><rfl:parameter name="` | `\n</rfl:parameter>\n<rfl:parameter name="` | `\n<rfl:parameter name="`
|
||||
tag `unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
optional `<|call|>`
|
||||
"#]].assert_eq(
|
||||
&grammar_with(
|
||||
HfPosition::Reasoning,
|
||||
&tools(),
|
||||
ToolChoice::function("get_weather"),
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_names_match_every_name_but_the_given_ones() {
|
||||
let regex = regex_automata::meta::Regex::new(&format!(
|
||||
"^(?:{})$",
|
||||
other_names(&["get", "get_time", "a.b"])
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
for name in [
|
||||
"g",
|
||||
"ge",
|
||||
"gets",
|
||||
"get_",
|
||||
"get_t",
|
||||
"get_times",
|
||||
"x",
|
||||
"a",
|
||||
"a.",
|
||||
"axb",
|
||||
"a.bc",
|
||||
] {
|
||||
assert!(regex.is_match(name), "{name}");
|
||||
}
|
||||
for name in ["get", "get_time", "a.b", "", "get time", "get<"] {
|
||||
assert!(!regex.is_match(name), "{name}");
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,12 @@
|
||||
//! Port of `content_parsers.py`. Each parser takes a chunk of captured text and
|
||||
//! parses it into a single key in the output message dictionary.
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use regex_automata::meta::Regex;
|
||||
use regex_automata::util::iter::Searcher;
|
||||
use regex_automata::util::syntax;
|
||||
use regex_automata::{Input, Match};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value};
|
||||
use thiserror_ext::AsReport as _;
|
||||
@@ -22,6 +26,10 @@ use crate::unified::{self, parsing_failed};
|
||||
const LAX_OPEN: char = '\u{1}';
|
||||
const LAX_CLOSE: char = '\u{2}';
|
||||
|
||||
/// Name of the group that stands for the lookahead ending an `xml-inline` tag
|
||||
/// pattern.
|
||||
const AHEAD_GROUP: &str = "lookahead";
|
||||
|
||||
/// A configured content parser (`content` plus `content_args` of one field).
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) enum ContentParser {
|
||||
@@ -66,6 +74,9 @@ pub(super) struct XmlInlineArgs {
|
||||
key_group: usize,
|
||||
/// Index of the `value` group, if the pattern declares one.
|
||||
value_group: Option<usize>,
|
||||
/// Index of the group standing for the lookahead that ends the pattern,
|
||||
/// and the lookahead's own pattern, if any.
|
||||
lookahead: Option<(usize, String)>,
|
||||
value_parser: Option<Box<ContentParser>>,
|
||||
/// Collect duplicate keys into a list.
|
||||
merge_duplicates: bool,
|
||||
@@ -143,7 +154,8 @@ impl ContentParser {
|
||||
}
|
||||
ContentKind::XmlInline => {
|
||||
let spec: XmlInlineSpec = deserialize(&args, &context)?;
|
||||
let tag_pattern = compile_regex(scope, &spec.tag_pattern)?;
|
||||
let (pattern, lookahead) = final_lookahead(&spec.tag_pattern);
|
||||
let tag_pattern = compile_regex(scope, &pattern)?;
|
||||
let group_info = tag_pattern.group_info();
|
||||
let Some(key_group) = group_info.to_index(Default::default(), "key") else {
|
||||
return Err(invalid!(
|
||||
@@ -152,10 +164,14 @@ impl ContentParser {
|
||||
));
|
||||
};
|
||||
let value_group = group_info.to_index(Default::default(), "value");
|
||||
let lookahead = group_info
|
||||
.to_index(Default::default(), AHEAD_GROUP)
|
||||
.zip(lookahead.map(str::to_string));
|
||||
Self::XmlInline(XmlInlineArgs {
|
||||
tag_pattern,
|
||||
key_group,
|
||||
value_group,
|
||||
lookahead,
|
||||
value_parser: ValueParserSpec::compile(scope, spec.value_parser)?,
|
||||
merge_duplicates: spec.merge_duplicates,
|
||||
})
|
||||
@@ -193,6 +209,32 @@ impl ContentParser {
|
||||
Self::KvLines(args) => args.parse(text),
|
||||
}
|
||||
}
|
||||
|
||||
/// The pattern of the lookahead that ends an `xml-inline` tag pattern, if
|
||||
/// any.
|
||||
pub fn lookahead(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::XmlInline(args) => args.lookahead.as_ref().map(|(_, ahead)| ahead.as_str()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `pattern` with a lookahead that ends it matched as the group
|
||||
/// [`AHEAD_GROUP`] instead, as the Rust regex engine has no lookahead, and
|
||||
/// the lookahead's own pattern. The engine rejects any other lookahead.
|
||||
fn final_lookahead(pattern: &str) -> (Cow<'_, str>, Option<&str>) {
|
||||
if let Some((head, rest)) = pattern.split_once("(?=") {
|
||||
// Only closing parentheses may follow a lookahead that ends the pattern.
|
||||
let ahead = rest.trim_end_matches(')');
|
||||
if ahead.len() < rest.len() && !ahead.contains(['(', ')']) {
|
||||
return (
|
||||
format!("{head}(?P<{AHEAD_GROUP}>{rest}").into(),
|
||||
Some(ahead),
|
||||
);
|
||||
}
|
||||
}
|
||||
(pattern.into(), None)
|
||||
}
|
||||
|
||||
impl ValueParserSpec {
|
||||
@@ -258,7 +300,21 @@ impl XmlInlineArgs {
|
||||
/// Parse shallow XML-ish tags into a dict.
|
||||
fn parse(&self, text: &str) -> unified::Result<Value> {
|
||||
let mut out = Map::new();
|
||||
for captures in self.tag_pattern.captures_iter(text) {
|
||||
let mut captures = self.tag_pattern.create_captures();
|
||||
let mut tags = Searcher::new(Input::new(text));
|
||||
while tags
|
||||
.advance(|input| {
|
||||
self.tag_pattern.search_captures(input, &mut captures);
|
||||
// The lookahead's text is not part of the tag.
|
||||
let ahead =
|
||||
self.lookahead.as_ref().and_then(|(group, _)| captures.get_group(*group));
|
||||
Ok(captures.get_match().map(|tag| match ahead {
|
||||
Some(ahead) => Match::new(tag.pattern(), tag.start()..ahead.start),
|
||||
None => tag,
|
||||
}))
|
||||
})
|
||||
.is_some()
|
||||
{
|
||||
let key =
|
||||
captures.get_group(self.key_group).map(|span| &text[span.range()]).ok_or_else(
|
||||
|| parsing_failed!("xml-inline: named group 'key' did not participate"),
|
||||
@@ -490,6 +546,25 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xml_inline_tags_may_end_where_the_pattern_looks_ahead() {
|
||||
let xml = parser(
|
||||
"xml-inline",
|
||||
json!({
|
||||
"tag_pattern": r#"<arg name="(?P<key>\w+)">(?P<value>.*?)\n(?:</arg>|(?=<arg name="))"#,
|
||||
}),
|
||||
);
|
||||
let value = xml
|
||||
.parse("<arg name=\"a\">1\n<arg name=\"b\">x <arg name=\"c\"> y\n</arg>")
|
||||
.unwrap();
|
||||
assert_eq!(value, json!({"a": "1", "b": "x <arg name=\"c\"> y"}));
|
||||
assert_eq!(xml.lookahead(), Some("<arg name=\""));
|
||||
|
||||
// Any other lookahead is unsupported.
|
||||
let args = json!({"tag_pattern": r"(?P<key>\w+)(?=:)(?P<value>.*)"});
|
||||
assert!(ContentParser::new("test", kind("xml-inline"), args.as_object().unwrap()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kv_lines_split_and_strip() {
|
||||
let kv = parser("kv-lines", json!({"value_parser": {"name": "int"}}));
|
||||
|
||||
@@ -25,7 +25,7 @@ mod transform;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub use parser::HfUnifiedParser;
|
||||
pub use parser::{HfPosition, HfUnifiedParser};
|
||||
pub use template::ResponseTemplate;
|
||||
use thiserror::Error;
|
||||
use thiserror_ext::Macro;
|
||||
|
||||
@@ -59,6 +59,20 @@ pub struct HfUnifiedParser {
|
||||
emitted_tool_count: usize,
|
||||
}
|
||||
|
||||
/// Where the parser is in the response; see [`HfUnifiedParser::position`].
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HfPosition {
|
||||
/// Nothing parsed since the start anchor, or since the start of the stream
|
||||
/// when there is none.
|
||||
ResponseStart,
|
||||
/// Inside an open reasoning field.
|
||||
Reasoning,
|
||||
/// Inside an open content field.
|
||||
Content,
|
||||
/// Anywhere else.
|
||||
Other,
|
||||
}
|
||||
|
||||
/// Parser mode: the region currently receiving text.
|
||||
enum HfMode {
|
||||
/// No implicit field is declared: text outside regions is discarded.
|
||||
@@ -156,6 +170,36 @@ impl HfUnifiedParser {
|
||||
parser
|
||||
}
|
||||
|
||||
/// Where the parser is in the response. Between
|
||||
/// [`UnifiedParser::initialize`] and the first generated text, this is where
|
||||
/// the prompt leaves the response.
|
||||
///
|
||||
/// Text outside every region, which a template without an implicit field
|
||||
/// discards, leaves no state behind and so does not move the parser from
|
||||
/// [`HfPosition::ResponseStart`].
|
||||
pub fn position(&self) -> HfPosition {
|
||||
if !self.buffer.text.is_empty() {
|
||||
return HfPosition::Other;
|
||||
}
|
||||
match &self.mode {
|
||||
HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) if occurrence.opened => {
|
||||
match &self.template.regions[occurrence.region].kind {
|
||||
RegionKind::Text(TextRegion {
|
||||
role: TextRole::Reasoning,
|
||||
..
|
||||
}) => HfPosition::Reasoning,
|
||||
RegionKind::Text(TextRegion {
|
||||
role: TextRole::Content,
|
||||
..
|
||||
}) => HfPosition::Content,
|
||||
_ => HfPosition::Other,
|
||||
}
|
||||
}
|
||||
_ if self.closed.contains(&true) || self.emitted_tool_count > 0 => HfPosition::Other,
|
||||
_ => HfPosition::ResponseStart,
|
||||
}
|
||||
}
|
||||
|
||||
/// Clear all per-stream state.
|
||||
fn reset_state(&mut self) {
|
||||
self.buffer.clear();
|
||||
|
||||
@@ -191,6 +191,31 @@ impl ResponseTemplate {
|
||||
})
|
||||
}
|
||||
|
||||
/// The pattern of the lookahead that ends the `xml-inline` tag pattern of
|
||||
/// tool-call arguments, such as for the opener of the next argument.
|
||||
pub fn argument_lookahead(&self) -> Option<&str> {
|
||||
self.regions.iter().find_map(|region| match ®ion.kind {
|
||||
RegionKind::ToolCalls(calls) => calls.content.lookahead(),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
/// The literal closers of the reasoning field, longest first, or none if it
|
||||
/// has no literal close.
|
||||
pub fn reasoning_closers(&self) -> &[String] {
|
||||
self.regions
|
||||
.iter()
|
||||
.find_map(|region| match (®ion.kind, ®ion.close) {
|
||||
(RegionKind::Text(text), Some(Boundary::Literals(closers)))
|
||||
if text.role == TextRole::Reasoning =>
|
||||
{
|
||||
Some(closers.as_slice())
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Return the start and end offsets of the last start-anchor match in
|
||||
/// `prompt`.
|
||||
pub(super) fn last_start_anchor(&self, prompt: &str) -> Option<(usize, usize)> {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
//! Unified parser interface for reasoning and tool-call deltas.
|
||||
|
||||
mod beam;
|
||||
mod combined;
|
||||
mod gemma4;
|
||||
mod hf;
|
||||
@@ -10,9 +11,10 @@ mod hy;
|
||||
mod inkling;
|
||||
mod kimi_k3;
|
||||
|
||||
pub use beam::BeamUnifiedParser;
|
||||
pub use combined::CombinedParser;
|
||||
pub use gemma4::Gemma4UnifiedParser;
|
||||
pub use hf::{HfTemplateError, HfUnifiedParser, ResponseTemplate};
|
||||
pub use hf::{HfPosition, HfTemplateError, HfUnifiedParser, ResponseTemplate};
|
||||
pub use hy::{HyV3UnifiedParser, HyV4UnifiedParser};
|
||||
pub use inkling::InklingUnifiedParser;
|
||||
pub use kimi_k3::KimiK3UnifiedParser;
|
||||
|
||||
Reference in New Issue
Block a user