[Frontend] Add the Beam reasoning and tool parsers (#62052)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com> Co-authored-by: neilkg <33635204+neilkg@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Woosuk Kwon
co-authored by
Bugen Zhao
Yoni Gozlan
neilkg
Cursor
Claude Opus 5.5
Bugen Zhao
Yoni Gozlan
neilkg
Cursor
Claude Opus 5.5
parent
9731f30a03
commit
153a17166e
@@ -201,6 +201,30 @@ checkpoint's output format.
|
||||
!!! note
|
||||
With `tool_choice="auto"`, structural-tag constraints require both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`, or a server-side floor set via `--tool-strict-level`. When these conditions are met and the selected parser supports structural tags, vLLM constrains the tool-call envelope and pins the argument schema of each tool that sets `strict: true` (or of every tool under `--tool-strict-level parameter`). Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema.
|
||||
|
||||
### Beam Models (`beam`)
|
||||
|
||||
Beam checkpoints describe their output format with `response_template` metadata, so the `beam` parser reads it like the `hf` parser does:
|
||||
|
||||
```bash
|
||||
vllm serve <model> \
|
||||
--enable-auto-tool-choice \
|
||||
--tool-call-parser beam \
|
||||
--reasoning-parser beam
|
||||
```
|
||||
|
||||
Unlike `hf`, the `beam` parser also constrains its output. For strict tools, required or named tool choice, and structured outputs, it derives a
|
||||
structural-tag grammar from the template. The grammar follows the template's tool call format, pins the arguments of strict tools when their
|
||||
encoding can be derived, allows a single call with `parallel_tool_calls=false`, and frames a structured output as the answer. Raw string arguments
|
||||
keep the `format` or `pattern` of their schema, as XGrammar keeps them in JSON strings, unless a match could end the value early. With
|
||||
`tool_choice="none"`, the grammar allows only reasoning and an answer, so a model that still sees the tools in its prompt cannot call them. When no
|
||||
grammar can be derived, required and named tool choice are rejected, and strict tools under `tool_choice="auto"` are left unconstrained with a
|
||||
warning.
|
||||
|
||||
It also constrains the output of requests without strict tools, forced tool choice, or structured outputs to the template's format: tool calls with
|
||||
free arguments or an answer, or just the rest of an answer the prompt already opened. This always-on grammar needs the `xgrammar` structured
|
||||
outputs backend, the default, and is disabled with `VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT=0`. From the message start, every `beam` grammar opens with
|
||||
reasoning that holds visible text, unless `enable_thinking` is false.
|
||||
|
||||
### Hermes Models (`hermes`)
|
||||
|
||||
All Nous Research Hermes-series models newer than Hermes 2 Pro should be supported.
|
||||
|
||||
@@ -434,7 +434,7 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hf, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_json, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: beam, deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hf, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_json, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -445,6 +445,6 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hf, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: beam, cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hf, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,8 @@ use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEvent
|
||||
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
|
||||
use crate::parser::tool::{ToolParser, ToolParserFactory};
|
||||
use crate::parser::unified::{
|
||||
HfTemplateError, HfUnifiedParser, ResponseTemplate, UnifiedParserFactory, names,
|
||||
BeamUnifiedParser, HfTemplateError, HfUnifiedParser, ResponseTemplate, UnifiedParserFactory,
|
||||
names,
|
||||
};
|
||||
use crate::parser::{ParserSelection, ToolStrictLevel};
|
||||
use crate::request::{ChatRequest, ChatTool};
|
||||
@@ -206,17 +207,19 @@ impl DefaultChatOutputProcessor {
|
||||
});
|
||||
}
|
||||
|
||||
// `hf` is built from the model's template, never by its registry constructor.
|
||||
let parser = if parser_name == names::HF {
|
||||
let template =
|
||||
response_template.as_ref().map_err(|error| Error::ParserInitialization {
|
||||
kind: "unified",
|
||||
name: parser_name.to_string(),
|
||||
error: Box::new(error.clone()),
|
||||
})?;
|
||||
Box::new(HfUnifiedParser::new(template.clone(), tools, tokenizer))
|
||||
} else {
|
||||
factory.create(parser_name, tools, tokenizer)?
|
||||
// Parsers driven by the model's template are built from it, never by
|
||||
// their registry constructors.
|
||||
let template = || {
|
||||
response_template.clone().map_err(|error| Error::ParserInitialization {
|
||||
kind: "unified",
|
||||
name: parser_name.to_string(),
|
||||
error: Box::new(error),
|
||||
})
|
||||
};
|
||||
let parser: Box<dyn UnifiedParser> = match parser_name {
|
||||
names::HF => Box::new(HfUnifiedParser::new(template()?, tools, tokenizer)),
|
||||
names::BEAM => Box::new(BeamUnifiedParser::new(template()?, tools, tokenizer)),
|
||||
_ => factory.create(parser_name, tools, tokenizer)?,
|
||||
};
|
||||
|
||||
UNIFIED_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using unified parser"));
|
||||
@@ -589,6 +592,20 @@ mod tests {
|
||||
.assert_eq(&error(invalid_template()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_beam_uses_the_model_response_template() {
|
||||
let beam = ParserSelection::Explicit("beam".to_string());
|
||||
crate::parser::validate_parser_overrides(&beam, &beam).unwrap();
|
||||
assert!(keeps_special_tokens("other-model", &loaded_template(), &beam, &beam).unwrap());
|
||||
|
||||
let error =
|
||||
keeps_special_tokens("other-model", &Err(HfTemplateError::Missing), &beam, &beam)
|
||||
.unwrap_err()
|
||||
.to_report_string();
|
||||
expect_test::expect!["failed to initialize unified parser `beam`: the model's tokenizer_config.json provides no response_template"]
|
||||
.assert_eq(&error);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_never_selects_hf() {
|
||||
let auto = ParserSelection::Auto;
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub use vllm_parser::unified::{
|
||||
Gemma4UnifiedParser, HfTemplateError, HfUnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser,
|
||||
InklingUnifiedParser, KimiK3UnifiedParser, ResponseTemplate, UnifiedParser,
|
||||
BeamUnifiedParser, Gemma4UnifiedParser, HfTemplateError, HfUnifiedParser, HyV3UnifiedParser,
|
||||
HyV4UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser, ResponseTemplate, UnifiedParser,
|
||||
};
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
@@ -26,6 +26,9 @@ pub mod names {
|
||||
/// name resolution only: it is built from the model's template, never by its
|
||||
/// registry constructor.
|
||||
pub const HF: &str = "hf";
|
||||
/// Beam checkpoints: [`HF`] with Beam's tool-call grammar, built from the
|
||||
/// model's template the same way.
|
||||
pub const BEAM: &str = "beam";
|
||||
}
|
||||
|
||||
/// Constructor signature for one registered unified parser implementation.
|
||||
@@ -55,6 +58,7 @@ impl UnifiedParserFactory {
|
||||
factory.register_parser::<KimiK3UnifiedParser>(names::KIMI_K3);
|
||||
// Opt-in only: no model patterns map to it.
|
||||
factory.register_parser::<HfUnifiedParser>(names::HF);
|
||||
factory.register_parser::<BeamUnifiedParser>(names::BEAM);
|
||||
|
||||
factory
|
||||
.register_pattern("gemma-4", names::GEMMA4)
|
||||
|
||||
@@ -207,7 +207,8 @@ pub(crate) fn full_format_from_builder_for_test(
|
||||
)
|
||||
}
|
||||
|
||||
fn tool_params(tools: &[Tool], strict_level: ToolStrictLevel) -> Vec<ToolParam> {
|
||||
/// Tools as builder inputs, strict when the tool or the server floor asks for it.
|
||||
pub(crate) fn tool_params(tools: &[Tool], strict_level: ToolStrictLevel) -> Vec<ToolParam> {
|
||||
tools
|
||||
.iter()
|
||||
.map(|tool| {
|
||||
@@ -240,7 +241,7 @@ pub(crate) fn normalize_tool_choice(
|
||||
}
|
||||
|
||||
/// Whether the request asks for a tool grammar at all.
|
||||
fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool {
|
||||
pub(crate) fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool {
|
||||
if ctx.tools.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
//! - `allOf` with more than one schema accepts any value.
|
||||
|
||||
use serde_json::{Value, json};
|
||||
use xgrammar_structural_tag::format::{Format, JsonSchemaFormat};
|
||||
use xgrammar_structural_tag::format::{EndBoundary, Format, JsonSchemaFormat, TokenFormat};
|
||||
|
||||
pub use crate::schema::JsonType;
|
||||
use crate::schema::{self, OptionSource, SchemaOption, SchemaRoot};
|
||||
@@ -60,7 +60,13 @@ impl ParameterKey<'_> {
|
||||
|
||||
/// `prefix KEY suffix content end`: a tag for a declared key, or a
|
||||
/// sequence matching [`Self::FREE_PATTERN`] for a free key.
|
||||
pub fn tag(&self, prefix: &str, suffix: &str, content: Format, end: &str) -> Format {
|
||||
pub fn tag(
|
||||
&self,
|
||||
prefix: &str,
|
||||
suffix: &str,
|
||||
content: Format,
|
||||
end: impl Into<EndBoundary>,
|
||||
) -> Format {
|
||||
match self {
|
||||
Self::Declared(key) => Format::tag(format!("{prefix}{key}{suffix}"), content, end),
|
||||
Self::Free => Format::sequence(vec![
|
||||
@@ -68,7 +74,13 @@ impl ParameterKey<'_> {
|
||||
Format::regex(Self::FREE_PATTERN),
|
||||
Format::const_string(suffix),
|
||||
content,
|
||||
Format::const_string(end),
|
||||
match end.into() {
|
||||
EndBoundary::Text(end) => Format::const_string(end),
|
||||
EndBoundary::Texts(ends) => {
|
||||
one_of(ends.into_iter().map(Format::const_string).collect())
|
||||
}
|
||||
EndBoundary::Token(end) => Format::Token(TokenFormat { token: end.token }),
|
||||
},
|
||||
]),
|
||||
}
|
||||
}
|
||||
@@ -183,6 +195,36 @@ impl ArgumentContext<'_> {
|
||||
/// Build the grammar of one call's arguments under `schema`, the tool's
|
||||
/// parameters schema, rendering each parameter with `syntax`.
|
||||
pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &ArgumentOptions) -> Format {
|
||||
parameter_list(
|
||||
options,
|
||||
syntax.separator(),
|
||||
parameters(schema, syntax, options),
|
||||
)
|
||||
}
|
||||
|
||||
/// The parameters of one call, each rendered by an [`ArgumentSyntax`], for a
|
||||
/// syntax that lists them itself rather than through [`arguments`].
|
||||
pub struct Parameters {
|
||||
/// The declared parameters, in declared order.
|
||||
pub declared: Vec<Parameter>,
|
||||
/// Any one additional parameter, if the schema allows them.
|
||||
pub additional: Option<Format>,
|
||||
}
|
||||
|
||||
/// One declared parameter.
|
||||
pub struct Parameter {
|
||||
pub key: String,
|
||||
pub format: Format,
|
||||
pub required: bool,
|
||||
}
|
||||
|
||||
/// The parameters of one call under `schema`, the tool's parameters schema,
|
||||
/// rendering each parameter with `syntax`.
|
||||
pub fn parameters(
|
||||
schema: &Value,
|
||||
syntax: &dyn ArgumentSyntax,
|
||||
options: &ArgumentOptions,
|
||||
) -> Parameters {
|
||||
let cx = ArgumentContext {
|
||||
options,
|
||||
root: SchemaRoot::new(schema),
|
||||
@@ -190,8 +232,7 @@ pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &Argument
|
||||
let parameter =
|
||||
|key: ParameterKey<'_>, schema: &Value| syntax.parameter(key, &cx.value_options(schema));
|
||||
|
||||
let schema = cx.root.resolved();
|
||||
let (properties, additional) = match schema {
|
||||
match cx.root.resolved() {
|
||||
Value::Object(schema) => {
|
||||
let additional = match schema.get("additionalProperties") {
|
||||
Some(Value::Bool(false)) => None,
|
||||
@@ -206,22 +247,33 @@ pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &Argument
|
||||
.and_then(Value::as_array)
|
||||
.map(|required| required.iter().filter_map(Value::as_str).collect::<Vec<_>>())
|
||||
.unwrap_or_default();
|
||||
let properties = schema
|
||||
let declared = schema
|
||||
.get("properties")
|
||||
.and_then(Value::as_object)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(|(key, schema)| {
|
||||
let parameter = parameter(ParameterKey::Declared(key), schema)?;
|
||||
Some((parameter, required.contains(&key.as_str())))
|
||||
Some(Parameter {
|
||||
key: key.clone(),
|
||||
format: parameter(ParameterKey::Declared(key), schema)?,
|
||||
required: required.contains(&key.as_str()),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
(properties, additional)
|
||||
Parameters {
|
||||
declared,
|
||||
additional,
|
||||
}
|
||||
}
|
||||
Value::Bool(false) => (vec![], None),
|
||||
_ => (vec![], parameter(ParameterKey::Free, &ANY_SCHEMA)),
|
||||
};
|
||||
parameter_list(options, syntax.separator(), properties, additional)
|
||||
Value::Bool(false) => Parameters {
|
||||
declared: vec![],
|
||||
additional: None,
|
||||
},
|
||||
_ => Parameters {
|
||||
declared: vec![],
|
||||
additional: parameter(ParameterKey::Free, &ANY_SCHEMA),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Parameters in declared order, required ones mandatory, followed by any
|
||||
@@ -230,8 +282,10 @@ pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &Argument
|
||||
fn parameter_list(
|
||||
options: &ArgumentOptions,
|
||||
separator: Option<Format>,
|
||||
properties: Vec<(Format, bool)>,
|
||||
additional: Option<Format>,
|
||||
Parameters {
|
||||
declared,
|
||||
additional,
|
||||
}: Parameters,
|
||||
) -> Format {
|
||||
let separated = |parameter: Format| match &separator {
|
||||
Some(separator) => Format::sequence(vec![parameter, separator.clone()]),
|
||||
@@ -239,10 +293,10 @@ fn parameter_list(
|
||||
};
|
||||
let mut elements = Vec::from_iter(separator.clone());
|
||||
if options.any_order {
|
||||
let any_required = properties.iter().any(|(_, required)| *required);
|
||||
let parameters = properties
|
||||
let any_required = declared.iter().any(|parameter| parameter.required);
|
||||
let parameters = declared
|
||||
.into_iter()
|
||||
.map(|(parameter, _)| parameter)
|
||||
.map(|parameter| parameter.format)
|
||||
.chain(additional)
|
||||
.collect::<Vec<_>>();
|
||||
if !parameters.is_empty() {
|
||||
@@ -254,8 +308,11 @@ fn parameter_list(
|
||||
});
|
||||
}
|
||||
} else {
|
||||
for (parameter, required) in properties {
|
||||
let parameter = separated(parameter);
|
||||
for Parameter {
|
||||
format, required, ..
|
||||
} in declared
|
||||
{
|
||||
let parameter = separated(format);
|
||||
elements.push(if required {
|
||||
parameter
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,543 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Unified parser for Beam checkpoints.
|
||||
//!
|
||||
//! Beam output is parsed by [`HfUnifiedParser`] with the checkpoint's
|
||||
//! `response_template`; this parser adds Beam's output grammar.
|
||||
//! A Beam message after `<|start|>assistant` is:
|
||||
//!
|
||||
//! ```text
|
||||
//! <|channel|>analysis<|message|>REASONING<|end|><|start|>assistant
|
||||
//! ( <|channel|>final<|message|>ANSWER<|return|>
|
||||
//! | CALL (<|end|><|start|>assistant CALL)* <|call|> )
|
||||
//! ```
|
||||
//!
|
||||
//! where reasoning may end at any closer of the template's reasoning field,
|
||||
//! and `CALL` is
|
||||
//! ` to=NAME<|channel|>analysis <|constrain|>xml<|message|>` followed by
|
||||
//! `<rfl:parameter name="KEY">\nVALUE\n</rfl:parameter>` parameters separated
|
||||
//! by newlines. Where the template looks ahead for the next parameter, a value
|
||||
//! may also end on a newline right before it.
|
||||
//!
|
||||
//! The grammar enforces the schema of every declared tool's arguments, in
|
||||
//! declared or alphabetical order, since Beam drifts from schemas it does not
|
||||
//! enforce. When the model may answer instead, calls may also name any other
|
||||
//! tool with any arguments, which the parser passes on so a harness can tell
|
||||
//! the model the tool does not exist. `ANSWER` follows the request's structured
|
||||
//! output, if any.
|
||||
//!
|
||||
//! The chat template rejects a history turn without reasoning, so a message the
|
||||
//! model opens holds non-empty reasoning: an empty one would come back as null
|
||||
//! reasoning and fail the next request.
|
||||
//!
|
||||
//! Unless `VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT` turns it off, the grammar
|
||||
//! applies to every message, including one the request does not constrain.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
|
||||
use serde_json::Value;
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer};
|
||||
use xgrammar_structural_tag::ToolChoice;
|
||||
use xgrammar_structural_tag::format::{EndBoundary, Format};
|
||||
use xgrammar_structural_tag::tool::{
|
||||
BuilderToolChoice, NormalizedToolChoice, ToolChoiceValue, normalize_tool_choice,
|
||||
};
|
||||
|
||||
use super::{
|
||||
HfPosition, HfUnifiedParser, ResponseTemplate, Result, UnifiedParser, UnifiedParserError,
|
||||
UnifiedParserOutput,
|
||||
};
|
||||
use crate::output_grammar::arguments::{
|
||||
self, ArgumentOptions, ArgumentSyntax, JsonType, ParameterKey, Parameters, ValueOption, one_of,
|
||||
};
|
||||
use crate::output_grammar::{
|
||||
self, BuiltOutputGrammar, OutputGrammarContext, tool_grammar_applies, tool_params,
|
||||
};
|
||||
use crate::tool::Tool;
|
||||
|
||||
const ANALYSIS_OPEN: &str = "<|channel|>analysis<|message|>";
|
||||
const FINAL_OPEN: &str = "<|channel|>final<|message|>";
|
||||
const END: &str = "<|end|>";
|
||||
/// Closes one message and opens the next assistant message.
|
||||
const NEXT_MESSAGE: &str = "<|end|><|start|>assistant";
|
||||
const RETURN: &str = "<|return|>";
|
||||
const CALL: &str = "<|call|>";
|
||||
/// Follows ` to=NAME` in a call.
|
||||
const CALL_HEADER_END: &str = "<|channel|>analysis <|constrain|>xml<|message|>";
|
||||
const PARAMETER_OPEN: &str = "<rfl:parameter name=\"";
|
||||
const PARAMETER_NAME_CLOSE: &str = "\">\n";
|
||||
const PARAMETER_CLOSE: &str = "\n</rfl:parameter>";
|
||||
/// Ends a value where the template looks ahead for the next parameter.
|
||||
const NEXT_PARAMETER: &str = "\n<rfl:parameter name=\"";
|
||||
/// Ends of a lenient value that another parameter follows: the closer and
|
||||
/// separator before the next opener, or the opener right after the value.
|
||||
const NEXT_PARAMETER_ENDS: [&str; 3] = [
|
||||
"\n</rfl:parameter><rfl:parameter name=\"",
|
||||
"\n</rfl:parameter>\n<rfl:parameter name=\"",
|
||||
NEXT_PARAMETER,
|
||||
];
|
||||
/// Text that ends the call in the parser, even inside a JSON value.
|
||||
const CALL_TERMINATORS: [&str; 2] = [END, CALL];
|
||||
|
||||
/// Whether every Beam message follows the grammar, even when the request
|
||||
/// constrains neither tool calls nor the answer. Read from
|
||||
/// `VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT` like the Python frontend: on unless
|
||||
/// set to something other than `true` or `1`.
|
||||
static ALWAYS_CONSTRAIN_OUTPUT: LazyLock<bool> = LazyLock::new(|| {
|
||||
std::env::var("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT").map_or(true, |value| {
|
||||
matches!(value.to_lowercase().as_str(), "true" | "1")
|
||||
})
|
||||
});
|
||||
|
||||
/// Beam parser: [`HfUnifiedParser`] with Beam's output grammar.
|
||||
pub struct BeamUnifiedParser {
|
||||
inner: HfUnifiedParser,
|
||||
/// Whether a value may end where the next parameter opens.
|
||||
lenient: bool,
|
||||
/// What ends the reasoning and opens the next message.
|
||||
reasoning_ends: Vec<String>,
|
||||
}
|
||||
|
||||
impl BeamUnifiedParser {
|
||||
/// Create a parser for one request from the checkpoint's `response_template`.
|
||||
pub fn new(template: Arc<ResponseTemplate>, tools: &[Tool], tokenizer: DynTokenizer) -> Self {
|
||||
Self {
|
||||
lenient: template.argument_lookahead() == Some(PARAMETER_OPEN),
|
||||
reasoning_ends: match template.reasoning_closers() {
|
||||
[] => vec![NEXT_MESSAGE.to_string()],
|
||||
closers => closers.to_vec(),
|
||||
},
|
||||
inner: HfUnifiedParser::new(template, tools, tokenizer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl UnifiedParser for BeamUnifiedParser {
|
||||
fn create(_tools: &[Tool], _tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>> {
|
||||
Err(UnifiedParserError::NoNamedConstructor {
|
||||
parser: "beam",
|
||||
built_from: "the checkpoint's response_template",
|
||||
})
|
||||
}
|
||||
|
||||
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
|
||||
self.inner.initialize(prompt_token_ids)
|
||||
}
|
||||
|
||||
fn preserve_special_tokens(&self) -> bool {
|
||||
self.inner.preserve_special_tokens()
|
||||
}
|
||||
|
||||
fn build_output_grammar(
|
||||
&self,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
|
||||
build_output_grammar(
|
||||
self.inner.position(),
|
||||
ctx,
|
||||
*ALWAYS_CONSTRAIN_OUTPUT,
|
||||
self.lenient,
|
||||
&self.reasoning_ends,
|
||||
)
|
||||
}
|
||||
|
||||
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
|
||||
self.inner.tool_call_id(tool_index)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, delta: DecodedText, output: &mut UnifiedParserOutput) -> Result<()> {
|
||||
self.inner.parse_into(delta, output)
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Result<UnifiedParserOutput> {
|
||||
self.inner.finish()
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> String {
|
||||
self.inner.reset()
|
||||
}
|
||||
}
|
||||
|
||||
/// The grammar of a Beam message from the first generated token, or `None`
|
||||
/// when the parser is neither at the start of the message nor inside its
|
||||
/// reasoning, or when the request constrains nothing and `always_constrain` is
|
||||
/// off. Values may end where the next parameter opens if `lenient`, and
|
||||
/// reasoning ends at any of `reasoning_ends`.
|
||||
fn build_output_grammar(
|
||||
position: HfPosition,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
always_constrain: bool,
|
||||
lenient: bool,
|
||||
reasoning_ends: &[String],
|
||||
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
|
||||
let forced = !ctx.tools.is_empty()
|
||||
&& matches!(
|
||||
ctx.tool_choice,
|
||||
ToolChoice::Value(ToolChoiceValue::Required) | ToolChoice::NamedFunction(_)
|
||||
);
|
||||
let answer_format = ctx.answer.filter(|_| !forced);
|
||||
// The prompt may still list the tools, so only the grammar keeps the model
|
||||
// from calling them.
|
||||
let tools_forbidden = !ctx.tools.is_empty()
|
||||
&& matches!(ctx.tool_choice, ToolChoice::Value(ToolChoiceValue::None));
|
||||
if !(always_constrain
|
||||
|| tool_grammar_applies(ctx)
|
||||
|| tools_forbidden
|
||||
|| answer_format.is_some())
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let text = Format::any_text();
|
||||
let reasoning_end = EndBoundary::from(reasoning_ends.to_vec());
|
||||
let reasoning = match position {
|
||||
HfPosition::ResponseStart => Format::tag(
|
||||
ANALYSIS_OPEN,
|
||||
// At least one character, which cannot start Beam's closers.
|
||||
Format::sequence(vec![Format::regex("[^<]"), text]),
|
||||
reasoning_end,
|
||||
),
|
||||
HfPosition::Reasoning => Format::tag("", text, reasoning_end),
|
||||
// Inside an answer the prompt opened, only the always-on grammar
|
||||
// applies, finishing a free answer. A tool choice that forces a call
|
||||
// cannot be met there, and the request's own constraint applies as is.
|
||||
HfPosition::Content => {
|
||||
return Ok(
|
||||
(always_constrain && !forced && answer_format.is_none()).then(|| {
|
||||
BuiltOutputGrammar::from_token_zero(Format::sequence(answer_rest(None)))
|
||||
}),
|
||||
);
|
||||
}
|
||||
HfPosition::Other => return Ok(None),
|
||||
};
|
||||
let NormalizedToolChoice {
|
||||
function_tools,
|
||||
choice,
|
||||
..
|
||||
} = normalize_tool_choice(
|
||||
&tool_params(ctx.tools, ctx.tool_strict_level),
|
||||
ctx.tool_choice.clone(),
|
||||
)?;
|
||||
if function_tools.is_empty() {
|
||||
return Ok(match choice {
|
||||
// `none`, or no tools to call: the model can only answer.
|
||||
BuilderToolChoice::Auto => {
|
||||
Some(BuiltOutputGrammar::from_token_zero(Format::sequence(vec![
|
||||
reasoning,
|
||||
answer(answer_format),
|
||||
])))
|
||||
}
|
||||
BuilderToolChoice::Required | BuilderToolChoice::Forced => None,
|
||||
});
|
||||
}
|
||||
|
||||
let mut calls = function_tools
|
||||
.iter()
|
||||
.map(|tool| {
|
||||
let header =
|
||||
Format::const_string(format!(" to={}{CALL_HEADER_END}", tool.function.name));
|
||||
let schema = tool.function.parameters.clone().unwrap_or(Value::Bool(true));
|
||||
Format::sequence(vec![header, arguments(&schema, lenient)])
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if matches!(choice, BuilderToolChoice::Auto) {
|
||||
let names = ctx.tools.iter().map(|tool| tool.name.as_str()).collect::<Vec<_>>();
|
||||
let header = Format::sequence(vec![
|
||||
Format::const_string(" to="),
|
||||
Format::regex(other_names(&names)),
|
||||
Format::const_string(CALL_HEADER_END),
|
||||
]);
|
||||
calls.push(Format::sequence(vec![
|
||||
header,
|
||||
arguments(&Value::Bool(true), lenient),
|
||||
]));
|
||||
}
|
||||
let call = one_of(calls);
|
||||
let mut calls = vec![call.clone()];
|
||||
if ctx.parallel_tool_calls && !choice.is_forced() {
|
||||
calls.push(Format::star(Format::sequence(vec![
|
||||
Format::const_string(NEXT_MESSAGE),
|
||||
call,
|
||||
])));
|
||||
}
|
||||
// The last call ends with the `<|call|>` stop token.
|
||||
calls.push(Format::optional(Format::const_string(CALL)));
|
||||
let calls = Format::sequence(calls);
|
||||
let output = match choice {
|
||||
BuilderToolChoice::Auto => Format::or(vec![calls, answer(answer_format)]),
|
||||
BuilderToolChoice::Required | BuilderToolChoice::Forced => calls,
|
||||
};
|
||||
Ok(Some(BuiltOutputGrammar::from_token_zero(Format::sequence(
|
||||
vec![reasoning, output],
|
||||
))))
|
||||
}
|
||||
|
||||
/// A final answer, ending with the `<|return|>` stop token.
|
||||
fn answer(format: Option<&Format>) -> Format {
|
||||
Format::sequence([vec![Format::const_string(FINAL_OPEN)], answer_rest(format)].concat())
|
||||
}
|
||||
|
||||
/// The rest of an open final answer, holding `format` or else free text.
|
||||
fn answer_rest(format: Option<&Format>) -> Vec<Format> {
|
||||
vec![
|
||||
// Without `<|end|>`, no unconstrained message can follow the answer.
|
||||
format.map_or_else(|| Format::any_text_excluding(&[END, RETURN]), Format::clone),
|
||||
Format::optional(Format::const_string(RETURN)),
|
||||
]
|
||||
}
|
||||
|
||||
/// The arguments under `schema`: its declared parameters in declared or
|
||||
/// alphabetical order, required ones mandatory, then any additional ones. If
|
||||
/// `lenient`, each value but the last may end by opening the next parameter.
|
||||
fn arguments(schema: &Value, lenient: bool) -> Format {
|
||||
let options = ArgumentOptions {
|
||||
excludes: CALL_TERMINATORS.map(str::to_string).to_vec(),
|
||||
..ArgumentOptions::default()
|
||||
};
|
||||
let render = |syntax: BeamArguments| arguments::parameters(schema, &syntax, &options);
|
||||
if lenient {
|
||||
lenient_arguments(render(BeamArguments::Joined), render(BeamArguments::Last))
|
||||
} else {
|
||||
closed_arguments(render(BeamArguments::Closed))
|
||||
}
|
||||
}
|
||||
|
||||
/// Arguments whose values all end with a closer.
|
||||
fn closed_arguments(
|
||||
Parameters {
|
||||
declared,
|
||||
additional,
|
||||
}: Parameters,
|
||||
) -> Format {
|
||||
let additional = additional.map(Format::star);
|
||||
let lists = orders(&declared, |parameter| ¶meter.key)
|
||||
.into_iter()
|
||||
.map(|order| {
|
||||
let listed = order
|
||||
.into_iter()
|
||||
.map(|parameter| listed(parameter.format.clone(), parameter.required));
|
||||
listed.chain(additional.clone()).collect::<Vec<_>>()
|
||||
})
|
||||
.filter(|list| !list.is_empty())
|
||||
.map(Format::sequence)
|
||||
.collect::<Vec<_>>();
|
||||
if lists.is_empty() {
|
||||
Format::const_string("")
|
||||
} else {
|
||||
one_of(lists)
|
||||
}
|
||||
}
|
||||
|
||||
/// A declared parameter as one another parameter follows, `joined`, and as the
|
||||
/// last one.
|
||||
struct LenientParameter {
|
||||
key: String,
|
||||
joined: Format,
|
||||
last: Format,
|
||||
required: bool,
|
||||
}
|
||||
|
||||
impl LenientParameter {
|
||||
fn joined(&self) -> Format {
|
||||
listed(self.joined.clone(), self.required)
|
||||
}
|
||||
}
|
||||
|
||||
/// Arguments whose values end by opening the next parameter, but for the last
|
||||
/// one, from the parameters as [`BeamArguments::Joined`] and as
|
||||
/// [`BeamArguments::Last`].
|
||||
fn lenient_arguments(joined: Parameters, last: Parameters) -> Format {
|
||||
// Both forms share their terminators, so they keep the same parameters.
|
||||
let declared = joined
|
||||
.declared
|
||||
.into_iter()
|
||||
.zip(last.declared)
|
||||
.map(|(joined, last)| LenientParameter {
|
||||
key: joined.key,
|
||||
joined: joined.format,
|
||||
last: last.format,
|
||||
required: joined.required,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let additional = joined.additional.zip(last.additional);
|
||||
|
||||
let mut lists = vec![];
|
||||
for order in orders(&declared, |parameter| ¶meter.key) {
|
||||
// The last parameter is a declared one that follows every required
|
||||
// one, or an additional one.
|
||||
if !order.is_empty() {
|
||||
let first = order.iter().rposition(|parameter| parameter.required).unwrap_or(0);
|
||||
let mut list =
|
||||
order[..first].iter().map(|parameter| parameter.joined()).collect::<Vec<_>>();
|
||||
list.push(last_present(&order[first..]));
|
||||
lists.push(Format::sequence(list));
|
||||
}
|
||||
if let Some((joined, last)) = &additional {
|
||||
let mut list = order.iter().map(|parameter| parameter.joined()).collect::<Vec<_>>();
|
||||
list.extend([Format::star(joined.clone()), last.clone()]);
|
||||
lists.push(Format::sequence(list));
|
||||
}
|
||||
}
|
||||
if lists.is_empty() {
|
||||
return Format::const_string("");
|
||||
}
|
||||
|
||||
// Each parameter starts past the opener, as the value before writes it.
|
||||
let arguments = Format::sequence(vec![
|
||||
separator(),
|
||||
Format::const_string(PARAMETER_OPEN),
|
||||
one_of(lists),
|
||||
]);
|
||||
if declared.iter().any(|parameter| parameter.required) {
|
||||
arguments
|
||||
} else {
|
||||
Format::optional(arguments)
|
||||
}
|
||||
}
|
||||
|
||||
/// `parameters`, at least one, in order and the last present one in its last
|
||||
/// form. Only the first may be required.
|
||||
fn last_present(parameters: &[&LenientParameter]) -> Format {
|
||||
if let [only] = parameters {
|
||||
return only.last.clone();
|
||||
}
|
||||
// The last one is in the first half, or after all of it. Halving keeps the
|
||||
// grammar O(n log n) in size and O(log n) deep.
|
||||
let (head, tail) = parameters.split_at(parameters.len() / 2);
|
||||
let mut after_head = head.iter().map(|parameter| parameter.joined()).collect::<Vec<_>>();
|
||||
after_head.push(last_present(tail));
|
||||
Format::or(vec![last_present(head), Format::sequence(after_head)])
|
||||
}
|
||||
|
||||
/// `parameters` in declared order, and in alphabetical order by `key` if that
|
||||
/// differs, since Beam often writes them sorted.
|
||||
fn orders<P>(parameters: &[P], key: impl Fn(&P) -> &str) -> Vec<Vec<&P>> {
|
||||
let declared = parameters.iter().collect::<Vec<_>>();
|
||||
let mut sorted = declared.clone();
|
||||
sorted.sort_by_key(|parameter| key(parameter));
|
||||
if sorted
|
||||
.iter()
|
||||
.map(|parameter| key(parameter))
|
||||
.eq(declared.iter().map(|parameter| key(parameter)))
|
||||
{
|
||||
vec![declared]
|
||||
} else {
|
||||
vec![declared, sorted]
|
||||
}
|
||||
}
|
||||
|
||||
/// `parameter` as listed: optional unless `required`.
|
||||
fn listed(parameter: Format, required: bool) -> Format {
|
||||
if required {
|
||||
parameter
|
||||
} else {
|
||||
Format::optional(parameter)
|
||||
}
|
||||
}
|
||||
|
||||
fn separator() -> Format {
|
||||
Format::optional(Format::const_string("\n"))
|
||||
}
|
||||
|
||||
/// A regex for the tool names the call header can spell (no whitespace or
|
||||
/// `<`) besides `names`.
|
||||
fn other_names(names: &[&str]) -> String {
|
||||
#[derive(Default)]
|
||||
struct Node {
|
||||
children: BTreeMap<char, Node>,
|
||||
end: bool,
|
||||
}
|
||||
|
||||
fn escape(char: char, meta: &str) -> String {
|
||||
if meta.contains(char) {
|
||||
format!("\\{char}")
|
||||
} else {
|
||||
char.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// A name character other than `chars`.
|
||||
fn other<'a>(chars: impl Iterator<Item = &'a char>) -> String {
|
||||
let chars = chars.map(|&char| escape(char, r"\]^-[")).collect::<String>();
|
||||
format!(r"[^\s<{chars}]")
|
||||
}
|
||||
|
||||
/// The rest of a name past `node` that ends on none of `names`.
|
||||
fn rest(node: &Node, first: bool) -> String {
|
||||
let any = other([].iter());
|
||||
let mut branches = vec![format!("{}{any}*", other(node.children.keys()))];
|
||||
branches.extend(node.children.iter().map(|(&char, child)| {
|
||||
format!("{}{}", escape(char, r".^$*+?{}[]\|()"), rest(child, false))
|
||||
}));
|
||||
let optional = if first || node.end { "" } else { "?" };
|
||||
format!("(?:{}){optional}", branches.join("|"))
|
||||
}
|
||||
|
||||
let mut root = Node::default();
|
||||
for name in names {
|
||||
let mut node = &mut root;
|
||||
for char in name.chars() {
|
||||
node = node.children.entry(char).or_default();
|
||||
}
|
||||
node.end = true;
|
||||
}
|
||||
rest(&root, true)
|
||||
}
|
||||
|
||||
/// How one parameter is written: `<rfl:parameter name="KEY">\nVALUE` followed
|
||||
/// by its end.
|
||||
///
|
||||
/// The parser reads every value as text and converts it by the tool schema, so
|
||||
/// string options are raw text and every other option is JSON.
|
||||
enum BeamArguments {
|
||||
/// The whole parameter, ending with `\n</rfl:parameter>`, after an optional
|
||||
/// newline.
|
||||
Closed,
|
||||
/// The parameter past its opener, where the template looks ahead for the
|
||||
/// next one: it ends by opening the next parameter, also right after the
|
||||
/// value's newline, so raw text holds no such opener.
|
||||
Joined,
|
||||
/// The last parameter past its opener under such a template, ending with
|
||||
/// `\n</rfl:parameter>`.
|
||||
Last,
|
||||
}
|
||||
|
||||
impl ArgumentSyntax for BeamArguments {
|
||||
fn separator(&self) -> Option<Format> {
|
||||
None
|
||||
}
|
||||
|
||||
fn parameter(&self, key: ParameterKey<'_>, options: &[ValueOption<'_>]) -> Option<Format> {
|
||||
let terminators: &[&str] = match self {
|
||||
Self::Closed => &["</rfl:parameter>"],
|
||||
Self::Joined | Self::Last => &["</rfl:parameter>", NEXT_PARAMETER],
|
||||
};
|
||||
let values = options
|
||||
.iter()
|
||||
.filter_map(|option| match option.ty {
|
||||
JsonType::String => option.raw_string(terminators),
|
||||
_ => Some(option.json()),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
(!values.is_empty()).then(|| {
|
||||
let value = one_of(values);
|
||||
match self {
|
||||
Self::Closed => Format::sequence(vec![
|
||||
separator(),
|
||||
key.tag(PARAMETER_OPEN, PARAMETER_NAME_CLOSE, value, PARAMETER_CLOSE),
|
||||
]),
|
||||
Self::Joined => key.tag(
|
||||
"",
|
||||
PARAMETER_NAME_CLOSE,
|
||||
value,
|
||||
NEXT_PARAMETER_ENDS.to_vec(),
|
||||
),
|
||||
Self::Last => key.tag("", PARAMETER_NAME_CLOSE, value, PARAMETER_CLOSE),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,897 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::{Value, json};
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use vllm_tokenizer::{DecodedText, Tokenizer as _};
|
||||
use xgrammar_structural_tag::ToolChoice;
|
||||
use xgrammar_structural_tag::format::Format;
|
||||
|
||||
use super::{BeamUnifiedParser, HfPosition, build_output_grammar, other_names};
|
||||
use crate::output_grammar::test_utils::outline;
|
||||
use crate::output_grammar::{GrammarCoverage, OutputGrammarContext, ToolStrictLevel};
|
||||
use crate::tool::Tool;
|
||||
use crate::unified::{ResponseTemplate, UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
|
||||
|
||||
/// The `tag_pattern` of the checkpoint of [`template`].
|
||||
const TAG_PATTERN: &str =
|
||||
r#"<rfl:parameter name="(?P<key>[^"]+)">\n(?P<value>.*?)\n</rfl:parameter>"#;
|
||||
/// [`TAG_PATTERN`], but a value may also end where the next parameter opens.
|
||||
const LENIENT_TAG_PATTERN: &str = r#"<rfl:parameter name="(?P<key>[^"]+)">\n(?P<value>.*?)\n(?:</rfl:parameter>|(?=<rfl:parameter name="))"#;
|
||||
|
||||
/// The Beam-1.0 `response_template`, with `tag_pattern`.
|
||||
fn template(tag_pattern: &str) -> Arc<ResponseTemplate> {
|
||||
template_with(tag_pattern, json!("<|end|><|start|>assistant"))
|
||||
}
|
||||
|
||||
/// [`template`] with `thinking_close` as the reasoning closer.
|
||||
fn template_with(tag_pattern: &str, thinking_close: Value) -> Arc<ResponseTemplate> {
|
||||
let template = json!({
|
||||
"defaults": {"role": "assistant"},
|
||||
"start_anchor_pattern": r"<\|start\|>assistant",
|
||||
"fields": {
|
||||
"thinking": {
|
||||
"open": "<|channel|>analysis<|message|>",
|
||||
"close": thinking_close,
|
||||
"content": "text"
|
||||
},
|
||||
"content": {
|
||||
"open": "<|channel|>final<|message|>",
|
||||
"close": ["<|return|>", "<|end|>", "<|call|>"],
|
||||
"content": "text"
|
||||
},
|
||||
"tool_calls": {
|
||||
"open_pattern": r" to=(?P<name>[^\s<]+)<\|channel\|>analysis <\|constrain\|>xml<\|message\|>",
|
||||
"close": ["<|end|><|start|>assistant", "<|call|>"],
|
||||
"repeats": true,
|
||||
"content": "xml-inline",
|
||||
"content_args": {
|
||||
"tag_pattern": tag_pattern,
|
||||
"value_parser": {"name": "text", "args": {"strip": false}}
|
||||
},
|
||||
"transform": {
|
||||
"type": "function",
|
||||
"function": {"name": "{name}", "arguments": "{content}"}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
Arc::new(ResponseTemplate::from_json(&template).unwrap())
|
||||
}
|
||||
|
||||
fn tools() -> Vec<Tool> {
|
||||
vec![
|
||||
Tool {
|
||||
name: "get_weather".to_string(),
|
||||
description: None,
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"},
|
||||
"days": {"type": "integer"},
|
||||
"unit": {"enum": ["c", "f"]}
|
||||
},
|
||||
"required": ["location"],
|
||||
"additionalProperties": false
|
||||
}),
|
||||
strict: Some(true),
|
||||
defer_loading: None,
|
||||
},
|
||||
Tool {
|
||||
name: "get_time".to_string(),
|
||||
description: None,
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {"zone": {"type": "string"}}
|
||||
}),
|
||||
strict: None,
|
||||
defer_loading: None,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/// A parser initialized from `prompt`, the rendered conversation.
|
||||
fn parser(prompt: &str) -> BeamUnifiedParser {
|
||||
parser_with(TAG_PATTERN, prompt)
|
||||
}
|
||||
|
||||
fn parser_with(tag_pattern: &str, prompt: &str) -> BeamUnifiedParser {
|
||||
let tokenizer = Arc::new(TestTokenizer::new());
|
||||
let prompt_ids = tokenizer.encode(prompt, false).unwrap();
|
||||
let mut parser = BeamUnifiedParser::new(template(tag_pattern), &tools(), tokenizer);
|
||||
parser.initialize(&prompt_ids).unwrap();
|
||||
parser
|
||||
}
|
||||
|
||||
const PROMPT: &str = "<|start|>user<|message|>Weather in Paris?<|end|><|start|>assistant";
|
||||
|
||||
fn grammar(position: HfPosition, tool_choice: ToolChoice, parallel: bool) -> Option<String> {
|
||||
grammar_with(position, &tools(), tool_choice, parallel, false, false)
|
||||
}
|
||||
|
||||
fn grammar_with(
|
||||
position: HfPosition,
|
||||
tools: &[Tool],
|
||||
tool_choice: ToolChoice,
|
||||
parallel: bool,
|
||||
always_constrain: bool,
|
||||
lenient: bool,
|
||||
) -> Option<String> {
|
||||
let ctx = OutputGrammarContext {
|
||||
tools,
|
||||
tool_choice: &tool_choice,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
parallel_tool_calls: parallel,
|
||||
answer: None,
|
||||
};
|
||||
grammar_in(position, &ctx, always_constrain, lenient)
|
||||
}
|
||||
|
||||
fn grammar_in(
|
||||
position: HfPosition,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
always_constrain: bool,
|
||||
lenient: bool,
|
||||
) -> Option<String> {
|
||||
let grammar = build_output_grammar(
|
||||
position,
|
||||
ctx,
|
||||
always_constrain,
|
||||
lenient,
|
||||
template(TAG_PATTERN).reasoning_closers(),
|
||||
)
|
||||
.unwrap()?;
|
||||
assert_eq!(grammar.coverage, GrammarCoverage::FromTokenZero);
|
||||
Some(outline(&grammar.format))
|
||||
}
|
||||
|
||||
fn constrained_grammar(
|
||||
position: HfPosition,
|
||||
tool_choice: ToolChoice,
|
||||
answer: Format,
|
||||
always_constrain: bool,
|
||||
) -> Option<String> {
|
||||
let tools = tools();
|
||||
let ctx = OutputGrammarContext {
|
||||
tools: &tools,
|
||||
tool_choice: &tool_choice,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
parallel_tool_calls: true,
|
||||
answer: Some(&answer),
|
||||
};
|
||||
grammar_in(position, &ctx, always_constrain, false)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn position_follows_the_prompt_and_then_the_output() {
|
||||
let position = |suffix: &str| parser(&format!("{PROMPT}{suffix}")).inner.position();
|
||||
|
||||
assert_eq!(position(""), HfPosition::ResponseStart);
|
||||
assert_eq!(
|
||||
position("<|channel|>analysis<|message|>Let me"),
|
||||
HfPosition::Reasoning
|
||||
);
|
||||
// The reasoning closer ends with the anchor, so the next message starts.
|
||||
assert_eq!(
|
||||
position("<|channel|>analysis<|message|>Done.<|end|><|start|>assistant"),
|
||||
HfPosition::ResponseStart
|
||||
);
|
||||
// A partial closer stays buffered.
|
||||
assert_eq!(
|
||||
position("<|channel|>analysis<|message|>Done.<|end|>"),
|
||||
HfPosition::Other
|
||||
);
|
||||
assert_eq!(
|
||||
position("<|channel|>final<|message|>It is"),
|
||||
HfPosition::Content
|
||||
);
|
||||
// A partial boundary stays buffered.
|
||||
assert_eq!(position("<|channel|>ana"), HfPosition::Other);
|
||||
// Without the anchor, the parser parses the output from the start.
|
||||
assert_eq!(
|
||||
parser("no anchor").inner.position(),
|
||||
HfPosition::ResponseStart
|
||||
);
|
||||
|
||||
let mut parser = parser(PROMPT);
|
||||
parser
|
||||
.parse_into(
|
||||
DecodedText::unattributed("<|channel|>analysis<|message|>Hmm"),
|
||||
&mut UnifiedParserOutput::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(parser.inner.position(), HfPosition::Reasoning);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn required_calls_any_tool_after_reasoning() {
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
|
||||
sequence
|
||||
/[^<]/
|
||||
text
|
||||
sequence
|
||||
or
|
||||
sequence
|
||||
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
or
|
||||
sequence
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
star
|
||||
sequence
|
||||
`<|end|><|start|>assistant`
|
||||
or
|
||||
sequence
|
||||
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
or
|
||||
sequence
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional `<|call|>`
|
||||
"#]].assert_eq(
|
||||
&grammar(HfPosition::ResponseStart, ToolChoice::required(), true).unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_with_a_strict_tool_also_allows_an_answer() {
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
|
||||
sequence
|
||||
/[^<]/
|
||||
text
|
||||
or
|
||||
sequence
|
||||
or
|
||||
sequence
|
||||
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
or
|
||||
sequence
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
sequence
|
||||
` to=`
|
||||
/(?:[^\s<g][^\s<]*|g(?:[^\s<e][^\s<]*|e(?:[^\s<t][^\s<]*|t(?:[^\s<_][^\s<]*|_(?:[^\s<tw][^\s<]*|t(?:[^\s<i][^\s<]*|i(?:[^\s<m][^\s<]*|m(?:[^\s<e][^\s<]*|e(?:[^\s<][^\s<]*))?)?)?|w(?:[^\s<e][^\s<]*|e(?:[^\s<a][^\s<]*|a(?:[^\s<t][^\s<]*|t(?:[^\s<h][^\s<]*|h(?:[^\s<e][^\s<]*|e(?:[^\s<r][^\s<]*|r(?:[^\s<][^\s<]*))?)?)?)?)?)?)?)?)?)?)/
|
||||
`<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
star
|
||||
sequence
|
||||
optional `\n`
|
||||
sequence
|
||||
`<rfl:parameter name="`
|
||||
/[a-zA-Z_][a-zA-Z0-9_]*/
|
||||
`">\n`
|
||||
or
|
||||
text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`]
|
||||
json(number) excluding [`<|end|>`, `<|call|>`]
|
||||
json(boolean) excluding [`<|end|>`, `<|call|>`]
|
||||
json(null) excluding [`<|end|>`, `<|call|>`]
|
||||
json({ ... }) excluding [`<|end|>`, `<|call|>`]
|
||||
json(any[]) excluding [`<|end|>`, `<|call|>`]
|
||||
`\n</rfl:parameter>`
|
||||
star
|
||||
sequence
|
||||
`<|end|><|start|>assistant`
|
||||
or
|
||||
sequence
|
||||
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
or
|
||||
sequence
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
sequence
|
||||
` to=`
|
||||
/(?:[^\s<g][^\s<]*|g(?:[^\s<e][^\s<]*|e(?:[^\s<t][^\s<]*|t(?:[^\s<_][^\s<]*|_(?:[^\s<tw][^\s<]*|t(?:[^\s<i][^\s<]*|i(?:[^\s<m][^\s<]*|m(?:[^\s<e][^\s<]*|e(?:[^\s<][^\s<]*))?)?)?|w(?:[^\s<e][^\s<]*|e(?:[^\s<a][^\s<]*|a(?:[^\s<t][^\s<]*|t(?:[^\s<h][^\s<]*|h(?:[^\s<e][^\s<]*|e(?:[^\s<r][^\s<]*|r(?:[^\s<][^\s<]*))?)?)?)?)?)?)?)?)?)?)/
|
||||
`<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
star
|
||||
sequence
|
||||
optional `\n`
|
||||
sequence
|
||||
`<rfl:parameter name="`
|
||||
/[a-zA-Z_][a-zA-Z0-9_]*/
|
||||
`">\n`
|
||||
or
|
||||
text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`]
|
||||
json(number) excluding [`<|end|>`, `<|call|>`]
|
||||
json(boolean) excluding [`<|end|>`, `<|call|>`]
|
||||
json(null) excluding [`<|end|>`, `<|call|>`]
|
||||
json({ ... }) excluding [`<|end|>`, `<|call|>`]
|
||||
json(any[]) excluding [`<|end|>`, `<|call|>`]
|
||||
`\n</rfl:parameter>`
|
||||
optional `<|call|>`
|
||||
sequence
|
||||
`<|channel|>final<|message|>`
|
||||
text excluding [`<|end|>`, `<|return|>`]
|
||||
optional `<|return|>`
|
||||
"#]]
|
||||
.assert_eq(&grammar(HfPosition::ResponseStart, ToolChoice::auto(), true).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn named_choice_calls_that_tool_once_after_prompted_reasoning() {
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `` text `<|end|><|start|>assistant`
|
||||
sequence
|
||||
sequence
|
||||
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
or
|
||||
sequence
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
optional `<|call|>`
|
||||
"#]].assert_eq(
|
||||
&grammar(HfPosition::Reasoning, ToolChoice::function("get_weather"), true).unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_grammar_without_a_constraining_request_or_known_position() {
|
||||
let non_strict = vec![tools().pop().unwrap()];
|
||||
for tools in [&[][..], &non_strict] {
|
||||
assert!(
|
||||
grammar_with(
|
||||
HfPosition::ResponseStart,
|
||||
tools,
|
||||
ToolChoice::auto(),
|
||||
true,
|
||||
false,
|
||||
false
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
grammar_with(
|
||||
HfPosition::Other,
|
||||
&tools(),
|
||||
ToolChoice::required(),
|
||||
true,
|
||||
true,
|
||||
false
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn none_with_tools_only_answers() {
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
|
||||
sequence
|
||||
/[^<]/
|
||||
text
|
||||
sequence
|
||||
`<|channel|>final<|message|>`
|
||||
text excluding [`<|end|>`, `<|return|>`]
|
||||
optional `<|return|>`
|
||||
"#]]
|
||||
.assert_eq(&grammar(HfPosition::ResponseStart, ToolChoice::none(), true).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn always_constrain_answers_without_tools() {
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `` text `<|end|><|start|>assistant`
|
||||
sequence
|
||||
`<|channel|>final<|message|>`
|
||||
text excluding [`<|end|>`, `<|return|>`]
|
||||
optional `<|return|>`
|
||||
"#]]
|
||||
.assert_eq(
|
||||
&grammar_with(
|
||||
HfPosition::Reasoning,
|
||||
&[],
|
||||
ToolChoice::auto(),
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn always_constrain_allows_non_strict_calls_or_an_answer() {
|
||||
let non_strict = vec![tools().pop().unwrap()];
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
|
||||
sequence
|
||||
/[^<]/
|
||||
text
|
||||
or
|
||||
sequence
|
||||
or
|
||||
sequence
|
||||
` to=get_time<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
optional
|
||||
sequence
|
||||
optional `\n`
|
||||
tag `<rfl:parameter name="zone">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
sequence
|
||||
` to=`
|
||||
/(?:[^\s<g][^\s<]*|g(?:[^\s<e][^\s<]*|e(?:[^\s<t][^\s<]*|t(?:[^\s<_][^\s<]*|_(?:[^\s<t][^\s<]*|t(?:[^\s<i][^\s<]*|i(?:[^\s<m][^\s<]*|m(?:[^\s<e][^\s<]*|e(?:[^\s<][^\s<]*))?)?)?)?)?)?)?)/
|
||||
`<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
star
|
||||
sequence
|
||||
optional `\n`
|
||||
sequence
|
||||
`<rfl:parameter name="`
|
||||
/[a-zA-Z_][a-zA-Z0-9_]*/
|
||||
`">\n`
|
||||
or
|
||||
text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`]
|
||||
json(number) excluding [`<|end|>`, `<|call|>`]
|
||||
json(boolean) excluding [`<|end|>`, `<|call|>`]
|
||||
json(null) excluding [`<|end|>`, `<|call|>`]
|
||||
json({ ... }) excluding [`<|end|>`, `<|call|>`]
|
||||
json(any[]) excluding [`<|end|>`, `<|call|>`]
|
||||
`\n</rfl:parameter>`
|
||||
optional `<|call|>`
|
||||
sequence
|
||||
`<|channel|>final<|message|>`
|
||||
text excluding [`<|end|>`, `<|return|>`]
|
||||
optional `<|return|>`
|
||||
"#]]
|
||||
.assert_eq(
|
||||
&grammar_with(
|
||||
HfPosition::ResponseStart,
|
||||
&non_strict,
|
||||
ToolChoice::auto(),
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn always_constrain_finishes_an_open_answer() {
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
text excluding [`<|end|>`, `<|return|>`]
|
||||
optional `<|return|>`
|
||||
"#]]
|
||||
.assert_eq(
|
||||
&grammar_with(
|
||||
HfPosition::Content,
|
||||
&tools(),
|
||||
ToolChoice::auto(),
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
assert!(
|
||||
grammar_with(
|
||||
HfPosition::Content,
|
||||
&tools(),
|
||||
ToolChoice::auto(),
|
||||
true,
|
||||
false,
|
||||
false
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
grammar_with(
|
||||
HfPosition::Content,
|
||||
&tools(),
|
||||
ToolChoice::required(),
|
||||
true,
|
||||
true,
|
||||
false
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answer_holds_the_requested_format() {
|
||||
let json = || Format::json_schema(json!({"type": "object"}));
|
||||
let answer = constrained_grammar(HfPosition::ResponseStart, ToolChoice::none(), json(), false);
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant`
|
||||
sequence
|
||||
/[^<]/
|
||||
text
|
||||
sequence
|
||||
`<|channel|>final<|message|>`
|
||||
json(object)
|
||||
optional `<|return|>`
|
||||
"#]]
|
||||
.assert_eq(&answer.unwrap());
|
||||
|
||||
// Under `auto`, the answer that may replace calls holds it too.
|
||||
let auto =
|
||||
constrained_grammar(HfPosition::ResponseStart, ToolChoice::auto(), json(), false).unwrap();
|
||||
assert!(auto.ends_with(
|
||||
"`<|channel|>final<|message|>`\n json(object)\n optional `<|return|>`\n"
|
||||
));
|
||||
|
||||
// A forced call leaves no answer to constrain.
|
||||
assert_eq!(
|
||||
constrained_grammar(
|
||||
HfPosition::ResponseStart,
|
||||
ToolChoice::required(),
|
||||
json(),
|
||||
false
|
||||
),
|
||||
grammar(HfPosition::ResponseStart, ToolChoice::required(), true)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_constraint_applies_alone_in_an_open_answer() {
|
||||
let json = Format::json_schema(json!({"type": "object"}));
|
||||
assert!(constrained_grammar(HfPosition::Content, ToolChoice::auto(), json, true).is_none());
|
||||
}
|
||||
|
||||
/// Parse `output` after the prompt under `tag_pattern`, returning the calls as
|
||||
/// `(name, arguments)`.
|
||||
fn calls(tag_pattern: &str, output: &str) -> Vec<(String, Value)> {
|
||||
let mut parser = parser_with(tag_pattern, PROMPT);
|
||||
let mut events = UnifiedParserOutput::default();
|
||||
parser.parse_into(DecodedText::unattributed(output), &mut events).unwrap();
|
||||
events.append(parser.finish().unwrap());
|
||||
let mut calls: Vec<(String, String)> = Vec::new();
|
||||
for event in events.events {
|
||||
if let UnifiedParserEvent::ToolCall(delta) = event {
|
||||
if let Some(name) = delta.name {
|
||||
calls.push((name, String::new()));
|
||||
}
|
||||
calls.last_mut().unwrap().1.push_str(&delta.arguments);
|
||||
}
|
||||
}
|
||||
calls
|
||||
.into_iter()
|
||||
.map(|(name, arguments)| (name, serde_json::from_str(&arguments).unwrap()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Output in the grammar's canonical spelling parses back to the values the
|
||||
/// grammar admitted.
|
||||
#[test]
|
||||
fn parser_reads_back_the_grammar_spelling() {
|
||||
let parameter = |key: &str, value: &str| {
|
||||
format!("<rfl:parameter name=\"{key}\">\n{value}\n</rfl:parameter>")
|
||||
};
|
||||
let call = |name: &str, parameters: &[String]| {
|
||||
format!(
|
||||
" to={name}<|channel|>analysis <|constrain|>xml<|message|>{}",
|
||||
parameters.join("\n")
|
||||
)
|
||||
};
|
||||
let output = format!(
|
||||
"<|channel|>analysis<|message|>Two tools.<|end|><|start|>assistant{}<|end|><|start|>assistant{}",
|
||||
call(
|
||||
"get_weather",
|
||||
&[
|
||||
parameter("location", "Paris"),
|
||||
parameter("days", "2"),
|
||||
parameter("unit", "c")
|
||||
]
|
||||
),
|
||||
call("get_time", &[parameter("zone", "multi\nline")]),
|
||||
);
|
||||
|
||||
for tag_pattern in [TAG_PATTERN, LENIENT_TAG_PATTERN] {
|
||||
assert_eq!(
|
||||
calls(tag_pattern, &output),
|
||||
vec![
|
||||
(
|
||||
"get_weather".to_string(),
|
||||
json!({"location": "Paris", "days": 2, "unit": "c"})
|
||||
),
|
||||
("get_time".to_string(), json!({"zone": "multi\nline"})),
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn values_end_before_the_next_parameter_where_the_template_looks_ahead() {
|
||||
let output = " to=get_weather<|channel|>analysis <|constrain|>xml<|message|>\
|
||||
<rfl:parameter name=\"days\">\n2\n\
|
||||
<rfl:parameter name=\"location\">\nParis\n</rfl:parameter>";
|
||||
|
||||
assert!(parser_with(LENIENT_TAG_PATTERN, PROMPT).lenient);
|
||||
assert_eq!(
|
||||
calls(LENIENT_TAG_PATTERN, output),
|
||||
vec![(
|
||||
"get_weather".to_string(),
|
||||
json!({"days": 2, "location": "Paris"})
|
||||
)]
|
||||
);
|
||||
assert!(!parser(PROMPT).lenient);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reasoning_ends_at_every_closer_of_the_template() {
|
||||
let closers = [
|
||||
"<|end|><|start|>assistant",
|
||||
"<|constrain|><|start|>assistant",
|
||||
"<|start|>assistant",
|
||||
];
|
||||
let tools = tools();
|
||||
let tokenizer = Arc::new(TestTokenizer::new());
|
||||
let mut parser = BeamUnifiedParser::new(
|
||||
template_with(TAG_PATTERN, json!(closers)),
|
||||
&tools,
|
||||
tokenizer.clone(),
|
||||
);
|
||||
parser.initialize(&tokenizer.encode(PROMPT, false).unwrap()).unwrap();
|
||||
let grammar = parser
|
||||
.build_output_grammar(&OutputGrammarContext {
|
||||
tools: &tools,
|
||||
tool_choice: &ToolChoice::required(),
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
parallel_tool_calls: false,
|
||||
answer: None,
|
||||
})
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
outline(&grammar.format).lines().nth(1).unwrap(),
|
||||
" tag `<|channel|>analysis<|message|>` .. `<|constrain|><|start|>assistant` | \
|
||||
`<|end|><|start|>assistant` | `<|start|>assistant`"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lenient_arguments_come_in_declared_or_sorted_order() {
|
||||
expect_test::expect![[r#"
|
||||
sequence
|
||||
tag `` text `<|end|><|start|>assistant`
|
||||
sequence
|
||||
sequence
|
||||
` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>`
|
||||
sequence
|
||||
optional `\n`
|
||||
`<rfl:parameter name="`
|
||||
or
|
||||
sequence
|
||||
or
|
||||
tag `location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`, `\n<rfl:parameter name="`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
tag `location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`, `\n<rfl:parameter name="`] `\n</rfl:parameter><rfl:parameter name="` | `\n</rfl:parameter>\n<rfl:parameter name="` | `\n<rfl:parameter name="`
|
||||
or
|
||||
tag `days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
optional tag `days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter><rfl:parameter name="` | `\n</rfl:parameter>\n<rfl:parameter name="` | `\n<rfl:parameter name="`
|
||||
tag `unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
sequence
|
||||
optional tag `days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n</rfl:parameter><rfl:parameter name="` | `\n</rfl:parameter>\n<rfl:parameter name="` | `\n<rfl:parameter name="`
|
||||
or
|
||||
tag `location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`, `\n<rfl:parameter name="`] `\n</rfl:parameter>`
|
||||
sequence
|
||||
tag `location">\n` text excluding [`<|end|>`, `<|call|>`, `</rfl:parameter>`, `\n<rfl:parameter name="`] `\n</rfl:parameter><rfl:parameter name="` | `\n</rfl:parameter>\n<rfl:parameter name="` | `\n<rfl:parameter name="`
|
||||
tag `unit">\n` .. `\n</rfl:parameter>`
|
||||
or
|
||||
`c`
|
||||
`f`
|
||||
optional `<|call|>`
|
||||
"#]].assert_eq(
|
||||
&grammar_with(
|
||||
HfPosition::Reasoning,
|
||||
&tools(),
|
||||
ToolChoice::function("get_weather"),
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_names_match_every_name_but_the_given_ones() {
|
||||
let regex = regex_automata::meta::Regex::new(&format!(
|
||||
"^(?:{})$",
|
||||
other_names(&["get", "get_time", "a.b"])
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
for name in [
|
||||
"g",
|
||||
"ge",
|
||||
"gets",
|
||||
"get_",
|
||||
"get_t",
|
||||
"get_times",
|
||||
"x",
|
||||
"a",
|
||||
"a.",
|
||||
"axb",
|
||||
"a.bc",
|
||||
] {
|
||||
assert!(regex.is_match(name), "{name}");
|
||||
}
|
||||
for name in ["get", "get_time", "a.b", "", "get time", "get<"] {
|
||||
assert!(!regex.is_match(name), "{name}");
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,12 @@
|
||||
//! Port of `content_parsers.py`. Each parser takes a chunk of captured text and
|
||||
//! parses it into a single key in the output message dictionary.
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use regex_automata::meta::Regex;
|
||||
use regex_automata::util::iter::Searcher;
|
||||
use regex_automata::util::syntax;
|
||||
use regex_automata::{Input, Match};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value};
|
||||
use thiserror_ext::AsReport as _;
|
||||
@@ -22,6 +26,10 @@ use crate::unified::{self, parsing_failed};
|
||||
const LAX_OPEN: char = '\u{1}';
|
||||
const LAX_CLOSE: char = '\u{2}';
|
||||
|
||||
/// Name of the group that stands for the lookahead ending an `xml-inline` tag
|
||||
/// pattern.
|
||||
const AHEAD_GROUP: &str = "lookahead";
|
||||
|
||||
/// A configured content parser (`content` plus `content_args` of one field).
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) enum ContentParser {
|
||||
@@ -66,6 +74,9 @@ pub(super) struct XmlInlineArgs {
|
||||
key_group: usize,
|
||||
/// Index of the `value` group, if the pattern declares one.
|
||||
value_group: Option<usize>,
|
||||
/// Index of the group standing for the lookahead that ends the pattern,
|
||||
/// and the lookahead's own pattern, if any.
|
||||
lookahead: Option<(usize, String)>,
|
||||
value_parser: Option<Box<ContentParser>>,
|
||||
/// Collect duplicate keys into a list.
|
||||
merge_duplicates: bool,
|
||||
@@ -143,7 +154,8 @@ impl ContentParser {
|
||||
}
|
||||
ContentKind::XmlInline => {
|
||||
let spec: XmlInlineSpec = deserialize(&args, &context)?;
|
||||
let tag_pattern = compile_regex(scope, &spec.tag_pattern)?;
|
||||
let (pattern, lookahead) = final_lookahead(&spec.tag_pattern);
|
||||
let tag_pattern = compile_regex(scope, &pattern)?;
|
||||
let group_info = tag_pattern.group_info();
|
||||
let Some(key_group) = group_info.to_index(Default::default(), "key") else {
|
||||
return Err(invalid!(
|
||||
@@ -152,10 +164,14 @@ impl ContentParser {
|
||||
));
|
||||
};
|
||||
let value_group = group_info.to_index(Default::default(), "value");
|
||||
let lookahead = group_info
|
||||
.to_index(Default::default(), AHEAD_GROUP)
|
||||
.zip(lookahead.map(str::to_string));
|
||||
Self::XmlInline(XmlInlineArgs {
|
||||
tag_pattern,
|
||||
key_group,
|
||||
value_group,
|
||||
lookahead,
|
||||
value_parser: ValueParserSpec::compile(scope, spec.value_parser)?,
|
||||
merge_duplicates: spec.merge_duplicates,
|
||||
})
|
||||
@@ -193,6 +209,32 @@ impl ContentParser {
|
||||
Self::KvLines(args) => args.parse(text),
|
||||
}
|
||||
}
|
||||
|
||||
/// The pattern of the lookahead that ends an `xml-inline` tag pattern, if
|
||||
/// any.
|
||||
pub fn lookahead(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::XmlInline(args) => args.lookahead.as_ref().map(|(_, ahead)| ahead.as_str()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `pattern` with a lookahead that ends it matched as the group
|
||||
/// [`AHEAD_GROUP`] instead, as the Rust regex engine has no lookahead, and
|
||||
/// the lookahead's own pattern. The engine rejects any other lookahead.
|
||||
fn final_lookahead(pattern: &str) -> (Cow<'_, str>, Option<&str>) {
|
||||
if let Some((head, rest)) = pattern.split_once("(?=") {
|
||||
// Only closing parentheses may follow a lookahead that ends the pattern.
|
||||
let ahead = rest.trim_end_matches(')');
|
||||
if ahead.len() < rest.len() && !ahead.contains(['(', ')']) {
|
||||
return (
|
||||
format!("{head}(?P<{AHEAD_GROUP}>{rest}").into(),
|
||||
Some(ahead),
|
||||
);
|
||||
}
|
||||
}
|
||||
(pattern.into(), None)
|
||||
}
|
||||
|
||||
impl ValueParserSpec {
|
||||
@@ -258,7 +300,21 @@ impl XmlInlineArgs {
|
||||
/// Parse shallow XML-ish tags into a dict.
|
||||
fn parse(&self, text: &str) -> unified::Result<Value> {
|
||||
let mut out = Map::new();
|
||||
for captures in self.tag_pattern.captures_iter(text) {
|
||||
let mut captures = self.tag_pattern.create_captures();
|
||||
let mut tags = Searcher::new(Input::new(text));
|
||||
while tags
|
||||
.advance(|input| {
|
||||
self.tag_pattern.search_captures(input, &mut captures);
|
||||
// The lookahead's text is not part of the tag.
|
||||
let ahead =
|
||||
self.lookahead.as_ref().and_then(|(group, _)| captures.get_group(*group));
|
||||
Ok(captures.get_match().map(|tag| match ahead {
|
||||
Some(ahead) => Match::new(tag.pattern(), tag.start()..ahead.start),
|
||||
None => tag,
|
||||
}))
|
||||
})
|
||||
.is_some()
|
||||
{
|
||||
let key =
|
||||
captures.get_group(self.key_group).map(|span| &text[span.range()]).ok_or_else(
|
||||
|| parsing_failed!("xml-inline: named group 'key' did not participate"),
|
||||
@@ -490,6 +546,25 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xml_inline_tags_may_end_where_the_pattern_looks_ahead() {
|
||||
let xml = parser(
|
||||
"xml-inline",
|
||||
json!({
|
||||
"tag_pattern": r#"<arg name="(?P<key>\w+)">(?P<value>.*?)\n(?:</arg>|(?=<arg name="))"#,
|
||||
}),
|
||||
);
|
||||
let value = xml
|
||||
.parse("<arg name=\"a\">1\n<arg name=\"b\">x <arg name=\"c\"> y\n</arg>")
|
||||
.unwrap();
|
||||
assert_eq!(value, json!({"a": "1", "b": "x <arg name=\"c\"> y"}));
|
||||
assert_eq!(xml.lookahead(), Some("<arg name=\""));
|
||||
|
||||
// Any other lookahead is unsupported.
|
||||
let args = json!({"tag_pattern": r"(?P<key>\w+)(?=:)(?P<value>.*)"});
|
||||
assert!(ContentParser::new("test", kind("xml-inline"), args.as_object().unwrap()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kv_lines_split_and_strip() {
|
||||
let kv = parser("kv-lines", json!({"value_parser": {"name": "int"}}));
|
||||
|
||||
@@ -25,7 +25,7 @@ mod transform;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub use parser::HfUnifiedParser;
|
||||
pub use parser::{HfPosition, HfUnifiedParser};
|
||||
pub use template::ResponseTemplate;
|
||||
use thiserror::Error;
|
||||
use thiserror_ext::Macro;
|
||||
|
||||
@@ -59,6 +59,20 @@ pub struct HfUnifiedParser {
|
||||
emitted_tool_count: usize,
|
||||
}
|
||||
|
||||
/// Where the parser is in the response; see [`HfUnifiedParser::position`].
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HfPosition {
|
||||
/// Nothing parsed since the start anchor, or since the start of the stream
|
||||
/// when there is none.
|
||||
ResponseStart,
|
||||
/// Inside an open reasoning field.
|
||||
Reasoning,
|
||||
/// Inside an open content field.
|
||||
Content,
|
||||
/// Anywhere else.
|
||||
Other,
|
||||
}
|
||||
|
||||
/// Parser mode: the region currently receiving text.
|
||||
enum HfMode {
|
||||
/// No implicit field is declared: text outside regions is discarded.
|
||||
@@ -156,6 +170,36 @@ impl HfUnifiedParser {
|
||||
parser
|
||||
}
|
||||
|
||||
/// Where the parser is in the response. Between
|
||||
/// [`UnifiedParser::initialize`] and the first generated text, this is where
|
||||
/// the prompt leaves the response.
|
||||
///
|
||||
/// Text outside every region, which a template without an implicit field
|
||||
/// discards, leaves no state behind and so does not move the parser from
|
||||
/// [`HfPosition::ResponseStart`].
|
||||
pub fn position(&self) -> HfPosition {
|
||||
if !self.buffer.text.is_empty() {
|
||||
return HfPosition::Other;
|
||||
}
|
||||
match &self.mode {
|
||||
HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) if occurrence.opened => {
|
||||
match &self.template.regions[occurrence.region].kind {
|
||||
RegionKind::Text(TextRegion {
|
||||
role: TextRole::Reasoning,
|
||||
..
|
||||
}) => HfPosition::Reasoning,
|
||||
RegionKind::Text(TextRegion {
|
||||
role: TextRole::Content,
|
||||
..
|
||||
}) => HfPosition::Content,
|
||||
_ => HfPosition::Other,
|
||||
}
|
||||
}
|
||||
_ if self.closed.contains(&true) || self.emitted_tool_count > 0 => HfPosition::Other,
|
||||
_ => HfPosition::ResponseStart,
|
||||
}
|
||||
}
|
||||
|
||||
/// Clear all per-stream state.
|
||||
fn reset_state(&mut self) {
|
||||
self.buffer.clear();
|
||||
|
||||
@@ -191,6 +191,31 @@ impl ResponseTemplate {
|
||||
})
|
||||
}
|
||||
|
||||
/// The pattern of the lookahead that ends the `xml-inline` tag pattern of
|
||||
/// tool-call arguments, such as for the opener of the next argument.
|
||||
pub fn argument_lookahead(&self) -> Option<&str> {
|
||||
self.regions.iter().find_map(|region| match ®ion.kind {
|
||||
RegionKind::ToolCalls(calls) => calls.content.lookahead(),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
/// The literal closers of the reasoning field, longest first, or none if it
|
||||
/// has no literal close.
|
||||
pub fn reasoning_closers(&self) -> &[String] {
|
||||
self.regions
|
||||
.iter()
|
||||
.find_map(|region| match (®ion.kind, ®ion.close) {
|
||||
(RegionKind::Text(text), Some(Boundary::Literals(closers)))
|
||||
if text.role == TextRole::Reasoning =>
|
||||
{
|
||||
Some(closers.as_slice())
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Return the start and end offsets of the last start-anchor match in
|
||||
/// `prompt`.
|
||||
pub(super) fn last_start_anchor(&self, prompt: &str) -> Option<(usize, usize)> {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
//! Unified parser interface for reasoning and tool-call deltas.
|
||||
|
||||
mod beam;
|
||||
mod combined;
|
||||
mod gemma4;
|
||||
mod hf;
|
||||
@@ -10,9 +11,10 @@ mod hy;
|
||||
mod inkling;
|
||||
mod kimi_k3;
|
||||
|
||||
pub use beam::BeamUnifiedParser;
|
||||
pub use combined::CombinedParser;
|
||||
pub use gemma4::Gemma4UnifiedParser;
|
||||
pub use hf::{HfTemplateError, HfUnifiedParser, ResponseTemplate};
|
||||
pub use hf::{HfPosition, HfTemplateError, HfUnifiedParser, ResponseTemplate};
|
||||
pub use hy::{HyV3UnifiedParser, HyV4UnifiedParser};
|
||||
pub use inkling::InklingUnifiedParser;
|
||||
pub use kimi_k3::KimiK3UnifiedParser;
|
||||
|
||||
@@ -0,0 +1,747 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from openai.types.responses import FunctionTool, ToolChoiceFunction
|
||||
from xgrammar import Grammar
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.config.reasoning import ReasoningConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponsesRequest,
|
||||
ResponsesResponse,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.parser.beam import BeamParser, BeamReasoningParser, BeamToolParser
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.parser.response_template import ResponseTemplateParser
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
|
||||
# The `response_template` of the Beam checkpoints.
|
||||
BEAM_RESPONSE_TEMPLATE: dict[str, Any] = {
|
||||
"defaults": {"role": "assistant"},
|
||||
"start_anchor_pattern": r"<\|start\|>assistant",
|
||||
"fields": {
|
||||
"thinking": {
|
||||
"open": "<|channel|>analysis<|message|>",
|
||||
"close": "<|end|><|start|>assistant",
|
||||
"content": "text",
|
||||
},
|
||||
"content": {
|
||||
"open": "<|channel|>final<|message|>",
|
||||
"close": ["<|return|>", "<|end|>", "<|call|>"],
|
||||
"content": "text",
|
||||
},
|
||||
"tool_calls": {
|
||||
"open_pattern": (
|
||||
r" to=(?P<name>[^\s<]+)"
|
||||
r"<\|channel\|>analysis <\|constrain\|>xml<\|message\|>"
|
||||
),
|
||||
"close": ["<|end|><|start|>assistant", "<|call|>"],
|
||||
"repeats": True,
|
||||
"content": "xml-inline",
|
||||
"content_args": {
|
||||
"tag_pattern": (
|
||||
r'<rfl:parameter name="(?P<key>[^"]+)">'
|
||||
r'\n(?P<value>.*?)\n(?:</rfl:parameter>|(?=<rfl:parameter name="))'
|
||||
),
|
||||
"value_parser": {"name": "text", "args": {"strip": False}},
|
||||
},
|
||||
"transform": {
|
||||
"type": "function",
|
||||
"function": {"name": "{name}", "arguments": "{content}"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
VOCAB = {
|
||||
"<|start|>": 200,
|
||||
"<|channel|>": 201,
|
||||
"<|message|>": 202,
|
||||
"<|end|>": 203,
|
||||
"<|call|>": 204,
|
||||
"<|return|>": 205,
|
||||
"<|constrain|>": 206,
|
||||
}
|
||||
TEXT_TOKENS = {100: "assistant", 101: "analysis", 102: "final", 103: "plan"}
|
||||
PROMPT_TOKEN_IDS = [200, 100]
|
||||
THOUGHT = "<|channel|>analysis<|message|>plan<|end|><|start|>assistant"
|
||||
FINAL = "<|channel|>final<|message|>"
|
||||
NEXT_MESSAGE = "<|end|><|start|>assistant"
|
||||
|
||||
|
||||
def _tool(name, properties, *, strict=None):
|
||||
return ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": name,
|
||||
"parameters": {"type": "object", "properties": properties},
|
||||
"strict": strict,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
TOOLS = [
|
||||
_tool("get_weather", {"city": {"type": "string"}}),
|
||||
_tool("set_count", {"count": {"type": "integer"}}),
|
||||
]
|
||||
STRICT_TOOLS = [
|
||||
_tool("get_weather", {"city": {"type": "string"}}),
|
||||
_tool("set_count", {"count": {"type": "integer"}}, strict=True),
|
||||
]
|
||||
|
||||
|
||||
class BeamTokenizer:
|
||||
def __init__(self, response_template=BEAM_RESPONSE_TEMPLATE):
|
||||
self.response_template = response_template
|
||||
self.init_kwargs = {}
|
||||
self.name_or_path = ""
|
||||
self.all_special_tokens = list(VOCAB)
|
||||
self.all_special_ids = list(VOCAB.values())
|
||||
|
||||
def decode(self, token_ids, skip_special_tokens=False, **_kwargs):
|
||||
inverse_vocab = {token_id: token for token, token_id in VOCAB.items()}
|
||||
return "".join(
|
||||
TEXT_TOKENS.get(token_id, inverse_vocab.get(token_id, ""))
|
||||
for token_id in token_ids
|
||||
)
|
||||
|
||||
def get_vocab(self):
|
||||
return VOCAB
|
||||
|
||||
def encode(self, text, add_special_tokens=False):
|
||||
del add_special_tokens
|
||||
return list(text.encode())
|
||||
|
||||
|
||||
def request(**overrides):
|
||||
values = dict(
|
||||
tool_choice="auto",
|
||||
include_reasoning=True,
|
||||
tools=TOOLS,
|
||||
parallel_tool_calls=True,
|
||||
)
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def parameter(name, value):
|
||||
return f'<rfl:parameter name="{name}">\n{value}\n</rfl:parameter>'
|
||||
|
||||
|
||||
def call(name, body, *, terminator="<|call|>"):
|
||||
return (
|
||||
f" to={name}<|channel|>analysis <|constrain|>xml<|message|>{body}{terminator}"
|
||||
)
|
||||
|
||||
|
||||
def make_parser(prompt_token_ids=PROMPT_TOKEN_IDS, tools=TOOLS, **kwargs):
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="beam",
|
||||
reasoning_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(),
|
||||
)
|
||||
assert parser_cls is not None
|
||||
parser = parser_cls(BeamTokenizer(), tools, **kwargs)
|
||||
parser.set_prompt_token_ids(prompt_token_ids)
|
||||
return parser
|
||||
|
||||
|
||||
def parse(text, prompt_token_ids=PROMPT_TOKEN_IDS):
|
||||
reasoning, content, calls = make_parser(prompt_token_ids).parse(
|
||||
text, request(), enable_auto_tools=True
|
||||
)
|
||||
return reasoning, content, normalize_calls(calls)
|
||||
|
||||
|
||||
def normalize_calls(calls):
|
||||
return [(call.name, json.loads(call.arguments)) for call in calls or []]
|
||||
|
||||
|
||||
def test_beam_parsers_are_registered():
|
||||
assert ToolParserManager.get_tool_parser("beam") is BeamToolParser
|
||||
assert ReasoningParserManager.get_reasoning_parser("beam") is BeamReasoningParser
|
||||
|
||||
|
||||
def test_parser_manager_selects_beam_parser():
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="beam",
|
||||
reasoning_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(),
|
||||
)
|
||||
|
||||
assert parser_cls is not None
|
||||
assert issubclass(parser_cls, BeamParser)
|
||||
assert parser_cls.reasoning_parser_cls is BeamReasoningParser
|
||||
assert parser_cls.tool_parser_cls is BeamToolParser
|
||||
assert parser_cls._enable_auto_tools
|
||||
|
||||
|
||||
@pytest.mark.parametrize("other", ["hermes", "hf"])
|
||||
def test_beam_parser_rejects_other_parsers(other):
|
||||
with pytest.raises(TypeError, match="cannot be combined"):
|
||||
ParserManager.get_parser(
|
||||
tool_parser_name=other,
|
||||
reasoning_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(),
|
||||
)
|
||||
|
||||
|
||||
def test_beam_parser_requires_checkpoint_metadata_at_startup():
|
||||
with pytest.raises(
|
||||
TypeError, match="--reasoning-parser beam requires `response_template`"
|
||||
):
|
||||
ParserManager.get_parser(
|
||||
reasoning_parser_name="beam",
|
||||
tokenizer=BeamTokenizer(response_template=None),
|
||||
)
|
||||
|
||||
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
|
||||
del template["fields"]["tool_calls"]
|
||||
with pytest.raises(TypeError, match="--tool-call-parser beam requires"):
|
||||
ParserManager.get_parser(
|
||||
tool_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(template),
|
||||
)
|
||||
|
||||
|
||||
def test_reasoning_and_final_answer():
|
||||
assert parse(THOUGHT + FINAL + "Sunny.<|return|>") == ("plan", "Sunny.", [])
|
||||
|
||||
|
||||
def test_prefilled_reasoning_continues_from_prompt():
|
||||
prompt = PROMPT_TOKEN_IDS + [201, 101, 202, 103]
|
||||
text = " more.<|end|><|start|>assistant" + FINAL + "Done.<|return|>"
|
||||
|
||||
assert parse(text, prompt) == (" more.", "Done.", [])
|
||||
|
||||
|
||||
def test_parallel_calls_are_coerced_to_their_schemas():
|
||||
text = (
|
||||
THOUGHT
|
||||
+ call(
|
||||
"get_weather",
|
||||
parameter("city", "Paris"),
|
||||
terminator="<|end|><|start|>assistant",
|
||||
)
|
||||
+ call("set_count", parameter("count", "7"))
|
||||
)
|
||||
|
||||
assert parse(text) == (
|
||||
"plan",
|
||||
None,
|
||||
[("get_weather", {"city": "Paris"}), ("set_count", {"count": 7})],
|
||||
)
|
||||
|
||||
|
||||
def test_call_stop_token_removed_from_the_text_still_ends_the_call():
|
||||
text = THOUGHT + call("get_weather", parameter("city", "Paris"), terminator="")
|
||||
|
||||
assert parse(text) == ("plan", None, [("get_weather", {"city": "Paris"})])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("end", ["<|end|>", "<|end|><|start|>"])
|
||||
def test_output_cut_before_the_next_message_start_ends_its_region(end):
|
||||
"""Output that stops after `<|end|>`, such as at `max_tokens`, has ended
|
||||
its reasoning or call and lacks only the next message's start."""
|
||||
weather = call("get_weather", parameter("city", "Paris"), terminator=end)
|
||||
|
||||
assert parse("<|channel|>analysis<|message|>plan" + end) == ("plan", None, [])
|
||||
assert parse(THOUGHT + weather) == (
|
||||
"plan",
|
||||
None,
|
||||
[("get_weather", {"city": "Paris"})],
|
||||
)
|
||||
|
||||
|
||||
THOUGHT_IDS = [201, 101, 202, 103, 203, 200, 100]
|
||||
FINAL_IDS = [201, 102, 202]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt_token_ids", "token_ids", "count"),
|
||||
[
|
||||
(PROMPT_TOKEN_IDS, [*THOUGHT_IDS, *FINAL_IDS, 205], 1),
|
||||
(PROMPT_TOKEN_IDS + [201, 101, 202], [103, 103, 203, 200, 100, *FINAL_IDS], 2),
|
||||
(PROMPT_TOKEN_IDS, [201, 101, 202, 103, 103, 103], 3),
|
||||
],
|
||||
ids=["reasoning_then_answer", "prefilled_reasoning", "cut_off"],
|
||||
)
|
||||
def test_reasoning_tokens_are_counted(prompt_token_ids, token_ids, count):
|
||||
parser = make_parser(prompt_token_ids)
|
||||
|
||||
assert parser.count_reasoning_tokens(token_ids) == count
|
||||
|
||||
|
||||
def test_reasoning_tokens_are_counted_as_the_generation_grows():
|
||||
token_ids = [*THOUGHT_IDS, *FINAL_IDS, 205]
|
||||
parser = make_parser()
|
||||
|
||||
assert [
|
||||
parser.count_reasoning_tokens(token_ids[:end])
|
||||
for end in range(1, len(token_ids) + 1)
|
||||
] == [
|
||||
make_parser().count_reasoning_tokens(token_ids[:end])
|
||||
for end in range(1, len(token_ids) + 1)
|
||||
]
|
||||
assert parser.count_reasoning_tokens(FINAL_IDS) == 0
|
||||
|
||||
|
||||
def test_call_stop_token_ends_the_answer():
|
||||
assert parse(FINAL + "done<|call|>") == (None, "done", [])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 7, 10_000])
|
||||
def test_streaming_matches_non_streaming(chunk_size):
|
||||
text = (
|
||||
THOUGHT
|
||||
+ call(
|
||||
"get_weather",
|
||||
parameter("city", "Paris"),
|
||||
terminator="<|end|><|start|>assistant",
|
||||
)
|
||||
+ call("set_count", parameter("count", "7"))
|
||||
)
|
||||
parser = make_parser()
|
||||
reasoning, calls = [], []
|
||||
for start in range(0, len(text), chunk_size):
|
||||
end = min(start + chunk_size, len(text))
|
||||
delta = parser.parse_delta(
|
||||
text[start:end], [], request(), finished=end == len(text)
|
||||
)
|
||||
if delta is not None:
|
||||
reasoning.append(delta.reasoning or "")
|
||||
calls.extend(delta.tool_calls)
|
||||
|
||||
assert "".join(reasoning) == "plan"
|
||||
assert [
|
||||
(call.function.name, json.loads(call.function.arguments)) for call in calls
|
||||
] == parse(text)[2]
|
||||
|
||||
|
||||
WEATHER_CALL = call("get_weather", parameter("city", "Paris"))
|
||||
WEATHER = [("get_weather", {"city": "Paris"})]
|
||||
|
||||
|
||||
def chat_request(tools=STRICT_TOOLS, **body):
|
||||
return ChatCompletionRequest(
|
||||
model="beam",
|
||||
messages=[{"role": "user", "content": "Weather?"}],
|
||||
tools=tools and [tool.model_dump() for tool in tools],
|
||||
**body,
|
||||
)
|
||||
|
||||
|
||||
def named(name):
|
||||
return {"type": "function", "function": {"name": name}}
|
||||
|
||||
|
||||
def adjust(req, prompt_token_ids=PROMPT_TOKEN_IDS, **kwargs):
|
||||
make_parser(prompt_token_ids, req.tools, **kwargs).adjust_request(req)
|
||||
return req
|
||||
|
||||
|
||||
def accepts(req, text):
|
||||
grammar = Grammar.from_structural_tag(
|
||||
req.extract_structured_outputs().structural_tag
|
||||
)
|
||||
return _is_grammar_accept_string(grammar, text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "text", "calls"),
|
||||
[
|
||||
("required", THOUGHT + WEATHER_CALL, WEATHER),
|
||||
("required", WEATHER_CALL, None),
|
||||
(
|
||||
"required",
|
||||
THOUGHT
|
||||
+ call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
|
||||
+ call("set_count", parameter("count", "7")),
|
||||
[*WEATHER, ("set_count", {"count": 7})],
|
||||
),
|
||||
("required", THOUGHT + call("set_count", parameter("count", "x")), None),
|
||||
(
|
||||
"required",
|
||||
THOUGHT + call("get_weather", parameter("city", "a<|call|>b")),
|
||||
None,
|
||||
),
|
||||
("required", THOUGHT + FINAL + "Paris<|return|>", None),
|
||||
(named("set_count"), THOUGHT + WEATHER_CALL, None),
|
||||
(named("get_weather"), THOUGHT + WEATHER_CALL, WEATHER),
|
||||
(
|
||||
named("get_weather"),
|
||||
THOUGHT
|
||||
+ call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
|
||||
+ WEATHER_CALL,
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_forced_tool_calls_follow_the_template_grammar(tool_choice, text, calls):
|
||||
"""From the message start, the grammar opens with the reasoning, and
|
||||
accepts only calls the parser reads back."""
|
||||
req = adjust(chat_request(tool_choice=tool_choice))
|
||||
|
||||
assert req._grammar_from_parser
|
||||
assert accepts(req, text) is (calls is not None)
|
||||
if calls is not None:
|
||||
assert parse(text)[2] == calls
|
||||
|
||||
|
||||
def test_reasoning_may_be_skipped_when_thinking_is_disabled():
|
||||
req = adjust(
|
||||
chat_request(tool_choice="required"),
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
)
|
||||
|
||||
assert accepts(req, WEATHER_CALL)
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL)
|
||||
|
||||
|
||||
def test_parallel_tool_calls_false_allows_a_single_call():
|
||||
req = adjust(chat_request(tool_choice="required", parallel_tool_calls=False))
|
||||
calls = call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
|
||||
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL)
|
||||
assert not accepts(req, THOUGHT + calls + WEATHER_CALL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("strict", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("separator", "accepted"), [("", True), ("\n", True), ("\n\n", False), (" ", False)]
|
||||
)
|
||||
def test_parameters_are_separated_by_at_most_a_newline(strict, separator, accepted):
|
||||
"""A model that skips a parameter's opener must not write whitespace until
|
||||
`max_tokens`."""
|
||||
write = _tool(
|
||||
"write",
|
||||
{"path": {"type": "string"}, "content": {"type": "string"}},
|
||||
strict=strict,
|
||||
)
|
||||
req = adjust(chat_request([write], tool_choice="required"))
|
||||
body = parameter("path", "a.py") + separator + parameter("content", "x")
|
||||
|
||||
assert accepts(req, THOUGHT + call("write", body)) is accepted
|
||||
assert accepts(req, THOUGHT + call("write", separator + body)) is accepted
|
||||
|
||||
|
||||
def test_forced_calls_in_prefilled_reasoning_start_after_it():
|
||||
req = adjust(
|
||||
chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 101, 202, 103]
|
||||
)
|
||||
|
||||
assert not req._grammar_from_parser
|
||||
assert accepts(req, WEATHER_CALL)
|
||||
|
||||
|
||||
def test_forced_calls_cannot_follow_a_prefilled_answer():
|
||||
with pytest.raises(VLLMValidationError, match="required tool choice"):
|
||||
adjust(chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 102, 202])
|
||||
|
||||
|
||||
def test_prefilled_answer_can_only_be_finished():
|
||||
req = adjust(chat_request(TOOLS), PROMPT_TOKEN_IDS + [201, 102, 202])
|
||||
|
||||
assert accepts(req, "Sunny.<|return|>")
|
||||
assert not accepts(req, "Sunny." + NEXT_MESSAGE + WEATHER_CALL)
|
||||
|
||||
|
||||
def test_forced_call_with_unencodable_tool_name_is_rejected():
|
||||
tools = [_tool("get weather", {"city": {"type": "string"}})]
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="tool name"):
|
||||
adjust(chat_request(tools, tool_choice="required"))
|
||||
|
||||
|
||||
def test_string_formats_and_patterns_are_held_in_raw_text():
|
||||
"""Like XGrammar, a strict string keeps its format or pattern, unless a
|
||||
match could end the value early."""
|
||||
contact = _tool(
|
||||
"set_contact",
|
||||
{
|
||||
"email": {"type": "string", "format": "email"},
|
||||
"code": {"type": "string", "pattern": "^[A-Z]{3}$"},
|
||||
"n": {"type": "integer"},
|
||||
},
|
||||
strict=True,
|
||||
)
|
||||
contact.function.parameters["required"] = ["email", "code", "n"]
|
||||
req = adjust(chat_request([contact], tool_choice="required"))
|
||||
|
||||
def contact_call(email, code, *rest):
|
||||
body = parameter("email", email) + parameter("code", code) + "".join(rest)
|
||||
return THOUGHT + call("set_contact", body)
|
||||
|
||||
text = contact_call("a.b@x.co", "ABC", parameter("n", "1"))
|
||||
calls = make_parser(tools=[contact]).parse(text, req, enable_auto_tools=True)[2]
|
||||
assert accepts(req, text)
|
||||
assert normalize_calls(calls) == [
|
||||
("set_contact", {"email": "a.b@x.co", "code": "ABC", "n": 1})
|
||||
]
|
||||
assert not accepts(req, contact_call("ab", "ABC", parameter("n", "1")))
|
||||
assert not accepts(req, contact_call("a.b@x.co", "ABCD", parameter("n", "1")))
|
||||
assert not accepts(req, contact_call("a.b@x.co", "ABC"))
|
||||
# Line breaks could end the value.
|
||||
dotted = _tool(
|
||||
"set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="required tool choice"):
|
||||
adjust(chat_request([dotted], tool_choice="required"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "accepted"),
|
||||
[
|
||||
(THOUGHT + WEATHER_CALL, True),
|
||||
(THOUGHT + FINAL + "Sunny.<|return|>", True),
|
||||
(THOUGHT + FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False),
|
||||
(THOUGHT + call("set_count", parameter("count", "x")), False),
|
||||
],
|
||||
)
|
||||
def test_strict_tools_constrain_automatic_tool_choice(text, accepted):
|
||||
assert accepts(adjust(chat_request()), text) is accepted
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "accepted"),
|
||||
[
|
||||
(THOUGHT + FINAL + "Sunny.<|return|>", True),
|
||||
(FINAL + "Sunny.<|return|>", False),
|
||||
(WEATHER_CALL, False),
|
||||
(THOUGHT + WEATHER_CALL, False),
|
||||
(FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False),
|
||||
],
|
||||
)
|
||||
def test_tool_choice_none_allows_only_an_answer(text, accepted):
|
||||
req = adjust(chat_request(TOOLS, tool_choice="none"))
|
||||
|
||||
assert req._grammar_from_parser
|
||||
assert accepts(req, text) is accepted
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("tools", "tool_choice"), [(TOOLS, "auto"), (None, None)])
|
||||
def test_always_constrained_output_follows_the_template_framing(tools, tool_choice):
|
||||
req = adjust(chat_request(tools, tool_choice=tool_choice))
|
||||
count_call = THOUGHT + call("set_count", parameter("count", "3"))
|
||||
|
||||
assert accepts(req, THOUGHT + FINAL + "Sunny.<|return|>")
|
||||
assert not accepts(req, FINAL + "Sunny.<|return|>")
|
||||
assert not accepts(req, THOUGHT.replace("plan", "") + FINAL + "Sunny.<|return|>")
|
||||
assert not accepts(req, THOUGHT + "Sunny.")
|
||||
assert accepts(req, count_call) is (tool_choice == "auto")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response_format", [None, {"type": "json_object"}])
|
||||
@pytest.mark.parametrize("strict", [None, True])
|
||||
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
|
||||
def test_automatic_choice_lets_calls_name_undeclared_tools(
|
||||
strict, tool_choice, response_format
|
||||
):
|
||||
"""Unless a forced choice limits calls to the declared tools. The parser
|
||||
passes such calls on, so the harness can tell the model the tool does not
|
||||
exist."""
|
||||
bash = _tool("bash", {"command": {"type": "string"}}, strict=strict)
|
||||
req = adjust(
|
||||
chat_request([bash], tool_choice=tool_choice, response_format=response_format)
|
||||
)
|
||||
undeclared = THOUGHT + call("read_file", parameter("path", "a.py"))
|
||||
parser = make_parser(tools=[bash])
|
||||
|
||||
assert accepts(req, THOUGHT + call("bash", parameter("command", "ls")))
|
||||
assert accepts(req, undeclared) is (tool_choice == "auto")
|
||||
calls = parser.parse(undeclared, req, enable_auto_tools=True)[2]
|
||||
assert normalize_calls(calls) == [("read_file", {"path": "a.py"})]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "body", "accepted"),
|
||||
[
|
||||
("set_count", parameter("count", "1"), True),
|
||||
("set_count", parameter("count", "x"), False),
|
||||
("get_weather", parameter("town", "Paris"), False),
|
||||
# The grammar cannot enforce `minProperties`.
|
||||
("find", parameter("x", "1") + parameter("x", "y"), True),
|
||||
("search_files", parameter("x", "1") + parameter("x", "y"), True),
|
||||
("set_coun", parameter("x", "1") + parameter("x", "y"), True),
|
||||
("set_counts", parameter("x", "1") + parameter("x", "y"), True),
|
||||
],
|
||||
)
|
||||
def test_automatic_calls_enforce_declared_arguments_beside_any_other_name(
|
||||
name, body, accepted
|
||||
):
|
||||
find = _tool("find", {"q": {"type": "string"}})
|
||||
find.function.parameters["minProperties"] = 1
|
||||
req = adjust(chat_request([*TOOLS, find]))
|
||||
|
||||
assert accepts(req, THOUGHT + call(name, body)) is accepted
|
||||
|
||||
|
||||
def test_value_ends_before_an_opener_on_the_next_line():
|
||||
write = _tool("write", {"path": {"type": "string"}, "content": {"type": "string"}})
|
||||
req = adjust(chat_request([write], tool_choice="required"))
|
||||
body = '<rfl:parameter name="content">\nprint(1)\n\n' + parameter("path", "a.py")
|
||||
text = THOUGHT + call("write", body)
|
||||
|
||||
assert accepts(req, text)
|
||||
calls = make_parser(tools=[write]).parse(text, req, enable_auto_tools=True)[2]
|
||||
assert normalize_calls(calls) == [
|
||||
("write", {"content": "print(1)\n", "path": "a.py"})
|
||||
]
|
||||
|
||||
|
||||
def test_always_constrained_output_can_be_disabled(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "0")
|
||||
|
||||
assert adjust(chat_request(TOOLS)).structured_outputs is None
|
||||
|
||||
|
||||
def test_strict_tools_the_template_cannot_hold_leave_automatic_choice_unconstrained():
|
||||
dotted = _tool(
|
||||
"set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True
|
||||
)
|
||||
|
||||
assert adjust(chat_request([dotted])).structured_outputs is None
|
||||
|
||||
|
||||
def test_generic_parser_does_not_derive_the_template_grammar():
|
||||
req = chat_request(TOOLS, tool_choice="required")
|
||||
parser = ResponseTemplateParser(BeamTokenizer(), req.tools, enable_auto_tools=True)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="required tool choice"):
|
||||
parser.adjust_request(req)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tools", [None, TOOLS])
|
||||
def test_structured_output_is_framed_as_the_answer(tools):
|
||||
schema = {"type": "object", "properties": {"a": {"type": "integer"}}}
|
||||
req = adjust(
|
||||
chat_request(
|
||||
tools,
|
||||
response_format={
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "answer", "schema": schema},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
assert req.response_format is None
|
||||
assert accepts(req, THOUGHT + FINAL + '{"a": 1}<|return|>')
|
||||
assert not accepts(req, THOUGHT + FINAL + "Sunny.<|return|>")
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL) is (tools is not None)
|
||||
|
||||
|
||||
def test_responses_named_tool_choice_follows_the_template_grammar():
|
||||
req = adjust(
|
||||
ResponsesRequest(
|
||||
input="Weather?",
|
||||
tools=[
|
||||
FunctionTool(
|
||||
type="function",
|
||||
name="get_weather",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
strict=True,
|
||||
)
|
||||
],
|
||||
tool_choice=ToolChoiceFunction(type="function", name="get_weather"),
|
||||
text={"format": {"type": "json_object"}},
|
||||
)
|
||||
)
|
||||
|
||||
assert req._grammar_from_parser
|
||||
assert req.text.format.type == "json_object"
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL)
|
||||
assert not accepts(req, FINAL + "{}<|return|>")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["in_progress", "completed"])
|
||||
@pytest.mark.parametrize("inside_reasoning", [False, True])
|
||||
def test_responses_preserves_text_config_with_template_grammar(
|
||||
status, inside_reasoning
|
||||
):
|
||||
text = {
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"name": "answer",
|
||||
"strict": True,
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"a": {"type": "integer"}},
|
||||
"required": ["a"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
"verbosity": "low",
|
||||
}
|
||||
req = ResponsesRequest(input="Answer", text=text)
|
||||
expected_text = req.text.model_dump(by_alias=True)
|
||||
prompt = PROMPT_TOKEN_IDS + ([201, 101, 202] if inside_reasoning else [])
|
||||
adjust(req, prompt_token_ids=prompt)
|
||||
params = req.to_sampling_params(default_max_tokens=128)
|
||||
grammar = Grammar.from_structural_tag(params.structured_outputs.structural_tag)
|
||||
prefix = "" if inside_reasoning else THOUGHT
|
||||
assert _is_grammar_accept_string(grammar, prefix + FINAL + '{"a": 1}<|return|>')
|
||||
assert not _is_grammar_accept_string(grammar, prefix + FINAL + "Sunny.<|return|>")
|
||||
response = ResponsesResponse.from_request(req, params, "beam", 0, [], status)
|
||||
assert response.model_dump(by_alias=True)["text"] == expected_text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reverse_alternatives", [False, True])
|
||||
def test_reasoning_budget_initializes_from_checkpoint_pattern(
|
||||
monkeypatch, reverse_alternatives
|
||||
):
|
||||
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
|
||||
alternatives = [
|
||||
r"^<\|channel\|>analysis<\|message\|>",
|
||||
r"<\|start\|>assistant<\|channel\|>analysis<\|message\|>",
|
||||
]
|
||||
if reverse_alternatives:
|
||||
alternatives.reverse()
|
||||
template["fields"]["thinking"] = {
|
||||
"open_pattern": "(?:" + "|".join(alternatives) + ")",
|
||||
"close": "<|end|>",
|
||||
"content": "text",
|
||||
}
|
||||
tokenizer = BeamTokenizer(template)
|
||||
monkeypatch.setattr(
|
||||
"vllm.config.reasoning.cached_tokenizer_from_config", lambda **_: tokenizer
|
||||
)
|
||||
config = ReasoningConfig(reasoning_parser="beam")
|
||||
config.initialize_token_ids(SimpleNamespace())
|
||||
|
||||
assert config.enabled
|
||||
# The marker must work both after the prompt's assistant header and in
|
||||
# subsequent complete assistant messages, regardless of regex branch order.
|
||||
assert config.reasoning_start_token_ids == tokenizer.encode(
|
||||
"<|channel|>analysis<|message|>"
|
||||
)
|
||||
assert config.reasoning_end_token_ids == tokenizer.encode("<|end|>")
|
||||
assert config.natural_reasoning_end_token_ids == tokenizer.encode("<|end|>")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern", [r"analysis.*", r"(?:analysis|thinking)"])
|
||||
def test_reasoning_budget_does_not_guess_a_marker_for_other_patterns(pattern):
|
||||
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
|
||||
thinking = template["fields"]["thinking"]
|
||||
del thinking["open"]
|
||||
thinking["open_pattern"] = pattern
|
||||
|
||||
assert BeamReasoningParser(BeamTokenizer(template)).reasoning_start_str is None
|
||||
@@ -0,0 +1,492 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from xgrammar import Grammar
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.parser.chat_parsing.response_templates import load_response_template
|
||||
from vllm.parser.response_template_grammar import (
|
||||
answer_rest_structural_tag,
|
||||
tool_call_structural_tag,
|
||||
)
|
||||
|
||||
TEMPLATE: dict[str, Any] = {
|
||||
"start_anchor": "<assistant>",
|
||||
"fields": {
|
||||
"thinking": {"open": "<think>", "close": "</think>", "content": "text"},
|
||||
"content": {"content": "text"},
|
||||
"tool_calls": {
|
||||
"open_pattern": r'<call name="(?P<name>[\w.-]+)">',
|
||||
"close": "</call>",
|
||||
"repeats": True,
|
||||
"content": "xml-inline",
|
||||
"content_args": {
|
||||
"tag_pattern": r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)</arg>'
|
||||
},
|
||||
"transform": {
|
||||
"type": "function",
|
||||
"function": {"name": "{name}", "arguments": "{content}"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
ANSWER_TEMPLATE = deepcopy(TEMPLATE)
|
||||
ANSWER_TEMPLATE["fields"]["content"] = {
|
||||
"open": "<answer>",
|
||||
"close": "</answer>",
|
||||
"content": "text",
|
||||
}
|
||||
|
||||
|
||||
def tool(parameters, name="f", strict=True) -> dict[str, Any]:
|
||||
return {"name": name, "parameters": parameters, "strict": strict}
|
||||
|
||||
|
||||
WEATHER = tool(
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}, "days": {"type": "integer"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
name="get.weather",
|
||||
)
|
||||
PING = {"name": "ping"}
|
||||
CALL = '<call name="get.weather"><arg name="city">Paris</arg></call>'
|
||||
|
||||
|
||||
def build(
|
||||
template=TEMPLATE,
|
||||
functions=(WEATHER, PING),
|
||||
*,
|
||||
parallel=True,
|
||||
thinking=True,
|
||||
answer=False,
|
||||
answer_format=None,
|
||||
sorted_arguments=False,
|
||||
):
|
||||
fields = load_response_template(template).fields
|
||||
return tool_call_structural_tag(
|
||||
fields["tool_calls"],
|
||||
list(functions),
|
||||
parallel_tool_calls=parallel,
|
||||
thinking=fields["thinking"] if thinking else None,
|
||||
answer=fields["content"] if answer else None,
|
||||
answer_format=answer_format,
|
||||
sorted_arguments=sorted_arguments,
|
||||
)
|
||||
|
||||
|
||||
def accepts(tag, text):
|
||||
return _is_grammar_accept_string(Grammar.from_structural_tag(json.dumps(tag)), text)
|
||||
|
||||
|
||||
def call(*arguments):
|
||||
body = "".join(f'<arg name="{key}">{value}</arg>' for key, value in arguments)
|
||||
return f'<call name="f">{body}</call>'
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "accepted"),
|
||||
[
|
||||
("<think>plan</think>" + CALL, True),
|
||||
(CALL, True),
|
||||
(CALL.removesuffix("</call>"), True),
|
||||
(
|
||||
'<call name="get.weather"><arg name="city">Paris</arg>\n'
|
||||
'<arg name="days">3</arg></call>',
|
||||
True,
|
||||
),
|
||||
(
|
||||
'<call name="get.weather"><arg name="city">Paris</arg>'
|
||||
'<arg name="days">soon</arg></call>',
|
||||
False,
|
||||
),
|
||||
('<call name="get.weather"><arg name="days">3</arg></call>', False),
|
||||
(CALL + '<call name="ping"><arg name="any">x</arg></call>', True),
|
||||
('<call name="other"></call>', False),
|
||||
("<think>plan</think>The answer.", False),
|
||||
],
|
||||
)
|
||||
def test_forced_calls_follow_the_template_format(text, accepted):
|
||||
assert accepts(build(), text) is accepted
|
||||
|
||||
|
||||
def test_single_call_when_parallel_calls_are_disabled():
|
||||
tag = build(parallel=False)
|
||||
|
||||
assert accepts(tag, CALL)
|
||||
assert not accepts(tag, CALL + CALL)
|
||||
|
||||
|
||||
def test_thinking_is_rejected_when_not_requested():
|
||||
assert not accepts(build(thinking=False), "<think>plan</think>" + CALL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "accepted"),
|
||||
[
|
||||
("<think>plan</think><answer>Sunny.", True),
|
||||
("<answer>Sunny.</answer>", True),
|
||||
("<think>plan</think>" + CALL, True),
|
||||
("<answer>Sunny.</answer>" + CALL, False),
|
||||
('<call name="get.weather"><arg name="days">soon</arg></call>', False),
|
||||
],
|
||||
)
|
||||
def test_answer_field_lets_the_model_reply_instead(text, accepted):
|
||||
assert accepts(build(ANSWER_TEMPLATE, answer=True), text) is accepted
|
||||
|
||||
|
||||
@pytest.mark.parametrize("required", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("text", "optional_only"),
|
||||
[
|
||||
("<think>plan</think>" + CALL, False),
|
||||
("<think>\nplan</think><answer>Sunny.", False),
|
||||
("<think> \n</think><answer>Sunny.", False),
|
||||
(CALL, True),
|
||||
("<answer>Sunny.", True),
|
||||
("<think></think><answer>Sunny.", True),
|
||||
("<think><</think><answer>Sunny.", True),
|
||||
],
|
||||
)
|
||||
def test_required_reasoning_opens_the_message_with_nonempty_reasoning(
|
||||
required, text, optional_only
|
||||
):
|
||||
fields = load_response_template(ANSWER_TEMPLATE).fields
|
||||
tag = tool_call_structural_tag(
|
||||
fields["tool_calls"],
|
||||
[WEATHER],
|
||||
parallel_tool_calls=True,
|
||||
thinking=fields["thinking"],
|
||||
answer=fields["content"],
|
||||
message_start=["<assistant>"],
|
||||
requires_reasoning=required,
|
||||
)
|
||||
|
||||
assert accepts(tag, text) is (not required or not optional_only)
|
||||
|
||||
|
||||
def test_answer_rest_finishes_the_open_answer():
|
||||
answer = load_response_template(ANSWER_TEMPLATE).fields["content"]
|
||||
tag = answer_rest_structural_tag(answer)
|
||||
|
||||
assert accepts(tag, "Sunny.</answer>")
|
||||
assert accepts(tag, "Sunny.")
|
||||
assert not accepts(tag, "Sunny.</answer>" + CALL)
|
||||
|
||||
|
||||
def test_answer_without_literal_opener_is_not_derived():
|
||||
assert build(answer=True) is None
|
||||
|
||||
|
||||
def test_answer_holds_the_answer_format():
|
||||
schema = {"type": "json_schema", "json_schema": {"type": "object"}}
|
||||
with_tools = build(ANSWER_TEMPLATE, answer=True, answer_format=schema)
|
||||
answer_only = build(ANSWER_TEMPLATE, (), answer=True, answer_format=schema)
|
||||
|
||||
for tag in (with_tools, answer_only):
|
||||
assert accepts(tag, '<think>plan</think><answer>{"a": 1}</answer>')
|
||||
assert not accepts(tag, "<answer>Sunny.</answer>")
|
||||
assert accepts(with_tools, CALL)
|
||||
assert not accepts(answer_only, CALL)
|
||||
|
||||
|
||||
def test_json_arguments_follow_the_schema():
|
||||
template = deepcopy(TEMPLATE)
|
||||
template["fields"]["tool_calls"].update(content="json", content_args={})
|
||||
tag = build(template, [WEATHER])
|
||||
|
||||
assert accepts(tag, '<call name="get.weather">{"city": "Paris"}</call>')
|
||||
assert not accepts(tag, '<call name="get.weather">{"days": 3}</call>')
|
||||
|
||||
|
||||
END_X = {"tag_pattern": r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)X'}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
{"content": "json", "content_args": {"unquoted_keys": True}},
|
||||
{
|
||||
"transform": {
|
||||
"type": "function",
|
||||
"function": {"name": "fixed", "arguments": "{content}"},
|
||||
}
|
||||
},
|
||||
{"content": "kv-lines", "content_args": {}},
|
||||
# Framing in which argument text can spell a closer, where the parser
|
||||
# would end the call.
|
||||
{"close": ["}X", "</call>"], "content_args": END_X},
|
||||
{
|
||||
"close": ["}X", "</call>"],
|
||||
"content_args": {**END_X, "value_parser": {"name": "json"}},
|
||||
},
|
||||
{"close": ["7X", "</call>"], "content_args": END_X},
|
||||
{"close": [">X", "</call>"]},
|
||||
{"close": ["></call>", "</call>"]},
|
||||
{"close": ["><a", "</call>"]},
|
||||
{"close": ["}</call>", "</call>"], "content": "json", "content_args": {}},
|
||||
],
|
||||
)
|
||||
def test_underivable_tool_formats_are_not_derived(change):
|
||||
template = deepcopy(TEMPLATE)
|
||||
template["fields"]["tool_calls"].update(change)
|
||||
|
||||
assert build(template) is None
|
||||
|
||||
|
||||
def test_non_literal_thinking_delimiters_are_not_derived():
|
||||
template = deepcopy(TEMPLATE)
|
||||
del template["fields"]["thinking"]["open"]
|
||||
template["fields"]["thinking"]["open_pattern"] = r"<think(?: mode=\w+)?>"
|
||||
|
||||
assert build(template) is None
|
||||
assert build(template, thinking=False) is not None
|
||||
|
||||
|
||||
LEVEL = {"type": "object", "properties": {"level": {"$ref": "#/$defs/level"}}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("schema", "value", "accepted"),
|
||||
[
|
||||
({"type": "integer", "nullable": True}, "null", True),
|
||||
({"type": "integer", "nullable": True}, "soon", False),
|
||||
({"type": ["integer", "null"]}, "null", True),
|
||||
({"anyOf": [{"type": "integer"}, {"type": "boolean"}]}, "true", True),
|
||||
({"anyOf": [{"type": "integer"}, {"type": "boolean"}]}, "soon", False),
|
||||
({"type": ["string", "integer"]}, "soon", True),
|
||||
({"enum": ["celsius", "fahrenheit"]}, "celsius", True),
|
||||
({"enum": ["celsius", "fahrenheit"]}, "kelvin", False),
|
||||
({"type": "string", "const": "on"}, "off", False),
|
||||
(LEVEL, '{"level": 2}', True),
|
||||
(LEVEL, '{"level": "high"}', False),
|
||||
],
|
||||
)
|
||||
def test_argument_values_follow_how_the_parser_reads_them(schema, value, accepted):
|
||||
parameters = {
|
||||
"type": "object",
|
||||
"properties": {"v": schema},
|
||||
"required": ["v"],
|
||||
"$defs": {"level": {"type": "integer"}},
|
||||
}
|
||||
|
||||
assert accepts(build(functions=[tool(parameters)]), call(("v", value))) is accepted
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("schema", "strict", "derivable"),
|
||||
[
|
||||
# The parser reads values without a type as text.
|
||||
({"enum": [1, 2]}, True, False),
|
||||
({"$ref": "#/$defs/level"}, True, False),
|
||||
({"$ref": "#/$defs/level"}, False, True),
|
||||
# Raw text cannot hold its closer.
|
||||
({"enum": ["a</arg>b"]}, True, False),
|
||||
({"type": "object", "properties": {"x": {"$ref": "#/x"}}}, True, False),
|
||||
(False, True, False),
|
||||
# A match could write the start of the value's end.
|
||||
({"type": "string", "pattern": "^[a-z<]+$"}, True, False),
|
||||
({"type": "string", "pattern": "^[a-z<]+$"}, False, True),
|
||||
({"type": "string", "pattern": "^a.b$"}, True, False),
|
||||
# XGrammar holds length bounds only in JSON.
|
||||
({"type": "string", "minLength": 3}, True, False),
|
||||
({"type": "string", "format": "ipv6"}, True, False),
|
||||
],
|
||||
)
|
||||
def test_schemas_text_values_cannot_hold_are_not_derived(schema, strict, derivable):
|
||||
parameters = {
|
||||
"type": "object",
|
||||
"properties": {"v": schema},
|
||||
"$defs": {"level": {"type": "integer"}},
|
||||
}
|
||||
|
||||
assert (build(functions=[tool(parameters, strict=strict)]) is not None) is derivable
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("schema", "value", "accepted"),
|
||||
[
|
||||
({"type": "string", "format": "email"}, "a.b@x.co", True),
|
||||
({"type": "string", "format": "email"}, "ab", False),
|
||||
({"type": "string", "format": "date-time"}, "2026-10-06T12:00:00Z", True),
|
||||
({"type": "string", "format": "date-time"}, "2026-13-06T12:00:00Z", False),
|
||||
({"type": "string", "pattern": "^[A-Z]{3}$"}, "ABC", True),
|
||||
({"type": "string", "pattern": "^[A-Z]{3}$"}, "ABCD", False),
|
||||
# Like XGrammar, the format overrides the pattern and the length bounds.
|
||||
(
|
||||
{"type": "string", "format": "uuid", "pattern": "^x$", "maxLength": 1},
|
||||
"123e4567-e89b-12d3-a456-426614174000",
|
||||
True,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_text_values_hold_string_formats_and_patterns(schema, value, accepted):
|
||||
parameters = {"type": "object", "properties": {"v": schema}, "required": ["v"]}
|
||||
|
||||
assert accepts(build(functions=[tool(parameters)]), call(("v", value))) is accepted
|
||||
|
||||
|
||||
def test_values_cannot_hold_a_closer():
|
||||
template = deepcopy(TEMPLATE)
|
||||
template["fields"]["tool_calls"]["close"] = ["</call>", "STOP"]
|
||||
text = {"type": "object", "properties": {"v": {"type": "string"}}}
|
||||
strings = {"type": "array", "items": {"type": "string"}}
|
||||
json_text = {"type": "object", "properties": {"v": strings}}
|
||||
|
||||
assert accepts(build(template, [tool(text)]), call(("v", "ab")))
|
||||
assert not accepts(build(template, [tool(text)]), call(("v", "aSTOPb")))
|
||||
assert not accepts(
|
||||
build(template, [tool(None, strict=False)]), call(("v", "aSTOPb"))
|
||||
)
|
||||
assert accepts(build(template, [tool(json_text)]), call(("v", '["ab"]')))
|
||||
assert not accepts(build(template, [tool(json_text)]), call(("v", '["aSTOPb"]')))
|
||||
|
||||
|
||||
def test_extra_arguments_need_additional_properties():
|
||||
parameters = {"type": "object", "properties": {"city": {"type": "string"}}}
|
||||
extra = call(("city", "Paris"), ("zip", "75001"))
|
||||
|
||||
assert not accepts(build(functions=[tool(parameters)]), extra)
|
||||
# Next to declared keys, an extra one could repeat them unchecked.
|
||||
allowed = {**parameters, "additionalProperties": True}
|
||||
assert build(functions=[tool(allowed)]) is None
|
||||
free_form = {"type": "object", "additionalProperties": True}
|
||||
assert accepts(build(functions=[tool(free_form)]), extra)
|
||||
|
||||
|
||||
def test_only_strict_tools_follow_their_schema():
|
||||
off_schema = call(("days", "soon"))
|
||||
parameters = WEATHER["parameters"]
|
||||
|
||||
assert accepts(build(functions=[tool(parameters, strict=False)]), off_schema)
|
||||
assert not accepts(build(functions=[tool(parameters)]), off_schema)
|
||||
assert accepts(build(functions=[tool(None)]), call())
|
||||
assert not accepts(build(functions=[tool(None)]), call(("a", "x")))
|
||||
|
||||
|
||||
def test_annotations_keep_strict_arguments_to_their_schema():
|
||||
parameters = {
|
||||
**WEATHER["parameters"],
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "weather",
|
||||
}
|
||||
tag = build(functions=[tool(parameters)])
|
||||
|
||||
assert accepts(tag, call(("city", "Paris"), ("days", "3")))
|
||||
assert not accepts(tag, call(("city", "Paris"), ("days", "soon")))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("arguments", "accepted", "accepted_sorted"),
|
||||
[
|
||||
((("path", "a"), ("content", "b")), True, True),
|
||||
((("content", "b"), ("path", "a")), False, True),
|
||||
((("path", "a"),), True, True),
|
||||
((("content", "b"),), False, False),
|
||||
((("path", "a"), ("path", "c")), False, False),
|
||||
((("path", "a"), ("mode", "c")), False, False),
|
||||
],
|
||||
)
|
||||
def test_strict_arguments_follow_declared_or_sorted_order(
|
||||
arguments, accepted, accepted_sorted
|
||||
):
|
||||
parameters = {
|
||||
"properties": {"path": {"type": "string"}, "content": {"type": "string"}},
|
||||
"required": ["path"],
|
||||
}
|
||||
tags = [
|
||||
build(functions=[tool(parameters)], sorted_arguments=sorted_arguments)
|
||||
for sorted_arguments in (False, True)
|
||||
]
|
||||
|
||||
assert [accepts(tag, call(*arguments)) for tag in tags] == [
|
||||
accepted,
|
||||
accepted_sorted,
|
||||
]
|
||||
|
||||
|
||||
LOOKAHEAD_TEMPLATE = deepcopy(TEMPLATE)
|
||||
LOOKAHEAD_TEMPLATE["fields"]["tool_calls"]["content_args"]["tag_pattern"] = (
|
||||
r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)\n(?:</arg>|(?=<arg name="))'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "accepted_loose", "accepted_strict"),
|
||||
[
|
||||
('<arg name="city">Paris\n</arg>\n<arg name="days">3\n</arg>', True, True),
|
||||
('<arg name="city">Paris\n<arg name="days">3\n</arg>', True, True),
|
||||
('<arg name="days">3\n<arg name="city">Paris\n</arg>', True, False),
|
||||
('<arg name="city">Paris\n<arg name="city">Lyon\n</arg>', True, False),
|
||||
('<arg name="city">Paris\n<arg name="hours">3\n</arg>', True, False),
|
||||
('<arg name="city">Paris\n<arg name="', False, False),
|
||||
('<arg name="city">Paris\n<arg name="days\n</arg>', False, False),
|
||||
('<arg name="city">a <arg name="b"> c\n</arg>', True, True),
|
||||
],
|
||||
)
|
||||
def test_values_end_where_a_lookahead_end_does(body, accepted_loose, accepted_strict):
|
||||
parameters = WEATHER["parameters"]
|
||||
tags = [
|
||||
build(LOOKAHEAD_TEMPLATE, [tool(parameters, strict=strict)])
|
||||
for strict in (False, True)
|
||||
]
|
||||
text = f'<call name="f">{body}</call>'
|
||||
|
||||
assert [accepts(tag, text) for tag in tags] == [accepted_loose, accepted_strict]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lookahead", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("keys", "accepted"),
|
||||
[
|
||||
("b", True),
|
||||
("ab", True),
|
||||
("bc", True),
|
||||
("bd", True),
|
||||
("be", True),
|
||||
("abde", True),
|
||||
("bcde", True),
|
||||
("", False),
|
||||
("a", False),
|
||||
("acd", False),
|
||||
("bdc", False),
|
||||
("bcc", False),
|
||||
],
|
||||
)
|
||||
def test_last_parameter_follows_every_required_one(keys, accepted, lookahead):
|
||||
parameters = {
|
||||
"properties": {key: {"type": "string"} for key in "abcde"},
|
||||
"required": ["b"],
|
||||
}
|
||||
tag = build(LOOKAHEAD_TEMPLATE, [tool(parameters)])
|
||||
ends = ["\n" if lookahead else "\n</arg>"] * (len(keys) - 1) + ["\n</arg>"]
|
||||
body = "".join(f'<arg name="{k}">x{e}' for k, e in zip(keys, ends))
|
||||
|
||||
assert accepts(tag, f'<call name="f">{body}</call>') is accepted
|
||||
|
||||
|
||||
def test_required_argument_without_schema_is_still_required():
|
||||
parameters = {"type": "object", "properties": {}, "required": ["a"]}
|
||||
tag = build(functions=[tool(parameters)])
|
||||
|
||||
assert accepts(tag, call(("a", "x")))
|
||||
assert not accepts(tag, call())
|
||||
assert build(functions=[tool({**parameters, "required": ["a", 1]})]) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("function", "match"),
|
||||
[
|
||||
(tool({}, name="get weather"), "tool name"),
|
||||
(tool({"properties": {'a"b': {"type": "string"}}}), "parameter name"),
|
||||
],
|
||||
)
|
||||
def test_unencodable_names_are_rejected(function, match):
|
||||
with pytest.raises(ValueError, match=match):
|
||||
build(functions=[function])
|
||||
@@ -53,6 +53,7 @@ from openai_harmony import Message as OpenAIHarmonyMessage
|
||||
from pydantic import (
|
||||
BeforeValidator,
|
||||
Field,
|
||||
PrivateAttr,
|
||||
ValidationError,
|
||||
field_serializer,
|
||||
model_validator,
|
||||
@@ -469,6 +470,14 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
max_output_tokens_param="max_output_tokens",
|
||||
)
|
||||
|
||||
_grammar_from_parser: bool = PrivateAttr(default=False)
|
||||
"""CAUTION: Should only be set by the parser's adjust_request."""
|
||||
|
||||
_parser_structured_outputs: StructuredOutputsParams | None = PrivateAttr(
|
||||
default=None
|
||||
)
|
||||
"""Parser-derived constraints that preserve the public request's text metadata."""
|
||||
|
||||
_DEFAULT_SAMPLING_PARAMS = {
|
||||
"temperature": 1.0,
|
||||
"top_p": 1.0,
|
||||
@@ -478,6 +487,8 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
|
||||
def extract_structured_outputs(self) -> StructuredOutputsParams | None:
|
||||
"""Normalize request constraints into ``StructuredOutputsParams``."""
|
||||
if self._parser_structured_outputs is not None:
|
||||
return self._parser_structured_outputs
|
||||
if self.text is None or self.text.format is None:
|
||||
return self.structured_outputs
|
||||
|
||||
|
||||
@@ -526,6 +526,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
trace_headers=trace_headers,
|
||||
session_id=session_id,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
reasoning_ended=True if request._grammar_from_parser else None,
|
||||
)
|
||||
generators.append(generator)
|
||||
|
||||
@@ -647,6 +648,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
session_id: str | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
reasoning_ended: bool | None = None,
|
||||
):
|
||||
max_model_len = self.model_config.max_model_len
|
||||
cache_salt = cast(str | None, engine_input.get("cache_salt"))
|
||||
@@ -673,6 +675,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
priority=priority,
|
||||
session_id=session_id,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
reasoning_ended=reasoning_ended,
|
||||
)
|
||||
|
||||
async for res in generator:
|
||||
|
||||
@@ -246,6 +246,7 @@ if TYPE_CHECKING:
|
||||
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
|
||||
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
|
||||
VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True
|
||||
VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT: bool = True
|
||||
VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
|
||||
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
|
||||
VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5
|
||||
@@ -1844,6 +1845,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: (
|
||||
os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Constrain every `beam` output to its response template's format, also
|
||||
# without tool or structured output constraints.
|
||||
"VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT": lambda: (
|
||||
os.getenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Control the max chunk bytes (in MB) for the rpc message queue.
|
||||
# Object larger than this threshold will be broadcast to worker
|
||||
# processes via zmq.
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Parser for Beam checkpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from openai.types.responses import ToolChoiceFunction
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.abstract_parser import (
|
||||
_xgrammar_supports,
|
||||
structured_outputs_to_format,
|
||||
)
|
||||
from vllm.parser.chat_parsing import ResponseParser
|
||||
from vllm.parser.chat_parsing.response_templates import ResponseTemplate
|
||||
from vllm.parser.response_template import (
|
||||
CONTENT_FIELD,
|
||||
THINKING_FIELD,
|
||||
TOOL_FIELD,
|
||||
ResponseTemplateParser,
|
||||
ResponseTemplateReasoningParser,
|
||||
ResponseTemplateToolParser,
|
||||
_as_tool_dicts,
|
||||
_is_named_tool_choice,
|
||||
_tool_parsing_enabled,
|
||||
)
|
||||
from vllm.parser.response_template_grammar import (
|
||||
_openers,
|
||||
answer_rest_structural_tag,
|
||||
message_start_anchors,
|
||||
tool_call_structural_tag,
|
||||
)
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _named_tool(request: ChatCompletionRequest | ResponsesRequest) -> str:
|
||||
choice = request.tool_choice
|
||||
if isinstance(choice, ToolChoiceFunction):
|
||||
return choice.name
|
||||
return cast(ChatCompletionNamedToolChoiceParam, choice).function.name
|
||||
|
||||
|
||||
def _starts_message(template: ResponseTemplate, prefix: str) -> bool:
|
||||
"""Whether output after `prefix` starts the assistant message."""
|
||||
ends = [match.end() for match in template.start_anchor_re.finditer(prefix)]
|
||||
return bool(ends) and ends[-1] == len(prefix)
|
||||
|
||||
|
||||
def _open_field(template: ResponseTemplate, prefix: str) -> str | None:
|
||||
"""The field that `prefix` leaves open with its explicit opener, if any."""
|
||||
field = None
|
||||
for event in ResponseParser(template, prefix=prefix).initial_events:
|
||||
if event["type"] == "region_open" and event["end"] > event["start"]:
|
||||
field = event["field"]
|
||||
elif event["type"] in {"region_close", "region_malformed"}:
|
||||
field = None
|
||||
return field
|
||||
|
||||
|
||||
class BeamReasoningParser(ResponseTemplateReasoningParser):
|
||||
"""Selects `BeamParser` as the reasoning parser."""
|
||||
|
||||
@classmethod
|
||||
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
|
||||
return BeamParser
|
||||
|
||||
@property
|
||||
def reasoning_start_str(self) -> str | None:
|
||||
if literal := super().reasoning_start_str:
|
||||
return literal
|
||||
thinking = self.response_template.fields.get(THINKING_FIELD)
|
||||
if thinking is None:
|
||||
return None
|
||||
openers = _openers(thinking, message_start_anchors(self.response_template))
|
||||
if not openers:
|
||||
return None
|
||||
# Budget tracking needs one marker for both a prompt continuation
|
||||
# and a complete assistant message in the checkpoint's pattern.
|
||||
marker = min(openers, key=len)
|
||||
return marker if all(opener.endswith(marker) for opener in openers) else None
|
||||
|
||||
|
||||
class BeamToolParser(ResponseTemplateToolParser):
|
||||
"""Selects `BeamParser` as the tool parser."""
|
||||
|
||||
@classmethod
|
||||
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
|
||||
return BeamParser
|
||||
|
||||
|
||||
class BeamParser(ResponseTemplateParser):
|
||||
"""Parse Beam output with the checkpoint's `response_template` metadata,
|
||||
and constrain it with a grammar derived from that template."""
|
||||
|
||||
reasoning_parser_cls = BeamReasoningParser
|
||||
tool_parser_cls = BeamToolParser
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
chat_template_kwargs = kwargs.get("chat_template_kwargs") or {}
|
||||
# From the message start, the grammar opens the message with non-empty
|
||||
# reasoning, unless the request turns thinking off.
|
||||
self._requires_reasoning = bool(
|
||||
chat_template_kwargs.get("enable_thinking", True)
|
||||
)
|
||||
|
||||
def _constrain_request(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
message_start = _starts_message(self.response_template, self._prefix)
|
||||
tag = self._template_tag(request, message_start=message_start)
|
||||
if tag is not None:
|
||||
grammar = StructuredOutputsParams(structural_tag=json.dumps(tag))
|
||||
if isinstance(request, ChatCompletionRequest):
|
||||
request.structured_outputs = grammar
|
||||
request.response_format = None
|
||||
else:
|
||||
request._parser_structured_outputs = grammar
|
||||
# From the message start, the tag also covers the reasoning.
|
||||
request._grammar_from_parser = message_start
|
||||
return request
|
||||
|
||||
def _template_tag(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
*,
|
||||
message_start: bool,
|
||||
) -> dict[str, Any] | None:
|
||||
"""The template grammar for `request`, or `None` to leave its output
|
||||
to the request's own constraint, if any."""
|
||||
fields = self.response_template.fields
|
||||
named = _is_named_tool_choice(request)
|
||||
tools_enabled = (
|
||||
self._tool_parser is not None
|
||||
and bool(request.tools)
|
||||
and _tool_parsing_enabled(
|
||||
request, enable_auto_tools=self._enable_auto_tools
|
||||
)
|
||||
)
|
||||
forced = tools_enabled and (request.tool_choice == "required" or named)
|
||||
# The prompt may still list the tools, so only the grammar keeps the
|
||||
# model from calling them.
|
||||
tools_forbidden = (
|
||||
bool(request.tools)
|
||||
and request.tool_choice == "none"
|
||||
and TOOL_FIELD in fields
|
||||
)
|
||||
constrained_tools = None
|
||||
if tools_enabled and (forced or envs.VLLM_ENFORCE_STRICT_TOOL_CALLING):
|
||||
assert self._tool_parser is not None
|
||||
strict_level = self.tool_strict_level
|
||||
if strict_level is None:
|
||||
strict_level = self._tool_parser.default_tool_strict_level
|
||||
constrained_tools = resolve_tool_strictness(
|
||||
request.tools or [], request.tool_choice, strict_level
|
||||
)
|
||||
output = request.extract_structured_outputs()
|
||||
if forced and output is not None:
|
||||
logger.warning_once(
|
||||
"structured outputs are ignored because tool_choice forces tool call."
|
||||
)
|
||||
output = None
|
||||
answer = fields.get(CONTENT_FIELD)
|
||||
if output is not None and (
|
||||
output.structural_tag is not None
|
||||
or answer is None
|
||||
or answer.open_re is None
|
||||
or not _xgrammar_supports(output)
|
||||
):
|
||||
if constrained_tools is not None:
|
||||
logger.warning_once(
|
||||
"response_template parsing cannot enforce strict tools "
|
||||
"alongside this structured output."
|
||||
)
|
||||
return None
|
||||
# Without a tool or structured output constraint, the output still
|
||||
# follows the template.
|
||||
always_constrain = envs.VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT and not forced
|
||||
if (
|
||||
constrained_tools is None
|
||||
and output is None
|
||||
and not tools_forbidden
|
||||
and not always_constrain
|
||||
):
|
||||
return None
|
||||
|
||||
# Inside an open answer, only the always-on grammar applies, finishing a
|
||||
# free answer. Inside open reasoning, the grammar needs the reasoning
|
||||
# gate to defer it to the reasoning's end.
|
||||
open_field = _open_field(self.response_template, self._prefix)
|
||||
reasoning_gated = (
|
||||
self._reasoning_parser is not None and request.include_reasoning
|
||||
)
|
||||
tag = None
|
||||
if open_field == CONTENT_FIELD:
|
||||
if always_constrain and output is None:
|
||||
tag = answer_rest_structural_tag(fields[CONTENT_FIELD])
|
||||
elif open_field != THINKING_FIELD or reasoning_gated:
|
||||
functions = []
|
||||
if tools_enabled:
|
||||
tools = _as_tool_dicts(constrained_tools or request.tools)
|
||||
functions = [tool.get("function", tool) for tool in tools]
|
||||
if named:
|
||||
name = _named_tool(request)
|
||||
functions = [f for f in functions if f.get("name") == name]
|
||||
answer_format = (
|
||||
None if output is None else structured_outputs_to_format(output)
|
||||
)
|
||||
try:
|
||||
tag = tool_call_structural_tag(
|
||||
fields.get(TOOL_FIELD),
|
||||
functions,
|
||||
parallel_tool_calls=(
|
||||
not named and request.parallel_tool_calls is not False
|
||||
),
|
||||
thinking=fields.get(THINKING_FIELD),
|
||||
answer=None if forced else answer,
|
||||
answer_format=(
|
||||
None if answer_format is None else answer_format.model_dump()
|
||||
),
|
||||
message_start=(
|
||||
message_start_anchors(self.response_template)
|
||||
if message_start
|
||||
else None
|
||||
),
|
||||
requires_reasoning=self._requires_reasoning,
|
||||
# Beam drifts from schemas the grammar does not enforce,
|
||||
# and also writes arguments sorted by name.
|
||||
sorted_arguments=True,
|
||||
strict_arguments=True,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise VLLMValidationError(str(exc), parameter="tools") from exc
|
||||
if tag is None and forced:
|
||||
kind = "named" if named else "required"
|
||||
raise VLLMValidationError(
|
||||
f"response_template parsing cannot guarantee {kind} tool choice "
|
||||
"for this request",
|
||||
parameter="tool_choice",
|
||||
)
|
||||
if tag is None and constrained_tools is not None:
|
||||
logger.warning_once(
|
||||
"response_template parsing cannot enforce strict tools for this "
|
||||
"request."
|
||||
)
|
||||
return tag
|
||||
@@ -16,8 +16,6 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
HF_PARSER = "hf"
|
||||
|
||||
|
||||
class ParserManager:
|
||||
"""Provides a unified Parser by composing reasoning and tool parser adapters."""
|
||||
@@ -122,8 +120,10 @@ class ParserManager:
|
||||
tags (``--tool-strict-level``), or ``None`` for the tool
|
||||
parser's default level.
|
||||
tokenizer: Tokenizer the composed parser is validated against
|
||||
at startup (see :meth:`validate_parser_tokenizer`). ``None``
|
||||
skips the check (e.g. ``skip_tokenizer_init``).
|
||||
at startup (see :meth:`validate_parser_tokenizer`), including
|
||||
its `response_template` metadata when a response-template
|
||||
parser is selected. ``None`` skips the check (e.g.
|
||||
``skip_tokenizer_init``).
|
||||
|
||||
Returns:
|
||||
A Parser class, or None if neither parser is specified.
|
||||
@@ -184,39 +184,40 @@ class ParserManager:
|
||||
HarmonyParser.tool_strict_level = strict_level
|
||||
return HarmonyParser
|
||||
|
||||
if HF_PARSER in (reasoning_parser_name, tool_parser_name):
|
||||
if {reasoning_parser_name, tool_parser_name} - {
|
||||
HF_PARSER,
|
||||
None,
|
||||
"",
|
||||
}:
|
||||
raise TypeError(
|
||||
"The hf parser cannot be combined with other "
|
||||
"reasoning or tool call parsers"
|
||||
)
|
||||
from vllm.parser.response_template import (
|
||||
ResponseTemplateParser,
|
||||
validate_tokenizer_response_template,
|
||||
)
|
||||
from vllm.parser.response_template import (
|
||||
response_template_parser_cls,
|
||||
validate_tokenizer_response_template,
|
||||
)
|
||||
|
||||
template_parsers = {
|
||||
response_template_parser_cls(parser_cls)
|
||||
for parser_cls in (reasoning_parser_cls, tool_parser_cls)
|
||||
if parser_cls is not None
|
||||
}
|
||||
template_parser_cls = next(iter(template_parsers - {None}), None)
|
||||
if template_parser_cls is not None:
|
||||
if len(template_parsers) > 1:
|
||||
raise TypeError(
|
||||
f"--reasoning-parser {reasoning_parser_name} and "
|
||||
f"--tool-call-parser {tool_parser_name} cannot be combined"
|
||||
)
|
||||
if tokenizer is not None:
|
||||
validate_tokenizer_response_template(
|
||||
tokenizer,
|
||||
reasoning=reasoning_parser_cls is not None,
|
||||
tools=tool_parser_cls is not None,
|
||||
reasoning_parser=reasoning_parser_name,
|
||||
# Without auto tool choice, no tool parser is selected.
|
||||
tool_parser=tool_parser_name if tool_parser_cls else None,
|
||||
)
|
||||
|
||||
r_cls = reasoning_parser_cls
|
||||
t_cls = tool_parser_cls
|
||||
auto_tools = enable_auto_tools
|
||||
|
||||
class _ResponseTemplateParser(ResponseTemplateParser):
|
||||
reasoning_parser_cls = r_cls
|
||||
tool_parser_cls = t_cls
|
||||
tool_strict_level = strict_level
|
||||
_enable_auto_tools = auto_tools
|
||||
|
||||
return _ResponseTemplateParser
|
||||
return type(
|
||||
template_parser_cls.__name__,
|
||||
(template_parser_cls,),
|
||||
{
|
||||
"reasoning_parser_cls": reasoning_parser_cls,
|
||||
"tool_parser_cls": tool_parser_cls,
|
||||
"tool_strict_level": strict_level,
|
||||
"_enable_auto_tools": enable_auto_tools,
|
||||
},
|
||||
)
|
||||
|
||||
if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
|
||||
from vllm.parser.kimi_k3 import KimiK3Parser
|
||||
|
||||
@@ -7,6 +7,7 @@ from __future__ import annotations
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Callable, Iterable, Sequence
|
||||
from math import inf
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
|
||||
@@ -28,6 +29,7 @@ from vllm.parser.chat_parsing.response_templates import (
|
||||
ResponseTemplate,
|
||||
load_response_template,
|
||||
)
|
||||
from vllm.parser.response_template_grammar import message_start_anchors
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.renderers.chat_utils import make_tool_call_id
|
||||
from vllm.tool_parsers.abstract_tool_parser import ToolParser
|
||||
@@ -135,27 +137,38 @@ def validate_response_template_for_serving(
|
||||
def validate_tokenizer_response_template(
|
||||
tokenizer: Any,
|
||||
*,
|
||||
reasoning: bool,
|
||||
tools: bool,
|
||||
reasoning_parser: str | None,
|
||||
tool_parser: str | None,
|
||||
) -> None:
|
||||
"""Fail at startup if the tokenizer cannot serve the requested parsing."""
|
||||
"""Fail at startup if the tokenizer cannot serve the selected parsers,
|
||||
named as given on the command line."""
|
||||
template = resolve_response_template(tokenizer)
|
||||
if template is None:
|
||||
selected = " ".join(
|
||||
f"{flag} {name}"
|
||||
for flag, name in (
|
||||
("--reasoning-parser", reasoning_parser),
|
||||
("--tool-call-parser", tool_parser),
|
||||
)
|
||||
if name
|
||||
)
|
||||
raise TypeError(
|
||||
"The hf parser requires `response_template` metadata "
|
||||
f"{selected} requires `response_template` metadata "
|
||||
"in the tokenizer configuration"
|
||||
)
|
||||
try:
|
||||
fields = validate_response_template_for_serving(template).fields
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise TypeError(f"Invalid response_template metadata: {exc}") from exc
|
||||
if reasoning and THINKING_FIELD not in fields:
|
||||
if reasoning_parser and THINKING_FIELD not in fields:
|
||||
raise TypeError(
|
||||
"--reasoning-parser hf requires a response_template with a thinking field"
|
||||
f"--reasoning-parser {reasoning_parser} requires a response_template "
|
||||
"with a thinking field"
|
||||
)
|
||||
if tools and TOOL_FIELD not in fields:
|
||||
if tool_parser and TOOL_FIELD not in fields:
|
||||
raise TypeError(
|
||||
"--tool-call-parser hf requires a response_template with a tool_calls field"
|
||||
f"--tool-call-parser {tool_parser} requires a response_template "
|
||||
"with a tool_calls field"
|
||||
)
|
||||
|
||||
|
||||
@@ -292,7 +305,7 @@ def _serving_template(
|
||||
template = resolve_response_template(tokenizer, template)
|
||||
if template is None:
|
||||
raise ValueError(
|
||||
"The hf parser requires `response_template` "
|
||||
"Parsing with a response template requires `response_template` "
|
||||
"metadata in the tokenizer configuration"
|
||||
)
|
||||
validate_response_template_for_serving(template)
|
||||
@@ -319,12 +332,17 @@ def _to_tool_calls(value: Any) -> list[tuple[str, str]]:
|
||||
|
||||
|
||||
class ResponseTemplateReasoningParser(ReasoningParser):
|
||||
"""Reasoning end detection for `--reasoning-parser hf`.
|
||||
"""Reasoning end detection for a response-template parser.
|
||||
|
||||
`ResponseTemplateParser` parses the output; this parser tells it and the
|
||||
structured-output engine where reasoning ends.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
|
||||
"""The unified parser that selecting this reasoning parser stands for."""
|
||||
return ResponseTemplateParser
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
@@ -400,8 +418,26 @@ class ResponseTemplateReasoningParser(ReasoningParser):
|
||||
|
||||
|
||||
class ResponseTemplateToolParser(ToolParser):
|
||||
"""Registers `--tool-call-parser hf`; `ResponseTemplateParser` extracts the
|
||||
tool calls."""
|
||||
"""Selects a response-template parser as the tool parser;
|
||||
`ResponseTemplateParser` extracts the tool calls."""
|
||||
|
||||
@classmethod
|
||||
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
|
||||
"""The unified parser that selecting this tool parser stands for."""
|
||||
return ResponseTemplateParser
|
||||
|
||||
|
||||
def response_template_parser_cls(
|
||||
split_parser_cls: type[ReasoningParser] | type[ToolParser],
|
||||
) -> type[ResponseTemplateParser] | None:
|
||||
"""The response-template parser that a registered reasoning or tool parser
|
||||
stands for, or `None` for any other parser."""
|
||||
if issubclass(
|
||||
split_parser_cls,
|
||||
(ResponseTemplateReasoningParser, ResponseTemplateToolParser),
|
||||
):
|
||||
return split_parser_cls.unified_parser_cls()
|
||||
return None
|
||||
|
||||
|
||||
class _DeltaBuilder:
|
||||
@@ -441,23 +477,50 @@ class _ResponseStream:
|
||||
self.include_reasoning = include_reasoning
|
||||
self.parse_tools = parse_tools
|
||||
self.new_tool_call_id = new_tool_call_id
|
||||
self.template = template
|
||||
self.parser = ResponseParser(template, prefix=prefix, tools=tools)
|
||||
self._prefix_end = len(self.parser.input_text)
|
||||
self._next_tool_index = 0
|
||||
self._finalized = False
|
||||
self._open_region: dict[str, Any] | None = None
|
||||
self._track(self.parser.initial_events)
|
||||
|
||||
def feed(self, text: str, *, finished: bool) -> DeltaMessage | None:
|
||||
if self._finalized:
|
||||
return None
|
||||
delta = _DeltaBuilder()
|
||||
self._route(self.parser.feed(text), delta)
|
||||
self._route(self._track(self.parser.feed(text)), delta)
|
||||
if finished:
|
||||
self._finalized = True
|
||||
if len(self.parser.input_text) > self._prefix_end:
|
||||
_, events = self.parser.finalize()
|
||||
self._route(events, delta)
|
||||
events = self.parser.feed(self._closer_rest())
|
||||
self._route(events + self.parser.finalize()[1], delta)
|
||||
return delta.build()
|
||||
|
||||
def _track(self, events: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
for event in events:
|
||||
if event["type"] == "region_open":
|
||||
self._open_region = event
|
||||
elif event["type"] in ("region_close", "region_malformed"):
|
||||
self._open_region = None
|
||||
return events
|
||||
|
||||
def _closer_rest(self) -> str:
|
||||
"""The rest of a closer the output stopped in after ending its region,
|
||||
when only the next message's start anchor is missing."""
|
||||
if self._open_region is None:
|
||||
return ""
|
||||
field = self.template.fields[self._open_region["field"]]
|
||||
text = self.parser.input_text[self._open_region["end"] :]
|
||||
for closer in field.close_literals or ():
|
||||
for anchor in message_start_anchors(self.template):
|
||||
if not anchor or not closer.endswith(anchor) or closer == anchor:
|
||||
continue
|
||||
for cut in range(len(closer) - 1, len(closer) - len(anchor) - 1, -1):
|
||||
if text.endswith(closer[:cut]):
|
||||
return closer[cut:]
|
||||
return ""
|
||||
|
||||
def _route(self, events: Sequence[dict[str, Any]], delta: _DeltaBuilder) -> None:
|
||||
for event in events:
|
||||
field = event.get("field")
|
||||
@@ -499,6 +562,54 @@ class _ResponseStream:
|
||||
self._next_tool_index += 1
|
||||
|
||||
|
||||
class _ReasoningTokenCounter:
|
||||
"""Counts the reasoning tokens of one growing generation, parsing each
|
||||
token once."""
|
||||
|
||||
def __init__(
|
||||
self, template: ResponseTemplate, prefix: str, tokenizer: TokenizerLike
|
||||
) -> None:
|
||||
self.tokenizer = tokenizer
|
||||
self.parser = ResponseParser(template, prefix=prefix)
|
||||
self.start = len(self.parser.input_text)
|
||||
self.token_ids: list[int] = []
|
||||
self.token_ends: list[int] = []
|
||||
# Input spans between the delimiters of thinking regions.
|
||||
self.spans: list[tuple[int, float]] = []
|
||||
self.open: int | None = None
|
||||
self._track(self.parser.initial_events)
|
||||
|
||||
def continues(self, token_ids: Sequence[int]) -> bool:
|
||||
return list(token_ids[: len(self.token_ids)]) == self.token_ids
|
||||
|
||||
def count(self, token_ids: Sequence[int]) -> int:
|
||||
new_ids = list(token_ids[len(self.token_ids) :])
|
||||
texts = [_decode(self.tokenizer, [token_id]) for token_id in new_ids]
|
||||
end = self.token_ends[-1] if self.token_ends else self.start
|
||||
for text in texts:
|
||||
end += len(text)
|
||||
self.token_ends.append(end)
|
||||
self.token_ids.extend(new_ids)
|
||||
self._track(self.parser.feed("".join(texts)))
|
||||
# Text held back as a possible closer counts until a closer arrives.
|
||||
spans = self.spans if self.open is None else [*self.spans, (self.open, inf)]
|
||||
count, start = 0, self.start
|
||||
for end in self.token_ends:
|
||||
count += any(begin < end and start < close for begin, close in spans)
|
||||
start = end
|
||||
return count
|
||||
|
||||
def _track(self, events: Iterable[dict[str, Any]]) -> None:
|
||||
for event in events:
|
||||
if event["field"] != THINKING_FIELD or event["type"] == "region_chunk":
|
||||
continue
|
||||
if event["type"] == "region_open":
|
||||
self.open = event["end"]
|
||||
elif self.open is not None:
|
||||
self.spans.append((self.open, event["start"]))
|
||||
self.open = None
|
||||
|
||||
|
||||
class ResponseTemplateParser(Parser):
|
||||
"""Parse reasoning, content, and tool calls with one response template."""
|
||||
|
||||
@@ -526,6 +637,7 @@ class ResponseTemplateParser(Parser):
|
||||
self._prompt_token_ids: list[int] | None = None
|
||||
self._prefix = ""
|
||||
self._stream: _ResponseStream | None = None
|
||||
self._reasoning_counter: _ReasoningTokenCounter | None = None
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
prompt_token_ids = list(prompt_token_ids)
|
||||
@@ -534,6 +646,7 @@ class ResponseTemplateParser(Parser):
|
||||
self._prompt_token_ids = prompt_token_ids
|
||||
self._prefix = _decode(self.model_tokenizer, prompt_token_ids)
|
||||
self._stream = None
|
||||
self._reasoning_counter = None
|
||||
|
||||
def adjust_request(
|
||||
self,
|
||||
@@ -547,6 +660,14 @@ class ResponseTemplateParser(Parser):
|
||||
request.skip_special_tokens = False
|
||||
if hasattr(request, "spaces_between_special_tokens"):
|
||||
request.spaces_between_special_tokens = False
|
||||
return self._constrain_request(request)
|
||||
|
||||
def _constrain_request(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
"""Constrain `request` to the tool guarantees it asks for, or reject
|
||||
those this parser cannot enforce."""
|
||||
if (
|
||||
self._tool_parser is not None
|
||||
and request.tools
|
||||
@@ -567,6 +688,21 @@ class ResponseTemplateParser(Parser):
|
||||
or self._reasoning_parser.is_reasoning_end(input_ids)
|
||||
)
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
"""Count the generated tokens whose text the template parses as
|
||||
reasoning, continuing reasoning left open by the prompt."""
|
||||
if (
|
||||
self._reasoning_parser is None
|
||||
or THINKING_FIELD not in self.response_template.fields
|
||||
):
|
||||
return 0
|
||||
counter = self._reasoning_counter
|
||||
if counter is None or not counter.continues(token_ids):
|
||||
counter = self._reasoning_counter = _ReasoningTokenCounter(
|
||||
self.response_template, self._prefix, self.model_tokenizer
|
||||
)
|
||||
return counter.count(token_ids)
|
||||
|
||||
def parse(
|
||||
self,
|
||||
model_output: str,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -84,6 +84,10 @@ _REASONING_PARSERS_TO_REGISTER = {
|
||||
"response_template_reasoning_parser",
|
||||
"ResponseTemplateReasoningParser",
|
||||
),
|
||||
"beam": (
|
||||
"beam_reasoning_parser",
|
||||
"BeamReasoningParser",
|
||||
),
|
||||
"holo2": (
|
||||
"deepseek_v3_reasoning_parser",
|
||||
"DeepSeekV3ReasoningWithThinkingParser",
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.parser.beam import BeamReasoningParser
|
||||
|
||||
__all__ = ["BeamReasoningParser"]
|
||||
@@ -60,6 +60,7 @@ from vllm.renderers.chat_utils import (
|
||||
has_non_text_content,
|
||||
)
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_prompt_components,
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
)
|
||||
@@ -844,13 +845,17 @@ class OnlineRenderer:
|
||||
f"but got {type(request).__name__}"
|
||||
)
|
||||
raise NotImplementedError(msg)
|
||||
request = parser(
|
||||
request_parser = parser(
|
||||
tokenizer,
|
||||
request.tools,
|
||||
model_config=self.model_config,
|
||||
chat_template_kwargs=chat_params.chat_template_kwargs,
|
||||
).adjust_request(
|
||||
request=request,
|
||||
)
|
||||
prompt_token_ids = extract_prompt_components(
|
||||
self.model_config, engine_input
|
||||
).token_ids
|
||||
if prompt_token_ids is not None:
|
||||
request_parser.set_prompt_token_ids(prompt_token_ids)
|
||||
request = request_parser.adjust_request(request=request)
|
||||
|
||||
return conversation, [engine_input]
|
||||
|
||||
@@ -90,6 +90,10 @@ _TOOL_PARSERS_TO_REGISTER = {
|
||||
"response_template_tool_parser",
|
||||
"ResponseTemplateToolParser",
|
||||
),
|
||||
"beam": (
|
||||
"beam_tool_parser",
|
||||
"BeamToolParser",
|
||||
),
|
||||
"poolside_v1": (
|
||||
"poolside_v1_tool_parser",
|
||||
"PoolsideV1ToolParser",
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.parser.beam import BeamToolParser
|
||||
|
||||
__all__ = ["BeamToolParser"]
|
||||
Reference in New Issue
Block a user