From 153a17166e5d07bc38184dedeea542d2e8885e55 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Fri, 16 Oct 2026 14:08:29 -0700 Subject: [PATCH] [Frontend] Add the Beam reasoning and tool parsers (#62052) Signed-off-by: Woosuk Kwon Co-authored-by: Bugen Zhao Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com> Co-authored-by: neilkg <33635204+neilkg@users.noreply.github.com> Co-authored-by: Cursor Co-authored-by: Claude Opus 5.5 --- docs/features/tool_calling.md | 24 + rust/src/chat/src/lib.rs | 4 +- rust/src/chat/src/output/default/mod.rs | 41 +- rust/src/chat/src/parser/unified.rs | 8 +- rust/src/parser/src/output_grammar.rs | 5 +- .../parser/src/output_grammar/arguments.rs | 97 +- rust/src/parser/src/unified/beam.rs | 543 +++++++++ rust/src/parser/src/unified/beam/tests.rs | 897 +++++++++++++++ rust/src/parser/src/unified/hf/content.rs | 79 +- rust/src/parser/src/unified/hf/mod.rs | 2 +- rust/src/parser/src/unified/hf/parser.rs | 44 + rust/src/parser/src/unified/hf/template.rs | 25 + rust/src/parser/src/unified/mod.rs | 4 +- tests/parser/test_beam_parser.py | 747 ++++++++++++ .../parser/test_response_template_grammar.py | 492 ++++++++ vllm/entrypoints/openai/responses/protocol.py | 11 + vllm/entrypoints/openai/responses/serving.py | 3 + vllm/envs.py | 6 + vllm/parser/beam.py | 262 +++++ vllm/parser/parser_manager.py | 65 +- vllm/parser/response_template.py | 166 ++- vllm/parser/response_template_grammar.py | 1000 +++++++++++++++++ vllm/reasoning/__init__.py | 4 + vllm/reasoning/beam_reasoning_parser.py | 6 + vllm/renderers/online_renderer.py | 11 +- vllm/tool_parsers/__init__.py | 4 + vllm/tool_parsers/beam_tool_parser.py | 6 + 27 files changed, 4464 insertions(+), 92 deletions(-) create mode 100644 rust/src/parser/src/unified/beam.rs create mode 100644 rust/src/parser/src/unified/beam/tests.rs create mode 100644 tests/parser/test_beam_parser.py create mode 100644 tests/parser/test_response_template_grammar.py create mode 100644 vllm/parser/beam.py create mode 100644 vllm/parser/response_template_grammar.py create mode 100644 vllm/reasoning/beam_reasoning_parser.py create mode 100644 vllm/tool_parsers/beam_tool_parser.py diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index dd56db0d57..9980429324 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -201,6 +201,30 @@ checkpoint's output format. !!! note With `tool_choice="auto"`, structural-tag constraints require both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`, or a server-side floor set via `--tool-strict-level`. When these conditions are met and the selected parser supports structural tags, vLLM constrains the tool-call envelope and pins the argument schema of each tool that sets `strict: true` (or of every tool under `--tool-strict-level parameter`). Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema. +### Beam Models (`beam`) + +Beam checkpoints describe their output format with `response_template` metadata, so the `beam` parser reads it like the `hf` parser does: + +```bash +vllm serve \ + --enable-auto-tool-choice \ + --tool-call-parser beam \ + --reasoning-parser beam +``` + +Unlike `hf`, the `beam` parser also constrains its output. For strict tools, required or named tool choice, and structured outputs, it derives a +structural-tag grammar from the template. The grammar follows the template's tool call format, pins the arguments of strict tools when their +encoding can be derived, allows a single call with `parallel_tool_calls=false`, and frames a structured output as the answer. Raw string arguments +keep the `format` or `pattern` of their schema, as XGrammar keeps them in JSON strings, unless a match could end the value early. With +`tool_choice="none"`, the grammar allows only reasoning and an answer, so a model that still sees the tools in its prompt cannot call them. When no +grammar can be derived, required and named tool choice are rejected, and strict tools under `tool_choice="auto"` are left unconstrained with a +warning. + +It also constrains the output of requests without strict tools, forced tool choice, or structured outputs to the template's format: tool calls with +free arguments or an answer, or just the rest of an answer the prompt already opened. This always-on grammar needs the `xgrammar` structured +outputs backend, the default, and is disabled with `VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT=0`. From the message start, every `beam` grammar opens with +reasoning that holds visible text, unless `enable_thinking` is false. + ### Hermes Models (`hermes`) All Nous Research Hermes-series models newer than Hermes 2 Pro should be supported. diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 1ffb95fe37..5abdd36cc9 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -434,7 +434,7 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hf, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_json, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string()); + expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: beam, deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hf, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_json, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string()); } #[test] @@ -445,6 +445,6 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hf, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); + expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: beam, cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hf, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); } } diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index 67a5a11eae..dbbedfd836 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -23,7 +23,8 @@ use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEvent use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory}; use crate::parser::tool::{ToolParser, ToolParserFactory}; use crate::parser::unified::{ - HfTemplateError, HfUnifiedParser, ResponseTemplate, UnifiedParserFactory, names, + BeamUnifiedParser, HfTemplateError, HfUnifiedParser, ResponseTemplate, UnifiedParserFactory, + names, }; use crate::parser::{ParserSelection, ToolStrictLevel}; use crate::request::{ChatRequest, ChatTool}; @@ -206,17 +207,19 @@ impl DefaultChatOutputProcessor { }); } - // `hf` is built from the model's template, never by its registry constructor. - let parser = if parser_name == names::HF { - let template = - response_template.as_ref().map_err(|error| Error::ParserInitialization { - kind: "unified", - name: parser_name.to_string(), - error: Box::new(error.clone()), - })?; - Box::new(HfUnifiedParser::new(template.clone(), tools, tokenizer)) - } else { - factory.create(parser_name, tools, tokenizer)? + // Parsers driven by the model's template are built from it, never by + // their registry constructors. + let template = || { + response_template.clone().map_err(|error| Error::ParserInitialization { + kind: "unified", + name: parser_name.to_string(), + error: Box::new(error), + }) + }; + let parser: Box = match parser_name { + names::HF => Box::new(HfUnifiedParser::new(template()?, tools, tokenizer)), + names::BEAM => Box::new(BeamUnifiedParser::new(template()?, tools, tokenizer)), + _ => factory.create(parser_name, tools, tokenizer)?, }; UNIFIED_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using unified parser")); @@ -589,6 +592,20 @@ mod tests { .assert_eq(&error(invalid_template())); } + #[test] + fn explicit_beam_uses_the_model_response_template() { + let beam = ParserSelection::Explicit("beam".to_string()); + crate::parser::validate_parser_overrides(&beam, &beam).unwrap(); + assert!(keeps_special_tokens("other-model", &loaded_template(), &beam, &beam).unwrap()); + + let error = + keeps_special_tokens("other-model", &Err(HfTemplateError::Missing), &beam, &beam) + .unwrap_err() + .to_report_string(); + expect_test::expect!["failed to initialize unified parser `beam`: the model's tokenizer_config.json provides no response_template"] + .assert_eq(&error); + } + #[test] fn auto_never_selects_hf() { let auto = ParserSelection::Auto; diff --git a/rust/src/chat/src/parser/unified.rs b/rust/src/chat/src/parser/unified.rs index 7db093c696..705133bb0a 100644 --- a/rust/src/chat/src/parser/unified.rs +++ b/rust/src/chat/src/parser/unified.rs @@ -6,8 +6,8 @@ use std::sync::LazyLock; pub use vllm_parser::unified::{ - Gemma4UnifiedParser, HfTemplateError, HfUnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser, - InklingUnifiedParser, KimiK3UnifiedParser, ResponseTemplate, UnifiedParser, + BeamUnifiedParser, Gemma4UnifiedParser, HfTemplateError, HfUnifiedParser, HyV3UnifiedParser, + HyV4UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser, ResponseTemplate, UnifiedParser, }; use vllm_tokenizer::DynTokenizer; @@ -26,6 +26,9 @@ pub mod names { /// name resolution only: it is built from the model's template, never by its /// registry constructor. pub const HF: &str = "hf"; + /// Beam checkpoints: [`HF`] with Beam's tool-call grammar, built from the + /// model's template the same way. + pub const BEAM: &str = "beam"; } /// Constructor signature for one registered unified parser implementation. @@ -55,6 +58,7 @@ impl UnifiedParserFactory { factory.register_parser::(names::KIMI_K3); // Opt-in only: no model patterns map to it. factory.register_parser::(names::HF); + factory.register_parser::(names::BEAM); factory .register_pattern("gemma-4", names::GEMMA4) diff --git a/rust/src/parser/src/output_grammar.rs b/rust/src/parser/src/output_grammar.rs index ba9b4968d5..0d05bb4dde 100644 --- a/rust/src/parser/src/output_grammar.rs +++ b/rust/src/parser/src/output_grammar.rs @@ -207,7 +207,8 @@ pub(crate) fn full_format_from_builder_for_test( ) } -fn tool_params(tools: &[Tool], strict_level: ToolStrictLevel) -> Vec { +/// Tools as builder inputs, strict when the tool or the server floor asks for it. +pub(crate) fn tool_params(tools: &[Tool], strict_level: ToolStrictLevel) -> Vec { tools .iter() .map(|tool| { @@ -240,7 +241,7 @@ pub(crate) fn normalize_tool_choice( } /// Whether the request asks for a tool grammar at all. -fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool { +pub(crate) fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool { if ctx.tools.is_empty() { return false; } diff --git a/rust/src/parser/src/output_grammar/arguments.rs b/rust/src/parser/src/output_grammar/arguments.rs index 0f621789a3..6e58c6956a 100644 --- a/rust/src/parser/src/output_grammar/arguments.rs +++ b/rust/src/parser/src/output_grammar/arguments.rs @@ -26,7 +26,7 @@ //! - `allOf` with more than one schema accepts any value. use serde_json::{Value, json}; -use xgrammar_structural_tag::format::{Format, JsonSchemaFormat}; +use xgrammar_structural_tag::format::{EndBoundary, Format, JsonSchemaFormat, TokenFormat}; pub use crate::schema::JsonType; use crate::schema::{self, OptionSource, SchemaOption, SchemaRoot}; @@ -60,7 +60,13 @@ impl ParameterKey<'_> { /// `prefix KEY suffix content end`: a tag for a declared key, or a /// sequence matching [`Self::FREE_PATTERN`] for a free key. - pub fn tag(&self, prefix: &str, suffix: &str, content: Format, end: &str) -> Format { + pub fn tag( + &self, + prefix: &str, + suffix: &str, + content: Format, + end: impl Into, + ) -> Format { match self { Self::Declared(key) => Format::tag(format!("{prefix}{key}{suffix}"), content, end), Self::Free => Format::sequence(vec![ @@ -68,7 +74,13 @@ impl ParameterKey<'_> { Format::regex(Self::FREE_PATTERN), Format::const_string(suffix), content, - Format::const_string(end), + match end.into() { + EndBoundary::Text(end) => Format::const_string(end), + EndBoundary::Texts(ends) => { + one_of(ends.into_iter().map(Format::const_string).collect()) + } + EndBoundary::Token(end) => Format::Token(TokenFormat { token: end.token }), + }, ]), } } @@ -183,6 +195,36 @@ impl ArgumentContext<'_> { /// Build the grammar of one call's arguments under `schema`, the tool's /// parameters schema, rendering each parameter with `syntax`. pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &ArgumentOptions) -> Format { + parameter_list( + options, + syntax.separator(), + parameters(schema, syntax, options), + ) +} + +/// The parameters of one call, each rendered by an [`ArgumentSyntax`], for a +/// syntax that lists them itself rather than through [`arguments`]. +pub struct Parameters { + /// The declared parameters, in declared order. + pub declared: Vec, + /// Any one additional parameter, if the schema allows them. + pub additional: Option, +} + +/// One declared parameter. +pub struct Parameter { + pub key: String, + pub format: Format, + pub required: bool, +} + +/// The parameters of one call under `schema`, the tool's parameters schema, +/// rendering each parameter with `syntax`. +pub fn parameters( + schema: &Value, + syntax: &dyn ArgumentSyntax, + options: &ArgumentOptions, +) -> Parameters { let cx = ArgumentContext { options, root: SchemaRoot::new(schema), @@ -190,8 +232,7 @@ pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &Argument let parameter = |key: ParameterKey<'_>, schema: &Value| syntax.parameter(key, &cx.value_options(schema)); - let schema = cx.root.resolved(); - let (properties, additional) = match schema { + match cx.root.resolved() { Value::Object(schema) => { let additional = match schema.get("additionalProperties") { Some(Value::Bool(false)) => None, @@ -206,22 +247,33 @@ pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &Argument .and_then(Value::as_array) .map(|required| required.iter().filter_map(Value::as_str).collect::>()) .unwrap_or_default(); - let properties = schema + let declared = schema .get("properties") .and_then(Value::as_object) .into_iter() .flatten() .filter_map(|(key, schema)| { - let parameter = parameter(ParameterKey::Declared(key), schema)?; - Some((parameter, required.contains(&key.as_str()))) + Some(Parameter { + key: key.clone(), + format: parameter(ParameterKey::Declared(key), schema)?, + required: required.contains(&key.as_str()), + }) }) .collect(); - (properties, additional) + Parameters { + declared, + additional, + } } - Value::Bool(false) => (vec![], None), - _ => (vec![], parameter(ParameterKey::Free, &ANY_SCHEMA)), - }; - parameter_list(options, syntax.separator(), properties, additional) + Value::Bool(false) => Parameters { + declared: vec![], + additional: None, + }, + _ => Parameters { + declared: vec![], + additional: parameter(ParameterKey::Free, &ANY_SCHEMA), + }, + } } /// Parameters in declared order, required ones mandatory, followed by any @@ -230,8 +282,10 @@ pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &Argument fn parameter_list( options: &ArgumentOptions, separator: Option, - properties: Vec<(Format, bool)>, - additional: Option, + Parameters { + declared, + additional, + }: Parameters, ) -> Format { let separated = |parameter: Format| match &separator { Some(separator) => Format::sequence(vec![parameter, separator.clone()]), @@ -239,10 +293,10 @@ fn parameter_list( }; let mut elements = Vec::from_iter(separator.clone()); if options.any_order { - let any_required = properties.iter().any(|(_, required)| *required); - let parameters = properties + let any_required = declared.iter().any(|parameter| parameter.required); + let parameters = declared .into_iter() - .map(|(parameter, _)| parameter) + .map(|parameter| parameter.format) .chain(additional) .collect::>(); if !parameters.is_empty() { @@ -254,8 +308,11 @@ fn parameter_list( }); } } else { - for (parameter, required) in properties { - let parameter = separated(parameter); + for Parameter { + format, required, .. + } in declared + { + let parameter = separated(format); elements.push(if required { parameter } else { diff --git a/rust/src/parser/src/unified/beam.rs b/rust/src/parser/src/unified/beam.rs new file mode 100644 index 0000000000..fb3b904e4e --- /dev/null +++ b/rust/src/parser/src/unified/beam.rs @@ -0,0 +1,543 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Unified parser for Beam checkpoints. +//! +//! Beam output is parsed by [`HfUnifiedParser`] with the checkpoint's +//! `response_template`; this parser adds Beam's output grammar. +//! A Beam message after `<|start|>assistant` is: +//! +//! ```text +//! <|channel|>analysis<|message|>REASONING<|end|><|start|>assistant +//! ( <|channel|>final<|message|>ANSWER<|return|> +//! | CALL (<|end|><|start|>assistant CALL)* <|call|> ) +//! ``` +//! +//! where reasoning may end at any closer of the template's reasoning field, +//! and `CALL` is +//! ` to=NAME<|channel|>analysis <|constrain|>xml<|message|>` followed by +//! `\nVALUE\n` parameters separated +//! by newlines. Where the template looks ahead for the next parameter, a value +//! may also end on a newline right before it. +//! +//! The grammar enforces the schema of every declared tool's arguments, in +//! declared or alphabetical order, since Beam drifts from schemas it does not +//! enforce. When the model may answer instead, calls may also name any other +//! tool with any arguments, which the parser passes on so a harness can tell +//! the model the tool does not exist. `ANSWER` follows the request's structured +//! output, if any. +//! +//! The chat template rejects a history turn without reasoning, so a message the +//! model opens holds non-empty reasoning: an empty one would come back as null +//! reasoning and fail the next request. +//! +//! Unless `VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT` turns it off, the grammar +//! applies to every message, including one the request does not constrain. + +use std::collections::BTreeMap; +use std::sync::{Arc, LazyLock}; + +use serde_json::Value; +use vllm_tokenizer::{DecodedText, DynTokenizer}; +use xgrammar_structural_tag::ToolChoice; +use xgrammar_structural_tag::format::{EndBoundary, Format}; +use xgrammar_structural_tag::tool::{ + BuilderToolChoice, NormalizedToolChoice, ToolChoiceValue, normalize_tool_choice, +}; + +use super::{ + HfPosition, HfUnifiedParser, ResponseTemplate, Result, UnifiedParser, UnifiedParserError, + UnifiedParserOutput, +}; +use crate::output_grammar::arguments::{ + self, ArgumentOptions, ArgumentSyntax, JsonType, ParameterKey, Parameters, ValueOption, one_of, +}; +use crate::output_grammar::{ + self, BuiltOutputGrammar, OutputGrammarContext, tool_grammar_applies, tool_params, +}; +use crate::tool::Tool; + +const ANALYSIS_OPEN: &str = "<|channel|>analysis<|message|>"; +const FINAL_OPEN: &str = "<|channel|>final<|message|>"; +const END: &str = "<|end|>"; +/// Closes one message and opens the next assistant message. +const NEXT_MESSAGE: &str = "<|end|><|start|>assistant"; +const RETURN: &str = "<|return|>"; +const CALL: &str = "<|call|>"; +/// Follows ` to=NAME` in a call. +const CALL_HEADER_END: &str = "<|channel|>analysis <|constrain|>xml<|message|>"; +const PARAMETER_OPEN: &str = "\n"; +const PARAMETER_CLOSE: &str = "\n"; +/// Ends a value where the template looks ahead for the next parameter. +const NEXT_PARAMETER: &str = "\n\n = LazyLock::new(|| { + std::env::var("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT").map_or(true, |value| { + matches!(value.to_lowercase().as_str(), "true" | "1") + }) +}); + +/// Beam parser: [`HfUnifiedParser`] with Beam's output grammar. +pub struct BeamUnifiedParser { + inner: HfUnifiedParser, + /// Whether a value may end where the next parameter opens. + lenient: bool, + /// What ends the reasoning and opens the next message. + reasoning_ends: Vec, +} + +impl BeamUnifiedParser { + /// Create a parser for one request from the checkpoint's `response_template`. + pub fn new(template: Arc, tools: &[Tool], tokenizer: DynTokenizer) -> Self { + Self { + lenient: template.argument_lookahead() == Some(PARAMETER_OPEN), + reasoning_ends: match template.reasoning_closers() { + [] => vec![NEXT_MESSAGE.to_string()], + closers => closers.to_vec(), + }, + inner: HfUnifiedParser::new(template, tools, tokenizer), + } + } +} + +impl UnifiedParser for BeamUnifiedParser { + fn create(_tools: &[Tool], _tokenizer: DynTokenizer) -> Result> { + Err(UnifiedParserError::NoNamedConstructor { + parser: "beam", + built_from: "the checkpoint's response_template", + }) + } + + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { + self.inner.initialize(prompt_token_ids) + } + + fn preserve_special_tokens(&self) -> bool { + self.inner.preserve_special_tokens() + } + + fn build_output_grammar( + &self, + ctx: &OutputGrammarContext<'_>, + ) -> output_grammar::Result> { + build_output_grammar( + self.inner.position(), + ctx, + *ALWAYS_CONSTRAIN_OUTPUT, + self.lenient, + &self.reasoning_ends, + ) + } + + fn tool_call_id(&self, tool_index: usize) -> Option<&str> { + self.inner.tool_call_id(tool_index) + } + + fn parse_into(&mut self, delta: DecodedText, output: &mut UnifiedParserOutput) -> Result<()> { + self.inner.parse_into(delta, output) + } + + fn finish(&mut self) -> Result { + self.inner.finish() + } + + fn reset(&mut self) -> String { + self.inner.reset() + } +} + +/// The grammar of a Beam message from the first generated token, or `None` +/// when the parser is neither at the start of the message nor inside its +/// reasoning, or when the request constrains nothing and `always_constrain` is +/// off. Values may end where the next parameter opens if `lenient`, and +/// reasoning ends at any of `reasoning_ends`. +fn build_output_grammar( + position: HfPosition, + ctx: &OutputGrammarContext<'_>, + always_constrain: bool, + lenient: bool, + reasoning_ends: &[String], +) -> output_grammar::Result> { + let forced = !ctx.tools.is_empty() + && matches!( + ctx.tool_choice, + ToolChoice::Value(ToolChoiceValue::Required) | ToolChoice::NamedFunction(_) + ); + let answer_format = ctx.answer.filter(|_| !forced); + // The prompt may still list the tools, so only the grammar keeps the model + // from calling them. + let tools_forbidden = !ctx.tools.is_empty() + && matches!(ctx.tool_choice, ToolChoice::Value(ToolChoiceValue::None)); + if !(always_constrain + || tool_grammar_applies(ctx) + || tools_forbidden + || answer_format.is_some()) + { + return Ok(None); + } + let text = Format::any_text(); + let reasoning_end = EndBoundary::from(reasoning_ends.to_vec()); + let reasoning = match position { + HfPosition::ResponseStart => Format::tag( + ANALYSIS_OPEN, + // At least one character, which cannot start Beam's closers. + Format::sequence(vec![Format::regex("[^<]"), text]), + reasoning_end, + ), + HfPosition::Reasoning => Format::tag("", text, reasoning_end), + // Inside an answer the prompt opened, only the always-on grammar + // applies, finishing a free answer. A tool choice that forces a call + // cannot be met there, and the request's own constraint applies as is. + HfPosition::Content => { + return Ok( + (always_constrain && !forced && answer_format.is_none()).then(|| { + BuiltOutputGrammar::from_token_zero(Format::sequence(answer_rest(None))) + }), + ); + } + HfPosition::Other => return Ok(None), + }; + let NormalizedToolChoice { + function_tools, + choice, + .. + } = normalize_tool_choice( + &tool_params(ctx.tools, ctx.tool_strict_level), + ctx.tool_choice.clone(), + )?; + if function_tools.is_empty() { + return Ok(match choice { + // `none`, or no tools to call: the model can only answer. + BuilderToolChoice::Auto => { + Some(BuiltOutputGrammar::from_token_zero(Format::sequence(vec![ + reasoning, + answer(answer_format), + ]))) + } + BuilderToolChoice::Required | BuilderToolChoice::Forced => None, + }); + } + + let mut calls = function_tools + .iter() + .map(|tool| { + let header = + Format::const_string(format!(" to={}{CALL_HEADER_END}", tool.function.name)); + let schema = tool.function.parameters.clone().unwrap_or(Value::Bool(true)); + Format::sequence(vec![header, arguments(&schema, lenient)]) + }) + .collect::>(); + if matches!(choice, BuilderToolChoice::Auto) { + let names = ctx.tools.iter().map(|tool| tool.name.as_str()).collect::>(); + let header = Format::sequence(vec![ + Format::const_string(" to="), + Format::regex(other_names(&names)), + Format::const_string(CALL_HEADER_END), + ]); + calls.push(Format::sequence(vec![ + header, + arguments(&Value::Bool(true), lenient), + ])); + } + let call = one_of(calls); + let mut calls = vec![call.clone()]; + if ctx.parallel_tool_calls && !choice.is_forced() { + calls.push(Format::star(Format::sequence(vec![ + Format::const_string(NEXT_MESSAGE), + call, + ]))); + } + // The last call ends with the `<|call|>` stop token. + calls.push(Format::optional(Format::const_string(CALL))); + let calls = Format::sequence(calls); + let output = match choice { + BuilderToolChoice::Auto => Format::or(vec![calls, answer(answer_format)]), + BuilderToolChoice::Required | BuilderToolChoice::Forced => calls, + }; + Ok(Some(BuiltOutputGrammar::from_token_zero(Format::sequence( + vec![reasoning, output], + )))) +} + +/// A final answer, ending with the `<|return|>` stop token. +fn answer(format: Option<&Format>) -> Format { + Format::sequence([vec![Format::const_string(FINAL_OPEN)], answer_rest(format)].concat()) +} + +/// The rest of an open final answer, holding `format` or else free text. +fn answer_rest(format: Option<&Format>) -> Vec { + vec![ + // Without `<|end|>`, no unconstrained message can follow the answer. + format.map_or_else(|| Format::any_text_excluding(&[END, RETURN]), Format::clone), + Format::optional(Format::const_string(RETURN)), + ] +} + +/// The arguments under `schema`: its declared parameters in declared or +/// alphabetical order, required ones mandatory, then any additional ones. If +/// `lenient`, each value but the last may end by opening the next parameter. +fn arguments(schema: &Value, lenient: bool) -> Format { + let options = ArgumentOptions { + excludes: CALL_TERMINATORS.map(str::to_string).to_vec(), + ..ArgumentOptions::default() + }; + let render = |syntax: BeamArguments| arguments::parameters(schema, &syntax, &options); + if lenient { + lenient_arguments(render(BeamArguments::Joined), render(BeamArguments::Last)) + } else { + closed_arguments(render(BeamArguments::Closed)) + } +} + +/// Arguments whose values all end with a closer. +fn closed_arguments( + Parameters { + declared, + additional, + }: Parameters, +) -> Format { + let additional = additional.map(Format::star); + let lists = orders(&declared, |parameter| ¶meter.key) + .into_iter() + .map(|order| { + let listed = order + .into_iter() + .map(|parameter| listed(parameter.format.clone(), parameter.required)); + listed.chain(additional.clone()).collect::>() + }) + .filter(|list| !list.is_empty()) + .map(Format::sequence) + .collect::>(); + if lists.is_empty() { + Format::const_string("") + } else { + one_of(lists) + } +} + +/// A declared parameter as one another parameter follows, `joined`, and as the +/// last one. +struct LenientParameter { + key: String, + joined: Format, + last: Format, + required: bool, +} + +impl LenientParameter { + fn joined(&self) -> Format { + listed(self.joined.clone(), self.required) + } +} + +/// Arguments whose values end by opening the next parameter, but for the last +/// one, from the parameters as [`BeamArguments::Joined`] and as +/// [`BeamArguments::Last`]. +fn lenient_arguments(joined: Parameters, last: Parameters) -> Format { + // Both forms share their terminators, so they keep the same parameters. + let declared = joined + .declared + .into_iter() + .zip(last.declared) + .map(|(joined, last)| LenientParameter { + key: joined.key, + joined: joined.format, + last: last.format, + required: joined.required, + }) + .collect::>(); + let additional = joined.additional.zip(last.additional); + + let mut lists = vec![]; + for order in orders(&declared, |parameter| ¶meter.key) { + // The last parameter is a declared one that follows every required + // one, or an additional one. + if !order.is_empty() { + let first = order.iter().rposition(|parameter| parameter.required).unwrap_or(0); + let mut list = + order[..first].iter().map(|parameter| parameter.joined()).collect::>(); + list.push(last_present(&order[first..])); + lists.push(Format::sequence(list)); + } + if let Some((joined, last)) = &additional { + let mut list = order.iter().map(|parameter| parameter.joined()).collect::>(); + list.extend([Format::star(joined.clone()), last.clone()]); + lists.push(Format::sequence(list)); + } + } + if lists.is_empty() { + return Format::const_string(""); + } + + // Each parameter starts past the opener, as the value before writes it. + let arguments = Format::sequence(vec![ + separator(), + Format::const_string(PARAMETER_OPEN), + one_of(lists), + ]); + if declared.iter().any(|parameter| parameter.required) { + arguments + } else { + Format::optional(arguments) + } +} + +/// `parameters`, at least one, in order and the last present one in its last +/// form. Only the first may be required. +fn last_present(parameters: &[&LenientParameter]) -> Format { + if let [only] = parameters { + return only.last.clone(); + } + // The last one is in the first half, or after all of it. Halving keeps the + // grammar O(n log n) in size and O(log n) deep. + let (head, tail) = parameters.split_at(parameters.len() / 2); + let mut after_head = head.iter().map(|parameter| parameter.joined()).collect::>(); + after_head.push(last_present(tail)); + Format::or(vec![last_present(head), Format::sequence(after_head)]) +} + +/// `parameters` in declared order, and in alphabetical order by `key` if that +/// differs, since Beam often writes them sorted. +fn orders

(parameters: &[P], key: impl Fn(&P) -> &str) -> Vec> { + let declared = parameters.iter().collect::>(); + let mut sorted = declared.clone(); + sorted.sort_by_key(|parameter| key(parameter)); + if sorted + .iter() + .map(|parameter| key(parameter)) + .eq(declared.iter().map(|parameter| key(parameter))) + { + vec![declared] + } else { + vec![declared, sorted] + } +} + +/// `parameter` as listed: optional unless `required`. +fn listed(parameter: Format, required: bool) -> Format { + if required { + parameter + } else { + Format::optional(parameter) + } +} + +fn separator() -> Format { + Format::optional(Format::const_string("\n")) +} + +/// A regex for the tool names the call header can spell (no whitespace or +/// `<`) besides `names`. +fn other_names(names: &[&str]) -> String { + #[derive(Default)] + struct Node { + children: BTreeMap, + end: bool, + } + + fn escape(char: char, meta: &str) -> String { + if meta.contains(char) { + format!("\\{char}") + } else { + char.to_string() + } + } + + /// A name character other than `chars`. + fn other<'a>(chars: impl Iterator) -> String { + let chars = chars.map(|&char| escape(char, r"\]^-[")).collect::(); + format!(r"[^\s<{chars}]") + } + + /// The rest of a name past `node` that ends on none of `names`. + fn rest(node: &Node, first: bool) -> String { + let any = other([].iter()); + let mut branches = vec![format!("{}{any}*", other(node.children.keys()))]; + branches.extend(node.children.iter().map(|(&char, child)| { + format!("{}{}", escape(char, r".^$*+?{}[]\|()"), rest(child, false)) + })); + let optional = if first || node.end { "" } else { "?" }; + format!("(?:{}){optional}", branches.join("|")) + } + + let mut root = Node::default(); + for name in names { + let mut node = &mut root; + for char in name.chars() { + node = node.children.entry(char).or_default(); + } + node.end = true; + } + rest(&root, true) +} + +/// How one parameter is written: `\nVALUE` followed +/// by its end. +/// +/// The parser reads every value as text and converts it by the tool schema, so +/// string options are raw text and every other option is JSON. +enum BeamArguments { + /// The whole parameter, ending with `\n`, after an optional + /// newline. + Closed, + /// The parameter past its opener, where the template looks ahead for the + /// next one: it ends by opening the next parameter, also right after the + /// value's newline, so raw text holds no such opener. + Joined, + /// The last parameter past its opener under such a template, ending with + /// `\n`. + Last, +} + +impl ArgumentSyntax for BeamArguments { + fn separator(&self) -> Option { + None + } + + fn parameter(&self, key: ParameterKey<'_>, options: &[ValueOption<'_>]) -> Option { + let terminators: &[&str] = match self { + Self::Closed => &[""], + Self::Joined | Self::Last => &["", NEXT_PARAMETER], + }; + let values = options + .iter() + .filter_map(|option| match option.ty { + JsonType::String => option.raw_string(terminators), + _ => Some(option.json()), + }) + .collect::>(); + (!values.is_empty()).then(|| { + let value = one_of(values); + match self { + Self::Closed => Format::sequence(vec![ + separator(), + key.tag(PARAMETER_OPEN, PARAMETER_NAME_CLOSE, value, PARAMETER_CLOSE), + ]), + Self::Joined => key.tag( + "", + PARAMETER_NAME_CLOSE, + value, + NEXT_PARAMETER_ENDS.to_vec(), + ), + Self::Last => key.tag("", PARAMETER_NAME_CLOSE, value, PARAMETER_CLOSE), + } + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/rust/src/parser/src/unified/beam/tests.rs b/rust/src/parser/src/unified/beam/tests.rs new file mode 100644 index 0000000000..906a01f334 --- /dev/null +++ b/rust/src/parser/src/unified/beam/tests.rs @@ -0,0 +1,897 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::sync::Arc; + +use serde_json::{Value, json}; +use vllm_tokenizer::test_utils::TestTokenizer; +use vllm_tokenizer::{DecodedText, Tokenizer as _}; +use xgrammar_structural_tag::ToolChoice; +use xgrammar_structural_tag::format::Format; + +use super::{BeamUnifiedParser, HfPosition, build_output_grammar, other_names}; +use crate::output_grammar::test_utils::outline; +use crate::output_grammar::{GrammarCoverage, OutputGrammarContext, ToolStrictLevel}; +use crate::tool::Tool; +use crate::unified::{ResponseTemplate, UnifiedParser, UnifiedParserEvent, UnifiedParserOutput}; + +/// The `tag_pattern` of the checkpoint of [`template`]. +const TAG_PATTERN: &str = + r#"\n(?P.*?)\n"#; +/// [`TAG_PATTERN`], but a value may also end where the next parameter opens. +const LENIENT_TAG_PATTERN: &str = r#"\n(?P.*?)\n(?:|(?= Arc { + template_with(tag_pattern, json!("<|end|><|start|>assistant")) +} + +/// [`template`] with `thinking_close` as the reasoning closer. +fn template_with(tag_pattern: &str, thinking_close: Value) -> Arc { + let template = json!({ + "defaults": {"role": "assistant"}, + "start_anchor_pattern": r"<\|start\|>assistant", + "fields": { + "thinking": { + "open": "<|channel|>analysis<|message|>", + "close": thinking_close, + "content": "text" + }, + "content": { + "open": "<|channel|>final<|message|>", + "close": ["<|return|>", "<|end|>", "<|call|>"], + "content": "text" + }, + "tool_calls": { + "open_pattern": r" to=(?P[^\s<]+)<\|channel\|>analysis <\|constrain\|>xml<\|message\|>", + "close": ["<|end|><|start|>assistant", "<|call|>"], + "repeats": true, + "content": "xml-inline", + "content_args": { + "tag_pattern": tag_pattern, + "value_parser": {"name": "text", "args": {"strip": false}} + }, + "transform": { + "type": "function", + "function": {"name": "{name}", "arguments": "{content}"} + } + } + } + }); + Arc::new(ResponseTemplate::from_json(&template).unwrap()) +} + +fn tools() -> Vec { + vec![ + Tool { + name: "get_weather".to_string(), + description: None, + parameters: json!({ + "type": "object", + "properties": { + "location": {"type": "string"}, + "days": {"type": "integer"}, + "unit": {"enum": ["c", "f"]} + }, + "required": ["location"], + "additionalProperties": false + }), + strict: Some(true), + defer_loading: None, + }, + Tool { + name: "get_time".to_string(), + description: None, + parameters: json!({ + "type": "object", + "properties": {"zone": {"type": "string"}} + }), + strict: None, + defer_loading: None, + }, + ] +} + +/// A parser initialized from `prompt`, the rendered conversation. +fn parser(prompt: &str) -> BeamUnifiedParser { + parser_with(TAG_PATTERN, prompt) +} + +fn parser_with(tag_pattern: &str, prompt: &str) -> BeamUnifiedParser { + let tokenizer = Arc::new(TestTokenizer::new()); + let prompt_ids = tokenizer.encode(prompt, false).unwrap(); + let mut parser = BeamUnifiedParser::new(template(tag_pattern), &tools(), tokenizer); + parser.initialize(&prompt_ids).unwrap(); + parser +} + +const PROMPT: &str = "<|start|>user<|message|>Weather in Paris?<|end|><|start|>assistant"; + +fn grammar(position: HfPosition, tool_choice: ToolChoice, parallel: bool) -> Option { + grammar_with(position, &tools(), tool_choice, parallel, false, false) +} + +fn grammar_with( + position: HfPosition, + tools: &[Tool], + tool_choice: ToolChoice, + parallel: bool, + always_constrain: bool, + lenient: bool, +) -> Option { + let ctx = OutputGrammarContext { + tools, + tool_choice: &tool_choice, + tool_strict_level: ToolStrictLevel::Auto, + parallel_tool_calls: parallel, + answer: None, + }; + grammar_in(position, &ctx, always_constrain, lenient) +} + +fn grammar_in( + position: HfPosition, + ctx: &OutputGrammarContext<'_>, + always_constrain: bool, + lenient: bool, +) -> Option { + let grammar = build_output_grammar( + position, + ctx, + always_constrain, + lenient, + template(TAG_PATTERN).reasoning_closers(), + ) + .unwrap()?; + assert_eq!(grammar.coverage, GrammarCoverage::FromTokenZero); + Some(outline(&grammar.format)) +} + +fn constrained_grammar( + position: HfPosition, + tool_choice: ToolChoice, + answer: Format, + always_constrain: bool, +) -> Option { + let tools = tools(); + let ctx = OutputGrammarContext { + tools: &tools, + tool_choice: &tool_choice, + tool_strict_level: ToolStrictLevel::Auto, + parallel_tool_calls: true, + answer: Some(&answer), + }; + grammar_in(position, &ctx, always_constrain, false) +} + +#[test] +fn position_follows_the_prompt_and_then_the_output() { + let position = |suffix: &str| parser(&format!("{PROMPT}{suffix}")).inner.position(); + + assert_eq!(position(""), HfPosition::ResponseStart); + assert_eq!( + position("<|channel|>analysis<|message|>Let me"), + HfPosition::Reasoning + ); + // The reasoning closer ends with the anchor, so the next message starts. + assert_eq!( + position("<|channel|>analysis<|message|>Done.<|end|><|start|>assistant"), + HfPosition::ResponseStart + ); + // A partial closer stays buffered. + assert_eq!( + position("<|channel|>analysis<|message|>Done.<|end|>"), + HfPosition::Other + ); + assert_eq!( + position("<|channel|>final<|message|>It is"), + HfPosition::Content + ); + // A partial boundary stays buffered. + assert_eq!(position("<|channel|>ana"), HfPosition::Other); + // Without the anchor, the parser parses the output from the start. + assert_eq!( + parser("no anchor").inner.position(), + HfPosition::ResponseStart + ); + + let mut parser = parser(PROMPT); + parser + .parse_into( + DecodedText::unattributed("<|channel|>analysis<|message|>Hmm"), + &mut UnifiedParserOutput::default(), + ) + .unwrap(); + assert_eq!(parser.inner.position(), HfPosition::Reasoning); +} + +#[test] +fn required_calls_any_tool_after_reasoning() { + expect_test::expect![[r#" + sequence + tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant` + sequence + /[^<]/ + text + sequence + or + sequence + ` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>` + or + sequence + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + optional + sequence + optional `\n` + tag `\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n` + optional + sequence + optional `\n` + tag `\n` .. `\n` + or + `c` + `f` + sequence + optional + sequence + optional `\n` + tag `\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n` + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + optional + sequence + optional `\n` + tag `\n` .. `\n` + or + `c` + `f` + sequence + ` to=get_time<|channel|>analysis <|constrain|>xml<|message|>` + sequence + optional + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + star + sequence + `<|end|><|start|>assistant` + or + sequence + ` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>` + or + sequence + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + optional + sequence + optional `\n` + tag `\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n` + optional + sequence + optional `\n` + tag `\n` .. `\n` + or + `c` + `f` + sequence + optional + sequence + optional `\n` + tag `\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n` + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + optional + sequence + optional `\n` + tag `\n` .. `\n` + or + `c` + `f` + sequence + ` to=get_time<|channel|>analysis <|constrain|>xml<|message|>` + sequence + optional + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + optional `<|call|>` + "#]].assert_eq( + &grammar(HfPosition::ResponseStart, ToolChoice::required(), true).unwrap(), + ); +} + +#[test] +fn auto_with_a_strict_tool_also_allows_an_answer() { + expect_test::expect![[r#" + sequence + tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant` + sequence + /[^<]/ + text + or + sequence + or + sequence + ` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>` + or + sequence + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + optional + sequence + optional `\n` + tag `\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n` + optional + sequence + optional `\n` + tag `\n` .. `\n` + or + `c` + `f` + sequence + optional + sequence + optional `\n` + tag `\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n` + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + optional + sequence + optional `\n` + tag `\n` .. `\n` + or + `c` + `f` + sequence + ` to=get_time<|channel|>analysis <|constrain|>xml<|message|>` + sequence + optional + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + sequence + sequence + ` to=` + /(?:[^\sanalysis <|constrain|>xml<|message|>` + sequence + star + sequence + optional `\n` + sequence + `\n` + or + text excluding [`<|end|>`, `<|call|>`, ``] + json(number) excluding [`<|end|>`, `<|call|>`] + json(boolean) excluding [`<|end|>`, `<|call|>`] + json(null) excluding [`<|end|>`, `<|call|>`] + json({ ... }) excluding [`<|end|>`, `<|call|>`] + json(any[]) excluding [`<|end|>`, `<|call|>`] + `\n` + star + sequence + `<|end|><|start|>assistant` + or + sequence + ` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>` + or + sequence + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + optional + sequence + optional `\n` + tag `\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n` + optional + sequence + optional `\n` + tag `\n` .. `\n` + or + `c` + `f` + sequence + optional + sequence + optional `\n` + tag `\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n` + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + optional + sequence + optional `\n` + tag `\n` .. `\n` + or + `c` + `f` + sequence + ` to=get_time<|channel|>analysis <|constrain|>xml<|message|>` + sequence + optional + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + sequence + sequence + ` to=` + /(?:[^\sanalysis <|constrain|>xml<|message|>` + sequence + star + sequence + optional `\n` + sequence + `\n` + or + text excluding [`<|end|>`, `<|call|>`, ``] + json(number) excluding [`<|end|>`, `<|call|>`] + json(boolean) excluding [`<|end|>`, `<|call|>`] + json(null) excluding [`<|end|>`, `<|call|>`] + json({ ... }) excluding [`<|end|>`, `<|call|>`] + json(any[]) excluding [`<|end|>`, `<|call|>`] + `\n` + optional `<|call|>` + sequence + `<|channel|>final<|message|>` + text excluding [`<|end|>`, `<|return|>`] + optional `<|return|>` + "#]] + .assert_eq(&grammar(HfPosition::ResponseStart, ToolChoice::auto(), true).unwrap()); +} + +#[test] +fn named_choice_calls_that_tool_once_after_prompted_reasoning() { + expect_test::expect![[r#" + sequence + tag `` text `<|end|><|start|>assistant` + sequence + sequence + ` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>` + or + sequence + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + optional + sequence + optional `\n` + tag `\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n` + optional + sequence + optional `\n` + tag `\n` .. `\n` + or + `c` + `f` + sequence + optional + sequence + optional `\n` + tag `\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n` + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + optional + sequence + optional `\n` + tag `\n` .. `\n` + or + `c` + `f` + optional `<|call|>` + "#]].assert_eq( + &grammar(HfPosition::Reasoning, ToolChoice::function("get_weather"), true).unwrap(), + ); +} + +#[test] +fn no_grammar_without_a_constraining_request_or_known_position() { + let non_strict = vec![tools().pop().unwrap()]; + for tools in [&[][..], &non_strict] { + assert!( + grammar_with( + HfPosition::ResponseStart, + tools, + ToolChoice::auto(), + true, + false, + false + ) + .is_none() + ); + } + assert!( + grammar_with( + HfPosition::Other, + &tools(), + ToolChoice::required(), + true, + true, + false + ) + .is_none() + ); +} + +#[test] +fn none_with_tools_only_answers() { + expect_test::expect![[r#" + sequence + tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant` + sequence + /[^<]/ + text + sequence + `<|channel|>final<|message|>` + text excluding [`<|end|>`, `<|return|>`] + optional `<|return|>` + "#]] + .assert_eq(&grammar(HfPosition::ResponseStart, ToolChoice::none(), true).unwrap()); +} + +#[test] +fn always_constrain_answers_without_tools() { + expect_test::expect![[r#" + sequence + tag `` text `<|end|><|start|>assistant` + sequence + `<|channel|>final<|message|>` + text excluding [`<|end|>`, `<|return|>`] + optional `<|return|>` + "#]] + .assert_eq( + &grammar_with( + HfPosition::Reasoning, + &[], + ToolChoice::auto(), + true, + true, + false, + ) + .unwrap(), + ); +} + +#[test] +fn always_constrain_allows_non_strict_calls_or_an_answer() { + let non_strict = vec![tools().pop().unwrap()]; + expect_test::expect![[r#" + sequence + tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant` + sequence + /[^<]/ + text + or + sequence + or + sequence + ` to=get_time<|channel|>analysis <|constrain|>xml<|message|>` + sequence + optional + sequence + optional `\n` + tag `\n` text excluding [`<|end|>`, `<|call|>`, ``] `\n` + sequence + sequence + ` to=` + /(?:[^\sanalysis <|constrain|>xml<|message|>` + sequence + star + sequence + optional `\n` + sequence + `\n` + or + text excluding [`<|end|>`, `<|call|>`, ``] + json(number) excluding [`<|end|>`, `<|call|>`] + json(boolean) excluding [`<|end|>`, `<|call|>`] + json(null) excluding [`<|end|>`, `<|call|>`] + json({ ... }) excluding [`<|end|>`, `<|call|>`] + json(any[]) excluding [`<|end|>`, `<|call|>`] + `\n` + optional `<|call|>` + sequence + `<|channel|>final<|message|>` + text excluding [`<|end|>`, `<|return|>`] + optional `<|return|>` + "#]] + .assert_eq( + &grammar_with( + HfPosition::ResponseStart, + &non_strict, + ToolChoice::auto(), + false, + true, + false, + ) + .unwrap(), + ); +} + +#[test] +fn always_constrain_finishes_an_open_answer() { + expect_test::expect![[r#" + sequence + text excluding [`<|end|>`, `<|return|>`] + optional `<|return|>` + "#]] + .assert_eq( + &grammar_with( + HfPosition::Content, + &tools(), + ToolChoice::auto(), + true, + true, + false, + ) + .unwrap(), + ); + assert!( + grammar_with( + HfPosition::Content, + &tools(), + ToolChoice::auto(), + true, + false, + false + ) + .is_none() + ); + assert!( + grammar_with( + HfPosition::Content, + &tools(), + ToolChoice::required(), + true, + true, + false + ) + .is_none() + ); +} + +#[test] +fn answer_holds_the_requested_format() { + let json = || Format::json_schema(json!({"type": "object"})); + let answer = constrained_grammar(HfPosition::ResponseStart, ToolChoice::none(), json(), false); + expect_test::expect![[r#" + sequence + tag `<|channel|>analysis<|message|>` .. `<|end|><|start|>assistant` + sequence + /[^<]/ + text + sequence + `<|channel|>final<|message|>` + json(object) + optional `<|return|>` + "#]] + .assert_eq(&answer.unwrap()); + + // Under `auto`, the answer that may replace calls holds it too. + let auto = + constrained_grammar(HfPosition::ResponseStart, ToolChoice::auto(), json(), false).unwrap(); + assert!(auto.ends_with( + "`<|channel|>final<|message|>`\n json(object)\n optional `<|return|>`\n" + )); + + // A forced call leaves no answer to constrain. + assert_eq!( + constrained_grammar( + HfPosition::ResponseStart, + ToolChoice::required(), + json(), + false + ), + grammar(HfPosition::ResponseStart, ToolChoice::required(), true) + ); +} + +#[test] +fn request_constraint_applies_alone_in_an_open_answer() { + let json = Format::json_schema(json!({"type": "object"})); + assert!(constrained_grammar(HfPosition::Content, ToolChoice::auto(), json, true).is_none()); +} + +/// Parse `output` after the prompt under `tag_pattern`, returning the calls as +/// `(name, arguments)`. +fn calls(tag_pattern: &str, output: &str) -> Vec<(String, Value)> { + let mut parser = parser_with(tag_pattern, PROMPT); + let mut events = UnifiedParserOutput::default(); + parser.parse_into(DecodedText::unattributed(output), &mut events).unwrap(); + events.append(parser.finish().unwrap()); + let mut calls: Vec<(String, String)> = Vec::new(); + for event in events.events { + if let UnifiedParserEvent::ToolCall(delta) = event { + if let Some(name) = delta.name { + calls.push((name, String::new())); + } + calls.last_mut().unwrap().1.push_str(&delta.arguments); + } + } + calls + .into_iter() + .map(|(name, arguments)| (name, serde_json::from_str(&arguments).unwrap())) + .collect() +} + +/// Output in the grammar's canonical spelling parses back to the values the +/// grammar admitted. +#[test] +fn parser_reads_back_the_grammar_spelling() { + let parameter = |key: &str, value: &str| { + format!("\n{value}\n") + }; + let call = |name: &str, parameters: &[String]| { + format!( + " to={name}<|channel|>analysis <|constrain|>xml<|message|>{}", + parameters.join("\n") + ) + }; + let output = format!( + "<|channel|>analysis<|message|>Two tools.<|end|><|start|>assistant{}<|end|><|start|>assistant{}", + call( + "get_weather", + &[ + parameter("location", "Paris"), + parameter("days", "2"), + parameter("unit", "c") + ] + ), + call("get_time", &[parameter("zone", "multi\nline")]), + ); + + for tag_pattern in [TAG_PATTERN, LENIENT_TAG_PATTERN] { + assert_eq!( + calls(tag_pattern, &output), + vec![ + ( + "get_weather".to_string(), + json!({"location": "Paris", "days": 2, "unit": "c"}) + ), + ("get_time".to_string(), json!({"zone": "multi\nline"})), + ] + ); + } +} + +#[test] +fn values_end_before_the_next_parameter_where_the_template_looks_ahead() { + let output = " to=get_weather<|channel|>analysis <|constrain|>xml<|message|>\ + \n2\n\ + \nParis\n"; + + assert!(parser_with(LENIENT_TAG_PATTERN, PROMPT).lenient); + assert_eq!( + calls(LENIENT_TAG_PATTERN, output), + vec![( + "get_weather".to_string(), + json!({"days": 2, "location": "Paris"}) + )] + ); + assert!(!parser(PROMPT).lenient); +} + +#[test] +fn reasoning_ends_at_every_closer_of_the_template() { + let closers = [ + "<|end|><|start|>assistant", + "<|constrain|><|start|>assistant", + "<|start|>assistant", + ]; + let tools = tools(); + let tokenizer = Arc::new(TestTokenizer::new()); + let mut parser = BeamUnifiedParser::new( + template_with(TAG_PATTERN, json!(closers)), + &tools, + tokenizer.clone(), + ); + parser.initialize(&tokenizer.encode(PROMPT, false).unwrap()).unwrap(); + let grammar = parser + .build_output_grammar(&OutputGrammarContext { + tools: &tools, + tool_choice: &ToolChoice::required(), + tool_strict_level: ToolStrictLevel::Auto, + parallel_tool_calls: false, + answer: None, + }) + .unwrap() + .unwrap(); + + assert_eq!( + outline(&grammar.format).lines().nth(1).unwrap(), + " tag `<|channel|>analysis<|message|>` .. `<|constrain|><|start|>assistant` | \ + `<|end|><|start|>assistant` | `<|start|>assistant`" + ); +} + +#[test] +fn lenient_arguments_come_in_declared_or_sorted_order() { + expect_test::expect![[r#" + sequence + tag `` text `<|end|><|start|>assistant` + sequence + sequence + ` to=get_weather<|channel|>analysis <|constrain|>xml<|message|>` + sequence + optional `\n` + `\n` text excluding [`<|end|>`, `<|call|>`, ``, `\n\n` text excluding [`<|end|>`, `<|call|>`, ``, `\n\n\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n` + sequence + optional tag `days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n\n` .. `\n` + or + `c` + `f` + sequence + optional tag `days">\n` json(integer) excluding [`<|end|>`, `<|call|>`] `\n\n` text excluding [`<|end|>`, `<|call|>`, ``, `\n\n` text excluding [`<|end|>`, `<|call|>`, ``, `\n\n\n` .. `\n` + or + `c` + `f` + optional `<|call|>` + "#]].assert_eq( + &grammar_with( + HfPosition::Reasoning, + &tools(), + ToolChoice::function("get_weather"), + true, + false, + true, + ) + .unwrap(), + ); +} + +#[test] +fn other_names_match_every_name_but_the_given_ones() { + let regex = regex_automata::meta::Regex::new(&format!( + "^(?:{})$", + other_names(&["get", "get_time", "a.b"]) + )) + .unwrap(); + + for name in [ + "g", + "ge", + "gets", + "get_", + "get_t", + "get_times", + "x", + "a", + "a.", + "axb", + "a.bc", + ] { + assert!(regex.is_match(name), "{name}"); + } + for name in ["get", "get_time", "a.b", "", "get time", "get<"] { + assert!(!regex.is_match(name), "{name}"); + } +} diff --git a/rust/src/parser/src/unified/hf/content.rs b/rust/src/parser/src/unified/hf/content.rs index d62ba9305e..598c6597e1 100644 --- a/rust/src/parser/src/unified/hf/content.rs +++ b/rust/src/parser/src/unified/hf/content.rs @@ -6,8 +6,12 @@ //! Port of `content_parsers.py`. Each parser takes a chunk of captured text and //! parses it into a single key in the output message dictionary. +use std::borrow::Cow; + use regex_automata::meta::Regex; +use regex_automata::util::iter::Searcher; use regex_automata::util::syntax; +use regex_automata::{Input, Match}; use serde::Deserialize; use serde_json::{Map, Value}; use thiserror_ext::AsReport as _; @@ -22,6 +26,10 @@ use crate::unified::{self, parsing_failed}; const LAX_OPEN: char = '\u{1}'; const LAX_CLOSE: char = '\u{2}'; +/// Name of the group that stands for the lookahead ending an `xml-inline` tag +/// pattern. +const AHEAD_GROUP: &str = "lookahead"; + /// A configured content parser (`content` plus `content_args` of one field). #[derive(Debug, Clone)] pub(super) enum ContentParser { @@ -66,6 +74,9 @@ pub(super) struct XmlInlineArgs { key_group: usize, /// Index of the `value` group, if the pattern declares one. value_group: Option, + /// Index of the group standing for the lookahead that ends the pattern, + /// and the lookahead's own pattern, if any. + lookahead: Option<(usize, String)>, value_parser: Option>, /// Collect duplicate keys into a list. merge_duplicates: bool, @@ -143,7 +154,8 @@ impl ContentParser { } ContentKind::XmlInline => { let spec: XmlInlineSpec = deserialize(&args, &context)?; - let tag_pattern = compile_regex(scope, &spec.tag_pattern)?; + let (pattern, lookahead) = final_lookahead(&spec.tag_pattern); + let tag_pattern = compile_regex(scope, &pattern)?; let group_info = tag_pattern.group_info(); let Some(key_group) = group_info.to_index(Default::default(), "key") else { return Err(invalid!( @@ -152,10 +164,14 @@ impl ContentParser { )); }; let value_group = group_info.to_index(Default::default(), "value"); + let lookahead = group_info + .to_index(Default::default(), AHEAD_GROUP) + .zip(lookahead.map(str::to_string)); Self::XmlInline(XmlInlineArgs { tag_pattern, key_group, value_group, + lookahead, value_parser: ValueParserSpec::compile(scope, spec.value_parser)?, merge_duplicates: spec.merge_duplicates, }) @@ -193,6 +209,32 @@ impl ContentParser { Self::KvLines(args) => args.parse(text), } } + + /// The pattern of the lookahead that ends an `xml-inline` tag pattern, if + /// any. + pub fn lookahead(&self) -> Option<&str> { + match self { + Self::XmlInline(args) => args.lookahead.as_ref().map(|(_, ahead)| ahead.as_str()), + _ => None, + } + } +} + +/// `pattern` with a lookahead that ends it matched as the group +/// [`AHEAD_GROUP`] instead, as the Rust regex engine has no lookahead, and +/// the lookahead's own pattern. The engine rejects any other lookahead. +fn final_lookahead(pattern: &str) -> (Cow<'_, str>, Option<&str>) { + if let Some((head, rest)) = pattern.split_once("(?=") { + // Only closing parentheses may follow a lookahead that ends the pattern. + let ahead = rest.trim_end_matches(')'); + if ahead.len() < rest.len() && !ahead.contains(['(', ')']) { + return ( + format!("{head}(?P<{AHEAD_GROUP}>{rest}").into(), + Some(ahead), + ); + } + } + (pattern.into(), None) } impl ValueParserSpec { @@ -258,7 +300,21 @@ impl XmlInlineArgs { /// Parse shallow XML-ish tags into a dict. fn parse(&self, text: &str) -> unified::Result { let mut out = Map::new(); - for captures in self.tag_pattern.captures_iter(text) { + let mut captures = self.tag_pattern.create_captures(); + let mut tags = Searcher::new(Input::new(text)); + while tags + .advance(|input| { + self.tag_pattern.search_captures(input, &mut captures); + // The lookahead's text is not part of the tag. + let ahead = + self.lookahead.as_ref().and_then(|(group, _)| captures.get_group(*group)); + Ok(captures.get_match().map(|tag| match ahead { + Some(ahead) => Match::new(tag.pattern(), tag.start()..ahead.start), + None => tag, + })) + }) + .is_some() + { let key = captures.get_group(self.key_group).map(|span| &text[span.range()]).ok_or_else( || parsing_failed!("xml-inline: named group 'key' did not participate"), @@ -490,6 +546,25 @@ mod tests { ); } + #[test] + fn xml_inline_tags_may_end_where_the_pattern_looks_ahead() { + let xml = parser( + "xml-inline", + json!({ + "tag_pattern": r#"(?P.*?)\n(?:|(?=1\nx y\n") + .unwrap(); + assert_eq!(value, json!({"a": "1", "b": "x y"})); + assert_eq!(xml.lookahead(), Some("\w+)(?=:)(?P.*)"}); + assert!(ContentParser::new("test", kind("xml-inline"), args.as_object().unwrap()).is_err()); + } + #[test] fn kv_lines_split_and_strip() { let kv = parser("kv-lines", json!({"value_parser": {"name": "int"}})); diff --git a/rust/src/parser/src/unified/hf/mod.rs b/rust/src/parser/src/unified/hf/mod.rs index 568a8371d0..9d9bd3bc06 100644 --- a/rust/src/parser/src/unified/hf/mod.rs +++ b/rust/src/parser/src/unified/hf/mod.rs @@ -25,7 +25,7 @@ mod transform; #[cfg(test)] mod tests; -pub use parser::HfUnifiedParser; +pub use parser::{HfPosition, HfUnifiedParser}; pub use template::ResponseTemplate; use thiserror::Error; use thiserror_ext::Macro; diff --git a/rust/src/parser/src/unified/hf/parser.rs b/rust/src/parser/src/unified/hf/parser.rs index 5d5065f109..febdd47175 100644 --- a/rust/src/parser/src/unified/hf/parser.rs +++ b/rust/src/parser/src/unified/hf/parser.rs @@ -59,6 +59,20 @@ pub struct HfUnifiedParser { emitted_tool_count: usize, } +/// Where the parser is in the response; see [`HfUnifiedParser::position`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HfPosition { + /// Nothing parsed since the start anchor, or since the start of the stream + /// when there is none. + ResponseStart, + /// Inside an open reasoning field. + Reasoning, + /// Inside an open content field. + Content, + /// Anywhere else. + Other, +} + /// Parser mode: the region currently receiving text. enum HfMode { /// No implicit field is declared: text outside regions is discarded. @@ -156,6 +170,36 @@ impl HfUnifiedParser { parser } + /// Where the parser is in the response. Between + /// [`UnifiedParser::initialize`] and the first generated text, this is where + /// the prompt leaves the response. + /// + /// Text outside every region, which a template without an implicit field + /// discards, leaves no state behind and so does not move the parser from + /// [`HfPosition::ResponseStart`]. + pub fn position(&self) -> HfPosition { + if !self.buffer.text.is_empty() { + return HfPosition::Other; + } + match &self.mode { + HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) if occurrence.opened => { + match &self.template.regions[occurrence.region].kind { + RegionKind::Text(TextRegion { + role: TextRole::Reasoning, + .. + }) => HfPosition::Reasoning, + RegionKind::Text(TextRegion { + role: TextRole::Content, + .. + }) => HfPosition::Content, + _ => HfPosition::Other, + } + } + _ if self.closed.contains(&true) || self.emitted_tool_count > 0 => HfPosition::Other, + _ => HfPosition::ResponseStart, + } + } + /// Clear all per-stream state. fn reset_state(&mut self) { self.buffer.clear(); diff --git a/rust/src/parser/src/unified/hf/template.rs b/rust/src/parser/src/unified/hf/template.rs index 4889f6f9b5..29c096ea32 100644 --- a/rust/src/parser/src/unified/hf/template.rs +++ b/rust/src/parser/src/unified/hf/template.rs @@ -191,6 +191,31 @@ impl ResponseTemplate { }) } + /// The pattern of the lookahead that ends the `xml-inline` tag pattern of + /// tool-call arguments, such as for the opener of the next argument. + pub fn argument_lookahead(&self) -> Option<&str> { + self.regions.iter().find_map(|region| match ®ion.kind { + RegionKind::ToolCalls(calls) => calls.content.lookahead(), + _ => None, + }) + } + + /// The literal closers of the reasoning field, longest first, or none if it + /// has no literal close. + pub fn reasoning_closers(&self) -> &[String] { + self.regions + .iter() + .find_map(|region| match (®ion.kind, ®ion.close) { + (RegionKind::Text(text), Some(Boundary::Literals(closers))) + if text.role == TextRole::Reasoning => + { + Some(closers.as_slice()) + } + _ => None, + }) + .unwrap_or_default() + } + /// Return the start and end offsets of the last start-anchor match in /// `prompt`. pub(super) fn last_start_anchor(&self, prompt: &str) -> Option<(usize, usize)> { diff --git a/rust/src/parser/src/unified/mod.rs b/rust/src/parser/src/unified/mod.rs index 74bef8e1f4..585f28cfde 100644 --- a/rust/src/parser/src/unified/mod.rs +++ b/rust/src/parser/src/unified/mod.rs @@ -3,6 +3,7 @@ //! Unified parser interface for reasoning and tool-call deltas. +mod beam; mod combined; mod gemma4; mod hf; @@ -10,9 +11,10 @@ mod hy; mod inkling; mod kimi_k3; +pub use beam::BeamUnifiedParser; pub use combined::CombinedParser; pub use gemma4::Gemma4UnifiedParser; -pub use hf::{HfTemplateError, HfUnifiedParser, ResponseTemplate}; +pub use hf::{HfPosition, HfTemplateError, HfUnifiedParser, ResponseTemplate}; pub use hy::{HyV3UnifiedParser, HyV4UnifiedParser}; pub use inkling::InklingUnifiedParser; pub use kimi_k3::KimiK3UnifiedParser; diff --git a/tests/parser/test_beam_parser.py b/tests/parser/test_beam_parser.py new file mode 100644 index 0000000000..3673b36090 --- /dev/null +++ b/tests/parser/test_beam_parser.py @@ -0,0 +1,747 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json +from copy import deepcopy +from types import SimpleNamespace +from typing import Any + +import pytest +from openai.types.responses import FunctionTool, ToolChoiceFunction +from xgrammar import Grammar +from xgrammar.testing import _is_grammar_accept_string + +from vllm.config.reasoning import ReasoningConfig +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ChatCompletionToolsParam, +) +from vllm.entrypoints.openai.responses.protocol import ( + ResponsesRequest, + ResponsesResponse, +) +from vllm.exceptions import VLLMValidationError +from vllm.parser.beam import BeamParser, BeamReasoningParser, BeamToolParser +from vllm.parser.parser_manager import ParserManager +from vllm.parser.response_template import ResponseTemplateParser +from vllm.reasoning import ReasoningParserManager +from vllm.tool_parsers import ToolParserManager + +# The `response_template` of the Beam checkpoints. +BEAM_RESPONSE_TEMPLATE: dict[str, Any] = { + "defaults": {"role": "assistant"}, + "start_anchor_pattern": r"<\|start\|>assistant", + "fields": { + "thinking": { + "open": "<|channel|>analysis<|message|>", + "close": "<|end|><|start|>assistant", + "content": "text", + }, + "content": { + "open": "<|channel|>final<|message|>", + "close": ["<|return|>", "<|end|>", "<|call|>"], + "content": "text", + }, + "tool_calls": { + "open_pattern": ( + r" to=(?P[^\s<]+)" + r"<\|channel\|>analysis <\|constrain\|>xml<\|message\|>" + ), + "close": ["<|end|><|start|>assistant", "<|call|>"], + "repeats": True, + "content": "xml-inline", + "content_args": { + "tag_pattern": ( + r'' + r'\n(?P.*?)\n(?:|(?=": 200, + "<|channel|>": 201, + "<|message|>": 202, + "<|end|>": 203, + "<|call|>": 204, + "<|return|>": 205, + "<|constrain|>": 206, +} +TEXT_TOKENS = {100: "assistant", 101: "analysis", 102: "final", 103: "plan"} +PROMPT_TOKEN_IDS = [200, 100] +THOUGHT = "<|channel|>analysis<|message|>plan<|end|><|start|>assistant" +FINAL = "<|channel|>final<|message|>" +NEXT_MESSAGE = "<|end|><|start|>assistant" + + +def _tool(name, properties, *, strict=None): + return ChatCompletionToolsParam( + type="function", + function={ + "name": name, + "parameters": {"type": "object", "properties": properties}, + "strict": strict, + }, + ) + + +TOOLS = [ + _tool("get_weather", {"city": {"type": "string"}}), + _tool("set_count", {"count": {"type": "integer"}}), +] +STRICT_TOOLS = [ + _tool("get_weather", {"city": {"type": "string"}}), + _tool("set_count", {"count": {"type": "integer"}}, strict=True), +] + + +class BeamTokenizer: + def __init__(self, response_template=BEAM_RESPONSE_TEMPLATE): + self.response_template = response_template + self.init_kwargs = {} + self.name_or_path = "" + self.all_special_tokens = list(VOCAB) + self.all_special_ids = list(VOCAB.values()) + + def decode(self, token_ids, skip_special_tokens=False, **_kwargs): + inverse_vocab = {token_id: token for token, token_id in VOCAB.items()} + return "".join( + TEXT_TOKENS.get(token_id, inverse_vocab.get(token_id, "")) + for token_id in token_ids + ) + + def get_vocab(self): + return VOCAB + + def encode(self, text, add_special_tokens=False): + del add_special_tokens + return list(text.encode()) + + +def request(**overrides): + values = dict( + tool_choice="auto", + include_reasoning=True, + tools=TOOLS, + parallel_tool_calls=True, + ) + values.update(overrides) + return SimpleNamespace(**values) + + +def parameter(name, value): + return f'\n{value}\n' + + +def call(name, body, *, terminator="<|call|>"): + return ( + f" to={name}<|channel|>analysis <|constrain|>xml<|message|>{body}{terminator}" + ) + + +def make_parser(prompt_token_ids=PROMPT_TOKEN_IDS, tools=TOOLS, **kwargs): + parser_cls = ParserManager.get_parser( + tool_parser_name="beam", + reasoning_parser_name="beam", + enable_auto_tools=True, + tokenizer=BeamTokenizer(), + ) + assert parser_cls is not None + parser = parser_cls(BeamTokenizer(), tools, **kwargs) + parser.set_prompt_token_ids(prompt_token_ids) + return parser + + +def parse(text, prompt_token_ids=PROMPT_TOKEN_IDS): + reasoning, content, calls = make_parser(prompt_token_ids).parse( + text, request(), enable_auto_tools=True + ) + return reasoning, content, normalize_calls(calls) + + +def normalize_calls(calls): + return [(call.name, json.loads(call.arguments)) for call in calls or []] + + +def test_beam_parsers_are_registered(): + assert ToolParserManager.get_tool_parser("beam") is BeamToolParser + assert ReasoningParserManager.get_reasoning_parser("beam") is BeamReasoningParser + + +def test_parser_manager_selects_beam_parser(): + parser_cls = ParserManager.get_parser( + tool_parser_name="beam", + reasoning_parser_name="beam", + enable_auto_tools=True, + tokenizer=BeamTokenizer(), + ) + + assert parser_cls is not None + assert issubclass(parser_cls, BeamParser) + assert parser_cls.reasoning_parser_cls is BeamReasoningParser + assert parser_cls.tool_parser_cls is BeamToolParser + assert parser_cls._enable_auto_tools + + +@pytest.mark.parametrize("other", ["hermes", "hf"]) +def test_beam_parser_rejects_other_parsers(other): + with pytest.raises(TypeError, match="cannot be combined"): + ParserManager.get_parser( + tool_parser_name=other, + reasoning_parser_name="beam", + enable_auto_tools=True, + tokenizer=BeamTokenizer(), + ) + + +def test_beam_parser_requires_checkpoint_metadata_at_startup(): + with pytest.raises( + TypeError, match="--reasoning-parser beam requires `response_template`" + ): + ParserManager.get_parser( + reasoning_parser_name="beam", + tokenizer=BeamTokenizer(response_template=None), + ) + + template = deepcopy(BEAM_RESPONSE_TEMPLATE) + del template["fields"]["tool_calls"] + with pytest.raises(TypeError, match="--tool-call-parser beam requires"): + ParserManager.get_parser( + tool_parser_name="beam", + enable_auto_tools=True, + tokenizer=BeamTokenizer(template), + ) + + +def test_reasoning_and_final_answer(): + assert parse(THOUGHT + FINAL + "Sunny.<|return|>") == ("plan", "Sunny.", []) + + +def test_prefilled_reasoning_continues_from_prompt(): + prompt = PROMPT_TOKEN_IDS + [201, 101, 202, 103] + text = " more.<|end|><|start|>assistant" + FINAL + "Done.<|return|>" + + assert parse(text, prompt) == (" more.", "Done.", []) + + +def test_parallel_calls_are_coerced_to_their_schemas(): + text = ( + THOUGHT + + call( + "get_weather", + parameter("city", "Paris"), + terminator="<|end|><|start|>assistant", + ) + + call("set_count", parameter("count", "7")) + ) + + assert parse(text) == ( + "plan", + None, + [("get_weather", {"city": "Paris"}), ("set_count", {"count": 7})], + ) + + +def test_call_stop_token_removed_from_the_text_still_ends_the_call(): + text = THOUGHT + call("get_weather", parameter("city", "Paris"), terminator="") + + assert parse(text) == ("plan", None, [("get_weather", {"city": "Paris"})]) + + +@pytest.mark.parametrize("end", ["<|end|>", "<|end|><|start|>"]) +def test_output_cut_before_the_next_message_start_ends_its_region(end): + """Output that stops after `<|end|>`, such as at `max_tokens`, has ended + its reasoning or call and lacks only the next message's start.""" + weather = call("get_weather", parameter("city", "Paris"), terminator=end) + + assert parse("<|channel|>analysis<|message|>plan" + end) == ("plan", None, []) + assert parse(THOUGHT + weather) == ( + "plan", + None, + [("get_weather", {"city": "Paris"})], + ) + + +THOUGHT_IDS = [201, 101, 202, 103, 203, 200, 100] +FINAL_IDS = [201, 102, 202] + + +@pytest.mark.parametrize( + ("prompt_token_ids", "token_ids", "count"), + [ + (PROMPT_TOKEN_IDS, [*THOUGHT_IDS, *FINAL_IDS, 205], 1), + (PROMPT_TOKEN_IDS + [201, 101, 202], [103, 103, 203, 200, 100, *FINAL_IDS], 2), + (PROMPT_TOKEN_IDS, [201, 101, 202, 103, 103, 103], 3), + ], + ids=["reasoning_then_answer", "prefilled_reasoning", "cut_off"], +) +def test_reasoning_tokens_are_counted(prompt_token_ids, token_ids, count): + parser = make_parser(prompt_token_ids) + + assert parser.count_reasoning_tokens(token_ids) == count + + +def test_reasoning_tokens_are_counted_as_the_generation_grows(): + token_ids = [*THOUGHT_IDS, *FINAL_IDS, 205] + parser = make_parser() + + assert [ + parser.count_reasoning_tokens(token_ids[:end]) + for end in range(1, len(token_ids) + 1) + ] == [ + make_parser().count_reasoning_tokens(token_ids[:end]) + for end in range(1, len(token_ids) + 1) + ] + assert parser.count_reasoning_tokens(FINAL_IDS) == 0 + + +def test_call_stop_token_ends_the_answer(): + assert parse(FINAL + "done<|call|>") == (None, "done", []) + + +@pytest.mark.parametrize("chunk_size", [1, 7, 10_000]) +def test_streaming_matches_non_streaming(chunk_size): + text = ( + THOUGHT + + call( + "get_weather", + parameter("city", "Paris"), + terminator="<|end|><|start|>assistant", + ) + + call("set_count", parameter("count", "7")) + ) + parser = make_parser() + reasoning, calls = [], [] + for start in range(0, len(text), chunk_size): + end = min(start + chunk_size, len(text)) + delta = parser.parse_delta( + text[start:end], [], request(), finished=end == len(text) + ) + if delta is not None: + reasoning.append(delta.reasoning or "") + calls.extend(delta.tool_calls) + + assert "".join(reasoning) == "plan" + assert [ + (call.function.name, json.loads(call.function.arguments)) for call in calls + ] == parse(text)[2] + + +WEATHER_CALL = call("get_weather", parameter("city", "Paris")) +WEATHER = [("get_weather", {"city": "Paris"})] + + +def chat_request(tools=STRICT_TOOLS, **body): + return ChatCompletionRequest( + model="beam", + messages=[{"role": "user", "content": "Weather?"}], + tools=tools and [tool.model_dump() for tool in tools], + **body, + ) + + +def named(name): + return {"type": "function", "function": {"name": name}} + + +def adjust(req, prompt_token_ids=PROMPT_TOKEN_IDS, **kwargs): + make_parser(prompt_token_ids, req.tools, **kwargs).adjust_request(req) + return req + + +def accepts(req, text): + grammar = Grammar.from_structural_tag( + req.extract_structured_outputs().structural_tag + ) + return _is_grammar_accept_string(grammar, text) + + +@pytest.mark.parametrize( + ("tool_choice", "text", "calls"), + [ + ("required", THOUGHT + WEATHER_CALL, WEATHER), + ("required", WEATHER_CALL, None), + ( + "required", + THOUGHT + + call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE) + + call("set_count", parameter("count", "7")), + [*WEATHER, ("set_count", {"count": 7})], + ), + ("required", THOUGHT + call("set_count", parameter("count", "x")), None), + ( + "required", + THOUGHT + call("get_weather", parameter("city", "a<|call|>b")), + None, + ), + ("required", THOUGHT + FINAL + "Paris<|return|>", None), + (named("set_count"), THOUGHT + WEATHER_CALL, None), + (named("get_weather"), THOUGHT + WEATHER_CALL, WEATHER), + ( + named("get_weather"), + THOUGHT + + call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE) + + WEATHER_CALL, + None, + ), + ], +) +def test_forced_tool_calls_follow_the_template_grammar(tool_choice, text, calls): + """From the message start, the grammar opens with the reasoning, and + accepts only calls the parser reads back.""" + req = adjust(chat_request(tool_choice=tool_choice)) + + assert req._grammar_from_parser + assert accepts(req, text) is (calls is not None) + if calls is not None: + assert parse(text)[2] == calls + + +def test_reasoning_may_be_skipped_when_thinking_is_disabled(): + req = adjust( + chat_request(tool_choice="required"), + chat_template_kwargs={"enable_thinking": False}, + ) + + assert accepts(req, WEATHER_CALL) + assert accepts(req, THOUGHT + WEATHER_CALL) + + +def test_parallel_tool_calls_false_allows_a_single_call(): + req = adjust(chat_request(tool_choice="required", parallel_tool_calls=False)) + calls = call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE) + + assert accepts(req, THOUGHT + WEATHER_CALL) + assert not accepts(req, THOUGHT + calls + WEATHER_CALL) + + +@pytest.mark.parametrize("strict", [False, True]) +@pytest.mark.parametrize( + ("separator", "accepted"), [("", True), ("\n", True), ("\n\n", False), (" ", False)] +) +def test_parameters_are_separated_by_at_most_a_newline(strict, separator, accepted): + """A model that skips a parameter's opener must not write whitespace until + `max_tokens`.""" + write = _tool( + "write", + {"path": {"type": "string"}, "content": {"type": "string"}}, + strict=strict, + ) + req = adjust(chat_request([write], tool_choice="required")) + body = parameter("path", "a.py") + separator + parameter("content", "x") + + assert accepts(req, THOUGHT + call("write", body)) is accepted + assert accepts(req, THOUGHT + call("write", separator + body)) is accepted + + +def test_forced_calls_in_prefilled_reasoning_start_after_it(): + req = adjust( + chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 101, 202, 103] + ) + + assert not req._grammar_from_parser + assert accepts(req, WEATHER_CALL) + + +def test_forced_calls_cannot_follow_a_prefilled_answer(): + with pytest.raises(VLLMValidationError, match="required tool choice"): + adjust(chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 102, 202]) + + +def test_prefilled_answer_can_only_be_finished(): + req = adjust(chat_request(TOOLS), PROMPT_TOKEN_IDS + [201, 102, 202]) + + assert accepts(req, "Sunny.<|return|>") + assert not accepts(req, "Sunny." + NEXT_MESSAGE + WEATHER_CALL) + + +def test_forced_call_with_unencodable_tool_name_is_rejected(): + tools = [_tool("get weather", {"city": {"type": "string"}})] + + with pytest.raises(VLLMValidationError, match="tool name"): + adjust(chat_request(tools, tool_choice="required")) + + +def test_string_formats_and_patterns_are_held_in_raw_text(): + """Like XGrammar, a strict string keeps its format or pattern, unless a + match could end the value early.""" + contact = _tool( + "set_contact", + { + "email": {"type": "string", "format": "email"}, + "code": {"type": "string", "pattern": "^[A-Z]{3}$"}, + "n": {"type": "integer"}, + }, + strict=True, + ) + contact.function.parameters["required"] = ["email", "code", "n"] + req = adjust(chat_request([contact], tool_choice="required")) + + def contact_call(email, code, *rest): + body = parameter("email", email) + parameter("code", code) + "".join(rest) + return THOUGHT + call("set_contact", body) + + text = contact_call("a.b@x.co", "ABC", parameter("n", "1")) + calls = make_parser(tools=[contact]).parse(text, req, enable_auto_tools=True)[2] + assert accepts(req, text) + assert normalize_calls(calls) == [ + ("set_contact", {"email": "a.b@x.co", "code": "ABC", "n": 1}) + ] + assert not accepts(req, contact_call("ab", "ABC", parameter("n", "1"))) + assert not accepts(req, contact_call("a.b@x.co", "ABCD", parameter("n", "1"))) + assert not accepts(req, contact_call("a.b@x.co", "ABC")) + # Line breaks could end the value. + dotted = _tool( + "set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True + ) + with pytest.raises(VLLMValidationError, match="required tool choice"): + adjust(chat_request([dotted], tool_choice="required")) + + +@pytest.mark.parametrize( + ("text", "accepted"), + [ + (THOUGHT + WEATHER_CALL, True), + (THOUGHT + FINAL + "Sunny.<|return|>", True), + (THOUGHT + FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False), + (THOUGHT + call("set_count", parameter("count", "x")), False), + ], +) +def test_strict_tools_constrain_automatic_tool_choice(text, accepted): + assert accepts(adjust(chat_request()), text) is accepted + + +@pytest.mark.parametrize( + ("text", "accepted"), + [ + (THOUGHT + FINAL + "Sunny.<|return|>", True), + (FINAL + "Sunny.<|return|>", False), + (WEATHER_CALL, False), + (THOUGHT + WEATHER_CALL, False), + (FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False), + ], +) +def test_tool_choice_none_allows_only_an_answer(text, accepted): + req = adjust(chat_request(TOOLS, tool_choice="none")) + + assert req._grammar_from_parser + assert accepts(req, text) is accepted + + +@pytest.mark.parametrize(("tools", "tool_choice"), [(TOOLS, "auto"), (None, None)]) +def test_always_constrained_output_follows_the_template_framing(tools, tool_choice): + req = adjust(chat_request(tools, tool_choice=tool_choice)) + count_call = THOUGHT + call("set_count", parameter("count", "3")) + + assert accepts(req, THOUGHT + FINAL + "Sunny.<|return|>") + assert not accepts(req, FINAL + "Sunny.<|return|>") + assert not accepts(req, THOUGHT.replace("plan", "") + FINAL + "Sunny.<|return|>") + assert not accepts(req, THOUGHT + "Sunny.") + assert accepts(req, count_call) is (tool_choice == "auto") + + +@pytest.mark.parametrize("response_format", [None, {"type": "json_object"}]) +@pytest.mark.parametrize("strict", [None, True]) +@pytest.mark.parametrize("tool_choice", ["auto", "required"]) +def test_automatic_choice_lets_calls_name_undeclared_tools( + strict, tool_choice, response_format +): + """Unless a forced choice limits calls to the declared tools. The parser + passes such calls on, so the harness can tell the model the tool does not + exist.""" + bash = _tool("bash", {"command": {"type": "string"}}, strict=strict) + req = adjust( + chat_request([bash], tool_choice=tool_choice, response_format=response_format) + ) + undeclared = THOUGHT + call("read_file", parameter("path", "a.py")) + parser = make_parser(tools=[bash]) + + assert accepts(req, THOUGHT + call("bash", parameter("command", "ls"))) + assert accepts(req, undeclared) is (tool_choice == "auto") + calls = parser.parse(undeclared, req, enable_auto_tools=True)[2] + assert normalize_calls(calls) == [("read_file", {"path": "a.py"})] + + +@pytest.mark.parametrize( + ("name", "body", "accepted"), + [ + ("set_count", parameter("count", "1"), True), + ("set_count", parameter("count", "x"), False), + ("get_weather", parameter("town", "Paris"), False), + # The grammar cannot enforce `minProperties`. + ("find", parameter("x", "1") + parameter("x", "y"), True), + ("search_files", parameter("x", "1") + parameter("x", "y"), True), + ("set_coun", parameter("x", "1") + parameter("x", "y"), True), + ("set_counts", parameter("x", "1") + parameter("x", "y"), True), + ], +) +def test_automatic_calls_enforce_declared_arguments_beside_any_other_name( + name, body, accepted +): + find = _tool("find", {"q": {"type": "string"}}) + find.function.parameters["minProperties"] = 1 + req = adjust(chat_request([*TOOLS, find])) + + assert accepts(req, THOUGHT + call(name, body)) is accepted + + +def test_value_ends_before_an_opener_on_the_next_line(): + write = _tool("write", {"path": {"type": "string"}, "content": {"type": "string"}}) + req = adjust(chat_request([write], tool_choice="required")) + body = '\nprint(1)\n\n' + parameter("path", "a.py") + text = THOUGHT + call("write", body) + + assert accepts(req, text) + calls = make_parser(tools=[write]).parse(text, req, enable_auto_tools=True)[2] + assert normalize_calls(calls) == [ + ("write", {"content": "print(1)\n", "path": "a.py"}) + ] + + +def test_always_constrained_output_can_be_disabled(monkeypatch): + monkeypatch.setenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "0") + + assert adjust(chat_request(TOOLS)).structured_outputs is None + + +def test_strict_tools_the_template_cannot_hold_leave_automatic_choice_unconstrained(): + dotted = _tool( + "set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True + ) + + assert adjust(chat_request([dotted])).structured_outputs is None + + +def test_generic_parser_does_not_derive_the_template_grammar(): + req = chat_request(TOOLS, tool_choice="required") + parser = ResponseTemplateParser(BeamTokenizer(), req.tools, enable_auto_tools=True) + + with pytest.raises(VLLMValidationError, match="required tool choice"): + parser.adjust_request(req) + + +@pytest.mark.parametrize("tools", [None, TOOLS]) +def test_structured_output_is_framed_as_the_answer(tools): + schema = {"type": "object", "properties": {"a": {"type": "integer"}}} + req = adjust( + chat_request( + tools, + response_format={ + "type": "json_schema", + "json_schema": {"name": "answer", "schema": schema}, + }, + ) + ) + + assert req.response_format is None + assert accepts(req, THOUGHT + FINAL + '{"a": 1}<|return|>') + assert not accepts(req, THOUGHT + FINAL + "Sunny.<|return|>") + assert accepts(req, THOUGHT + WEATHER_CALL) is (tools is not None) + + +def test_responses_named_tool_choice_follows_the_template_grammar(): + req = adjust( + ResponsesRequest( + input="Weather?", + tools=[ + FunctionTool( + type="function", + name="get_weather", + parameters={ + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + strict=True, + ) + ], + tool_choice=ToolChoiceFunction(type="function", name="get_weather"), + text={"format": {"type": "json_object"}}, + ) + ) + + assert req._grammar_from_parser + assert req.text.format.type == "json_object" + assert accepts(req, THOUGHT + WEATHER_CALL) + assert not accepts(req, FINAL + "{}<|return|>") + + +@pytest.mark.parametrize("status", ["in_progress", "completed"]) +@pytest.mark.parametrize("inside_reasoning", [False, True]) +def test_responses_preserves_text_config_with_template_grammar( + status, inside_reasoning +): + text = { + "format": { + "type": "json_schema", + "name": "answer", + "strict": True, + "schema": { + "type": "object", + "properties": {"a": {"type": "integer"}}, + "required": ["a"], + "additionalProperties": False, + }, + }, + "verbosity": "low", + } + req = ResponsesRequest(input="Answer", text=text) + expected_text = req.text.model_dump(by_alias=True) + prompt = PROMPT_TOKEN_IDS + ([201, 101, 202] if inside_reasoning else []) + adjust(req, prompt_token_ids=prompt) + params = req.to_sampling_params(default_max_tokens=128) + grammar = Grammar.from_structural_tag(params.structured_outputs.structural_tag) + prefix = "" if inside_reasoning else THOUGHT + assert _is_grammar_accept_string(grammar, prefix + FINAL + '{"a": 1}<|return|>') + assert not _is_grammar_accept_string(grammar, prefix + FINAL + "Sunny.<|return|>") + response = ResponsesResponse.from_request(req, params, "beam", 0, [], status) + assert response.model_dump(by_alias=True)["text"] == expected_text + + +@pytest.mark.parametrize("reverse_alternatives", [False, True]) +def test_reasoning_budget_initializes_from_checkpoint_pattern( + monkeypatch, reverse_alternatives +): + template = deepcopy(BEAM_RESPONSE_TEMPLATE) + alternatives = [ + r"^<\|channel\|>analysis<\|message\|>", + r"<\|start\|>assistant<\|channel\|>analysis<\|message\|>", + ] + if reverse_alternatives: + alternatives.reverse() + template["fields"]["thinking"] = { + "open_pattern": "(?:" + "|".join(alternatives) + ")", + "close": "<|end|>", + "content": "text", + } + tokenizer = BeamTokenizer(template) + monkeypatch.setattr( + "vllm.config.reasoning.cached_tokenizer_from_config", lambda **_: tokenizer + ) + config = ReasoningConfig(reasoning_parser="beam") + config.initialize_token_ids(SimpleNamespace()) + + assert config.enabled + # The marker must work both after the prompt's assistant header and in + # subsequent complete assistant messages, regardless of regex branch order. + assert config.reasoning_start_token_ids == tokenizer.encode( + "<|channel|>analysis<|message|>" + ) + assert config.reasoning_end_token_ids == tokenizer.encode("<|end|>") + assert config.natural_reasoning_end_token_ids == tokenizer.encode("<|end|>") + + +@pytest.mark.parametrize("pattern", [r"analysis.*", r"(?:analysis|thinking)"]) +def test_reasoning_budget_does_not_guess_a_marker_for_other_patterns(pattern): + template = deepcopy(BEAM_RESPONSE_TEMPLATE) + thinking = template["fields"]["thinking"] + del thinking["open"] + thinking["open_pattern"] = pattern + + assert BeamReasoningParser(BeamTokenizer(template)).reasoning_start_str is None diff --git a/tests/parser/test_response_template_grammar.py b/tests/parser/test_response_template_grammar.py new file mode 100644 index 0000000000..29ae227262 --- /dev/null +++ b/tests/parser/test_response_template_grammar.py @@ -0,0 +1,492 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json +from copy import deepcopy +from typing import Any + +import pytest +from xgrammar import Grammar +from xgrammar.testing import _is_grammar_accept_string + +from vllm.parser.chat_parsing.response_templates import load_response_template +from vllm.parser.response_template_grammar import ( + answer_rest_structural_tag, + tool_call_structural_tag, +) + +TEMPLATE: dict[str, Any] = { + "start_anchor": "", + "fields": { + "thinking": {"open": "", "close": "", "content": "text"}, + "content": {"content": "text"}, + "tool_calls": { + "open_pattern": r'', + "close": "", + "repeats": True, + "content": "xml-inline", + "content_args": { + "tag_pattern": r'(?P.*?)' + }, + "transform": { + "type": "function", + "function": {"name": "{name}", "arguments": "{content}"}, + }, + }, + }, +} +ANSWER_TEMPLATE = deepcopy(TEMPLATE) +ANSWER_TEMPLATE["fields"]["content"] = { + "open": "", + "close": "", + "content": "text", +} + + +def tool(parameters, name="f", strict=True) -> dict[str, Any]: + return {"name": name, "parameters": parameters, "strict": strict} + + +WEATHER = tool( + { + "type": "object", + "properties": {"city": {"type": "string"}, "days": {"type": "integer"}}, + "required": ["city"], + }, + name="get.weather", +) +PING = {"name": "ping"} +CALL = 'Paris' + + +def build( + template=TEMPLATE, + functions=(WEATHER, PING), + *, + parallel=True, + thinking=True, + answer=False, + answer_format=None, + sorted_arguments=False, +): + fields = load_response_template(template).fields + return tool_call_structural_tag( + fields["tool_calls"], + list(functions), + parallel_tool_calls=parallel, + thinking=fields["thinking"] if thinking else None, + answer=fields["content"] if answer else None, + answer_format=answer_format, + sorted_arguments=sorted_arguments, + ) + + +def accepts(tag, text): + return _is_grammar_accept_string(Grammar.from_structural_tag(json.dumps(tag)), text) + + +def call(*arguments): + body = "".join(f'{value}' for key, value in arguments) + return f'{body}' + + +@pytest.mark.parametrize( + ("text", "accepted"), + [ + ("plan" + CALL, True), + (CALL, True), + (CALL.removesuffix(""), True), + ( + 'Paris\n' + '3', + True, + ), + ( + 'Paris' + 'soon', + False, + ), + ('3', False), + (CALL + 'x', True), + ('', False), + ("planThe answer.", False), + ], +) +def test_forced_calls_follow_the_template_format(text, accepted): + assert accepts(build(), text) is accepted + + +def test_single_call_when_parallel_calls_are_disabled(): + tag = build(parallel=False) + + assert accepts(tag, CALL) + assert not accepts(tag, CALL + CALL) + + +def test_thinking_is_rejected_when_not_requested(): + assert not accepts(build(thinking=False), "plan" + CALL) + + +@pytest.mark.parametrize( + ("text", "accepted"), + [ + ("planSunny.", True), + ("Sunny.", True), + ("plan" + CALL, True), + ("Sunny." + CALL, False), + ('soon', False), + ], +) +def test_answer_field_lets_the_model_reply_instead(text, accepted): + assert accepts(build(ANSWER_TEMPLATE, answer=True), text) is accepted + + +@pytest.mark.parametrize("required", [False, True]) +@pytest.mark.parametrize( + ("text", "optional_only"), + [ + ("plan" + CALL, False), + ("\nplanSunny.", False), + (" \nSunny.", False), + (CALL, True), + ("Sunny.", True), + ("Sunny.", True), + ("<Sunny.", True), + ], +) +def test_required_reasoning_opens_the_message_with_nonempty_reasoning( + required, text, optional_only +): + fields = load_response_template(ANSWER_TEMPLATE).fields + tag = tool_call_structural_tag( + fields["tool_calls"], + [WEATHER], + parallel_tool_calls=True, + thinking=fields["thinking"], + answer=fields["content"], + message_start=[""], + requires_reasoning=required, + ) + + assert accepts(tag, text) is (not required or not optional_only) + + +def test_answer_rest_finishes_the_open_answer(): + answer = load_response_template(ANSWER_TEMPLATE).fields["content"] + tag = answer_rest_structural_tag(answer) + + assert accepts(tag, "Sunny.") + assert accepts(tag, "Sunny.") + assert not accepts(tag, "Sunny." + CALL) + + +def test_answer_without_literal_opener_is_not_derived(): + assert build(answer=True) is None + + +def test_answer_holds_the_answer_format(): + schema = {"type": "json_schema", "json_schema": {"type": "object"}} + with_tools = build(ANSWER_TEMPLATE, answer=True, answer_format=schema) + answer_only = build(ANSWER_TEMPLATE, (), answer=True, answer_format=schema) + + for tag in (with_tools, answer_only): + assert accepts(tag, 'plan{"a": 1}') + assert not accepts(tag, "Sunny.") + assert accepts(with_tools, CALL) + assert not accepts(answer_only, CALL) + + +def test_json_arguments_follow_the_schema(): + template = deepcopy(TEMPLATE) + template["fields"]["tool_calls"].update(content="json", content_args={}) + tag = build(template, [WEATHER]) + + assert accepts(tag, '{"city": "Paris"}') + assert not accepts(tag, '{"days": 3}') + + +END_X = {"tag_pattern": r'(?P.*?)X'} + + +@pytest.mark.parametrize( + "change", + [ + {"content": "json", "content_args": {"unquoted_keys": True}}, + { + "transform": { + "type": "function", + "function": {"name": "fixed", "arguments": "{content}"}, + } + }, + {"content": "kv-lines", "content_args": {}}, + # Framing in which argument text can spell a closer, where the parser + # would end the call. + {"close": ["}X", ""], "content_args": END_X}, + { + "close": ["}X", ""], + "content_args": {**END_X, "value_parser": {"name": "json"}}, + }, + {"close": ["7X", ""], "content_args": END_X}, + {"close": [">X", ""]}, + {"close": [">", ""]}, + {"close": [">"]}, + {"close": ["}", ""], "content": "json", "content_args": {}}, + ], +) +def test_underivable_tool_formats_are_not_derived(change): + template = deepcopy(TEMPLATE) + template["fields"]["tool_calls"].update(change) + + assert build(template) is None + + +def test_non_literal_thinking_delimiters_are_not_derived(): + template = deepcopy(TEMPLATE) + del template["fields"]["thinking"]["open"] + template["fields"]["thinking"]["open_pattern"] = r"" + + assert build(template) is None + assert build(template, thinking=False) is not None + + +LEVEL = {"type": "object", "properties": {"level": {"$ref": "#/$defs/level"}}} + + +@pytest.mark.parametrize( + ("schema", "value", "accepted"), + [ + ({"type": "integer", "nullable": True}, "null", True), + ({"type": "integer", "nullable": True}, "soon", False), + ({"type": ["integer", "null"]}, "null", True), + ({"anyOf": [{"type": "integer"}, {"type": "boolean"}]}, "true", True), + ({"anyOf": [{"type": "integer"}, {"type": "boolean"}]}, "soon", False), + ({"type": ["string", "integer"]}, "soon", True), + ({"enum": ["celsius", "fahrenheit"]}, "celsius", True), + ({"enum": ["celsius", "fahrenheit"]}, "kelvin", False), + ({"type": "string", "const": "on"}, "off", False), + (LEVEL, '{"level": 2}', True), + (LEVEL, '{"level": "high"}', False), + ], +) +def test_argument_values_follow_how_the_parser_reads_them(schema, value, accepted): + parameters = { + "type": "object", + "properties": {"v": schema}, + "required": ["v"], + "$defs": {"level": {"type": "integer"}}, + } + + assert accepts(build(functions=[tool(parameters)]), call(("v", value))) is accepted + + +@pytest.mark.parametrize( + ("schema", "strict", "derivable"), + [ + # The parser reads values without a type as text. + ({"enum": [1, 2]}, True, False), + ({"$ref": "#/$defs/level"}, True, False), + ({"$ref": "#/$defs/level"}, False, True), + # Raw text cannot hold its closer. + ({"enum": ["ab"]}, True, False), + ({"type": "object", "properties": {"x": {"$ref": "#/x"}}}, True, False), + (False, True, False), + # A match could write the start of the value's end. + ({"type": "string", "pattern": "^[a-z<]+$"}, True, False), + ({"type": "string", "pattern": "^[a-z<]+$"}, False, True), + ({"type": "string", "pattern": "^a.b$"}, True, False), + # XGrammar holds length bounds only in JSON. + ({"type": "string", "minLength": 3}, True, False), + ({"type": "string", "format": "ipv6"}, True, False), + ], +) +def test_schemas_text_values_cannot_hold_are_not_derived(schema, strict, derivable): + parameters = { + "type": "object", + "properties": {"v": schema}, + "$defs": {"level": {"type": "integer"}}, + } + + assert (build(functions=[tool(parameters, strict=strict)]) is not None) is derivable + + +@pytest.mark.parametrize( + ("schema", "value", "accepted"), + [ + ({"type": "string", "format": "email"}, "a.b@x.co", True), + ({"type": "string", "format": "email"}, "ab", False), + ({"type": "string", "format": "date-time"}, "2026-10-06T12:00:00Z", True), + ({"type": "string", "format": "date-time"}, "2026-13-06T12:00:00Z", False), + ({"type": "string", "pattern": "^[A-Z]{3}$"}, "ABC", True), + ({"type": "string", "pattern": "^[A-Z]{3}$"}, "ABCD", False), + # Like XGrammar, the format overrides the pattern and the length bounds. + ( + {"type": "string", "format": "uuid", "pattern": "^x$", "maxLength": 1}, + "123e4567-e89b-12d3-a456-426614174000", + True, + ), + ], +) +def test_text_values_hold_string_formats_and_patterns(schema, value, accepted): + parameters = {"type": "object", "properties": {"v": schema}, "required": ["v"]} + + assert accepts(build(functions=[tool(parameters)]), call(("v", value))) is accepted + + +def test_values_cannot_hold_a_closer(): + template = deepcopy(TEMPLATE) + template["fields"]["tool_calls"]["close"] = ["", "STOP"] + text = {"type": "object", "properties": {"v": {"type": "string"}}} + strings = {"type": "array", "items": {"type": "string"}} + json_text = {"type": "object", "properties": {"v": strings}} + + assert accepts(build(template, [tool(text)]), call(("v", "ab"))) + assert not accepts(build(template, [tool(text)]), call(("v", "aSTOPb"))) + assert not accepts( + build(template, [tool(None, strict=False)]), call(("v", "aSTOPb")) + ) + assert accepts(build(template, [tool(json_text)]), call(("v", '["ab"]'))) + assert not accepts(build(template, [tool(json_text)]), call(("v", '["aSTOPb"]'))) + + +def test_extra_arguments_need_additional_properties(): + parameters = {"type": "object", "properties": {"city": {"type": "string"}}} + extra = call(("city", "Paris"), ("zip", "75001")) + + assert not accepts(build(functions=[tool(parameters)]), extra) + # Next to declared keys, an extra one could repeat them unchecked. + allowed = {**parameters, "additionalProperties": True} + assert build(functions=[tool(allowed)]) is None + free_form = {"type": "object", "additionalProperties": True} + assert accepts(build(functions=[tool(free_form)]), extra) + + +def test_only_strict_tools_follow_their_schema(): + off_schema = call(("days", "soon")) + parameters = WEATHER["parameters"] + + assert accepts(build(functions=[tool(parameters, strict=False)]), off_schema) + assert not accepts(build(functions=[tool(parameters)]), off_schema) + assert accepts(build(functions=[tool(None)]), call()) + assert not accepts(build(functions=[tool(None)]), call(("a", "x"))) + + +def test_annotations_keep_strict_arguments_to_their_schema(): + parameters = { + **WEATHER["parameters"], + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "weather", + } + tag = build(functions=[tool(parameters)]) + + assert accepts(tag, call(("city", "Paris"), ("days", "3"))) + assert not accepts(tag, call(("city", "Paris"), ("days", "soon"))) + + +@pytest.mark.parametrize( + ("arguments", "accepted", "accepted_sorted"), + [ + ((("path", "a"), ("content", "b")), True, True), + ((("content", "b"), ("path", "a")), False, True), + ((("path", "a"),), True, True), + ((("content", "b"),), False, False), + ((("path", "a"), ("path", "c")), False, False), + ((("path", "a"), ("mode", "c")), False, False), + ], +) +def test_strict_arguments_follow_declared_or_sorted_order( + arguments, accepted, accepted_sorted +): + parameters = { + "properties": {"path": {"type": "string"}, "content": {"type": "string"}}, + "required": ["path"], + } + tags = [ + build(functions=[tool(parameters)], sorted_arguments=sorted_arguments) + for sorted_arguments in (False, True) + ] + + assert [accepts(tag, call(*arguments)) for tag in tags] == [ + accepted, + accepted_sorted, + ] + + +LOOKAHEAD_TEMPLATE = deepcopy(TEMPLATE) +LOOKAHEAD_TEMPLATE["fields"]["tool_calls"]["content_args"]["tag_pattern"] = ( + r'(?P.*?)\n(?:|(?=Paris\n\n3\n', True, True), + ('Paris\n3\n', True, True), + ('3\nParis\n', True, False), + ('Paris\nLyon\n', True, False), + ('Paris\n3\n', True, False), + ('Paris\nParis\na c\n', True, True), + ], +) +def test_values_end_where_a_lookahead_end_does(body, accepted_loose, accepted_strict): + parameters = WEATHER["parameters"] + tags = [ + build(LOOKAHEAD_TEMPLATE, [tool(parameters, strict=strict)]) + for strict in (False, True) + ] + text = f'{body}' + + assert [accepts(tag, text) for tag in tags] == [accepted_loose, accepted_strict] + + +@pytest.mark.parametrize("lookahead", [False, True]) +@pytest.mark.parametrize( + ("keys", "accepted"), + [ + ("b", True), + ("ab", True), + ("bc", True), + ("bd", True), + ("be", True), + ("abde", True), + ("bcde", True), + ("", False), + ("a", False), + ("acd", False), + ("bdc", False), + ("bcc", False), + ], +) +def test_last_parameter_follows_every_required_one(keys, accepted, lookahead): + parameters = { + "properties": {key: {"type": "string"} for key in "abcde"}, + "required": ["b"], + } + tag = build(LOOKAHEAD_TEMPLATE, [tool(parameters)]) + ends = ["\n" if lookahead else "\n"] * (len(keys) - 1) + ["\n"] + body = "".join(f'x{e}' for k, e in zip(keys, ends)) + + assert accepts(tag, f'{body}') is accepted + + +def test_required_argument_without_schema_is_still_required(): + parameters = {"type": "object", "properties": {}, "required": ["a"]} + tag = build(functions=[tool(parameters)]) + + assert accepts(tag, call(("a", "x"))) + assert not accepts(tag, call()) + assert build(functions=[tool({**parameters, "required": ["a", 1]})]) is None + + +@pytest.mark.parametrize( + ("function", "match"), + [ + (tool({}, name="get weather"), "tool name"), + (tool({"properties": {'a"b': {"type": "string"}}}), "parameter name"), + ], +) +def test_unencodable_names_are_rejected(function, match): + with pytest.raises(ValueError, match=match): + build(functions=[function]) diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index 3ddf81b288..39d5ff9961 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -53,6 +53,7 @@ from openai_harmony import Message as OpenAIHarmonyMessage from pydantic import ( BeforeValidator, Field, + PrivateAttr, ValidationError, field_serializer, model_validator, @@ -469,6 +470,14 @@ class ResponsesRequest(OpenAIBaseModel): max_output_tokens_param="max_output_tokens", ) + _grammar_from_parser: bool = PrivateAttr(default=False) + """CAUTION: Should only be set by the parser's adjust_request.""" + + _parser_structured_outputs: StructuredOutputsParams | None = PrivateAttr( + default=None + ) + """Parser-derived constraints that preserve the public request's text metadata.""" + _DEFAULT_SAMPLING_PARAMS = { "temperature": 1.0, "top_p": 1.0, @@ -478,6 +487,8 @@ class ResponsesRequest(OpenAIBaseModel): def extract_structured_outputs(self) -> StructuredOutputsParams | None: """Normalize request constraints into ``StructuredOutputsParams``.""" + if self._parser_structured_outputs is not None: + return self._parser_structured_outputs if self.text is None or self.text.format is None: return self.structured_outputs diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index dd154f968b..ba9d95b428 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -526,6 +526,7 @@ class OpenAIServingResponses(GenerateBaseServing): trace_headers=trace_headers, session_id=session_id, reasoning_parser_kwargs=reasoning_parser_kwargs, + reasoning_ended=True if request._grammar_from_parser else None, ) generators.append(generator) @@ -647,6 +648,7 @@ class OpenAIServingResponses(GenerateBaseServing): trace_headers: Mapping[str, str] | None = None, session_id: str | None = None, reasoning_parser_kwargs: dict[str, Any] | None = None, + reasoning_ended: bool | None = None, ): max_model_len = self.model_config.max_model_len cache_salt = cast(str | None, engine_input.get("cache_salt")) @@ -673,6 +675,7 @@ class OpenAIServingResponses(GenerateBaseServing): priority=priority, session_id=session_id, reasoning_parser_kwargs=reasoning_parser_kwargs, + reasoning_ended=reasoning_ended, ) async for res in generator: diff --git a/vllm/envs.py b/vllm/envs.py index ab04f405d7..5cd1d3eecb 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -246,6 +246,7 @@ if TYPE_CHECKING: VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840 VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1 VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True + VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT: bool = True VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16 VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300 VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5 @@ -1844,6 +1845,11 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: ( os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "True").lower() in ("true", "1") ), + # Constrain every `beam` output to its response template's format, also + # without tool or structured output constraints. + "VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT": lambda: ( + os.getenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "True").lower() in ("true", "1") + ), # Control the max chunk bytes (in MB) for the rpc message queue. # Object larger than this threshold will be broadcast to worker # processes via zmq. diff --git a/vllm/parser/beam.py b/vllm/parser/beam.py new file mode 100644 index 0000000000..9f99e321f1 --- /dev/null +++ b/vllm/parser/beam.py @@ -0,0 +1,262 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Parser for Beam checkpoints.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, cast + +from openai.types.responses import ToolChoiceFunction + +from vllm import envs +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionNamedToolChoiceParam, + ChatCompletionRequest, +) +from vllm.exceptions import VLLMValidationError +from vllm.logger import init_logger +from vllm.parser.abstract_parser import ( + _xgrammar_supports, + structured_outputs_to_format, +) +from vllm.parser.chat_parsing import ResponseParser +from vllm.parser.chat_parsing.response_templates import ResponseTemplate +from vllm.parser.response_template import ( + CONTENT_FIELD, + THINKING_FIELD, + TOOL_FIELD, + ResponseTemplateParser, + ResponseTemplateReasoningParser, + ResponseTemplateToolParser, + _as_tool_dicts, + _is_named_tool_choice, + _tool_parsing_enabled, +) +from vllm.parser.response_template_grammar import ( + _openers, + answer_rest_structural_tag, + message_start_anchors, + tool_call_structural_tag, +) +from vllm.sampling_params import StructuredOutputsParams +from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness + +if TYPE_CHECKING: + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + +logger = init_logger(__name__) + + +def _named_tool(request: ChatCompletionRequest | ResponsesRequest) -> str: + choice = request.tool_choice + if isinstance(choice, ToolChoiceFunction): + return choice.name + return cast(ChatCompletionNamedToolChoiceParam, choice).function.name + + +def _starts_message(template: ResponseTemplate, prefix: str) -> bool: + """Whether output after `prefix` starts the assistant message.""" + ends = [match.end() for match in template.start_anchor_re.finditer(prefix)] + return bool(ends) and ends[-1] == len(prefix) + + +def _open_field(template: ResponseTemplate, prefix: str) -> str | None: + """The field that `prefix` leaves open with its explicit opener, if any.""" + field = None + for event in ResponseParser(template, prefix=prefix).initial_events: + if event["type"] == "region_open" and event["end"] > event["start"]: + field = event["field"] + elif event["type"] in {"region_close", "region_malformed"}: + field = None + return field + + +class BeamReasoningParser(ResponseTemplateReasoningParser): + """Selects `BeamParser` as the reasoning parser.""" + + @classmethod + def unified_parser_cls(cls) -> type[ResponseTemplateParser]: + return BeamParser + + @property + def reasoning_start_str(self) -> str | None: + if literal := super().reasoning_start_str: + return literal + thinking = self.response_template.fields.get(THINKING_FIELD) + if thinking is None: + return None + openers = _openers(thinking, message_start_anchors(self.response_template)) + if not openers: + return None + # Budget tracking needs one marker for both a prompt continuation + # and a complete assistant message in the checkpoint's pattern. + marker = min(openers, key=len) + return marker if all(opener.endswith(marker) for opener in openers) else None + + +class BeamToolParser(ResponseTemplateToolParser): + """Selects `BeamParser` as the tool parser.""" + + @classmethod + def unified_parser_cls(cls) -> type[ResponseTemplateParser]: + return BeamParser + + +class BeamParser(ResponseTemplateParser): + """Parse Beam output with the checkpoint's `response_template` metadata, + and constrain it with a grammar derived from that template.""" + + reasoning_parser_cls = BeamReasoningParser + tool_parser_cls = BeamToolParser + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + chat_template_kwargs = kwargs.get("chat_template_kwargs") or {} + # From the message start, the grammar opens the message with non-empty + # reasoning, unless the request turns thinking off. + self._requires_reasoning = bool( + chat_template_kwargs.get("enable_thinking", True) + ) + + def _constrain_request( + self, + request: ChatCompletionRequest | ResponsesRequest, + ) -> ChatCompletionRequest | ResponsesRequest: + message_start = _starts_message(self.response_template, self._prefix) + tag = self._template_tag(request, message_start=message_start) + if tag is not None: + grammar = StructuredOutputsParams(structural_tag=json.dumps(tag)) + if isinstance(request, ChatCompletionRequest): + request.structured_outputs = grammar + request.response_format = None + else: + request._parser_structured_outputs = grammar + # From the message start, the tag also covers the reasoning. + request._grammar_from_parser = message_start + return request + + def _template_tag( + self, + request: ChatCompletionRequest | ResponsesRequest, + *, + message_start: bool, + ) -> dict[str, Any] | None: + """The template grammar for `request`, or `None` to leave its output + to the request's own constraint, if any.""" + fields = self.response_template.fields + named = _is_named_tool_choice(request) + tools_enabled = ( + self._tool_parser is not None + and bool(request.tools) + and _tool_parsing_enabled( + request, enable_auto_tools=self._enable_auto_tools + ) + ) + forced = tools_enabled and (request.tool_choice == "required" or named) + # The prompt may still list the tools, so only the grammar keeps the + # model from calling them. + tools_forbidden = ( + bool(request.tools) + and request.tool_choice == "none" + and TOOL_FIELD in fields + ) + constrained_tools = None + if tools_enabled and (forced or envs.VLLM_ENFORCE_STRICT_TOOL_CALLING): + assert self._tool_parser is not None + strict_level = self.tool_strict_level + if strict_level is None: + strict_level = self._tool_parser.default_tool_strict_level + constrained_tools = resolve_tool_strictness( + request.tools or [], request.tool_choice, strict_level + ) + output = request.extract_structured_outputs() + if forced and output is not None: + logger.warning_once( + "structured outputs are ignored because tool_choice forces tool call." + ) + output = None + answer = fields.get(CONTENT_FIELD) + if output is not None and ( + output.structural_tag is not None + or answer is None + or answer.open_re is None + or not _xgrammar_supports(output) + ): + if constrained_tools is not None: + logger.warning_once( + "response_template parsing cannot enforce strict tools " + "alongside this structured output." + ) + return None + # Without a tool or structured output constraint, the output still + # follows the template. + always_constrain = envs.VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT and not forced + if ( + constrained_tools is None + and output is None + and not tools_forbidden + and not always_constrain + ): + return None + + # Inside an open answer, only the always-on grammar applies, finishing a + # free answer. Inside open reasoning, the grammar needs the reasoning + # gate to defer it to the reasoning's end. + open_field = _open_field(self.response_template, self._prefix) + reasoning_gated = ( + self._reasoning_parser is not None and request.include_reasoning + ) + tag = None + if open_field == CONTENT_FIELD: + if always_constrain and output is None: + tag = answer_rest_structural_tag(fields[CONTENT_FIELD]) + elif open_field != THINKING_FIELD or reasoning_gated: + functions = [] + if tools_enabled: + tools = _as_tool_dicts(constrained_tools or request.tools) + functions = [tool.get("function", tool) for tool in tools] + if named: + name = _named_tool(request) + functions = [f for f in functions if f.get("name") == name] + answer_format = ( + None if output is None else structured_outputs_to_format(output) + ) + try: + tag = tool_call_structural_tag( + fields.get(TOOL_FIELD), + functions, + parallel_tool_calls=( + not named and request.parallel_tool_calls is not False + ), + thinking=fields.get(THINKING_FIELD), + answer=None if forced else answer, + answer_format=( + None if answer_format is None else answer_format.model_dump() + ), + message_start=( + message_start_anchors(self.response_template) + if message_start + else None + ), + requires_reasoning=self._requires_reasoning, + # Beam drifts from schemas the grammar does not enforce, + # and also writes arguments sorted by name. + sorted_arguments=True, + strict_arguments=True, + ) + except ValueError as exc: + raise VLLMValidationError(str(exc), parameter="tools") from exc + if tag is None and forced: + kind = "named" if named else "required" + raise VLLMValidationError( + f"response_template parsing cannot guarantee {kind} tool choice " + "for this request", + parameter="tool_choice", + ) + if tag is None and constrained_tools is not None: + logger.warning_once( + "response_template parsing cannot enforce strict tools for this " + "request." + ) + return tag diff --git a/vllm/parser/parser_manager.py b/vllm/parser/parser_manager.py index d6fdbce19c..b6f00cacd8 100644 --- a/vllm/parser/parser_manager.py +++ b/vllm/parser/parser_manager.py @@ -16,8 +16,6 @@ if TYPE_CHECKING: logger = init_logger(__name__) -HF_PARSER = "hf" - class ParserManager: """Provides a unified Parser by composing reasoning and tool parser adapters.""" @@ -122,8 +120,10 @@ class ParserManager: tags (``--tool-strict-level``), or ``None`` for the tool parser's default level. tokenizer: Tokenizer the composed parser is validated against - at startup (see :meth:`validate_parser_tokenizer`). ``None`` - skips the check (e.g. ``skip_tokenizer_init``). + at startup (see :meth:`validate_parser_tokenizer`), including + its `response_template` metadata when a response-template + parser is selected. ``None`` skips the check (e.g. + ``skip_tokenizer_init``). Returns: A Parser class, or None if neither parser is specified. @@ -184,39 +184,40 @@ class ParserManager: HarmonyParser.tool_strict_level = strict_level return HarmonyParser - if HF_PARSER in (reasoning_parser_name, tool_parser_name): - if {reasoning_parser_name, tool_parser_name} - { - HF_PARSER, - None, - "", - }: - raise TypeError( - "The hf parser cannot be combined with other " - "reasoning or tool call parsers" - ) - from vllm.parser.response_template import ( - ResponseTemplateParser, - validate_tokenizer_response_template, - ) + from vllm.parser.response_template import ( + response_template_parser_cls, + validate_tokenizer_response_template, + ) + template_parsers = { + response_template_parser_cls(parser_cls) + for parser_cls in (reasoning_parser_cls, tool_parser_cls) + if parser_cls is not None + } + template_parser_cls = next(iter(template_parsers - {None}), None) + if template_parser_cls is not None: + if len(template_parsers) > 1: + raise TypeError( + f"--reasoning-parser {reasoning_parser_name} and " + f"--tool-call-parser {tool_parser_name} cannot be combined" + ) if tokenizer is not None: validate_tokenizer_response_template( tokenizer, - reasoning=reasoning_parser_cls is not None, - tools=tool_parser_cls is not None, + reasoning_parser=reasoning_parser_name, + # Without auto tool choice, no tool parser is selected. + tool_parser=tool_parser_name if tool_parser_cls else None, ) - - r_cls = reasoning_parser_cls - t_cls = tool_parser_cls - auto_tools = enable_auto_tools - - class _ResponseTemplateParser(ResponseTemplateParser): - reasoning_parser_cls = r_cls - tool_parser_cls = t_cls - tool_strict_level = strict_level - _enable_auto_tools = auto_tools - - return _ResponseTemplateParser + return type( + template_parser_cls.__name__, + (template_parser_cls,), + { + "reasoning_parser_cls": reasoning_parser_cls, + "tool_parser_cls": tool_parser_cls, + "tool_strict_level": strict_level, + "_enable_auto_tools": enable_auto_tools, + }, + ) if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3": from vllm.parser.kimi_k3 import KimiK3Parser diff --git a/vllm/parser/response_template.py b/vllm/parser/response_template.py index b1665f1772..ea95567305 100644 --- a/vllm/parser/response_template.py +++ b/vllm/parser/response_template.py @@ -7,6 +7,7 @@ from __future__ import annotations import copy import json from collections.abc import Callable, Iterable, Sequence +from math import inf from typing import TYPE_CHECKING, Any, cast from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction @@ -28,6 +29,7 @@ from vllm.parser.chat_parsing.response_templates import ( ResponseTemplate, load_response_template, ) +from vllm.parser.response_template_grammar import message_start_anchors from vllm.reasoning.abs_reasoning_parsers import ReasoningParser from vllm.renderers.chat_utils import make_tool_call_id from vllm.tool_parsers.abstract_tool_parser import ToolParser @@ -135,27 +137,38 @@ def validate_response_template_for_serving( def validate_tokenizer_response_template( tokenizer: Any, *, - reasoning: bool, - tools: bool, + reasoning_parser: str | None, + tool_parser: str | None, ) -> None: - """Fail at startup if the tokenizer cannot serve the requested parsing.""" + """Fail at startup if the tokenizer cannot serve the selected parsers, + named as given on the command line.""" template = resolve_response_template(tokenizer) if template is None: + selected = " ".join( + f"{flag} {name}" + for flag, name in ( + ("--reasoning-parser", reasoning_parser), + ("--tool-call-parser", tool_parser), + ) + if name + ) raise TypeError( - "The hf parser requires `response_template` metadata " + f"{selected} requires `response_template` metadata " "in the tokenizer configuration" ) try: fields = validate_response_template_for_serving(template).fields except (TypeError, ValueError) as exc: raise TypeError(f"Invalid response_template metadata: {exc}") from exc - if reasoning and THINKING_FIELD not in fields: + if reasoning_parser and THINKING_FIELD not in fields: raise TypeError( - "--reasoning-parser hf requires a response_template with a thinking field" + f"--reasoning-parser {reasoning_parser} requires a response_template " + "with a thinking field" ) - if tools and TOOL_FIELD not in fields: + if tool_parser and TOOL_FIELD not in fields: raise TypeError( - "--tool-call-parser hf requires a response_template with a tool_calls field" + f"--tool-call-parser {tool_parser} requires a response_template " + "with a tool_calls field" ) @@ -292,7 +305,7 @@ def _serving_template( template = resolve_response_template(tokenizer, template) if template is None: raise ValueError( - "The hf parser requires `response_template` " + "Parsing with a response template requires `response_template` " "metadata in the tokenizer configuration" ) validate_response_template_for_serving(template) @@ -319,12 +332,17 @@ def _to_tool_calls(value: Any) -> list[tuple[str, str]]: class ResponseTemplateReasoningParser(ReasoningParser): - """Reasoning end detection for `--reasoning-parser hf`. + """Reasoning end detection for a response-template parser. `ResponseTemplateParser` parses the output; this parser tells it and the structured-output engine where reasoning ends. """ + @classmethod + def unified_parser_cls(cls) -> type[ResponseTemplateParser]: + """The unified parser that selecting this reasoning parser stands for.""" + return ResponseTemplateParser + def __init__( self, tokenizer: TokenizerLike, @@ -400,8 +418,26 @@ class ResponseTemplateReasoningParser(ReasoningParser): class ResponseTemplateToolParser(ToolParser): - """Registers `--tool-call-parser hf`; `ResponseTemplateParser` extracts the - tool calls.""" + """Selects a response-template parser as the tool parser; + `ResponseTemplateParser` extracts the tool calls.""" + + @classmethod + def unified_parser_cls(cls) -> type[ResponseTemplateParser]: + """The unified parser that selecting this tool parser stands for.""" + return ResponseTemplateParser + + +def response_template_parser_cls( + split_parser_cls: type[ReasoningParser] | type[ToolParser], +) -> type[ResponseTemplateParser] | None: + """The response-template parser that a registered reasoning or tool parser + stands for, or `None` for any other parser.""" + if issubclass( + split_parser_cls, + (ResponseTemplateReasoningParser, ResponseTemplateToolParser), + ): + return split_parser_cls.unified_parser_cls() + return None class _DeltaBuilder: @@ -441,23 +477,50 @@ class _ResponseStream: self.include_reasoning = include_reasoning self.parse_tools = parse_tools self.new_tool_call_id = new_tool_call_id + self.template = template self.parser = ResponseParser(template, prefix=prefix, tools=tools) self._prefix_end = len(self.parser.input_text) self._next_tool_index = 0 self._finalized = False + self._open_region: dict[str, Any] | None = None + self._track(self.parser.initial_events) def feed(self, text: str, *, finished: bool) -> DeltaMessage | None: if self._finalized: return None delta = _DeltaBuilder() - self._route(self.parser.feed(text), delta) + self._route(self._track(self.parser.feed(text)), delta) if finished: self._finalized = True if len(self.parser.input_text) > self._prefix_end: - _, events = self.parser.finalize() - self._route(events, delta) + events = self.parser.feed(self._closer_rest()) + self._route(events + self.parser.finalize()[1], delta) return delta.build() + def _track(self, events: list[dict[str, Any]]) -> list[dict[str, Any]]: + for event in events: + if event["type"] == "region_open": + self._open_region = event + elif event["type"] in ("region_close", "region_malformed"): + self._open_region = None + return events + + def _closer_rest(self) -> str: + """The rest of a closer the output stopped in after ending its region, + when only the next message's start anchor is missing.""" + if self._open_region is None: + return "" + field = self.template.fields[self._open_region["field"]] + text = self.parser.input_text[self._open_region["end"] :] + for closer in field.close_literals or (): + for anchor in message_start_anchors(self.template): + if not anchor or not closer.endswith(anchor) or closer == anchor: + continue + for cut in range(len(closer) - 1, len(closer) - len(anchor) - 1, -1): + if text.endswith(closer[:cut]): + return closer[cut:] + return "" + def _route(self, events: Sequence[dict[str, Any]], delta: _DeltaBuilder) -> None: for event in events: field = event.get("field") @@ -499,6 +562,54 @@ class _ResponseStream: self._next_tool_index += 1 +class _ReasoningTokenCounter: + """Counts the reasoning tokens of one growing generation, parsing each + token once.""" + + def __init__( + self, template: ResponseTemplate, prefix: str, tokenizer: TokenizerLike + ) -> None: + self.tokenizer = tokenizer + self.parser = ResponseParser(template, prefix=prefix) + self.start = len(self.parser.input_text) + self.token_ids: list[int] = [] + self.token_ends: list[int] = [] + # Input spans between the delimiters of thinking regions. + self.spans: list[tuple[int, float]] = [] + self.open: int | None = None + self._track(self.parser.initial_events) + + def continues(self, token_ids: Sequence[int]) -> bool: + return list(token_ids[: len(self.token_ids)]) == self.token_ids + + def count(self, token_ids: Sequence[int]) -> int: + new_ids = list(token_ids[len(self.token_ids) :]) + texts = [_decode(self.tokenizer, [token_id]) for token_id in new_ids] + end = self.token_ends[-1] if self.token_ends else self.start + for text in texts: + end += len(text) + self.token_ends.append(end) + self.token_ids.extend(new_ids) + self._track(self.parser.feed("".join(texts))) + # Text held back as a possible closer counts until a closer arrives. + spans = self.spans if self.open is None else [*self.spans, (self.open, inf)] + count, start = 0, self.start + for end in self.token_ends: + count += any(begin < end and start < close for begin, close in spans) + start = end + return count + + def _track(self, events: Iterable[dict[str, Any]]) -> None: + for event in events: + if event["field"] != THINKING_FIELD or event["type"] == "region_chunk": + continue + if event["type"] == "region_open": + self.open = event["end"] + elif self.open is not None: + self.spans.append((self.open, event["start"])) + self.open = None + + class ResponseTemplateParser(Parser): """Parse reasoning, content, and tool calls with one response template.""" @@ -526,6 +637,7 @@ class ResponseTemplateParser(Parser): self._prompt_token_ids: list[int] | None = None self._prefix = "" self._stream: _ResponseStream | None = None + self._reasoning_counter: _ReasoningTokenCounter | None = None def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None: prompt_token_ids = list(prompt_token_ids) @@ -534,6 +646,7 @@ class ResponseTemplateParser(Parser): self._prompt_token_ids = prompt_token_ids self._prefix = _decode(self.model_tokenizer, prompt_token_ids) self._stream = None + self._reasoning_counter = None def adjust_request( self, @@ -547,6 +660,14 @@ class ResponseTemplateParser(Parser): request.skip_special_tokens = False if hasattr(request, "spaces_between_special_tokens"): request.spaces_between_special_tokens = False + return self._constrain_request(request) + + def _constrain_request( + self, + request: ChatCompletionRequest | ResponsesRequest, + ) -> ChatCompletionRequest | ResponsesRequest: + """Constrain `request` to the tool guarantees it asks for, or reject + those this parser cannot enforce.""" if ( self._tool_parser is not None and request.tools @@ -567,6 +688,21 @@ class ResponseTemplateParser(Parser): or self._reasoning_parser.is_reasoning_end(input_ids) ) + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + """Count the generated tokens whose text the template parses as + reasoning, continuing reasoning left open by the prompt.""" + if ( + self._reasoning_parser is None + or THINKING_FIELD not in self.response_template.fields + ): + return 0 + counter = self._reasoning_counter + if counter is None or not counter.continues(token_ids): + counter = self._reasoning_counter = _ReasoningTokenCounter( + self.response_template, self._prefix, self.model_tokenizer + ) + return counter.count(token_ids) + def parse( self, model_output: str, diff --git a/vllm/parser/response_template_grammar.py b/vllm/parser/response_template_grammar.py new file mode 100644 index 0000000000..40317023f4 --- /dev/null +++ b/vllm/parser/response_template_grammar.py @@ -0,0 +1,1000 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Structural tags for tool calls and answers, derived from a response template.""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from functools import lru_cache +from typing import Any + +import regex as re + +try: + from re import _parser # type: ignore[attr-defined] +except ImportError: # Python 3.10 + import sre_parse as _parser + +from vllm.parser.chat_parsing.content_parsers import parse_content +from vllm.parser.chat_parsing.response_parser import _schema_types +from vllm.parser.chat_parsing.response_templates import ( + ResponseTemplate, + ResponseTemplateField, +) + +_sre_parse = _parser +_REGEX_META = re.compile(r"[.^$*+?{}\[\]\\|()]") +_LITERAL_ESCAPES = {"n": "\n", "r": "\r", "t": "\t"} +# Stands for `^` while expanding a delimiter pattern. +_START = "\0" +# Precedes the text of a final lookahead while expanding a delimiter pattern. +_AHEAD = "\1" +# Arbitrary bound on the strings expanded from one delimiter pattern. +_MAX_LITERALS = 16 +# Templates write at most a newline between parameters. Unbounded whitespace +# would let a model that skipped a parameter's opener write it until `max_tokens`. +_SEPARATOR = {"type": "regex", "pattern": r"\n?"} +_ANY_TEXT = {"type": "any_text"} +# Character classes in XGrammar regexes take whitespace only as escapes. +_BLANKS = {" ": " ", "\t": r"\t", "\n": r"\n", "\r": r"\r"} +# Keywords that describe a schema without constraining its values. +_ANNOTATIONS = frozenset( + { + "title", + "description", + "default", + "examples", + "deprecated", + "$comment", + "$schema", + "$id", + } +) +_OBJECT_KEYWORDS = frozenset( + {"type", "properties", "required", "additionalProperties", "$defs", "definitions"} +) +# Characters JSON writes outside strings. +_JSON_SYNTAX = frozenset(' \t\n\r{}[],:"-+.0123456789eEtrufalsn') +# Pieces of the text of a call for `_spells_closer`. Text and JSON hold no +# closer, and JSON ends with a character it writes outside strings. +_TEXT_PIECE = (r"[\s\S]*", r"[\s\S]+", False) +_JSON_PIECE = ( + r"[\s\S]*", + r"[\s\S]*[" + "".join(map(re.escape, sorted(_JSON_SYNTAX))) + "]", + False, +) +_REPEATS = tuple( + getattr(_sre_parse, name) + for name in ("MAX_REPEAT", "MIN_REPEAT", "POSSESSIVE_REPEAT") + if hasattr(_sre_parse, name) +) +_CATEGORIES = { + _sre_parse.CATEGORY_DIGIT: r"\d", + _sre_parse.CATEGORY_NOT_DIGIT: r"\D", + _sre_parse.CATEGORY_SPACE: r"\s", + _sre_parse.CATEGORY_NOT_SPACE: r"\S", + _sre_parse.CATEGORY_WORD: r"\w", + _sre_parse.CATEGORY_NOT_WORD: r"\W", +} +_HOST_LABEL = r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?" +_DATE = r"[0-9]{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])" +_TIME = ( + r"(?:[01][0-9]|2[0-3]):[0-5][0-9]:(?:[0-5][0-9]|60)(?:\.[0-9]+)?" + r"(?:Z|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9])" +) +_OCTET = r"(?:25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])" +_ATOM = r"[A-Za-z0-9_!#$%&'*+/=?^`{|}~-]+" +_URI_CHARS = r"A-Za-z0-9_.~!$&'()*+,;=" +_ESCAPED = r"%[0-9A-Fa-f]{2}" +_PATH = rf"(?:[{_URI_CHARS}:@-]|{_ESCAPED})" +_QUERY = rf"(?:[{_URI_CHARS}:@/?-]|{_ESCAPED})*" +# Raw string forms of the formats XGrammar holds in JSON strings. Emails leave +# out quoted local parts, whose quotes XGrammar writes as JSON escapes. +_STRING_FORMATS = { + "date": _DATE, + "time": _TIME, + "date-time": f"{_DATE}T{_TIME}", + "email": ( + rf"{_ATOM}(?:\.{_ATOM})*@[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?" + r"(?:\.[A-Za-z0-9][A-Za-z0-9-]*[A-Za-z0-9])*" + ), + "hostname": rf"{_HOST_LABEL}(?:\.{_HOST_LABEL})*", + "ipv4": rf"(?:{_OCTET}\.){{3}}{_OCTET}", + "uuid": r"[0-9A-Fa-f]{8}(?:-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12}", + "uri": ( + rf"[A-Za-z][A-Za-z+.-]*:(?://(?:(?:[{_URI_CHARS}:-]|{_ESCAPED})*@)?" + rf"(?:[{_URI_CHARS}-]|{_ESCAPED})*(?::[0-9]*)?(?:/{_PATH}*)*" + rf"|/?(?:{_PATH}+(?:/{_PATH}*)*)?)(?:\?{_QUERY})?(?:#{_QUERY})?" + ), +} + + +def _split_group(pattern: str, name: str) -> tuple[str, str, str] | None: + """Split a named regex group into its prefix, body and suffix.""" + opener = f"(?P<{name}>" + start = pattern.find(opener) + if start < 0: + return None + depth = 0 + in_class = False + index = start + while index < len(pattern): + char = pattern[index] + if char == "\\": + index += 2 + continue + if in_class: + in_class = char != "]" + elif char == "[": + in_class = True + elif char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth == 0: + body = pattern[start + len(opener) : index] + return pattern[:start], body, pattern[index + 1 :] + index += 1 + return None + + +def _literals( + pattern: str, *, message_start: bool = False, lookahead: bool = False +) -> list[str] | None: + """Expand literal alternatives, allowing `^` only at the message start and + a final lookahead only with `lookahead`.""" + literals, index = _alternatives(pattern, 0) + if literals is None or index != len(pattern): + return None + if any(literal.count(_AHEAD) > lookahead for literal in literals): + return None + if not message_start: + return None if any(_START in literal for literal in literals) else literals + return [ + literal.removeprefix(_START) + for literal in literals + if _START not in literal[1:] and literal != _START + ] + + +def _alternatives(pattern: str, index: int) -> tuple[list[str] | None, int]: + literals = [] + while True: + branch, index = _concatenation(pattern, index) + if branch is None: + return None, index + literals.extend(branch) + if not pattern.startswith("|", index): + break + index += 1 + literals = list(dict.fromkeys(literals)) + return (literals if len(literals) <= _MAX_LITERALS else None), index + + +def _concatenation(pattern: str, index: int) -> tuple[list[str] | None, int]: + literals = [""] + while index < len(pattern) and pattern[index] not in "|)": + if pattern.startswith(("(?:", "(?="), index): + ahead = pattern[index + 2] == "=" + atom, index = _alternatives(pattern, index + 3) + if atom is None or not pattern.startswith(")", index): + return None, index + index += 1 + if ahead: + atom = [_AHEAD + literal for literal in atom] + elif pattern[index] == "\\" and index + 1 < len(pattern): + escaped = pattern[index + 1] + if escaped.isalnum() and escaped not in _LITERAL_ESCAPES: + return None, index + atom = [_LITERAL_ESCAPES.get(escaped, escaped)] + index += 2 + elif pattern[index] == "^": + atom = [_START] + index += 1 + elif _REGEX_META.fullmatch(pattern[index]): + return None, index + else: + atom = [pattern[index]] + index += 1 + if pattern.startswith("?", index): + atom = ["", *atom] + index += 1 + # A lookahead reads as a literal only at the end of the pattern. + if any(_AHEAD in prefix for prefix in literals): + return None, index + literals = [prefix + suffix for prefix in literals for suffix in atom] + if len(literals) > _MAX_LITERALS: + return None, index + return literals, index + + +def _openers( + field: ResponseTemplateField, message_start: list[str] | None = None +) -> list[str] | None: + """Derive field openers, including continuations of message-start anchors.""" + if field.open_literals or field.open_re is None: + openers = field.open_literals + else: + openers = _literals( + field.open_re.pattern, message_start=message_start is not None + ) + return _continuing(openers, message_start, field.open_re) + + +def _continuing( + literals: list[str] | None, anchors: list[str] | None, opener: re.Pattern +) -> list[str] | None: + """Keep anchor continuations only where the parser finds the opener.""" + if not literals or not anchors: + return literals + rests = [ + literal[len(anchor) :] + for literal in literals + for anchor in anchors + if len(literal) > len(anchor) and literal.startswith(anchor) + ] + rests = [ + rest + for rest in rests + if any(opener.fullmatch(rest, index) for index in range(len(rest))) + ] + return list(dict.fromkeys([*literals, *rests])) + + +def _escape(text: str) -> str: + return _REGEX_META.sub(lambda match: "\\" + match.group(), text) + + +def _call_opener( + opener: tuple[str, str, str], name: str, message_start: list[str] | None +) -> str: + before, _, after = opener + if message_start is not None and before: + literals = _literals(before, message_start=True) + # Text before a name inside a group is no pattern of its own. + if literals: + literals = _continuing(literals, message_start, re.compile(before)) + before = "(?:" + "|".join(map(_escape, literals or ())) + ")" + return before + name + after + + +def _other_names(pattern: str, names: list[str]) -> str | None: + """A regex for the names `pattern` matches besides `names`, or `None` if + `pattern` is no negatable character class repeated.""" + if not names: + return pattern + match = re.fullmatch(r"\\([wS])\+|\[\^((?:\\.|[^\]\\])+)\]\+", pattern) + if match is None: + return None + negated = {"w": r"\W", "S": r"\s"}[match[1]] if match[1] else match[2] + + def other(chars: set[str]) -> str: + return f"[^{negated}{''.join(map(_class_escape, sorted(chars)))}]" + + trie: dict[str, dict] = {} + for name in names: + node = trie + for char in name: + node = node.setdefault(char, {}) + node[""] = {} + + def rest(node: dict[str, dict], first: bool) -> str: + """The rest of a name past `node` that ends on none of `names`.""" + branches = [other(set(node) - {""}) + other(set()) + "*"] + branches += [ + _escape(char) + rest(child, False) for char, child in node.items() if char + ] + return f"(?:{'|'.join(branches)})" + ("" if first or "" in node else "?") + + return rest(trie, True) + + +def _thinking_format( + thinking: ResponseTemplateField, + message_start: list[str] | None, + nonempty: bool = False, +) -> dict[str, Any] | None: + """The reasoning, which holds at least a character if `nonempty`.""" + openers, closers = _openers(thinking, message_start), _closers(thinking) + if not openers or not closers: + return None + text = _sequence(_nonempty_start(closers), _ANY_TEXT) if nonempty else _ANY_TEXT + return _one_of([_tag(opener, text, closers) for opener in openers]) + + +def _nonempty_start(closers: list[str]) -> dict[str, Any]: + """The start of text that is not empty and does not begin with one of the + `closers`. Each branch leaves every closer's spelling before completing it, + on a character that starts no closer, so the text after it cannot hide a + closer's start and a closer written as one token cannot pass for the text.""" + closers = [closer for closer in closers if closer] + starts = {closer[0] for closer in closers} + prefixes = dict.fromkeys( + [""] + [closer[:end] for closer in closers for end in range(1, len(closer))] + ) + branches = [] + for prefix in prefixes: + # A prefix holding a whole closer has already closed the text. + if any(closer in prefix for closer in closers): + continue + nexts = { + closer[len(prefix)] + for closer in closers + if len(closer) > len(prefix) and closer.startswith(prefix) + } + excluded = "".join(map(_class_escape, sorted(starts | nexts))) + branches.append(f"{_escape(prefix)}[^{excluded}]") + return {"type": "regex", "pattern": "|".join(branches)} + + +def _class_escape(char: str) -> str: + if char in _BLANKS: + return _BLANKS[char] + return "\\" + char if char in "\\]^-[" else char + + +def message_start_anchors(template: ResponseTemplate) -> list[str]: + """The strings the assistant message starts after, if literal.""" + if template.start_anchor_literals: + return template.start_anchor_literals + return _literals(template.start_anchor_re.pattern) or [] + + +def _closers(field: ResponseTemplateField) -> list[str] | None: + """Return literal closers, an empty list if absent, or `None` if underivable.""" + if field.close_re is None: + return [] + return field.close_literals or _literals(field.close_re.pattern) + + +# Every request derives the tags for its tools anew. +@lru_cache(maxsize=512) +def _endings(literals: tuple[str, ...]) -> str: + return "|".join( + re.escape(literal[index:]) + for literal in literals + for index in range(len(literal)) + ) + + +def _literal_piece(literals: list[str], before: str = "") -> tuple[str, str, bool]: + either = before + "(?:" + "|".join(map(re.escape, literals)) + ")" + return either, f"{either}|{_endings(tuple(literals))}", True + + +@lru_cache(maxsize=512) +def _spells_closer( + closers: tuple[str, ...], + pieces: tuple[tuple[str, str, bool], ...], + *, + repeats: bool = False, +) -> bool: + """Check whether argument framing can spell a call closer across pieces.""" + texts = [text for text, _, _ in pieces] + loop = f"(?:{''.join(texts)})*" if repeats else "" + # Another call or an answer may follow the call's own closer. + closed = "(?:" + "|".join(map(re.escape, closers)) + r")[\s\S]*" + for index, (_, endings, holds) in enumerate(pieces): + ends = re.compile(endings) + rest = re.compile("".join(texts[index + 1 :]) + loop + closed, re.DOTALL) + for closer in closers: + if holds and ends.fullmatch(closer, partial=True): + return True + # A closer that starts in this piece and ends in the text after it. + for split in range(1, len(closer)): + if ends.fullmatch(closer[:split]) and rest.fullmatch( + closer[split:], partial=True + ): + return True + return False + + +def _const_strings(values: list[str]) -> dict[str, Any]: + return _one_of([{"type": "const_string", "value": value} for value in values]) + + +def _sequence(*elements: dict[str, Any]) -> dict[str, Any]: + return {"type": "sequence", "elements": list(elements)} + + +def _one_of(formats: list[dict[str, Any]]) -> dict[str, Any]: + return formats[0] if len(formats) == 1 else {"type": "or", "elements": formats} + + +def _optional(content: dict[str, Any]) -> dict[str, Any]: + return {"type": "optional", "content": content} + + +def _tag(begin: str, content: dict[str, Any], end: str | list[str]) -> dict[str, Any]: + return {"type": "tag", "begin": begin, "content": content, "end": end} + + +def _json_value_schema(schema: Any, parameters: dict[str, Any]) -> dict[str, Any]: + if not isinstance(schema, dict): + return {} + if schema.get("nullable"): + # XGrammar ignores `nullable`. + schema = {"anyOf": [schema, {"type": "null"}]} + definitions = { + key: parameters[key] for key in ("$defs", "definitions") if key in parameters + } + return {**definitions, **schema} + + +def _string_choices(schema: Any) -> list[str] | None: + if not isinstance(schema, dict): + return None + choices = [schema["const"]] if "const" in schema else schema.get("enum") + if not isinstance(choices, list) or not choices: + return None + return choices if all(isinstance(choice, str) for choice in choices) else None + + +def _typed_by_parser(schema: Any) -> bool: + """Whether the parser reads schema values with their JSON type.""" + if schema is True or _schema_types(schema): + return True + return isinstance(schema, dict) and ( + not set(schema) - _ANNOTATIONS or _string_choices(schema) is not None + ) + + +def _text_format(schema: Any, excludes: list[str]) -> dict[str, Any] | None: + """Constrain raw text, which the `excludes` end, without changing its + parsed value.""" + if schema is True: + return _ANY_TEXT + if isinstance(schema, dict) and {"format", "pattern"} & set(schema): + return _string_regex(schema, excludes) + if not isinstance(schema, dict) or set(schema) - _ANNOTATIONS - { + "type", + "nullable", + "enum", + "const", + "anyOf", + "oneOf", + }: + return None + branches = [*(schema.get("anyOf") or []), *(schema.get("oneOf") or [])] + if "enum" in schema or "const" in schema: + choices = _string_choices(schema) + if choices is None or branches or set(_schema_types(schema)) - {"string"}: + return None + return _const_strings(choices) + if any(_text_format(branch, excludes) is not _ANY_TEXT for branch in branches): + return None + return _ANY_TEXT + + +def _string_regex(schema: dict[str, Any], excludes: list[str]) -> dict[str, Any] | None: + """Hold the `format` or `pattern` of a raw string. Like XGrammar, a format + overrides a pattern, either one drops length bounds, and the pattern + matches the whole string.""" + if set(schema) - _ANNOTATIONS - { + "type", + "format", + "pattern", + "minLength", + "maxLength", + } or _schema_types(schema) != ("string",): + return None + pattern = ( + _STRING_FORMATS.get(schema["format"]) + if "format" in schema + else schema["pattern"] + ) + if ( + not isinstance(pattern, str) + or _may_write(pattern, {text[0] for text in excludes if text}) + or not _regex_compiles(pattern) + ): + return None + return {"type": "regex", "pattern": pattern} + + +def _may_write(pattern: str, chars: set[str]) -> bool: + """Whether a match of `pattern` may hold one of `chars`, as any pattern may + that Python cannot parse.""" + try: + parsed = _sre_parse.parse(pattern) + except (_sre_parse.error, OverflowError, RecursionError): + return True + if parsed.state.flags & _sre_parse.SRE_FLAG_IGNORECASE: + return True + + def may(items: Any) -> bool: + for op, arg in items: + if op is _sre_parse.LITERAL: + written = {chr(arg)} + elif op is _sre_parse.IN: + written = {char for char in chars if _in_class(arg, char)} + elif op in _REPEATS: + written = chars if arg[1] and may(arg[2]) else set() + elif op is _sre_parse.SUBPATTERN: + if arg[1] & _sre_parse.SRE_FLAG_IGNORECASE: + return True + written = chars if may(arg[3]) else set() + elif op is _sre_parse.BRANCH: + written = chars if any(map(may, arg[1])) else set() + elif op is _sre_parse.AT: + continue + else: + # Such as `.`, which XGrammar lets match line breaks. + return True + if written & chars: + return True + return False + + return may(parsed) + + +def _in_class(items: list[tuple[Any, Any]], char: str) -> bool: + negated = bool(items) and items[0][0] is _sre_parse.NEGATE + matched = False + for op, arg in items[negated:]: + if op is _sre_parse.LITERAL: + matched |= ord(char) == arg + elif op is _sre_parse.RANGE: + matched |= arg[0] <= ord(char) <= arg[1] + elif op is _sre_parse.CATEGORY and arg in _CATEGORIES: + matched |= bool(re.fullmatch(_CATEGORIES[arg], char)) + else: + return True + return matched != negated + + +# Every request derives the tags for its tools anew. +@lru_cache(maxsize=512) +def _regex_compiles(pattern: str) -> bool: + from xgrammar import Grammar + + try: + Grammar.from_regex(pattern) + except RuntimeError: + return False + return True + + +def _value_format( + schema: Any, parameters: dict[str, Any], json_values: bool, closers: list[str] +) -> dict[str, Any] | None: + """Use JSON for decoded values and raw text for string values.""" + if schema is False: + return None + types = _schema_types(schema) + if json_values or (types and "string" not in types): + return _json_format(_json_value_schema(schema, parameters), closers) + return _text_format(schema, closers) + + +def _json_format(schema: Any, closers: list[str]) -> dict[str, Any] | None: + """Use JSON unless a closer is ambiguous with its syntax or the schema is + unsupported. Strings hold no closer, as the parser ends the call there.""" + if closers and ( + any(set(closer) <= _JSON_SYNTAX for closer in closers) + or not _json_compiles(json.dumps(schema)) + ): + return None + return {"type": "json_schema", "json_schema": schema, "excludes": closers} + + +# Every request derives the tags for its tools anew. +@lru_cache(maxsize=512) +def _json_compiles(schema: str) -> bool: + from xgrammar import Grammar + + try: + Grammar.from_json_schema(schema) + except RuntimeError: + return False + return True + + +def _xml_arguments_format( + content_args: dict[str, Any], + parameters: dict[str, Any], + *, + strict: bool, + closers: list[str], + sorted_arguments: bool, +) -> dict[str, Any] | None: + if set(content_args) - {"tag_pattern", "value_parser", "merge_duplicates"}: + return None + value_parser = content_args.get("value_parser") or {"name": "text"} + if value_parser.get("name") not in ("text", "json") or ( + value_parser.get("name") == "json" and value_parser.get("args") + ): + return None + key_split = _split_group(content_args["tag_pattern"], "key") + value_split = key_split and _split_group(key_split[2], "value") + if not key_split or not value_split: + return None + key_pattern = key_split[1] + begins, middles = _literals(key_split[0]), _literals(value_split[0]) + ends = _literals(value_split[2], lookahead=True) + if not begins or not middles or not ends: + return None + aheads = [end.split(_AHEAD) for end in ends if _AHEAD in end] + ends = [end for end in ends if _AHEAD not in end] + # A value may also end where the next parameter opens, if the template + # looks ahead for its only opener. + if not ends or ( + aheads and (len(begins) > 1 or {a for _, a in aheads} != {*begins}) + ): + return None + follows = [before + ahead for before, ahead in aheads] + properties = parameters.get("properties") + json_values = value_parser.get("name") == "json" + typed = properties is None or ( + isinstance(properties, dict) + and (json_values or all(map(_typed_by_parser, properties.values()))) + ) + # Other schema shapes cannot be typed by the XML parser. + if strict and (set(parameters) - _ANNOTATIONS - _OBJECT_KEYWORDS or not typed): + return None + required = parameters.get("required") or [] + names: tuple[str, ...] = () + if strict and properties is not None: + # Required entries that are no names leave the tool to the JSON schema. + if not isinstance(required, list) or not all( + isinstance(key, str) for key in required + ): + return None + names = (*properties, *(key for key in required if key not in properties)) + pieces = ( + _literal_piece(begins, before=_SEPARATOR["pattern"]), + (f"(?:{key_pattern})", _endings(names), False), + _literal_piece(middles), + # Values of every format are some text. + _TEXT_PIECE, + _literal_piece(ends), + ) + if _spells_closer(tuple(closers), pieces, repeats=True): + return None + + # Raw text has no escaping for a call closer, nor, where a value may end + # before the next opener, for any value end. + excludes = closers + follows + ends if follows else closers + text = {"type": "any_text", "excludes": excludes} if excludes else _ANY_TEXT + # Each value but the last then ends by opening the next parameter, whose + # tag starts after the opener. + starts = [""] if follows else begins + + def tag( + begin: str, value: dict[str, Any], fits: list[str], last: bool + ) -> dict[str, Any]: + if follows and not last: + fits = [e + sep + begins[0] for e in fits for sep in ("", "\n")] + follows + return _tag(begin, value, fits) + + def separated(tags: list[dict[str, Any]]) -> dict[str, Any]: + return _one_of(tags) if follows else _sequence(_SEPARATOR, _one_of(tags)) + + def arguments(body: dict[str, Any], optional: bool) -> dict[str, Any]: + """`body` after the opener of its first parameter.""" + if not follows: + return body + body = _sequence(_SEPARATOR, _const_strings(begins), body) + return _optional(body) if optional else body + + def any_parameter(last: bool) -> dict[str, Any]: + name = {"type": "regex", "pattern": key_pattern} + content = _sequence(name, _const_strings(middles), text) + return separated([tag(start, content, ends, last) for start in starts]) + + def any_parameters() -> dict[str, Any]: + more = {"type": "star", "content": any_parameter(False)} + return ( + arguments(_sequence(more, any_parameter(True)), True) if follows else more + ) + + if not strict: + return any_parameters() + if properties is None: + extra = parameters.get("additionalProperties", True) + if extra is False: + return {"type": "const_string", "value": ""} + return any_parameters() if extra is True or extra == {} else None + # A declared key repeated as an extra one would bypass its schema. + if parameters.get("additionalProperties", False) is not False: + return None + + def read_back(written: str) -> dict[str, Any] | None: + """The arguments the parser reads from `written`, unless a closer ends + the call in it.""" + if any(closer in written for closer in closers): + return None + try: + return parse_content(written, "xml-inline", content_args) + except ValueError: + return None + + def parameter( + key: str, value: dict[str, Any], choices: list[str] | None, last: bool + ) -> dict[str, Any] | None: + if not re.fullmatch(key_pattern, key) or any( + list(read_back(b + key + m + "0" + e) or ()) != [key] + for b in begins + for m in middles + for e in ends + ): + raise ValueError(f"parameter name {key!r} cannot be encoded in tool calls") + if choices is None: + return separated( + [tag(s + key + m, value, ends, last) for s in starts for m in middles] + ) + # Optional framing whitespace must not change a text choice. + tags, framed = [], set() + for b, s in zip(begins, starts): + for m in middles: + by_ends: dict[tuple[str, ...], list[str]] = {} + for choice in choices: + fits = tuple( + e + for e in ends + if read_back(b + key + m + choice + e) == {key: choice} + ) + by_ends.setdefault(fits, []).append(choice) + for fits, group in by_ends.items(): + if fits: + tags.append( + tag(s + key + m, _const_strings(group), list(fits), last) + ) + framed.update(group) + if framed != set(choices): + return None + return separated(tags) + + if not names: + return {"type": "const_string", "value": ""} + more: dict[str, dict[str, Any]] = {} + final: dict[str, dict[str, Any]] = {} + for key in names: + # A required name without a schema takes any value. + schema = properties.get(key, True) + value = _value_format(schema, parameters, json_values, closers + follows + ends) + if value is None: + return None + choices = None + if value["type"] in ("const_string", "or"): + choices = _string_choices(schema) + value = text if value == _ANY_TEXT else value + joined = parameter(key, value, choices, last=False) + ending = parameter(key, value, choices, last=True) + if joined is None or ending is None: + return None + more[key], final[key] = joined, ending + + def ordered(keys: tuple[str, ...]) -> dict[str, Any]: + before = [more[k] if k in required else _optional(more[k]) for k in keys] + if not follows: + return _sequence(*before) + + def last(start: int, stop: int) -> dict[str, Any]: + """Parameters of `keys[start:stop]` that end at one of them.""" + if stop - start == 1: + return final[keys[start]] + # Halving keeps the tag O(n log n) in size and O(log n) deep. + middle = (start + stop) // 2 + return _one_of( + [ + last(start, middle), + _sequence(*before[start:middle], last(middle, stop)), + ] + ) + + # The last parameter follows every required one. + first = max((i for i, k in enumerate(keys) if k in required), default=0) + return _sequence(*before[:first], last(first, len(keys))) + + orders = [names] + if sorted_arguments and sorted(names) != list(names): + orders.append(tuple(sorted(names))) + return arguments(_one_of([ordered(keys) for keys in orders]), not required) + + +def _arguments_format( + field: ResponseTemplateField, + parameters: dict[str, Any], + *, + strict: bool, + sorted_arguments: bool = False, +) -> dict[str, Any] | None: + closers = _closers(field) or [] + if field.content == "json" and not field.content_args: + if _spells_closer(tuple(closers), (_JSON_PIECE,)): + return None + schema = parameters if strict else {"type": "object"} + return _json_format(schema, closers) + if field.content == "xml-inline": + return _xml_arguments_format( + field.content_args, + parameters, + strict=strict, + closers=closers, + sorted_arguments=sorted_arguments, + ) + return None + + +def _names_function_by_opener(field: ResponseTemplateField) -> bool: + function = field.transform.get("function") if field.transform else None + return ( + field.open_re is not None + and "name" in field.open_re.groupindex + and not field.transform_each + and isinstance(function, dict) + and function.get("name") == "{name}" + and function.get("arguments") == "{content}" + ) + + +def _reply_format( + answer: ResponseTemplateField, + message_start: list[str] | None, + content: dict[str, Any] | None, +) -> dict[str, Any] | None: + """The `answer` field holding `content`, or free text if not given, or + `None` if it cannot be derived.""" + openers, closers = _openers(answer, message_start), _closers(answer) + if not openers or closers is None: + return None + return _sequence(_const_strings(openers), *_answer_text(closers, content)) + + +def _answer_text( + closers: list[str], content: dict[str, Any] | None = None +) -> list[dict[str, Any]]: + # Text cannot close the answer early, so no unconstrained call can follow + # it. Like a call, the answer may end at a stop token instead of a closer. + elements = [content or {"type": "any_text", "excludes": closers}] + if closers: + elements.append(_optional(_const_strings(closers))) + return elements + + +def answer_rest_structural_tag(answer: ResponseTemplateField) -> dict[str, Any] | None: + """The rest of an `answer` that is already open, or `None` if underivable.""" + closers = _closers(answer) + if closers is None: + return None + return {"type": "structural_tag", "format": _sequence(*_answer_text(closers))} + + +def tool_call_structural_tag( + field: ResponseTemplateField | None, + functions: list[dict[str, Any]], + *, + parallel_tool_calls: bool, + thinking: ResponseTemplateField | None = None, + answer: ResponseTemplateField | None = None, + answer_format: dict[str, Any] | None = None, + message_start: list[str] | None = None, + requires_reasoning: bool = False, + sorted_arguments: bool = False, + strict_arguments: bool = False, +) -> dict[str, Any] | None: + """Derive native calls to `functions` in the format of the tool call + `field`, respecting strict arguments, count and reasoning. + + `functions` are OpenAI function definitions. `strict_arguments` enforces + the schema of every function's arguments where the template can. Strict + `xml-inline` arguments come in declared order, or also alphabetical with + `sorted_arguments`. An `answer` field lets the model answer instead, in + `answer_format` if given; without `functions`, only the answer is derived. + When the model may answer, calls may also name any other function the + template can spell, with any arguments, so a harness can tell the model + the function does not exist. At the `message_start`, + the tag covers the output from its first token, which opens with non-empty + reasoning if `requires_reasoning`. Returns `None` if the template format + cannot be derived, and raises `ValueError` for names it cannot encode.""" + if not functions and answer is not None: + + def reply(start: list[str] | None) -> list[dict[str, Any]] | None: + format = _reply_format(answer, start, answer_format) + return None if format is None else [format] + + return _structural_tag(reply, thinking, message_start, requires_reasoning) + if not functions or field is None or not _names_function_by_opener(field): + return None + opener = _split_group(field.open_re.pattern, "name") + closer_literals = _closers(field) + if opener is None or closer_literals is None: + return None + arguments, enforced = [], [] + for function in functions: + # Like the generic JSON schema, a function without parameters takes none. + parameters = function.get("parameters") or {"type": "object", "properties": {}} + strict = bool(function.get("strict")) + function_arguments = None + if strict or strict_arguments: + try: + function_arguments = _arguments_format( + field, parameters, strict=True, sorted_arguments=sorted_arguments + ) + except ValueError: + if strict: + raise + enforced.append(function_arguments is not None) + if function_arguments is None and not strict: + function_arguments = _arguments_format(field, parameters, strict=False) + if function_arguments is None: + return None + arguments.append(function_arguments) + names = [] + for function in functions: + name = function["name"] + if not re.fullmatch(opener[1], name): + raise ValueError(f"tool name {name!r} cannot be encoded in tool calls") + names.append(_escape(name)) + if answer is not None: + others = _other_names( + opener[1], [f["name"] for f, e in zip(functions, enforced) if e] + ) + loose = _arguments_format(field, {}, strict=False) + if others is not None and loose is not None: + # Calls to any function whose schema is not enforced take any arguments. + names = [name for name, e in zip(names, enforced) if e] + [f"(?:{others})"] + arguments = [args for args, e in zip(arguments, enforced) if e] + [loose] + closers = _const_strings(closer_literals) if closer_literals else None + + def call(start: list[str] | None) -> dict[str, Any]: + return _one_of( + [ + _sequence( + {"type": "regex", "pattern": _call_opener(opener, name, start)}, + function_arguments, + ) + for name, function_arguments in zip(names, arguments) + ] + ) + + def output(start: list[str] | None) -> list[dict[str, Any]] | None: + elements = [call(start)] + if parallel_tool_calls and field.repeats: + repeated = _sequence(closers, call(None)) if closers else call(None) + elements.append({"type": "star", "content": repeated}) + if closers: + # EOS can terminate a call without writing its closer. + elements.append(_optional(closers)) + if answer is not None: + reply = _reply_format(answer, start, answer_format) + if reply is None: + return None + elements = [_one_of([_sequence(*elements), reply])] + return elements + + return _structural_tag(output, thinking, message_start, requires_reasoning) + + +def _structural_tag( + output: Callable[[list[str] | None], list[dict[str, Any]] | None], + thinking: ResponseTemplateField | None, + message_start: list[str] | None, + requires_reasoning: bool = False, +) -> dict[str, Any] | None: + """Frame output after reasoning, from the first token at the + `message_start`. The reasoning is optional unless `requires_reasoning` + makes the message open with non-empty reasoning.""" + rest = output(None) + if rest is None: + return None + think = None + if thinking is not None: + think = _thinking_format(thinking, message_start, requires_reasoning) + if think is None: + return None + if message_start is None: + if think is not None: + rest.insert(0, _optional(think)) + return {"type": "structural_tag", "format": _sequence(*rest)} + if think is not None and requires_reasoning: + return {"type": "structural_tag", "format": _sequence(think, *rest)} + start = output(message_start) + if start is None: + return None + format = _sequence(*start) + if think is not None: + format = _one_of([_sequence(think, *rest), format]) + return {"type": "structural_tag", "format": format} diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index 9e75e9b77e..796fc3ba3f 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -84,6 +84,10 @@ _REASONING_PARSERS_TO_REGISTER = { "response_template_reasoning_parser", "ResponseTemplateReasoningParser", ), + "beam": ( + "beam_reasoning_parser", + "BeamReasoningParser", + ), "holo2": ( "deepseek_v3_reasoning_parser", "DeepSeekV3ReasoningWithThinkingParser", diff --git a/vllm/reasoning/beam_reasoning_parser.py b/vllm/reasoning/beam_reasoning_parser.py new file mode 100644 index 0000000000..b5d87170f0 --- /dev/null +++ b/vllm/reasoning/beam_reasoning_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.beam import BeamReasoningParser + +__all__ = ["BeamReasoningParser"] diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index babd0ff6e5..65771211d0 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -60,6 +60,7 @@ from vllm.renderers.chat_utils import ( has_non_text_content, ) from vllm.renderers.inputs.preprocess import ( + extract_prompt_components, parse_model_prompt, prompt_to_seq, ) @@ -844,13 +845,17 @@ class OnlineRenderer: f"but got {type(request).__name__}" ) raise NotImplementedError(msg) - request = parser( + request_parser = parser( tokenizer, request.tools, model_config=self.model_config, chat_template_kwargs=chat_params.chat_template_kwargs, - ).adjust_request( - request=request, ) + prompt_token_ids = extract_prompt_components( + self.model_config, engine_input + ).token_ids + if prompt_token_ids is not None: + request_parser.set_prompt_token_ids(prompt_token_ids) + request = request_parser.adjust_request(request=request) return conversation, [engine_input] diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index 4b9cb33502..9e2e4a6d9f 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -90,6 +90,10 @@ _TOOL_PARSERS_TO_REGISTER = { "response_template_tool_parser", "ResponseTemplateToolParser", ), + "beam": ( + "beam_tool_parser", + "BeamToolParser", + ), "poolside_v1": ( "poolside_v1_tool_parser", "PoolsideV1ToolParser", diff --git a/vllm/tool_parsers/beam_tool_parser.py b/vllm/tool_parsers/beam_tool_parser.py new file mode 100644 index 0000000000..dbfd820688 --- /dev/null +++ b/vllm/tool_parsers/beam_tool_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.beam import BeamToolParser + +__all__ = ["BeamToolParser"]