From 24fb5fc08c2bcae8575bd2e861f463d8cf919ffb Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 1 Oct 2026 16:16:38 +0800 Subject: [PATCH] [Rust Frontend] Add `hf` parser for response templates (#59005) Signed-off-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 --- rust/Cargo.lock | 4 + rust/Cargo.toml | 3 + rust/src/chat/src/backend/hf.rs | 56 +- rust/src/chat/src/lib.rs | 10 +- rust/src/chat/src/output/default/mod.rs | 150 ++- rust/src/chat/src/parser/unified.rs | 11 +- rust/src/chat/tests/roundtrip.rs | 10 + rust/src/parser/Cargo.toml | 4 + rust/src/parser/src/tool/mod.rs | 1 + rust/src/parser/src/tool/parameters.rs | 32 +- rust/src/parser/src/unified/combined.rs | 5 +- rust/src/parser/src/unified/hf/content.rs | 537 ++++++++++ .../src/unified/hf/fixtures/differential.json | 440 +++++++++ .../hf/fixtures/generate_differential.py | 264 +++++ rust/src/parser/src/unified/hf/mod.rs | 53 + rust/src/parser/src/unified/hf/parser.rs | 730 ++++++++++++++ rust/src/parser/src/unified/hf/pattern.rs | 403 ++++++++ rust/src/parser/src/unified/hf/spec.rs | 202 ++++ rust/src/parser/src/unified/hf/template.rs | 481 +++++++++ rust/src/parser/src/unified/hf/tests.rs | 933 ++++++++++++++++++ rust/src/parser/src/unified/hf/transform.rs | 263 +++++ rust/src/parser/src/unified/mod.rs | 11 +- rust/src/text/src/backend/hf/config.rs | 3 + 23 files changed, 4573 insertions(+), 33 deletions(-) create mode 100644 rust/src/parser/src/unified/hf/content.rs create mode 100644 rust/src/parser/src/unified/hf/fixtures/differential.json create mode 100644 rust/src/parser/src/unified/hf/fixtures/generate_differential.py create mode 100644 rust/src/parser/src/unified/hf/mod.rs create mode 100644 rust/src/parser/src/unified/hf/parser.rs create mode 100644 rust/src/parser/src/unified/hf/pattern.rs create mode 100644 rust/src/parser/src/unified/hf/spec.rs create mode 100644 rust/src/parser/src/unified/hf/template.rs create mode 100644 rust/src/parser/src/unified/hf/tests.rs create mode 100644 rust/src/parser/src/unified/hf/transform.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index bd5a63e81d..2ba9832b6b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5971,12 +5971,16 @@ dependencies = [ "expect-test", "futures", "openai-protocol", + "regex-automata", + "regex-syntax", "serde", "serde_json", + "serde_path_to_error", "serde_with", "thiserror 2.0.18", "thiserror-ext", "tool-parser", + "tracing", "vllm-tokenizer", "winnow", "xgrammar-structural-tag", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index a2e6d0c1ed..e9935d9e28 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -85,6 +85,8 @@ rand = "0.9.3" rand_distr = "0.5.1" rayon = "1.11.0" reasoning-parser = "1.2.2" +regex-automata = "0.4.14" +regex-syntax = "0.8.10" reqwest = { version = "0.12.8", default-features = false, features = ["native-tls"] } reqwest-0-13 = { package = "reqwest", version = "0.13.4", default-features = false, features = ["native-tls"] } riptoken = { version = "0.3.0", default-features = false } @@ -96,6 +98,7 @@ serde = { version = "1.0.228", features = ["derive", "rc"] } serde-json-fmt = "0.1.0" serde_default = "0.2.0" serde_json = { version = "1.0.145", features = ["preserve_order"] } +serde_path_to_error = "0.1.20" serde_repr = "0.1.20" serde_tuple = "1.1.3" serde_with = "3.21.0" diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index 07f0b2dce8..174115693a 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -3,8 +3,11 @@ use std::sync::Arc; -use tracing::info; -use vllm_text::backend::hf::{HfTextBackend, ResolvedModelFiles, load_model_config}; +use thiserror_ext::AsReport as _; +use tracing::{info, warn}; +use vllm_text::backend::hf::{ + HfTextBackend, ResolvedModelFiles, load_model_config, load_tokenizer_config, +}; use vllm_text::tokenizer::DynTokenizer; use vllm_text::{DynTextBackend, TextBackend as _}; @@ -17,6 +20,7 @@ use crate::multimodal::{MultimodalConfigFiles, MultimodalModelInfo}; use crate::output::{ DefaultChatOutputProcessor, HarmonyChatOutputProcessor, validate_harmony_parser_overrides, }; +use crate::parser::unified::{HfTemplateError, ResponseTemplate}; use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo}; use crate::renderer::{ DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer, DynChatRenderer, @@ -32,6 +36,9 @@ pub struct HfChatBackend { tokenizer: DynTokenizer, chat_renderer: DynChatRenderer, multimodal_model_info: Option, + /// `response_template` from `tokenizer_config.json` for the `hf` parser, or + /// why it is unavailable. + response_template: std::result::Result, HfTemplateError>, } impl HfChatBackend { @@ -61,6 +68,7 @@ impl HfChatBackend { )? }; let multimodal_render_info = resolve_multimodal_render_info(multimodal_model_info.as_ref()); + let response_template = load_response_template(&files)?; let renderer = options.renderer.resolve(model_type); let chat_renderer: DynChatRenderer = match renderer { @@ -105,10 +113,35 @@ impl HfChatBackend { tokenizer, chat_renderer, multimodal_model_info, + response_template, }) } } +/// Load and compile the checkpoint's `response_template`, if any. +/// +/// A missing or unusable template is kept as an error, so that only requests +/// selecting the `hf` parser fail. +fn load_response_template( + files: &ResolvedModelFiles, +) -> Result, HfTemplateError>> { + let Some(template) = + load_tokenizer_config(files.tokenizer_config_path.as_deref())?.response_template + else { + return Ok(Err(HfTemplateError::Missing)); + }; + Ok(match ResponseTemplate::from_json(&template) { + Ok(template) => { + info!("loaded response_template for the `hf` parser"); + Ok(Arc::new(template)) + } + Err(error) => { + warn!(error = %error.as_report(), "the `hf` parser cannot execute the model's response_template"); + Err(error) + } + }) +} + impl ChatBackend for HfChatBackend { fn chat_renderer(&self) -> DynChatRenderer { self.chat_renderer.clone() @@ -128,14 +161,17 @@ impl ChatBackend for HfChatBackend { return Ok(Box::new(HarmonyChatOutputProcessor::new(request)?)); } - Ok(Box::new(DefaultChatOutputProcessor::new( - request, - &self.model_id, - self.tokenizer.clone(), - options.tool_call_parser, - options.reasoning_parser, - options.tool_strict_level, - )?)) + Ok(Box::new( + DefaultChatOutputProcessor::with_response_template( + request, + &self.model_id, + self.tokenizer.clone(), + &self.response_template, + options.tool_call_parser, + options.reasoning_parser, + options.tool_strict_level, + )?, + )) } } diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 86b94b6b9c..f437745e39 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -412,6 +412,12 @@ mod tests { validate_parser_overrides(&selection, &selection).unwrap(); } + #[test] + fn validate_parser_overrides_accepts_explicit_hf() { + let selection = ParserSelection::Explicit("hf".to_string()); + validate_parser_overrides(&selection, &selection).unwrap(); + } + #[test] fn validate_parser_overrides_accepts_auto_and_none() { validate_parser_overrides(&ParserSelection::Auto, &ParserSelection::None).unwrap(); @@ -425,7 +431,7 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string()); + expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hf, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string()); } #[test] @@ -436,6 +442,6 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); + expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hf, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); } } diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index ee067764cf..3f75aed63d 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -5,7 +5,7 @@ mod unified; -use std::sync::Once; +use std::sync::{Arc, Once}; use futures::StreamExt as _; use tracing::info; @@ -20,7 +20,9 @@ use crate::error::Result; use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream}; use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory}; use crate::parser::tool::{ToolParser, ToolParserFactory}; -use crate::parser::unified::UnifiedParserFactory; +use crate::parser::unified::{ + HfTemplateError, HfUnifiedParser, ResponseTemplate, UnifiedParserFactory, names, +}; use crate::parser::{ParserSelection, ToolStrictLevel}; use crate::request::{ChatRequest, ChatTool}; use crate::{Error, Result as ChatResult}; @@ -64,11 +66,48 @@ impl DefaultChatOutputProcessor { reasoning_parser: &ParserSelection, tool_strict_level: ToolStrictLevel, ) -> ChatResult { + Self::with_response_template( + request, + model_id, + tokenizer, + &Err(HfTemplateError::Missing), + tool_call_parser, + reasoning_parser, + tool_strict_level, + ) + } + + /// Like [`Self::new`], additionally providing the model's `response_template` + /// for the `hf` parser, or why it is unavailable. + pub fn with_response_template( + request: &mut ChatRequest, + model_id: &str, + tokenizer: DynTokenizer, + response_template: &std::result::Result, HfTemplateError>, + tool_call_parser: &ParserSelection, + reasoning_parser: &ParserSelection, + tool_strict_level: ToolStrictLevel, + ) -> ChatResult { + let tool_name = tool_call_parser.resolve_tool_name(model_id); + let reasoning_name = reasoning_parser.resolve_reasoning_name(model_id); + // `hf` is opt-in: `Auto` never selects it, but a usable template is worth a hint. + let auto = [tool_call_parser, reasoning_parser] + .iter() + .any(|selection| matches!(selection, ParserSelection::Auto)); + if auto && tool_name.is_none() && reasoning_name.is_none() && response_template.is_ok() { + RESPONSE_TEMPLATE_HINT_ONCE.call_once(|| { + info!( + "the model ships a response_template; pass `--tool-call-parser hf \ + --reasoning-parser hf` to parse with it" + ); + }); + } let parser = if let Some(parser) = Self::resolve_optional_unified_parser( request.tools(), tokenizer.clone(), - tool_call_parser.resolve_tool_name(model_id), - reasoning_parser.resolve_reasoning_name(model_id), + response_template, + tool_name, + reasoning_name, )? { parser } else { @@ -141,6 +180,7 @@ impl DefaultChatOutputProcessor { fn resolve_optional_unified_parser( tools: &[ChatTool], tokenizer: DynTokenizer, + response_template: &std::result::Result, HfTemplateError>, tool_name: Option<&str>, reasoning_name: Option<&str>, ) -> ChatResult>> { @@ -157,7 +197,18 @@ impl DefaultChatOutputProcessor { }); } - let parser = factory.create(parser_name, tools, tokenizer)?; + // `hf` is built from the model's template, never by its registry constructor. + let parser = if parser_name == names::HF { + let template = + response_template.as_ref().map_err(|error| Error::ParserInitialization { + kind: "unified", + name: parser_name.to_string(), + error: Box::new(error.clone()), + })?; + Box::new(HfUnifiedParser::new(template.clone(), tools, tokenizer)) + } else { + factory.create(parser_name, tools, tokenizer)? + }; UNIFIED_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using unified parser")); Ok(Some(parser)) @@ -186,6 +237,7 @@ impl DefaultChatOutputProcessor { static TOOL_PARSER_LOG_ONCE: Once = Once::new(); static REASONING_PARSER_LOG_ONCE: Once = Once::new(); static UNIFIED_PARSER_LOG_ONCE: Once = Once::new(); +static RESPONSE_TEMPLATE_HINT_ONCE: Once = Once::new(); impl ChatOutputProcessor for DefaultChatOutputProcessor { fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { @@ -230,10 +282,12 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor { mod tests { use std::sync::Arc; + use thiserror_ext::AsReport as _; use vllm_tokenizer::test_utils::TestTokenizer; use super::DefaultChatOutputProcessor; use crate::output::ChatOutputProcessor; + use crate::parser::unified::{HfTemplateError, ResponseTemplate}; use crate::parser::{ParserSelection, ToolStrictLevel}; use crate::request::{ChatRequest, ChatTool, ChatToolChoice, ResolvedToolContext}; @@ -390,4 +444,90 @@ mod tests { expect_test::expect!["unified parsing requires the tool and reasoning selections to resolve to the same parser; resolved tool=qwen3_xml, reasoning=gemma4"] .assert_eq(&format!("{error}")); } + + type TemplateResult = Result, HfTemplateError>; + + fn loaded_template() -> TemplateResult { + Ok(Arc::new( + ResponseTemplate::from_json(&serde_json::json!({ + "start_anchor": "<|turn>model\n", + "fields": { + "thinking": {"open": "<|channel>thought\n", "close": ""}, + "content": {"close": ""}, + }, + })) + .unwrap(), + )) + } + + fn invalid_template() -> TemplateResult { + let error = ResponseTemplate::from_json(&serde_json::json!({"fields": {"content": {}}})) + .unwrap_err(); + Err(error) + } + + /// Build a processor for `model_id`, returning whether it keeps special + /// tokens (only parsers that need them, such as `hf`, turn this on). + fn keeps_special_tokens( + model_id: &str, + template: &TemplateResult, + tool: &ParserSelection, + reasoning: &ParserSelection, + ) -> crate::Result { + let mut request = ChatRequest::for_test(); + DefaultChatOutputProcessor::with_response_template( + &mut request, + model_id, + tokenizer(), + template, + tool, + reasoning, + ToolStrictLevel::Auto, + )?; + Ok(!request.decode_options.skip_special_tokens) + } + + #[test] + fn explicit_hf_uses_the_model_response_template() { + let hf = ParserSelection::Explicit("hf".to_string()); + assert!(keeps_special_tokens("other-model", &loaded_template(), &hf, &hf).unwrap()); + } + + #[test] + fn explicit_hf_without_usable_template_fails() { + let hf = ParserSelection::Explicit("hf".to_string()); + let error = |template| { + keeps_special_tokens("other-model", &template, &hf, &hf) + .unwrap_err() + .to_report_string() + }; + expect_test::expect!["failed to initialize unified parser `hf`: the model's tokenizer_config.json provides no response_template"] + .assert_eq(&error(Err(HfTemplateError::Missing))); + expect_test::expect!["failed to initialize unified parser `hf`: invalid response_template: response_template must define 'start_anchor' or 'start_anchor_pattern'."] + .assert_eq(&error(invalid_template())); + } + + #[test] + fn auto_never_selects_hf() { + let auto = ParserSelection::Auto; + // Without a matching parser, parsing stays disabled even with a usable template. + assert!(!keeps_special_tokens("other-model", &loaded_template(), &auto, &auto).unwrap()); + // An unusable template is not consulted unless `hf` is selected. + assert!(!keeps_special_tokens("other-model", &invalid_template(), &auto, &auto).unwrap()); + keeps_special_tokens("google/gemma-4-27b-it", &invalid_template(), &auto, &auto).unwrap(); + } + + #[test] + fn hf_requires_matching_parser_selections() { + let error = keeps_special_tokens( + "other-model", + &loaded_template(), + &ParserSelection::Explicit("hf".to_string()), + &ParserSelection::None, + ) + .unwrap_err(); + + expect_test::expect!["unified parsing requires the tool and reasoning selections to resolve to the same parser; resolved tool=hf, reasoning=none"] + .assert_eq(&format!("{error}")); + } } diff --git a/rust/src/chat/src/parser/unified.rs b/rust/src/chat/src/parser/unified.rs index 52cabaf162..7db093c696 100644 --- a/rust/src/chat/src/parser/unified.rs +++ b/rust/src/chat/src/parser/unified.rs @@ -6,8 +6,8 @@ use std::sync::LazyLock; pub use vllm_parser::unified::{ - Gemma4UnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser, InklingUnifiedParser, - KimiK3UnifiedParser, UnifiedParser, + Gemma4UnifiedParser, HfTemplateError, HfUnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser, + InklingUnifiedParser, KimiK3UnifiedParser, ResponseTemplate, UnifiedParser, }; use vllm_tokenizer::DynTokenizer; @@ -21,6 +21,11 @@ pub mod names { pub const HY_V4: &str = "hy_v4"; pub const INKLING: &str = "inkling"; pub const KIMI_K3: &str = "kimi_k3"; + + /// Executes the checkpoint's Hugging Face `response_template`. Registered for + /// name resolution only: it is built from the model's template, never by its + /// registry constructor. + pub const HF: &str = "hf"; } /// Constructor signature for one registered unified parser implementation. @@ -48,6 +53,8 @@ impl UnifiedParserFactory { factory.register_parser::(names::HY_V4); factory.register_parser::(names::INKLING); factory.register_parser::(names::KIMI_K3); + // Opt-in only: no model patterns map to it. + factory.register_parser::(names::HF); factory .register_pattern("gemma-4", names::GEMMA4) diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index de813e3045..b1a6505cf8 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -309,6 +309,15 @@ impl RoundtripCase { } } + /// Gemma4 through the checkpoint's `response_template` (`hf` parser). + fn gemma4_hf() -> Self { + Self { + tool_call_parser: ParserSelection::Explicit("hf".to_string()), + reasoning_parser: ParserSelection::Explicit("hf".to_string()), + ..Self::gemma4() + } + } + /// Kimi K2.5 tool-call format with `` reasoning tags. #[allow(dead_code)] fn kimi_k25() -> Self { @@ -459,6 +468,7 @@ roundtrip_tests! { step3p5 => [reasoning_and_content], nemotron_v3 => [reasoning_and_content], gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call + gemma4_hf => [tool_call_mix], kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history // K3 drops plain-assistant reasoning in history; tool-call turns keep it. kimi_k3 => [tool_call_mix], diff --git a/rust/src/parser/Cargo.toml b/rust/src/parser/Cargo.toml index 3086de03e9..aff4c795ab 100644 --- a/rust/src/parser/Cargo.toml +++ b/rust/src/parser/Cargo.toml @@ -11,11 +11,15 @@ test-util = [] auto_impl.workspace = true easy-ext.workspace = true educe.workspace = true +regex-automata.workspace = true +regex-syntax.workspace = true serde.workspace = true serde_json.workspace = true +serde_path_to_error.workspace = true serde_with.workspace = true thiserror.workspace = true thiserror-ext.workspace = true +tracing.workspace = true vllm-tokenizer.workspace = true winnow.workspace = true xgrammar-structural-tag.workspace = true diff --git a/rust/src/parser/src/tool/mod.rs b/rust/src/parser/src/tool/mod.rs index 94f0b46711..7789752626 100644 --- a/rust/src/parser/src/tool/mod.rs +++ b/rust/src/parser/src/tool/mod.rs @@ -34,6 +34,7 @@ pub use kimi_k2::KimiK2ToolParser; pub use mimo::MiMoToolParser; pub use minimax_m2::MinimaxM2ToolParser; pub use minimax_m3::MinimaxM3ToolParser; +pub(crate) use parameters::{ToolSchemas, convert_integer_text, convert_number_text}; pub use qwen_coder::Qwen3CoderToolParser; pub use seed_oss::SeedOssToolParser; use serde::{Deserialize, Serialize}; diff --git a/rust/src/parser/src/tool/parameters.rs b/rust/src/parser/src/tool/parameters.rs index 71601ca8af..19d700830b 100644 --- a/rust/src/parser/src/tool/parameters.rs +++ b/rust/src/parser/src/tool/parameters.rs @@ -9,7 +9,7 @@ use crate::tool::Tool; /// Normalized parameter schemas for all tools in one request. #[derive(Debug, Clone, Default, PartialEq, Eq)] -pub(super) struct ToolSchemas { +pub(crate) struct ToolSchemas { tools: BTreeMap, } @@ -28,7 +28,7 @@ pub(super) struct ToolSchema { /// /// It can be either a raw text string, or a structured input with named child elements. #[derive(Debug, Clone, PartialEq, Eq)] -pub(super) enum ParamInput { +pub(crate) enum ParamInput { Text(String), #[allow(dead_code)] Elements(Vec), @@ -42,7 +42,7 @@ impl From for ParamInput { /// One named structured parameter child. #[derive(Debug, Clone, PartialEq, Eq)] -pub(super) struct ParamElement { +pub(crate) struct ParamElement { pub name: String, pub value: ParamInput, } @@ -67,7 +67,7 @@ pub(super) enum JsonParamType { impl ToolSchemas { /// Normalize OpenAI-style tool parameter JSON schemas for one request. - pub(super) fn from_tools(tools: &[Tool]) -> Self { + pub(crate) fn from_tools(tools: &[Tool]) -> Self { let tools = tools .iter() .map(|tool| (tool.name.clone(), ToolSchema::from_schema(&tool.parameters))) @@ -98,7 +98,7 @@ impl ToolSchemas { } /// Convert one parameter value for one named tool. - pub(super) fn convert_param_with_schema

( + pub(crate) fn convert_param_with_schema

( &self, function_name: &str, name: &str, @@ -361,12 +361,8 @@ fn try_convert_value(param_type: &JsonParamType, input: &ParamInput) -> Option Option { match param_type { JsonParamType::String => Some(Value::String(value.to_string())), - JsonParamType::Integer => without_digit_separators(value.trim())? - .parse::() - .ok() - .map(Number::from) - .map(Value::Number), - JsonParamType::Number => try_convert_number(&without_digit_separators(value.trim())?), + JsonParamType::Integer => convert_integer_text(value), + JsonParamType::Number => convert_number_text(value), JsonParamType::Boolean => try_convert_boolean(value), JsonParamType::Object { .. } if value.trim().is_empty() => Some(Value::Object(Map::new())), JsonParamType::Array { .. } if value.trim().is_empty() => Some(Value::Array(Vec::new())), @@ -461,6 +457,20 @@ fn without_digit_separators(value: &str) -> Option> { valid.then(|| value.replace('_', "").into()) } +/// Convert raw text to a JSON integer, as for an `integer` parameter. +pub(crate) fn convert_integer_text(value: &str) -> Option { + without_digit_separators(value.trim())? + .parse::() + .ok() + .map(Number::from) + .map(Value::Number) +} + +/// Convert raw text to a JSON number, as for a `number` parameter. +pub(crate) fn convert_number_text(value: &str) -> Option { + try_convert_number(&without_digit_separators(value.trim())?) +} + /// Convert one raw string value to a JSON number. fn try_convert_number(value: &str) -> Option { serde_json::from_str::(value) diff --git a/rust/src/parser/src/unified/combined.rs b/rust/src/parser/src/unified/combined.rs index 48c77ef900..6f6849ec95 100644 --- a/rust/src/parser/src/unified/combined.rs +++ b/rust/src/parser/src/unified/combined.rs @@ -65,7 +65,10 @@ impl UnifiedParser for CombinedParser { where Self: Sized + 'static, { - Err(UnifiedParserError::CombinedParserConstructor) + Err(UnifiedParserError::NoNamedConstructor { + parser: "combined", + built_from: "split parser instances", + }) } fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { diff --git a/rust/src/parser/src/unified/hf/content.rs b/rust/src/parser/src/unified/hf/content.rs new file mode 100644 index 0000000000..d62ba9305e --- /dev/null +++ b/rust/src/parser/src/unified/hf/content.rs @@ -0,0 +1,537 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Content parsers: turn one region's captured text into a JSON value. +//! +//! Port of `content_parsers.py`. Each parser takes a chunk of captured text and +//! parses it into a single key in the output message dictionary. + +use regex_automata::meta::Regex; +use regex_automata::util::syntax; +use serde::Deserialize; +use serde_json::{Map, Value}; +use thiserror_ext::AsReport as _; + +use super::spec::{ContentKind, deserialize}; +use super::{Result, invalid, unsupported}; +use crate::tool::{convert_integer_text, convert_number_text}; +use crate::unified::{self, parsing_failed}; + +// Sentinel characters for lax-JSON string pre-extraction — ASCII control chars +// that should never appear in real LLM output. +const LAX_OPEN: char = '\u{1}'; +const LAX_CLOSE: char = '\u{2}'; + +/// A configured content parser (`content` plus `content_args` of one field). +#[derive(Debug, Clone)] +pub(super) enum ContentParser { + Text(TextArgs), + /// Converted like an `integer` tool parameter. + Int, + /// Converted like a `number` tool parameter. + Float, + Bool(TextArgs), + Json(JsonArgs), + XmlInline(XmlInlineArgs), + KvLines(KvLinesArgs), +} + +/// The `strip` argument shared by parsers that go through `_text`. +#[derive(Debug, Clone, Copy, Deserialize)] +pub(super) struct TextArgs { + #[serde(default = "default_strip")] + pub strip: bool, +} + +#[derive(Debug, Clone, Deserialize)] +pub(super) struct JsonArgs { + #[serde(flatten)] + text: TextArgs, + /// Quote bare-identifier keys before parsing. + #[serde(default)] + unquoted_keys: bool, + /// Strings delimited by these custom markers are pre-extracted, then restored + /// as standard JSON strings. + #[serde(default)] + string_delims: Vec<(String, String)>, + /// Return stripped text if parsing fails. + #[serde(default)] + allow_non_json: bool, +} + +#[derive(Debug, Clone)] +pub(super) struct XmlInlineArgs { + tag_pattern: Regex, + /// Index of the required `key` group. + key_group: usize, + /// Index of the `value` group, if the pattern declares one. + value_group: Option, + value_parser: Option>, + /// Collect duplicate keys into a list. + merge_duplicates: bool, +} + +#[derive(Debug, Clone)] +pub(super) struct KvLinesArgs { + text: TextArgs, + line_sep: String, + kv_sep: String, + value_parser: Option>, +} + +/// `content_args` of `xml-inline`. +#[derive(Deserialize)] +struct XmlInlineSpec { + /// Must have named groups `key` and `value`. + tag_pattern: String, + value_parser: Option, + #[serde(default)] + merge_duplicates: bool, +} + +/// `content_args` of `kv-lines`. +#[derive(Deserialize)] +struct KvLinesSpec { + #[serde(flatten)] + text: TextArgs, + #[serde(default = "default_line_sep")] + line_sep: String, + #[serde(default = "default_kv_sep")] + kv_sep: String, + value_parser: Option, +} + +/// A nested `value_parser`: `{"name": ..., "args": {...}}`. +#[derive(Deserialize)] +struct ValueParserSpec { + #[serde(default)] + name: ContentKind, + #[serde(default)] + args: Map, +} + +impl ContentParser { + /// Build a parser from its kind and `content_args`. + /// + /// Unknown argument keys are ignored, as in Transformers. Argument types and the + /// `xml-inline` tag pattern are checked here rather than at parse time. + pub fn new(scope: &str, kind: ContentKind, args: &Map) -> Result { + let context = format!("{scope}: content_args"); + let args = Value::Object(args.clone()); + Ok(match kind { + ContentKind::Text => Self::Text(deserialize(&args, &context)?), + // Number conversion ignores `strip`, but a malformed value is still rejected. + ContentKind::Int => { + deserialize::(&args, &context)?; + Self::Int + } + ContentKind::Float => { + deserialize::(&args, &context)?; + Self::Float + } + ContentKind::Bool => Self::Bool(deserialize(&args, &context)?), + ContentKind::Json => { + let args: JsonArgs = deserialize(&args, &context)?; + if args + .string_delims + .iter() + .any(|(open, close)| open.is_empty() || close.is_empty()) + { + return Err(invalid!("{context}: string_delims cannot be empty strings")); + } + Self::Json(args) + } + ContentKind::XmlInline => { + let spec: XmlInlineSpec = deserialize(&args, &context)?; + let tag_pattern = compile_regex(scope, &spec.tag_pattern)?; + let group_info = tag_pattern.group_info(); + let Some(key_group) = group_info.to_index(Default::default(), "key") else { + return Err(invalid!( + "{scope}: xml-inline: tag_pattern must have a named group 'key'. Pattern: {}", + spec.tag_pattern + )); + }; + let value_group = group_info.to_index(Default::default(), "value"); + Self::XmlInline(XmlInlineArgs { + tag_pattern, + key_group, + value_group, + value_parser: ValueParserSpec::compile(scope, spec.value_parser)?, + merge_duplicates: spec.merge_duplicates, + }) + } + ContentKind::KvLines => { + let spec: KvLinesSpec = deserialize(&args, &context)?; + if spec.line_sep.is_empty() { + return Err(invalid!("{context}: line_sep cannot be empty")); + } + Self::KvLines(KvLinesArgs { + text: spec.text, + line_sep: spec.line_sep, + kv_sep: spec.kv_sep, + value_parser: ValueParserSpec::compile(scope, spec.value_parser)?, + }) + } + }) + } + + /// Parse one region body. + pub fn parse(&self, text: &str) -> unified::Result { + match self { + Self::Text(args) => Ok(Value::String(args.apply(text).to_string())), + // Numbers follow the tool-parameter conversion. + Self::Int => convert_integer_text(text) + .ok_or_else(|| parsing_failed!("int: invalid literal {text:?}")), + Self::Float => convert_number_text(text) + .ok_or_else(|| parsing_failed!("float: invalid or non-finite literal {text:?}")), + Self::Bool(args) => { + let text = args.apply(text).to_lowercase(); + Ok(Value::Bool(text == "true" || text == "1")) + } + Self::Json(args) => args.parse(text), + Self::XmlInline(args) => args.parse(text), + Self::KvLines(args) => args.parse(text), + } + } +} + +impl ValueParserSpec { + /// Build the nested parser, if any. + fn compile(scope: &str, spec: Option) -> Result>> { + spec.map(|spec| { + ContentParser::new(&format!("{scope}: value_parser"), spec.name, &spec.args) + .map(Box::new) + }) + .transpose() + } +} + +impl TextArgs { + /// Apply `_text`: strip surrounding whitespace unless disabled. + pub fn apply(self, text: &str) -> &str { + if self.strip { text.trim() } else { text } + } +} + +impl JsonArgs { + /// JSON parser with optional dialect knobs for LLM-emitted quirks. + fn parse(&self, text: &str) -> unified::Result { + if !self.string_delims.is_empty() && text.contains([LAX_OPEN, LAX_CLOSE]) { + return Err(parsing_failed!( + "json: input contains reserved sentinel characters (\\x01/\\x02); cannot parse safely." + )); + } + + let mut working = text.to_string(); + let mut captured: Vec = Vec::new(); + for (open, close) in &self.string_delims { + working = extract_delimited(&working, open, close, &mut captured); + } + + if self.unquoted_keys { + working = quote_bare_keys(&working); + } + + for (index, string) in captured.iter().enumerate() { + let quoted = serde_json::to_string(string).map_err(|error| { + parsing_failed!("json: failed to quote string: {}", error.as_report()) + })?; + working = working.replace(&format!("{LAX_OPEN}{index}{LAX_CLOSE}"), "ed); + } + + match serde_json::from_str(&working) { + Ok(value) => Ok(value), + Err(_) if self.allow_non_json => Ok(Value::String(self.text.apply(text).to_string())), + Err(error) if working == text => Err(parsing_failed!( + "json parser could not parse region as JSON.\nContent: {text:?}\nError: {}", + error.as_report() + )), + Err(error) => Err(parsing_failed!( + "json: could not parse after dialect transforms.\nOriginal: {text:?}\nTransformed: {working:?}\nError: {}", + error.as_report() + )), + } + } +} + +impl XmlInlineArgs { + /// Parse shallow XML-ish tags into a dict. + fn parse(&self, text: &str) -> unified::Result { + let mut out = Map::new(); + for captures in self.tag_pattern.captures_iter(text) { + let key = + captures.get_group(self.key_group).map(|span| &text[span.range()]).ok_or_else( + || parsing_failed!("xml-inline: named group 'key' did not participate"), + )?; + let raw = match self.value_group { + // The pattern has no `value` group. + None => Some(""), + Some(index) => captures.get_group(index).map(|span| &text[span.range()]), + }; + let value = match raw { + Some(raw) => sub_parse(raw, self.value_parser.as_deref())?, + None if self.value_parser.is_none() => Value::Null, + None => { + return Err(parsing_failed!( + "xml-inline: named group 'value' did not participate" + )); + } + }; + match out.get_mut(key) { + Some(existing) if self.merge_duplicates => match existing { + Value::Array(values) => values.push(value), + _ => { + let first = existing.take(); + *existing = Value::Array(vec![first, value]); + } + }, + _ => { + out.insert(key.to_string(), value); + } + } + } + Ok(Value::Object(out)) + } +} + +impl KvLinesArgs { + /// Parse line-delimited `keyvalue` pairs into a dict. + fn parse(&self, text: &str) -> unified::Result { + let mut out = Map::new(); + for line in text.split(self.line_sep.as_str()) { + let line = self.text.apply(line); + let Some((key, value)) = line.split_once(self.kv_sep.as_str()) else { + continue; + }; + let (key, value) = (self.text.apply(key), self.text.apply(value)); + out.insert( + key.to_string(), + sub_parse(value, self.value_parser.as_deref())?, + ); + } + Ok(Value::Object(out)) + } +} + +/// Parse `raw` with an optional nested value parser. +fn sub_parse(raw: &str, value_parser: Option<&ContentParser>) -> unified::Result { + match value_parser { + None => Ok(Value::String(raw.to_string())), + Some(parser) => parser.parse(raw), + } +} + +/// Replace every `open(.*?)close` span with an indexed sentinel, collecting the +/// inner strings. Equivalent to the non-overlapping, left-to-right `re.sub` with +/// `re.DOTALL` used by Transformers. +fn extract_delimited(text: &str, open: &str, close: &str, captured: &mut Vec) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(start) = rest.find(open) { + let inner_start = start + open.len(); + // No close after this open means no close after any later open either. + let Some(inner_len) = rest[inner_start..].find(close) else { + break; + }; + out.push_str(&rest[..start]); + captured.push(rest[inner_start..inner_start + inner_len].to_string()); + out.push(LAX_OPEN); + out.push_str(&(captured.len() - 1).to_string()); + out.push(LAX_CLOSE); + rest = &rest[inner_start + inner_len + close.len()..]; + } + out.push_str(rest); + out +} + +/// Quote bare-identifier keys: Transformers' `(?<=[{,])(\w+):` → `"\1":`. +/// +/// The Rust regex engine has no lookbehind, so the preceding `{` or `,` is +/// matched and written back. Matches cannot overlap either way, so the rewrite is +/// equivalent. +fn quote_bare_keys(text: &str) -> String { + static BARE_KEY: std::sync::LazyLock = + std::sync::LazyLock::new(|| Regex::new(r"([{,])(\w+):").expect("valid regex")); + + let mut out = String::with_capacity(text.len()); + let mut last = 0; + for captures in BARE_KEY.captures_iter(text) { + let whole = captures.get_match().expect("match"); + let separator = captures.get_group(1).expect("group 1"); + let key = captures.get_group(2).expect("group 2"); + out.push_str(&text[last..whole.start()]); + out.push_str(&text[separator.range()]); + out.push('"'); + out.push_str(&text[key.range()]); + out.push_str("\":"); + last = whole.end(); + } + out.push_str(&text[last..]); + out +} + +/// Compile a Python-syntax regex with `re.DOTALL`, as Transformers does. +pub(super) fn compile_regex(scope: &str, pattern: &str) -> Result { + Regex::builder() + .syntax(syntax::Config::new().dot_matches_new_line(true)) + .build(pattern) + .map_err(|error| { + unsupported!( + "{scope}: regex {pattern:?} is not supported by the Rust regex engine: {}", + error.as_report() + ) + }) +} + +fn default_strip() -> bool { + true +} + +fn default_line_sep() -> String { + "\n".to_string() +} + +fn default_kv_sep() -> String { + ":".to_string() +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn kind(name: &str) -> ContentKind { + serde_json::from_value(json!(name)).unwrap() + } + + fn parser(name: &str, args: Value) -> ContentParser { + ContentParser::new("test", kind(name), args.as_object().unwrap()).unwrap() + } + + #[test] + fn scalar_parsers_convert_like_tool_parameters() { + let int = parser("int", json!({})); + assert_eq!(int.parse(" 42\n").unwrap(), json!(42)); + assert_eq!(int.parse("-1_000").unwrap(), json!(-1000)); + assert!(int.parse("1__0").is_err()); + assert!(int.parse("4.2").is_err()); + + let float = parser("float", json!({})); + assert_eq!(float.parse("1.5").unwrap(), json!(1.5)); + assert_eq!(float.parse(" 1e3 ").unwrap(), json!(1000.0)); + assert!(float.parse("inf").is_err(), "JSON cannot represent inf"); + + let boolean = parser("bool", json!({})); + assert_eq!(boolean.parse(" TRUE ").unwrap(), json!(true)); + assert_eq!(boolean.parse("1").unwrap(), json!(true)); + assert_eq!(boolean.parse("yes").unwrap(), json!(false)); + + let text = parser("text", json!({"strip": false})); + assert_eq!(text.parse(" a \n").unwrap(), json!(" a \n")); + assert_eq!( + parser("text", json!({})).parse(" a \n").unwrap(), + json!("a") + ); + } + + #[test] + fn json_dialects_match_gemma4_syntax() { + let json = parser( + "json", + json!({"unquoted_keys": true, "string_delims": [["<|\"|>", "<|\"|>"]]}), + ); + let value = json + .parse( + r#"{bool_value:true,list_value:[<|"|>foo<|"|>,<|"|>bar<|"|>],null_value:null,number_value:1,string_value:<|"|>a, b: {c}<|"|>,struct_value:{foo:<|"|>bar<|"|>}}"#, + ) + .unwrap(); + assert_eq!( + value, + json!({ + "bool_value": true, + "list_value": ["foo", "bar"], + "null_value": null, + "number_value": 1, + "string_value": "a, b: {c}", + "struct_value": {"foo": "bar"}, + }) + ); + assert!(json.parse("{a:\u{1}}").is_err()); + } + + #[test] + fn json_allow_non_json_returns_stripped_text() { + let json = parser("json", json!({"allow_non_json": true})); + assert_eq!(json.parse(" celsius \n").unwrap(), json!("celsius")); + assert_eq!(json.parse(" [1, 2] ").unwrap(), json!([1, 2])); + assert!(parser("json", json!({})).parse("celsius").is_err()); + } + + #[test] + fn xml_inline_parses_tags_with_value_parser_and_merging() { + let xml = parser( + "xml-inline", + json!({ + "tag_pattern": r"\w+)>\s*(?P.*?)\s*", + "value_parser": {"name": "json", "args": {"allow_non_json": true}}, + "merge_duplicates": true, + }), + ); + let value = xml + .parse( + "\n[{\"city\": \"Paris\"}]\n\n\ + \ncelsius\n2", + ) + .unwrap(); + assert_eq!( + value, + json!({"locations": [{"city": "Paris"}], "unit": ["celsius", 2]}) + ); + } + + #[test] + fn kv_lines_split_and_strip() { + let kv = parser("kv-lines", json!({"value_parser": {"name": "int"}})); + assert_eq!( + kv.parse(" a : 1 \n\nnot a pair\nb:2").unwrap(), + json!({"a": 1, "b": 2}) + ); + } + + #[test] + fn invalid_arguments_are_rejected_at_load() { + let error = |name: &str, args: Value| { + ContentParser::new("Field 'x'", kind(name), args.as_object().unwrap()).unwrap_err() + }; + expect_test::expect![[r#" + [ + Invalid { + message: "Field 'x': content_args.strip: invalid type: string \"no\", expected a boolean", + }, + Invalid { + message: "Field 'x': content_args.string_delims[0]: invalid length 1, expected a tuple of size 2", + }, + Invalid { + message: "Field 'x': content_args: missing field `tag_pattern`", + }, + Invalid { + message: "Field 'x': xml-inline: tag_pattern must have a named group 'key'. Pattern: <(?P\\w+)>", + }, + Invalid { + message: "Field 'x': content_args.value_parser.name: unknown variant `yaml`, expected one of `text`, `int`, `float`, `bool`, `json`, `xml-inline`, `kv-lines`", + }, + ] + "#]].assert_debug_eq(&[ + error("text", json!({"strip": "no"})), + error("json", json!({"string_delims": [[""]]})), + error("xml-inline", json!({})), + error("xml-inline", json!({"tag_pattern": r"<(?P\w+)>"})), + error("kv-lines", json!({"value_parser": {"name": "yaml"}})), + ]); + assert!(matches!( + error("xml-inline", json!({"tag_pattern": r"(?<=a)b"})), + super::super::HfTemplateError::Unsupported { .. } + )); + } +} diff --git a/rust/src/parser/src/unified/hf/fixtures/differential.json b/rust/src/parser/src/unified/hf/fixtures/differential.json new file mode 100644 index 0000000000..9b24cb295f --- /dev/null +++ b/rust/src/parser/src/unified/hf/fixtures/differential.json @@ -0,0 +1,440 @@ +{ + "templates": { + "gemma4": { + "defaults": { + "role": "assistant" + }, + "start_anchor": [ + "<|turn>model\n", + "" + ], + "fields": { + "thinking": { + "open": "<|channel>thought\n", + "close": "", + "content": "text" + }, + "tool_calls": { + "open_pattern": "<\\|tool_call>call:(?P\\w+)", + "close": "", + "repeats": true, + "content": "json", + "content_args": { + "unquoted_keys": true, + "string_delims": [ + [ + "<|\"|>", + "<|\"|>" + ] + ] + }, + "transform": { + "type": "function", + "function": { + "name": "{name}", + "arguments": "{content}" + } + } + }, + "content": { + "close": [ + "", + "<|tool_response>", + "" + ], + "content": "text" + } + } + }, + "invoke": { + "defaults": { + "role": "assistant" + }, + "start_anchor": "<|begin|>assistant", + "fields": { + "reasoning_content": { + "open_pattern": "to=self<\\|msg\\|>", + "close": "<|pause|>", + "content": "text" + }, + "tool_calls": { + "open_pattern": "]*?\\bname=\"(?P[^\"]+)\">", + "close": "", + "repeats": true, + "content": "xml-inline", + "content_args": { + "tag_pattern": "]*?\\bname=\"(?P[^\"]+)\"[^>]*?>(?P.*?)", + "value_parser": { + "name": "json", + "args": { + "allow_non_json": true + } + } + }, + "transform": { + "type": "function", + "function": { + "name": "{name}", + "arguments": "{content}" + } + } + }, + "content": { + "open_pattern": "to=(?:user|note)<\\|msg\\|>", + "close": [ + "<|end|>", + "<|pause|>" + ], + "repeats": true, + "join": "", + "content": "text" + } + } + }, + "kv_tools": { + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "thinking": { + "open": "", + "close": "" + }, + "tool_calls": { + "open_pattern": "\\s*\\w+)>\\n", + "close": "", + "repeats": true, + "content": "kv-lines", + "transform": { + "type": "function", + "function": { + "name": "{name}", + "arguments": "{content}" + } + } + }, + "content": { + "close": "<|im_end|>" + } + } + } + }, + "cases": [ + { + "template": "gemma4", + "prefix": "", + "text": "Hello there.", + "tools": null, + "expected": { + "content": "Hello there." + } + }, + { + "template": "gemma4", + "prefix": "", + "text": " \n Hello\n\n", + "tools": null, + "expected": { + "content": "Hello" + } + }, + { + "template": "gemma4", + "prefix": "", + "text": "<|channel>thought\n plan\n\n\nAnswer: 天气很好 ", + "tools": null, + "expected": { + "reasoning": "plan", + "content": "Answer: 天气很好" + } + }, + { + "template": "gemma4", + "prefix": "", + "text": "<|channel>thought\nunterminated reasoning", + "tools": null, + "expected": { + "reasoning": "unterminated reasoning" + } + }, + { + "template": "gemma4", + "prefix": "", + "text": "Before <|tool_call>call:f{x:<|\"|>a, b<|\"|>,y:[1,2.5,-3],z:{w:true}} after", + "tools": null, + "expected": { + "content": "after", + "tool_calls": [ + { + "name": "f", + "arguments": { + "x": "a, b", + "y": [ + 1, + 2.5, + -3 + ], + "z": { + "w": true + } + } + } + ] + } + }, + { + "template": "gemma4", + "prefix": "", + "text": "<|tool_call>call:f{}<|tool_call>call:g{n:1}", + "tools": null, + "expected": { + "tool_calls": [ + { + "name": "f", + "arguments": {} + }, + { + "name": "g", + "arguments": { + "n": 1 + } + } + ] + } + }, + { + "template": "gemma4", + "prefix": "", + "text": "<|tool_call>call:f{a:<|\"|>unterminated", + "tools": null, + "expected": { + "error": "ValueError" + } + }, + { + "template": "gemma4", + "prefix": "", + "text": "text <|tool_call> not a call", + "tools": null, + "expected": { + "content": "text <|tool_call> not a call" + } + }, + { + "template": "gemma4", + "prefix": "", + "text": "<|tool_call>call:f{a:1}", + "tools": null, + "expected": { + "tool_calls": [ + { + "name": "f", + "arguments": { + "a": 1 + } + } + ] + } + }, + { + "template": "gemma4", + "prefix": "<|turn>user\nhi\n<|turn>model\n", + "text": "Hi!", + "tools": null, + "expected": { + "content": "Hi!" + } + }, + { + "template": "gemma4", + "prefix": "<|turn>model\n<|tool_call>call:f{}<|tool_response>response:f{}<|channel>thought\n", + "text": "Done.It is sunny.", + "tools": null, + "expected": { + "reasoning": "Done.", + "content": "It is sunny." + } + }, + { + "template": "gemma4", + "prefix": "<|turn>model\n<|channel>thought\n", + "text": "Plain answer", + "tools": null, + "expected": { + "content": "Plain answer" + } + }, + { + "template": "invoke", + "prefix": "", + "text": "to=self<|msg|>think<|pause|>to=user<|msg|>A<|pause|>to=note<|msg|> B <|end|>", + "tools": null, + "expected": { + "reasoning": "think", + "content": "AB" + } + }, + { + "template": "invoke", + "prefix": "", + "text": "noise \"Paris\"3not json", + "tools": null, + "expected": {} + }, + { + "template": "invoke", + "prefix": "", + "text": "12", + "tools": null, + "expected": { + "tool_calls": [ + { + "name": "get", + "arguments": { + "k": 2 + } + } + ] + } + }, + { + "template": "invoke", + "prefix": "", + "text": "bare text only", + "tools": null, + "expected": {} + }, + { + "template": "invoke", + "prefix": "", + "text": "to=user<|msg|>unterminated content", + "tools": null, + "expected": { + "content": "unterminated content" + } + }, + { + "template": "invoke", + "prefix": "<|begin|>user<|end|><|begin|>assistant", + "text": "to=user<|msg|>hi<|end|>", + "tools": null, + "expected": { + "content": "hi" + } + }, + { + "template": "kv_tools", + "prefix": "", + "text": "\n\nhour: 7\nratio: 0.5\nenabled: 1\nlabel: 7\nmaybe: None\n", + "tools": [ + { + "type": "function", + "function": { + "name": "set_alarm", + "parameters": { + "type": "object", + "properties": { + "hour": { + "type": "integer" + }, + "ratio": { + "type": "number" + }, + "enabled": { + "type": "boolean" + }, + "label": { + "type": "string" + }, + "maybe": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + } + } + } + } + ], + "expected": { + "tool_calls": [ + { + "name": "set_alarm", + "arguments": { + "hour": 7, + "ratio": 0.5, + "enabled": true, + "label": "7", + "maybe": null + } + } + ] + } + }, + { + "template": "kv_tools", + "prefix": "", + "text": "\n\nhour: seven\nratio: 1e3\nenabled: nope\n", + "tools": [ + { + "type": "function", + "function": { + "name": "set_alarm", + "parameters": { + "type": "object", + "properties": { + "hour": { + "type": "integer" + }, + "ratio": { + "type": "number" + }, + "enabled": { + "type": "boolean" + }, + "label": { + "type": "string" + }, + "maybe": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + } + } + } + } + ], + "expected": { + "tool_calls": [ + { + "name": "set_alarm", + "arguments": { + "hour": "seven", + "ratio": 1000, + "enabled": "nope" + } + } + ] + } + }, + { + "template": "kv_tools", + "prefix": "<|im_start|>assistant\n\n", + "text": "\n reasoning \n\n\nanswer<|im_end|>", + "tools": null, + "expected": { + "reasoning": "reasoning", + "content": "answer" + } + } + ] +} diff --git a/rust/src/parser/src/unified/hf/fixtures/generate_differential.py b/rust/src/parser/src/unified/hf/fixtures/generate_differential.py new file mode 100644 index 0000000000..35df98e083 --- /dev/null +++ b/rust/src/parser/src/unified/hf/fixtures/generate_differential.py @@ -0,0 +1,264 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regenerate `differential.json` from Transformers `parse_response`. + +Requires Transformers with `transformers.utils.chat_parsing` (>= 5.13); the +expected values in the checked-in file were produced at commit 6d43ab4008. + + python generate_differential.py > differential.json +""" + +import json + +from transformers.utils.chat_parsing import parse_response + +GEMMA4 = { + "defaults": {"role": "assistant"}, + "start_anchor": ["<|turn>model\n", ""], + "fields": { + "thinking": { + "open": "<|channel>thought\n", + "close": "", + "content": "text", + }, + "tool_calls": { + "open_pattern": r"<\|tool_call>call:(?P\w+)", + "close": "", + "repeats": True, + "content": "json", + "content_args": { + "unquoted_keys": True, + "string_delims": [['<|"|>', '<|"|>']], + }, + "transform": { + "type": "function", + "function": {"name": "{name}", "arguments": "{content}"}, + }, + }, + "content": { + "close": ["", "<|tool_response>", ""], + "content": "text", + }, + }, +} + +INVOKE = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|begin|>assistant", + "fields": { + "reasoning_content": { + "open_pattern": r"to=self<\|msg\|>", + "close": "<|pause|>", + "content": "text", + }, + "tool_calls": { + "open_pattern": r']*?\bname="(?P[^"]+)">', + "close": "", + "repeats": True, + "content": "xml-inline", + "content_args": { + "tag_pattern": ( + r']*?\bname="(?P[^"]+)"[^>]*?>' + r"(?P.*?)" + ), + "value_parser": {"name": "json", "args": {"allow_non_json": True}}, + }, + "transform": { + "type": "function", + "function": {"name": "{name}", "arguments": "{content}"}, + }, + }, + "content": { + "open_pattern": r"to=(?:user|note)<\|msg\|>", + "close": ["<|end|>", "<|pause|>"], + "repeats": True, + "join": "", + "content": "text", + }, + }, +} + +KV_TOOLS = { + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "thinking": {"open": "", "close": ""}, + "tool_calls": { + "open_pattern": r"\s*\w+)>\n", + "close": "", + "repeats": True, + "content": "kv-lines", + "transform": { + "type": "function", + "function": {"name": "{name}", "arguments": "{content}"}, + }, + }, + "content": {"close": "<|im_end|>"}, + }, +} + +ALARM_TOOLS = [ + { + "type": "function", + "function": { + "name": "set_alarm", + "parameters": { + "type": "object", + "properties": { + "hour": {"type": "integer"}, + "ratio": {"type": "number"}, + "enabled": {"type": "boolean"}, + "label": {"type": "string"}, + "maybe": {"anyOf": [{"type": "integer"}, {"type": "null"}]}, + }, + }, + }, + } +] + +CASES = [ + ("gemma4", GEMMA4, "", "Hello there.", None), + ("gemma4", GEMMA4, "", " \n Hello\n\n", None), + ( + "gemma4", + GEMMA4, + "", + "<|channel>thought\n plan\n\n\nAnswer: 天气很好 ", + None, + ), + ("gemma4", GEMMA4, "", "<|channel>thought\nunterminated reasoning", None), + ( + "gemma4", + GEMMA4, + "", + 'Before <|tool_call>call:f{x:<|"|>a, b<|"|>,y:[1,2.5,-3],z:{w:true}}' + " after", + None, + ), + ( + "gemma4", + GEMMA4, + "", + "<|tool_call>call:f{}<|tool_call>call:g{n:1}", + None, + ), + ("gemma4", GEMMA4, "", '<|tool_call>call:f{a:<|"|>unterminated', None), + ("gemma4", GEMMA4, "", "text <|tool_call> not a call", None), + ("gemma4", GEMMA4, "", "<|tool_call>call:f{a:1}", None), + ("gemma4", GEMMA4, "<|turn>user\nhi\n<|turn>model\n", "Hi!", None), + ( + "gemma4", + GEMMA4, + "<|turn>model\n<|tool_call>call:f{}<|tool_response>response:f{}<|channel>thought\n", + "Done.It is sunny.", + None, + ), + ( + "gemma4", + GEMMA4, + "<|turn>model\n<|channel>thought\n", + "Plain answer", + None, + ), + ( + "invoke", + INVOKE, + "", + "to=self<|msg|>think<|pause|>to=user<|msg|>A<|pause|>to=note<|msg|> B <|end|>", + None, + ), + ( + "invoke", + INVOKE, + "", + 'noise "Paris"' + '3not json', + None, + ), + ( + "invoke", + INVOKE, + "", + '1' + '2', + None, + ), + ("invoke", INVOKE, "", "bare text only", None), + ("invoke", INVOKE, "", "to=user<|msg|>unterminated content", None), + ( + "invoke", + INVOKE, + "<|begin|>user<|end|><|begin|>assistant", + "to=user<|msg|>hi<|end|>", + None, + ), + ( + "kv_tools", + KV_TOOLS, + "", + "\n\nhour: 7\nratio: 0.5\nenabled: 1\n" + "label: 7\nmaybe: None\n", + ALARM_TOOLS, + ), + ( + "kv_tools", + KV_TOOLS, + "", + "\n\nhour: seven\nratio: 1e3\n" + "enabled: nope\n", + ALARM_TOOLS, + ), + ( + "kv_tools", + KV_TOOLS, + "<|im_start|>assistant\n\n", + "\n reasoning \n\n\nanswer<|im_end|>", + None, + ), +] + + +def normalize(message): + out = {} + reasoning = message.get("thinking", message.get("reasoning_content")) + if reasoning: + out["reasoning"] = reasoning + if message.get("content"): + out["content"] = message["content"] + calls = [ + {"name": call["function"]["name"], "arguments": call["function"]["arguments"]} + for call in message.get("tool_calls", []) + ] + if calls: + out["tool_calls"] = calls + return out + + +def main(): + templates = {} + cases = [] + for template_name, template, prefix, text, tools in CASES: + templates[template_name] = template + try: + expected = normalize( + parse_response(text, template, prefix=prefix, tools=tools) + ) + except Exception as error: # noqa: BLE001 + expected = {"error": type(error).__name__} + cases.append( + { + "template": template_name, + "prefix": prefix, + "text": text, + "tools": tools, + "expected": expected, + } + ) + print( + json.dumps( + {"templates": templates, "cases": cases}, indent=2, ensure_ascii=False + ) + ) + + +if __name__ == "__main__": + main() diff --git a/rust/src/parser/src/unified/hf/mod.rs b/rust/src/parser/src/unified/hf/mod.rs new file mode 100644 index 0000000000..568a8371d0 --- /dev/null +++ b/rust/src/parser/src/unified/hf/mod.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Unified parser driven by a checkpoint's Hugging Face `response_template`. +//! +//! The renderer executes a checkpoint's `chat_template`; this parser executes its +//! `response_template`, the declarative output-protocol description introduced by +//! Transformers `chat_parsing` (`ResponseParser`). +//! +//! The template language and its value semantics (content parsers, transforms, +//! tool-schema coercion) follow Transformers at +//! . +//! The streaming executor is native: it emits ordered [`UnifiedParserEvent`]s +//! instead of Transformers' region events and aggregated message dict. +//! +//! [`UnifiedParserEvent`]: super::UnifiedParserEvent + +mod content; +mod parser; +mod pattern; +mod spec; +mod template; +mod transform; + +#[cfg(test)] +mod tests; + +pub use parser::HfUnifiedParser; +pub use template::ResponseTemplate; +use thiserror::Error; +use thiserror_ext::Macro; + +/// Result alias for response-template operations. +pub type Result = std::result::Result; + +/// Errors produced while loading a `response_template`. +/// +/// Failures while parsing generated output are +/// [`UnifiedParserError::ParsingFailed`](super::UnifiedParserError::ParsingFailed), as for +/// the other unified parsers. +#[derive(Debug, Clone, Error, Macro)] +#[thiserror_ext(macro(path = "crate::unified::hf", mangle))] +pub enum HfTemplateError { + /// The template violates the Transformers `response_template` schema. + #[error("invalid response_template: {message}")] + Invalid { message: String }, + /// The template is valid but uses a feature this implementation does not support. + #[error("unsupported response_template: {message}")] + Unsupported { message: String }, + /// The model provides no template. + #[error("the model's tokenizer_config.json provides no response_template")] + Missing, +} diff --git a/rust/src/parser/src/unified/hf/parser.rs b/rust/src/parser/src/unified/hf/parser.rs new file mode 100644 index 0000000000..f523c0f31a --- /dev/null +++ b/rust/src/parser/src/unified/hf/parser.rs @@ -0,0 +1,730 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Streaming executor for a compiled [`ResponseTemplate`]. +//! +//! Follows `ResponseParser` in `response_parser.py`: a flat region state +//! machine that, outside explicit regions, watches every explicit open plus the +//! implicit region's close, and inside an explicit region watches only that +//! region's close. Instead of region events and an aggregated dict, it emits +//! ordered unified parser events: +//! +//! - text and reasoning stream as they arrive, with `strip` applied so that the +//! concatenated deltas equal Transformers' stripped value. For a field without +//! `repeats`, stripping spans all its occurrences: an interrupted field (text, +//! tool call, more text) streams every occurrence, where Transformers keeps only +//! the last; +//! - a tool call starts at its opener when the opener captures the function +//! name, and its arguments are emitted when the region closes. + +use std::mem::take; +use std::sync::Arc; + +use serde::Deserialize; +use serde_json::{Map, Value}; +use thiserror_ext::AsReport as _; +use vllm_tokenizer::{DecodedText, DynTokenizer, TokenAnchor, TokenAttribution}; +use winnow::error::{ErrMode, ModalResult}; +use winnow::stream::{Partial, Stream}; + +use super::pattern::{Captures, Resolution}; +use super::template::{ + FieldName, Region, RegionKind, Repeat, ResponseTemplate, TextRegion, TextRole, ToolCallRegion, + Watch, WatchKind, WatchSet, +}; +use crate::tool::{Tool, ToolCallDelta, ToolSchemas}; +use crate::unified::{ + Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput, parsing_failed, +}; +use crate::utils::{incomplete, parse_buffered_event, safe_text_len_mul}; + +/// Prompt tail windows, in tokens, searched for the start anchor before +/// decoding the whole prompt. +const PROMPT_WINDOWS: [usize; 3] = [64, 512, 4096]; + +/// Unified parser executing a checkpoint's `response_template`. +/// +/// Original Python implementation: +/// +pub struct HfUnifiedParser { + template: Arc, + tool_schemas: ToolSchemas, + tokenizer: DynTokenizer, + buffer: DecodedText, + mode: HfMode, + /// Whether each region has closed an occurrence. + closed: Vec, + /// Strip state per region: per occurrence with `repeats`, per stream otherwise. + strip: Vec, + emitted_tool_count: usize, +} + +/// Parser mode: the region currently receiving text. +enum HfMode { + /// No implicit field is declared: text outside regions is discarded. + Discard, + /// The implicit region, opened lazily on its first byte. + Implicit(Occurrence), + /// An explicit region, opened by its open boundary. + Explicit(Occurrence), +} + +/// One occurrence of a region. +struct Occurrence { + region: usize, + /// Whether the occurrence has content to close (`_opened`). + opened: bool, + captures: Captures, + /// Opener text, returned by [`UnifiedParser::reset`]. + raw_open: String, + /// Buffered body of a tool-call region. + body: String, + /// Index of the tool call started at the opener, if any. + active_tool_index: Option, + /// `join` separator still to emit before this occurrence's content. + separator: Option, +} + +/// Strip-compatible streaming state of a text or reasoning occurrence. +#[derive(Default)] +struct StripState { + enabled: bool, + /// Whether a non-whitespace character has been seen. + started: bool, + /// Trailing whitespace held until more text arrives or the region closes. + held: DecodedText, +} + +/// One parsed event of the buffered input. +enum HfEvent { + /// Text routed into the current region. + Text, + /// A boundary of the current mode's watch list. + Boundary { watch: Watch, captures: Captures }, +} + +/// A committable boundary match at the current position. +struct BoundaryMatch { + watch: Watch, + len: usize, + captures: Captures, +} + +impl BoundaryMatch { + /// Preference order of `_scan`: longest first, opens before closes, then by + /// field name. + fn key(&self, template: &ResponseTemplate) -> (std::cmp::Reverse, bool, FieldName) { + ( + std::cmp::Reverse(self.len), + self.watch.kind == WatchKind::Close, + template.regions[self.watch.region].name, + ) + } +} + +/// How the event loop treats its input. +#[derive(Clone, Copy)] +enum Phase { + /// Prompt text after the start anchor: state transitions only, no events. + Prefill, + /// Generated text; more may follow. + Stream, + /// Generated text at the end of the stream. + Eof, +} + +impl Phase { + fn emits(self) -> bool { + !matches!(self, Self::Prefill) + } +} + +impl HfUnifiedParser { + /// Create a parser for one request. + pub fn new(template: Arc, tools: &[Tool], tokenizer: DynTokenizer) -> Self { + let mut parser = Self { + closed: vec![false; template.regions.len()], + strip: template.regions.iter().map(StripState::new).collect(), + mode: HfMode::Discard, + template, + tool_schemas: ToolSchemas::from_tools(tools), + tokenizer, + buffer: DecodedText::default(), + emitted_tool_count: 0, + }; + parser.reset_to_implicit(); + parser + } + + /// Clear all per-stream state. + fn reset_state(&mut self) { + self.buffer.clear(); + self.closed.fill(false); + self.strip = self.template.regions.iter().map(StripState::new).collect(); + self.emitted_tool_count = 0; + self.reset_to_implicit(); + } + + /// Return to the implicit region, or discard text when there is none. + fn reset_to_implicit(&mut self) { + self.mode = match self.template.implicit { + Some(region) => { + HfMode::Implicit(self.occurrence(region, false, Captures::new(), String::new())) + } + None => HfMode::Discard, + }; + } + + /// Start a new occurrence of `region`. + fn occurrence( + &mut self, + region: usize, + opened: bool, + captures: Captures, + raw_open: String, + ) -> Occurrence { + let spec = &self.template.regions[region]; + let mut separator = None; + if let RegionKind::Text(text) = &spec.kind { + if let Repeat::Join(join) = &text.repeat + && !join.is_empty() + && self.closed[region] + { + separator = Some(join.clone()); + } + if text.repeat != Repeat::Once { + self.strip[region] = StripState::new(spec); + } + } + Occurrence { + region, + opened, + captures, + raw_open, + body: String::new(), + active_tool_index: None, + separator, + } + } + + /// Parse buffered input until more is needed. + fn drive(&mut self, output: &mut UnifiedParserOutput, phase: Phase) -> Result<()> { + let template = Arc::clone(&self.template); + loop { + let watch = match &self.mode { + HfMode::Explicit(occurrence) => &template.regions[occurrence.region].close_watch, + HfMode::Discard | HfMode::Implicit(_) => &template.idle_watch, + }; + let eof = matches!(phase, Phase::Eof); + let Some((step, consumed_len)) = parse_buffered_event(&self.buffer.text, |input| { + parse_next_event(input, &template, watch, eof) + })? + else { + return Ok(()); + }; + let piece = self.buffer.drain_prefix(consumed_len); + match step { + HfEvent::Text => self.route(piece, output, phase), + HfEvent::Boundary { watch, captures } => { + self.close_current(output, phase)?; + match watch.kind { + WatchKind::Open => { + self.open_explicit(watch.region, captures, piece.text, output, phase) + } + // The implicit region's close: start a fresh implicit occurrence. + WatchKind::Close => {} + } + } + } + } + } + + /// Route a text piece into the current region. + fn route(&mut self, piece: DecodedText, output: &mut UnifiedParserOutput, phase: Phase) { + let (occurrence, region) = match &mut self.mode { + HfMode::Discard => return, + HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) => { + let region = &self.template.regions[occurrence.region]; + (occurrence, region) + } + }; + let RegionKind::Text(text) = ®ion.kind else { + if !piece.text.is_empty() { + occurrence.opened = true; + occurrence.body.push_str(&piece.text); + } + return; + }; + let reasoning = text.role == TextRole::Reasoning; + if piece.text.is_empty() { + // Zero-width tokens still count as reasoning. + if reasoning && phase.emits() { + output.push_reasoning(piece); + } + return; + } + occurrence.opened = true; + + let strip = &mut self.strip[occurrence.region]; + if !phase.emits() { + // Prompt text is never emitted; it only decides whether later + // leading whitespace is still stripped. + strip.started |= !piece.text.trim_start().is_empty(); + return; + } + let (visible, dropped) = strip.feed(piece); + if reasoning { + push_dropped_reasoning(output, dropped); + } + if visible.text.is_empty() { + return; + } + if let Some(separator) = occurrence.separator.take() { + push_text(output, text.role, DecodedText::unattributed(separator)); + } + push_text(output, text.role, visible); + } + + /// Open an explicit region. + fn open_explicit( + &mut self, + region: usize, + captures: Captures, + raw_open: String, + output: &mut UnifiedParserOutput, + phase: Phase, + ) { + let mut occurrence = self.occurrence(region, true, captures, raw_open); + let opener_name = match &self.template.regions[region].kind { + RegionKind::ToolCalls(ToolCallRegion { + name_group: Some(group), + .. + }) => occurrence.captures.get(group).and_then(Value::as_str).map(str::to_string), + _ => None, + }; + if let Some(name) = opener_name + && phase.emits() + { + let tool_index = self.allocate_tool_index(); + occurrence.active_tool_index = Some(tool_index); + output.push_call(ToolCallDelta { + tool_index, + name: Some(name), + arguments: String::new(), + }); + } + self.mode = HfMode::Explicit(occurrence); + } + + /// Close the current region and reset to the implicit region. + /// + /// Skipped (aside from the reset) when the current region never opened -- + /// avoids vacuous open/close pairs at every explicit boundary. + fn close_current(&mut self, output: &mut UnifiedParserOutput, phase: Phase) -> Result<()> { + let template = Arc::clone(&self.template); + let mode = std::mem::replace(&mut self.mode, HfMode::Discard); + let occurrence = match mode { + HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) if occurrence.opened => { + Some(occurrence) + } + _ => None, + }; + // Without `repeats`, trailing whitespace stays held in case the field + // continues; it is dropped at the end of the stream. + let held = occurrence + .as_ref() + .filter(|occurrence| { + matches!(&template.regions[occurrence.region].kind, + RegionKind::Text(text) if text.repeat != Repeat::Once) + }) + .map(|occurrence| take(&mut self.strip[occurrence.region].held)); + if let Some(occurrence) = &occurrence { + self.closed[occurrence.region] = true; + } + self.reset_to_implicit(); + // Regions closed inside the prompt contribute nothing to the output. + let Some(occurrence) = occurrence.filter(|_| phase.emits()) else { + return Ok(()); + }; + + let region = &template.regions[occurrence.region]; + match ®ion.kind { + RegionKind::Text(text) => { + if text.role == TextRole::Reasoning + && let Some(held) = held + { + push_dropped_reasoning(output, held); + } + // An empty occurrence still joins: `previous + join + ""`. + if let Some(separator) = occurrence.separator { + push_text(output, text.role, DecodedText::unattributed(separator)); + } + } + RegionKind::ToolCalls(calls) => { + let parsed = calls.content.parse(&occurrence.body)?; + let value = match &calls.transform { + Some(transform) => { + transform.apply(region.name.as_str(), parsed, &occurrence.captures)? + } + None => parsed, + }; + self.emit_calls(value, occurrence.active_tool_index, output)?; + } + } + Ok(()) + } + + /// Emit the tool calls of a closed tool-call region. + // TODO: stream arguments incrementally instead of once at region close; the + // dialects (e.g. Gemma 4's unquoted keys and `<|"|>` strings, `xml-inline`) + // need incremental conversion rather than raw passthrough. + fn emit_calls( + &mut self, + value: Value, + started: Option, + output: &mut UnifiedParserOutput, + ) -> Result<()> { + let calls = match value { + Value::Array(calls) => calls, + call => vec![call], + }; + for (position, call) in calls.into_iter().enumerate() { + let (name, arguments) = call_parts(call, &self.tool_schemas)?; + let (tool_index, name) = match (position, started) { + (0, Some(tool_index)) => (tool_index, None), + _ => (self.allocate_tool_index(), Some(name)), + }; + output.push_call(ToolCallDelta { + tool_index, + name, + arguments, + }); + } + Ok(()) + } + + fn allocate_tool_index(&mut self) -> usize { + let tool_index = self.emitted_tool_count; + self.emitted_tool_count += 1; + tool_index + } + + /// Return the prompt text after the last start-anchor match, if any. + /// + /// Searches geometrically growing tail windows before decoding the whole + /// prompt. A decoded suffix can differ from the full decode only in its first + /// character, so a match is accepted only after that character unless the + /// window is the whole prompt. + fn prompt_remainder(&self, prompt_token_ids: &[u32]) -> Result> { + let len = prompt_token_ids.len(); + let windows = PROMPT_WINDOWS.into_iter().filter(|window| *window < len).chain([len]); + for window in windows { + let text = self.tokenizer.decode(&prompt_token_ids[len - window..], false).map_err( + |error| parsing_failed!("failed to decode prompt: {}", error.as_report()), + )?; + let whole = window == len; + if let Some((start, end)) = self.template.last_start_anchor(&text) { + let first_char = text.chars().next().map_or(0, char::len_utf8); + if whole || start >= first_char { + return Ok(Some(text[end..].to_string())); + } + } + } + Ok(None) + } +} + +impl StripState { + fn new(region: &Region) -> Self { + Self { + enabled: matches!( + region.kind, + RegionKind::Text(TextRegion { strip: true, .. }) + ), + ..Self::default() + } + } + + /// Split a piece into its visible part and whitespace dropped by `strip`. + /// + /// Leading whitespace is dropped until the first non-whitespace character; + /// trailing whitespace is held until more non-whitespace text arrives (then + /// released) or the region closes (then dropped). + fn feed(&mut self, mut piece: DecodedText) -> (DecodedText, DecodedText) { + if !self.enabled { + return (piece, DecodedText::default()); + } + let mut dropped = DecodedText::default(); + if !self.started { + let lead = piece.text.len() - piece.text.trim_start().len(); + if lead == piece.text.len() { + return (DecodedText::default(), piece); + } + dropped = piece.drain_prefix(lead); + self.started = true; + } + let body_len = piece.text.trim_end().len(); + if body_len == 0 { + self.held.append(piece); + return (DecodedText::default(), dropped); + } + let head = piece.drain_prefix(body_len); + let mut visible = take(&mut self.held); + visible.append(head); + self.held = piece; + (visible, dropped) + } +} + +/// Push visible text of a text region. +fn push_text(output: &mut UnifiedParserOutput, role: TextRole, piece: DecodedText) { + match role { + TextRole::Content => output.push_text(piece.text), + TextRole::Reasoning => output.push_reasoning(piece), + } +} + +/// Keep the tokens of stripped reasoning whitespace in the reasoning count. +fn push_dropped_reasoning(output: &mut UnifiedParserOutput, dropped: DecodedText) { + if dropped.attributions.is_empty() { + return; + } + output.push_reasoning(DecodedText { + text: String::new(), + attributions: dropped + .attributions + .into_iter() + .map(|attribution| TokenAttribution { + token_id: attribution.token_id, + anchor: TokenAnchor::ZeroWidth { byte_offset: 0 }, + }) + .collect(), + }); +} + +/// One tool call produced by a tool-call region, after its transform. +#[derive(Deserialize)] +#[serde(untagged)] +enum ToolCallValue { + /// The OpenAI shape `{"type": "function", "function": {...}}`. + Wrapped { function: FunctionValue }, + /// The bare `{"name", "arguments"}` shape. + Bare(FunctionValue), +} + +#[derive(Deserialize)] +struct FunctionValue { + name: String, + /// Missing or `null` arguments become `{}`. + #[serde(default)] + arguments: Option, +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum ArgumentsValue { + /// An argument object, typed by the tool schema. + Object(Map), + /// Pre-serialized or non-JSON arguments (e.g. an `allow_non_json` + /// fallback), passed through unchanged. + Text(String), + /// Any other JSON value, serialized as is. + Other(Value), +} + +/// Extract the function name and serialized arguments of one tool-call value. +/// +/// String argument values are converted by the calling tool's parameter +/// schema, as for the other tool parsers. +fn call_parts(call: Value, tool_schemas: &ToolSchemas) -> Result<(String, String)> { + let (ToolCallValue::Wrapped { function } | ToolCallValue::Bare(function)) = + ToolCallValue::deserialize(&call).map_err(|_| { + parsing_failed!( + "tool call must be {{\"function\": {{\"name\": ..., \"arguments\": ...}}}} or \ + {{\"name\": ..., \"arguments\": ...}} with a string name, got {call}" + ) + })?; + let FunctionValue { name, arguments } = function; + let arguments = match arguments { + None => "{}".to_string(), + Some(ArgumentsValue::Text(arguments)) => arguments, + Some(ArgumentsValue::Object(mut arguments)) => { + convert_text_arguments(tool_schemas, &name, &mut arguments); + serialize_arguments(&arguments)? + } + Some(ArgumentsValue::Other(arguments)) => serialize_arguments(&arguments)?, + }; + Ok((name, arguments)) +} + +/// Convert the argument values a content parser left as raw text by the tool +/// schema: top-level strings, and string lists collected from duplicate keys +/// (`merge_duplicates`). +/// +/// Deeper values are the content parser's own JSON and are kept, as in +/// Transformers' `_coerce_tool_calls`. +fn convert_text_arguments( + tool_schemas: &ToolSchemas, + function_name: &str, + arguments: &mut Map, +) { + for (name, value) in arguments.iter_mut() { + let values = match value { + Value::Array(items) => items.as_mut_slice(), + value => std::slice::from_mut(value), + }; + for value in values { + if let Value::String(text) = value { + *value = tool_schemas.convert_param_with_schema(function_name, name, take(text)); + } + } + } +} + +/// Serialize tool-call arguments to JSON text. +fn serialize_arguments(arguments: &impl serde::Serialize) -> Result { + serde_json::to_string(arguments).map_err(|error| { + parsing_failed!("failed to serialize tool arguments: {}", error.as_report()) + }) +} + +/// Parse one event: safe text before the next marker, or a boundary at it. +fn parse_next_event( + input: &mut Partial<&str>, + template: &ResponseTemplate, + watch: &WatchSet, + eof: bool, +) -> ModalResult { + let markers: Vec<&str> = watch.markers.iter().map(String::as_str).collect(); + match safe_text_len_mul(input, &markers) { + Ok(_) => return Ok(HfEvent::Text), + // A marker, complete or partial, starts here (or the input is empty). + Err(ErrMode::Incomplete(_)) => {} + Err(error) => return Err(error), + } + + // Any pending boundary blocks the position; otherwise the preferred match + // wins. At the end of the stream, a partial marker matches nothing and is + // released as text one character at a time. + let text = **input; + let mut best: Option = None; + for &watch in &watch.boundaries { + match template.boundary(watch).resolve(text, eof) { + Resolution::Pending => return incomplete(), + Resolution::NoMatch => {} + Resolution::Matched { len, captures } => { + let candidate = BoundaryMatch { + watch, + len, + captures, + }; + if best.as_ref().is_none_or(|best| candidate.key(template) < best.key(template)) { + best = Some(candidate); + } + } + } + } + + match best { + Some(BoundaryMatch { + watch, + len, + captures, + }) => { + input.next_slice(len); + Ok(HfEvent::Boundary { watch, captures }) + } + None => { + // Not a boundary: the marker's first character is plain text. + let len = text.chars().next().map_or(0, char::len_utf8); + input.next_slice(len); + Ok(HfEvent::Text) + } + } +} + +impl UnifiedParser for HfUnifiedParser { + fn create(_tools: &[Tool], _tokenizer: DynTokenizer) -> Result> + where + Self: Sized + 'static, + { + Err(UnifiedParserError::NoNamedConstructor { + parser: "hf", + built_from: "the checkpoint's response_template", + }) + } + + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { + self.reset_state(); + if prompt_token_ids.is_empty() { + return Ok(()); + } + let Some(remainder) = self.prompt_remainder(prompt_token_ids)? else { + self.template.warn_missing_anchor_once(); + return Ok(()); + }; + // A partial boundary at the end of the prompt stays buffered, so the + // generated text can complete it. + self.buffer = DecodedText::unattributed(remainder); + self.drive(&mut UnifiedParserOutput::default(), Phase::Prefill) + } + + fn preserve_special_tokens(&self) -> bool { + // Templates spell boundaries with special tokens and expect decoding + // with them kept. + true + } + + // TODO: derive a structural-tag output grammar from literal-bounded templates. + + fn parse_into(&mut self, delta: DecodedText, output: &mut UnifiedParserOutput) -> Result<()> { + self.buffer.append(delta); + self.drive(output, Phase::Stream) + } + + fn finish(&mut self) -> Result { + let mut output = UnifiedParserOutput::default(); + self.drive(&mut output, Phase::Eof)?; + // Trailing zero-width tokens carry no text for the step loop to consume. + let rest = self.buffer.take(); + self.route(rest, &mut output, Phase::Eof); + self.close_current(&mut output, Phase::Eof)?; + for (region, strip) in self.template.regions.iter().zip(&mut self.strip) { + let held = take(&mut strip.held); + if matches!(®ion.kind, RegionKind::Text(text) if text.role == TextRole::Reasoning) { + push_dropped_reasoning(&mut output, held); + } + } + + let missing: Vec<_> = self + .template + .regions + .iter() + .zip(&self.closed) + .filter(|(region, closed)| !region.optional && !**closed) + .map(|(region, _)| region.name.as_str()) + .collect(); + if !missing.is_empty() { + return Err(parsing_failed!( + "Required response_template fields missing from parsed output: {missing:?}" + )); + } + + self.reset_state(); + Ok(output) + } + + fn reset(&mut self) -> String { + let mut raw = String::new(); + if let HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) = + std::mem::replace(&mut self.mode, HfMode::Discard) + { + raw.push_str(&occurrence.raw_open); + raw.push_str(&occurrence.body); + raw.push_str(&self.strip[occurrence.region].held.text); + } + raw.push_str(&self.buffer.take().text); + self.reset_state(); + raw + } +} diff --git a/rust/src/parser/src/unified/hf/pattern.rs b/rust/src/parser/src/unified/hf/pattern.rs new file mode 100644 index 0000000000..143545ca12 --- /dev/null +++ b/rust/src/parser/src/unified/hf/pattern.rs @@ -0,0 +1,403 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Streaming resolution of `open_pattern` / `close_pattern` boundaries. +//! +//! Transformers relies on the Python `regex` module's `partial=True` search to +//! decide whether a match at the end of the buffer is final, could still appear, +//! or is impossible. The Rust `regex` crate has no partial-match API, so this +//! module answers that question by stepping a `regex-automata` DFA, and uses a +//! `meta::Regex` for the actual match span and captures. + +use regex_automata::dfa::{Automaton, StartKind, dense}; +use regex_automata::meta::Regex; +use regex_automata::nfa::thompson::{self, WhichCaptures}; +use regex_automata::util::primitives::StateID; +use regex_automata::{Anchored, Input, MatchKind, PatternID}; +use regex_syntax::hir::literal::{ExtractKind, Extractor}; +use regex_syntax::hir::{Hir, HirKind}; +use serde_json::{Map, Value}; +use thiserror_ext::AsReport as _; + +use super::{Result, unsupported}; + +/// Upper bound for the memory of one boundary DFA. +const DFA_SIZE_LIMIT: usize = 16 << 20; + +/// Named capture groups of a boundary match, as string values: the scope a +/// transform template reads them from. +pub(super) type Captures = Map; + +/// Outcome of resolving a boundary at a candidate position. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum Resolution { + /// The boundary matches `len` bytes; more input cannot change the match. + Matched { len: usize, captures: Captures }, + /// More input could still produce or change a match. + Pending, + /// No match starts here, whatever input follows. + NoMatch, +} + +/// A regex boundary that can be resolved against a growing buffer. +#[derive(Debug, Clone)] +pub(super) struct StreamingPattern { + /// Finite set of non-empty literals, one of which starts every match. + prefixes: Vec, + /// Anchored leftmost-first DFA of the pattern: decides whether the + /// leftmost-first match is final. Unicode word boundaries are handled + /// heuristically, quitting on non-ASCII bytes. + first: dense::DFA>, + /// Anchored DFA with every look-around assertion removed and + /// `MatchKind::All`, used when `first` quits. Relaxing an assertion only + /// enlarges the language, so once this DFA is dead no continuation can + /// produce a later-ending match of the real pattern. `MatchKind::All` is + /// required for that argument: leftmost-first pruning could discard the + /// thread the real pattern takes when an assertion fails (`a\b|ab` on `ab`). + fallback: Option>>, + /// The pattern itself, for the match span and named captures. + exact: Regex, +} + +/// Result of walking a DFA over the available bytes. +enum Walk { + /// No continuation can change the result. + Final, + /// More input could still change the result. + Alive, + /// The DFA gave up (non-ASCII byte near a Unicode word boundary). + Quit, +} + +impl StreamingPattern { + /// Compile a boundary pattern in Python `regex` syntax, with `re.DOTALL`. + /// + /// Streaming resolution only starts at occurrences of the pattern's literal + /// prefixes, so every match must start with one of a finite set of non-empty + /// literals. + pub fn new(scope: &str, pattern: &str) -> Result { + let hir = regex_syntax::ParserBuilder::new() + .dot_matches_new_line(true) + .build() + .parse(pattern) + .map_err(|error| { + unsupported!( + "{scope}: regex {pattern:?} is not supported by the Rust regex engine: {}", + error.as_report() + ) + })?; + + let prefixes = literal_prefixes(&hir).ok_or_else(|| { + unsupported!( + "{scope}: pattern {pattern:?} must start with a literal prefix for streaming \ + matching (e.g. `` in `(?P\\w+)`)" + ) + })?; + + let first = build_dfa(scope, pattern, &hir, MatchKind::LeftmostFirst)?; + let fallback = hir + .properties() + .look_set() + .contains_word_unicode() + .then(|| build_dfa(scope, pattern, &relax_assertions(&hir), MatchKind::All)) + .transpose()?; + let exact = Regex::builder().build_from_hir(&hir).map_err(|error| { + unsupported!( + "{scope}: failed to compile regex {pattern:?}: {}", + error.as_report() + ) + })?; + + Ok(Self { + prefixes, + first, + fallback, + exact, + }) + } + + /// Literals, one of which starts every match. + pub fn prefixes(&self) -> &[String] { + &self.prefixes + } + + /// Named capture groups of the pattern. + pub fn group_names(&self) -> impl Iterator { + self.exact.group_info().pattern_names(PatternID::ZERO).flatten() + } + + /// Resolve a match starting at the beginning of `text`. + /// + /// `eof` means no more input will arrive. + pub fn resolve(&self, text: &str, eof: bool) -> Resolution { + let walk = match walk(&self.first, text) { + Walk::Quit => walk( + self.fallback.as_ref().expect("quits imply a fallback"), + text, + ), + walk => walk, + }; + if matches!(walk, Walk::Alive) && !eof { + return Resolution::Pending; + } + + let input = Input::new(text).anchored(Anchored::Yes); + let mut captures = self.exact.create_captures(); + self.exact.search_captures(&input, &mut captures); + let Some(found) = captures.get_match() else { + return Resolution::NoMatch; + }; + let names = self.exact.group_info().pattern_names(PatternID::ZERO); + let captures = names + .enumerate() + .filter_map(|(index, name)| { + let span = captures.get_group(index)?; + Some(( + name?.to_string(), + Value::String(text[span.range()].to_string()), + )) + }) + .collect(); + Resolution::Matched { + len: found.end(), + captures, + } + } +} + +/// Walk `dfa` from an anchored start over `text`. +/// +/// The walk is final once the DFA is dead, or once it is in a match state whose +/// transitions are all dead: that removes the one byte of holdback that delayed +/// DFA match reporting would otherwise add. A quit transition counts as live. +fn walk(dfa: &dense::DFA>, text: &str) -> Walk { + let input = Input::new(text).anchored(Anchored::Yes); + let Ok(mut state) = dfa.start_state_forward(&input) else { + return Walk::Quit; + }; + for byte in text.bytes() { + state = dfa.next_state(state, byte); + if dfa.is_quit_state(state) { + return Walk::Quit; + } + if dfa.is_dead_state(state) || is_terminal_match(dfa, state) { + return Walk::Final; + } + } + Walk::Alive +} + +/// Return whether `state` reports a match and cannot continue. +fn is_terminal_match(dfa: &dense::DFA>, state: StateID) -> bool { + dfa.is_match_state(state) + && (0..=u8::MAX).all(|byte| dfa.is_dead_state(dfa.next_state(state, byte))) +} + +/// Build an anchored dense DFA for `hir`. +fn build_dfa( + scope: &str, + pattern: &str, + hir: &Hir, + kind: MatchKind, +) -> Result>> { + let nfa = thompson::Compiler::new() + .configure(thompson::Config::new().which_captures(WhichCaptures::None)) + .build_from_hir(hir) + .map_err(|error| { + unsupported!( + "{scope}: failed to compile regex {pattern:?}: {}", + error.as_report() + ) + })?; + dense::Builder::new() + .configure( + dense::Config::new() + .match_kind(kind) + .start_kind(StartKind::Anchored) + .unicode_word_boundary(true) + .dfa_size_limit(Some(DFA_SIZE_LIMIT)) + .determinize_size_limit(Some(DFA_SIZE_LIMIT)), + ) + .build_from_nfa(&nfa) + .map_err(|error| { + unsupported!( + "{scope}: regex {pattern:?} cannot be matched incrementally: {}", + error.as_report() + ) + }) +} + +/// Replace every look-around assertion with the empty regex. +fn relax_assertions(hir: &Hir) -> Hir { + match hir.kind() { + HirKind::Look(_) => Hir::empty(), + HirKind::Repetition(repetition) => { + let mut repetition = repetition.clone(); + repetition.sub = Box::new(relax_assertions(&repetition.sub)); + Hir::repetition(repetition) + } + HirKind::Capture(capture) => { + let mut capture = capture.clone(); + capture.sub = Box::new(relax_assertions(&capture.sub)); + Hir::capture(capture) + } + HirKind::Concat(subs) => Hir::concat(subs.iter().map(relax_assertions).collect()), + HirKind::Alternation(subs) => Hir::alternation(subs.iter().map(relax_assertions).collect()), + HirKind::Empty | HirKind::Literal(_) | HirKind::Class(_) => hir.clone(), + } +} + +/// Extract the finite set of non-empty literal prefixes starting every match. +fn literal_prefixes(hir: &Hir) -> Option> { + let seq = Extractor::new().kind(ExtractKind::Prefix).extract(hir); + let mut prefixes: Vec = Vec::new(); + for literal in seq.literals()? { + let bytes = literal.as_bytes(); + // Truncated inexact literals may end inside a UTF-8 sequence; any valid + // prefix of a prefix is still a prefix. + let valid = match std::str::from_utf8(bytes) { + Ok(text) => text, + Err(error) => std::str::from_utf8(&bytes[..error.valid_up_to()]).ok()?, + }; + if valid.is_empty() { + return None; + } + if !prefixes.iter().any(|prefix| prefix == valid) { + prefixes.push(valid.to_string()); + } + } + (!prefixes.is_empty()).then_some(prefixes) +} + +#[cfg(test)] +mod tests { + use super::*; + + const MUSE_INVOKE: &str = r#"]*?\bname="(?P[^"]+)">"#; + const GEMMA4_CALL: &str = r"<\|tool_call>call:(?P\w+)"; + const GPT_OSS_CALL: &str = + r"<\|channel\|>commentary to=functions\.(?P\w+).*?<\|message\|>"; + + fn pattern(source: &str) -> StreamingPattern { + StreamingPattern::new("test", source).unwrap() + } + + fn matched(len: usize, name: &str) -> Resolution { + Resolution::Matched { + len, + captures: Captures::from_iter([("name".to_string(), name.into())]), + } + } + + #[test] + fn extracts_literal_prefixes() { + assert_eq!(pattern(GEMMA4_CALL).prefixes(), ["<|tool_call>call:"]); + assert_eq!(pattern(MUSE_INVOKE).prefixes(), ["").prefixes(), + ["to=user<|message|>", "to=commentary<|message|>"] + ); + assert_eq!( + pattern(r"\n?").prefixes(), + ["\n", ""] + ); + for source in [r"(?:^|\s*)", r"(?:<\|m\|>)?[^<]*<\|c\|>", r".*x"] { + let result = StreamingPattern::new("test", source); + assert!( + matches!( + result, + Err(super::super::HfTemplateError::Unsupported { .. }) + ), + "{source}" + ); + } + } + + #[test] + fn word_boundaries_follow_unicode_semantics() { + let muse = pattern(MUSE_INVOKE); + assert_eq!( + muse.resolve(r#"rest"#, false), + matched(24, "get") + ); + let unicode_attr = r#"rest"#; + assert_eq!( + muse.resolve(unicode_attr, false), + matched(unicode_attr.len() - 4, "获取") + ); + assert_eq!( + muse.resolve(r#"r"#, false), + Resolution::NoMatch + ); + assert_eq!( + muse.resolve(r#"r"#, false), + Resolution::NoMatch + ); + assert_eq!( + muse.resolve(r#"r"#, false), + matched(25, "x") + ); + assert_eq!( + muse.resolve(r#"r"#, false), + Resolution::NoMatch + ); + assert_eq!( + muse.resolve(" 正文", false), + Resolution::NoMatch + ); + + let alternation = pattern(r"a\b|ab"); + assert_eq!( + alternation.resolve("ab!", true), + Resolution::Matched { + len: 2, + captures: Captures::new() + } + ); + } + + #[test] + fn greedy_and_lazy_tails_commit_like_transformers() { + let gemma4 = pattern(GEMMA4_CALL); + assert_eq!( + gemma4.resolve("<|tool_call>call:get_wea", false), + Resolution::Pending + ); + assert_eq!( + gemma4.resolve("<|tool_call>call:get_weather{", false), + matched(28, "get_weather") + ); + assert_eq!( + gemma4.resolve("<|tool_call>call:天气{", false), + matched(23, "天气") + ); + assert_eq!( + gemma4.resolve("<|tool_call>call:{", false), + Resolution::NoMatch + ); + assert_eq!( + gemma4.resolve("<|tool_call>call:get", true), + matched(20, "get") + ); + + let gpt_oss = pattern(GPT_OSS_CALL); + let opener = "<|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>"; + assert_eq!(gpt_oss.resolve(opener, false), Resolution::Pending); + assert_eq!( + gpt_oss.resolve(&format!("{opener}{{\"a\""), false), + matched(opener.len(), "get_weather") + ); + } +} diff --git a/rust/src/parser/src/unified/hf/spec.rs b/rust/src/parser/src/unified/hf/spec.rs new file mode 100644 index 0000000000..20738b95c7 --- /dev/null +++ b/rust/src/parser/src/unified/hf/spec.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Serde mirror of the `response_template` JSON. +//! +//! Key sets and defaults follow `response_templates.py`; semantic validation +//! happens when compiling into [`super::ResponseTemplate`]. + +use serde::Deserialize; +use serde::de::DeserializeOwned; +use serde_json::{Map, Value}; +use serde_with::serde_as; +use thiserror_ext::AsReport as _; + +use super::{Result, invalid}; + +/// Top-level `response_template` object. +#[serde_as] +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct TemplateSpec { + #[serde(default = "default_version")] + version: u64, + /// Constant keys of the Transformers output dict (e.g. `role`). Unused by the + /// event-based executor, but validated as part of the schema. + #[serde(default, rename = "defaults")] + _defaults: Map, + /// Field specs in declaration order. + #[serde_as(as = "serde_with::Map<_, _>")] + pub fields: Vec<(String, FieldSpec)>, + start_anchor: Option, + start_anchor_pattern: Option, +} + +/// One entry of `fields`. +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct FieldSpec { + open: Option, + open_pattern: Option, + close: Option, + close_pattern: Option, + #[serde(default)] + pub content: ContentKind, + /// Arguments of the `content` parser; their keys depend on the parser. + #[serde(default)] + pub content_args: Map, + #[serde(default)] + pub repeats: bool, + pub join: Option, + #[serde(default = "default_optional")] + pub optional: bool, + /// A free-form JSON template, compiled into a [`super::transform::Transform`]. + pub transform: Option, + #[serde(default)] + pub transform_each: bool, +} + +/// Name of a content parser. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(super) enum ContentKind { + #[default] + Text, + Int, + Float, + Bool, + Json, + XmlInline, + KvLines, +} + +/// A literal anchor: one string, or a list of alternatives. +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +enum Literals { + One(String), + Many(Vec), +} + +/// An anchor after resolving its literal-or-pattern form. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum AnchorSpec<'a> { + /// Deduplicated literals in first-seen order. + Literals(Vec), + /// A regex in Python `regex` syntax. + Pattern(&'a str), +} + +impl TemplateSpec { + /// Parse the top-level template object. + pub fn from_value(value: &Value) -> Result { + let spec: Self = deserialize(value, "response_template")?; + if spec.version != 1 { + return Err(invalid!( + "unsupported response_template version: {}", + spec.version + )); + } + if spec.fields.is_empty() { + return Err(invalid!( + "response_template.fields must be a non-empty dict" + )); + } + Ok(spec) + } + + /// The required start anchor. + pub fn start_anchor(&self) -> Result> { + anchor_spec( + "response_template", + (self.start_anchor.as_ref(), "start_anchor"), + (self.start_anchor_pattern.as_deref(), "start_anchor_pattern"), + )? + .ok_or_else(|| { + invalid!("response_template must define 'start_anchor' or 'start_anchor_pattern'.") + }) + } +} + +impl FieldSpec { + /// The open anchor; `None` for the implicit field. + pub fn open(&self, scope: &str) -> Result>> { + anchor_spec( + scope, + (self.open.as_ref(), "open"), + (self.open_pattern.as_deref(), "open_pattern"), + ) + } + + /// The close anchor; `None` when the field runs to the end of the stream. + pub fn close(&self, scope: &str) -> Result>> { + anchor_spec( + scope, + (self.close.as_ref(), "close"), + (self.close_pattern.as_deref(), "close_pattern"), + ) + } +} + +/// Deserialize `value`, reporting `context` and the JSON path of any error. +pub(super) fn deserialize(value: &Value, context: &str) -> Result { + serde_path_to_error::deserialize(value).map_err(|error| { + let path = error.path().to_string(); + let error = error.into_inner(); + if path == "." { + invalid!("{context}: {}", error.as_report()) + } else { + invalid!("{context}.{path}: {}", error.as_report()) + } + }) +} + +/// Resolve an anchor given in literal form or regex form. +/// +/// Mirrors `_compile_anchor`: both forms at once is an error, literal lists must be +/// non-empty and contain no empty strings, and duplicates are dropped in first-seen +/// order. +fn anchor_spec<'a>( + scope: &str, + (literals, literal_key): (Option<&Literals>, &str), + (pattern, pattern_key): (Option<&'a str>, &str), +) -> Result>> { + match (literals, pattern) { + (Some(_), Some(_)) => Err(invalid!( + "{scope}: cannot specify both '{literal_key}' and '{pattern_key}'" + )), + (Some(literals), None) => { + let raw = match literals { + Literals::One(literal) => std::slice::from_ref(literal), + Literals::Many(literals) => literals.as_slice(), + }; + if raw.is_empty() { + return Err(invalid!( + "{scope}: '{literal_key}' list must contain at least one literal" + )); + } + if raw.iter().any(String::is_empty) { + return Err(invalid!( + "{scope}: '{literal_key}' literals cannot be empty strings" + )); + } + let mut deduped: Vec = Vec::with_capacity(raw.len()); + for literal in raw { + if !deduped.contains(literal) { + deduped.push(literal.clone()); + } + } + Ok(Some(AnchorSpec::Literals(deduped))) + } + (None, Some(pattern)) => Ok(Some(AnchorSpec::Pattern(pattern))), + (None, None) => Ok(None), + } +} + +fn default_version() -> u64 { + 1 +} + +fn default_optional() -> bool { + true +} diff --git a/rust/src/parser/src/unified/hf/template.rs b/rust/src/parser/src/unified/hf/template.rs new file mode 100644 index 0000000000..4889f6f9b5 --- /dev/null +++ b/rust/src/parser/src/unified/hf/template.rs @@ -0,0 +1,481 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Compiled `response_template`. +//! +//! Validation follows `response_templates.py`; the compiled form additionally +//! maps fields to parser roles and precomputes each state's boundary markers. + +use std::fmt; +use std::sync::atomic::{AtomicBool, Ordering}; + +use regex_automata::meta::Regex; +use serde_json::Value; + +use super::content::{ContentParser, TextArgs, compile_regex}; +use super::pattern::{Captures, Resolution, StreamingPattern}; +use super::spec::{AnchorSpec, FieldSpec, TemplateSpec}; +use super::transform::{FieldTransform, Transform}; +use super::{Result, invalid, unsupported}; + +/// A compiled `response_template`, shared by all requests of one model. +#[derive(Debug)] +pub struct ResponseTemplate { + /// Matches the start of the current assistant turn in the prompt. + start_anchor: Regex, + pub(super) regions: Vec, + /// The field without an opener: the implicit-open / leftover sink. + pub(super) implicit: Option, + /// Boundaries watched outside explicit regions: every explicit open plus the + /// implicit region's close. + pub(super) idle_watch: WatchSet, + warned_missing_anchor: AtomicBool, +} + +/// One compiled field. +#[derive(Debug)] +pub(super) struct Region { + pub name: FieldName, + /// `None`: the implicit region. + pub open: Option, + /// `None`: the region runs to the end of the stream. + pub close: Option, + /// Boundaries watched inside this region: its close, if any. + pub close_watch: WatchSet, + pub optional: bool, + pub kind: RegionKind, +} + +/// A field this parser can report, in the lexical order of its name, which +/// Transformers uses to break ties between boundaries. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(super) enum FieldName { + Content, + ReasoningContent, + Thinking, + ToolCalls, +} + +/// How a region's content is reported. +#[derive(Debug)] +pub(super) enum RegionKind { + /// Streamed as it arrives. + Text(TextRegion), + /// Buffered, and parsed into tool calls at close. + ToolCalls(ToolCallRegion), +} + +#[derive(Debug)] +pub(super) struct TextRegion { + pub role: TextRole, + /// Whether the value is stripped (`content_args.strip`). + pub strip: bool, + pub repeat: Repeat, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum TextRole { + Content, + Reasoning, +} + +/// How repeated occurrences of a text field combine. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum Repeat { + /// No `repeats`: occurrences stream as one value. + Once, + /// `repeats` without `join`: each occurrence is its own value. + Each, + /// `repeats` with `join`: occurrences are joined by the separator. + Join(String), +} + +#[derive(Debug)] +pub(super) struct ToolCallRegion { + pub content: ContentParser, + pub transform: Option, + /// Opener capture group that provides the function name, allowing the tool + /// call to start before its arguments are complete. + pub name_group: Option, +} + +/// A region delimiter. +#[derive(Debug)] +pub(super) enum Boundary { + /// Literal alternatives, longest first. + Literals(Vec), + Pattern(Box), +} + +/// One boundary the executor watches for in a given state. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct Watch { + pub region: usize, + pub kind: WatchKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum WatchKind { + Open, + Close, +} + +/// The boundaries watched in one parser mode. +#[derive(Debug, Default)] +pub(super) struct WatchSet { + pub boundaries: Vec, + /// Literals, deduplicated, whose occurrences are the only positions any of + /// the boundaries can match at. + pub markers: Vec, +} + +impl ResponseTemplate { + /// Compile the `response_template` object from `tokenizer_config.json`. + pub fn from_json(value: &Value) -> Result { + let spec = TemplateSpec::from_value(value)?; + + let regions = spec + .fields + .iter() + .enumerate() + .map(|(index, (name, field))| Region::compile(index, name, field)) + .collect::>>()?; + + // A field without an open anchor is the implicit-open / leftover sink; at most one is allowed. + let implicit_fields: Vec<_> = + regions.iter().enumerate().filter(|(_, region)| region.open.is_none()).collect(); + if implicit_fields.len() > 1 { + let names: Vec<_> = + implicit_fields.iter().map(|(_, region)| region.name.as_str()).collect(); + return Err(invalid!( + "At most one field may omit 'open'/'open_pattern' (that field becomes the \ + implicit-open / leftover sink). Found: {}", + names.join(", ") + )); + } + let implicit = implicit_fields.first().map(|(index, _)| *index); + + let start_anchor = match spec.start_anchor()? { + AnchorSpec::Literals(mut literals) => { + // Sort longest-first so alternation prefers the longer alternative when both could match. + literals.sort_by_key(|literal| std::cmp::Reverse(literal.len())); + let alternation: Vec<_> = + literals.iter().map(|literal| regex_syntax::escape(literal)).collect(); + compile_regex("response_template", &alternation.join("|"))? + } + AnchorSpec::Pattern(pattern) => compile_regex("response_template", pattern)?, + }; + + let opens = regions.iter().enumerate().filter_map(|(region, spec)| { + let watch = Watch { + region, + kind: WatchKind::Open, + }; + Some((watch, spec.open.as_ref()?)) + }); + let implicit_close = implicit.and_then(|region| { + let watch = Watch { + region, + kind: WatchKind::Close, + }; + Some((watch, regions[region].close.as_ref()?)) + }); + let idle_watch = WatchSet::new(opens.chain(implicit_close)); + + Ok(Self { + start_anchor, + regions, + implicit, + idle_watch, + warned_missing_anchor: AtomicBool::new(false), + }) + } + + /// Return the start and end offsets of the last start-anchor match in + /// `prompt`. + pub(super) fn last_start_anchor(&self, prompt: &str) -> Option<(usize, usize)> { + self.start_anchor + .find_iter(prompt) + .last() + .map(|found| (found.start(), found.end())) + } + + /// Log once per template that the prompt carried no start anchor. + pub(super) fn warn_missing_anchor_once(&self) { + if !self.warned_missing_anchor.swap(true, Ordering::Relaxed) { + tracing::warn!( + "response_template start anchor not found in the prompt; parsing starts from the \ + template's initial state (the chat template may not match the checkpoint's \ + response_template)" + ); + } + } + + /// The boundary of a watch entry. + pub(super) fn boundary(&self, watch: Watch) -> &Boundary { + let region = &self.regions[watch.region]; + match watch.kind { + WatchKind::Open => region.open.as_ref(), + WatchKind::Close => region.close.as_ref(), + } + .expect("watched boundaries exist") + } +} + +impl Region { + /// Validate the spec of the field at `index` and compile it. + fn compile(index: usize, name: &str, field: &FieldSpec) -> Result { + let scope = format!("Field '{name}'"); + let open = field.open(&scope)?.map(|spec| Boundary::compile(&scope, spec)).transpose()?; + let close = field.close(&scope)?.map(|spec| Boundary::compile(&scope, spec)).transpose()?; + + let content = ContentParser::new(&scope, field.content, &field.content_args)?; + let repeat = match (field.repeats, &field.join) { + (false, None) => Repeat::Once, + (true, None) => Repeat::Each, + (true, Some(join)) => Repeat::Join(join.clone()), + (false, Some(_)) => return Err(invalid!("{scope}: 'join' requires 'repeats': true")), + }; + let transform = match (&field.transform, field.transform_each) { + (Some(template), each) => Some(FieldTransform { + template: Transform::compile(&scope, template)?, + each, + }), + (None, false) => None, + (None, true) => { + return Err(invalid!( + "{scope}: transform_each is set but no transform was provided" + )); + } + }; + + let captured: Vec<&str> = + [&open, &close].into_iter().flatten().flat_map(Boundary::group_names).collect(); + match &transform { + None if !captured.is_empty() => { + // Named captures only reach the output through a transform, so flag any that would be silently dropped. + return Err(invalid!( + "{scope}: open_pattern/close_pattern declares named group(s) {captured:?}, but \ + the field has no 'transform'. Named captures are only surfaced through a \ + 'transform' template (where they appear alongside 'content'). Either add a \ + 'transform' that uses the captures, or remove the named groups from the pattern." + )); + } + // Without `transform_each`, the scope is the opener's captures plus `content`; + // any other placeholder fails on every match. + Some(FieldTransform { + template, + each: false, + }) => { + let open_groups: Vec<&str> = open.iter().flat_map(Boundary::group_names).collect(); + if let Some(root) = template + .placeholder_roots() + .find(|root| *root != "content" && !open_groups.contains(root)) + { + return Err(invalid!( + "{scope}: transform placeholder '{{{root}}}' is neither 'content' nor a \ + named group of open_pattern" + )); + } + } + _ => {} + } + + let name = FieldName::parse(&scope, name)?; + let kind = match name.text_role() { + Some(role) => { + let strip = match (content, &transform) { + (ContentParser::Text(TextArgs { strip }), None) => strip, + _ => { + return Err(unsupported!( + "{scope}: text and reasoning fields must use the 'text' content \ + parser without a transform" + )); + } + }; + RegionKind::Text(TextRegion { + role, + strip, + repeat, + }) + } + None => { + if matches!(repeat, Repeat::Join(_)) { + return Err(unsupported!( + "{scope}: 'join' requires each match to parse to a string" + )); + } + let name_group = name_group(transform.as_ref(), open.as_ref()); + RegionKind::ToolCalls(ToolCallRegion { + content, + transform, + name_group, + }) + } + }; + + let close_watch = WatchSet::new(close.as_ref().map(|close| { + let watch = Watch { + region: index, + kind: WatchKind::Close, + }; + (watch, close) + })); + + Ok(Self { + name, + open, + close, + close_watch, + optional: field.optional, + kind, + }) + } +} + +/// The opener capture group that a non-`transform_each` transform uses as the +/// function name, if any. +fn name_group(transform: Option<&FieldTransform>, open: Option<&Boundary>) -> Option { + let ( + FieldTransform { + template, + each: false, + }, + open, + ) = (transform?, open?) + else { + return None; + }; + let path = template + .placeholder_at(&["function", "name"]) + .or_else(|| template.placeholder_at(&["name"]))?; + match path { + [root] if open.group_names().any(|group| group == root) => Some(root.clone()), + _ => None, + } +} + +impl FieldName { + /// Map a field name to a reportable field. + fn parse(scope: &str, name: &str) -> Result { + Ok(match name { + "content" => Self::Content, + "reasoning_content" => Self::ReasoningContent, + "thinking" => Self::Thinking, + "tool_calls" => Self::ToolCalls, + _ => { + return Err(unsupported!( + "{scope}: only 'content', 'reasoning_content'/'thinking', and 'tool_calls' \ + fields can be reported" + )); + } + }) + } + + /// The role of a text field; `None` for `tool_calls`. + fn text_role(self) -> Option { + match self { + Self::Content => Some(TextRole::Content), + Self::ReasoningContent | Self::Thinking => Some(TextRole::Reasoning), + Self::ToolCalls => None, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Content => "content", + Self::ReasoningContent => "reasoning_content", + Self::Thinking => "thinking", + Self::ToolCalls => "tool_calls", + } + } +} + +impl fmt::Display for FieldName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl Boundary { + /// Compile an anchor into a boundary. + fn compile(scope: &str, spec: AnchorSpec<'_>) -> Result { + Ok(match spec { + AnchorSpec::Literals(mut literals) => { + // Sort longest-first so the longer alternative wins when both match. + literals.sort_by_key(|literal| std::cmp::Reverse(literal.len())); + Self::Literals(literals) + } + AnchorSpec::Pattern(pattern) => { + Self::Pattern(Box::new(StreamingPattern::new(scope, pattern)?)) + } + }) + } + + /// Literals whose occurrences are the only positions this boundary can match at. + pub fn markers(&self) -> &[String] { + match self { + Self::Literals(literals) => literals, + Self::Pattern(pattern) => pattern.prefixes(), + } + } + + /// Named capture groups. + pub fn group_names(&self) -> impl Iterator { + match self { + Self::Literals(_) => None, + Self::Pattern(pattern) => Some(pattern.group_names()), + } + .into_iter() + .flatten() + } + + /// Resolve this boundary at the start of `text`. + pub fn resolve(&self, text: &str, eof: bool) -> Resolution { + let markers = self.markers(); + // A marker the available text is a proper prefix of: more input decides. + let partial = !eof + && markers + .iter() + .any(|marker| marker.len() > text.len() && marker.starts_with(text)); + match self { + Self::Literals(literals) => { + if partial { + return Resolution::Pending; + } + match literals.iter().find(|literal| text.starts_with(literal.as_str())) { + Some(literal) => Resolution::Matched { + len: literal.len(), + captures: Captures::new(), + }, + None => Resolution::NoMatch, + } + } + Self::Pattern(pattern) => { + if markers.iter().any(|marker| text.starts_with(marker.as_str())) { + pattern.resolve(text, eof) + } else if partial { + Resolution::Pending + } else { + Resolution::NoMatch + } + } + } + } +} + +impl WatchSet { + /// Collect the watched boundaries and their deduplicated markers. + fn new<'a>(entries: impl IntoIterator) -> Self { + let mut set = Self::default(); + for (watch, boundary) in entries { + set.boundaries.push(watch); + for marker in boundary.markers() { + if !set.markers.contains(marker) { + set.markers.push(marker.clone()); + } + } + } + set + } +} diff --git a/rust/src/parser/src/unified/hf/tests.rs b/rust/src/parser/src/unified/hf/tests.rs new file mode 100644 index 0000000000..7e27a42805 --- /dev/null +++ b/rust/src/parser/src/unified/hf/tests.rs @@ -0,0 +1,933 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Executor tests. +//! +//! Template fixtures and expected values are ported from Transformers +//! `tests/utils/test_chat_parsing.py` at 6d43ab4008, with the expected output +//! dict re-expressed as aggregated unified parser events (`thinking` becomes +//! `reasoning`). + +use std::collections::BTreeMap; +use std::sync::Arc; + +use serde_json::{Value, json}; +use vllm_tokenizer::test_utils::TestTokenizer; +use vllm_tokenizer::{DecodedText, TokenAnchor, TokenAttribution, Tokenizer as _}; + +use super::{HfTemplateError, HfUnifiedParser, ResponseTemplate}; +use crate::tool::Tool; +use crate::tool::test_utils::split_by_chars; +use crate::unified::{UnifiedParser, UnifiedParserError, UnifiedParserEvent, UnifiedParserOutput}; + +fn cohere_template() -> Value { + json!({ + "defaults": {"role": "assistant"}, + "start_anchor": "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", + "fields": { + "content": {"open": "<|START_RESPONSE|>", "close": "<|END_RESPONSE|>", "content": "text"}, + "thinking": {"open": "<|START_THINKING|>", "close": "<|END_THINKING|>", "content": "text"}, + "tool_calls": { + "open": "<|START_ACTION|>", + "close": "<|END_ACTION|>", + "content": "json", + "transform_each": true, + "transform": {"type": "function", "function": {"name": "{tool_name}", "arguments": "{parameters}"}}, + }, + }, + }) +} + +fn gpt_oss_template() -> Value { + json!({ + "defaults": {"role": "assistant"}, + "start_anchor": "<|start|>assistant", + "fields": { + "thinking": {"open": "<|channel|>analysis<|message|>", "close": "<|end|>", "content": "text"}, + "content": {"open": "<|channel|>final<|message|>", "close": "<|end|>", "content": "text"}, + "tool_calls": { + "open_pattern": r"<\|channel\|>commentary to=functions\.(?P\w+).*?<\|message\|>", + "close": "<|call|>", + "repeats": true, + "content": "json", + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + }, + }) +} + +fn smollm_template() -> Value { + json!({ + "defaults": {"role": "assistant"}, + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "thinking": {"open": "", "close": "", "content": "text"}, + "tool_calls": { + "open": "", + "close": "", + "repeats": true, + "content": "json", + "transform": {"type": "function", "function": "{content}"}, + }, + "content": {"close": "<|im_end|>", "content": "text"}, + }, + }) +} + +fn qwen3_template() -> Value { + json!({ + "defaults": {"role": "assistant"}, + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "thinking": {"open": "", "close": "", "content": "text"}, + "tool_calls": { + "open_pattern": r"\s*\w+)>", + "close": "", + "repeats": true, + "content": "xml-inline", + "content_args": { + "tag_pattern": r"\w+)>\s*(?P.*?)\s*", + "value_parser": {"name": "json", "args": {"allow_non_json": true}}, + }, + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + }, + }) +} + +/// Identical to the template shipped in the Gemma 4 IT checkpoints. +fn gemma4_template() -> Value { + json!({ + "defaults": {"role": "assistant"}, + "start_anchor": ["<|turn>model\n", ""], + "fields": { + "thinking": {"open": "<|channel>thought\n", "close": "", "content": "text"}, + "tool_calls": { + "open_pattern": r"<\|tool_call>call:(?P\w+)", + "close": "", + "repeats": true, + "content": "json", + "content_args": {"unquoted_keys": true, "string_delims": [["<|\"|>", "<|\"|>"]]}, + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + "content": {"close": ["", "<|tool_response>", ""], "content": "text"}, + }, + }) +} + +/// A routed-message protocol with XML-style invocations: no implicit field +/// (text outside regions is discarded), Unicode word boundaries and lazy +/// quantifiers in the opener, `xml-inline` arguments, and joined content. +fn invoke_template() -> Value { + json!({ + "defaults": {"role": "assistant"}, + "start_anchor": "<|begin|>assistant", + "fields": { + "reasoning_content": {"open_pattern": r"to=self<\|msg\|>", "close": "<|pause|>", "content": "text"}, + "tool_calls": { + "open_pattern": r#"]*?\bname="(?P[^"]+)">"#, + "close": "", + "repeats": true, + "content": "xml-inline", + "content_args": { + "tag_pattern": r#"]*?\bname="(?P[^"]+)"[^>]*?>(?P.*?)"#, + "value_parser": {"name": "json", "args": {"allow_non_json": true}}, + }, + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + "content": { + "open_pattern": r"to=(?:user|note)<\|msg\|>", + "close": ["<|end|>", "<|pause|>"], + "repeats": true, + "join": "", + "content": "text", + }, + }, + }) +} + +fn compile(template: Value) -> Arc { + Arc::new(ResponseTemplate::from_json(&template).unwrap()) +} + +fn tokenizer() -> Arc { + Arc::new(TestTokenizer::new()) +} + +/// Parse `chunks` after initializing from `prompt`, returning all events. +fn parse_events( + template: &Arc, + tools: &[Tool], + prompt: &str, + chunks: &[&str], +) -> Result, UnifiedParserError> { + let tokenizer = tokenizer(); + let prompt_ids = tokenizer.encode(prompt, false).unwrap(); + let mut parser = HfUnifiedParser::new(template.clone(), tools, tokenizer); + parser.initialize(&prompt_ids)?; + let mut output = UnifiedParserOutput::default(); + for chunk in chunks { + parser.parse_into(DecodedText::unattributed(*chunk), &mut output)?; + } + output.append(parser.finish()?); + Ok(output.events) +} + +/// Aggregate events into the shape of the Transformers output dict. +fn message(events: &[UnifiedParserEvent]) -> Value { + let mut reasoning = String::new(); + let mut content = String::new(); + let mut calls: BTreeMap = BTreeMap::new(); + for event in events { + match event { + UnifiedParserEvent::Text(text) => content.push_str(text), + UnifiedParserEvent::Reasoning(piece) => reasoning.push_str(&piece.text), + UnifiedParserEvent::ToolCall(delta) => { + let call = calls.entry(delta.tool_index).or_default(); + if let Some(name) = &delta.name { + call.0.clone_from(name); + } + call.1.push_str(&delta.arguments); + } + } + } + let mut out = serde_json::Map::new(); + if !reasoning.is_empty() { + out.insert("reasoning".to_string(), reasoning.into()); + } + if !content.is_empty() { + out.insert("content".to_string(), content.into()); + } + if !calls.is_empty() { + let calls = calls + .into_values() + .map(|(name, arguments)| { + let arguments = + serde_json::from_str(&arguments).unwrap_or(Value::String(arguments)); + json!({"name": name, "arguments": arguments}) + }) + .collect(); + out.insert("tool_calls".to_string(), Value::Array(calls)); + } + Value::Object(out) +} + +fn parse(template: Value, text: &str) -> Value { + message(&parse_events(&compile(template), &[], "", &[text]).unwrap()) +} + +/// Split `text` at pseudo-random char boundaries (xorshift, seeded). +fn random_chunks(text: &str, seed: u64) -> Vec<&str> { + let mut state = seed | 1; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + let mut chunks = Vec::new(); + let mut start = 0; + for (index, _) in text.char_indices().skip(1) { + if next() % 3 == 0 { + chunks.push(&text[start..index]); + start = index; + } + } + chunks.push(&text[start..]); + chunks +} + +#[test] +fn upstream_fixtures_match_transformers_values() { + let cases = [ + ( + cohere_template(), + "<|START_THINKING|>I should call a tool.<|END_THINKING|><|START_ACTION|>[\n {\"tool_call_id\": \"0\", \"tool_name\": \"simple_tool\", \"parameters\": {\"temperature_format\": \"Celsius\"}}\n]<|END_ACTION|><|END_OF_TURN_TOKEN|>", + json!({ + "reasoning": "I should call a tool.", + "tool_calls": [{"name": "simple_tool", "arguments": {"temperature_format": "Celsius"}}], + }), + ), + ( + gpt_oss_template(), + "<|channel|>analysis<|message|>We will call function get_current_weather.<|end|><|start|>assistant<|channel|>commentary to=functions.get_current_weather <|constrain|>json<|message|>{\n \"location\": \"San Francisco, CA\"\n}", + json!({ + "reasoning": "We will call function get_current_weather.", + "tool_calls": [{"name": "get_current_weather", "arguments": {"location": "San Francisco, CA"}}], + }), + ), + ( + gpt_oss_template(), + "<|channel|>analysis<|message|>User asks a simple math question: 2+2 = 4. Provide answer.<|end|><|start|>assistant<|channel|>final<|message|>2", + json!({ + "reasoning": "User asks a simple math question: 2+2 = 4. Provide answer.", + "content": "2", + }), + ), + ( + smollm_template(), + "\nOkay, let me greet them.\n\n\n{\"name\": \"greet_user\", \"arguments\": {\"greeting\": \"Hello!\"}}", + json!({ + "reasoning": "Okay, let me greet them.", + "tool_calls": [{"name": "greet_user", "arguments": {"greeting": "Hello!"}}], + }), + ), + ( + smollm_template(), + "{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Paris\"}}", + json!({"tool_calls": [{"name": "get_weather", "arguments": {"city": "Paris"}}]}), + ), + ( + smollm_template(), + "\nOkay, gravity.\nSome content about gravity goes here!", + json!({"reasoning": "Okay, gravity.", "content": "Some content about gravity goes here!"}), + ), + ( + qwen3_template(), + "\n\n\n[{\"country\": \"France\", \"city\": \"Paris\"}]\n\n\ncelsius\n\n\n", + json!({ + "tool_calls": [{ + "name": "get_weather", + "arguments": {"locations": [{"country": "France", "city": "Paris"}], "temp_units": "celsius"}, + }], + }), + ), + ( + gemma4_template(), + "<|channel>thought\nThe user is asking for the temperature.<|tool_call>call:get_current_temperature{detail_level:0,location:<|\"|>Paris, France<|\"|>,unit:<|\"|>celsius<|\"|>}<|tool_response>", + json!({ + "reasoning": "The user is asking for the temperature.", + "tool_calls": [{ + "name": "get_current_temperature", + "arguments": {"detail_level": 0, "location": "Paris, France", "unit": "celsius"}, + }], + }), + ), + ( + gemma4_template(), + "<|channel>thought\nLet me call the tool.<|tool_call>call:foo{bool_value:true,list_value:[<|\"|>foo<|\"|>,<|\"|>bar<|\"|>],null_value:null,number_value:1,string_value:<|\"|>foo<|\"|>,struct_value:{foo:<|\"|>bar<|\"|>}}", + json!({ + "reasoning": "Let me call the tool.", + "tool_calls": [{ + "name": "foo", + "arguments": { + "bool_value": true, + "list_value": ["foo", "bar"], + "null_value": null, + "number_value": 1, + "string_value": "foo", + "struct_value": {"foo": "bar"}, + }, + }], + }), + ), + ]; + for (template, text, expected) in cases { + assert_eq!(parse(template, text), expected, "{text}"); + } +} + +/// Upstream fixtures whose patterns have no literal prefix are rejected at load. +#[test] +fn patterns_without_literal_prefix_are_unsupported() { + let ernie = json!({ + "start_anchor": "Assistant:", + "fields": {"thinking": {"open_pattern": r"(?:^|\s*)", "close": ""}}, + }); + let inkling = json!({ + "start_anchor": "<|message_model|>", + "fields": {"content": {"open_pattern": r"(?:<\|message_model\|>)?[^<]*<\|content_text\|>", "close": "<|end_message|>"}}, + }); + for template in [ernie, inkling] { + assert!(matches!( + ResponseTemplate::from_json(&template), + Err(HfTemplateError::Unsupported { .. }) + )); + } +} + +#[test] +fn streaming_matches_whole_string_for_every_chunking() { + let fixtures = [ + ( + cohere_template(), + "<|START_THINKING|>I should call a tool.<|END_THINKING|><|START_ACTION|>[{\"tool_call_id\": \"0\", \"tool_name\": \"simple_tool\", \"parameters\": {\"a\": 1}}]<|END_ACTION|>", + ), + ( + gpt_oss_template(), + "<|channel|>analysis<|message|>thinking chunk<|end|><|channel|>final<|message|>done text", + ), + ( + gpt_oss_template(), + "<|channel|>analysis<|message|>Let me check.<|end|><|start|>assistant<|channel|>commentary to=functions.get_current_weather <|constrain|>json<|message|>{\"location\": \"San Francisco, CA\"}<|call|>", + ), + ( + smollm_template(), + "thinking\n{\"name\": \"fn\", \"arguments\": {\"x\": 1}}", + ), + ( + qwen3_template(), + "short thought\n\n\n\nParis\n\n\n", + ), + ( + gemma4_template(), + "<|channel>thought\nhi 天气前言 <|tool_call>call:foo{a:1,b:<|\"|>bar<|\"|>} after", + ), + ( + invoke_template(), + "header to=self<|msg|>plan it<|pause|>to=user<|msg|>Hello <|pause|>\"Paris\"3to=note<|msg|>world<|end|>", + ), + ]; + for (template, text) in fixtures { + let template = compile(template); + let expected = message(&parse_events(&template, &[], "", &[text]).unwrap()); + let mut chunkings: Vec> = + [1, 2, 3, 5, 7, 13, 31].map(|step| split_by_chars(text, step)).into(); + chunkings.extend((0..30).map(|seed| random_chunks(text, 0xC0DE_5EED + seed))); + for chunks in chunkings { + assert_eq!(chunks.concat(), text); + let actual = message(&parse_events(&template, &[], "", &chunks).unwrap()); + assert_eq!(actual, expected, "{chunks:?}"); + } + } +} + +#[test] +fn gemma4_streams_text_and_starts_calls_at_the_opener() { + let template = compile(gemma4_template()); + let chunks = [ + "<|channel>thought\n", + "Check ", + "the weather. ", + "Sure", + ". <|tool_call>call:get_weather", + "{location:<|\"|>Paris<|\"|>}", + "", + ]; + let tokenizer = tokenizer(); + let mut parser = HfUnifiedParser::new(template, &[], tokenizer); + parser.initialize(&[]).unwrap(); + let steps: Vec<_> = chunks + .iter() + .map(|chunk| { + let mut output = UnifiedParserOutput::default(); + parser.parse_into(DecodedText::unattributed(*chunk), &mut output).unwrap(); + (*chunk, output.events) + }) + .collect(); + expect_test::expect![[r#" + [ + ( + "<|channel>thought\n", + [], + ), + ( + "Check ", + [ + Reasoning( + DecodedText { + text: "Check", + attributions: [], + }, + ), + ], + ), + ( + "the weather. ", + [ + Reasoning( + DecodedText { + text: " the weather.", + attributions: [], + }, + ), + ], + ), + ( + "Sure", + [ + Text( + "Sure", + ), + ], + ), + ( + ". <|tool_call>call:get_weather", + [ + Text( + ".", + ), + ], + ), + ( + "{location:<|\"|>Paris<|\"|>}", + [ + ToolCall( + ToolCallDelta { + tool_index: 0, + name: Some( + "get_weather", + ), + arguments: "", + }, + ), + ], + ), + ( + "", + [ + ToolCall( + ToolCallDelta { + tool_index: 0, + name: None, + arguments: "{\"location\":\"Paris\"}", + }, + ), + ], + ), + ] + "#]] + .assert_debug_eq(&steps); +} + +#[test] +fn invoke_protocol_discards_unrouted_text_and_joins_content() { + let text = "stray to=self<|msg|> plan <|pause|>to=user<|msg|>Hello, <|pause|>betweenxto=note<|msg|>world<|end|>tail"; + expect_test::expect![[r#" + Object { + "reasoning": String("plan"), + "content": String("Hello,world"), + "tool_calls": Array [ + Object { + "name": String("get"), + "arguments": Object { + "n": String("x"), + }, + }, + ], + } + "#]] + .assert_debug_eq(&parse(invoke_template(), text)); + + // Unicode word boundaries: `é` is a word character, so `éname=` is not the `name` attribute. + let text = "北京"; + assert_eq!( + parse(invoke_template(), text), + json!({"tool_calls": [{"name": "获取", "arguments": {"城市": "北京"}}]}) + ); +} + +#[test] +fn join_concatenates_repeated_matches() { + let template = json!({ + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "thinking": {"open": "", "close": "", "repeats": true, "join": " "}, + "content": {"repeats": true, "join": " "}, + }, + }); + assert_eq!( + parse( + template.clone(), + "firstmiddleseconddone" + ), + json!({"reasoning": "first second", "content": "middle done"}) + ); + assert_eq!( + parse(template, "only"), + json!({"reasoning": "only"}) + ); +} + +#[test] +fn transforms_with_dotted_paths() { + let template = json!({ + "start_anchor": "<|assistant|>", + "fields": { + "tool_calls": { + "open": "", + "close": "", + "content": "json", + "transform_each": true, + "transform": {"type": "function", "function": {"name": "{fn.name}", "arguments": "{fn.args}"}}, + }, + }, + }); + assert_eq!( + parse( + template, + r#"[{"fn": {"name": "a", "args": {"x": 1}}}, {"fn": {"name": "b", "args": {}}}]"# + ), + json!({"tool_calls": [{"name": "a", "arguments": {"x": 1}}, {"name": "b", "arguments": {}}]}) + ); +} + +#[test] +fn literal_boundaries() { + let template = json!({ + "start_anchor": "<|assistant|>", + "fields": {"content": {"open": ["", ""], "close": ["", ""]}}, + }); + for (opener, closer) in [("", ""), ("", ""), ("", "")] { + assert_eq!( + parse(template.clone(), &format!("{opener}hi{closer}")), + json!({"content": "hi"}) + ); + } + + // A literal that is a strict prefix of another in the same list defers at the buffer edge. + let template = compile(json!({ + "start_anchor": "<|assistant|>", + "fields": {"content": {"open": "", "close": ["END", "ENDX"]}}, + })); + let events = parse_events(&template, &[], "", &["hiEND", " more"]).unwrap(); + assert_eq!(message(&events), json!({"content": "hi"})); + + // A field without a close runs to the end of the stream. + let template = + json!({"start_anchor": "<|assistant|>", "fields": {"content": {"open": ""}}}); + assert_eq!( + parse(template, "hello world"), + json!({"content": "hello world"}) + ); +} + +#[test] +fn required_field_missing_fails_at_finish() { + let template = compile(json!({ + "start_anchor": "<|assistant|>", + "fields": {"content": {"open": "", "close": "", "optional": false}}, + })); + let error = parse_events(&template, &[], "", &["no response here"]).unwrap_err(); + expect_test::expect![[r#" + ParsingFailed { + message: "Required response_template fields missing from parsed output: [\"content\"]", + } + "#]] + .assert_debug_eq(&error); +} + +#[test] +fn prefix_sets_the_initial_state() { + let template = compile(qwen3_template()); + + // The prompt ends inside the thinking block; the prefilled newline is stripped. + let prompt = "<|im_start|>system\nYou are helpful<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n\n"; + let events = parse_events(&template, &[], prompt, &["Let me think..."]).unwrap(); + assert_eq!(message(&events), json!({"reasoning": "Let me think..."})); + + // Only the slice after the last anchor matters. + let prompt = "<|im_start|>assistant\nEarlier reply<|im_end|>\n<|im_start|>user\nFollowup<|im_end|>\n<|im_start|>assistant\n\n"; + let events = parse_events(&template, &[], prompt, &["done"]).unwrap(); + assert_eq!(message(&events), json!({"reasoning": "done"})); + + // A prefix ending mid-delimiter is completed by the generated text. + let events = parse_events( + &template, + &[], + "<|im_start|>assistant\nreal body"], + ) + .unwrap(); + assert_eq!(message(&events), json!({"reasoning": "real body"})); + + // Gemma 4 continues inside the same turn after a tool response. + let template = compile(gemma4_template()); + let prompt = "<|turn>model\n<|tool_call>call:f{}<|tool_response>response:f{}<|channel>thought\n"; + let events = parse_events(&template, &[], prompt, &["Done.It is sunny."]).unwrap(); + assert_eq!( + message(&events), + json!({"reasoning": "Done.", "content": "It is sunny."}) + ); + + // Prompt text is never emitted, but it counts for leading-whitespace stripping. + let events = parse_events(&template, &[], "<|turn>model\nSure,", &[" here", " it is"]).unwrap(); + assert_eq!(message(&events), json!({"content": " here it is"})); +} + +/// Diverges from Transformers, which runs the whole prompt through the parser +/// when the start anchor is absent. +#[test] +fn missing_start_anchor_starts_from_the_initial_state() { + let template = compile(smollm_template()); + let events = parse_events(&template, &[], "\n", &["hi"]).unwrap(); + assert_eq!(message(&events), json!({"content": "hi"})); +} + +#[test] +fn tools_convert_string_arguments() { + let template = compile(json!({ + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "tool_calls": { + "open_pattern": r"\s*\w+)>", + "close": "", + "repeats": true, + "content": "xml-inline", + "content_args": { + "tag_pattern": r"\w+)>\s*(?P.*?)\s*", + "merge_duplicates": true, + }, + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + }, + })); + let tools = [Tool { + name: "set_alarm".to_string(), + description: None, + parameters: json!({ + "type": "object", + "properties": { + "hour": {"type": "integer"}, + "enabled": {"type": "boolean"}, + "label": {"type": "string"}, + "weekday": {"type": "integer"}, + }, + }), + strict: None, + }]; + let text = "\n\n\n7\n\n\ntrue\n\n\nwake up\n\n12\n\n"; + assert_eq!( + message(&parse_events(&template, &tools, "", &[text]).unwrap()), + json!({"tool_calls": [{"name": "set_alarm", "arguments": {"hour": 7, "enabled": true, "label": "wake up", "weekday": [1, 2]}}]}) + ); + // Without tools, values stay strings. + assert_eq!( + message(&parse_events(&template, &[], "", &[text]).unwrap()), + json!({"tool_calls": [{"name": "set_alarm", "arguments": {"hour": "7", "enabled": "true", "label": "wake up", "weekday": ["1", "2"]}}]}) + ); +} + +#[test] +fn stripped_reasoning_whitespace_keeps_its_tokens() { + let template = compile(gemma4_template()); + let mut parser = HfUnifiedParser::new(template, &[], tokenizer()); + parser.initialize(&[]).unwrap(); + let text = "<|channel>thought\n \n plan \n"; + let mut output = UnifiedParserOutput::default(); + for (index, c) in text.chars().enumerate() { + let piece = DecodedText { + text: c.to_string(), + attributions: [TokenAttribution { + token_id: u32::try_from(index).unwrap(), + anchor: TokenAnchor::Visible { byte_offset: 0 }, + }] + .into_iter() + .collect(), + }; + parser.parse_into(piece, &mut output).unwrap(); + } + output.append(parser.finish().unwrap()); + let (reasoning, tokens) = + output + .events + .iter() + .fold((String::new(), 0), |(text, tokens), event| match event { + UnifiedParserEvent::Reasoning(piece) => { + (text + &piece.text, tokens + piece.attributions.len()) + } + _ => (text, tokens), + }); + assert_eq!(reasoning, "plan"); + // Every character between the markers is a reasoning token, stripped or not. + assert_eq!(tokens, " \n plan \n".chars().count()); +} + +#[test] +fn reset_returns_uncommitted_text() { + let template = compile(gemma4_template()); + let mut parser = HfUnifiedParser::new(template, &[], tokenizer()); + parser.initialize(&[]).unwrap(); + let mut output = UnifiedParserOutput::default(); + parser + .parse_into( + DecodedText::unattributed("Hi <|tool_call>call:f{a:<|\"|>x"), + &mut output, + ) + .unwrap(); + assert_eq!(parser.reset(), "<|tool_call>call:f{a:<|\"|>x"); +} + +#[test] +fn malformed_tool_arguments_fail_the_parser() { + let template = compile(gemma4_template()); + let error = + parse_events(&template, &[], "", &["<|tool_call>call:f{a:}"]).unwrap_err(); + assert!( + matches!(error, UnifiedParserError::ParsingFailed { .. }), + "{error:?}" + ); +} + +#[test] +fn template_validation_errors() { + let error = |template: Value| ResponseTemplate::from_json(&template).unwrap_err(); + expect_test::expect![[r#" + [ + Invalid { + message: "unsupported response_template version: 2", + }, + Invalid { + message: "At most one field may omit 'open'/'open_pattern' (that field becomes the implicit-open / leftover sink). Found: content, thinking", + }, + Invalid { + message: "response_template must define 'start_anchor' or 'start_anchor_pattern'.", + }, + Invalid { + message: "Field 'content': 'join' requires 'repeats': true", + }, + Invalid { + message: "Field 'tool_calls': open_pattern/close_pattern declares named group(s) [\"name\"], but the field has no 'transform'. Named captures are only surfaced through a 'transform' template (where they appear alongside 'content'). Either add a 'transform' that uses the captures, or remove the named groups from the pattern.", + }, + Invalid { + message: "Field 'tool_calls': transform string \"name: {content}\" mixes a {placeholder} with literal text. Use either a whole-string placeholder (e.g. \"{content}\") or a plain literal; string interpolation is not supported.", + }, + Invalid { + message: "Field 'content': 'open' literals cannot be empty strings", + }, + Unsupported { + message: "Field 'count': only 'content', 'reasoning_content'/'thinking', and 'tool_calls' fields can be reported", + }, + Unsupported { + message: "Field 'thinking': text and reasoning fields must use the 'text' content parser without a transform", + }, + Invalid { + message: "response_template.fields.content.content: unknown variant `yaml`, expected one of `text`, `int`, `float`, `bool`, `json`, `xml-inline`, `kv-lines`", + }, + Invalid { + message: "response_template.fields.content.repeats: invalid type: string \"yes\", expected a boolean", + }, + Invalid { + message: "Field 'content': cannot specify both 'open' and 'open_pattern'", + }, + Invalid { + message: "Field 'tool_calls': transform_each is set but no transform was provided", + }, + Unsupported { + message: "Field 'tool_calls': 'join' requires each match to parse to a string", + }, + Invalid { + message: "Field 'tool_calls': transform placeholder '{id}' is neither 'content' nor a named group of open_pattern", + }, + ] + "#]] + .assert_debug_eq(&[ + error(json!({"version": 2, "start_anchor": "a", "fields": {"content": {}}})), + error(json!({"start_anchor": "a", "fields": {"content": {}, "thinking": {}}})), + error(json!({"fields": {"content": {}}})), + error(json!({"start_anchor": "a", "fields": {"content": {"open": "", "join": ""}}})), + error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open_pattern": r"\w+)>", "close": ""}}})), + error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open": "", "transform": {"label": "name: {content}"}}}})), + error(json!({"start_anchor": "a", "fields": {"content": {"open": [""]}}})), + error(json!({"start_anchor": "a", "fields": {"count": {"open": "", "content": "int"}}})), + error(json!({"start_anchor": "a", "fields": {"thinking": {"open": "", "content": "json"}}})), + error(json!({"start_anchor": "a", "fields": {"content": {"content": "yaml"}}})), + error(json!({"start_anchor": "a", "fields": {"content": {"repeats": "yes"}}})), + error(json!({"start_anchor": "a", "fields": {"content": {"open": "", "open_pattern": ""}}})), + error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open": "", "transform_each": true}}})), + error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open": "", "repeats": true, "join": ""}}})), + error(json!({ + "start_anchor": "a", + "fields": {"tool_calls": { + "open_pattern": r"\w+)>", + "close_pattern": r"\d+)>", + "content": "json", + "transform": {"function": {"name": "{name}", "arguments": "{content}", "id": "{id}"}}, + }}, + })), + ]); +} + +/// Cases where the Rust parser intentionally differs from Transformers, keyed by +/// generated text. +const DIVERGENCES: &[(&str, &str)] = &[ + ( + // A field without `repeats` keeps only its last occurrence in Transformers; + // streamed text cannot be retracted, so every occurrence is reported. + r#"Before <|tool_call>call:f{x:<|"|>a, b<|"|>,y:[1,2.5,-3],z:{w:true}} after"#, + r#"{"content": "Before after", "tool_calls": [{"name": "f", "arguments": {"x": "a, b", "y": [1, 2.5, -3], "z": {"w": true}}}]}"#, + ), + ( + // Arguments use the tool parsers' shared conversion, which keeps a JSON + // number's spelling; Transformers turns `1e3` into the integer 1000. + "\n\nhour: seven\nratio: 1e3\nenabled: nope\n", + r#"{"tool_calls": [{"name": "set_alarm", "arguments": {"hour": "seven", "ratio": 1000.0, "enabled": "nope"}}]}"#, + ), +]; + +/// Compare against `fixtures/differential.json`, generated from Transformers +/// `parse_response` by `fixtures/generate_differential.py`. +#[test] +fn differential_against_transformers() { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("src/unified/hf/fixtures/differential.json"); + let corpus: Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let templates: BTreeMap<&str, Arc> = corpus["templates"] + .as_object() + .unwrap() + .iter() + .map(|(name, template)| { + ( + name.as_str(), + Arc::new(ResponseTemplate::from_json(template).unwrap()), + ) + }) + .collect(); + + for case in corpus["cases"].as_array().unwrap() { + let template = &templates[case["template"].as_str().unwrap()]; + let prefix = case["prefix"].as_str().unwrap(); + let text = case["text"].as_str().unwrap(); + let tools: Vec = case["tools"] + .as_array() + .into_iter() + .flatten() + .map(|tool| Tool { + name: tool["function"]["name"].as_str().unwrap().to_string(), + description: None, + parameters: tool["function"]["parameters"].clone(), + strict: None, + }) + .collect(); + let expected = match DIVERGENCES.iter().find(|(divergent, _)| *divergent == text) { + Some((_, expected)) => serde_json::from_str(expected).unwrap(), + None => case["expected"].clone(), + }; + + for chunks in [vec![text], split_by_chars(text, 1)] { + let actual = parse_events(template, &tools, prefix, &chunks); + match (&actual, expected.get("error")) { + (Err(_), Some(_)) => {} + (Ok(events), None) => assert_eq!(message(events), expected, "{text:?}"), + _ => panic!("{text:?}: expected {expected}, got {actual:?}"), + } + } + } +} + +#[test] +fn tool_call_value_shapes() { + // No transform: the parsed JSON is the tool call itself. + let template = compile(json!({ + "start_anchor": "<|assistant|>", + "fields": {"tool_calls": {"open": "", "close": "", "content": "json"}}, + })); + let parse = + |text: &str| parse_events(&template, &[], "", &[text]).map(|events| message(&events)); + + assert_eq!( + parse(r#"{"type": "function", "function": {"name": "f", "arguments": {"a": 1}}}"#).unwrap(), + json!({"tool_calls": [{"name": "f", "arguments": {"a": 1}}]}) + ); + assert_eq!( + parse(r#"[{"name": "f", "arguments": null}, {"name": "g", "arguments": "raw"}]"#).unwrap(), + json!({"tool_calls": [{"name": "f", "arguments": {}}, {"name": "g", "arguments": "raw"}]}) + ); + expect_test::expect![[r#" + ParsingFailed { + message: "tool call must be {\"function\": {\"name\": ..., \"arguments\": ...}} or {\"name\": ..., \"arguments\": ...} with a string name, got {\"name\":1}", + } + "#]] + .assert_debug_eq(&parse(r#"{"name": 1}"#).unwrap_err()); +} diff --git a/rust/src/parser/src/unified/hf/transform.rs b/rust/src/parser/src/unified/hf/transform.rs new file mode 100644 index 0000000000..9d3ab6913f --- /dev/null +++ b/rust/src/parser/src/unified/hf/transform.rs @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Transform templates: restructure a parsed region value into the output shape. +//! +//! Port of `_apply_transform`, `validate_transform_strings`, and `process_field` +//! from `content_parsers.py`. + +use std::sync::LazyLock; + +use regex_automata::meta::Regex; +use serde_json::{Map, Value}; + +use super::pattern::Captures; +use super::{Result, invalid}; +use crate::unified::{self, parsing_failed}; + +/// `\{(\w+(?:\.\w+)*)\}`: a whole-string placeholder such as `{content}` or +/// `{content.args}`. +static PLACEHOLDER: LazyLock = + LazyLock::new(|| Regex::new(r"\{\w+(?:\.\w+)*\}").expect("valid regex")); + +/// A compiled transform template. +#[derive(Debug, Clone, PartialEq)] +pub(super) enum Transform { + Object(Vec<(String, Transform)>), + Array(Vec), + /// A dotted placeholder path, e.g. `["content", "args"]`. + Placeholder(Vec), + /// Any other JSON value, copied verbatim. + Literal(Value), +} + +impl Transform { + /// Compile a transform template, rejecting any string that mixes a + /// `{name}` placeholder with literal text. + pub fn compile(scope: &str, template: &Value) -> Result { + Ok(match template { + Value::Object(object) => Self::Object( + object + .iter() + .map(|(key, value)| Ok((key.clone(), Self::compile(scope, value)?))) + .collect::>()?, + ), + Value::Array(values) => Self::Array( + values.iter().map(|value| Self::compile(scope, value)).collect::>()?, + ), + Value::String(string) => match PLACEHOLDER.find(string) { + Some(found) if found.range() == (0..string.len()) => Self::Placeholder( + string[1..string.len() - 1].split('.').map(str::to_string).collect(), + ), + Some(_) => { + return Err(invalid!( + "{scope}: transform string {string:?} mixes a {{placeholder}} with literal text. \ + Use either a whole-string placeholder (e.g. \"{{content}}\") or a plain literal; \ + string interpolation is not supported." + )); + } + None => Self::Literal(template.clone()), + }, + other => Self::Literal(other.clone()), + }) + } + + /// Return the placeholder path found at `key_path` inside this template, if any. + pub fn placeholder_at(&self, key_path: &[&str]) -> Option<&[String]> { + match (self, key_path) { + (Self::Placeholder(path), []) => Some(path), + (Self::Object(entries), [key, rest @ ..]) => entries + .iter() + .find(|(entry, _)| entry == key) + .and_then(|(_, transform)| transform.placeholder_at(rest)), + _ => None, + } + } + + /// Root names of every placeholder in this template. + pub fn placeholder_roots(&self) -> Box + '_> { + match self { + Self::Object(entries) => { + Box::new(entries.iter().flat_map(|(_, transform)| transform.placeholder_roots())) + } + Self::Array(transforms) => { + Box::new(transforms.iter().flat_map(Self::placeholder_roots)) + } + Self::Placeholder(path) => Box::new(path.first().map(String::as_str).into_iter()), + Self::Literal(_) => Box::new(std::iter::empty()), + } + } + + /// Recursively instantiate this template against `scope`. + pub fn apply(&self, scope: &Map) -> unified::Result { + Ok(match self { + Self::Object(entries) => Value::Object( + entries + .iter() + .map(|(key, transform)| Ok((key.clone(), transform.apply(scope)?))) + .collect::>()?, + ), + Self::Array(transforms) => Value::Array( + transforms + .iter() + .map(|transform| transform.apply(scope)) + .collect::>()?, + ), + Self::Literal(value) => value.clone(), + Self::Placeholder(path) => { + let dotted = path.join("."); + let (root, keys) = path.split_first().expect("placeholder path is non-empty"); + let Some(mut value) = scope.get(root) else { + let available: Vec<_> = scope.keys().collect(); + return Err(parsing_failed!( + "transform placeholder '{{{dotted}}}' is not defined. Available: {available:?}" + )); + }; + for key in keys { + let Some(object) = value.as_object() else { + return Err(parsing_failed!( + "transform placeholder '{{{dotted}}}' cannot index into {} at '{key}'", + json_type_name(value) + )); + }; + let Some(next) = object.get(key) else { + let available: Vec<_> = object.keys().collect(); + return Err(parsing_failed!( + "transform placeholder '{{{dotted}}}' is missing key '{key}'. Available: {available:?}" + )); + }; + value = next; + } + value.clone() + } + }) + } +} + +/// A field's transform: applied to the parsed value, or to each of its elements. +#[derive(Debug, Clone, PartialEq)] +pub(super) struct FieldTransform { + pub template: Transform, + /// `transform_each`: the parsed content must be a list of dicts, and the + /// template is applied to each element (with the element's keys unpacked into + /// the template scope, alongside any regex captures). + pub each: bool, +} + +impl FieldTransform { + /// Apply the transform to a parsed region value. + pub fn apply( + &self, + field_name: &str, + parsed: Value, + captures: &Captures, + ) -> unified::Result { + if !self.each { + let mut scope = captures.clone(); + scope.insert("content".to_string(), parsed); + return self.template.apply(&scope); + } + + let Value::Array(items) = parsed else { + return Err(parsing_failed!( + "Field '{field_name}': transform_each requires the parsed content to be an array, got {}.", + json_type_name(&parsed) + )); + }; + items + .into_iter() + .map(|item| { + let Value::Object(item) = item else { + return Err(parsing_failed!( + "Field '{field_name}': transform_each requires each array element to be an object, got {}.", + json_type_name(&item) + )); + }; + let mut scope = captures.clone(); + scope.extend(item); + self.template.apply(&scope) + }) + .collect::>() + .map(Value::Array) + } +} + +/// JSON type name of a value, for error messages. +fn json_type_name(value: &Value) -> &'static str { + match value { + Value::Null => "null", + Value::Bool(_) => "boolean", + Value::Number(_) => "number", + Value::String(_) => "string", + Value::Array(_) => "array", + Value::Object(_) => "object", + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn scope(value: Value) -> Map { + value.as_object().unwrap().clone() + } + + #[test] + fn dotted_placeholders_descend_into_objects() { + let transform = Transform::compile( + "test", + &json!({"type": "function", "function": {"name": "{content.name}", "arguments": "{content.args}"}}), + ) + .unwrap(); + let value = transform + .apply(&scope(json!({"content": {"name": "f", "args": {"a": 1}}}))) + .unwrap(); + assert_eq!( + value, + json!({"type": "function", "function": {"name": "f", "arguments": {"a": 1}}}) + ); + assert_eq!( + transform.placeholder_at(&["function", "name"]), + Some(["content".to_string(), "name".to_string()].as_slice()) + ); + } + + #[test] + fn placeholder_errors_match_transformers() { + let apply = |template: Value, scope_value: Value| { + Transform::compile("test", &template) + .unwrap() + .apply(&scope(scope_value)) + .unwrap_err() + }; + expect_test::expect![[r#" + ParsingFailed { + message: "transform placeholder '{name}' is not defined. Available: [\"content\"]", + } + "#]] + .assert_debug_eq(&apply(json!("{name}"), json!({"content": 1}))); + expect_test::expect![[r#" + ParsingFailed { + message: "transform placeholder '{content.a}' cannot index into number at 'a'", + } + "#]] + .assert_debug_eq(&apply(json!("{content.a}"), json!({"content": 1}))); + expect_test::expect![[r#" + ParsingFailed { + message: "transform placeholder '{content.b}' is missing key 'b'. Available: [\"a\"]", + } + "#]] + .assert_debug_eq(&apply(json!("{content.b}"), json!({"content": {"a": 1}}))); + } + + #[test] + fn interpolation_is_rejected_at_load() { + assert!(Transform::compile("test", &json!({"name": "fn_{name}"})).is_err()); + assert_eq!( + Transform::compile("test", &json!("no placeholder")).unwrap(), + Transform::Literal(json!("no placeholder")) + ); + } +} diff --git a/rust/src/parser/src/unified/mod.rs b/rust/src/parser/src/unified/mod.rs index c621075a02..aa0e92e6e1 100644 --- a/rust/src/parser/src/unified/mod.rs +++ b/rust/src/parser/src/unified/mod.rs @@ -5,12 +5,14 @@ mod combined; mod gemma4; +mod hf; mod hy; mod inkling; mod kimi_k3; pub use combined::CombinedParser; pub use gemma4::Gemma4UnifiedParser; +pub use hf::{HfTemplateError, HfUnifiedParser, ResponseTemplate}; pub use hy::{HyV3UnifiedParser, HyV4UnifiedParser}; pub use inkling::InklingUnifiedParser; pub use kimi_k3::{KimiK3StructuralTagBuilder, KimiK3UnifiedParser}; @@ -271,8 +273,13 @@ pub trait UnifiedParser: Send { #[derive(Debug, Error, Macro)] #[thiserror_ext(macro(path = "crate::unified", mangle))] pub enum UnifiedParserError { - #[error("combined parser is constructed from split parser instances")] - CombinedParserConstructor, + /// The parser is built from inputs other than tools and a tokenizer, so + /// [`UnifiedParser::create`] cannot construct it. + #[error("the `{parser}` unified parser is built from {built_from}, not by name")] + NoNamedConstructor { + parser: &'static str, + built_from: &'static str, + }, #[error("tokenizer is missing unified parser token `{token}`")] MissingToken { token: String }, #[error("unified parser parsing failed: {message}")] diff --git a/rust/src/text/src/backend/hf/config.rs b/rust/src/text/src/backend/hf/config.rs index 0768adf4ae..0864df641b 100644 --- a/rust/src/text/src/backend/hf/config.rs +++ b/rust/src/text/src/backend/hf/config.rs @@ -23,6 +23,9 @@ pub struct HfTokenizerConfig { /// containing "Tiktoken" which can be used as a hint for backend /// selection. pub tokenizer_class: Option, + /// The Hugging Face `response_template`: a declarative description of the + /// model's output protocol, executed by the `hf` unified parser. + pub response_template: Option, } /// Hugging Face named special tokens may be serialized as a string or an