iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Rust Frontend] Add hf parser for response templates (#59005)

Browse Source 
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-10-01 08:16:38 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 91fcec1235
commit 24fb5fc08c
23 changed files with 4573 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/Cargo.lock
Generated
+4
View File
@@ -5971,12 +5971,16 @@ dependencies = [
"expect-test",
"futures",
"openai-protocol",
"regex-automata",
"regex-syntax",
"serde",
"serde_json",
"serde_path_to_error",
"serde_with",
"thiserror 2.0.18",
"thiserror-ext",
"tool-parser",
"tracing",
"vllm-tokenizer",
"winnow",
"xgrammar-structural-tag",
rust/Cargo.toml
+3
View File
@@ -85,6 +85,8 @@ rand = "0.9.3"
rand_distr = "0.5.1"
rayon = "1.11.0"
reasoning-parser = "1.2.2"
regex-automata = "0.4.14"
regex-syntax = "0.8.10"
reqwest = { version = "0.12.8", default-features = false, features = ["native-tls"] }
reqwest-0-13 = { package = "reqwest", version = "0.13.4", default-features = false, features = ["native-tls"] }
riptoken = { version = "0.3.0", default-features = false }
@@ -96,6 +98,7 @@ serde = { version = "1.0.228", features = ["derive", "rc"] }
serde-json-fmt = "0.1.0"
serde_default = "0.2.0"
serde_json = { version = "1.0.145", features = ["preserve_order"] }
serde_path_to_error = "0.1.20"
serde_repr = "0.1.20"
serde_tuple = "1.1.3"
serde_with = "3.21.0"
rust/src/chat/src/backend/hf.rs
+46 -10
View File
@@ -3,8 +3,11 @@
use std::sync::Arc;
use tracing::info;
use vllm_text::backend::hf::{HfTextBackend, ResolvedModelFiles, load_model_config};
use thiserror_ext::AsReport as _;
use tracing::{info, warn};
use vllm_text::backend::hf::{
HfTextBackend, ResolvedModelFiles, load_model_config, load_tokenizer_config,
};
use vllm_text::tokenizer::DynTokenizer;
use vllm_text::{DynTextBackend, TextBackend as _};
@@ -17,6 +20,7 @@ use crate::multimodal::{MultimodalConfigFiles, MultimodalModelInfo};
use crate::output::{
DefaultChatOutputProcessor, HarmonyChatOutputProcessor, validate_harmony_parser_overrides,
};
use crate::parser::unified::{HfTemplateError, ResponseTemplate};
use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
use crate::renderer::{
DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer, DynChatRenderer,
@@ -32,6 +36,9 @@ pub struct HfChatBackend {
tokenizer: DynTokenizer,
chat_renderer: DynChatRenderer,
multimodal_model_info: Option<MultimodalModelInfo>,
/// `response_template` from `tokenizer_config.json` for the `hf` parser, or
/// why it is unavailable.
response_template: std::result::Result<Arc<ResponseTemplate>, HfTemplateError>,
}
impl HfChatBackend {
@@ -61,6 +68,7 @@ impl HfChatBackend {
)?
};
let multimodal_render_info = resolve_multimodal_render_info(multimodal_model_info.as_ref());
let response_template = load_response_template(&files)?;
let renderer = options.renderer.resolve(model_type);
let chat_renderer: DynChatRenderer = match renderer {
@@ -105,10 +113,35 @@ impl HfChatBackend {
tokenizer,
chat_renderer,
multimodal_model_info,
response_template,
})
}
}
/// Load and compile the checkpoint's `response_template`, if any.
///
/// A missing or unusable template is kept as an error, so that only requests
/// selecting the `hf` parser fail.
fn load_response_template(
files: &ResolvedModelFiles,
) -> Result<std::result::Result<Arc<ResponseTemplate>, HfTemplateError>> {
let Some(template) =
load_tokenizer_config(files.tokenizer_config_path.as_deref())?.response_template
else {
return Ok(Err(HfTemplateError::Missing));
};
Ok(match ResponseTemplate::from_json(&template) {
Ok(template) => {
info!("loaded response_template for the `hf` parser");
Ok(Arc::new(template))
}
Err(error) => {
warn!(error = %error.as_report(), "the `hf` parser cannot execute the model's response_template");
Err(error)
}
})
}
impl ChatBackend for HfChatBackend {
fn chat_renderer(&self) -> DynChatRenderer {
self.chat_renderer.clone()
@@ -128,14 +161,17 @@ impl ChatBackend for HfChatBackend {
return Ok(Box::new(HarmonyChatOutputProcessor::new(request)?));
}
Ok(Box::new(DefaultChatOutputProcessor::new(
request,
&self.model_id,
self.tokenizer.clone(),
options.tool_call_parser,
options.reasoning_parser,
options.tool_strict_level,
)?))
Ok(Box::new(
DefaultChatOutputProcessor::with_response_template(
request,
&self.model_id,
self.tokenizer.clone(),
&self.response_template,
options.tool_call_parser,
options.reasoning_parser,
options.tool_strict_level,
)?,
))
}
}
rust/src/chat/src/lib.rs
+8 -2
View File
@@ -412,6 +412,12 @@ mod tests {
validate_parser_overrides(&selection, &selection).unwrap();
}
#[test]
fn validate_parser_overrides_accepts_explicit_hf() {
let selection = ParserSelection::Explicit("hf".to_string());
validate_parser_overrides(&selection, &selection).unwrap();
}
#[test]
fn validate_parser_overrides_accepts_auto_and_none() {
validate_parser_overrides(&ParserSelection::Auto, &ParserSelection::None).unwrap();
@@ -425,7 +431,7 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hf, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
}
#[test]
@@ -436,6 +442,6 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hf, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
}
}
rust/src/chat/src/output/default/mod.rs
+145 -5
View File
@@ -5,7 +5,7 @@
mod unified;
use std::sync::Once;
use std::sync::{Arc, Once};
use futures::StreamExt as _;
use tracing::info;
@@ -20,7 +20,9 @@ use crate::error::Result;
use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream};
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
use crate::parser::tool::{ToolParser, ToolParserFactory};
use crate::parser::unified::UnifiedParserFactory;
use crate::parser::unified::{
HfTemplateError, HfUnifiedParser, ResponseTemplate, UnifiedParserFactory, names,
};
use crate::parser::{ParserSelection, ToolStrictLevel};
use crate::request::{ChatRequest, ChatTool};
use crate::{Error, Result as ChatResult};
@@ -64,11 +66,48 @@ impl DefaultChatOutputProcessor {
reasoning_parser: &ParserSelection,
tool_strict_level: ToolStrictLevel,
) -> ChatResult<Self> {
Self::with_response_template(
request,
model_id,
tokenizer,
&Err(HfTemplateError::Missing),
tool_call_parser,
reasoning_parser,
tool_strict_level,
)
}
/// Like [`Self::new`], additionally providing the model's `response_template`
/// for the `hf` parser, or why it is unavailable.
pub fn with_response_template(
request: &mut ChatRequest,
model_id: &str,
tokenizer: DynTokenizer,
response_template: &std::result::Result<Arc<ResponseTemplate>, HfTemplateError>,
tool_call_parser: &ParserSelection,
reasoning_parser: &ParserSelection,
tool_strict_level: ToolStrictLevel,
) -> ChatResult<Self> {
let tool_name = tool_call_parser.resolve_tool_name(model_id);
let reasoning_name = reasoning_parser.resolve_reasoning_name(model_id);
// `hf` is opt-in: `Auto` never selects it, but a usable template is worth a hint.
let auto = [tool_call_parser, reasoning_parser]
.iter()
.any(|selection| matches!(selection, ParserSelection::Auto));
if auto && tool_name.is_none() && reasoning_name.is_none() && response_template.is_ok() {
RESPONSE_TEMPLATE_HINT_ONCE.call_once(|| {
info!(
"the model ships a response_template; pass `--tool-call-parser hf \
--reasoning-parser hf` to parse with it"
);
});
}
let parser = if let Some(parser) = Self::resolve_optional_unified_parser(
request.tools(),
tokenizer.clone(),
tool_call_parser.resolve_tool_name(model_id),
reasoning_parser.resolve_reasoning_name(model_id),
response_template,
tool_name,
reasoning_name,
)? {
parser
} else {
@@ -141,6 +180,7 @@ impl DefaultChatOutputProcessor {
fn resolve_optional_unified_parser(
tools: &[ChatTool],
tokenizer: DynTokenizer,
response_template: &std::result::Result<Arc<ResponseTemplate>, HfTemplateError>,
tool_name: Option<&str>,
reasoning_name: Option<&str>,
) -> ChatResult<Option<Box<dyn UnifiedParser>>> {
@@ -157,7 +197,18 @@ impl DefaultChatOutputProcessor {
});
}
let parser = factory.create(parser_name, tools, tokenizer)?;
// `hf` is built from the model's template, never by its registry constructor.
let parser = if parser_name == names::HF {
let template =
response_template.as_ref().map_err(|error| Error::ParserInitialization {
kind: "unified",
name: parser_name.to_string(),
error: Box::new(error.clone()),
})?;
Box::new(HfUnifiedParser::new(template.clone(), tools, tokenizer))
} else {
factory.create(parser_name, tools, tokenizer)?
};
UNIFIED_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using unified parser"));
Ok(Some(parser))
@@ -186,6 +237,7 @@ impl DefaultChatOutputProcessor {
static TOOL_PARSER_LOG_ONCE: Once = Once::new();
static REASONING_PARSER_LOG_ONCE: Once = Once::new();
static UNIFIED_PARSER_LOG_ONCE: Once = Once::new();
static RESPONSE_TEMPLATE_HINT_ONCE: Once = Once::new();
impl ChatOutputProcessor for DefaultChatOutputProcessor {
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
@@ -230,10 +282,12 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor {
mod tests {
use std::sync::Arc;
use thiserror_ext::AsReport as _;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::DefaultChatOutputProcessor;
use crate::output::ChatOutputProcessor;
use crate::parser::unified::{HfTemplateError, ResponseTemplate};
use crate::parser::{ParserSelection, ToolStrictLevel};
use crate::request::{ChatRequest, ChatTool, ChatToolChoice, ResolvedToolContext};
@@ -390,4 +444,90 @@ mod tests {
expect_test::expect!["unified parsing requires the tool and reasoning selections to resolve to the same parser; resolved tool=qwen3_xml, reasoning=gemma4"]
.assert_eq(&format!("{error}"));
}
type TemplateResult = Result<Arc<ResponseTemplate>, HfTemplateError>;
fn loaded_template() -> TemplateResult {
Ok(Arc::new(
ResponseTemplate::from_json(&serde_json::json!({
"start_anchor": "<|turn>model\n",
"fields": {
"thinking": {"open": "<|channel>thought\n", "close": "<channel|>"},
"content": {"close": "<turn|>"},
},
}))
.unwrap(),
))
}
fn invalid_template() -> TemplateResult {
let error = ResponseTemplate::from_json(&serde_json::json!({"fields": {"content": {}}}))
.unwrap_err();
Err(error)
}
/// Build a processor for `model_id`, returning whether it keeps special
/// tokens (only parsers that need them, such as `hf`, turn this on).
fn keeps_special_tokens(
model_id: &str,
template: &TemplateResult,
tool: &ParserSelection,
reasoning: &ParserSelection,
) -> crate::Result<bool> {
let mut request = ChatRequest::for_test();
DefaultChatOutputProcessor::with_response_template(
&mut request,
model_id,
tokenizer(),
template,
tool,
reasoning,
ToolStrictLevel::Auto,
)?;
Ok(!request.decode_options.skip_special_tokens)
}
#[test]
fn explicit_hf_uses_the_model_response_template() {
let hf = ParserSelection::Explicit("hf".to_string());
assert!(keeps_special_tokens("other-model", &loaded_template(), &hf, &hf).unwrap());
}
#[test]
fn explicit_hf_without_usable_template_fails() {
let hf = ParserSelection::Explicit("hf".to_string());
let error = |template| {
keeps_special_tokens("other-model", &template, &hf, &hf)
.unwrap_err()
.to_report_string()
};
expect_test::expect!["failed to initialize unified parser `hf`: the model's tokenizer_config.json provides no response_template"]
.assert_eq(&error(Err(HfTemplateError::Missing)));
expect_test::expect!["failed to initialize unified parser `hf`: invalid response_template: response_template must define 'start_anchor' or 'start_anchor_pattern'."]
.assert_eq(&error(invalid_template()));
}
#[test]
fn auto_never_selects_hf() {
let auto = ParserSelection::Auto;
// Without a matching parser, parsing stays disabled even with a usable template.
assert!(!keeps_special_tokens("other-model", &loaded_template(), &auto, &auto).unwrap());
// An unusable template is not consulted unless `hf` is selected.
assert!(!keeps_special_tokens("other-model", &invalid_template(), &auto, &auto).unwrap());
keeps_special_tokens("google/gemma-4-27b-it", &invalid_template(), &auto, &auto).unwrap();
}
#[test]
fn hf_requires_matching_parser_selections() {
let error = keeps_special_tokens(
"other-model",
&loaded_template(),
&ParserSelection::Explicit("hf".to_string()),
&ParserSelection::None,
)
.unwrap_err();
expect_test::expect!["unified parsing requires the tool and reasoning selections to resolve to the same parser; resolved tool=hf, reasoning=none"]
.assert_eq(&format!("{error}"));
}
}
rust/src/chat/src/parser/unified.rs
+9 -2
View File
@@ -6,8 +6,8 @@
use std::sync::LazyLock;
pub use vllm_parser::unified::{
Gemma4UnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser, InklingUnifiedParser,
KimiK3UnifiedParser, UnifiedParser,
Gemma4UnifiedParser, HfTemplateError, HfUnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser,
InklingUnifiedParser, KimiK3UnifiedParser, ResponseTemplate, UnifiedParser,
};
use vllm_tokenizer::DynTokenizer;
@@ -21,6 +21,11 @@ pub mod names {
pub const HY_V4: &str = "hy_v4";
pub const INKLING: &str = "inkling";
pub const KIMI_K3: &str = "kimi_k3";
/// Executes the checkpoint's Hugging Face `response_template`. Registered for
/// name resolution only: it is built from the model's template, never by its
/// registry constructor.
pub const HF: &str = "hf";
}
/// Constructor signature for one registered unified parser implementation.
@@ -48,6 +53,8 @@ impl UnifiedParserFactory {
factory.register_parser::<HyV4UnifiedParser>(names::HY_V4);
factory.register_parser::<InklingUnifiedParser>(names::INKLING);
factory.register_parser::<KimiK3UnifiedParser>(names::KIMI_K3);
// Opt-in only: no model patterns map to it.
factory.register_parser::<HfUnifiedParser>(names::HF);
factory
.register_pattern("gemma-4", names::GEMMA4)
rust/src/chat/tests/roundtrip.rs
+10
View File
@@ -309,6 +309,15 @@ impl RoundtripCase {
}
}
/// Gemma4 through the checkpoint's `response_template` (`hf` parser).
fn gemma4_hf() -> Self {
Self {
tool_call_parser: ParserSelection::Explicit("hf".to_string()),
reasoning_parser: ParserSelection::Explicit("hf".to_string()),
..Self::gemma4()
}
}
/// Kimi K2.5 tool-call format with `<think>` reasoning tags.
#[allow(dead_code)]
fn kimi_k25() -> Self {
@@ -459,6 +468,7 @@ roundtrip_tests! {
step3p5 => [reasoning_and_content],
nemotron_v3 => [reasoning_and_content],
gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call
gemma4_hf => [tool_call_mix],
kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history
// K3 drops plain-assistant reasoning in history; tool-call turns keep it.
kimi_k3 => [tool_call_mix],
rust/src/parser/Cargo.toml
+4
View File
@@ -11,11 +11,15 @@ test-util = []
auto_impl.workspace = true
easy-ext.workspace = true
educe.workspace = true
regex-automata.workspace = true
regex-syntax.workspace = true
serde.workspace = true
serde_json.workspace = true
serde_path_to_error.workspace = true
serde_with.workspace = true
thiserror.workspace = true
thiserror-ext.workspace = true
tracing.workspace = true
vllm-tokenizer.workspace = true
winnow.workspace = true
xgrammar-structural-tag.workspace = true
rust/src/parser/src/tool/mod.rs
+1
View File
@@ -34,6 +34,7 @@ pub use kimi_k2::KimiK2ToolParser;
pub use mimo::MiMoToolParser;
pub use minimax_m2::MinimaxM2ToolParser;
pub use minimax_m3::MinimaxM3ToolParser;
pub(crate) use parameters::{ToolSchemas, convert_integer_text, convert_number_text};
pub use qwen_coder::Qwen3CoderToolParser;
pub use seed_oss::SeedOssToolParser;
use serde::{Deserialize, Serialize};
rust/src/parser/src/tool/parameters.rs
+21 -11
View File
@@ -9,7 +9,7 @@ use crate::tool::Tool;
/// Normalized parameter schemas for all tools in one request.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub(super) struct ToolSchemas {
pub(crate) struct ToolSchemas {
tools: BTreeMap<String, ToolSchema>,
}
@@ -28,7 +28,7 @@ pub(super) struct ToolSchema {
///
/// It can be either a raw text string, or a structured input with named child elements.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) enum ParamInput {
pub(crate) enum ParamInput {
Text(String),
#[allow(dead_code)]
Elements(Vec<ParamElement>),
@@ -42,7 +42,7 @@ impl From<String> for ParamInput {
/// One named structured parameter child.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) struct ParamElement {
pub(crate) struct ParamElement {
pub name: String,
pub value: ParamInput,
}
@@ -67,7 +67,7 @@ pub(super) enum JsonParamType {
impl ToolSchemas {
/// Normalize OpenAI-style tool parameter JSON schemas for one request.
pub(super) fn from_tools(tools: &[Tool]) -> Self {
pub(crate) fn from_tools(tools: &[Tool]) -> Self {
let tools = tools
.iter()
.map(|tool| (tool.name.clone(), ToolSchema::from_schema(&tool.parameters)))
@@ -98,7 +98,7 @@ impl ToolSchemas {
}
/// Convert one parameter value for one named tool.
pub(super) fn convert_param_with_schema<P>(
pub(crate) fn convert_param_with_schema<P>(
&self,
function_name: &str,
name: &str,
@@ -361,12 +361,8 @@ fn try_convert_value(param_type: &JsonParamType, input: &ParamInput) -> Option<V
fn try_convert_text_value(param_type: &JsonParamType, value: &str) -> Option<Value> {
match param_type {
JsonParamType::String => Some(Value::String(value.to_string())),
JsonParamType::Integer => without_digit_separators(value.trim())?
.parse::<i64>()
.ok()
.map(Number::from)
.map(Value::Number),
JsonParamType::Number => try_convert_number(&without_digit_separators(value.trim())?),
JsonParamType::Integer => convert_integer_text(value),
JsonParamType::Number => convert_number_text(value),
JsonParamType::Boolean => try_convert_boolean(value),
JsonParamType::Object { .. } if value.trim().is_empty() => Some(Value::Object(Map::new())),
JsonParamType::Array { .. } if value.trim().is_empty() => Some(Value::Array(Vec::new())),
@@ -461,6 +457,20 @@ fn without_digit_separators(value: &str) -> Option<std::borrow::Cow<'_, str>> {
valid.then(|| value.replace('_', "").into())
}
/// Convert raw text to a JSON integer, as for an `integer` parameter.
pub(crate) fn convert_integer_text(value: &str) -> Option<Value> {
without_digit_separators(value.trim())?
.parse::<i64>()
.ok()
.map(Number::from)
.map(Value::Number)
}
/// Convert raw text to a JSON number, as for a `number` parameter.
pub(crate) fn convert_number_text(value: &str) -> Option<Value> {
try_convert_number(&without_digit_separators(value.trim())?)
}
/// Convert one raw string value to a JSON number.
fn try_convert_number(value: &str) -> Option<Value> {
serde_json::from_str::<Number>(value)
rust/src/parser/src/unified/combined.rs
+4 -1
View File
@@ -65,7 +65,10 @@ impl UnifiedParser for CombinedParser {
where
Self: Sized + 'static,
{
Err(UnifiedParserError::CombinedParserConstructor)
Err(UnifiedParserError::NoNamedConstructor {
parser: "combined",
built_from: "split parser instances",
})
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
rust/src/parser/src/unified/hf/content.rs
+537
View File
@@ -0,0 +1,537 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Content parsers: turn one region's captured text into a JSON value.
//!
//! Port of `content_parsers.py`. Each parser takes a chunk of captured text and
//! parses it into a single key in the output message dictionary.
use regex_automata::meta::Regex;
use regex_automata::util::syntax;
use serde::Deserialize;
use serde_json::{Map, Value};
use thiserror_ext::AsReport as _;
use super::spec::{ContentKind, deserialize};
use super::{Result, invalid, unsupported};
use crate::tool::{convert_integer_text, convert_number_text};
use crate::unified::{self, parsing_failed};
// Sentinel characters for lax-JSON string pre-extraction — ASCII control chars
// that should never appear in real LLM output.
const LAX_OPEN: char = '\u{1}';
const LAX_CLOSE: char = '\u{2}';
/// A configured content parser (`content` plus `content_args` of one field).
#[derive(Debug, Clone)]
pub(super) enum ContentParser {
Text(TextArgs),
/// Converted like an `integer` tool parameter.
Int,
/// Converted like a `number` tool parameter.
Float,
Bool(TextArgs),
Json(JsonArgs),
XmlInline(XmlInlineArgs),
KvLines(KvLinesArgs),
}
/// The `strip` argument shared by parsers that go through `_text`.
#[derive(Debug, Clone, Copy, Deserialize)]
pub(super) struct TextArgs {
#[serde(default = "default_strip")]
pub strip: bool,
}
#[derive(Debug, Clone, Deserialize)]
pub(super) struct JsonArgs {
#[serde(flatten)]
text: TextArgs,
/// Quote bare-identifier keys before parsing.
#[serde(default)]
unquoted_keys: bool,
/// Strings delimited by these custom markers are pre-extracted, then restored
/// as standard JSON strings.
#[serde(default)]
string_delims: Vec<(String, String)>,
/// Return stripped text if parsing fails.
#[serde(default)]
allow_non_json: bool,
}
#[derive(Debug, Clone)]
pub(super) struct XmlInlineArgs {
tag_pattern: Regex,
/// Index of the required `key` group.
key_group: usize,
/// Index of the `value` group, if the pattern declares one.
value_group: Option<usize>,
value_parser: Option<Box<ContentParser>>,
/// Collect duplicate keys into a list.
merge_duplicates: bool,
}
#[derive(Debug, Clone)]
pub(super) struct KvLinesArgs {
text: TextArgs,
line_sep: String,
kv_sep: String,
value_parser: Option<Box<ContentParser>>,
}
/// `content_args` of `xml-inline`.
#[derive(Deserialize)]
struct XmlInlineSpec {
/// Must have named groups `key` and `value`.
tag_pattern: String,
value_parser: Option<ValueParserSpec>,
#[serde(default)]
merge_duplicates: bool,
}
/// `content_args` of `kv-lines`.
#[derive(Deserialize)]
struct KvLinesSpec {
#[serde(flatten)]
text: TextArgs,
#[serde(default = "default_line_sep")]
line_sep: String,
#[serde(default = "default_kv_sep")]
kv_sep: String,
value_parser: Option<ValueParserSpec>,
}
/// A nested `value_parser`: `{"name": ..., "args": {...}}`.
#[derive(Deserialize)]
struct ValueParserSpec {
#[serde(default)]
name: ContentKind,
#[serde(default)]
args: Map<String, Value>,
}
impl ContentParser {
/// Build a parser from its kind and `content_args`.
///
/// Unknown argument keys are ignored, as in Transformers. Argument types and the
/// `xml-inline` tag pattern are checked here rather than at parse time.
pub fn new(scope: &str, kind: ContentKind, args: &Map<String, Value>) -> Result<Self> {
let context = format!("{scope}: content_args");
let args = Value::Object(args.clone());
Ok(match kind {
ContentKind::Text => Self::Text(deserialize(&args, &context)?),
// Number conversion ignores `strip`, but a malformed value is still rejected.
ContentKind::Int => {
deserialize::<TextArgs>(&args, &context)?;
Self::Int
}
ContentKind::Float => {
deserialize::<TextArgs>(&args, &context)?;
Self::Float
}
ContentKind::Bool => Self::Bool(deserialize(&args, &context)?),
ContentKind::Json => {
let args: JsonArgs = deserialize(&args, &context)?;
if args
.string_delims
.iter()
.any(|(open, close)| open.is_empty() || close.is_empty())
{
return Err(invalid!("{context}: string_delims cannot be empty strings"));
}
Self::Json(args)
}
ContentKind::XmlInline => {
let spec: XmlInlineSpec = deserialize(&args, &context)?;
let tag_pattern = compile_regex(scope, &spec.tag_pattern)?;
let group_info = tag_pattern.group_info();
let Some(key_group) = group_info.to_index(Default::default(), "key") else {
return Err(invalid!(
"{scope}: xml-inline: tag_pattern must have a named group 'key'. Pattern: {}",
spec.tag_pattern
));
};
let value_group = group_info.to_index(Default::default(), "value");
Self::XmlInline(XmlInlineArgs {
tag_pattern,
key_group,
value_group,
value_parser: ValueParserSpec::compile(scope, spec.value_parser)?,
merge_duplicates: spec.merge_duplicates,
})
}
ContentKind::KvLines => {
let spec: KvLinesSpec = deserialize(&args, &context)?;
if spec.line_sep.is_empty() {
return Err(invalid!("{context}: line_sep cannot be empty"));
}
Self::KvLines(KvLinesArgs {
text: spec.text,
line_sep: spec.line_sep,
kv_sep: spec.kv_sep,
value_parser: ValueParserSpec::compile(scope, spec.value_parser)?,
})
}
})
}
/// Parse one region body.
pub fn parse(&self, text: &str) -> unified::Result<Value> {
match self {
Self::Text(args) => Ok(Value::String(args.apply(text).to_string())),
// Numbers follow the tool-parameter conversion.
Self::Int => convert_integer_text(text)
.ok_or_else(|| parsing_failed!("int: invalid literal {text:?}")),
Self::Float => convert_number_text(text)
.ok_or_else(|| parsing_failed!("float: invalid or non-finite literal {text:?}")),
Self::Bool(args) => {
let text = args.apply(text).to_lowercase();
Ok(Value::Bool(text == "true" || text == "1"))
}
Self::Json(args) => args.parse(text),
Self::XmlInline(args) => args.parse(text),
Self::KvLines(args) => args.parse(text),
}
}
}
impl ValueParserSpec {
/// Build the nested parser, if any.
fn compile(scope: &str, spec: Option<Self>) -> Result<Option<Box<ContentParser>>> {
spec.map(|spec| {
ContentParser::new(&format!("{scope}: value_parser"), spec.name, &spec.args)
.map(Box::new)
})
.transpose()
}
}
impl TextArgs {
/// Apply `_text`: strip surrounding whitespace unless disabled.
pub fn apply(self, text: &str) -> &str {
if self.strip { text.trim() } else { text }
}
}
impl JsonArgs {
/// JSON parser with optional dialect knobs for LLM-emitted quirks.
fn parse(&self, text: &str) -> unified::Result<Value> {
if !self.string_delims.is_empty() && text.contains([LAX_OPEN, LAX_CLOSE]) {
return Err(parsing_failed!(
"json: input contains reserved sentinel characters (\\x01/\\x02); cannot parse safely."
));
}
let mut working = text.to_string();
let mut captured: Vec<String> = Vec::new();
for (open, close) in &self.string_delims {
working = extract_delimited(&working, open, close, &mut captured);
}
if self.unquoted_keys {
working = quote_bare_keys(&working);
}
for (index, string) in captured.iter().enumerate() {
let quoted = serde_json::to_string(string).map_err(|error| {
parsing_failed!("json: failed to quote string: {}", error.as_report())
})?;
working = working.replace(&format!("{LAX_OPEN}{index}{LAX_CLOSE}"), &quoted);
}
match serde_json::from_str(&working) {
Ok(value) => Ok(value),
Err(_) if self.allow_non_json => Ok(Value::String(self.text.apply(text).to_string())),
Err(error) if working == text => Err(parsing_failed!(
"json parser could not parse region as JSON.\nContent: {text:?}\nError: {}",
error.as_report()
)),
Err(error) => Err(parsing_failed!(
"json: could not parse after dialect transforms.\nOriginal: {text:?}\nTransformed: {working:?}\nError: {}",
error.as_report()
)),
}
}
}
impl XmlInlineArgs {
/// Parse shallow XML-ish tags into a dict.
fn parse(&self, text: &str) -> unified::Result<Value> {
let mut out = Map::new();
for captures in self.tag_pattern.captures_iter(text) {
let key =
captures.get_group(self.key_group).map(|span| &text[span.range()]).ok_or_else(
|| parsing_failed!("xml-inline: named group 'key' did not participate"),
)?;
let raw = match self.value_group {
// The pattern has no `value` group.
None => Some(""),
Some(index) => captures.get_group(index).map(|span| &text[span.range()]),
};
let value = match raw {
Some(raw) => sub_parse(raw, self.value_parser.as_deref())?,
None if self.value_parser.is_none() => Value::Null,
None => {
return Err(parsing_failed!(
"xml-inline: named group 'value' did not participate"
));
}
};
match out.get_mut(key) {
Some(existing) if self.merge_duplicates => match existing {
Value::Array(values) => values.push(value),
_ => {
let first = existing.take();
*existing = Value::Array(vec![first, value]);
}
},
_ => {
out.insert(key.to_string(), value);
}
}
}
Ok(Value::Object(out))
}
}
impl KvLinesArgs {
/// Parse line-delimited `key<sep>value` pairs into a dict.
fn parse(&self, text: &str) -> unified::Result<Value> {
let mut out = Map::new();
for line in text.split(self.line_sep.as_str()) {
let line = self.text.apply(line);
let Some((key, value)) = line.split_once(self.kv_sep.as_str()) else {
continue;
};
let (key, value) = (self.text.apply(key), self.text.apply(value));
out.insert(
key.to_string(),
sub_parse(value, self.value_parser.as_deref())?,
);
}
Ok(Value::Object(out))
}
}
/// Parse `raw` with an optional nested value parser.
fn sub_parse(raw: &str, value_parser: Option<&ContentParser>) -> unified::Result<Value> {
match value_parser {
None => Ok(Value::String(raw.to_string())),
Some(parser) => parser.parse(raw),
}
}
/// Replace every `open(.*?)close` span with an indexed sentinel, collecting the
/// inner strings. Equivalent to the non-overlapping, left-to-right `re.sub` with
/// `re.DOTALL` used by Transformers.
fn extract_delimited(text: &str, open: &str, close: &str, captured: &mut Vec<String>) -> String {
let mut out = String::with_capacity(text.len());
let mut rest = text;
while let Some(start) = rest.find(open) {
let inner_start = start + open.len();
// No close after this open means no close after any later open either.
let Some(inner_len) = rest[inner_start..].find(close) else {
break;
};
out.push_str(&rest[..start]);
captured.push(rest[inner_start..inner_start + inner_len].to_string());
out.push(LAX_OPEN);
out.push_str(&(captured.len() - 1).to_string());
out.push(LAX_CLOSE);
rest = &rest[inner_start + inner_len + close.len()..];
}
out.push_str(rest);
out
}
/// Quote bare-identifier keys: Transformers' `(?<=[{,])(\w+):` → `"\1":`.
///
/// The Rust regex engine has no lookbehind, so the preceding `{` or `,` is
/// matched and written back. Matches cannot overlap either way, so the rewrite is
/// equivalent.
fn quote_bare_keys(text: &str) -> String {
static BARE_KEY: std::sync::LazyLock<Regex> =
std::sync::LazyLock::new(|| Regex::new(r"([{,])(\w+):").expect("valid regex"));
let mut out = String::with_capacity(text.len());
let mut last = 0;
for captures in BARE_KEY.captures_iter(text) {
let whole = captures.get_match().expect("match");
let separator = captures.get_group(1).expect("group 1");
let key = captures.get_group(2).expect("group 2");
out.push_str(&text[last..whole.start()]);
out.push_str(&text[separator.range()]);
out.push('"');
out.push_str(&text[key.range()]);
out.push_str("\":");
last = whole.end();
}
out.push_str(&text[last..]);
out
}
/// Compile a Python-syntax regex with `re.DOTALL`, as Transformers does.
pub(super) fn compile_regex(scope: &str, pattern: &str) -> Result<Regex> {
Regex::builder()
.syntax(syntax::Config::new().dot_matches_new_line(true))
.build(pattern)
.map_err(|error| {
unsupported!(
"{scope}: regex {pattern:?} is not supported by the Rust regex engine: {}",
error.as_report()
)
})
}
fn default_strip() -> bool {
true
}
fn default_line_sep() -> String {
"\n".to_string()
}
fn default_kv_sep() -> String {
":".to_string()
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::*;
fn kind(name: &str) -> ContentKind {
serde_json::from_value(json!(name)).unwrap()
}
fn parser(name: &str, args: Value) -> ContentParser {
ContentParser::new("test", kind(name), args.as_object().unwrap()).unwrap()
}
#[test]
fn scalar_parsers_convert_like_tool_parameters() {
let int = parser("int", json!({}));
assert_eq!(int.parse(" 42\n").unwrap(), json!(42));
assert_eq!(int.parse("-1_000").unwrap(), json!(-1000));
assert!(int.parse("1__0").is_err());
assert!(int.parse("4.2").is_err());
let float = parser("float", json!({}));
assert_eq!(float.parse("1.5").unwrap(), json!(1.5));
assert_eq!(float.parse(" 1e3 ").unwrap(), json!(1000.0));
assert!(float.parse("inf").is_err(), "JSON cannot represent inf");
let boolean = parser("bool", json!({}));
assert_eq!(boolean.parse(" TRUE ").unwrap(), json!(true));
assert_eq!(boolean.parse("1").unwrap(), json!(true));
assert_eq!(boolean.parse("yes").unwrap(), json!(false));
let text = parser("text", json!({"strip": false}));
assert_eq!(text.parse(" a \n").unwrap(), json!(" a \n"));
assert_eq!(
parser("text", json!({})).parse(" a \n").unwrap(),
json!("a")
);
}
#[test]
fn json_dialects_match_gemma4_syntax() {
let json = parser(
"json",
json!({"unquoted_keys": true, "string_delims": [["<|\"|>", "<|\"|>"]]}),
);
let value = json
.parse(
r#"{bool_value:true,list_value:[<|"|>foo<|"|>,<|"|>bar<|"|>],null_value:null,number_value:1,string_value:<|"|>a, b: {c}<|"|>,struct_value:{foo:<|"|>bar<|"|>}}"#,
)
.unwrap();
assert_eq!(
value,
json!({
"bool_value": true,
"list_value": ["foo", "bar"],
"null_value": null,
"number_value": 1,
"string_value": "a, b: {c}",
"struct_value": {"foo": "bar"},
})
);
assert!(json.parse("{a:\u{1}}").is_err());
}
#[test]
fn json_allow_non_json_returns_stripped_text() {
let json = parser("json", json!({"allow_non_json": true}));
assert_eq!(json.parse(" celsius \n").unwrap(), json!("celsius"));
assert_eq!(json.parse(" [1, 2] ").unwrap(), json!([1, 2]));
assert!(parser("json", json!({})).parse("celsius").is_err());
}
#[test]
fn xml_inline_parses_tags_with_value_parser_and_merging() {
let xml = parser(
"xml-inline",
json!({
"tag_pattern": r"<parameter=(?P<key>\w+)>\s*(?P<value>.*?)\s*</parameter>",
"value_parser": {"name": "json", "args": {"allow_non_json": true}},
"merge_duplicates": true,
}),
);
let value = xml
.parse(
"<parameter=locations>\n[{\"city\": \"Paris\"}]\n</parameter>\n\
<parameter=unit>\ncelsius\n</parameter><parameter=unit>2</parameter>",
)
.unwrap();
assert_eq!(
value,
json!({"locations": [{"city": "Paris"}], "unit": ["celsius", 2]})
);
}
#[test]
fn kv_lines_split_and_strip() {
let kv = parser("kv-lines", json!({"value_parser": {"name": "int"}}));
assert_eq!(
kv.parse(" a : 1 \n\nnot a pair\nb:2").unwrap(),
json!({"a": 1, "b": 2})
);
}
#[test]
fn invalid_arguments_are_rejected_at_load() {
let error = |name: &str, args: Value| {
ContentParser::new("Field 'x'", kind(name), args.as_object().unwrap()).unwrap_err()
};
expect_test::expect![[r#"
[
Invalid {
message: "Field 'x': content_args.strip: invalid type: string \"no\", expected a boolean",
},
Invalid {
message: "Field 'x': content_args.string_delims[0]: invalid length 1, expected a tuple of size 2",
},
Invalid {
message: "Field 'x': content_args: missing field `tag_pattern`",
},
Invalid {
message: "Field 'x': xml-inline: tag_pattern must have a named group 'key'. Pattern: <(?P<k>\\w+)>",
},
Invalid {
message: "Field 'x': content_args.value_parser.name: unknown variant `yaml`, expected one of `text`, `int`, `float`, `bool`, `json`, `xml-inline`, `kv-lines`",
},
]
"#]].assert_debug_eq(&[
error("text", json!({"strip": "no"})),
error("json", json!({"string_delims": [["<s>"]]})),
error("xml-inline", json!({})),
error("xml-inline", json!({"tag_pattern": r"<(?P<k>\w+)>"})),
error("kv-lines", json!({"value_parser": {"name": "yaml"}})),
]);
assert!(matches!(
error("xml-inline", json!({"tag_pattern": r"(?<=a)b"})),
super::super::HfTemplateError::Unsupported { .. }
));
}
}
rust/src/parser/src/unified/hf/fixtures/differential.json
+440
View File
@@ -0,0 +1,440 @@
{
"templates": {
"gemma4": {
"defaults": {
"role": "assistant"
},
"start_anchor": [
"<|turn>model\n",
"<tool_response|>"
],
"fields": {
"thinking": {
"open": "<|channel>thought\n",
"close": "<channel|>",
"content": "text"
},
"tool_calls": {
"open_pattern": "<\\|tool_call>call:(?P<name>\\w+)",
"close": "<tool_call|>",
"repeats": true,
"content": "json",
"content_args": {
"unquoted_keys": true,
"string_delims": [
[
"<|\"|>",
"<|\"|>"
]
]
},
"transform": {
"type": "function",
"function": {
"name": "{name}",
"arguments": "{content}"
}
}
},
"content": {
"close": [
"<turn|>",
"<|tool_response>",
"<eos>"
],
"content": "text"
}
}
},
"invoke": {
"defaults": {
"role": "assistant"
},
"start_anchor": "<|begin|>assistant",
"fields": {
"reasoning_content": {
"open_pattern": "to=self<\\|msg\\|>",
"close": "<|pause|>",
"content": "text"
},
"tool_calls": {
"open_pattern": "<invoke\\b[^>]*?\\bname=\"(?P<name>[^\"]+)\">",
"close": "</invoke>",
"repeats": true,
"content": "xml-inline",
"content_args": {
"tag_pattern": "<param\\b[^>]*?\\bname=\"(?P<key>[^\"]+)\"[^>]*?>(?P<value>.*?)</param>",
"value_parser": {
"name": "json",
"args": {
"allow_non_json": true
}
}
},
"transform": {
"type": "function",
"function": {
"name": "{name}",
"arguments": "{content}"
}
}
},
"content": {
"open_pattern": "to=(?:user|note)<\\|msg\\|>",
"close": [
"<|end|>",
"<|pause|>"
],
"repeats": true,
"join": "",
"content": "text"
}
}
},
"kv_tools": {
"start_anchor": "<|im_start|>assistant\n",
"fields": {
"thinking": {
"open": "<think>",
"close": "</think>"
},
"tool_calls": {
"open_pattern": "<tool_call>\\s*<function=(?P<name>\\w+)>\\n",
"close": "</tool_call>",
"repeats": true,
"content": "kv-lines",
"transform": {
"type": "function",
"function": {
"name": "{name}",
"arguments": "{content}"
}
}
},
"content": {
"close": "<|im_end|>"
}
}
}
},
"cases": [
{
"template": "gemma4",
"prefix": "",
"text": "Hello there.",
"tools": null,
"expected": {
"content": "Hello there."
}
},
{
"template": "gemma4",
"prefix": "",
"text": " \n Hello\n\n",
"tools": null,
"expected": {
"content": "Hello"
}
},
{
"template": "gemma4",
"prefix": "",
"text": "<|channel>thought\n plan\n<channel|>\n\nAnswer: 天气很好 ",
"tools": null,
"expected": {
"reasoning": "plan",
"content": "Answer: 天气很好"
}
},
{
"template": "gemma4",
"prefix": "",
"text": "<|channel>thought\nunterminated reasoning",
"tools": null,
"expected": {
"reasoning": "unterminated reasoning"
}
},
{
"template": "gemma4",
"prefix": "",
"text": "Before <|tool_call>call:f{x:<|\"|>a, b<|\"|>,y:[1,2.5,-3],z:{w:true}}<tool_call|> after",
"tools": null,
"expected": {
"content": "after",
"tool_calls": [
{
"name": "f",
"arguments": {
"x": "a, b",
"y": [
1,
2.5,
-3
],
"z": {
"w": true
}
}
}
]
}
},
{
"template": "gemma4",
"prefix": "",
"text": "<|tool_call>call:f{}<tool_call|><|tool_call>call:g{n:1}<tool_call|>",
"tools": null,
"expected": {
"tool_calls": [
{
"name": "f",
"arguments": {}
},
{
"name": "g",
"arguments": {
"n": 1
}
}
]
}
},
{
"template": "gemma4",
"prefix": "",
"text": "<|tool_call>call:f{a:<|\"|>unterminated",
"tools": null,
"expected": {
"error": "ValueError"
}
},
{
"template": "gemma4",
"prefix": "",
"text": "text <|tool_call> not a call",
"tools": null,
"expected": {
"content": "text <|tool_call> not a call"
}
},
{
"template": "gemma4",
"prefix": "",
"text": "<|tool_call>call:f{a:1}",
"tools": null,
"expected": {
"tool_calls": [
{
"name": "f",
"arguments": {
"a": 1
}
}
]
}
},
{
"template": "gemma4",
"prefix": "<|turn>user\nhi<turn|>\n<|turn>model\n",
"text": "Hi!",
"tools": null,
"expected": {
"content": "Hi!"
}
},
{
"template": "gemma4",
"prefix": "<|turn>model\n<|tool_call>call:f{}<tool_call|><|tool_response>response:f{}<tool_response|><|channel>thought\n",
"text": "Done.<channel|>It is sunny.",
"tools": null,
"expected": {
"reasoning": "Done.",
"content": "It is sunny."
}
},
{
"template": "gemma4",
"prefix": "<|turn>model\n<|channel>thought\n<channel|>",
"text": "Plain answer",
"tools": null,
"expected": {
"content": "Plain answer"
}
},
{
"template": "invoke",
"prefix": "",
"text": "to=self<|msg|>think<|pause|>to=user<|msg|>A<|pause|>to=note<|msg|> B <|end|>",
"tools": null,
"expected": {
"reasoning": "think",
"content": "AB"
}
},
{
"template": "invoke",
"prefix": "",
"text": "noise <invoke name=\"get\" id=\"1\"><param name=\"city\">\"Paris\"</param><param name=\"n\">3</param><param name=\"raw\">not json</param></invoke>",
"tools": null,
"expected": {}
},
{
"template": "invoke",
"prefix": "",
"text": "<invoke name=\"get\"><param name=\"k\">1</param><param name=\"k\">2</param></invoke>",
"tools": null,
"expected": {
"tool_calls": [
{
"name": "get",
"arguments": {
"k": 2
}
}
]
}
},
{
"template": "invoke",
"prefix": "",
"text": "bare text only",
"tools": null,
"expected": {}
},
{
"template": "invoke",
"prefix": "",
"text": "to=user<|msg|>unterminated content",
"tools": null,
"expected": {
"content": "unterminated content"
}
},
{
"template": "invoke",
"prefix": "<|begin|>user<|end|><|begin|>assistant",
"text": "to=user<|msg|>hi<|end|>",
"tools": null,
"expected": {
"content": "hi"
}
},
{
"template": "kv_tools",
"prefix": "",
"text": "<tool_call>\n<function=set_alarm>\nhour: 7\nratio: 0.5\nenabled: 1\nlabel: 7\nmaybe: None\n</tool_call>",
"tools": [
{
"type": "function",
"function": {
"name": "set_alarm",
"parameters": {
"type": "object",
"properties": {
"hour": {
"type": "integer"
},
"ratio": {
"type": "number"
},
"enabled": {
"type": "boolean"
},
"label": {
"type": "string"
},
"maybe": {
"anyOf": [
{
"type": "integer"
},
{
"type": "null"
}
]
}
}
}
}
}
],
"expected": {
"tool_calls": [
{
"name": "set_alarm",
"arguments": {
"hour": 7,
"ratio": 0.5,
"enabled": true,
"label": "7",
"maybe": null
}
}
]
}
},
{
"template": "kv_tools",
"prefix": "",
"text": "<tool_call>\n<function=set_alarm>\nhour: seven\nratio: 1e3\nenabled: nope\n</tool_call>",
"tools": [
{
"type": "function",
"function": {
"name": "set_alarm",
"parameters": {
"type": "object",
"properties": {
"hour": {
"type": "integer"
},
"ratio": {
"type": "number"
},
"enabled": {
"type": "boolean"
},
"label": {
"type": "string"
},
"maybe": {
"anyOf": [
{
"type": "integer"
},
{
"type": "null"
}
]
}
}
}
}
}
],
"expected": {
"tool_calls": [
{
"name": "set_alarm",
"arguments": {
"hour": "seven",
"ratio": 1000,
"enabled": "nope"
}
}
]
}
},
{
"template": "kv_tools",
"prefix": "<|im_start|>assistant\n<think>\n",
"text": "\n reasoning \n</think>\n\nanswer<|im_end|>",
"tools": null,
"expected": {
"reasoning": "reasoning",
"content": "answer"
}
}
]
}
rust/src/parser/src/unified/hf/fixtures/generate_differential.py
+264
View File
@@ -0,0 +1,264 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regenerate `differential.json` from Transformers `parse_response`.
Requires Transformers with `transformers.utils.chat_parsing` (>= 5.13); the
expected values in the checked-in file were produced at commit 6d43ab4008.
python generate_differential.py > differential.json
"""
import json
from transformers.utils.chat_parsing import parse_response
GEMMA4 = {
"defaults": {"role": "assistant"},
"start_anchor": ["<|turn>model\n", "<tool_response|>"],
"fields": {
"thinking": {
"open": "<|channel>thought\n",
"close": "<channel|>",
"content": "text",
},
"tool_calls": {
"open_pattern": r"<\|tool_call>call:(?P<name>\w+)",
"close": "<tool_call|>",
"repeats": True,
"content": "json",
"content_args": {
"unquoted_keys": True,
"string_delims": [['<|"|>', '<|"|>']],
},
"transform": {
"type": "function",
"function": {"name": "{name}", "arguments": "{content}"},
},
},
"content": {
"close": ["<turn|>", "<|tool_response>", "<eos>"],
"content": "text",
},
},
}
INVOKE = {
"defaults": {"role": "assistant"},
"start_anchor": "<|begin|>assistant",
"fields": {
"reasoning_content": {
"open_pattern": r"to=self<\|msg\|>",
"close": "<|pause|>",
"content": "text",
},
"tool_calls": {
"open_pattern": r'<invoke\b[^>]*?\bname="(?P<name>[^"]+)">',
"close": "</invoke>",
"repeats": True,
"content": "xml-inline",
"content_args": {
"tag_pattern": (
r'<param\b[^>]*?\bname="(?P<key>[^"]+)"[^>]*?>'
r"(?P<value>.*?)</param>"
),
"value_parser": {"name": "json", "args": {"allow_non_json": True}},
},
"transform": {
"type": "function",
"function": {"name": "{name}", "arguments": "{content}"},
},
},
"content": {
"open_pattern": r"to=(?:user|note)<\|msg\|>",
"close": ["<|end|>", "<|pause|>"],
"repeats": True,
"join": "",
"content": "text",
},
},
}
KV_TOOLS = {
"start_anchor": "<|im_start|>assistant\n",
"fields": {
"thinking": {"open": "<think>", "close": "</think>"},
"tool_calls": {
"open_pattern": r"<tool_call>\s*<function=(?P<name>\w+)>\n",
"close": "</tool_call>",
"repeats": True,
"content": "kv-lines",
"transform": {
"type": "function",
"function": {"name": "{name}", "arguments": "{content}"},
},
},
"content": {"close": "<|im_end|>"},
},
}
ALARM_TOOLS = [
{
"type": "function",
"function": {
"name": "set_alarm",
"parameters": {
"type": "object",
"properties": {
"hour": {"type": "integer"},
"ratio": {"type": "number"},
"enabled": {"type": "boolean"},
"label": {"type": "string"},
"maybe": {"anyOf": [{"type": "integer"}, {"type": "null"}]},
},
},
},
}
]
CASES = [
("gemma4", GEMMA4, "", "Hello there.", None),
("gemma4", GEMMA4, "", " \n Hello\n\n", None),
(
"gemma4",
GEMMA4,
"",
"<|channel>thought\n plan\n<channel|>\n\nAnswer: 天气很好 ",
None,
),
("gemma4", GEMMA4, "", "<|channel>thought\nunterminated reasoning", None),
(
"gemma4",
GEMMA4,
"",
'Before <|tool_call>call:f{x:<|"|>a, b<|"|>,y:[1,2.5,-3],z:{w:true}}'
"<tool_call|> after",
None,
),
(
"gemma4",
GEMMA4,
"",
"<|tool_call>call:f{}<tool_call|><|tool_call>call:g{n:1}<tool_call|>",
None,
),
("gemma4", GEMMA4, "", '<|tool_call>call:f{a:<|"|>unterminated', None),
("gemma4", GEMMA4, "", "text <|tool_call> not a call", None),
("gemma4", GEMMA4, "", "<|tool_call>call:f{a:1}", None),
("gemma4", GEMMA4, "<|turn>user\nhi<turn|>\n<|turn>model\n", "Hi!", None),
(
"gemma4",
GEMMA4,
"<|turn>model\n<|tool_call>call:f{}<tool_call|><|tool_response>response:f{}<tool_response|><|channel>thought\n",
"Done.<channel|>It is sunny.",
None,
),
(
"gemma4",
GEMMA4,
"<|turn>model\n<|channel>thought\n<channel|>",
"Plain answer",
None,
),
(
"invoke",
INVOKE,
"",
"to=self<|msg|>think<|pause|>to=user<|msg|>A<|pause|>to=note<|msg|> B <|end|>",
None,
),
(
"invoke",
INVOKE,
"",
'noise <invoke name="get" id="1"><param name="city">"Paris"</param>'
'<param name="n">3</param><param name="raw">not json</param></invoke>',
None,
),
(
"invoke",
INVOKE,
"",
'<invoke name="get"><param name="k">1</param>'
'<param name="k">2</param></invoke>',
None,
),
("invoke", INVOKE, "", "bare text only", None),
("invoke", INVOKE, "", "to=user<|msg|>unterminated content", None),
(
"invoke",
INVOKE,
"<|begin|>user<|end|><|begin|>assistant",
"to=user<|msg|>hi<|end|>",
None,
),
(
"kv_tools",
KV_TOOLS,
"",
"<tool_call>\n<function=set_alarm>\nhour: 7\nratio: 0.5\nenabled: 1\n"
"label: 7\nmaybe: None\n</tool_call>",
ALARM_TOOLS,
),
(
"kv_tools",
KV_TOOLS,
"",
"<tool_call>\n<function=set_alarm>\nhour: seven\nratio: 1e3\n"
"enabled: nope\n</tool_call>",
ALARM_TOOLS,
),
(
"kv_tools",
KV_TOOLS,
"<|im_start|>assistant\n<think>\n",
"\n reasoning \n</think>\n\nanswer<|im_end|>",
None,
),
]
def normalize(message):
out = {}
reasoning = message.get("thinking", message.get("reasoning_content"))
if reasoning:
out["reasoning"] = reasoning
if message.get("content"):
out["content"] = message["content"]
calls = [
{"name": call["function"]["name"], "arguments": call["function"]["arguments"]}
for call in message.get("tool_calls", [])
]
if calls:
out["tool_calls"] = calls
return out
def main():
templates = {}
cases = []
for template_name, template, prefix, text, tools in CASES:
templates[template_name] = template
try:
expected = normalize(
parse_response(text, template, prefix=prefix, tools=tools)
)
except Exception as error: # noqa: BLE001
expected = {"error": type(error).__name__}
cases.append(
{
"template": template_name,
"prefix": prefix,
"text": text,
"tools": tools,
"expected": expected,
}
)
print(
json.dumps(
{"templates": templates, "cases": cases}, indent=2, ensure_ascii=False
)
)
if __name__ == "__main__":
main()
rust/src/parser/src/unified/hf/mod.rs
+53
View File
@@ -0,0 +1,53 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Unified parser driven by a checkpoint's Hugging Face `response_template`.
//!
//! The renderer executes a checkpoint's `chat_template`; this parser executes its
//! `response_template`, the declarative output-protocol description introduced by
//! Transformers `chat_parsing` (`ResponseParser`).
//!
//! The template language and its value semantics (content parsers, transforms,
//! tool-schema coercion) follow Transformers at
//! <https://github.com/huggingface/transformers/tree/6d43ab4008/src/transformers/utils/chat_parsing>.
//! The streaming executor is native: it emits ordered [`UnifiedParserEvent`]s
//! instead of Transformers' region events and aggregated message dict.
//!
//! [`UnifiedParserEvent`]: super::UnifiedParserEvent
mod content;
mod parser;
mod pattern;
mod spec;
mod template;
mod transform;
#[cfg(test)]
mod tests;
pub use parser::HfUnifiedParser;
pub use template::ResponseTemplate;
use thiserror::Error;
use thiserror_ext::Macro;
/// Result alias for response-template operations.
pub type Result<T> = std::result::Result<T, HfTemplateError>;
/// Errors produced while loading a `response_template`.
///
/// Failures while parsing generated output are
/// [`UnifiedParserError::ParsingFailed`](super::UnifiedParserError::ParsingFailed), as for
/// the other unified parsers.
#[derive(Debug, Clone, Error, Macro)]
#[thiserror_ext(macro(path = "crate::unified::hf", mangle))]
pub enum HfTemplateError {
/// The template violates the Transformers `response_template` schema.
#[error("invalid response_template: {message}")]
Invalid { message: String },
/// The template is valid but uses a feature this implementation does not support.
#[error("unsupported response_template: {message}")]
Unsupported { message: String },
/// The model provides no template.
#[error("the model's tokenizer_config.json provides no response_template")]
Missing,
}
rust/src/parser/src/unified/hf/parser.rs
+730
View File
@@ -0,0 +1,730 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Streaming executor for a compiled [`ResponseTemplate`].
//!
//! Follows `ResponseParser` in `response_parser.py`: a flat region state
//! machine that, outside explicit regions, watches every explicit open plus the
//! implicit region's close, and inside an explicit region watches only that
//! region's close. Instead of region events and an aggregated dict, it emits
//! ordered unified parser events:
//!
//! - text and reasoning stream as they arrive, with `strip` applied so that the
//! concatenated deltas equal Transformers' stripped value. For a field without
//! `repeats`, stripping spans all its occurrences: an interrupted field (text,
//! tool call, more text) streams every occurrence, where Transformers keeps only
//! the last;
//! - a tool call starts at its opener when the opener captures the function
//! name, and its arguments are emitted when the region closes.
use std::mem::take;
use std::sync::Arc;
use serde::Deserialize;
use serde_json::{Map, Value};
use thiserror_ext::AsReport as _;
use vllm_tokenizer::{DecodedText, DynTokenizer, TokenAnchor, TokenAttribution};
use winnow::error::{ErrMode, ModalResult};
use winnow::stream::{Partial, Stream};
use super::pattern::{Captures, Resolution};
use super::template::{
FieldName, Region, RegionKind, Repeat, ResponseTemplate, TextRegion, TextRole, ToolCallRegion,
Watch, WatchKind, WatchSet,
};
use crate::tool::{Tool, ToolCallDelta, ToolSchemas};
use crate::unified::{
Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput, parsing_failed,
};
use crate::utils::{incomplete, parse_buffered_event, safe_text_len_mul};
/// Prompt tail windows, in tokens, searched for the start anchor before
/// decoding the whole prompt.
const PROMPT_WINDOWS: [usize; 3] = [64, 512, 4096];
/// Unified parser executing a checkpoint's `response_template`.
///
/// Original Python implementation:
/// <https://github.com/huggingface/transformers/blob/6d43ab4008/src/transformers/utils/chat_parsing/response_parser.py>
pub struct HfUnifiedParser {
template: Arc<ResponseTemplate>,
tool_schemas: ToolSchemas,
tokenizer: DynTokenizer,
buffer: DecodedText,
mode: HfMode,
/// Whether each region has closed an occurrence.
closed: Vec<bool>,
/// Strip state per region: per occurrence with `repeats`, per stream otherwise.
strip: Vec<StripState>,
emitted_tool_count: usize,
}
/// Parser mode: the region currently receiving text.
enum HfMode {
/// No implicit field is declared: text outside regions is discarded.
Discard,
/// The implicit region, opened lazily on its first byte.
Implicit(Occurrence),
/// An explicit region, opened by its open boundary.
Explicit(Occurrence),
}
/// One occurrence of a region.
struct Occurrence {
region: usize,
/// Whether the occurrence has content to close (`_opened`).
opened: bool,
captures: Captures,
/// Opener text, returned by [`UnifiedParser::reset`].
raw_open: String,
/// Buffered body of a tool-call region.
body: String,
/// Index of the tool call started at the opener, if any.
active_tool_index: Option<usize>,
/// `join` separator still to emit before this occurrence's content.
separator: Option<String>,
}
/// Strip-compatible streaming state of a text or reasoning occurrence.
#[derive(Default)]
struct StripState {
enabled: bool,
/// Whether a non-whitespace character has been seen.
started: bool,
/// Trailing whitespace held until more text arrives or the region closes.
held: DecodedText,
}
/// One parsed event of the buffered input.
enum HfEvent {
/// Text routed into the current region.
Text,
/// A boundary of the current mode's watch list.
Boundary { watch: Watch, captures: Captures },
}
/// A committable boundary match at the current position.
struct BoundaryMatch {
watch: Watch,
len: usize,
captures: Captures,
}
impl BoundaryMatch {
/// Preference order of `_scan`: longest first, opens before closes, then by
/// field name.
fn key(&self, template: &ResponseTemplate) -> (std::cmp::Reverse<usize>, bool, FieldName) {
(
std::cmp::Reverse(self.len),
self.watch.kind == WatchKind::Close,
template.regions[self.watch.region].name,
)
}
}
/// How the event loop treats its input.
#[derive(Clone, Copy)]
enum Phase {
/// Prompt text after the start anchor: state transitions only, no events.
Prefill,
/// Generated text; more may follow.
Stream,
/// Generated text at the end of the stream.
Eof,
}
impl Phase {
fn emits(self) -> bool {
!matches!(self, Self::Prefill)
}
}
impl HfUnifiedParser {
/// Create a parser for one request.
pub fn new(template: Arc<ResponseTemplate>, tools: &[Tool], tokenizer: DynTokenizer) -> Self {
let mut parser = Self {
closed: vec![false; template.regions.len()],
strip: template.regions.iter().map(StripState::new).collect(),
mode: HfMode::Discard,
template,
tool_schemas: ToolSchemas::from_tools(tools),
tokenizer,
buffer: DecodedText::default(),
emitted_tool_count: 0,
};
parser.reset_to_implicit();
parser
}
/// Clear all per-stream state.
fn reset_state(&mut self) {
self.buffer.clear();
self.closed.fill(false);
self.strip = self.template.regions.iter().map(StripState::new).collect();
self.emitted_tool_count = 0;
self.reset_to_implicit();
}
/// Return to the implicit region, or discard text when there is none.
fn reset_to_implicit(&mut self) {
self.mode = match self.template.implicit {
Some(region) => {
HfMode::Implicit(self.occurrence(region, false, Captures::new(), String::new()))
}
None => HfMode::Discard,
};
}
/// Start a new occurrence of `region`.
fn occurrence(
&mut self,
region: usize,
opened: bool,
captures: Captures,
raw_open: String,
) -> Occurrence {
let spec = &self.template.regions[region];
let mut separator = None;
if let RegionKind::Text(text) = &spec.kind {
if let Repeat::Join(join) = &text.repeat
&& !join.is_empty()
&& self.closed[region]
{
separator = Some(join.clone());
}
if text.repeat != Repeat::Once {
self.strip[region] = StripState::new(spec);
}
}
Occurrence {
region,
opened,
captures,
raw_open,
body: String::new(),
active_tool_index: None,
separator,
}
}
/// Parse buffered input until more is needed.
fn drive(&mut self, output: &mut UnifiedParserOutput, phase: Phase) -> Result<()> {
let template = Arc::clone(&self.template);
loop {
let watch = match &self.mode {
HfMode::Explicit(occurrence) => &template.regions[occurrence.region].close_watch,
HfMode::Discard | HfMode::Implicit(_) => &template.idle_watch,
};
let eof = matches!(phase, Phase::Eof);
let Some((step, consumed_len)) = parse_buffered_event(&self.buffer.text, |input| {
parse_next_event(input, &template, watch, eof)
})?
else {
return Ok(());
};
let piece = self.buffer.drain_prefix(consumed_len);
match step {
HfEvent::Text => self.route(piece, output, phase),
HfEvent::Boundary { watch, captures } => {
self.close_current(output, phase)?;
match watch.kind {
WatchKind::Open => {
self.open_explicit(watch.region, captures, piece.text, output, phase)
}
// The implicit region's close: start a fresh implicit occurrence.
WatchKind::Close => {}
}
}
}
}
}
/// Route a text piece into the current region.
fn route(&mut self, piece: DecodedText, output: &mut UnifiedParserOutput, phase: Phase) {
let (occurrence, region) = match &mut self.mode {
HfMode::Discard => return,
HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) => {
let region = &self.template.regions[occurrence.region];
(occurrence, region)
}
};
let RegionKind::Text(text) = &region.kind else {
if !piece.text.is_empty() {
occurrence.opened = true;
occurrence.body.push_str(&piece.text);
}
return;
};
let reasoning = text.role == TextRole::Reasoning;
if piece.text.is_empty() {
// Zero-width tokens still count as reasoning.
if reasoning && phase.emits() {
output.push_reasoning(piece);
}
return;
}
occurrence.opened = true;
let strip = &mut self.strip[occurrence.region];
if !phase.emits() {
// Prompt text is never emitted; it only decides whether later
// leading whitespace is still stripped.
strip.started |= !piece.text.trim_start().is_empty();
return;
}
let (visible, dropped) = strip.feed(piece);
if reasoning {
push_dropped_reasoning(output, dropped);
}
if visible.text.is_empty() {
return;
}
if let Some(separator) = occurrence.separator.take() {
push_text(output, text.role, DecodedText::unattributed(separator));
}
push_text(output, text.role, visible);
}
/// Open an explicit region.
fn open_explicit(
&mut self,
region: usize,
captures: Captures,
raw_open: String,
output: &mut UnifiedParserOutput,
phase: Phase,
) {
let mut occurrence = self.occurrence(region, true, captures, raw_open);
let opener_name = match &self.template.regions[region].kind {
RegionKind::ToolCalls(ToolCallRegion {
name_group: Some(group),
..
}) => occurrence.captures.get(group).and_then(Value::as_str).map(str::to_string),
_ => None,
};
if let Some(name) = opener_name
&& phase.emits()
{
let tool_index = self.allocate_tool_index();
occurrence.active_tool_index = Some(tool_index);
output.push_call(ToolCallDelta {
tool_index,
name: Some(name),
arguments: String::new(),
});
}
self.mode = HfMode::Explicit(occurrence);
}
/// Close the current region and reset to the implicit region.
///
/// Skipped (aside from the reset) when the current region never opened --
/// avoids vacuous open/close pairs at every explicit boundary.
fn close_current(&mut self, output: &mut UnifiedParserOutput, phase: Phase) -> Result<()> {
let template = Arc::clone(&self.template);
let mode = std::mem::replace(&mut self.mode, HfMode::Discard);
let occurrence = match mode {
HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) if occurrence.opened => {
Some(occurrence)
}
_ => None,
};
// Without `repeats`, trailing whitespace stays held in case the field
// continues; it is dropped at the end of the stream.
let held = occurrence
.as_ref()
.filter(|occurrence| {
matches!(&template.regions[occurrence.region].kind,
RegionKind::Text(text) if text.repeat != Repeat::Once)
})
.map(|occurrence| take(&mut self.strip[occurrence.region].held));
if let Some(occurrence) = &occurrence {
self.closed[occurrence.region] = true;
}
self.reset_to_implicit();
// Regions closed inside the prompt contribute nothing to the output.
let Some(occurrence) = occurrence.filter(|_| phase.emits()) else {
return Ok(());
};
let region = &template.regions[occurrence.region];
match &region.kind {
RegionKind::Text(text) => {
if text.role == TextRole::Reasoning
&& let Some(held) = held
{
push_dropped_reasoning(output, held);
}
// An empty occurrence still joins: `previous + join + ""`.
if let Some(separator) = occurrence.separator {
push_text(output, text.role, DecodedText::unattributed(separator));
}
}
RegionKind::ToolCalls(calls) => {
let parsed = calls.content.parse(&occurrence.body)?;
let value = match &calls.transform {
Some(transform) => {
transform.apply(region.name.as_str(), parsed, &occurrence.captures)?
}
None => parsed,
};
self.emit_calls(value, occurrence.active_tool_index, output)?;
}
}
Ok(())
}
/// Emit the tool calls of a closed tool-call region.
// TODO: stream arguments incrementally instead of once at region close; the
// dialects (e.g. Gemma 4's unquoted keys and `<|"|>` strings, `xml-inline`)
// need incremental conversion rather than raw passthrough.
fn emit_calls(
&mut self,
value: Value,
started: Option<usize>,
output: &mut UnifiedParserOutput,
) -> Result<()> {
let calls = match value {
Value::Array(calls) => calls,
call => vec![call],
};
for (position, call) in calls.into_iter().enumerate() {
let (name, arguments) = call_parts(call, &self.tool_schemas)?;
let (tool_index, name) = match (position, started) {
(0, Some(tool_index)) => (tool_index, None),
_ => (self.allocate_tool_index(), Some(name)),
};
output.push_call(ToolCallDelta {
tool_index,
name,
arguments,
});
}
Ok(())
}
fn allocate_tool_index(&mut self) -> usize {
let tool_index = self.emitted_tool_count;
self.emitted_tool_count += 1;
tool_index
}
/// Return the prompt text after the last start-anchor match, if any.
///
/// Searches geometrically growing tail windows before decoding the whole
/// prompt. A decoded suffix can differ from the full decode only in its first
/// character, so a match is accepted only after that character unless the
/// window is the whole prompt.
fn prompt_remainder(&self, prompt_token_ids: &[u32]) -> Result<Option<String>> {
let len = prompt_token_ids.len();
let windows = PROMPT_WINDOWS.into_iter().filter(|window| *window < len).chain([len]);
for window in windows {
let text = self.tokenizer.decode(&prompt_token_ids[len - window..], false).map_err(
|error| parsing_failed!("failed to decode prompt: {}", error.as_report()),
)?;
let whole = window == len;
if let Some((start, end)) = self.template.last_start_anchor(&text) {
let first_char = text.chars().next().map_or(0, char::len_utf8);
if whole || start >= first_char {
return Ok(Some(text[end..].to_string()));
}
}
}
Ok(None)
}
}
impl StripState {
fn new(region: &Region) -> Self {
Self {
enabled: matches!(
region.kind,
RegionKind::Text(TextRegion { strip: true, .. })
),
..Self::default()
}
}
/// Split a piece into its visible part and whitespace dropped by `strip`.
///
/// Leading whitespace is dropped until the first non-whitespace character;
/// trailing whitespace is held until more non-whitespace text arrives (then
/// released) or the region closes (then dropped).
fn feed(&mut self, mut piece: DecodedText) -> (DecodedText, DecodedText) {
if !self.enabled {
return (piece, DecodedText::default());
}
let mut dropped = DecodedText::default();
if !self.started {
let lead = piece.text.len() - piece.text.trim_start().len();
if lead == piece.text.len() {
return (DecodedText::default(), piece);
}
dropped = piece.drain_prefix(lead);
self.started = true;
}
let body_len = piece.text.trim_end().len();
if body_len == 0 {
self.held.append(piece);
return (DecodedText::default(), dropped);
}
let head = piece.drain_prefix(body_len);
let mut visible = take(&mut self.held);
visible.append(head);
self.held = piece;
(visible, dropped)
}
}
/// Push visible text of a text region.
fn push_text(output: &mut UnifiedParserOutput, role: TextRole, piece: DecodedText) {
match role {
TextRole::Content => output.push_text(piece.text),
TextRole::Reasoning => output.push_reasoning(piece),
}
}
/// Keep the tokens of stripped reasoning whitespace in the reasoning count.
fn push_dropped_reasoning(output: &mut UnifiedParserOutput, dropped: DecodedText) {
if dropped.attributions.is_empty() {
return;
}
output.push_reasoning(DecodedText {
text: String::new(),
attributions: dropped
.attributions
.into_iter()
.map(|attribution| TokenAttribution {
token_id: attribution.token_id,
anchor: TokenAnchor::ZeroWidth { byte_offset: 0 },
})
.collect(),
});
}
/// One tool call produced by a tool-call region, after its transform.
#[derive(Deserialize)]
#[serde(untagged)]
enum ToolCallValue {
/// The OpenAI shape `{"type": "function", "function": {...}}`.
Wrapped { function: FunctionValue },
/// The bare `{"name", "arguments"}` shape.
Bare(FunctionValue),
}
#[derive(Deserialize)]
struct FunctionValue {
name: String,
/// Missing or `null` arguments become `{}`.
#[serde(default)]
arguments: Option<ArgumentsValue>,
}
#[derive(Deserialize)]
#[serde(untagged)]
enum ArgumentsValue {
/// An argument object, typed by the tool schema.
Object(Map<String, Value>),
/// Pre-serialized or non-JSON arguments (e.g. an `allow_non_json`
/// fallback), passed through unchanged.
Text(String),
/// Any other JSON value, serialized as is.
Other(Value),
}
/// Extract the function name and serialized arguments of one tool-call value.
///
/// String argument values are converted by the calling tool's parameter
/// schema, as for the other tool parsers.
fn call_parts(call: Value, tool_schemas: &ToolSchemas) -> Result<(String, String)> {
let (ToolCallValue::Wrapped { function } | ToolCallValue::Bare(function)) =
ToolCallValue::deserialize(&call).map_err(|_| {
parsing_failed!(
"tool call must be {{\"function\": {{\"name\": ..., \"arguments\": ...}}}} or \
{{\"name\": ..., \"arguments\": ...}} with a string name, got {call}"
)
})?;
let FunctionValue { name, arguments } = function;
let arguments = match arguments {
None => "{}".to_string(),
Some(ArgumentsValue::Text(arguments)) => arguments,
Some(ArgumentsValue::Object(mut arguments)) => {
convert_text_arguments(tool_schemas, &name, &mut arguments);
serialize_arguments(&arguments)?
}
Some(ArgumentsValue::Other(arguments)) => serialize_arguments(&arguments)?,
};
Ok((name, arguments))
}
/// Convert the argument values a content parser left as raw text by the tool
/// schema: top-level strings, and string lists collected from duplicate keys
/// (`merge_duplicates`).
///
/// Deeper values are the content parser's own JSON and are kept, as in
/// Transformers' `_coerce_tool_calls`.
fn convert_text_arguments(
tool_schemas: &ToolSchemas,
function_name: &str,
arguments: &mut Map<String, Value>,
) {
for (name, value) in arguments.iter_mut() {
let values = match value {
Value::Array(items) => items.as_mut_slice(),
value => std::slice::from_mut(value),
};
for value in values {
if let Value::String(text) = value {
*value = tool_schemas.convert_param_with_schema(function_name, name, take(text));
}
}
}
}
/// Serialize tool-call arguments to JSON text.
fn serialize_arguments(arguments: &impl serde::Serialize) -> Result<String> {
serde_json::to_string(arguments).map_err(|error| {
parsing_failed!("failed to serialize tool arguments: {}", error.as_report())
})
}
/// Parse one event: safe text before the next marker, or a boundary at it.
fn parse_next_event(
input: &mut Partial<&str>,
template: &ResponseTemplate,
watch: &WatchSet,
eof: bool,
) -> ModalResult<HfEvent> {
let markers: Vec<&str> = watch.markers.iter().map(String::as_str).collect();
match safe_text_len_mul(input, &markers) {
Ok(_) => return Ok(HfEvent::Text),
// A marker, complete or partial, starts here (or the input is empty).
Err(ErrMode::Incomplete(_)) => {}
Err(error) => return Err(error),
}
// Any pending boundary blocks the position; otherwise the preferred match
// wins. At the end of the stream, a partial marker matches nothing and is
// released as text one character at a time.
let text = **input;
let mut best: Option<BoundaryMatch> = None;
for &watch in &watch.boundaries {
match template.boundary(watch).resolve(text, eof) {
Resolution::Pending => return incomplete(),
Resolution::NoMatch => {}
Resolution::Matched { len, captures } => {
let candidate = BoundaryMatch {
watch,
len,
captures,
};
if best.as_ref().is_none_or(|best| candidate.key(template) < best.key(template)) {
best = Some(candidate);
}
}
}
}
match best {
Some(BoundaryMatch {
watch,
len,
captures,
}) => {
input.next_slice(len);
Ok(HfEvent::Boundary { watch, captures })
}
None => {
// Not a boundary: the marker's first character is plain text.
let len = text.chars().next().map_or(0, char::len_utf8);
input.next_slice(len);
Ok(HfEvent::Text)
}
}
}
impl UnifiedParser for HfUnifiedParser {
fn create(_tools: &[Tool], _tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
where
Self: Sized + 'static,
{
Err(UnifiedParserError::NoNamedConstructor {
parser: "hf",
built_from: "the checkpoint's response_template",
})
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.reset_state();
if prompt_token_ids.is_empty() {
return Ok(());
}
let Some(remainder) = self.prompt_remainder(prompt_token_ids)? else {
self.template.warn_missing_anchor_once();
return Ok(());
};
// A partial boundary at the end of the prompt stays buffered, so the
// generated text can complete it.
self.buffer = DecodedText::unattributed(remainder);
self.drive(&mut UnifiedParserOutput::default(), Phase::Prefill)
}
fn preserve_special_tokens(&self) -> bool {
// Templates spell boundaries with special tokens and expect decoding
// with them kept.
true
}
// TODO: derive a structural-tag output grammar from literal-bounded templates.
fn parse_into(&mut self, delta: DecodedText, output: &mut UnifiedParserOutput) -> Result<()> {
self.buffer.append(delta);
self.drive(output, Phase::Stream)
}
fn finish(&mut self) -> Result<UnifiedParserOutput> {
let mut output = UnifiedParserOutput::default();
self.drive(&mut output, Phase::Eof)?;
// Trailing zero-width tokens carry no text for the step loop to consume.
let rest = self.buffer.take();
self.route(rest, &mut output, Phase::Eof);
self.close_current(&mut output, Phase::Eof)?;
for (region, strip) in self.template.regions.iter().zip(&mut self.strip) {
let held = take(&mut strip.held);
if matches!(&region.kind, RegionKind::Text(text) if text.role == TextRole::Reasoning) {
push_dropped_reasoning(&mut output, held);
}
}
let missing: Vec<_> = self
.template
.regions
.iter()
.zip(&self.closed)
.filter(|(region, closed)| !region.optional && !**closed)
.map(|(region, _)| region.name.as_str())
.collect();
if !missing.is_empty() {
return Err(parsing_failed!(
"Required response_template fields missing from parsed output: {missing:?}"
));
}
self.reset_state();
Ok(output)
}
fn reset(&mut self) -> String {
let mut raw = String::new();
if let HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) =
std::mem::replace(&mut self.mode, HfMode::Discard)
{
raw.push_str(&occurrence.raw_open);
raw.push_str(&occurrence.body);
raw.push_str(&self.strip[occurrence.region].held.text);
}
raw.push_str(&self.buffer.take().text);
self.reset_state();
raw
}
}
rust/src/parser/src/unified/hf/pattern.rs
+403
View File
@@ -0,0 +1,403 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Streaming resolution of `open_pattern` / `close_pattern` boundaries.
//!
//! Transformers relies on the Python `regex` module's `partial=True` search to
//! decide whether a match at the end of the buffer is final, could still appear,
//! or is impossible. The Rust `regex` crate has no partial-match API, so this
//! module answers that question by stepping a `regex-automata` DFA, and uses a
//! `meta::Regex` for the actual match span and captures.
use regex_automata::dfa::{Automaton, StartKind, dense};
use regex_automata::meta::Regex;
use regex_automata::nfa::thompson::{self, WhichCaptures};
use regex_automata::util::primitives::StateID;
use regex_automata::{Anchored, Input, MatchKind, PatternID};
use regex_syntax::hir::literal::{ExtractKind, Extractor};
use regex_syntax::hir::{Hir, HirKind};
use serde_json::{Map, Value};
use thiserror_ext::AsReport as _;
use super::{Result, unsupported};
/// Upper bound for the memory of one boundary DFA.
const DFA_SIZE_LIMIT: usize = 16 << 20;
/// Named capture groups of a boundary match, as string values: the scope a
/// transform template reads them from.
pub(super) type Captures = Map<String, Value>;
/// Outcome of resolving a boundary at a candidate position.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) enum Resolution {
/// The boundary matches `len` bytes; more input cannot change the match.
Matched { len: usize, captures: Captures },
/// More input could still produce or change a match.
Pending,
/// No match starts here, whatever input follows.
NoMatch,
}
/// A regex boundary that can be resolved against a growing buffer.
#[derive(Debug, Clone)]
pub(super) struct StreamingPattern {
/// Finite set of non-empty literals, one of which starts every match.
prefixes: Vec<String>,
/// Anchored leftmost-first DFA of the pattern: decides whether the
/// leftmost-first match is final. Unicode word boundaries are handled
/// heuristically, quitting on non-ASCII bytes.
first: dense::DFA<Vec<u32>>,
/// Anchored DFA with every look-around assertion removed and
/// `MatchKind::All`, used when `first` quits. Relaxing an assertion only
/// enlarges the language, so once this DFA is dead no continuation can
/// produce a later-ending match of the real pattern. `MatchKind::All` is
/// required for that argument: leftmost-first pruning could discard the
/// thread the real pattern takes when an assertion fails (`a\b|ab` on `ab`).
fallback: Option<dense::DFA<Vec<u32>>>,
/// The pattern itself, for the match span and named captures.
exact: Regex,
}
/// Result of walking a DFA over the available bytes.
enum Walk {
/// No continuation can change the result.
Final,
/// More input could still change the result.
Alive,
/// The DFA gave up (non-ASCII byte near a Unicode word boundary).
Quit,
}
impl StreamingPattern {
/// Compile a boundary pattern in Python `regex` syntax, with `re.DOTALL`.
///
/// Streaming resolution only starts at occurrences of the pattern's literal
/// prefixes, so every match must start with one of a finite set of non-empty
/// literals.
pub fn new(scope: &str, pattern: &str) -> Result<Self> {
let hir = regex_syntax::ParserBuilder::new()
.dot_matches_new_line(true)
.build()
.parse(pattern)
.map_err(|error| {
unsupported!(
"{scope}: regex {pattern:?} is not supported by the Rust regex engine: {}",
error.as_report()
)
})?;
let prefixes = literal_prefixes(&hir).ok_or_else(|| {
unsupported!(
"{scope}: pattern {pattern:?} must start with a literal prefix for streaming \
matching (e.g. `<tool_call>` in `<tool_call>(?P<name>\\w+)`)"
)
})?;
let first = build_dfa(scope, pattern, &hir, MatchKind::LeftmostFirst)?;
let fallback = hir
.properties()
.look_set()
.contains_word_unicode()
.then(|| build_dfa(scope, pattern, &relax_assertions(&hir), MatchKind::All))
.transpose()?;
let exact = Regex::builder().build_from_hir(&hir).map_err(|error| {
unsupported!(
"{scope}: failed to compile regex {pattern:?}: {}",
error.as_report()
)
})?;
Ok(Self {
prefixes,
first,
fallback,
exact,
})
}
/// Literals, one of which starts every match.
pub fn prefixes(&self) -> &[String] {
&self.prefixes
}
/// Named capture groups of the pattern.
pub fn group_names(&self) -> impl Iterator<Item = &str> {
self.exact.group_info().pattern_names(PatternID::ZERO).flatten()
}
/// Resolve a match starting at the beginning of `text`.
///
/// `eof` means no more input will arrive.
pub fn resolve(&self, text: &str, eof: bool) -> Resolution {
let walk = match walk(&self.first, text) {
Walk::Quit => walk(
self.fallback.as_ref().expect("quits imply a fallback"),
text,
),
walk => walk,
};
if matches!(walk, Walk::Alive) && !eof {
return Resolution::Pending;
}
let input = Input::new(text).anchored(Anchored::Yes);
let mut captures = self.exact.create_captures();
self.exact.search_captures(&input, &mut captures);
let Some(found) = captures.get_match() else {
return Resolution::NoMatch;
};
let names = self.exact.group_info().pattern_names(PatternID::ZERO);
let captures = names
.enumerate()
.filter_map(|(index, name)| {
let span = captures.get_group(index)?;
Some((
name?.to_string(),
Value::String(text[span.range()].to_string()),
))
})
.collect();
Resolution::Matched {
len: found.end(),
captures,
}
}
}
/// Walk `dfa` from an anchored start over `text`.
///
/// The walk is final once the DFA is dead, or once it is in a match state whose
/// transitions are all dead: that removes the one byte of holdback that delayed
/// DFA match reporting would otherwise add. A quit transition counts as live.
fn walk(dfa: &dense::DFA<Vec<u32>>, text: &str) -> Walk {
let input = Input::new(text).anchored(Anchored::Yes);
let Ok(mut state) = dfa.start_state_forward(&input) else {
return Walk::Quit;
};
for byte in text.bytes() {
state = dfa.next_state(state, byte);
if dfa.is_quit_state(state) {
return Walk::Quit;
}
if dfa.is_dead_state(state) || is_terminal_match(dfa, state) {
return Walk::Final;
}
}
Walk::Alive
}
/// Return whether `state` reports a match and cannot continue.
fn is_terminal_match(dfa: &dense::DFA<Vec<u32>>, state: StateID) -> bool {
dfa.is_match_state(state)
&& (0..=u8::MAX).all(|byte| dfa.is_dead_state(dfa.next_state(state, byte)))
}
/// Build an anchored dense DFA for `hir`.
fn build_dfa(
scope: &str,
pattern: &str,
hir: &Hir,
kind: MatchKind,
) -> Result<dense::DFA<Vec<u32>>> {
let nfa = thompson::Compiler::new()
.configure(thompson::Config::new().which_captures(WhichCaptures::None))
.build_from_hir(hir)
.map_err(|error| {
unsupported!(
"{scope}: failed to compile regex {pattern:?}: {}",
error.as_report()
)
})?;
dense::Builder::new()
.configure(
dense::Config::new()
.match_kind(kind)
.start_kind(StartKind::Anchored)
.unicode_word_boundary(true)
.dfa_size_limit(Some(DFA_SIZE_LIMIT))
.determinize_size_limit(Some(DFA_SIZE_LIMIT)),
)
.build_from_nfa(&nfa)
.map_err(|error| {
unsupported!(
"{scope}: regex {pattern:?} cannot be matched incrementally: {}",
error.as_report()
)
})
}
/// Replace every look-around assertion with the empty regex.
fn relax_assertions(hir: &Hir) -> Hir {
match hir.kind() {
HirKind::Look(_) => Hir::empty(),
HirKind::Repetition(repetition) => {
let mut repetition = repetition.clone();
repetition.sub = Box::new(relax_assertions(&repetition.sub));
Hir::repetition(repetition)
}
HirKind::Capture(capture) => {
let mut capture = capture.clone();
capture.sub = Box::new(relax_assertions(&capture.sub));
Hir::capture(capture)
}
HirKind::Concat(subs) => Hir::concat(subs.iter().map(relax_assertions).collect()),
HirKind::Alternation(subs) => Hir::alternation(subs.iter().map(relax_assertions).collect()),
HirKind::Empty | HirKind::Literal(_) | HirKind::Class(_) => hir.clone(),
}
}
/// Extract the finite set of non-empty literal prefixes starting every match.
fn literal_prefixes(hir: &Hir) -> Option<Vec<String>> {
let seq = Extractor::new().kind(ExtractKind::Prefix).extract(hir);
let mut prefixes: Vec<String> = Vec::new();
for literal in seq.literals()? {
let bytes = literal.as_bytes();
// Truncated inexact literals may end inside a UTF-8 sequence; any valid
// prefix of a prefix is still a prefix.
let valid = match std::str::from_utf8(bytes) {
Ok(text) => text,
Err(error) => std::str::from_utf8(&bytes[..error.valid_up_to()]).ok()?,
};
if valid.is_empty() {
return None;
}
if !prefixes.iter().any(|prefix| prefix == valid) {
prefixes.push(valid.to_string());
}
}
(!prefixes.is_empty()).then_some(prefixes)
}
#[cfg(test)]
mod tests {
use super::*;
const MUSE_INVOKE: &str = r#"<atem:invoke\b[^>]*?\bname="(?P<name>[^"]+)">"#;
const GEMMA4_CALL: &str = r"<\|tool_call>call:(?P<name>\w+)";
const GPT_OSS_CALL: &str =
r"<\|channel\|>commentary to=functions\.(?P<name>\w+).*?<\|message\|>";
fn pattern(source: &str) -> StreamingPattern {
StreamingPattern::new("test", source).unwrap()
}
fn matched(len: usize, name: &str) -> Resolution {
Resolution::Matched {
len,
captures: Captures::from_iter([("name".to_string(), name.into())]),
}
}
#[test]
fn extracts_literal_prefixes() {
assert_eq!(pattern(GEMMA4_CALL).prefixes(), ["<|tool_call>call:"]);
assert_eq!(pattern(MUSE_INVOKE).prefixes(), ["<atem:invoke"]);
assert_eq!(
pattern(r"to=(?:user|commentary)<\|message\|>").prefixes(),
["to=user<|message|>", "to=commentary<|message|>"]
);
assert_eq!(
pattern(r"\n?</response>").prefixes(),
["\n</response>", "</response>"]
);
for source in [r"(?:^|<think>\s*)", r"(?:<\|m\|>)?[^<]*<\|c\|>", r".*x"] {
let result = StreamingPattern::new("test", source);
assert!(
matches!(
result,
Err(super::super::HfTemplateError::Unsupported { .. })
),
"{source}"
);
}
}
#[test]
fn word_boundaries_follow_unicode_semantics() {
let muse = pattern(MUSE_INVOKE);
assert_eq!(
muse.resolve(r#"<atem:invoke name="get">rest"#, false),
matched(24, "get")
);
let unicode_attr = r#"<atem:invoke title="天气" name="获取">rest"#;
assert_eq!(
muse.resolve(unicode_attr, false),
matched(unicode_attr.len() - 4, "获取")
);
assert_eq!(
muse.resolve(r#"<atem:invoke name="get"#, false),
Resolution::Pending
);
assert_eq!(
muse.resolve(r#"<atem:invoker name="x">r"#, false),
Resolution::NoMatch
);
assert_eq!(
muse.resolve(r#"<atem:invoke xname="x">r"#, false),
Resolution::NoMatch
);
assert_eq!(
muse.resolve(r#"<atem:invoke é name="x">r"#, false),
matched(25, "x")
);
assert_eq!(
muse.resolve(r#"<atem:invoke éname="x">r"#, false),
Resolution::NoMatch
);
assert_eq!(
muse.resolve("<atem:invoke 使用标签", false),
Resolution::Pending
);
assert_eq!(
muse.resolve("<atem:invoke 使用标签", true),
Resolution::NoMatch
);
assert_eq!(
muse.resolve("<atem:invoke 使用> 正文", false),
Resolution::NoMatch
);
let alternation = pattern(r"a\b|ab");
assert_eq!(
alternation.resolve("ab!", true),
Resolution::Matched {
len: 2,
captures: Captures::new()
}
);
}
#[test]
fn greedy_and_lazy_tails_commit_like_transformers() {
let gemma4 = pattern(GEMMA4_CALL);
assert_eq!(
gemma4.resolve("<|tool_call>call:get_wea", false),
Resolution::Pending
);
assert_eq!(
gemma4.resolve("<|tool_call>call:get_weather{", false),
matched(28, "get_weather")
);
assert_eq!(
gemma4.resolve("<|tool_call>call:天气{", false),
matched(23, "天气")
);
assert_eq!(
gemma4.resolve("<|tool_call>call:{", false),
Resolution::NoMatch
);
assert_eq!(
gemma4.resolve("<|tool_call>call:get", true),
matched(20, "get")
);
let gpt_oss = pattern(GPT_OSS_CALL);
let opener = "<|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>";
assert_eq!(gpt_oss.resolve(opener, false), Resolution::Pending);
assert_eq!(
gpt_oss.resolve(&format!("{opener}{{\"a\""), false),
matched(opener.len(), "get_weather")
);
}
}
rust/src/parser/src/unified/hf/spec.rs
+202
View File
@@ -0,0 +1,202 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Serde mirror of the `response_template` JSON.
//!
//! Key sets and defaults follow `response_templates.py`; semantic validation
//! happens when compiling into [`super::ResponseTemplate`].
use serde::Deserialize;
use serde::de::DeserializeOwned;
use serde_json::{Map, Value};
use serde_with::serde_as;
use thiserror_ext::AsReport as _;
use super::{Result, invalid};
/// Top-level `response_template` object.
#[serde_as]
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) struct TemplateSpec {
#[serde(default = "default_version")]
version: u64,
/// Constant keys of the Transformers output dict (e.g. `role`). Unused by the
/// event-based executor, but validated as part of the schema.
#[serde(default, rename = "defaults")]
_defaults: Map<String, Value>,
/// Field specs in declaration order.
#[serde_as(as = "serde_with::Map<_, _>")]
pub fields: Vec<(String, FieldSpec)>,
start_anchor: Option<Literals>,
start_anchor_pattern: Option<String>,
}
/// One entry of `fields`.
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) struct FieldSpec {
open: Option<Literals>,
open_pattern: Option<String>,
close: Option<Literals>,
close_pattern: Option<String>,
#[serde(default)]
pub content: ContentKind,
/// Arguments of the `content` parser; their keys depend on the parser.
#[serde(default)]
pub content_args: Map<String, Value>,
#[serde(default)]
pub repeats: bool,
pub join: Option<String>,
#[serde(default = "default_optional")]
pub optional: bool,
/// A free-form JSON template, compiled into a [`super::transform::Transform`].
pub transform: Option<Value>,
#[serde(default)]
pub transform_each: bool,
}
/// Name of a content parser.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub(super) enum ContentKind {
#[default]
Text,
Int,
Float,
Bool,
Json,
XmlInline,
KvLines,
}
/// A literal anchor: one string, or a list of alternatives.
#[derive(Debug, Clone, Deserialize)]
#[serde(untagged)]
enum Literals {
One(String),
Many(Vec<String>),
}
/// An anchor after resolving its literal-or-pattern form.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) enum AnchorSpec<'a> {
/// Deduplicated literals in first-seen order.
Literals(Vec<String>),
/// A regex in Python `regex` syntax.
Pattern(&'a str),
}
impl TemplateSpec {
/// Parse the top-level template object.
pub fn from_value(value: &Value) -> Result<Self> {
let spec: Self = deserialize(value, "response_template")?;
if spec.version != 1 {
return Err(invalid!(
"unsupported response_template version: {}",
spec.version
));
}
if spec.fields.is_empty() {
return Err(invalid!(
"response_template.fields must be a non-empty dict"
));
}
Ok(spec)
}
/// The required start anchor.
pub fn start_anchor(&self) -> Result<AnchorSpec<'_>> {
anchor_spec(
"response_template",
(self.start_anchor.as_ref(), "start_anchor"),
(self.start_anchor_pattern.as_deref(), "start_anchor_pattern"),
)?
.ok_or_else(|| {
invalid!("response_template must define 'start_anchor' or 'start_anchor_pattern'.")
})
}
}
impl FieldSpec {
/// The open anchor; `None` for the implicit field.
pub fn open(&self, scope: &str) -> Result<Option<AnchorSpec<'_>>> {
anchor_spec(
scope,
(self.open.as_ref(), "open"),
(self.open_pattern.as_deref(), "open_pattern"),
)
}
/// The close anchor; `None` when the field runs to the end of the stream.
pub fn close(&self, scope: &str) -> Result<Option<AnchorSpec<'_>>> {
anchor_spec(
scope,
(self.close.as_ref(), "close"),
(self.close_pattern.as_deref(), "close_pattern"),
)
}
}
/// Deserialize `value`, reporting `context` and the JSON path of any error.
pub(super) fn deserialize<T: DeserializeOwned>(value: &Value, context: &str) -> Result<T> {
serde_path_to_error::deserialize(value).map_err(|error| {
let path = error.path().to_string();
let error = error.into_inner();
if path == "." {
invalid!("{context}: {}", error.as_report())
} else {
invalid!("{context}.{path}: {}", error.as_report())
}
})
}
/// Resolve an anchor given in literal form or regex form.
///
/// Mirrors `_compile_anchor`: both forms at once is an error, literal lists must be
/// non-empty and contain no empty strings, and duplicates are dropped in first-seen
/// order.
fn anchor_spec<'a>(
scope: &str,
(literals, literal_key): (Option<&Literals>, &str),
(pattern, pattern_key): (Option<&'a str>, &str),
) -> Result<Option<AnchorSpec<'a>>> {
match (literals, pattern) {
(Some(_), Some(_)) => Err(invalid!(
"{scope}: cannot specify both '{literal_key}' and '{pattern_key}'"
)),
(Some(literals), None) => {
let raw = match literals {
Literals::One(literal) => std::slice::from_ref(literal),
Literals::Many(literals) => literals.as_slice(),
};
if raw.is_empty() {
return Err(invalid!(
"{scope}: '{literal_key}' list must contain at least one literal"
));
}
if raw.iter().any(String::is_empty) {
return Err(invalid!(
"{scope}: '{literal_key}' literals cannot be empty strings"
));
}
let mut deduped: Vec<String> = Vec::with_capacity(raw.len());
for literal in raw {
if !deduped.contains(literal) {
deduped.push(literal.clone());
}
}
Ok(Some(AnchorSpec::Literals(deduped)))
}
(None, Some(pattern)) => Ok(Some(AnchorSpec::Pattern(pattern))),
(None, None) => Ok(None),
}
}
fn default_version() -> u64 {
1
}
fn default_optional() -> bool {
true
}
rust/src/parser/src/unified/hf/template.rs
+481
View File
@@ -0,0 +1,481 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Compiled `response_template`.
//!
//! Validation follows `response_templates.py`; the compiled form additionally
//! maps fields to parser roles and precomputes each state's boundary markers.
use std::fmt;
use std::sync::atomic::{AtomicBool, Ordering};
use regex_automata::meta::Regex;
use serde_json::Value;
use super::content::{ContentParser, TextArgs, compile_regex};
use super::pattern::{Captures, Resolution, StreamingPattern};
use super::spec::{AnchorSpec, FieldSpec, TemplateSpec};
use super::transform::{FieldTransform, Transform};
use super::{Result, invalid, unsupported};
/// A compiled `response_template`, shared by all requests of one model.
#[derive(Debug)]
pub struct ResponseTemplate {
/// Matches the start of the current assistant turn in the prompt.
start_anchor: Regex,
pub(super) regions: Vec<Region>,
/// The field without an opener: the implicit-open / leftover sink.
pub(super) implicit: Option<usize>,
/// Boundaries watched outside explicit regions: every explicit open plus the
/// implicit region's close.
pub(super) idle_watch: WatchSet,
warned_missing_anchor: AtomicBool,
}
/// One compiled field.
#[derive(Debug)]
pub(super) struct Region {
pub name: FieldName,
/// `None`: the implicit region.
pub open: Option<Boundary>,
/// `None`: the region runs to the end of the stream.
pub close: Option<Boundary>,
/// Boundaries watched inside this region: its close, if any.
pub close_watch: WatchSet,
pub optional: bool,
pub kind: RegionKind,
}
/// A field this parser can report, in the lexical order of its name, which
/// Transformers uses to break ties between boundaries.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub(super) enum FieldName {
Content,
ReasoningContent,
Thinking,
ToolCalls,
}
/// How a region's content is reported.
#[derive(Debug)]
pub(super) enum RegionKind {
/// Streamed as it arrives.
Text(TextRegion),
/// Buffered, and parsed into tool calls at close.
ToolCalls(ToolCallRegion),
}
#[derive(Debug)]
pub(super) struct TextRegion {
pub role: TextRole,
/// Whether the value is stripped (`content_args.strip`).
pub strip: bool,
pub repeat: Repeat,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum TextRole {
Content,
Reasoning,
}
/// How repeated occurrences of a text field combine.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) enum Repeat {
/// No `repeats`: occurrences stream as one value.
Once,
/// `repeats` without `join`: each occurrence is its own value.
Each,
/// `repeats` with `join`: occurrences are joined by the separator.
Join(String),
}
#[derive(Debug)]
pub(super) struct ToolCallRegion {
pub content: ContentParser,
pub transform: Option<FieldTransform>,
/// Opener capture group that provides the function name, allowing the tool
/// call to start before its arguments are complete.
pub name_group: Option<String>,
}
/// A region delimiter.
#[derive(Debug)]
pub(super) enum Boundary {
/// Literal alternatives, longest first.
Literals(Vec<String>),
Pattern(Box<StreamingPattern>),
}
/// One boundary the executor watches for in a given state.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) struct Watch {
pub region: usize,
pub kind: WatchKind,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum WatchKind {
Open,
Close,
}
/// The boundaries watched in one parser mode.
#[derive(Debug, Default)]
pub(super) struct WatchSet {
pub boundaries: Vec<Watch>,
/// Literals, deduplicated, whose occurrences are the only positions any of
/// the boundaries can match at.
pub markers: Vec<String>,
}
impl ResponseTemplate {
/// Compile the `response_template` object from `tokenizer_config.json`.
pub fn from_json(value: &Value) -> Result<Self> {
let spec = TemplateSpec::from_value(value)?;
let regions = spec
.fields
.iter()
.enumerate()
.map(|(index, (name, field))| Region::compile(index, name, field))
.collect::<Result<Vec<_>>>()?;
// A field without an open anchor is the implicit-open / leftover sink; at most one is allowed.
let implicit_fields: Vec<_> =
regions.iter().enumerate().filter(|(_, region)| region.open.is_none()).collect();
if implicit_fields.len() > 1 {
let names: Vec<_> =
implicit_fields.iter().map(|(_, region)| region.name.as_str()).collect();
return Err(invalid!(
"At most one field may omit 'open'/'open_pattern' (that field becomes the \
implicit-open / leftover sink). Found: {}",
names.join(", ")
));
}
let implicit = implicit_fields.first().map(|(index, _)| *index);
let start_anchor = match spec.start_anchor()? {
AnchorSpec::Literals(mut literals) => {
// Sort longest-first so alternation prefers the longer alternative when both could match.
literals.sort_by_key(|literal| std::cmp::Reverse(literal.len()));
let alternation: Vec<_> =
literals.iter().map(|literal| regex_syntax::escape(literal)).collect();
compile_regex("response_template", &alternation.join("|"))?
}
AnchorSpec::Pattern(pattern) => compile_regex("response_template", pattern)?,
};
let opens = regions.iter().enumerate().filter_map(|(region, spec)| {
let watch = Watch {
region,
kind: WatchKind::Open,
};
Some((watch, spec.open.as_ref()?))
});
let implicit_close = implicit.and_then(|region| {
let watch = Watch {
region,
kind: WatchKind::Close,
};
Some((watch, regions[region].close.as_ref()?))
});
let idle_watch = WatchSet::new(opens.chain(implicit_close));
Ok(Self {
start_anchor,
regions,
implicit,
idle_watch,
warned_missing_anchor: AtomicBool::new(false),
})
}
/// Return the start and end offsets of the last start-anchor match in
/// `prompt`.
pub(super) fn last_start_anchor(&self, prompt: &str) -> Option<(usize, usize)> {
self.start_anchor
.find_iter(prompt)
.last()
.map(|found| (found.start(), found.end()))
}
/// Log once per template that the prompt carried no start anchor.
pub(super) fn warn_missing_anchor_once(&self) {
if !self.warned_missing_anchor.swap(true, Ordering::Relaxed) {
tracing::warn!(
"response_template start anchor not found in the prompt; parsing starts from the \
template's initial state (the chat template may not match the checkpoint's \
response_template)"
);
}
}
/// The boundary of a watch entry.
pub(super) fn boundary(&self, watch: Watch) -> &Boundary {
let region = &self.regions[watch.region];
match watch.kind {
WatchKind::Open => region.open.as_ref(),
WatchKind::Close => region.close.as_ref(),
}
.expect("watched boundaries exist")
}
}
impl Region {
/// Validate the spec of the field at `index` and compile it.
fn compile(index: usize, name: &str, field: &FieldSpec) -> Result<Self> {
let scope = format!("Field '{name}'");
let open = field.open(&scope)?.map(|spec| Boundary::compile(&scope, spec)).transpose()?;
let close = field.close(&scope)?.map(|spec| Boundary::compile(&scope, spec)).transpose()?;
let content = ContentParser::new(&scope, field.content, &field.content_args)?;
let repeat = match (field.repeats, &field.join) {
(false, None) => Repeat::Once,
(true, None) => Repeat::Each,
(true, Some(join)) => Repeat::Join(join.clone()),
(false, Some(_)) => return Err(invalid!("{scope}: 'join' requires 'repeats': true")),
};
let transform = match (&field.transform, field.transform_each) {
(Some(template), each) => Some(FieldTransform {
template: Transform::compile(&scope, template)?,
each,
}),
(None, false) => None,
(None, true) => {
return Err(invalid!(
"{scope}: transform_each is set but no transform was provided"
));
}
};
let captured: Vec<&str> =
[&open, &close].into_iter().flatten().flat_map(Boundary::group_names).collect();
match &transform {
None if !captured.is_empty() => {
// Named captures only reach the output through a transform, so flag any that would be silently dropped.
return Err(invalid!(
"{scope}: open_pattern/close_pattern declares named group(s) {captured:?}, but \
the field has no 'transform'. Named captures are only surfaced through a \
'transform' template (where they appear alongside 'content'). Either add a \
'transform' that uses the captures, or remove the named groups from the pattern."
));
}
// Without `transform_each`, the scope is the opener's captures plus `content`;
// any other placeholder fails on every match.
Some(FieldTransform {
template,
each: false,
}) => {
let open_groups: Vec<&str> = open.iter().flat_map(Boundary::group_names).collect();
if let Some(root) = template
.placeholder_roots()
.find(|root| *root != "content" && !open_groups.contains(root))
{
return Err(invalid!(
"{scope}: transform placeholder '{{{root}}}' is neither 'content' nor a \
named group of open_pattern"
));
}
}
_ => {}
}
let name = FieldName::parse(&scope, name)?;
let kind = match name.text_role() {
Some(role) => {
let strip = match (content, &transform) {
(ContentParser::Text(TextArgs { strip }), None) => strip,
_ => {
return Err(unsupported!(
"{scope}: text and reasoning fields must use the 'text' content \
parser without a transform"
));
}
};
RegionKind::Text(TextRegion {
role,
strip,
repeat,
})
}
None => {
if matches!(repeat, Repeat::Join(_)) {
return Err(unsupported!(
"{scope}: 'join' requires each match to parse to a string"
));
}
let name_group = name_group(transform.as_ref(), open.as_ref());
RegionKind::ToolCalls(ToolCallRegion {
content,
transform,
name_group,
})
}
};
let close_watch = WatchSet::new(close.as_ref().map(|close| {
let watch = Watch {
region: index,
kind: WatchKind::Close,
};
(watch, close)
}));
Ok(Self {
name,
open,
close,
close_watch,
optional: field.optional,
kind,
})
}
}
/// The opener capture group that a non-`transform_each` transform uses as the
/// function name, if any.
fn name_group(transform: Option<&FieldTransform>, open: Option<&Boundary>) -> Option<String> {
let (
FieldTransform {
template,
each: false,
},
open,
) = (transform?, open?)
else {
return None;
};
let path = template
.placeholder_at(&["function", "name"])
.or_else(|| template.placeholder_at(&["name"]))?;
match path {
[root] if open.group_names().any(|group| group == root) => Some(root.clone()),
_ => None,
}
}
impl FieldName {
/// Map a field name to a reportable field.
fn parse(scope: &str, name: &str) -> Result<Self> {
Ok(match name {
"content" => Self::Content,
"reasoning_content" => Self::ReasoningContent,
"thinking" => Self::Thinking,
"tool_calls" => Self::ToolCalls,
_ => {
return Err(unsupported!(
"{scope}: only 'content', 'reasoning_content'/'thinking', and 'tool_calls' \
fields can be reported"
));
}
})
}
/// The role of a text field; `None` for `tool_calls`.
fn text_role(self) -> Option<TextRole> {
match self {
Self::Content => Some(TextRole::Content),
Self::ReasoningContent | Self::Thinking => Some(TextRole::Reasoning),
Self::ToolCalls => None,
}
}
pub fn as_str(self) -> &'static str {
match self {
Self::Content => "content",
Self::ReasoningContent => "reasoning_content",
Self::Thinking => "thinking",
Self::ToolCalls => "tool_calls",
}
}
}
impl fmt::Display for FieldName {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
impl Boundary {
/// Compile an anchor into a boundary.
fn compile(scope: &str, spec: AnchorSpec<'_>) -> Result<Self> {
Ok(match spec {
AnchorSpec::Literals(mut literals) => {
// Sort longest-first so the longer alternative wins when both match.
literals.sort_by_key(|literal| std::cmp::Reverse(literal.len()));
Self::Literals(literals)
}
AnchorSpec::Pattern(pattern) => {
Self::Pattern(Box::new(StreamingPattern::new(scope, pattern)?))
}
})
}
/// Literals whose occurrences are the only positions this boundary can match at.
pub fn markers(&self) -> &[String] {
match self {
Self::Literals(literals) => literals,
Self::Pattern(pattern) => pattern.prefixes(),
}
}
/// Named capture groups.
pub fn group_names(&self) -> impl Iterator<Item = &str> {
match self {
Self::Literals(_) => None,
Self::Pattern(pattern) => Some(pattern.group_names()),
}
.into_iter()
.flatten()
}
/// Resolve this boundary at the start of `text`.
pub fn resolve(&self, text: &str, eof: bool) -> Resolution {
let markers = self.markers();
// A marker the available text is a proper prefix of: more input decides.
let partial = !eof
&& markers
.iter()
.any(|marker| marker.len() > text.len() && marker.starts_with(text));
match self {
Self::Literals(literals) => {
if partial {
return Resolution::Pending;
}
match literals.iter().find(|literal| text.starts_with(literal.as_str())) {
Some(literal) => Resolution::Matched {
len: literal.len(),
captures: Captures::new(),
},
None => Resolution::NoMatch,
}
}
Self::Pattern(pattern) => {
if markers.iter().any(|marker| text.starts_with(marker.as_str())) {
pattern.resolve(text, eof)
} else if partial {
Resolution::Pending
} else {
Resolution::NoMatch
}
}
}
}
}
impl WatchSet {
/// Collect the watched boundaries and their deduplicated markers.
fn new<'a>(entries: impl IntoIterator<Item = (Watch, &'a Boundary)>) -> Self {
let mut set = Self::default();
for (watch, boundary) in entries {
set.boundaries.push(watch);
for marker in boundary.markers() {
if !set.markers.contains(marker) {
set.markers.push(marker.clone());
}
}
}
set
}
}
rust/src/parser/src/unified/hf/tests.rs
+933
View File
@@ -0,0 +1,933 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Executor tests.
//!
//! Template fixtures and expected values are ported from Transformers
//! `tests/utils/test_chat_parsing.py` at 6d43ab4008, with the expected output
//! dict re-expressed as aggregated unified parser events (`thinking` becomes
//! `reasoning`).
use std::collections::BTreeMap;
use std::sync::Arc;
use serde_json::{Value, json};
use vllm_tokenizer::test_utils::TestTokenizer;
use vllm_tokenizer::{DecodedText, TokenAnchor, TokenAttribution, Tokenizer as _};
use super::{HfTemplateError, HfUnifiedParser, ResponseTemplate};
use crate::tool::Tool;
use crate::tool::test_utils::split_by_chars;
use crate::unified::{UnifiedParser, UnifiedParserError, UnifiedParserEvent, UnifiedParserOutput};
fn cohere_template() -> Value {
json!({
"defaults": {"role": "assistant"},
"start_anchor": "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",
"fields": {
"content": {"open": "<|START_RESPONSE|>", "close": "<|END_RESPONSE|>", "content": "text"},
"thinking": {"open": "<|START_THINKING|>", "close": "<|END_THINKING|>", "content": "text"},
"tool_calls": {
"open": "<|START_ACTION|>",
"close": "<|END_ACTION|>",
"content": "json",
"transform_each": true,
"transform": {"type": "function", "function": {"name": "{tool_name}", "arguments": "{parameters}"}},
},
},
})
}
fn gpt_oss_template() -> Value {
json!({
"defaults": {"role": "assistant"},
"start_anchor": "<|start|>assistant",
"fields": {
"thinking": {"open": "<|channel|>analysis<|message|>", "close": "<|end|>", "content": "text"},
"content": {"open": "<|channel|>final<|message|>", "close": "<|end|>", "content": "text"},
"tool_calls": {
"open_pattern": r"<\|channel\|>commentary to=functions\.(?P<name>\w+).*?<\|message\|>",
"close": "<|call|>",
"repeats": true,
"content": "json",
"transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},
},
},
})
}
fn smollm_template() -> Value {
json!({
"defaults": {"role": "assistant"},
"start_anchor": "<|im_start|>assistant\n",
"fields": {
"thinking": {"open": "<think>", "close": "</think>", "content": "text"},
"tool_calls": {
"open": "<tool_call>",
"close": "</tool_call>",
"repeats": true,
"content": "json",
"transform": {"type": "function", "function": "{content}"},
},
"content": {"close": "<|im_end|>", "content": "text"},
},
})
}
fn qwen3_template() -> Value {
json!({
"defaults": {"role": "assistant"},
"start_anchor": "<|im_start|>assistant\n",
"fields": {
"thinking": {"open": "<think>", "close": "</think>", "content": "text"},
"tool_calls": {
"open_pattern": r"<tool_call>\s*<function=(?P<name>\w+)>",
"close": "</tool_call>",
"repeats": true,
"content": "xml-inline",
"content_args": {
"tag_pattern": r"<parameter=(?P<key>\w+)>\s*(?P<value>.*?)\s*</parameter>",
"value_parser": {"name": "json", "args": {"allow_non_json": true}},
},
"transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},
},
},
})
}
/// Identical to the template shipped in the Gemma 4 IT checkpoints.
fn gemma4_template() -> Value {
json!({
"defaults": {"role": "assistant"},
"start_anchor": ["<|turn>model\n", "<tool_response|>"],
"fields": {
"thinking": {"open": "<|channel>thought\n", "close": "<channel|>", "content": "text"},
"tool_calls": {
"open_pattern": r"<\|tool_call>call:(?P<name>\w+)",
"close": "<tool_call|>",
"repeats": true,
"content": "json",
"content_args": {"unquoted_keys": true, "string_delims": [["<|\"|>", "<|\"|>"]]},
"transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},
},
"content": {"close": ["<turn|>", "<|tool_response>", "<eos>"], "content": "text"},
},
})
}
/// A routed-message protocol with XML-style invocations: no implicit field
/// (text outside regions is discarded), Unicode word boundaries and lazy
/// quantifiers in the opener, `xml-inline` arguments, and joined content.
fn invoke_template() -> Value {
json!({
"defaults": {"role": "assistant"},
"start_anchor": "<|begin|>assistant",
"fields": {
"reasoning_content": {"open_pattern": r"to=self<\|msg\|>", "close": "<|pause|>", "content": "text"},
"tool_calls": {
"open_pattern": r#"<invoke\b[^>]*?\bname="(?P<name>[^"]+)">"#,
"close": "</invoke>",
"repeats": true,
"content": "xml-inline",
"content_args": {
"tag_pattern": r#"<param\b[^>]*?\bname="(?P<key>[^"]+)"[^>]*?>(?P<value>.*?)</param>"#,
"value_parser": {"name": "json", "args": {"allow_non_json": true}},
},
"transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},
},
"content": {
"open_pattern": r"to=(?:user|note)<\|msg\|>",
"close": ["<|end|>", "<|pause|>"],
"repeats": true,
"join": "",
"content": "text",
},
},
})
}
fn compile(template: Value) -> Arc<ResponseTemplate> {
Arc::new(ResponseTemplate::from_json(&template).unwrap())
}
fn tokenizer() -> Arc<TestTokenizer> {
Arc::new(TestTokenizer::new())
}
/// Parse `chunks` after initializing from `prompt`, returning all events.
fn parse_events(
template: &Arc<ResponseTemplate>,
tools: &[Tool],
prompt: &str,
chunks: &[&str],
) -> Result<Vec<UnifiedParserEvent>, UnifiedParserError> {
let tokenizer = tokenizer();
let prompt_ids = tokenizer.encode(prompt, false).unwrap();
let mut parser = HfUnifiedParser::new(template.clone(), tools, tokenizer);
parser.initialize(&prompt_ids)?;
let mut output = UnifiedParserOutput::default();
for chunk in chunks {
parser.parse_into(DecodedText::unattributed(*chunk), &mut output)?;
}
output.append(parser.finish()?);
Ok(output.events)
}
/// Aggregate events into the shape of the Transformers output dict.
fn message(events: &[UnifiedParserEvent]) -> Value {
let mut reasoning = String::new();
let mut content = String::new();
let mut calls: BTreeMap<usize, (String, String)> = BTreeMap::new();
for event in events {
match event {
UnifiedParserEvent::Text(text) => content.push_str(text),
UnifiedParserEvent::Reasoning(piece) => reasoning.push_str(&piece.text),
UnifiedParserEvent::ToolCall(delta) => {
let call = calls.entry(delta.tool_index).or_default();
if let Some(name) = &delta.name {
call.0.clone_from(name);
}
call.1.push_str(&delta.arguments);
}
}
}
let mut out = serde_json::Map::new();
if !reasoning.is_empty() {
out.insert("reasoning".to_string(), reasoning.into());
}
if !content.is_empty() {
out.insert("content".to_string(), content.into());
}
if !calls.is_empty() {
let calls = calls
.into_values()
.map(|(name, arguments)| {
let arguments =
serde_json::from_str(&arguments).unwrap_or(Value::String(arguments));
json!({"name": name, "arguments": arguments})
})
.collect();
out.insert("tool_calls".to_string(), Value::Array(calls));
}
Value::Object(out)
}
fn parse(template: Value, text: &str) -> Value {
message(&parse_events(&compile(template), &[], "", &[text]).unwrap())
}
/// Split `text` at pseudo-random char boundaries (xorshift, seeded).
fn random_chunks(text: &str, seed: u64) -> Vec<&str> {
let mut state = seed | 1;
let mut next = move || {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
state
};
let mut chunks = Vec::new();
let mut start = 0;
for (index, _) in text.char_indices().skip(1) {
if next() % 3 == 0 {
chunks.push(&text[start..index]);
start = index;
}
}
chunks.push(&text[start..]);
chunks
}
#[test]
fn upstream_fixtures_match_transformers_values() {
let cases = [
(
cohere_template(),
"<|START_THINKING|>I should call a tool.<|END_THINKING|><|START_ACTION|>[\n {\"tool_call_id\": \"0\", \"tool_name\": \"simple_tool\", \"parameters\": {\"temperature_format\": \"Celsius\"}}\n]<|END_ACTION|><|END_OF_TURN_TOKEN|>",
json!({
"reasoning": "I should call a tool.",
"tool_calls": [{"name": "simple_tool", "arguments": {"temperature_format": "Celsius"}}],
}),
),
(
gpt_oss_template(),
"<|channel|>analysis<|message|>We will call function get_current_weather.<|end|><|start|>assistant<|channel|>commentary to=functions.get_current_weather <|constrain|>json<|message|>{\n \"location\": \"San Francisco, CA\"\n}",
json!({
"reasoning": "We will call function get_current_weather.",
"tool_calls": [{"name": "get_current_weather", "arguments": {"location": "San Francisco, CA"}}],
}),
),
(
gpt_oss_template(),
"<|channel|>analysis<|message|>User asks a simple math question: 2+2 = 4. Provide answer.<|end|><|start|>assistant<|channel|>final<|message|>2",
json!({
"reasoning": "User asks a simple math question: 2+2 = 4. Provide answer.",
"content": "2",
}),
),
(
smollm_template(),
"<think>\nOkay, let me greet them.\n</think>\n\n<tool_call>{\"name\": \"greet_user\", \"arguments\": {\"greeting\": \"Hello!\"}}</tool_call>",
json!({
"reasoning": "Okay, let me greet them.",
"tool_calls": [{"name": "greet_user", "arguments": {"greeting": "Hello!"}}],
}),
),
(
smollm_template(),
"<tool_call>{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Paris\"}}</tool_call>",
json!({"tool_calls": [{"name": "get_weather", "arguments": {"city": "Paris"}}]}),
),
(
smollm_template(),
"<think>\nOkay, gravity.</think>\nSome content about gravity goes here!",
json!({"reasoning": "Okay, gravity.", "content": "Some content about gravity goes here!"}),
),
(
qwen3_template(),
"<tool_call>\n<function=get_weather>\n<parameter=locations>\n[{\"country\": \"France\", \"city\": \"Paris\"}]\n</parameter>\n<parameter=temp_units>\ncelsius\n</parameter>\n</function>\n</tool_call>",
json!({
"tool_calls": [{
"name": "get_weather",
"arguments": {"locations": [{"country": "France", "city": "Paris"}], "temp_units": "celsius"},
}],
}),
),
(
gemma4_template(),
"<|channel>thought\nThe user is asking for the temperature.<channel|><|tool_call>call:get_current_temperature{detail_level:0,location:<|\"|>Paris, France<|\"|>,unit:<|\"|>celsius<|\"|>}<tool_call|><|tool_response>",
json!({
"reasoning": "The user is asking for the temperature.",
"tool_calls": [{
"name": "get_current_temperature",
"arguments": {"detail_level": 0, "location": "Paris, France", "unit": "celsius"},
}],
}),
),
(
gemma4_template(),
"<|channel>thought\nLet me call the tool.<channel|><|tool_call>call:foo{bool_value:true,list_value:[<|\"|>foo<|\"|>,<|\"|>bar<|\"|>],null_value:null,number_value:1,string_value:<|\"|>foo<|\"|>,struct_value:{foo:<|\"|>bar<|\"|>}}<tool_call|>",
json!({
"reasoning": "Let me call the tool.",
"tool_calls": [{
"name": "foo",
"arguments": {
"bool_value": true,
"list_value": ["foo", "bar"],
"null_value": null,
"number_value": 1,
"string_value": "foo",
"struct_value": {"foo": "bar"},
},
}],
}),
),
];
for (template, text, expected) in cases {
assert_eq!(parse(template, text), expected, "{text}");
}
}
/// Upstream fixtures whose patterns have no literal prefix are rejected at load.
#[test]
fn patterns_without_literal_prefix_are_unsupported() {
let ernie = json!({
"start_anchor": "Assistant:",
"fields": {"thinking": {"open_pattern": r"(?:^|<think>\s*)", "close": "</think>"}},
});
let inkling = json!({
"start_anchor": "<|message_model|>",
"fields": {"content": {"open_pattern": r"(?:<\|message_model\|>)?[^<]*<\|content_text\|>", "close": "<|end_message|>"}},
});
for template in [ernie, inkling] {
assert!(matches!(
ResponseTemplate::from_json(&template),
Err(HfTemplateError::Unsupported { .. })
));
}
}
#[test]
fn streaming_matches_whole_string_for_every_chunking() {
let fixtures = [
(
cohere_template(),
"<|START_THINKING|>I should call a tool.<|END_THINKING|><|START_ACTION|>[{\"tool_call_id\": \"0\", \"tool_name\": \"simple_tool\", \"parameters\": {\"a\": 1}}]<|END_ACTION|>",
),
(
gpt_oss_template(),
"<|channel|>analysis<|message|>thinking chunk<|end|><|channel|>final<|message|>done text",
),
(
gpt_oss_template(),
"<|channel|>analysis<|message|>Let me check.<|end|><|start|>assistant<|channel|>commentary to=functions.get_current_weather <|constrain|>json<|message|>{\"location\": \"San Francisco, CA\"}<|call|>",
),
(
smollm_template(),
"<think>thinking</think>\n<tool_call>{\"name\": \"fn\", \"arguments\": {\"x\": 1}}</tool_call>",
),
(
qwen3_template(),
"<think>short thought</think>\n<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n</parameter>\n</function>\n</tool_call>",
),
(
gemma4_template(),
"<|channel>thought\nhi 天气<channel|>前言 <|tool_call>call:foo{a:1,b:<|\"|>bar<|\"|>}<tool_call|> after",
),
(
invoke_template(),
"header to=self<|msg|>plan it<|pause|>to=user<|msg|>Hello <|pause|><invoke id=\"1\" name=\"get\"><param name=\"city\">\"Paris\"</param><param name=\"days\">3</param></invoke>to=note<|msg|>world<|end|>",
),
];
for (template, text) in fixtures {
let template = compile(template);
let expected = message(&parse_events(&template, &[], "", &[text]).unwrap());
let mut chunkings: Vec<Vec<&str>> =
[1, 2, 3, 5, 7, 13, 31].map(|step| split_by_chars(text, step)).into();
chunkings.extend((0..30).map(|seed| random_chunks(text, 0xC0DE_5EED + seed)));
for chunks in chunkings {
assert_eq!(chunks.concat(), text);
let actual = message(&parse_events(&template, &[], "", &chunks).unwrap());
assert_eq!(actual, expected, "{chunks:?}");
}
}
}
#[test]
fn gemma4_streams_text_and_starts_calls_at_the_opener() {
let template = compile(gemma4_template());
let chunks = [
"<|channel>thought\n",
"Check ",
"the weather. ",
"<channel|>Sure",
". <|tool_call>call:get_weather",
"{location:<|\"|>Paris<|\"|>}",
"<tool_call|>",
];
let tokenizer = tokenizer();
let mut parser = HfUnifiedParser::new(template, &[], tokenizer);
parser.initialize(&[]).unwrap();
let steps: Vec<_> = chunks
.iter()
.map(|chunk| {
let mut output = UnifiedParserOutput::default();
parser.parse_into(DecodedText::unattributed(*chunk), &mut output).unwrap();
(*chunk, output.events)
})
.collect();
expect_test::expect![[r#"
[
(
"<|channel>thought\n",
[],
),
(
"Check ",
[
Reasoning(
DecodedText {
text: "Check",
attributions: [],
},
),
],
),
(
"the weather. ",
[
Reasoning(
DecodedText {
text: " the weather.",
attributions: [],
},
),
],
),
(
"<channel|>Sure",
[
Text(
"Sure",
),
],
),
(
". <|tool_call>call:get_weather",
[
Text(
".",
),
],
),
(
"{location:<|\"|>Paris<|\"|>}",
[
ToolCall(
ToolCallDelta {
tool_index: 0,
name: Some(
"get_weather",
),
arguments: "",
},
),
],
),
(
"<tool_call|>",
[
ToolCall(
ToolCallDelta {
tool_index: 0,
name: None,
arguments: "{\"location\":\"Paris\"}",
},
),
],
),
]
"#]]
.assert_debug_eq(&steps);
}
#[test]
fn invoke_protocol_discards_unrouted_text_and_joins_content() {
let text = "stray to=self<|msg|> plan <|pause|>to=user<|msg|>Hello, <|pause|>between<invoke name=\"get\"><param name=\"n\">x</param></invoke>to=note<|msg|>world<|end|>tail";
expect_test::expect![[r#"
Object {
"reasoning": String("plan"),
"content": String("Hello,world"),
"tool_calls": Array [
Object {
"name": String("get"),
"arguments": Object {
"n": String("x"),
},
},
],
}
"#]]
.assert_debug_eq(&parse(invoke_template(), text));
// Unicode word boundaries: `é` is a word character, so `éname=` is not the `name` attribute.
let text = "<invoke title=\"天气\" name=\"获取\"><param name=\"城市\">北京</param></invoke><invoke éname=\"x\">";
assert_eq!(
parse(invoke_template(), text),
json!({"tool_calls": [{"name": "获取", "arguments": {"城市": "北京"}}]})
);
}
#[test]
fn join_concatenates_repeated_matches() {
let template = json!({
"defaults": {"role": "assistant"},
"start_anchor": "<|assistant|>",
"fields": {
"thinking": {"open": "<think>", "close": "</think>", "repeats": true, "join": " "},
"content": {"repeats": true, "join": " "},
},
});
assert_eq!(
parse(
template.clone(),
"<think>first</think>middle<think>second</think>done"
),
json!({"reasoning": "first second", "content": "middle done"})
);
assert_eq!(
parse(template, "<think>only</think>"),
json!({"reasoning": "only"})
);
}
#[test]
fn transforms_with_dotted_paths() {
let template = json!({
"start_anchor": "<|assistant|>",
"fields": {
"tool_calls": {
"open": "<actions>",
"close": "</actions>",
"content": "json",
"transform_each": true,
"transform": {"type": "function", "function": {"name": "{fn.name}", "arguments": "{fn.args}"}},
},
},
});
assert_eq!(
parse(
template,
r#"<actions>[{"fn": {"name": "a", "args": {"x": 1}}}, {"fn": {"name": "b", "args": {}}}]</actions>"#
),
json!({"tool_calls": [{"name": "a", "arguments": {"x": 1}}, {"name": "b", "arguments": {}}]})
);
}
#[test]
fn literal_boundaries() {
let template = json!({
"start_anchor": "<|assistant|>",
"fields": {"content": {"open": ["<a>", "<bb>"], "close": ["</a>", "</bb>"]}},
});
for (opener, closer) in [("<a>", "</a>"), ("<bb>", "</bb>"), ("<a>", "</bb>")] {
assert_eq!(
parse(template.clone(), &format!("{opener}hi{closer}")),
json!({"content": "hi"})
);
}
// A literal that is a strict prefix of another in the same list defers at the buffer edge.
let template = compile(json!({
"start_anchor": "<|assistant|>",
"fields": {"content": {"open": "<x>", "close": ["END", "ENDX"]}},
}));
let events = parse_events(&template, &[], "", &["<x>hiEND", " more"]).unwrap();
assert_eq!(message(&events), json!({"content": "hi"}));
// A field without a close runs to the end of the stream.
let template =
json!({"start_anchor": "<|assistant|>", "fields": {"content": {"open": "<resp>"}}});
assert_eq!(
parse(template, "<resp>hello world"),
json!({"content": "hello world"})
);
}
#[test]
fn required_field_missing_fails_at_finish() {
let template = compile(json!({
"start_anchor": "<|assistant|>",
"fields": {"content": {"open": "<response>", "close": "</response>", "optional": false}},
}));
let error = parse_events(&template, &[], "", &["no response here"]).unwrap_err();
expect_test::expect![[r#"
ParsingFailed {
message: "Required response_template fields missing from parsed output: [\"content\"]",
}
"#]]
.assert_debug_eq(&error);
}
#[test]
fn prefix_sets_the_initial_state() {
let template = compile(qwen3_template());
// The prompt ends inside the thinking block; the prefilled newline is stripped.
let prompt = "<|im_start|>system\nYou are helpful<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n<think>\n";
let events = parse_events(&template, &[], prompt, &["Let me think...</think>"]).unwrap();
assert_eq!(message(&events), json!({"reasoning": "Let me think..."}));
// Only the slice after the last anchor matters.
let prompt = "<|im_start|>assistant\nEarlier reply<|im_end|>\n<|im_start|>user\nFollowup<|im_end|>\n<|im_start|>assistant\n<think>\n";
let events = parse_events(&template, &[], prompt, &["done</think>"]).unwrap();
assert_eq!(message(&events), json!({"reasoning": "done"}));
// A prefix ending mid-delimiter is completed by the generated text.
let events = parse_events(
&template,
&[],
"<|im_start|>assistant\n<thi",
&["nk>real body</think>"],
)
.unwrap();
assert_eq!(message(&events), json!({"reasoning": "real body"}));
// Gemma 4 continues inside the same turn after a tool response.
let template = compile(gemma4_template());
let prompt = "<|turn>model\n<|tool_call>call:f{}<tool_call|><|tool_response>response:f{}<tool_response|><|channel>thought\n";
let events = parse_events(&template, &[], prompt, &["Done.<channel|>It is sunny."]).unwrap();
assert_eq!(
message(&events),
json!({"reasoning": "Done.", "content": "It is sunny."})
);
// Prompt text is never emitted, but it counts for leading-whitespace stripping.
let events = parse_events(&template, &[], "<|turn>model\nSure,", &[" here", " it is"]).unwrap();
assert_eq!(message(&events), json!({"content": " here it is"}));
}
/// Diverges from Transformers, which runs the whole prompt through the parser
/// when the start anchor is absent.
#[test]
fn missing_start_anchor_starts_from_the_initial_state() {
let template = compile(smollm_template());
let events = parse_events(&template, &[], "<think>\n", &["hi</think>"]).unwrap();
assert_eq!(message(&events), json!({"content": "hi</think>"}));
}
#[test]
fn tools_convert_string_arguments() {
let template = compile(json!({
"start_anchor": "<|im_start|>assistant\n",
"fields": {
"tool_calls": {
"open_pattern": r"<tool_call>\s*<function=(?P<name>\w+)>",
"close": "</tool_call>",
"repeats": true,
"content": "xml-inline",
"content_args": {
"tag_pattern": r"<parameter=(?P<key>\w+)>\s*(?P<value>.*?)\s*</parameter>",
"merge_duplicates": true,
},
"transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},
},
},
}));
let tools = [Tool {
name: "set_alarm".to_string(),
description: None,
parameters: json!({
"type": "object",
"properties": {
"hour": {"type": "integer"},
"enabled": {"type": "boolean"},
"label": {"type": "string"},
"weekday": {"type": "integer"},
},
}),
strict: None,
}];
let text = "<tool_call>\n<function=set_alarm>\n<parameter=hour>\n7\n</parameter>\n<parameter=enabled>\ntrue\n</parameter>\n<parameter=label>\nwake up\n</parameter>\n<parameter=weekday>1</parameter><parameter=weekday>2</parameter>\n</function>\n</tool_call>";
assert_eq!(
message(&parse_events(&template, &tools, "", &[text]).unwrap()),
json!({"tool_calls": [{"name": "set_alarm", "arguments": {"hour": 7, "enabled": true, "label": "wake up", "weekday": [1, 2]}}]})
);
// Without tools, values stay strings.
assert_eq!(
message(&parse_events(&template, &[], "", &[text]).unwrap()),
json!({"tool_calls": [{"name": "set_alarm", "arguments": {"hour": "7", "enabled": "true", "label": "wake up", "weekday": ["1", "2"]}}]})
);
}
#[test]
fn stripped_reasoning_whitespace_keeps_its_tokens() {
let template = compile(gemma4_template());
let mut parser = HfUnifiedParser::new(template, &[], tokenizer());
parser.initialize(&[]).unwrap();
let text = "<|channel>thought\n \n plan \n<channel|>";
let mut output = UnifiedParserOutput::default();
for (index, c) in text.chars().enumerate() {
let piece = DecodedText {
text: c.to_string(),
attributions: [TokenAttribution {
token_id: u32::try_from(index).unwrap(),
anchor: TokenAnchor::Visible { byte_offset: 0 },
}]
.into_iter()
.collect(),
};
parser.parse_into(piece, &mut output).unwrap();
}
output.append(parser.finish().unwrap());
let (reasoning, tokens) =
output
.events
.iter()
.fold((String::new(), 0), |(text, tokens), event| match event {
UnifiedParserEvent::Reasoning(piece) => {
(text + &piece.text, tokens + piece.attributions.len())
}
_ => (text, tokens),
});
assert_eq!(reasoning, "plan");
// Every character between the markers is a reasoning token, stripped or not.
assert_eq!(tokens, " \n plan \n".chars().count());
}
#[test]
fn reset_returns_uncommitted_text() {
let template = compile(gemma4_template());
let mut parser = HfUnifiedParser::new(template, &[], tokenizer());
parser.initialize(&[]).unwrap();
let mut output = UnifiedParserOutput::default();
parser
.parse_into(
DecodedText::unattributed("Hi <|tool_call>call:f{a:<|\"|>x"),
&mut output,
)
.unwrap();
assert_eq!(parser.reset(), "<|tool_call>call:f{a:<|\"|>x");
}
#[test]
fn malformed_tool_arguments_fail_the_parser() {
let template = compile(gemma4_template());
let error =
parse_events(&template, &[], "", &["<|tool_call>call:f{a:}<tool_call|>"]).unwrap_err();
assert!(
matches!(error, UnifiedParserError::ParsingFailed { .. }),
"{error:?}"
);
}
#[test]
fn template_validation_errors() {
let error = |template: Value| ResponseTemplate::from_json(&template).unwrap_err();
expect_test::expect![[r#"
[
Invalid {
message: "unsupported response_template version: 2",
},
Invalid {
message: "At most one field may omit 'open'/'open_pattern' (that field becomes the implicit-open / leftover sink). Found: content, thinking",
},
Invalid {
message: "response_template must define 'start_anchor' or 'start_anchor_pattern'.",
},
Invalid {
message: "Field 'content': 'join' requires 'repeats': true",
},
Invalid {
message: "Field 'tool_calls': open_pattern/close_pattern declares named group(s) [\"name\"], but the field has no 'transform'. Named captures are only surfaced through a 'transform' template (where they appear alongside 'content'). Either add a 'transform' that uses the captures, or remove the named groups from the pattern.",
},
Invalid {
message: "Field 'tool_calls': transform string \"name: {content}\" mixes a {placeholder} with literal text. Use either a whole-string placeholder (e.g. \"{content}\") or a plain literal; string interpolation is not supported.",
},
Invalid {
message: "Field 'content': 'open' literals cannot be empty strings",
},
Unsupported {
message: "Field 'count': only 'content', 'reasoning_content'/'thinking', and 'tool_calls' fields can be reported",
},
Unsupported {
message: "Field 'thinking': text and reasoning fields must use the 'text' content parser without a transform",
},
Invalid {
message: "response_template.fields.content.content: unknown variant `yaml`, expected one of `text`, `int`, `float`, `bool`, `json`, `xml-inline`, `kv-lines`",
},
Invalid {
message: "response_template.fields.content.repeats: invalid type: string \"yes\", expected a boolean",
},
Invalid {
message: "Field 'content': cannot specify both 'open' and 'open_pattern'",
},
Invalid {
message: "Field 'tool_calls': transform_each is set but no transform was provided",
},
Unsupported {
message: "Field 'tool_calls': 'join' requires each match to parse to a string",
},
Invalid {
message: "Field 'tool_calls': transform placeholder '{id}' is neither 'content' nor a named group of open_pattern",
},
]
"#]]
.assert_debug_eq(&[
error(json!({"version": 2, "start_anchor": "a", "fields": {"content": {}}})),
error(json!({"start_anchor": "a", "fields": {"content": {}, "thinking": {}}})),
error(json!({"fields": {"content": {}}})),
error(json!({"start_anchor": "a", "fields": {"content": {"open": "<x>", "join": ""}}})),
error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open_pattern": r"<t (?P<name>\w+)>", "close": "</t>"}}})),
error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open": "<t>", "transform": {"label": "name: {content}"}}}})),
error(json!({"start_anchor": "a", "fields": {"content": {"open": [""]}}})),
error(json!({"start_anchor": "a", "fields": {"count": {"open": "<n>", "content": "int"}}})),
error(json!({"start_anchor": "a", "fields": {"thinking": {"open": "<n>", "content": "json"}}})),
error(json!({"start_anchor": "a", "fields": {"content": {"content": "yaml"}}})),
error(json!({"start_anchor": "a", "fields": {"content": {"repeats": "yes"}}})),
error(json!({"start_anchor": "a", "fields": {"content": {"open": "<x>", "open_pattern": "<y>"}}})),
error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open": "<t>", "transform_each": true}}})),
error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open": "<t>", "repeats": true, "join": ""}}})),
error(json!({
"start_anchor": "a",
"fields": {"tool_calls": {
"open_pattern": r"<t (?P<name>\w+)>",
"close_pattern": r"</t (?P<id>\d+)>",
"content": "json",
"transform": {"function": {"name": "{name}", "arguments": "{content}", "id": "{id}"}},
}},
})),
]);
}
/// Cases where the Rust parser intentionally differs from Transformers, keyed by
/// generated text.
const DIVERGENCES: &[(&str, &str)] = &[
(
// A field without `repeats` keeps only its last occurrence in Transformers;
// streamed text cannot be retracted, so every occurrence is reported.
r#"Before <|tool_call>call:f{x:<|"|>a, b<|"|>,y:[1,2.5,-3],z:{w:true}}<tool_call|> after"#,
r#"{"content": "Before after", "tool_calls": [{"name": "f", "arguments": {"x": "a, b", "y": [1, 2.5, -3], "z": {"w": true}}}]}"#,
),
(
// Arguments use the tool parsers' shared conversion, which keeps a JSON
// number's spelling; Transformers turns `1e3` into the integer 1000.
"<tool_call>\n<function=set_alarm>\nhour: seven\nratio: 1e3\nenabled: nope\n</tool_call>",
r#"{"tool_calls": [{"name": "set_alarm", "arguments": {"hour": "seven", "ratio": 1000.0, "enabled": "nope"}}]}"#,
),
];
/// Compare against `fixtures/differential.json`, generated from Transformers
/// `parse_response` by `fixtures/generate_differential.py`.
#[test]
fn differential_against_transformers() {
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("src/unified/hf/fixtures/differential.json");
let corpus: Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap();
let templates: BTreeMap<&str, Arc<ResponseTemplate>> = corpus["templates"]
.as_object()
.unwrap()
.iter()
.map(|(name, template)| {
(
name.as_str(),
Arc::new(ResponseTemplate::from_json(template).unwrap()),
)
})
.collect();
for case in corpus["cases"].as_array().unwrap() {
let template = &templates[case["template"].as_str().unwrap()];
let prefix = case["prefix"].as_str().unwrap();
let text = case["text"].as_str().unwrap();
let tools: Vec<Tool> = case["tools"]
.as_array()
.into_iter()
.flatten()
.map(|tool| Tool {
name: tool["function"]["name"].as_str().unwrap().to_string(),
description: None,
parameters: tool["function"]["parameters"].clone(),
strict: None,
})
.collect();
let expected = match DIVERGENCES.iter().find(|(divergent, _)| *divergent == text) {
Some((_, expected)) => serde_json::from_str(expected).unwrap(),
None => case["expected"].clone(),
};
for chunks in [vec![text], split_by_chars(text, 1)] {
let actual = parse_events(template, &tools, prefix, &chunks);
match (&actual, expected.get("error")) {
(Err(_), Some(_)) => {}
(Ok(events), None) => assert_eq!(message(events), expected, "{text:?}"),
_ => panic!("{text:?}: expected {expected}, got {actual:?}"),
}
}
}
}
#[test]
fn tool_call_value_shapes() {
// No transform: the parsed JSON is the tool call itself.
let template = compile(json!({
"start_anchor": "<|assistant|>",
"fields": {"tool_calls": {"open": "<tool_call>", "close": "</tool_call>", "content": "json"}},
}));
let parse =
|text: &str| parse_events(&template, &[], "", &[text]).map(|events| message(&events));
assert_eq!(
parse(r#"<tool_call>{"type": "function", "function": {"name": "f", "arguments": {"a": 1}}}</tool_call>"#).unwrap(),
json!({"tool_calls": [{"name": "f", "arguments": {"a": 1}}]})
);
assert_eq!(
parse(r#"<tool_call>[{"name": "f", "arguments": null}, {"name": "g", "arguments": "raw"}]</tool_call>"#).unwrap(),
json!({"tool_calls": [{"name": "f", "arguments": {}}, {"name": "g", "arguments": "raw"}]})
);
expect_test::expect![[r#"
ParsingFailed {
message: "tool call must be {\"function\": {\"name\": ..., \"arguments\": ...}} or {\"name\": ..., \"arguments\": ...} with a string name, got {\"name\":1}",
}
"#]]
.assert_debug_eq(&parse(r#"<tool_call>{"name": 1}</tool_call>"#).unwrap_err());
}
rust/src/parser/src/unified/hf/transform.rs
+263
View File
@@ -0,0 +1,263 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Transform templates: restructure a parsed region value into the output shape.
//!
//! Port of `_apply_transform`, `validate_transform_strings`, and `process_field`
//! from `content_parsers.py`.
use std::sync::LazyLock;
use regex_automata::meta::Regex;
use serde_json::{Map, Value};
use super::pattern::Captures;
use super::{Result, invalid};
use crate::unified::{self, parsing_failed};
/// `\{(\w+(?:\.\w+)*)\}`: a whole-string placeholder such as `{content}` or
/// `{content.args}`.
static PLACEHOLDER: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"\{\w+(?:\.\w+)*\}").expect("valid regex"));
/// A compiled transform template.
#[derive(Debug, Clone, PartialEq)]
pub(super) enum Transform {
Object(Vec<(String, Transform)>),
Array(Vec<Transform>),
/// A dotted placeholder path, e.g. `["content", "args"]`.
Placeholder(Vec<String>),
/// Any other JSON value, copied verbatim.
Literal(Value),
}
impl Transform {
/// Compile a transform template, rejecting any string that mixes a
/// `{name}` placeholder with literal text.
pub fn compile(scope: &str, template: &Value) -> Result<Self> {
Ok(match template {
Value::Object(object) => Self::Object(
object
.iter()
.map(|(key, value)| Ok((key.clone(), Self::compile(scope, value)?)))
.collect::<Result<_>>()?,
),
Value::Array(values) => Self::Array(
values.iter().map(|value| Self::compile(scope, value)).collect::<Result<_>>()?,
),
Value::String(string) => match PLACEHOLDER.find(string) {
Some(found) if found.range() == (0..string.len()) => Self::Placeholder(
string[1..string.len() - 1].split('.').map(str::to_string).collect(),
),
Some(_) => {
return Err(invalid!(
"{scope}: transform string {string:?} mixes a {{placeholder}} with literal text. \
Use either a whole-string placeholder (e.g. \"{{content}}\") or a plain literal; \
string interpolation is not supported."
));
}
None => Self::Literal(template.clone()),
},
other => Self::Literal(other.clone()),
})
}
/// Return the placeholder path found at `key_path` inside this template, if any.
pub fn placeholder_at(&self, key_path: &[&str]) -> Option<&[String]> {
match (self, key_path) {
(Self::Placeholder(path), []) => Some(path),
(Self::Object(entries), [key, rest @ ..]) => entries
.iter()
.find(|(entry, _)| entry == key)
.and_then(|(_, transform)| transform.placeholder_at(rest)),
_ => None,
}
}
/// Root names of every placeholder in this template.
pub fn placeholder_roots(&self) -> Box<dyn Iterator<Item = &str> + '_> {
match self {
Self::Object(entries) => {
Box::new(entries.iter().flat_map(|(_, transform)| transform.placeholder_roots()))
}
Self::Array(transforms) => {
Box::new(transforms.iter().flat_map(Self::placeholder_roots))
}
Self::Placeholder(path) => Box::new(path.first().map(String::as_str).into_iter()),
Self::Literal(_) => Box::new(std::iter::empty()),
}
}
/// Recursively instantiate this template against `scope`.
pub fn apply(&self, scope: &Map<String, Value>) -> unified::Result<Value> {
Ok(match self {
Self::Object(entries) => Value::Object(
entries
.iter()
.map(|(key, transform)| Ok((key.clone(), transform.apply(scope)?)))
.collect::<unified::Result<_>>()?,
),
Self::Array(transforms) => Value::Array(
transforms
.iter()
.map(|transform| transform.apply(scope))
.collect::<unified::Result<_>>()?,
),
Self::Literal(value) => value.clone(),
Self::Placeholder(path) => {
let dotted = path.join(".");
let (root, keys) = path.split_first().expect("placeholder path is non-empty");
let Some(mut value) = scope.get(root) else {
let available: Vec<_> = scope.keys().collect();
return Err(parsing_failed!(
"transform placeholder '{{{dotted}}}' is not defined. Available: {available:?}"
));
};
for key in keys {
let Some(object) = value.as_object() else {
return Err(parsing_failed!(
"transform placeholder '{{{dotted}}}' cannot index into {} at '{key}'",
json_type_name(value)
));
};
let Some(next) = object.get(key) else {
let available: Vec<_> = object.keys().collect();
return Err(parsing_failed!(
"transform placeholder '{{{dotted}}}' is missing key '{key}'. Available: {available:?}"
));
};
value = next;
}
value.clone()
}
})
}
}
/// A field's transform: applied to the parsed value, or to each of its elements.
#[derive(Debug, Clone, PartialEq)]
pub(super) struct FieldTransform {
pub template: Transform,
/// `transform_each`: the parsed content must be a list of dicts, and the
/// template is applied to each element (with the element's keys unpacked into
/// the template scope, alongside any regex captures).
pub each: bool,
}
impl FieldTransform {
/// Apply the transform to a parsed region value.
pub fn apply(
&self,
field_name: &str,
parsed: Value,
captures: &Captures,
) -> unified::Result<Value> {
if !self.each {
let mut scope = captures.clone();
scope.insert("content".to_string(), parsed);
return self.template.apply(&scope);
}
let Value::Array(items) = parsed else {
return Err(parsing_failed!(
"Field '{field_name}': transform_each requires the parsed content to be an array, got {}.",
json_type_name(&parsed)
));
};
items
.into_iter()
.map(|item| {
let Value::Object(item) = item else {
return Err(parsing_failed!(
"Field '{field_name}': transform_each requires each array element to be an object, got {}.",
json_type_name(&item)
));
};
let mut scope = captures.clone();
scope.extend(item);
self.template.apply(&scope)
})
.collect::<unified::Result<_>>()
.map(Value::Array)
}
}
/// JSON type name of a value, for error messages.
fn json_type_name(value: &Value) -> &'static str {
match value {
Value::Null => "null",
Value::Bool(_) => "boolean",
Value::Number(_) => "number",
Value::String(_) => "string",
Value::Array(_) => "array",
Value::Object(_) => "object",
}
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::*;
fn scope(value: Value) -> Map<String, Value> {
value.as_object().unwrap().clone()
}
#[test]
fn dotted_placeholders_descend_into_objects() {
let transform = Transform::compile(
"test",
&json!({"type": "function", "function": {"name": "{content.name}", "arguments": "{content.args}"}}),
)
.unwrap();
let value = transform
.apply(&scope(json!({"content": {"name": "f", "args": {"a": 1}}})))
.unwrap();
assert_eq!(
value,
json!({"type": "function", "function": {"name": "f", "arguments": {"a": 1}}})
);
assert_eq!(
transform.placeholder_at(&["function", "name"]),
Some(["content".to_string(), "name".to_string()].as_slice())
);
}
#[test]
fn placeholder_errors_match_transformers() {
let apply = |template: Value, scope_value: Value| {
Transform::compile("test", &template)
.unwrap()
.apply(&scope(scope_value))
.unwrap_err()
};
expect_test::expect![[r#"
ParsingFailed {
message: "transform placeholder '{name}' is not defined. Available: [\"content\"]",
}
"#]]
.assert_debug_eq(&apply(json!("{name}"), json!({"content": 1})));
expect_test::expect![[r#"
ParsingFailed {
message: "transform placeholder '{content.a}' cannot index into number at 'a'",
}
"#]]
.assert_debug_eq(&apply(json!("{content.a}"), json!({"content": 1})));
expect_test::expect![[r#"
ParsingFailed {
message: "transform placeholder '{content.b}' is missing key 'b'. Available: [\"a\"]",
}
"#]]
.assert_debug_eq(&apply(json!("{content.b}"), json!({"content": {"a": 1}})));
}
#[test]
fn interpolation_is_rejected_at_load() {
assert!(Transform::compile("test", &json!({"name": "fn_{name}"})).is_err());
assert_eq!(
Transform::compile("test", &json!("no placeholder")).unwrap(),
Transform::Literal(json!("no placeholder"))
);
}
}
rust/src/parser/src/unified/mod.rs
+9 -2
View File
@@ -5,12 +5,14 @@
mod combined;
mod gemma4;
mod hf;
mod hy;
mod inkling;
mod kimi_k3;
pub use combined::CombinedParser;
pub use gemma4::Gemma4UnifiedParser;
pub use hf::{HfTemplateError, HfUnifiedParser, ResponseTemplate};
pub use hy::{HyV3UnifiedParser, HyV4UnifiedParser};
pub use inkling::InklingUnifiedParser;
pub use kimi_k3::{KimiK3StructuralTagBuilder, KimiK3UnifiedParser};
@@ -271,8 +273,13 @@ pub trait UnifiedParser: Send {
#[derive(Debug, Error, Macro)]
#[thiserror_ext(macro(path = "crate::unified", mangle))]
pub enum UnifiedParserError {
#[error("combined parser is constructed from split parser instances")]
CombinedParserConstructor,
/// The parser is built from inputs other than tools and a tokenizer, so
/// [`UnifiedParser::create`] cannot construct it.
#[error("the `{parser}` unified parser is built from {built_from}, not by name")]
NoNamedConstructor {
parser: &'static str,
built_from: &'static str,
},
#[error("tokenizer is missing unified parser token `{token}`")]
MissingToken { token: String },
#[error("unified parser parsing failed: {message}")]
rust/src/text/src/backend/hf/config.rs
+3
View File
@@ -23,6 +23,9 @@ pub struct HfTokenizerConfig {
/// containing "Tiktoken" which can be used as a hint for backend
/// selection.
pub tokenizer_class: Option<String>,
/// The Hugging Face `response_template`: a declarative description of the
/// model's output protocol, executed by the `hf` unified parser.
pub response_template: Option<serde_json::Value>,
}
/// Hugging Face named special tokens may be serialized as a string or an