[Rust Frontend] Fix several hf chat template rendering issues (#44311)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -10,6 +10,7 @@ asynk-strim-attr.workspace = true
|
||||
easy-ext.workspace = true
|
||||
futures.workspace = true
|
||||
half.workspace = true
|
||||
indexmap.workspace = true
|
||||
itertools.workspace = true
|
||||
llm-multimodal.workspace = true
|
||||
minijinja.workspace = true
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use serde_json::Value as JsonValue;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tracing::{info, trace, warn};
|
||||
use vllm_text::Prompt;
|
||||
@@ -13,6 +13,7 @@ use self::format::{
|
||||
ChatTemplateContentFormat, ChatTemplateContentFormatOption as ContentFormatOption,
|
||||
};
|
||||
use self::template::{CompiledChatTemplate, TemplateContext};
|
||||
use self::value::{TemplateValue, to_template_value};
|
||||
use super::{ChatRenderer, RenderedPrompt};
|
||||
use crate::error::Result;
|
||||
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
|
||||
@@ -24,6 +25,7 @@ mod error;
|
||||
mod format;
|
||||
mod template;
|
||||
mod tojson;
|
||||
mod value;
|
||||
|
||||
pub use template::{load_chat_template, resolve_chat_template};
|
||||
|
||||
@@ -38,7 +40,7 @@ pub struct MultimodalRenderInfo {
|
||||
/// state.
|
||||
pub struct HfChatRenderer {
|
||||
default_template: Option<CompiledChatTemplate>,
|
||||
default_template_kwargs: HashMap<String, Value>,
|
||||
default_template_kwargs: HashMap<String, JsonValue>,
|
||||
content_format: ContentFormatOption,
|
||||
special_tokens: Option<HfSpecialTokens>,
|
||||
multimodal: Option<MultimodalRenderInfo>,
|
||||
@@ -48,7 +50,7 @@ impl HfChatRenderer {
|
||||
/// Create a renderer from the given template string.
|
||||
pub fn new(
|
||||
template: Option<String>,
|
||||
default_template_kwargs: HashMap<String, Value>,
|
||||
default_template_kwargs: HashMap<String, JsonValue>,
|
||||
content_format: ContentFormatOption,
|
||||
) -> Result<Self> {
|
||||
Ok(Self {
|
||||
@@ -245,7 +247,7 @@ struct TemplateToolCall {
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TemplateToolFunction {
|
||||
name: String,
|
||||
arguments: Value,
|
||||
arguments: TemplateValue,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
@@ -259,7 +261,7 @@ pub(super) struct TemplateTool {
|
||||
struct TemplateToolDefinition {
|
||||
name: String,
|
||||
description: Option<String>,
|
||||
parameters: Value,
|
||||
parameters: TemplateValue,
|
||||
strict: Option<bool>,
|
||||
}
|
||||
|
||||
@@ -345,13 +347,14 @@ fn to_template_tool_calls(
|
||||
let mut tool_calls = Vec::new();
|
||||
|
||||
for tool_call in content.tool_calls() {
|
||||
let arguments = serde_json::from_str::<Value>(&tool_call.arguments).map_err(|error| {
|
||||
let arguments = serde_json::from_str(&tool_call.arguments).map_err(|error| {
|
||||
Error::ChatTemplate(format!(
|
||||
"assistant tool call `{}` has invalid JSON arguments: {}",
|
||||
tool_call.id,
|
||||
error.as_report()
|
||||
))
|
||||
})?;
|
||||
let arguments = to_template_value(arguments);
|
||||
|
||||
tool_calls.push(TemplateToolCall {
|
||||
id: tool_call.id.clone(),
|
||||
@@ -434,7 +437,7 @@ fn to_template_tools(tools: &[ChatTool]) -> Vec<TemplateTool> {
|
||||
function: TemplateToolDefinition {
|
||||
name: tool.name.clone(),
|
||||
description: tool.description.clone(),
|
||||
parameters: tool.parameters.clone(),
|
||||
parameters: to_template_value(tool.parameters.clone()),
|
||||
strict: tool.strict,
|
||||
},
|
||||
})
|
||||
@@ -909,6 +912,29 @@ mod tests {
|
||||
assert_eq!(rendered, "get_weather|Paris|call_1|Sunny");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_tool_call_argument_items_method_is_not_shadowed_by_field() {
|
||||
let request = sample_request(vec![ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::ToolCall(crate::AssistantToolCall {
|
||||
id: "call_1".to_string(),
|
||||
name: "add".to_string(),
|
||||
arguments: r#"{"items":"operands","x":2,"y":1.0}"#.to_string(),
|
||||
}),
|
||||
])]);
|
||||
|
||||
let rendered = render(
|
||||
Some(
|
||||
"{%- set arguments = messages[0].tool_calls[0].function.arguments -%}
|
||||
{%- for key, value in arguments.items() -%}{{ key }}={{ value }};{%- endfor -%}
|
||||
|{{ arguments['items'] }}",
|
||||
),
|
||||
&request,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(rendered, "items=operands;x=2;y=1.0;|operands");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen35_template_renders_prefilled_reasoning_start_when_thinking_enabled() {
|
||||
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
|
||||
@@ -208,11 +208,27 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tojson_preserves_arbitrary_precision_number_spelling() {
|
||||
fn tojson_uses_standard_serde_json_number_spelling() {
|
||||
let payload = serde_json::from_str(r#"{"x":2,"y":1.00}"#).unwrap();
|
||||
let rendered = render("{{ payload|tojson }}", payload);
|
||||
|
||||
assert_eq!(rendered, "{\"x\": 2, \"y\": 1.00}");
|
||||
// TODO: we cannot preserve the original number precision by enabling `serde_json`'s
|
||||
// `arbitrary_precision` feature, otherwise the following test
|
||||
// `serialized_json_numbers_do_not_leak_serde_private_representation` will fail.
|
||||
// See issue: https://github.com/mitsuhiko/minijinja/issues/641
|
||||
assert_eq!(rendered, "{\"x\": 2, \"y\": 1.0}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serialized_json_numbers_do_not_leak_serde_private_representation() {
|
||||
let payload: serde_json::Value = serde_json::from_str(r#"{"x":2,"y":1.00}"#).unwrap();
|
||||
let rendered = render("{{ payload }}", payload);
|
||||
|
||||
// TODO: we cannot preserve the original number precision by enabling `serde_json`'s
|
||||
// `arbitrary_precision` feature, otherwise this will fail.
|
||||
// See issue: https://github.com/mitsuhiko/minijinja/issues/641
|
||||
assert!(!rendered.contains("$serde_json::private::Number"));
|
||||
assert_eq!(rendered, r#"{"x": 2, "y": 1.0}"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use indexmap::IndexMap;
|
||||
use minijinja::value::{Enumerator, Object, ObjectExt, ObjectRepr};
|
||||
use minijinja::{Error as TemplateError, ErrorKind as TemplateErrorKind, State};
|
||||
use serde::Serialize;
|
||||
use serde_json::Value as JsonValue;
|
||||
|
||||
/// A wrapper around `minijinja::Value` that can be constructed with `to_template_value` and used
|
||||
/// as a value in the chat template.
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub(super) struct TemplateValue(minijinja::Value);
|
||||
|
||||
pub(super) fn to_template_value(value: JsonValue) -> TemplateValue {
|
||||
TemplateValue(match value {
|
||||
JsonValue::Array(values) => values
|
||||
.into_iter()
|
||||
.map(to_template_value)
|
||||
.map(|value| value.0)
|
||||
.collect::<minijinja::Value>(),
|
||||
JsonValue::Object(values) => minijinja::Value::from_object(TemplateMap(
|
||||
values
|
||||
.into_iter()
|
||||
.map(|(key, value)| (key, to_template_value(value).0))
|
||||
.collect(),
|
||||
)),
|
||||
// For primitive values, directly convert them to `minijinja::Value` using `from_serialize`.
|
||||
value => minijinja::Value::from_serialize(value),
|
||||
})
|
||||
}
|
||||
|
||||
/// A custom map type that always returns `UnknownMethod` for method calls, so that pycompat can
|
||||
/// always handle dict methods through the unknown-method callback.
|
||||
///
|
||||
/// Use `IndexMap` to preserve the original key order when iterating.
|
||||
///
|
||||
/// MiniJinja's default map can resolve a same-named field before Python dict methods. HF templates
|
||||
/// commonly call `dict.items()`, which would fail if the map had an `items` field.
|
||||
/// See issue: https://github.com/mitsuhiko/minijinja/issues/903
|
||||
#[derive(Debug)]
|
||||
struct TemplateMap(IndexMap<String, minijinja::Value>);
|
||||
|
||||
impl Object for TemplateMap {
|
||||
fn repr(self: &Arc<Self>) -> ObjectRepr {
|
||||
ObjectRepr::Map
|
||||
}
|
||||
|
||||
fn get_value(self: &Arc<Self>, key: &minijinja::Value) -> Option<minijinja::Value> {
|
||||
self.0.get(key.as_str()?).cloned()
|
||||
}
|
||||
|
||||
fn get_value_by_str(self: &Arc<Self>, key: &str) -> Option<minijinja::Value> {
|
||||
self.0.get(key).cloned()
|
||||
}
|
||||
|
||||
fn enumerate(self: &Arc<Self>) -> Enumerator {
|
||||
self.mapped_rev_enumerator(|this| {
|
||||
Box::new(this.0.keys().map(|key| minijinja::Value::from(key.as_str())))
|
||||
})
|
||||
}
|
||||
|
||||
fn enumerator_len(self: &Arc<Self>) -> Option<usize> {
|
||||
Some(self.0.len())
|
||||
}
|
||||
|
||||
fn call_method(
|
||||
self: &Arc<Self>,
|
||||
_state: &State<'_, '_>,
|
||||
_method: &str,
|
||||
_args: &[minijinja::Value],
|
||||
) -> std::result::Result<minijinja::Value, TemplateError> {
|
||||
// Always return `UnknownMethod` for method calls,
|
||||
// so that pycompat can handle dict methods through the unknown-method callback.
|
||||
Err(TemplateError::from(TemplateErrorKind::UnknownMethod))
|
||||
}
|
||||
}
|
||||
@@ -233,7 +233,7 @@ async fn run_roundtrip_tool_call_mix(case: RoundtripCase) -> Result<()> {
|
||||
"roundtrip-reasoning-tools",
|
||||
vec![ChatMessage::text(
|
||||
ChatRole::User,
|
||||
"Check Shanghai weather and add 1.00 plus 2.",
|
||||
"Check Shanghai weather and add 1.0 plus 2.",
|
||||
)],
|
||||
test_tools(),
|
||||
Some(true), // always enable thinking in this fixture
|
||||
@@ -261,9 +261,10 @@ async fn run_roundtrip_tool_call_mix(case: RoundtripCase) -> Result<()> {
|
||||
AssistantContentBlock::ToolCall(AssistantToolCall {
|
||||
id: "functions.add:1".to_string(),
|
||||
name: "add".to_string(),
|
||||
// Intentionally use a non-lexical order of keys and a different number
|
||||
// formatting style to verify text-level fidelity of the roundtrip.
|
||||
arguments: r#"{"y":1.00,"x":2}"#.to_string(),
|
||||
// Intentionally use a non-lexical order of keys to verify text-level
|
||||
// fidelity of the roundtrip where JSON formatting remains stable. The
|
||||
// `items` key also exercises templates that call `arguments.items()`.
|
||||
arguments: r#"{"y":1.0,"x":2,"items":["left","right"]}"#.to_string(),
|
||||
}),
|
||||
],
|
||||
},
|
||||
@@ -291,7 +292,7 @@ async fn run_roundtrip_tool_call_mix(case: RoundtripCase) -> Result<()> {
|
||||
assert_eq!(tool_calls[1].name, "add");
|
||||
assert_eq!(
|
||||
tool_calls[1].arguments,
|
||||
expected_arguments(&case, r#"{"y": 1.00, "x": 2}"#)?,
|
||||
expected_arguments(&case, r#"{"y": 1.0, "x": 2, "items": ["left", "right"]}"#)?,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
@@ -585,9 +586,13 @@ fn test_tools() -> Vec<ChatTool> {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"y": { "type": "number" },
|
||||
"x": { "type": "number" }
|
||||
"x": { "type": "number" },
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"required": ["y", "x"]
|
||||
"required": ["y", "x", "items"]
|
||||
}),
|
||||
strict: None,
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user