[Rust Frontend] Accept custom chat roles for HF templates (#58311)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-24 13:43:34 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent a5603f21ae
commit a1b6763df6
14 changed files with 215 additions and 20 deletions
+4 -1
View File
@@ -21,6 +21,8 @@ pub enum Error {
InvalidReasoningEffort(String),
#[error("{message}")]
InvalidReasoningControl { message: String },
#[error("chat role `{role}` is not supported by this chat renderer")]
UnsupportedChatRole { role: String },
#[error("multimodal input is not supported by this chat renderer")]
UnsupportedMultimodalRenderer,
#[error("unsupported multimodal content: {0}")]
@@ -114,7 +116,8 @@ impl Error {
| Self::InvalidReasoningControl { .. }
| Self::DuplicateToolName { .. }
| Self::ToolChoiceRequiresTools
| Self::ToolChoiceFunctionNotFound { .. } => true,
| Self::ToolChoiceFunctionNotFound { .. }
| Self::UnsupportedChatRole { .. } => true,
Self::Text(error) => error.is_request_validation_error(),
Self::UnsupportedMultimodalRenderer
| Self::UnsupportedMultimodalContent(_)
+4 -2
View File
@@ -599,7 +599,8 @@ fn extract_media_parts(
ChatMessage::System { content }
| ChatMessage::Developer { content, .. }
| ChatMessage::User { content }
| ChatMessage::ToolResponse { content, .. } => content,
| ChatMessage::ToolResponse { content, .. }
| ChatMessage::Custom { content, .. } => content,
ChatMessage::Assistant { .. } => {
bail_multimodal!("renderer reported multimodal assistant content")
}
@@ -622,7 +623,8 @@ fn extract_media_parts(
ChatMessage::System { content }
| ChatMessage::Developer { content, .. }
| ChatMessage::User { content }
| ChatMessage::ToolResponse { content, .. } => Some(content),
| ChatMessage::ToolResponse { content, .. }
| ChatMessage::Custom { content, .. } => Some(content),
ChatMessage::Assistant { .. } => None,
};
match content {
+3
View File
@@ -227,6 +227,9 @@ pub(super) fn render_request_with_media_order(
);
}
}
ChatMessage::Custom { role, .. } => {
return Err(Error::UnsupportedChatRole { role: role.clone() });
}
}
if is_user_like_entry(message, current_render_index as usize, dialect)
@@ -158,6 +158,7 @@ fn render_message(
thinking_mode == ThinkingMode::Thinking && after_or_at_last_user_turn,
content,
),
ChatMessage::Custom { role, .. } => Err(Error::UnsupportedChatRole { role: role.clone() }),
}
}
@@ -5,6 +5,7 @@ use std::path::PathBuf;
use expect_test::{ExpectFile, expect, expect_file};
use serde_json::Value;
use thiserror_ext::AsReport as _;
use super::DeepSeekV4ChatRenderer;
use crate::ChatRenderer;
@@ -764,3 +765,20 @@ fn last_developer_omits_generation_prompt_when_disabled() {
expect!["<|begin▁of▁sentence|><|User|>latest instruction"].assert_eq(&rendered);
}
#[test]
fn custom_roles_are_rejected_as_request_errors() {
let request = ChatRequest {
messages: vec![
ChatMessage::custom("root", "identity"),
ChatMessage::user("hello"),
],
..ChatRequest::for_test()
};
let error = DeepSeekV4ChatRenderer::default().render(&request).unwrap_err();
assert!(error.is_request_validation_error());
expect!["chat role `root` is not supported by this chat renderer"]
.assert_eq(&error.to_report_string());
}
+5 -1
View File
@@ -188,7 +188,8 @@ fn peel_leading_instructions(
)),
ChatMessage::User { .. }
| ChatMessage::Assistant { .. }
| ChatMessage::ToolResponse { .. } => Ok((None, None, messages)),
| ChatMessage::ToolResponse { .. }
| ChatMessage::Custom { .. } => Ok((None, None, messages)),
}
}
@@ -328,6 +329,9 @@ fn to_harmony_message(
.with_recipient("assistant"),
]
}
ChatMessage::Custom { role, .. } => {
return Err(Error::UnsupportedChatRole { role: role.clone() });
}
})
}
+35 -9
View File
@@ -258,8 +258,8 @@ impl ChatRenderer for HfChatRenderer {
// TODO: borrow more fields directly from the original `ChatMessage`.
#[serde_with::skip_serializing_none]
#[derive(Debug, Serialize)]
struct TemplateMessage {
role: &'static str,
struct TemplateMessage<'a> {
role: &'a str,
content: TemplateContent,
// Developer-role messages may provide message-local tools in the same shape
// as top-level request tools.
@@ -322,22 +322,22 @@ struct TemplateToolDefinition {
}
/// Convert chat messages into the JSON shape expected by Jinja chat templates.
fn to_template_messages(
messages: &[ChatMessage],
fn to_template_messages<'a>(
messages: &'a [ChatMessage],
content_format: ChatTemplateContentFormat,
multimodal: Option<&MultimodalRenderInfo>,
) -> Result<Vec<TemplateMessage>> {
) -> Result<Vec<TemplateMessage<'a>>> {
messages
.iter()
.map(|message| to_template_message(message, content_format, multimodal))
.collect()
}
fn to_template_message(
message: &ChatMessage,
fn to_template_message<'a>(
message: &'a ChatMessage,
content_format: ChatTemplateContentFormat,
multimodal: Option<&MultimodalRenderInfo>,
) -> Result<TemplateMessage> {
) -> Result<TemplateMessage<'a>> {
Ok(match message {
ChatMessage::System { content } => TemplateMessage {
role: "system",
@@ -394,6 +394,15 @@ fn to_template_message(
tool_calls: None,
tool_call_id: Some(tool_call_id.clone()),
},
ChatMessage::Custom { role, content } => TemplateMessage {
role,
content: to_template_content(content, content_format, multimodal)?,
tools: None,
reasoning: None,
reasoning_content: None,
tool_calls: None,
tool_call_id: None,
},
})
}
@@ -524,7 +533,7 @@ const CONTINUE_FINAL_MESSAGE_TAG: &str = "CONTINUE_FINAL_MESSAGE_TAG ";
// TODO: transformers v5 also allows continuing a non-`content` field (e.g.
// `reasoning_content`) by passing a field name; only the boolean form is
// supported here.
fn append_continue_final_message_tag(message: &mut TemplateMessage) -> Result<String> {
fn append_continue_final_message_tag(message: &mut TemplateMessage<'_>) -> Result<String> {
let text = match &mut message.content {
TemplateContent::String(text) => Some(text),
// Pick the last text part in the message.
@@ -1001,6 +1010,23 @@ mod tests {
assert_eq!(rendered, "developer|policy|get_weather|city");
}
#[test]
fn chat_template_passes_custom_roles_unchanged() {
let request = sample_request(vec![
ChatMessage::custom("root", "identity"),
ChatMessage::system("policy"),
ChatMessage::user("hello"),
]);
let rendered = render(
Some("{% for m in messages %}{{ m.role }}={{ m.content }};{% endfor %}"),
&request,
)
.unwrap();
assert_eq!(rendered, "root=identity;system=policy;user=hello;");
}
#[test]
fn chat_template_keeps_string_text_for_openai_detected_templates() {
let request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
+1 -1
View File
@@ -45,7 +45,7 @@ fn build_environment(template: String) -> Result<Environment<'static>> {
#[serde_with::skip_serializing_none]
#[derive(Default, Serialize)]
pub(super) struct TemplateContext<'a> {
pub(super) messages: &'a [TemplateMessage],
pub(super) messages: &'a [TemplateMessage<'a>],
pub(super) add_generation_prompt: bool,
pub(super) continue_final_message: bool,
pub(super) tools: Option<&'a [TemplateTool]>,
@@ -345,6 +345,9 @@ impl ChatRenderer for InklingChatRenderer {
&tool_call_id_to_name,
)?;
}
ChatMessage::Custom { role, .. } => {
return Err(Error::UnsupportedChatRole { role: role.clone() });
}
}
}
@@ -205,6 +205,9 @@ pub(super) fn render_request_with_media_order(
write_assistant_message(&mut out, content, thinking)?;
}
ChatMessage::ToolResponse { .. } => unreachable!("handled above"),
ChatMessage::Custom { role, .. } => {
return Err(Error::UnsupportedChatRole { role: role.clone() });
}
}
}
flush_tool_run(&mut out, &mut pending_tool_run, &tool_call_id_index)?;
+25 -4
View File
@@ -24,6 +24,9 @@ pub enum ChatRole {
User,
Assistant,
ToolResponse,
/// Role outside the standard set, passed to chat templates unchanged.
#[serde(untagged)]
Custom(String),
}
/// One text-only chat content part in OpenAI-style block format.
@@ -212,6 +215,10 @@ pub enum ChatMessage {
content: ChatContent,
tool_call_id: String,
},
/// Message content under a role outside the standard set, passed to chat
/// templates unchanged.
#[serde(untagged)]
Custom { role: String, content: ChatContent },
}
impl ChatMessage {
@@ -230,6 +237,7 @@ impl ChatMessage {
use ChatMessage::tool_response() instead"
)
}
ChatRole::Custom(role) => Self::custom(role, content),
}
}
@@ -286,6 +294,14 @@ impl ChatMessage {
}
}
/// Construct one chat message with a role outside the standard set.
pub fn custom(role: impl Into<String>, content: impl Into<ChatContent>) -> Self {
Self::Custom {
role: role.into(),
content: content.into(),
}
}
/// Return the chat role of this message.
pub fn role(&self) -> ChatRole {
match self {
@@ -294,6 +310,7 @@ impl ChatMessage {
Self::User { .. } => ChatRole::User,
Self::Assistant { .. } => ChatRole::Assistant,
Self::ToolResponse { .. } => ChatRole::ToolResponse,
Self::Custom { role, .. } => ChatRole::Custom(role.clone()),
}
}
@@ -303,7 +320,8 @@ impl ChatMessage {
Self::System { content }
| Self::Developer { content, .. }
| Self::User { content }
| Self::ToolResponse { content, .. } => content.try_flatten_to_text(),
| Self::ToolResponse { content, .. }
| Self::Custom { content, .. } => content.try_flatten_to_text(),
Self::Assistant { content } => Ok(content.text()),
}
}
@@ -315,7 +333,8 @@ impl ChatMessage {
Self::System { .. }
| Self::Developer { .. }
| Self::User { .. }
| Self::ToolResponse { .. } => None,
| Self::ToolResponse { .. }
| Self::Custom { .. } => None,
}
}
@@ -325,7 +344,8 @@ impl ChatMessage {
Self::System { content }
| Self::Developer { content, .. }
| Self::User { content }
| Self::ToolResponse { content, .. } => content.has_multimodal(),
| Self::ToolResponse { content, .. }
| Self::Custom { content, .. } => content.has_multimodal(),
Self::Assistant { .. } => false,
}
}
@@ -673,13 +693,14 @@ impl ChatRequest {
impl ChatRole {
/// Return the chat-template role string used by the current text-only chat
/// backend.
pub fn as_str(&self) -> &'static str {
pub fn as_str(&self) -> &str {
match self {
Self::System => "system",
Self::Developer => "developer",
Self::User => "user",
Self::Assistant => "assistant",
Self::ToolResponse => "tool_response",
Self::Custom(role) => role,
}
}
}
@@ -317,6 +317,9 @@ pub(crate) fn convert_message(message: ChatMessage) -> Result<VllmChatMessage, A
convert_content(content)?,
convert_message_tools(tools)?,
)),
ChatMessage::Custom { role, content } => {
Ok(VllmChatMessage::custom(role, convert_content(content)?))
}
}
}
@@ -442,6 +445,7 @@ mod tests {
use expect_test::expect;
use llm_multimodal::ImageDetail;
use serde_json::json;
use thiserror_ext::AsReport as _;
use validator::Validate;
use vllm_chat::{
AssistantContentBlock, AssistantToolCall, ChatContentPart, ChatMessage as VllmChatMessage,
@@ -1047,6 +1051,43 @@ mod tests {
);
}
#[test]
fn prepare_chat_request_accepts_custom_role_messages() {
let request: ChatCompletionRequest = serde_json::from_value(json!({
"messages": [
{"role": "root", "content": "Custom identity."},
{"role": "user", "content": "hello"},
],
}))
.expect("parse custom role message");
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.messages,
vec![
VllmChatMessage::custom("root", "Custom identity."),
VllmChatMessage::user("hello"),
]
);
}
#[test]
fn chat_http_message_keeps_standard_role_errors() {
let error = serde_json::from_value::<ChatMessage>(json!({
"role": "tool",
"content": "Sunny",
}))
.unwrap_err();
expect!["missing field `tool_call_id`"].assert_eq(&error.to_report_string());
}
#[test]
fn prepare_chat_request_maps_image_url_content_parts() {
let request = ChatCompletionRequest {
@@ -410,9 +410,17 @@ impl ToolReference {
// Chat Messages
// ============================================================================
/// One chat message, tagged by `role`.
///
/// The derived serde impls use `remote = "Self"` and cover only the standard
/// roles; the trait impls below dispatch on the role tag and handle
/// [`ChatMessage::Custom`] separately. A `#[serde(untagged)]` fallback variant
/// would also accept malformed standard messages, such as a `tool` message
/// without `tool_call_id`, and replace their field errors with serde's generic
/// untagged-enum error.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(tag = "role")]
#[serde(tag = "role", remote = "Self")]
pub enum ChatMessage {
#[serde(rename = "system")]
System {
@@ -447,6 +455,67 @@ pub enum ChatMessage {
tools: Option<Vec<Tool>>,
name: Option<String>,
},
/// Message with a role outside the OpenAI set, such as `root`. Chat
/// templates receive the role string unchanged.
#[serde(skip)]
Custom {
role: String,
content: MessageContent,
},
}
/// Role tags of the standard [`ChatMessage`] variants.
const STANDARD_CHAT_ROLES: &[&str] = &[
"system",
"user",
"assistant",
"tool",
"function",
"developer",
];
/// Wire shape of [`ChatMessage::Custom`].
#[derive(Deserialize, Serialize)]
struct CustomChatMessage<R, C> {
role: R,
content: C,
}
impl Serialize for ChatMessage {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match self {
Self::Custom { role, content } => {
CustomChatMessage { role, content }.serialize(serializer)
}
_ => Self::serialize(self, serializer),
}
}
}
impl<'de> Deserialize<'de> for ChatMessage {
/// Dispatch on the role tag: standard roles keep their derived errors,
/// and any other string role becomes [`ChatMessage::Custom`].
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let value = Value::deserialize(deserializer)?;
let is_custom = value
.get("role")
.and_then(Value::as_str)
.is_some_and(|role| !STANDARD_CHAT_ROLES.contains(&role));
if is_custom {
let CustomChatMessage { role, content } =
CustomChatMessage::deserialize(value).map_err(serde::de::Error::custom)?;
Ok(Self::Custom { role, content })
} else {
Self::deserialize(value).map_err(serde::de::Error::custom)
}
}
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
+2 -1
View File
@@ -555,7 +555,8 @@ fn render_fake_message_content(
ChatMessage::System { content }
| ChatMessage::Developer { content, .. }
| ChatMessage::User { content }
| ChatMessage::ToolResponse { content, .. } => render_fake_content(content, placeholder),
| ChatMessage::ToolResponse { content, .. }
| ChatMessage::Custom { content, .. } => render_fake_content(content, placeholder),
ChatMessage::Assistant { .. } => message.text_content(),
}
}