From a1b6763df65981474ba5c4dc27e16566eddb41e3 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 24 Sep 2026 13:43:34 +0800 Subject: [PATCH] [Rust Frontend] Accept custom chat roles for HF templates (#58311) Co-authored-by: Claude Opus 5.5 Signed-off-by: Bugen Zhao --- rust/src/chat/src/error.rs | 5 +- rust/src/chat/src/multimodal.rs | 6 +- rust/src/chat/src/renderer/deepseek.rs | 3 + .../src/renderer/deepseek_v32/encoding.rs | 1 + .../chat/src/renderer/deepseek_v4/tests.rs | 18 +++++ rust/src/chat/src/renderer/harmony/mod.rs | 6 +- rust/src/chat/src/renderer/hf/mod.rs | 44 +++++++++--- rust/src/chat/src/renderer/hf/template.rs | 2 +- rust/src/chat/src/renderer/inkling/mod.rs | 3 + .../src/chat/src/renderer/kimi_k3/encoding.rs | 3 + rust/src/chat/src/request.rs | 29 ++++++-- .../routes/openai/chat_completions/convert.rs | 41 +++++++++++ .../server/src/routes/openai/utils/types.rs | 71 ++++++++++++++++++- rust/src/server/src/routes/tests.rs | 3 +- 14 files changed, 215 insertions(+), 20 deletions(-) diff --git a/rust/src/chat/src/error.rs b/rust/src/chat/src/error.rs index 52e26370af..7d124aaaf0 100644 --- a/rust/src/chat/src/error.rs +++ b/rust/src/chat/src/error.rs @@ -21,6 +21,8 @@ pub enum Error { InvalidReasoningEffort(String), #[error("{message}")] InvalidReasoningControl { message: String }, + #[error("chat role `{role}` is not supported by this chat renderer")] + UnsupportedChatRole { role: String }, #[error("multimodal input is not supported by this chat renderer")] UnsupportedMultimodalRenderer, #[error("unsupported multimodal content: {0}")] @@ -114,7 +116,8 @@ impl Error { | Self::InvalidReasoningControl { .. } | Self::DuplicateToolName { .. } | Self::ToolChoiceRequiresTools - | Self::ToolChoiceFunctionNotFound { .. } => true, + | Self::ToolChoiceFunctionNotFound { .. } + | Self::UnsupportedChatRole { .. } => true, Self::Text(error) => error.is_request_validation_error(), Self::UnsupportedMultimodalRenderer | Self::UnsupportedMultimodalContent(_) diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index 4af49aa5c6..55acfea93b 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -599,7 +599,8 @@ fn extract_media_parts( ChatMessage::System { content } | ChatMessage::Developer { content, .. } | ChatMessage::User { content } - | ChatMessage::ToolResponse { content, .. } => content, + | ChatMessage::ToolResponse { content, .. } + | ChatMessage::Custom { content, .. } => content, ChatMessage::Assistant { .. } => { bail_multimodal!("renderer reported multimodal assistant content") } @@ -622,7 +623,8 @@ fn extract_media_parts( ChatMessage::System { content } | ChatMessage::Developer { content, .. } | ChatMessage::User { content } - | ChatMessage::ToolResponse { content, .. } => Some(content), + | ChatMessage::ToolResponse { content, .. } + | ChatMessage::Custom { content, .. } => Some(content), ChatMessage::Assistant { .. } => None, }; match content { diff --git a/rust/src/chat/src/renderer/deepseek.rs b/rust/src/chat/src/renderer/deepseek.rs index 7892fd2497..2683c37ec3 100644 --- a/rust/src/chat/src/renderer/deepseek.rs +++ b/rust/src/chat/src/renderer/deepseek.rs @@ -227,6 +227,9 @@ pub(super) fn render_request_with_media_order( ); } } + ChatMessage::Custom { role, .. } => { + return Err(Error::UnsupportedChatRole { role: role.clone() }); + } } if is_user_like_entry(message, current_render_index as usize, dialect) diff --git a/rust/src/chat/src/renderer/deepseek_v32/encoding.rs b/rust/src/chat/src/renderer/deepseek_v32/encoding.rs index 74c7e4cbf2..33e6eb99d8 100644 --- a/rust/src/chat/src/renderer/deepseek_v32/encoding.rs +++ b/rust/src/chat/src/renderer/deepseek_v32/encoding.rs @@ -158,6 +158,7 @@ fn render_message( thinking_mode == ThinkingMode::Thinking && after_or_at_last_user_turn, content, ), + ChatMessage::Custom { role, .. } => Err(Error::UnsupportedChatRole { role: role.clone() }), } } diff --git a/rust/src/chat/src/renderer/deepseek_v4/tests.rs b/rust/src/chat/src/renderer/deepseek_v4/tests.rs index 21a33ec2d5..8702e8d73c 100644 --- a/rust/src/chat/src/renderer/deepseek_v4/tests.rs +++ b/rust/src/chat/src/renderer/deepseek_v4/tests.rs @@ -5,6 +5,7 @@ use std::path::PathBuf; use expect_test::{ExpectFile, expect, expect_file}; use serde_json::Value; +use thiserror_ext::AsReport as _; use super::DeepSeekV4ChatRenderer; use crate::ChatRenderer; @@ -764,3 +765,20 @@ fn last_developer_omits_generation_prompt_when_disabled() { expect!["<|begin▁of▁sentence|><|User|>latest instruction"].assert_eq(&rendered); } + +#[test] +fn custom_roles_are_rejected_as_request_errors() { + let request = ChatRequest { + messages: vec![ + ChatMessage::custom("root", "identity"), + ChatMessage::user("hello"), + ], + ..ChatRequest::for_test() + }; + + let error = DeepSeekV4ChatRenderer::default().render(&request).unwrap_err(); + + assert!(error.is_request_validation_error()); + expect!["chat role `root` is not supported by this chat renderer"] + .assert_eq(&error.to_report_string()); +} diff --git a/rust/src/chat/src/renderer/harmony/mod.rs b/rust/src/chat/src/renderer/harmony/mod.rs index 83b59e9198..d8dafadca7 100644 --- a/rust/src/chat/src/renderer/harmony/mod.rs +++ b/rust/src/chat/src/renderer/harmony/mod.rs @@ -188,7 +188,8 @@ fn peel_leading_instructions( )), ChatMessage::User { .. } | ChatMessage::Assistant { .. } - | ChatMessage::ToolResponse { .. } => Ok((None, None, messages)), + | ChatMessage::ToolResponse { .. } + | ChatMessage::Custom { .. } => Ok((None, None, messages)), } } @@ -328,6 +329,9 @@ fn to_harmony_message( .with_recipient("assistant"), ] } + ChatMessage::Custom { role, .. } => { + return Err(Error::UnsupportedChatRole { role: role.clone() }); + } }) } diff --git a/rust/src/chat/src/renderer/hf/mod.rs b/rust/src/chat/src/renderer/hf/mod.rs index 911a73928a..f406e32da6 100644 --- a/rust/src/chat/src/renderer/hf/mod.rs +++ b/rust/src/chat/src/renderer/hf/mod.rs @@ -258,8 +258,8 @@ impl ChatRenderer for HfChatRenderer { // TODO: borrow more fields directly from the original `ChatMessage`. #[serde_with::skip_serializing_none] #[derive(Debug, Serialize)] -struct TemplateMessage { - role: &'static str, +struct TemplateMessage<'a> { + role: &'a str, content: TemplateContent, // Developer-role messages may provide message-local tools in the same shape // as top-level request tools. @@ -322,22 +322,22 @@ struct TemplateToolDefinition { } /// Convert chat messages into the JSON shape expected by Jinja chat templates. -fn to_template_messages( - messages: &[ChatMessage], +fn to_template_messages<'a>( + messages: &'a [ChatMessage], content_format: ChatTemplateContentFormat, multimodal: Option<&MultimodalRenderInfo>, -) -> Result> { +) -> Result>> { messages .iter() .map(|message| to_template_message(message, content_format, multimodal)) .collect() } -fn to_template_message( - message: &ChatMessage, +fn to_template_message<'a>( + message: &'a ChatMessage, content_format: ChatTemplateContentFormat, multimodal: Option<&MultimodalRenderInfo>, -) -> Result { +) -> Result> { Ok(match message { ChatMessage::System { content } => TemplateMessage { role: "system", @@ -394,6 +394,15 @@ fn to_template_message( tool_calls: None, tool_call_id: Some(tool_call_id.clone()), }, + ChatMessage::Custom { role, content } => TemplateMessage { + role, + content: to_template_content(content, content_format, multimodal)?, + tools: None, + reasoning: None, + reasoning_content: None, + tool_calls: None, + tool_call_id: None, + }, }) } @@ -524,7 +533,7 @@ const CONTINUE_FINAL_MESSAGE_TAG: &str = "CONTINUE_FINAL_MESSAGE_TAG "; // TODO: transformers v5 also allows continuing a non-`content` field (e.g. // `reasoning_content`) by passing a field name; only the boolean form is // supported here. -fn append_continue_final_message_tag(message: &mut TemplateMessage) -> Result { +fn append_continue_final_message_tag(message: &mut TemplateMessage<'_>) -> Result { let text = match &mut message.content { TemplateContent::String(text) => Some(text), // Pick the last text part in the message. @@ -1001,6 +1010,23 @@ mod tests { assert_eq!(rendered, "developer|policy|get_weather|city"); } + #[test] + fn chat_template_passes_custom_roles_unchanged() { + let request = sample_request(vec![ + ChatMessage::custom("root", "identity"), + ChatMessage::system("policy"), + ChatMessage::user("hello"), + ]); + + let rendered = render( + Some("{% for m in messages %}{{ m.role }}={{ m.content }};{% endfor %}"), + &request, + ) + .unwrap(); + + assert_eq!(rendered, "root=identity;system=policy;user=hello;"); + } + #[test] fn chat_template_keeps_string_text_for_openai_detected_templates() { let request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]); diff --git a/rust/src/chat/src/renderer/hf/template.rs b/rust/src/chat/src/renderer/hf/template.rs index b890783701..de447e2c34 100644 --- a/rust/src/chat/src/renderer/hf/template.rs +++ b/rust/src/chat/src/renderer/hf/template.rs @@ -45,7 +45,7 @@ fn build_environment(template: String) -> Result> { #[serde_with::skip_serializing_none] #[derive(Default, Serialize)] pub(super) struct TemplateContext<'a> { - pub(super) messages: &'a [TemplateMessage], + pub(super) messages: &'a [TemplateMessage<'a>], pub(super) add_generation_prompt: bool, pub(super) continue_final_message: bool, pub(super) tools: Option<&'a [TemplateTool]>, diff --git a/rust/src/chat/src/renderer/inkling/mod.rs b/rust/src/chat/src/renderer/inkling/mod.rs index 9feee371aa..f792f70d32 100644 --- a/rust/src/chat/src/renderer/inkling/mod.rs +++ b/rust/src/chat/src/renderer/inkling/mod.rs @@ -345,6 +345,9 @@ impl ChatRenderer for InklingChatRenderer { &tool_call_id_to_name, )?; } + ChatMessage::Custom { role, .. } => { + return Err(Error::UnsupportedChatRole { role: role.clone() }); + } } } diff --git a/rust/src/chat/src/renderer/kimi_k3/encoding.rs b/rust/src/chat/src/renderer/kimi_k3/encoding.rs index b434d72c53..2424782947 100644 --- a/rust/src/chat/src/renderer/kimi_k3/encoding.rs +++ b/rust/src/chat/src/renderer/kimi_k3/encoding.rs @@ -205,6 +205,9 @@ pub(super) fn render_request_with_media_order( write_assistant_message(&mut out, content, thinking)?; } ChatMessage::ToolResponse { .. } => unreachable!("handled above"), + ChatMessage::Custom { role, .. } => { + return Err(Error::UnsupportedChatRole { role: role.clone() }); + } } } flush_tool_run(&mut out, &mut pending_tool_run, &tool_call_id_index)?; diff --git a/rust/src/chat/src/request.rs b/rust/src/chat/src/request.rs index 59e7ec6fba..f7ddd9b128 100644 --- a/rust/src/chat/src/request.rs +++ b/rust/src/chat/src/request.rs @@ -24,6 +24,9 @@ pub enum ChatRole { User, Assistant, ToolResponse, + /// Role outside the standard set, passed to chat templates unchanged. + #[serde(untagged)] + Custom(String), } /// One text-only chat content part in OpenAI-style block format. @@ -212,6 +215,10 @@ pub enum ChatMessage { content: ChatContent, tool_call_id: String, }, + /// Message content under a role outside the standard set, passed to chat + /// templates unchanged. + #[serde(untagged)] + Custom { role: String, content: ChatContent }, } impl ChatMessage { @@ -230,6 +237,7 @@ impl ChatMessage { use ChatMessage::tool_response() instead" ) } + ChatRole::Custom(role) => Self::custom(role, content), } } @@ -286,6 +294,14 @@ impl ChatMessage { } } + /// Construct one chat message with a role outside the standard set. + pub fn custom(role: impl Into, content: impl Into) -> Self { + Self::Custom { + role: role.into(), + content: content.into(), + } + } + /// Return the chat role of this message. pub fn role(&self) -> ChatRole { match self { @@ -294,6 +310,7 @@ impl ChatMessage { Self::User { .. } => ChatRole::User, Self::Assistant { .. } => ChatRole::Assistant, Self::ToolResponse { .. } => ChatRole::ToolResponse, + Self::Custom { role, .. } => ChatRole::Custom(role.clone()), } } @@ -303,7 +320,8 @@ impl ChatMessage { Self::System { content } | Self::Developer { content, .. } | Self::User { content } - | Self::ToolResponse { content, .. } => content.try_flatten_to_text(), + | Self::ToolResponse { content, .. } + | Self::Custom { content, .. } => content.try_flatten_to_text(), Self::Assistant { content } => Ok(content.text()), } } @@ -315,7 +333,8 @@ impl ChatMessage { Self::System { .. } | Self::Developer { .. } | Self::User { .. } - | Self::ToolResponse { .. } => None, + | Self::ToolResponse { .. } + | Self::Custom { .. } => None, } } @@ -325,7 +344,8 @@ impl ChatMessage { Self::System { content } | Self::Developer { content, .. } | Self::User { content } - | Self::ToolResponse { content, .. } => content.has_multimodal(), + | Self::ToolResponse { content, .. } + | Self::Custom { content, .. } => content.has_multimodal(), Self::Assistant { .. } => false, } } @@ -673,13 +693,14 @@ impl ChatRequest { impl ChatRole { /// Return the chat-template role string used by the current text-only chat /// backend. - pub fn as_str(&self) -> &'static str { + pub fn as_str(&self) -> &str { match self { Self::System => "system", Self::Developer => "developer", Self::User => "user", Self::Assistant => "assistant", Self::ToolResponse => "tool_response", + Self::Custom(role) => role, } } } diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 1815856636..d4df614270 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -317,6 +317,9 @@ pub(crate) fn convert_message(message: ChatMessage) -> Result { + Ok(VllmChatMessage::custom(role, convert_content(content)?)) + } } } @@ -442,6 +445,7 @@ mod tests { use expect_test::expect; use llm_multimodal::ImageDetail; use serde_json::json; + use thiserror_ext::AsReport as _; use validator::Validate; use vllm_chat::{ AssistantContentBlock, AssistantToolCall, ChatContentPart, ChatMessage as VllmChatMessage, @@ -1047,6 +1051,43 @@ mod tests { ); } + #[test] + fn prepare_chat_request_accepts_custom_role_messages() { + let request: ChatCompletionRequest = serde_json::from_value(json!({ + "messages": [ + {"role": "root", "content": "Custom identity."}, + {"role": "user", "content": "hello"}, + ], + })) + .expect("parse custom role message"); + + let prepared = prepare_chat_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("request is valid"); + + assert_eq!( + prepared.chat_request.messages, + vec![ + VllmChatMessage::custom("root", "Custom identity."), + VllmChatMessage::user("hello"), + ] + ); + } + + #[test] + fn chat_http_message_keeps_standard_role_errors() { + let error = serde_json::from_value::(json!({ + "role": "tool", + "content": "Sunny", + })) + .unwrap_err(); + + expect!["missing field `tool_call_id`"].assert_eq(&error.to_report_string()); + } + #[test] fn prepare_chat_request_maps_image_url_content_parts() { let request = ChatCompletionRequest { diff --git a/rust/src/server/src/routes/openai/utils/types.rs b/rust/src/server/src/routes/openai/utils/types.rs index 5554b5d366..85a891341c 100644 --- a/rust/src/server/src/routes/openai/utils/types.rs +++ b/rust/src/server/src/routes/openai/utils/types.rs @@ -410,9 +410,17 @@ impl ToolReference { // Chat Messages // ============================================================================ +/// One chat message, tagged by `role`. +/// +/// The derived serde impls use `remote = "Self"` and cover only the standard +/// roles; the trait impls below dispatch on the role tag and handle +/// [`ChatMessage::Custom`] separately. A `#[serde(untagged)]` fallback variant +/// would also accept malformed standard messages, such as a `tool` message +/// without `tool_call_id`, and replace their field errors with serde's generic +/// untagged-enum error. #[serde_with::skip_serializing_none] #[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(tag = "role")] +#[serde(tag = "role", remote = "Self")] pub enum ChatMessage { #[serde(rename = "system")] System { @@ -447,6 +455,67 @@ pub enum ChatMessage { tools: Option>, name: Option, }, + /// Message with a role outside the OpenAI set, such as `root`. Chat + /// templates receive the role string unchanged. + #[serde(skip)] + Custom { + role: String, + content: MessageContent, + }, +} + +/// Role tags of the standard [`ChatMessage`] variants. +const STANDARD_CHAT_ROLES: &[&str] = &[ + "system", + "user", + "assistant", + "tool", + "function", + "developer", +]; + +/// Wire shape of [`ChatMessage::Custom`]. +#[derive(Deserialize, Serialize)] +struct CustomChatMessage { + role: R, + content: C, +} + +impl Serialize for ChatMessage { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + Self::Custom { role, content } => { + CustomChatMessage { role, content }.serialize(serializer) + } + _ => Self::serialize(self, serializer), + } + } +} + +impl<'de> Deserialize<'de> for ChatMessage { + /// Dispatch on the role tag: standard roles keep their derived errors, + /// and any other string role becomes [`ChatMessage::Custom`]. + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = Value::deserialize(deserializer)?; + let is_custom = value + .get("role") + .and_then(Value::as_str) + .is_some_and(|role| !STANDARD_CHAT_ROLES.contains(&role)); + + if is_custom { + let CustomChatMessage { role, content } = + CustomChatMessage::deserialize(value).map_err(serde::de::Error::custom)?; + Ok(Self::Custom { role, content }) + } else { + Self::deserialize(value).map_err(serde::de::Error::custom) + } + } } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 89ce35ce68..5c1fff2cae 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -555,7 +555,8 @@ fn render_fake_message_content( ChatMessage::System { content } | ChatMessage::Developer { content, .. } | ChatMessage::User { content } - | ChatMessage::ToolResponse { content, .. } => render_fake_content(content, placeholder), + | ChatMessage::ToolResponse { content, .. } + | ChatMessage::Custom { content, .. } => render_fake_content(content, placeholder), ChatMessage::Assistant { .. } => message.text_content(), } }