[Rust Frontend] Accept custom chat roles for HF templates (#58311)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a5603f21ae
commit
a1b6763df6
@@ -21,6 +21,8 @@ pub enum Error {
|
||||
InvalidReasoningEffort(String),
|
||||
#[error("{message}")]
|
||||
InvalidReasoningControl { message: String },
|
||||
#[error("chat role `{role}` is not supported by this chat renderer")]
|
||||
UnsupportedChatRole { role: String },
|
||||
#[error("multimodal input is not supported by this chat renderer")]
|
||||
UnsupportedMultimodalRenderer,
|
||||
#[error("unsupported multimodal content: {0}")]
|
||||
@@ -114,7 +116,8 @@ impl Error {
|
||||
| Self::InvalidReasoningControl { .. }
|
||||
| Self::DuplicateToolName { .. }
|
||||
| Self::ToolChoiceRequiresTools
|
||||
| Self::ToolChoiceFunctionNotFound { .. } => true,
|
||||
| Self::ToolChoiceFunctionNotFound { .. }
|
||||
| Self::UnsupportedChatRole { .. } => true,
|
||||
Self::Text(error) => error.is_request_validation_error(),
|
||||
Self::UnsupportedMultimodalRenderer
|
||||
| Self::UnsupportedMultimodalContent(_)
|
||||
|
||||
@@ -599,7 +599,8 @@ fn extract_media_parts(
|
||||
ChatMessage::System { content }
|
||||
| ChatMessage::Developer { content, .. }
|
||||
| ChatMessage::User { content }
|
||||
| ChatMessage::ToolResponse { content, .. } => content,
|
||||
| ChatMessage::ToolResponse { content, .. }
|
||||
| ChatMessage::Custom { content, .. } => content,
|
||||
ChatMessage::Assistant { .. } => {
|
||||
bail_multimodal!("renderer reported multimodal assistant content")
|
||||
}
|
||||
@@ -622,7 +623,8 @@ fn extract_media_parts(
|
||||
ChatMessage::System { content }
|
||||
| ChatMessage::Developer { content, .. }
|
||||
| ChatMessage::User { content }
|
||||
| ChatMessage::ToolResponse { content, .. } => Some(content),
|
||||
| ChatMessage::ToolResponse { content, .. }
|
||||
| ChatMessage::Custom { content, .. } => Some(content),
|
||||
ChatMessage::Assistant { .. } => None,
|
||||
};
|
||||
match content {
|
||||
|
||||
@@ -227,6 +227,9 @@ pub(super) fn render_request_with_media_order(
|
||||
);
|
||||
}
|
||||
}
|
||||
ChatMessage::Custom { role, .. } => {
|
||||
return Err(Error::UnsupportedChatRole { role: role.clone() });
|
||||
}
|
||||
}
|
||||
|
||||
if is_user_like_entry(message, current_render_index as usize, dialect)
|
||||
|
||||
@@ -158,6 +158,7 @@ fn render_message(
|
||||
thinking_mode == ThinkingMode::Thinking && after_or_at_last_user_turn,
|
||||
content,
|
||||
),
|
||||
ChatMessage::Custom { role, .. } => Err(Error::UnsupportedChatRole { role: role.clone() }),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ use std::path::PathBuf;
|
||||
|
||||
use expect_test::{ExpectFile, expect, expect_file};
|
||||
use serde_json::Value;
|
||||
use thiserror_ext::AsReport as _;
|
||||
|
||||
use super::DeepSeekV4ChatRenderer;
|
||||
use crate::ChatRenderer;
|
||||
@@ -764,3 +765,20 @@ fn last_developer_omits_generation_prompt_when_disabled() {
|
||||
|
||||
expect!["<|begin▁of▁sentence|><|User|>latest instruction"].assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_roles_are_rejected_as_request_errors() {
|
||||
let request = ChatRequest {
|
||||
messages: vec![
|
||||
ChatMessage::custom("root", "identity"),
|
||||
ChatMessage::user("hello"),
|
||||
],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
let error = DeepSeekV4ChatRenderer::default().render(&request).unwrap_err();
|
||||
|
||||
assert!(error.is_request_validation_error());
|
||||
expect!["chat role `root` is not supported by this chat renderer"]
|
||||
.assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
@@ -188,7 +188,8 @@ fn peel_leading_instructions(
|
||||
)),
|
||||
ChatMessage::User { .. }
|
||||
| ChatMessage::Assistant { .. }
|
||||
| ChatMessage::ToolResponse { .. } => Ok((None, None, messages)),
|
||||
| ChatMessage::ToolResponse { .. }
|
||||
| ChatMessage::Custom { .. } => Ok((None, None, messages)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -328,6 +329,9 @@ fn to_harmony_message(
|
||||
.with_recipient("assistant"),
|
||||
]
|
||||
}
|
||||
ChatMessage::Custom { role, .. } => {
|
||||
return Err(Error::UnsupportedChatRole { role: role.clone() });
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -258,8 +258,8 @@ impl ChatRenderer for HfChatRenderer {
|
||||
// TODO: borrow more fields directly from the original `ChatMessage`.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TemplateMessage {
|
||||
role: &'static str,
|
||||
struct TemplateMessage<'a> {
|
||||
role: &'a str,
|
||||
content: TemplateContent,
|
||||
// Developer-role messages may provide message-local tools in the same shape
|
||||
// as top-level request tools.
|
||||
@@ -322,22 +322,22 @@ struct TemplateToolDefinition {
|
||||
}
|
||||
|
||||
/// Convert chat messages into the JSON shape expected by Jinja chat templates.
|
||||
fn to_template_messages(
|
||||
messages: &[ChatMessage],
|
||||
fn to_template_messages<'a>(
|
||||
messages: &'a [ChatMessage],
|
||||
content_format: ChatTemplateContentFormat,
|
||||
multimodal: Option<&MultimodalRenderInfo>,
|
||||
) -> Result<Vec<TemplateMessage>> {
|
||||
) -> Result<Vec<TemplateMessage<'a>>> {
|
||||
messages
|
||||
.iter()
|
||||
.map(|message| to_template_message(message, content_format, multimodal))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn to_template_message(
|
||||
message: &ChatMessage,
|
||||
fn to_template_message<'a>(
|
||||
message: &'a ChatMessage,
|
||||
content_format: ChatTemplateContentFormat,
|
||||
multimodal: Option<&MultimodalRenderInfo>,
|
||||
) -> Result<TemplateMessage> {
|
||||
) -> Result<TemplateMessage<'a>> {
|
||||
Ok(match message {
|
||||
ChatMessage::System { content } => TemplateMessage {
|
||||
role: "system",
|
||||
@@ -394,6 +394,15 @@ fn to_template_message(
|
||||
tool_calls: None,
|
||||
tool_call_id: Some(tool_call_id.clone()),
|
||||
},
|
||||
ChatMessage::Custom { role, content } => TemplateMessage {
|
||||
role,
|
||||
content: to_template_content(content, content_format, multimodal)?,
|
||||
tools: None,
|
||||
reasoning: None,
|
||||
reasoning_content: None,
|
||||
tool_calls: None,
|
||||
tool_call_id: None,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -524,7 +533,7 @@ const CONTINUE_FINAL_MESSAGE_TAG: &str = "CONTINUE_FINAL_MESSAGE_TAG ";
|
||||
// TODO: transformers v5 also allows continuing a non-`content` field (e.g.
|
||||
// `reasoning_content`) by passing a field name; only the boolean form is
|
||||
// supported here.
|
||||
fn append_continue_final_message_tag(message: &mut TemplateMessage) -> Result<String> {
|
||||
fn append_continue_final_message_tag(message: &mut TemplateMessage<'_>) -> Result<String> {
|
||||
let text = match &mut message.content {
|
||||
TemplateContent::String(text) => Some(text),
|
||||
// Pick the last text part in the message.
|
||||
@@ -1001,6 +1010,23 @@ mod tests {
|
||||
assert_eq!(rendered, "developer|policy|get_weather|city");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_passes_custom_roles_unchanged() {
|
||||
let request = sample_request(vec![
|
||||
ChatMessage::custom("root", "identity"),
|
||||
ChatMessage::system("policy"),
|
||||
ChatMessage::user("hello"),
|
||||
]);
|
||||
|
||||
let rendered = render(
|
||||
Some("{% for m in messages %}{{ m.role }}={{ m.content }};{% endfor %}"),
|
||||
&request,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(rendered, "root=identity;system=policy;user=hello;");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_keeps_string_text_for_openai_detected_templates() {
|
||||
let request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
|
||||
@@ -45,7 +45,7 @@ fn build_environment(template: String) -> Result<Environment<'static>> {
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Default, Serialize)]
|
||||
pub(super) struct TemplateContext<'a> {
|
||||
pub(super) messages: &'a [TemplateMessage],
|
||||
pub(super) messages: &'a [TemplateMessage<'a>],
|
||||
pub(super) add_generation_prompt: bool,
|
||||
pub(super) continue_final_message: bool,
|
||||
pub(super) tools: Option<&'a [TemplateTool]>,
|
||||
|
||||
@@ -345,6 +345,9 @@ impl ChatRenderer for InklingChatRenderer {
|
||||
&tool_call_id_to_name,
|
||||
)?;
|
||||
}
|
||||
ChatMessage::Custom { role, .. } => {
|
||||
return Err(Error::UnsupportedChatRole { role: role.clone() });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -205,6 +205,9 @@ pub(super) fn render_request_with_media_order(
|
||||
write_assistant_message(&mut out, content, thinking)?;
|
||||
}
|
||||
ChatMessage::ToolResponse { .. } => unreachable!("handled above"),
|
||||
ChatMessage::Custom { role, .. } => {
|
||||
return Err(Error::UnsupportedChatRole { role: role.clone() });
|
||||
}
|
||||
}
|
||||
}
|
||||
flush_tool_run(&mut out, &mut pending_tool_run, &tool_call_id_index)?;
|
||||
|
||||
@@ -24,6 +24,9 @@ pub enum ChatRole {
|
||||
User,
|
||||
Assistant,
|
||||
ToolResponse,
|
||||
/// Role outside the standard set, passed to chat templates unchanged.
|
||||
#[serde(untagged)]
|
||||
Custom(String),
|
||||
}
|
||||
|
||||
/// One text-only chat content part in OpenAI-style block format.
|
||||
@@ -212,6 +215,10 @@ pub enum ChatMessage {
|
||||
content: ChatContent,
|
||||
tool_call_id: String,
|
||||
},
|
||||
/// Message content under a role outside the standard set, passed to chat
|
||||
/// templates unchanged.
|
||||
#[serde(untagged)]
|
||||
Custom { role: String, content: ChatContent },
|
||||
}
|
||||
|
||||
impl ChatMessage {
|
||||
@@ -230,6 +237,7 @@ impl ChatMessage {
|
||||
use ChatMessage::tool_response() instead"
|
||||
)
|
||||
}
|
||||
ChatRole::Custom(role) => Self::custom(role, content),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -286,6 +294,14 @@ impl ChatMessage {
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct one chat message with a role outside the standard set.
|
||||
pub fn custom(role: impl Into<String>, content: impl Into<ChatContent>) -> Self {
|
||||
Self::Custom {
|
||||
role: role.into(),
|
||||
content: content.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the chat role of this message.
|
||||
pub fn role(&self) -> ChatRole {
|
||||
match self {
|
||||
@@ -294,6 +310,7 @@ impl ChatMessage {
|
||||
Self::User { .. } => ChatRole::User,
|
||||
Self::Assistant { .. } => ChatRole::Assistant,
|
||||
Self::ToolResponse { .. } => ChatRole::ToolResponse,
|
||||
Self::Custom { role, .. } => ChatRole::Custom(role.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -303,7 +320,8 @@ impl ChatMessage {
|
||||
Self::System { content }
|
||||
| Self::Developer { content, .. }
|
||||
| Self::User { content }
|
||||
| Self::ToolResponse { content, .. } => content.try_flatten_to_text(),
|
||||
| Self::ToolResponse { content, .. }
|
||||
| Self::Custom { content, .. } => content.try_flatten_to_text(),
|
||||
Self::Assistant { content } => Ok(content.text()),
|
||||
}
|
||||
}
|
||||
@@ -315,7 +333,8 @@ impl ChatMessage {
|
||||
Self::System { .. }
|
||||
| Self::Developer { .. }
|
||||
| Self::User { .. }
|
||||
| Self::ToolResponse { .. } => None,
|
||||
| Self::ToolResponse { .. }
|
||||
| Self::Custom { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -325,7 +344,8 @@ impl ChatMessage {
|
||||
Self::System { content }
|
||||
| Self::Developer { content, .. }
|
||||
| Self::User { content }
|
||||
| Self::ToolResponse { content, .. } => content.has_multimodal(),
|
||||
| Self::ToolResponse { content, .. }
|
||||
| Self::Custom { content, .. } => content.has_multimodal(),
|
||||
Self::Assistant { .. } => false,
|
||||
}
|
||||
}
|
||||
@@ -673,13 +693,14 @@ impl ChatRequest {
|
||||
impl ChatRole {
|
||||
/// Return the chat-template role string used by the current text-only chat
|
||||
/// backend.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
pub fn as_str(&self) -> &str {
|
||||
match self {
|
||||
Self::System => "system",
|
||||
Self::Developer => "developer",
|
||||
Self::User => "user",
|
||||
Self::Assistant => "assistant",
|
||||
Self::ToolResponse => "tool_response",
|
||||
Self::Custom(role) => role,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -317,6 +317,9 @@ pub(crate) fn convert_message(message: ChatMessage) -> Result<VllmChatMessage, A
|
||||
convert_content(content)?,
|
||||
convert_message_tools(tools)?,
|
||||
)),
|
||||
ChatMessage::Custom { role, content } => {
|
||||
Ok(VllmChatMessage::custom(role, convert_content(content)?))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -442,6 +445,7 @@ mod tests {
|
||||
use expect_test::expect;
|
||||
use llm_multimodal::ImageDetail;
|
||||
use serde_json::json;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use validator::Validate;
|
||||
use vllm_chat::{
|
||||
AssistantContentBlock, AssistantToolCall, ChatContentPart, ChatMessage as VllmChatMessage,
|
||||
@@ -1047,6 +1051,43 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_accepts_custom_role_messages() {
|
||||
let request: ChatCompletionRequest = serde_json::from_value(json!({
|
||||
"messages": [
|
||||
{"role": "root", "content": "Custom identity."},
|
||||
{"role": "user", "content": "hello"},
|
||||
],
|
||||
}))
|
||||
.expect("parse custom role message");
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(
|
||||
prepared.chat_request.messages,
|
||||
vec![
|
||||
VllmChatMessage::custom("root", "Custom identity."),
|
||||
VllmChatMessage::user("hello"),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_http_message_keeps_standard_role_errors() {
|
||||
let error = serde_json::from_value::<ChatMessage>(json!({
|
||||
"role": "tool",
|
||||
"content": "Sunny",
|
||||
}))
|
||||
.unwrap_err();
|
||||
|
||||
expect!["missing field `tool_call_id`"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_maps_image_url_content_parts() {
|
||||
let request = ChatCompletionRequest {
|
||||
|
||||
@@ -410,9 +410,17 @@ impl ToolReference {
|
||||
// Chat Messages
|
||||
// ============================================================================
|
||||
|
||||
/// One chat message, tagged by `role`.
|
||||
///
|
||||
/// The derived serde impls use `remote = "Self"` and cover only the standard
|
||||
/// roles; the trait impls below dispatch on the role tag and handle
|
||||
/// [`ChatMessage::Custom`] separately. A `#[serde(untagged)]` fallback variant
|
||||
/// would also accept malformed standard messages, such as a `tool` message
|
||||
/// without `tool_call_id`, and replace their field errors with serde's generic
|
||||
/// untagged-enum error.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "role")]
|
||||
#[serde(tag = "role", remote = "Self")]
|
||||
pub enum ChatMessage {
|
||||
#[serde(rename = "system")]
|
||||
System {
|
||||
@@ -447,6 +455,67 @@ pub enum ChatMessage {
|
||||
tools: Option<Vec<Tool>>,
|
||||
name: Option<String>,
|
||||
},
|
||||
/// Message with a role outside the OpenAI set, such as `root`. Chat
|
||||
/// templates receive the role string unchanged.
|
||||
#[serde(skip)]
|
||||
Custom {
|
||||
role: String,
|
||||
content: MessageContent,
|
||||
},
|
||||
}
|
||||
|
||||
/// Role tags of the standard [`ChatMessage`] variants.
|
||||
const STANDARD_CHAT_ROLES: &[&str] = &[
|
||||
"system",
|
||||
"user",
|
||||
"assistant",
|
||||
"tool",
|
||||
"function",
|
||||
"developer",
|
||||
];
|
||||
|
||||
/// Wire shape of [`ChatMessage::Custom`].
|
||||
#[derive(Deserialize, Serialize)]
|
||||
struct CustomChatMessage<R, C> {
|
||||
role: R,
|
||||
content: C,
|
||||
}
|
||||
|
||||
impl Serialize for ChatMessage {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
match self {
|
||||
Self::Custom { role, content } => {
|
||||
CustomChatMessage { role, content }.serialize(serializer)
|
||||
}
|
||||
_ => Self::serialize(self, serializer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for ChatMessage {
|
||||
/// Dispatch on the role tag: standard roles keep their derived errors,
|
||||
/// and any other string role becomes [`ChatMessage::Custom`].
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let value = Value::deserialize(deserializer)?;
|
||||
let is_custom = value
|
||||
.get("role")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|role| !STANDARD_CHAT_ROLES.contains(&role));
|
||||
|
||||
if is_custom {
|
||||
let CustomChatMessage { role, content } =
|
||||
CustomChatMessage::deserialize(value).map_err(serde::de::Error::custom)?;
|
||||
Ok(Self::Custom { role, content })
|
||||
} else {
|
||||
Self::deserialize(value).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
|
||||
@@ -555,7 +555,8 @@ fn render_fake_message_content(
|
||||
ChatMessage::System { content }
|
||||
| ChatMessage::Developer { content, .. }
|
||||
| ChatMessage::User { content }
|
||||
| ChatMessage::ToolResponse { content, .. } => render_fake_content(content, placeholder),
|
||||
| ChatMessage::ToolResponse { content, .. }
|
||||
| ChatMessage::Custom { content, .. } => render_fake_content(content, placeholder),
|
||||
ChatMessage::Assistant { .. } => message.text_content(),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user