[Rust Frontend] Integrate MM video support (#47959)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -290,7 +290,10 @@ fn convert_content(content: MessageContent) -> Result<ChatContent, ApiError> {
|
||||
detail: image_url.detail,
|
||||
uuid,
|
||||
}),
|
||||
_ => bail_invalid_request!("Only text and image_url content parts are supported."),
|
||||
ContentPart::VideoUrl { video_url, uuid } => Ok(ChatContentPart::VideoUrl {
|
||||
video_url: video_url.url,
|
||||
uuid,
|
||||
}),
|
||||
})
|
||||
.try_collect()
|
||||
.map(ChatContent::Parts),
|
||||
@@ -772,28 +775,34 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_rejects_video_content_parts() {
|
||||
fn prepare_chat_request_accepts_video_content_parts() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![ChatMessage::User {
|
||||
content: MessageContent::Parts(vec![ContentPart::VideoUrl {
|
||||
video_url: VideoUrl {
|
||||
url: "https://example.com/video.mp4".to_string(),
|
||||
},
|
||||
uuid: Some("video-uuid".to_string()),
|
||||
}]),
|
||||
name: None,
|
||||
}],
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let error = prepare_chat_request(
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.unwrap_err();
|
||||
.expect("request is valid");
|
||||
|
||||
expect!["Only text and image_url content parts are supported."]
|
||||
.assert_eq(&error.to_error_response().error.message);
|
||||
assert_eq!(
|
||||
prepared.chat_request.messages,
|
||||
vec![VllmChatMessage::user(vec![ChatContentPart::VideoUrl {
|
||||
video_url: "https://example.com/video.mp4".to_string(),
|
||||
uuid: Some("video-uuid".to_string()),
|
||||
}])]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -91,7 +91,11 @@ pub enum ContentPart {
|
||||
uuid: Option<String>,
|
||||
},
|
||||
#[serde(rename = "video_url")]
|
||||
VideoUrl { video_url: VideoUrl },
|
||||
VideoUrl {
|
||||
video_url: VideoUrl,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
uuid: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
|
||||
@@ -504,7 +504,7 @@ impl ChatRenderer for FakeChatBackend {
|
||||
let placeholder = self
|
||||
.multimodal_model_info
|
||||
.as_ref()
|
||||
.map(|info| info.placeholder_token())
|
||||
.and_then(|info| info.placeholder_token(llm_multimodal::Modality::Image))
|
||||
.unwrap_or("<image>");
|
||||
let mut prompt = String::new();
|
||||
for message in &request.messages {
|
||||
@@ -544,7 +544,9 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R
|
||||
for part in parts {
|
||||
match part {
|
||||
ChatContentPart::Text { text } => out.push_str(text),
|
||||
ChatContentPart::ImageUrl { .. } => out.push_str(placeholder),
|
||||
ChatContentPart::ImageUrl { .. } | ChatContentPart::VideoUrl { .. } => {
|
||||
out.push_str(placeholder)
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
@@ -565,8 +567,10 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
|
||||
let info = vllm_chat::multimodal::MultimodalModelInfo::from_paths(
|
||||
"qwen2-vl-test".to_string(),
|
||||
Some("qwen2_vl".to_string()),
|
||||
Some(&config_path),
|
||||
None,
|
||||
vllm_chat::multimodal::MultimodalConfigFiles {
|
||||
config: Some(&config_path),
|
||||
..Default::default()
|
||||
},
|
||||
Arc::new(fake_chat_tokenizer()),
|
||||
)
|
||||
.expect("load multimodal info")
|
||||
|
||||
Reference in New Issue
Block a user