[Rust Frontend] Integrate MM video support (#47959)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-10 08:15:33 +00:00
committed by GitHub
parent 216ee58780
commit 074bdd0d99
16 changed files with 1576 additions and 521 deletions
@@ -290,7 +290,10 @@ fn convert_content(content: MessageContent) -> Result<ChatContent, ApiError> {
detail: image_url.detail,
uuid,
}),
_ => bail_invalid_request!("Only text and image_url content parts are supported."),
ContentPart::VideoUrl { video_url, uuid } => Ok(ChatContentPart::VideoUrl {
video_url: video_url.url,
uuid,
}),
})
.try_collect()
.map(ChatContent::Parts),
@@ -772,28 +775,34 @@ mod tests {
}
#[test]
fn prepare_chat_request_rejects_video_content_parts() {
fn prepare_chat_request_accepts_video_content_parts() {
let request = ChatCompletionRequest {
messages: vec![ChatMessage::User {
content: MessageContent::Parts(vec![ContentPart::VideoUrl {
video_url: VideoUrl {
url: "https://example.com/video.mp4".to_string(),
},
uuid: Some("video-uuid".to_string()),
}]),
name: None,
}],
..base_request()
};
let error = prepare_chat_request(
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.unwrap_err();
.expect("request is valid");
expect!["Only text and image_url content parts are supported."]
.assert_eq(&error.to_error_response().error.message);
assert_eq!(
prepared.chat_request.messages,
vec![VllmChatMessage::user(vec![ChatContentPart::VideoUrl {
video_url: "https://example.com/video.mp4".to_string(),
uuid: Some("video-uuid".to_string()),
}])]
);
}
#[test]
@@ -91,7 +91,11 @@ pub enum ContentPart {
uuid: Option<String>,
},
#[serde(rename = "video_url")]
VideoUrl { video_url: VideoUrl },
VideoUrl {
video_url: VideoUrl,
#[serde(skip_serializing_if = "Option::is_none")]
uuid: Option<String>,
},
}
#[serde_with::skip_serializing_none]
+8 -4
View File
@@ -504,7 +504,7 @@ impl ChatRenderer for FakeChatBackend {
let placeholder = self
.multimodal_model_info
.as_ref()
.map(|info| info.placeholder_token())
.and_then(|info| info.placeholder_token(llm_multimodal::Modality::Image))
.unwrap_or("<image>");
let mut prompt = String::new();
for message in &request.messages {
@@ -544,7 +544,9 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R
for part in parts {
match part {
ChatContentPart::Text { text } => out.push_str(text),
ChatContentPart::ImageUrl { .. } => out.push_str(placeholder),
ChatContentPart::ImageUrl { .. } | ChatContentPart::VideoUrl { .. } => {
out.push_str(placeholder)
}
}
}
out
@@ -565,8 +567,10 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
let info = vllm_chat::multimodal::MultimodalModelInfo::from_paths(
"qwen2-vl-test".to_string(),
Some("qwen2_vl".to_string()),
Some(&config_path),
None,
vllm_chat::multimodal::MultimodalConfigFiles {
config: Some(&config_path),
..Default::default()
},
Arc::new(fake_chat_tokenizer()),
)
.expect("load multimodal info")