[Rust Frontend] Integrate MM audio support (#48554)

Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Bugen Zhao
2026-07-15 15:00:17 +08:00
committed by GitHub
co-authored by Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent adce068118
commit e281ac663a
18 changed files with 1275 additions and 258 deletions
@@ -299,6 +299,15 @@ fn convert_content(content: MessageContent) -> Result<ChatContent, ApiError> {
video_url: video_url.url,
uuid,
}),
ContentPart::AudioUrl { audio_url, uuid } => Ok(ChatContentPart::AudioUrl {
audio_url: audio_url.url,
uuid,
}),
ContentPart::InputAudio { input_audio, uuid } => Ok(ChatContentPart::InputAudio {
data: input_audio.data,
format: input_audio.format,
uuid,
}),
})
.try_collect()
.map(ChatContent::Parts),
@@ -402,8 +411,8 @@ mod tests {
AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
};
use crate::routes::openai::utils::types::{
ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, MessageContent,
StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
AudioUrl, ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, InputAudio,
MessageContent, StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
};
use crate::utils::{ResolvedRequestContext, resolve_request_context};
@@ -810,6 +819,51 @@ mod tests {
);
}
#[test]
fn prepare_chat_request_accepts_audio_content_parts() {
let request = ChatCompletionRequest {
model: "Qwen/Qwen3-ASR-1.7B".to_string(),
messages: vec![ChatMessage::User {
content: MessageContent::Parts(vec![
ContentPart::InputAudio {
input_audio: InputAudio {
data: "dGVzdA==".to_string(),
format: Some("wav".to_string()),
},
uuid: Some("audio-1".to_string()),
},
ContentPart::AudioUrl {
audio_url: AudioUrl {
url: "https://example.com/audio.mp3".to_string(),
},
uuid: None,
},
]),
name: None,
}],
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen3-ASR-1.7B"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.messages,
vec![VllmChatMessage::user(vec![
ChatContentPart::InputAudio {
data: "dGVzdA==".to_string(),
format: Some("wav".to_string()),
uuid: Some("audio-1".to_string()),
},
ChatContentPart::audio_url("https://example.com/audio.mp3"),
])]
);
}
#[test]
fn prepare_chat_request_rejects_assistant_image_url_content_parts() {
let request = ChatCompletionRequest {
@@ -99,6 +99,18 @@ pub enum ContentPart {
#[serde(skip_serializing_if = "Option::is_none")]
uuid: Option<String>,
},
#[serde(rename = "audio_url")]
AudioUrl {
audio_url: AudioUrl,
#[serde(skip_serializing_if = "Option::is_none")]
uuid: Option<String>,
},
#[serde(rename = "input_audio")]
InputAudio {
input_audio: InputAudio,
#[serde(skip_serializing_if = "Option::is_none")]
uuid: Option<String>,
},
}
#[serde_with::skip_serializing_none]
@@ -113,6 +125,19 @@ pub struct VideoUrl {
pub url: String,
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
pub struct AudioUrl {
pub url: String,
}
/// Base64-encoded audio bytes in OpenAI `input_audio` form.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
pub struct InputAudio {
pub data: String,
pub format: Option<String>,
}
// ============================================================================
// Streaming
// ============================================================================
+4 -3
View File
@@ -551,9 +551,10 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R
for part in parts {
match part {
ChatContentPart::Text { text } => out.push_str(text),
ChatContentPart::ImageUrl { .. } | ChatContentPart::VideoUrl { .. } => {
out.push_str(placeholder)
}
ChatContentPart::ImageUrl { .. }
| ChatContentPart::VideoUrl { .. }
| ChatContentPart::InputAudio { .. }
| ChatContentPart::AudioUrl { .. } => out.push_str(placeholder),
}
}
out