[Rust Frontend] Integrate MM audio support (#48554)
Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Isotr0py
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
adce068118
commit
e281ac663a
@@ -299,6 +299,15 @@ fn convert_content(content: MessageContent) -> Result<ChatContent, ApiError> {
|
||||
video_url: video_url.url,
|
||||
uuid,
|
||||
}),
|
||||
ContentPart::AudioUrl { audio_url, uuid } => Ok(ChatContentPart::AudioUrl {
|
||||
audio_url: audio_url.url,
|
||||
uuid,
|
||||
}),
|
||||
ContentPart::InputAudio { input_audio, uuid } => Ok(ChatContentPart::InputAudio {
|
||||
data: input_audio.data,
|
||||
format: input_audio.format,
|
||||
uuid,
|
||||
}),
|
||||
})
|
||||
.try_collect()
|
||||
.map(ChatContent::Parts),
|
||||
@@ -402,8 +411,8 @@ mod tests {
|
||||
AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
|
||||
};
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, MessageContent,
|
||||
StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
|
||||
AudioUrl, ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, InputAudio,
|
||||
MessageContent, StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
|
||||
};
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context};
|
||||
|
||||
@@ -810,6 +819,51 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_accepts_audio_content_parts() {
|
||||
let request = ChatCompletionRequest {
|
||||
model: "Qwen/Qwen3-ASR-1.7B".to_string(),
|
||||
messages: vec![ChatMessage::User {
|
||||
content: MessageContent::Parts(vec![
|
||||
ContentPart::InputAudio {
|
||||
input_audio: InputAudio {
|
||||
data: "dGVzdA==".to_string(),
|
||||
format: Some("wav".to_string()),
|
||||
},
|
||||
uuid: Some("audio-1".to_string()),
|
||||
},
|
||||
ContentPart::AudioUrl {
|
||||
audio_url: AudioUrl {
|
||||
url: "https://example.com/audio.mp3".to_string(),
|
||||
},
|
||||
uuid: None,
|
||||
},
|
||||
]),
|
||||
name: None,
|
||||
}],
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen3-ASR-1.7B"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(
|
||||
prepared.chat_request.messages,
|
||||
vec![VllmChatMessage::user(vec![
|
||||
ChatContentPart::InputAudio {
|
||||
data: "dGVzdA==".to_string(),
|
||||
format: Some("wav".to_string()),
|
||||
uuid: Some("audio-1".to_string()),
|
||||
},
|
||||
ChatContentPart::audio_url("https://example.com/audio.mp3"),
|
||||
])]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_rejects_assistant_image_url_content_parts() {
|
||||
let request = ChatCompletionRequest {
|
||||
|
||||
@@ -99,6 +99,18 @@ pub enum ContentPart {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
uuid: Option<String>,
|
||||
},
|
||||
#[serde(rename = "audio_url")]
|
||||
AudioUrl {
|
||||
audio_url: AudioUrl,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
uuid: Option<String>,
|
||||
},
|
||||
#[serde(rename = "input_audio")]
|
||||
InputAudio {
|
||||
input_audio: InputAudio,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
uuid: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
@@ -113,6 +125,19 @@ pub struct VideoUrl {
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct AudioUrl {
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
/// Base64-encoded audio bytes in OpenAI `input_audio` form.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct InputAudio {
|
||||
pub data: String,
|
||||
pub format: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Streaming
|
||||
// ============================================================================
|
||||
|
||||
@@ -551,9 +551,10 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R
|
||||
for part in parts {
|
||||
match part {
|
||||
ChatContentPart::Text { text } => out.push_str(text),
|
||||
ChatContentPart::ImageUrl { .. } | ChatContentPart::VideoUrl { .. } => {
|
||||
out.push_str(placeholder)
|
||||
}
|
||||
ChatContentPart::ImageUrl { .. }
|
||||
| ChatContentPart::VideoUrl { .. }
|
||||
| ChatContentPart::InputAudio { .. }
|
||||
| ChatContentPart::AudioUrl { .. } => out.push_str(placeholder),
|
||||
}
|
||||
}
|
||||
out
|
||||
|
||||
Reference in New Issue
Block a user