[Rust Frontend] Integrate MM audio support (#48554)
Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Isotr0py
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
adce068118
commit
e281ac663a
@@ -16,7 +16,7 @@ llm-multimodal.workspace = true
|
||||
minijinja.workspace = true
|
||||
minijinja-contrib.workspace = true
|
||||
openai-harmony.workspace = true
|
||||
reqwest.workspace = true
|
||||
reqwest-0-13.workspace = true
|
||||
serde.workspace = true
|
||||
serde-json-fmt.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
@@ -151,6 +151,7 @@ fn resolve_multimodal_render_info(
|
||||
info.map(|info| MultimodalRenderInfo {
|
||||
image_token: info.placeholder_token(Modality::Image).map(str::to_string),
|
||||
video_token: info.placeholder_token(Modality::Video).map(str::to_string),
|
||||
audio_token: info.placeholder_token(Modality::Audio).map(str::to_string),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+203
-92
@@ -5,24 +5,24 @@
|
||||
//!
|
||||
//! This module owns the multimodal path for chat requests: it extracts media
|
||||
//! parts from structured chat messages, fetches and preprocesses them through
|
||||
//! `llm-multimodal` one modality at a time, expands rendered prompt
|
||||
//! placeholders after tokenization, and builds the engine-facing
|
||||
//! `MmFeatures` payload.
|
||||
//! `llm-multimodal`, expands rendered prompt placeholders after tokenization,
|
||||
//! and builds the engine-facing `MmFeatures` payload.
|
||||
//!
|
||||
//! Raw media stays above `vllm-text`; this module lowers it into token IDs and
|
||||
//! opaque tensor payloads before the request is handed to text generation.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashSet;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
|
||||
use itertools::izip;
|
||||
use llm_multimodal::{
|
||||
AsyncMultiModalTracker, FieldLayout, ImageFrame, MediaConnector, MediaConnectorConfig,
|
||||
MediaContentPart, Modality, ModelMetadata, ModelProcessorSpec, ModelRegistry,
|
||||
PreProcessorConfig, PreprocessedEncoderInputs, PromptReplacement, Tokenizer as TokenResolver,
|
||||
TrackedMedia, VideoClip, VisionPreProcessor, VisionProcessorRegistry,
|
||||
AsyncMultiModalTracker, AudioClip, AudioPreProcessor, EncoderFieldLayouts, FieldLayout,
|
||||
ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata,
|
||||
ModelProcessorSpec, ModelRegistry, PreProcessorConfig, PreprocessedEncoderInputs,
|
||||
PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor,
|
||||
VisionProcessorRegistry,
|
||||
};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tracing::warn;
|
||||
@@ -35,8 +35,10 @@ use crate::error::{Error, Result, bail_multimodal, multimodal};
|
||||
use crate::renderer::RenderedPrompt;
|
||||
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
|
||||
|
||||
mod audio;
|
||||
mod expand;
|
||||
mod image;
|
||||
mod item;
|
||||
mod tensor;
|
||||
mod video;
|
||||
|
||||
@@ -46,9 +48,9 @@ use self::expand::expand_prompt_token_ids;
|
||||
#[derive(Clone)]
|
||||
pub struct MultimodalModelInfo {
|
||||
context: MultimodalModelContext,
|
||||
spec: ResolvedMultimodalSpec,
|
||||
image: Option<ModalitySupport>,
|
||||
video: Option<ModalitySupport>,
|
||||
audio: Option<AudioModalitySupport>,
|
||||
media_connector: Arc<MediaConnector>,
|
||||
}
|
||||
|
||||
@@ -88,22 +90,50 @@ impl MultimodalModelContext {
|
||||
LazyLock::new(VisionProcessorRegistry::with_defaults);
|
||||
REGISTRY.find(&self.model_id, self.model_type.as_deref())
|
||||
}
|
||||
|
||||
/// Resolve an audio preprocessor for one loaded model.
|
||||
fn resolve_audio_processor(
|
||||
&self,
|
||||
model_spec: &'static dyn ModelProcessorSpec,
|
||||
preprocessor_config: &PreProcessorConfig,
|
||||
) -> Option<Arc<dyn AudioPreProcessor>> {
|
||||
model_spec.audio_processor(&self.config, preprocessor_config).map(Arc::from)
|
||||
}
|
||||
}
|
||||
|
||||
/// Static model-specific tensor-layout behavior shared across modalities.
|
||||
/// Static model-specific tensor-layout behavior for one modality.
|
||||
#[derive(Clone)]
|
||||
struct ResolvedMultimodalSpec {
|
||||
raw: &'static dyn ModelProcessorSpec,
|
||||
field_layouts: HashMap<String, FieldLayout>,
|
||||
modality: Modality,
|
||||
field_layouts: EncoderFieldLayouts,
|
||||
keep_on_cpu_keys: HashSet<String>,
|
||||
}
|
||||
|
||||
impl ResolvedMultimodalSpec {
|
||||
fn new(raw: &'static dyn ModelProcessorSpec) -> Self {
|
||||
fn new(raw: &'static dyn ModelProcessorSpec, modality: Modality) -> Self {
|
||||
Self {
|
||||
raw,
|
||||
field_layouts: raw.field_layouts(),
|
||||
keep_on_cpu_keys: raw.keep_on_cpu_keys().into_iter().collect(),
|
||||
modality,
|
||||
field_layouts: raw.encoder_field_layouts_for(modality),
|
||||
keep_on_cpu_keys: raw.keep_on_cpu_keys_for(modality).into_iter().collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn primary_key(&self) -> &'static str {
|
||||
match self.modality {
|
||||
Modality::Image => image::IMAGE_PRIMARY_KEY,
|
||||
Modality::Video => video::VIDEO_PRIMARY_KEY,
|
||||
Modality::Audio => audio::AUDIO_PRIMARY_KEY,
|
||||
Modality::ImageEmbeds => unreachable!("image embeds use no encoder preprocessor"),
|
||||
}
|
||||
}
|
||||
|
||||
fn field_layout_for(&self, key: &str) -> Option<&FieldLayout> {
|
||||
if key == self.primary_key() {
|
||||
Some(&self.field_layouts.encoder_input)
|
||||
} else {
|
||||
self.field_layouts.model_specific.get(key)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,14 +141,15 @@ impl ResolvedMultimodalSpec {
|
||||
&self,
|
||||
context: &MultimodalModelContext,
|
||||
preprocessed: &PreprocessedEncoderInputs,
|
||||
modality: Modality,
|
||||
) -> Result<Vec<PromptReplacement>> {
|
||||
Ok(self.raw.prompt_replacements_for(&context.metadata(), preprocessed, modality)?)
|
||||
Ok(self
|
||||
.raw
|
||||
.prompt_replacements_for(&context.metadata(), preprocessed, self.modality)?)
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolved placeholder tokens for one modality.
|
||||
#[derive(Clone)]
|
||||
#[derive(Debug, Clone)]
|
||||
struct ResolvedPlaceholder {
|
||||
token: String,
|
||||
/// The token ID emitted for `token` in the rendered prompt.
|
||||
@@ -152,19 +183,30 @@ impl ResolvedPlaceholder {
|
||||
}
|
||||
}
|
||||
|
||||
/// Static per-modality vision preprocessor plus its loaded config and
|
||||
/// resolved placeholder tokens.
|
||||
/// Static per-modality vision preprocessor plus its loaded config, resolved
|
||||
/// placeholder tokens, and the model's shared tensor-layout spec.
|
||||
#[derive(Clone)]
|
||||
struct ModalitySupport {
|
||||
spec: ResolvedMultimodalSpec,
|
||||
placeholder: ResolvedPlaceholder,
|
||||
processor: &'static dyn VisionPreProcessor,
|
||||
config: PreProcessorConfig,
|
||||
}
|
||||
|
||||
/// Static audio preprocessor plus the resolved model contract for its output.
|
||||
#[derive(Clone)]
|
||||
struct AudioModalitySupport {
|
||||
spec: ResolvedMultimodalSpec,
|
||||
placeholder: ResolvedPlaceholder,
|
||||
processor: Arc<dyn AudioPreProcessor>,
|
||||
}
|
||||
|
||||
/// Model-repo config file locations consumed by multimodal support.
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
pub struct MultimodalConfigFiles<'a> {
|
||||
pub config: Option<&'a Path>,
|
||||
/// Shared preprocessor config (`preprocessor_config.json`), consumed by
|
||||
/// image/audio processors and used as the video fallback.
|
||||
pub preprocessor_config: Option<&'a Path>,
|
||||
/// Video-specific preprocessor config (`video_preprocessor_config.json`).
|
||||
pub video_preprocessor_config: Option<&'a Path>,
|
||||
@@ -212,6 +254,8 @@ struct FetchedMedia {
|
||||
image_uuids: Vec<Option<String>>,
|
||||
videos: Vec<Arc<VideoClip>>,
|
||||
video_uuids: Vec<Option<String>>,
|
||||
audios: Vec<Arc<AudioClip>>,
|
||||
audio_uuids: Vec<Option<String>>,
|
||||
}
|
||||
|
||||
/// One modality's preprocessed output, ready for the shared expansion and
|
||||
@@ -253,7 +297,7 @@ impl MultimodalModelInfo {
|
||||
}
|
||||
None => serde_json::Value::Object(Default::default()),
|
||||
};
|
||||
let image_preprocessor_config = load_preprocessor_config(
|
||||
let preprocessor_config = load_preprocessor_config(
|
||||
files.preprocessor_config,
|
||||
"preprocessor_config.json",
|
||||
files.processor_config,
|
||||
@@ -266,7 +310,7 @@ impl MultimodalModelInfo {
|
||||
files.processor_config,
|
||||
"video_processor",
|
||||
)?
|
||||
.unwrap_or_else(|| image_preprocessor_config.clone());
|
||||
.unwrap_or_else(|| preprocessor_config.clone());
|
||||
|
||||
let context = MultimodalModelContext {
|
||||
model_id,
|
||||
@@ -275,41 +319,76 @@ impl MultimodalModelInfo {
|
||||
tokenizer: TokenizerResolver(tokenizer),
|
||||
};
|
||||
|
||||
Self::from_loaded(
|
||||
context,
|
||||
image_preprocessor_config,
|
||||
video_preprocessor_config,
|
||||
)
|
||||
Self::from_loaded(context, preprocessor_config, video_preprocessor_config)
|
||||
}
|
||||
|
||||
/// Resolve multimodal support from an assembled context and parsed
|
||||
/// preprocessor configs.
|
||||
fn from_loaded(
|
||||
context: MultimodalModelContext,
|
||||
image_preprocessor_config: PreProcessorConfig,
|
||||
preprocessor_config: PreProcessorConfig,
|
||||
video_preprocessor_config: PreProcessorConfig,
|
||||
) -> Result<Option<Self>> {
|
||||
let (image, video) = Self::resolve_vision_lanes(
|
||||
&context,
|
||||
preprocessor_config.clone(),
|
||||
video_preprocessor_config,
|
||||
);
|
||||
let audio = Self::resolve_audio_lane(&context, &preprocessor_config)?;
|
||||
|
||||
if image.is_none() && video.is_none() && audio.is_none() {
|
||||
warn!(
|
||||
model_id = context.model_id,
|
||||
model_type = context.model_type,
|
||||
"no multimodal modality resolved; disabling multimodal support for this model"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let media_connector = Arc::new(MediaConnector::new(
|
||||
reqwest_0_13::Client::new(),
|
||||
MediaConnectorConfig::default(),
|
||||
)?);
|
||||
|
||||
Ok(Some(Self {
|
||||
context,
|
||||
image,
|
||||
video,
|
||||
audio,
|
||||
media_connector,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Resolve the spec-backed vision lanes (image and video).
|
||||
///
|
||||
/// A missing model spec or vision processor disables both lanes but is
|
||||
/// not an error: unsupported model families are logged and skipped.
|
||||
fn resolve_vision_lanes(
|
||||
context: &MultimodalModelContext,
|
||||
preprocessor_config: PreProcessorConfig,
|
||||
video_preprocessor_config: PreProcessorConfig,
|
||||
) -> (Option<ModalitySupport>, Option<ModalitySupport>) {
|
||||
let Some(raw_spec) = context.resolve_model_spec() else {
|
||||
warn!(
|
||||
model_id = context.model_id,
|
||||
model_type = context.model_type,
|
||||
"multimodal model spec is not registered; disabling multimodal support for this model"
|
||||
"multimodal model spec is not registered; disabling image/video support for this model"
|
||||
);
|
||||
return Ok(None);
|
||||
return (None, None);
|
||||
};
|
||||
|
||||
let Some(processor) = context.resolve_vision_processor() else {
|
||||
warn!(
|
||||
model_id = context.model_id,
|
||||
model_type = context.model_type,
|
||||
"vision processor is not registered; disabling multimodal support for this model"
|
||||
"vision processor is not registered; disabling image/video support for this model"
|
||||
);
|
||||
return Ok(None);
|
||||
return (None, None);
|
||||
};
|
||||
|
||||
// Warn and disable the modality if the placeholder resolution fails.
|
||||
let resolve_placeholder =
|
||||
|modality: Modality| match ResolvedPlaceholder::resolve(raw_spec, &context, modality) {
|
||||
|modality: Modality| match ResolvedPlaceholder::resolve(raw_spec, context, modality) {
|
||||
Ok(placeholder) => Some(placeholder),
|
||||
Err(error) => {
|
||||
warn!(
|
||||
@@ -323,9 +402,10 @@ impl MultimodalModelInfo {
|
||||
};
|
||||
|
||||
let image = resolve_placeholder(Modality::Image).map(|placeholder| ModalitySupport {
|
||||
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Image),
|
||||
placeholder,
|
||||
processor,
|
||||
config: image_preprocessor_config,
|
||||
config: preprocessor_config.clone(),
|
||||
});
|
||||
|
||||
let video = resolve_placeholder(Modality::Video).and_then(|placeholder| {
|
||||
@@ -341,6 +421,7 @@ impl MultimodalModelInfo {
|
||||
None
|
||||
} else {
|
||||
Some(ModalitySupport {
|
||||
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Video),
|
||||
placeholder,
|
||||
processor,
|
||||
config: video_preprocessor_config,
|
||||
@@ -348,26 +429,33 @@ impl MultimodalModelInfo {
|
||||
}
|
||||
});
|
||||
|
||||
if image.is_none() && video.is_none() {
|
||||
warn!(
|
||||
model_id = context.model_id,
|
||||
model_type = context.model_type,
|
||||
"no multimodal modality resolved; disabling multimodal support for this model"
|
||||
);
|
||||
(image, video)
|
||||
}
|
||||
|
||||
/// Resolve a spec-backed audio lane when the model advertises audio support.
|
||||
fn resolve_audio_lane(
|
||||
context: &MultimodalModelContext,
|
||||
preprocessor_config: &PreProcessorConfig,
|
||||
) -> Result<Option<AudioModalitySupport>> {
|
||||
let Some(raw_spec) = context.resolve_model_spec() else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !raw_spec.modality_limits(&context.metadata())?.contains_key(&Modality::Audio) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let media_connector = Arc::new(MediaConnector::new(
|
||||
reqwest::Client::new(),
|
||||
MediaConnectorConfig::default(),
|
||||
)?);
|
||||
|
||||
Ok(Some(Self {
|
||||
context,
|
||||
spec: ResolvedMultimodalSpec::new(raw_spec),
|
||||
image,
|
||||
video,
|
||||
media_connector,
|
||||
let Some(processor) = context.resolve_audio_processor(raw_spec, preprocessor_config) else {
|
||||
warn!(
|
||||
model_id = context.model_id,
|
||||
model_spec = raw_spec.name(),
|
||||
"model spec does not provide an audio processor; disabling audio support for this model"
|
||||
);
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(AudioModalitySupport {
|
||||
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Audio),
|
||||
placeholder: ResolvedPlaceholder::resolve(raw_spec, context, Modality::Audio)?,
|
||||
processor,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -380,7 +468,8 @@ impl MultimodalModelInfo {
|
||||
match modality {
|
||||
Modality::Image => self.image.as_ref()?.placeholder.token.as_str().into(),
|
||||
Modality::Video => self.video.as_ref()?.placeholder.token.as_str().into(),
|
||||
_ => None,
|
||||
Modality::Audio => self.audio.as_ref()?.placeholder.token.as_str().into(),
|
||||
Modality::ImageEmbeds => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -449,12 +538,35 @@ fn extract_media_parts(request: &ChatRequest) -> Result<Vec<MediaContentPart>> {
|
||||
uuid: uuid.clone(),
|
||||
})
|
||||
}
|
||||
ChatContentPart::InputAudio { data, format, uuid } => {
|
||||
all_parts.push(MediaContentPart::AudioUrl {
|
||||
url: input_audio_data_url(data, format.as_deref())?,
|
||||
uuid: uuid.clone(),
|
||||
})
|
||||
}
|
||||
ChatContentPart::AudioUrl { audio_url, uuid } => {
|
||||
all_parts.push(MediaContentPart::AudioUrl {
|
||||
url: audio_url.clone(),
|
||||
uuid: uuid.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(all_parts)
|
||||
}
|
||||
|
||||
/// Wrap OpenAI base64 audio in a data URL consumed by `MediaConnector`.
|
||||
fn input_audio_data_url(data: &str, format: Option<&str>) -> Result<String> {
|
||||
let mime_type = match format {
|
||||
Some(format) if format.eq_ignore_ascii_case("wav") => "audio/wav",
|
||||
Some(format) if format.eq_ignore_ascii_case("mp3") => "audio/mpeg",
|
||||
Some(format) => bail_multimodal!("unsupported input_audio format `{format}`"),
|
||||
None => "application/octet-stream",
|
||||
};
|
||||
Ok(format!("data:{mime_type};base64,{data}"))
|
||||
}
|
||||
|
||||
impl MultimodalModelInfo {
|
||||
/// Run media fetch, per-modality preprocessing, prompt expansion, and
|
||||
/// feature build.
|
||||
@@ -468,11 +580,10 @@ impl MultimodalModelInfo {
|
||||
prompt_token_ids: &mut Vec<u32>,
|
||||
model_dtype: ModelDtype,
|
||||
) -> Result<MmFeatures> {
|
||||
if media_parts.is_empty() {
|
||||
let media_parts_len = media_parts.len();
|
||||
if media_parts_len == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let media_parts_len = media_parts.len();
|
||||
|
||||
// TODO: enforce per-modality item-count limits, aligned with the
|
||||
// engine's `--limit-mm-per-prompt` semantics.
|
||||
let fetched = self.fetch_media(media_parts).await?;
|
||||
@@ -486,6 +597,9 @@ impl MultimodalModelInfo {
|
||||
prepared
|
||||
.push(self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype).await?);
|
||||
}
|
||||
if !fetched.audios.is_empty() {
|
||||
prepared.push(self.prepare_audios(fetched.audios, fetched.audio_uuids).await?);
|
||||
}
|
||||
|
||||
let mut ranges = expand_prompt_token_ids(prompt_token_ids, &prepared)?;
|
||||
|
||||
@@ -524,8 +638,8 @@ impl MultimodalModelInfo {
|
||||
Ok(features)
|
||||
}
|
||||
|
||||
/// Fetch all media parts and split them per modality, preserving their
|
||||
/// request-order UUID metadata.
|
||||
/// Fetch all connector-backed media parts and split them per modality,
|
||||
/// preserving their request-order UUID metadata.
|
||||
async fn fetch_media(&self, media_parts: Vec<MediaContentPart>) -> Result<FetchedMedia> {
|
||||
let mut tracker = AsyncMultiModalTracker::new(Arc::clone(&self.media_connector));
|
||||
for part in media_parts {
|
||||
@@ -562,11 +676,27 @@ impl MultimodalModelInfo {
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let video_uuids = tracker_output.uuids.remove(&Modality::Video).unwrap_or_default();
|
||||
|
||||
let audios = tracker_output
|
||||
.data
|
||||
.remove(&Modality::Audio)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|media| match media {
|
||||
TrackedMedia::Audio(clip) => Ok(clip),
|
||||
_ => Err(multimodal!(
|
||||
"tracker returned non-audio media for the audio modality"
|
||||
)),
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let audio_uuids = tracker_output.uuids.remove(&Modality::Audio).unwrap_or_default();
|
||||
|
||||
Ok(FetchedMedia {
|
||||
images,
|
||||
image_uuids,
|
||||
videos,
|
||||
video_uuids,
|
||||
audios,
|
||||
audio_uuids,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -665,42 +795,6 @@ mod tests {
|
||||
test_info("qwen3_vl", config, qwen3_vl_tokenizer())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_paths_resolves_image_config_from_processor_config() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let config_path = dir.path().join("config.json");
|
||||
std::fs::write(
|
||||
&config_path,
|
||||
serde_json::json!({
|
||||
"model_type": "qwen3_vl",
|
||||
"image_token_id": QWEN3_IMAGE_PAD_ID,
|
||||
})
|
||||
.to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
let processor_config_path = dir.path().join("processor_config.json");
|
||||
std::fs::write(
|
||||
&processor_config_path,
|
||||
r#"{"image_processor":{"size":{"shortest_edge":64}}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let info = MultimodalModelInfo::from_paths(
|
||||
"qwen3-vl-test".to_string(),
|
||||
Some("qwen3_vl".to_string()),
|
||||
MultimodalConfigFiles {
|
||||
config: Some(&config_path),
|
||||
processor_config: Some(&processor_config_path),
|
||||
..Default::default()
|
||||
},
|
||||
Arc::new(qwen3_vl_tokenizer()),
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(info.image.unwrap().config.get_shortest_edge(), Some(64));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_vl_resolves_image_and_video_support() {
|
||||
let info = qwen3_vl_info();
|
||||
@@ -742,4 +836,21 @@ mod tests {
|
||||
assert_eq!(info.placeholder_token(Modality::Image), Some("<|image|>"));
|
||||
assert_eq!(info.placeholder_token(Modality::Video), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_audio_uses_connector_data_urls() {
|
||||
assert_eq!(
|
||||
input_audio_data_url("AAAA", Some("wav")).unwrap(),
|
||||
"data:audio/wav;base64,AAAA"
|
||||
);
|
||||
assert_eq!(
|
||||
input_audio_data_url("BBBB", Some("MP3")).unwrap(),
|
||||
"data:audio/mpeg;base64,BBBB"
|
||||
);
|
||||
assert_eq!(
|
||||
input_audio_data_url("CCCC", None).unwrap(),
|
||||
"data:application/octet-stream;base64,CCCC"
|
||||
);
|
||||
assert!(input_audio_data_url("AAAA", Some("flac")).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Audio-modality preparation through `llm-multimodal`.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use llm_multimodal::{AudioClip, Modality, PreprocessedEncoderInputs};
|
||||
use vllm_engine_core_client::protocol::dtype::ModelDtype;
|
||||
|
||||
use super::{AudioModalitySupport, MultimodalModelInfo, PreparedMedia, item};
|
||||
use crate::error::{Error, Result, bail_multimodal, multimodal};
|
||||
|
||||
/// Forward-kwargs name of the primary audio encoder input.
|
||||
pub(super) const AUDIO_PRIMARY_KEY: &str = "input_audio_features";
|
||||
|
||||
impl MultimodalModelInfo {
|
||||
/// Preprocess fetched audio clips as one batch and build per-item features.
|
||||
pub(super) async fn prepare_audios(
|
||||
&self,
|
||||
clips: Vec<Arc<AudioClip>>,
|
||||
uuids: Vec<Option<String>>,
|
||||
) -> Result<PreparedMedia> {
|
||||
let support = self.audio.as_ref().ok_or_else(|| Error::UnsupportedModality {
|
||||
modality: Modality::Audio.to_string(),
|
||||
})?;
|
||||
let preprocessed = self.preprocess_audios(support, &clips).await?;
|
||||
let replacements = support.spec.prompt_replacements_for(&self.context, &preprocessed)?;
|
||||
if replacements.len() != clips.len() {
|
||||
bail_multimodal!(
|
||||
"number of audio prompt replacements {} does not match number of audio clips {}",
|
||||
replacements.len(),
|
||||
clips.len()
|
||||
);
|
||||
}
|
||||
|
||||
let hashes = clips.iter().map(|clip| clip.hash.clone()).collect();
|
||||
let items = item::build_batched_items(
|
||||
&support.spec,
|
||||
preprocessed,
|
||||
hashes,
|
||||
uuids,
|
||||
ModelDtype::Float32,
|
||||
)?;
|
||||
|
||||
Ok(PreparedMedia {
|
||||
modality: Modality::Audio,
|
||||
placeholder: support.placeholder.clone(),
|
||||
replacements,
|
||||
items,
|
||||
})
|
||||
}
|
||||
|
||||
/// Run CPU-heavy audio preprocessing in a blocking task.
|
||||
async fn preprocess_audios(
|
||||
&self,
|
||||
support: &AudioModalitySupport,
|
||||
clips: &[Arc<AudioClip>],
|
||||
) -> Result<PreprocessedEncoderInputs> {
|
||||
let processor = Arc::clone(&support.processor);
|
||||
let clips = clips.to_vec();
|
||||
tokio::task::spawn_blocking(move || Ok(processor.preprocess(&clips)?))
|
||||
.await
|
||||
.map_err(|error| multimodal!("audio preprocessing task failed: {error}"))?
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use llm_multimodal::{MediaContentPart, ModelSpecificValue, PreProcessorConfig};
|
||||
use ndarray::ArrayD;
|
||||
use vllm_engine_core_client::protocol::multimodal::{MmField, MmKwargValue};
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::super::{MultimodalModelContext, TokenizerResolver};
|
||||
use super::*;
|
||||
|
||||
const AUDIO_PAD_ID: u32 = 151_676;
|
||||
|
||||
fn qwen3_asr_info() -> MultimodalModelInfo {
|
||||
let context = MultimodalModelContext {
|
||||
model_id: "Qwen/Qwen3-ASR-1.7B".to_string(),
|
||||
model_type: Some("qwen3_asr".to_string()),
|
||||
config: serde_json::json!({"model_type": "qwen3_asr"}),
|
||||
tokenizer: TokenizerResolver(Arc::new(
|
||||
TestTokenizer::new().with_regular_token("<|audio_pad|>", AUDIO_PAD_ID),
|
||||
)),
|
||||
};
|
||||
|
||||
MultimodalModelInfo::from_loaded(
|
||||
context,
|
||||
PreProcessorConfig::default(),
|
||||
PreProcessorConfig::default(),
|
||||
)
|
||||
.unwrap()
|
||||
.expect("Qwen3-ASR multimodal support")
|
||||
}
|
||||
|
||||
fn wav_i16_mono(sample_rate: u32, samples: &[i16]) -> Vec<u8> {
|
||||
let data_bytes = samples.len() as u32 * 2;
|
||||
let mut bytes = Vec::new();
|
||||
bytes.extend_from_slice(b"RIFF");
|
||||
bytes.extend_from_slice(&(36 + data_bytes).to_le_bytes());
|
||||
bytes.extend_from_slice(b"WAVEfmt ");
|
||||
bytes.extend_from_slice(&16_u32.to_le_bytes());
|
||||
bytes.extend_from_slice(&1_u16.to_le_bytes());
|
||||
bytes.extend_from_slice(&1_u16.to_le_bytes());
|
||||
bytes.extend_from_slice(&sample_rate.to_le_bytes());
|
||||
bytes.extend_from_slice(&(sample_rate * 2).to_le_bytes());
|
||||
bytes.extend_from_slice(&2_u16.to_le_bytes());
|
||||
bytes.extend_from_slice(&16_u16.to_le_bytes());
|
||||
bytes.extend_from_slice(b"data");
|
||||
bytes.extend_from_slice(&data_bytes.to_le_bytes());
|
||||
for sample in samples {
|
||||
bytes.extend_from_slice(&sample.to_le_bytes());
|
||||
}
|
||||
bytes
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_qwen_audio_from_model_spec() {
|
||||
let info = qwen3_asr_info();
|
||||
let support = info.audio.as_ref().expect("audio support");
|
||||
|
||||
assert_eq!(
|
||||
info.placeholder_token(Modality::Audio),
|
||||
Some("<|audio_pad|>")
|
||||
);
|
||||
assert_eq!(support.placeholder.marker_token_id, AUDIO_PAD_ID);
|
||||
assert_eq!(support.placeholder.embed_token_id, AUDIO_PAD_ID);
|
||||
assert!(matches!(
|
||||
support.spec.field_layouts.encoder_input,
|
||||
llm_multimodal::FieldLayout::Batched
|
||||
));
|
||||
|
||||
let preprocessed = PreprocessedEncoderInputs {
|
||||
encoder_input: ArrayD::zeros(vec![1, 128, 4]),
|
||||
feature_token_counts: vec![1],
|
||||
item_sizes: vec![(128, 4)],
|
||||
model_specific: HashMap::from([
|
||||
(
|
||||
"feature_attention_mask".to_string(),
|
||||
ModelSpecificValue::int_2d(vec![1; 4], 1, 4),
|
||||
),
|
||||
(
|
||||
"audio_feature_lengths".to_string(),
|
||||
ModelSpecificValue::int_1d(vec![4]),
|
||||
),
|
||||
]),
|
||||
};
|
||||
let item = item::build_batched_items(
|
||||
&support.spec,
|
||||
preprocessed,
|
||||
vec!["<hash>".to_string()],
|
||||
vec![None],
|
||||
ModelDtype::Float32,
|
||||
)
|
||||
.unwrap()
|
||||
.pop()
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
&item.data[AUDIO_PRIMARY_KEY].field,
|
||||
MmField::Batched(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tracker_processor_and_lowering_preserve_audio_contract() {
|
||||
let info = qwen3_asr_info();
|
||||
let wav = wav_i16_mono(16_000, &[0; 1_600]);
|
||||
let expected_hash = llm_multimodal::hasher::hash_audio(&wav);
|
||||
let fetched = info
|
||||
.fetch_media(vec![MediaContentPart::AudioData {
|
||||
data: wav,
|
||||
mime_type: Some("audio/wav".to_string()),
|
||||
uuid: Some("audio-1".to_string()),
|
||||
}])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let prepared = info.prepare_audios(fetched.audios, fetched.audio_uuids).await.unwrap();
|
||||
|
||||
assert_eq!(prepared.replacements.len(), 1);
|
||||
assert!(
|
||||
prepared.replacements[0]
|
||||
.tokens
|
||||
.iter()
|
||||
.all(|token| *token == AUDIO_PAD_ID as i32)
|
||||
);
|
||||
let item = &prepared.items[0];
|
||||
assert_eq!(item.hash, expected_hash);
|
||||
assert_eq!(item.uuid.as_deref(), Some("audio-1"));
|
||||
|
||||
let features = &item.data[AUDIO_PRIMARY_KEY];
|
||||
assert!(matches!(&features.field, MmField::Batched(_)));
|
||||
assert!(matches!(
|
||||
features.data.as_ref(),
|
||||
Some(MmKwargValue::Tensor(tensor))
|
||||
if tensor.dtype == "float32" && tensor.shape.first() == Some(&128)
|
||||
));
|
||||
let lengths = &item.data["audio_feature_lengths"];
|
||||
assert!(matches!(&lengths.field, MmField::Batched(_)));
|
||||
assert!(matches!(
|
||||
lengths.data.as_ref(),
|
||||
Some(MmKwargValue::Tensor(tensor))
|
||||
if tensor.dtype == "int64" && tensor.shape.is_empty()
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -418,6 +418,67 @@ mod tests {
|
||||
assert_bool_mask(&video_ranges[0], &[true, true, true, true]);
|
||||
}
|
||||
|
||||
const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
|
||||
|
||||
fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
|
||||
prepared_media(
|
||||
Modality::Audio,
|
||||
"<|audio_pad|>",
|
||||
QWEN3_AUDIO_PAD_ID,
|
||||
QWEN3_AUDIO_PAD_ID,
|
||||
replacements,
|
||||
)
|
||||
}
|
||||
|
||||
fn qwen3_audio_replacement(tokens: usize) -> PromptReplacement {
|
||||
PromptReplacement::repeated(
|
||||
Modality::Audio,
|
||||
"<|audio_pad|>",
|
||||
QWEN3_AUDIO_PAD_ID as TokenId,
|
||||
tokens,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expand_prompt_tokens_interleaves_audio_and_image_prepared_media() {
|
||||
// Audio marker precedes the image marker: the image range offset must
|
||||
// account for the earlier audio expansion growth.
|
||||
let mut prompt_token_ids = vec![1, QWEN3_AUDIO_PAD_ID, 2, LLAMA4_IMAGE_ID, 3];
|
||||
let prepared = vec![
|
||||
llama4_prepared(vec![llama4_single_tile_replacement()]),
|
||||
qwen3_audio_prepared(vec![qwen3_audio_replacement(3)]),
|
||||
];
|
||||
|
||||
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
prompt_token_ids,
|
||||
vec![
|
||||
1,
|
||||
QWEN3_AUDIO_PAD_ID,
|
||||
QWEN3_AUDIO_PAD_ID,
|
||||
QWEN3_AUDIO_PAD_ID,
|
||||
2,
|
||||
LLAMA4_IMAGE_START_ID,
|
||||
LLAMA4_IMAGE_ID,
|
||||
LLAMA4_PATCH_ID,
|
||||
LLAMA4_PATCH_ID,
|
||||
LLAMA4_IMAGE_END_ID,
|
||||
3,
|
||||
]
|
||||
);
|
||||
|
||||
let audio_ranges = &ranges[&Modality::Audio];
|
||||
assert_eq!(audio_ranges[0].offset, 1);
|
||||
assert_eq!(audio_ranges[0].length, 3);
|
||||
assert_bool_mask(&audio_ranges[0], &[true, true, true]);
|
||||
|
||||
let image_ranges = &ranges[&Modality::Image];
|
||||
assert_eq!(image_ranges[0].offset, 5);
|
||||
assert_eq!(image_ranges[0].length, 5);
|
||||
assert_bool_mask(&image_ranges[0], &[false, false, true, true, false]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expand_prompt_tokens_error_names_modality_with_leftover_replacements() {
|
||||
let mut prompt_token_ids = vec![1, QWEN3_IMAGE_PAD_ID, 2];
|
||||
|
||||
@@ -6,17 +6,15 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use itertools::izip;
|
||||
use llm_multimodal::{FieldLayout, ImageFrame, Modality, PreprocessedEncoderInputs};
|
||||
use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs};
|
||||
use vllm_engine_core_client::protocol::dtype::ModelDtype;
|
||||
use vllm_engine_core_client::protocol::multimodal::{
|
||||
MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice,
|
||||
SliceSpec,
|
||||
};
|
||||
|
||||
use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor};
|
||||
use super::{ModalitySupport, MultimodalModelInfo, PreparedMedia, item};
|
||||
use crate::error::{Error, Result, bail_multimodal, multimodal};
|
||||
|
||||
/// Forward-kwargs name of the primary image encoder input.
|
||||
pub(super) const IMAGE_PRIMARY_KEY: &str = "pixel_values";
|
||||
|
||||
impl MultimodalModelInfo {
|
||||
/// Preprocess all fetched image frames as one batch and build per-item
|
||||
/// features.
|
||||
@@ -30,9 +28,7 @@ impl MultimodalModelInfo {
|
||||
modality: Modality::Image.to_string(),
|
||||
})?;
|
||||
let preprocessed = self.preprocess_images(support, &frames).await?;
|
||||
let replacements =
|
||||
self.spec
|
||||
.prompt_replacements_for(&self.context, &preprocessed, Modality::Image)?;
|
||||
let replacements = support.spec.prompt_replacements_for(&self.context, &preprocessed)?;
|
||||
if replacements.len() != frames.len() {
|
||||
bail_multimodal!(
|
||||
"number of image prompt replacements {} does not match number of images {}",
|
||||
@@ -40,7 +36,9 @@ impl MultimodalModelInfo {
|
||||
frames.len()
|
||||
);
|
||||
}
|
||||
let items = self.build_image_items(preprocessed, &frames, uuids, model_dtype)?;
|
||||
let hashes = frames.iter().map(|frame| frame.hash.clone()).collect();
|
||||
let items =
|
||||
item::build_batched_items(&support.spec, preprocessed, hashes, uuids, model_dtype)?;
|
||||
|
||||
Ok(PreparedMedia {
|
||||
modality: Modality::Image,
|
||||
@@ -70,75 +68,4 @@ impl MultimodalModelInfo {
|
||||
.await
|
||||
.map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
|
||||
}
|
||||
|
||||
/// Convert one batch of preprocessed image tensors into per-item engine
|
||||
/// kwargs.
|
||||
///
|
||||
/// Tensor fields are sliced per item according to the model spec's field
|
||||
/// layout declarations.
|
||||
fn build_image_items(
|
||||
&self,
|
||||
preprocessed: PreprocessedEncoderInputs,
|
||||
frames: &[Arc<ImageFrame>],
|
||||
uuids: Vec<Option<String>>,
|
||||
model_dtype: ModelDtype,
|
||||
) -> Result<Vec<PreparedItem>> {
|
||||
let len = frames.len();
|
||||
let tensors = tensor::collect_tensors(preprocessed, "pixel_values", model_dtype)?;
|
||||
|
||||
let mut items = Vec::with_capacity(len);
|
||||
for (index, (frame, uuid)) in izip!(frames, uuids).enumerate() {
|
||||
let mut data = MmKwargsItem::new();
|
||||
for (key, tensor) in &tensors {
|
||||
let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(key);
|
||||
let (value, field) = match self.spec.field_layouts.get(key) {
|
||||
Some(FieldLayout::Batched) => (
|
||||
tensor.batched_value_at(index)?,
|
||||
MmField::Batched(MmBatchedField { keep_on_cpu }),
|
||||
),
|
||||
Some(FieldLayout::Flat { sizes_key }) => {
|
||||
let sizes = tensors.get(sizes_key).ok_or_else(|| {
|
||||
multimodal!("flat tensor sizes key `{sizes_key}` is missing")
|
||||
})?;
|
||||
let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?;
|
||||
(
|
||||
tensor.flat_value_range(start, end)?,
|
||||
MmField::Flat(MmFlatField {
|
||||
slices: vec![MmSlice::Slice(SliceSpec {
|
||||
start: Some(0),
|
||||
stop: Some((end - start) as isize),
|
||||
step: None,
|
||||
})],
|
||||
dim: 0,
|
||||
keep_on_cpu,
|
||||
}),
|
||||
)
|
||||
}
|
||||
None => (
|
||||
tensor.clone(),
|
||||
MmField::Shared(MmSharedField {
|
||||
batch_size: len,
|
||||
keep_on_cpu,
|
||||
}),
|
||||
),
|
||||
};
|
||||
|
||||
data.insert(
|
||||
key.clone(),
|
||||
MmFieldElem {
|
||||
data: Some(value.try_into()?),
|
||||
field,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
items.push(PreparedItem {
|
||||
data,
|
||||
hash: frame.hash.clone(),
|
||||
uuid,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Shared lowering from batched preprocessor output to per-item engine kwargs.
|
||||
|
||||
use itertools::izip;
|
||||
use llm_multimodal::{FieldLayout, PreprocessedEncoderInputs};
|
||||
use vllm_engine_core_client::protocol::dtype::ModelDtype;
|
||||
use vllm_engine_core_client::protocol::multimodal::{
|
||||
MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice,
|
||||
SliceSpec,
|
||||
};
|
||||
|
||||
use super::{PreparedItem, ResolvedMultimodalSpec, tensor};
|
||||
use crate::error::{Result, bail_multimodal, multimodal};
|
||||
|
||||
/// Split one batch of preprocessed tensors into engine kwargs per media item.
|
||||
pub(super) fn build_batched_items(
|
||||
spec: &ResolvedMultimodalSpec,
|
||||
preprocessed: PreprocessedEncoderInputs,
|
||||
hashes: Vec<String>,
|
||||
uuids: Vec<Option<String>>,
|
||||
float_dtype: ModelDtype,
|
||||
) -> Result<Vec<PreparedItem>> {
|
||||
let len = hashes.len();
|
||||
if uuids.len() != len {
|
||||
bail_multimodal!(
|
||||
"number of media UUIDs {} does not match number of media items {len}",
|
||||
uuids.len()
|
||||
);
|
||||
}
|
||||
let tensors = tensor::collect_tensors(preprocessed, spec.primary_key(), float_dtype)?;
|
||||
|
||||
let mut items = Vec::with_capacity(len);
|
||||
for (index, (hash, uuid)) in izip!(hashes, uuids).enumerate() {
|
||||
let mut data = MmKwargsItem::new();
|
||||
for (key, tensor) in &tensors {
|
||||
let keep_on_cpu = spec.keep_on_cpu_keys.contains(key);
|
||||
let (value, field) = match spec.field_layout_for(key) {
|
||||
Some(FieldLayout::Batched) => (
|
||||
tensor.batched_value_at(index)?,
|
||||
MmField::Batched(MmBatchedField { keep_on_cpu }),
|
||||
),
|
||||
Some(FieldLayout::Flat { sizes_key }) => {
|
||||
let sizes = tensors.get(sizes_key).ok_or_else(|| {
|
||||
multimodal!("flat tensor sizes key `{sizes_key}` is missing")
|
||||
})?;
|
||||
let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?;
|
||||
(
|
||||
tensor.flat_value_range(start, end)?,
|
||||
MmField::Flat(MmFlatField {
|
||||
slices: vec![MmSlice::Slice(SliceSpec {
|
||||
start: Some(0),
|
||||
stop: Some((end - start) as isize),
|
||||
step: None,
|
||||
})],
|
||||
dim: 0,
|
||||
keep_on_cpu,
|
||||
}),
|
||||
)
|
||||
}
|
||||
None => (
|
||||
tensor.clone(),
|
||||
MmField::Shared(MmSharedField {
|
||||
batch_size: len,
|
||||
keep_on_cpu,
|
||||
}),
|
||||
),
|
||||
};
|
||||
|
||||
data.insert(
|
||||
key.clone(),
|
||||
MmFieldElem {
|
||||
data: Some(value.try_into()?),
|
||||
field,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
items.push(PreparedItem { data, hash, uuid });
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
@@ -23,7 +23,7 @@ use crate::error::{Error, Result, bail_multimodal, multimodal};
|
||||
///
|
||||
/// Video-capable vLLM models read `pixel_values_videos` alongside
|
||||
/// `video_grid_thw`, mirroring the HF processor output naming.
|
||||
const VIDEO_PRIMARY_KEY: &str = "pixel_values_videos";
|
||||
pub(super) const VIDEO_PRIMARY_KEY: &str = "pixel_values_videos";
|
||||
|
||||
impl MultimodalModelInfo {
|
||||
/// Preprocess fetched video clips one at a time and build per-item
|
||||
@@ -47,8 +47,7 @@ impl MultimodalModelInfo {
|
||||
for (clip, uuid) in izip!(&clips, uuids) {
|
||||
let preprocessed = self.preprocess_video_clip(support, Arc::clone(clip)).await?;
|
||||
let mut clip_replacements =
|
||||
self.spec
|
||||
.prompt_replacements_for(&self.context, &preprocessed, Modality::Video)?;
|
||||
support.spec.prompt_replacements_for(&self.context, &preprocessed)?;
|
||||
if clip_replacements.len() != 1 {
|
||||
bail_multimodal!(
|
||||
"expected exactly one prompt replacement per video clip, got {}",
|
||||
@@ -56,7 +55,8 @@ impl MultimodalModelInfo {
|
||||
);
|
||||
}
|
||||
replacements.push(clip_replacements.pop().unwrap());
|
||||
items.push(self.build_video_item(
|
||||
items.push(build_video_item(
|
||||
support,
|
||||
preprocessed,
|
||||
clip.hash.clone(),
|
||||
uuid,
|
||||
@@ -107,30 +107,36 @@ impl MultimodalModelInfo {
|
||||
.await
|
||||
.map_err(|error| multimodal!("video preprocessing task failed: {error}"))?
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert one preprocessed video clip into engine kwargs.
|
||||
///
|
||||
/// The clip is a batch of one, so no per-item slicing is required: the
|
||||
/// primary tensor ships as a full-range flat field (the engine re-batches
|
||||
/// flat fields by concatenating along the declared dim, matching vLLM's
|
||||
/// `flat_from_sizes` treatment of video patches), and batched metadata
|
||||
/// tensors drop their singleton batch axis.
|
||||
fn build_video_item(
|
||||
&self,
|
||||
preprocessed: PreprocessedEncoderInputs,
|
||||
hash: String,
|
||||
uuid: Option<String>,
|
||||
model_dtype: ModelDtype,
|
||||
) -> Result<PreparedItem> {
|
||||
let tensors = tensor::collect_tensors(preprocessed, VIDEO_PRIMARY_KEY, model_dtype)?;
|
||||
/// Convert one preprocessed video clip into engine kwargs.
|
||||
///
|
||||
/// The clip is a batch of one, so no per-item slicing is required: the
|
||||
/// primary tensor ships as a full-range flat field (the engine re-batches
|
||||
/// flat fields by concatenating along the declared dim, matching vLLM's
|
||||
/// `flat_from_sizes` treatment of video patches), and batched metadata
|
||||
/// tensors drop their singleton batch axis.
|
||||
fn build_video_item(
|
||||
support: &ModalitySupport,
|
||||
preprocessed: PreprocessedEncoderInputs,
|
||||
hash: String,
|
||||
uuid: Option<String>,
|
||||
model_dtype: ModelDtype,
|
||||
) -> Result<PreparedItem> {
|
||||
let tensors = tensor::collect_tensors(preprocessed, support.spec.primary_key(), model_dtype)?;
|
||||
|
||||
let mut data = MmKwargsItem::new();
|
||||
for (key, tensor) in tensors {
|
||||
let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(&key);
|
||||
let (value, field) = if key == VIDEO_PRIMARY_KEY {
|
||||
let mut data = MmKwargsItem::new();
|
||||
for (key, tensor) in tensors {
|
||||
let keep_on_cpu = support.spec.keep_on_cpu_keys.contains(&key);
|
||||
let (value, field) = match support.spec.field_layout_for(&key) {
|
||||
Some(FieldLayout::Batched) => (
|
||||
tensor.batched_value_at(0)?,
|
||||
MmField::Batched(MmBatchedField { keep_on_cpu }),
|
||||
),
|
||||
Some(FieldLayout::Flat { .. }) => {
|
||||
let len = tensor
|
||||
.first_dim()
|
||||
.ok_or_else(|| multimodal!("video encoder input `{key}` is not a tensor"))?;
|
||||
.ok_or_else(|| multimodal!("flat video input `{key}` is not a tensor"))?;
|
||||
(
|
||||
tensor,
|
||||
MmField::Flat(MmFlatField {
|
||||
@@ -143,35 +149,26 @@ impl MultimodalModelInfo {
|
||||
keep_on_cpu,
|
||||
}),
|
||||
)
|
||||
} else if matches!(
|
||||
self.spec.field_layouts.get(&key),
|
||||
Some(FieldLayout::Batched)
|
||||
) {
|
||||
(
|
||||
tensor.batched_value_at(0)?,
|
||||
MmField::Batched(MmBatchedField { keep_on_cpu }),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
tensor,
|
||||
MmField::Shared(MmSharedField {
|
||||
batch_size: 1,
|
||||
keep_on_cpu,
|
||||
}),
|
||||
)
|
||||
};
|
||||
}
|
||||
None => (
|
||||
tensor,
|
||||
MmField::Shared(MmSharedField {
|
||||
batch_size: 1,
|
||||
keep_on_cpu,
|
||||
}),
|
||||
),
|
||||
};
|
||||
|
||||
data.insert(
|
||||
key,
|
||||
MmFieldElem {
|
||||
data: Some(value.try_into()?),
|
||||
field,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
Ok(PreparedItem { data, hash, uuid })
|
||||
data.insert(
|
||||
key,
|
||||
MmFieldElem {
|
||||
data: Some(value.try_into()?),
|
||||
field,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
Ok(PreparedItem { data, hash, uuid })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -287,14 +284,14 @@ mod tests {
|
||||
]),
|
||||
};
|
||||
|
||||
let item = info
|
||||
.build_video_item(
|
||||
preprocessed,
|
||||
"<hash>".to_string(),
|
||||
None,
|
||||
ModelDtype::Float32,
|
||||
)
|
||||
.unwrap();
|
||||
let item = build_video_item(
|
||||
info.video.as_ref().unwrap(),
|
||||
preprocessed,
|
||||
"<hash>".to_string(),
|
||||
None,
|
||||
ModelDtype::Float32,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let primary = &item.data[VIDEO_PRIMARY_KEY];
|
||||
assert!(matches!(
|
||||
|
||||
@@ -42,6 +42,7 @@ pub use self::format::ChatTemplateContentFormatOption;
|
||||
pub struct MultimodalRenderInfo {
|
||||
pub image_token: Option<String>,
|
||||
pub video_token: Option<String>,
|
||||
pub audio_token: Option<String>,
|
||||
}
|
||||
|
||||
/// Hugging Face chat-template renderer backed by the local Jinja chat-template
|
||||
@@ -263,6 +264,7 @@ enum TemplateContentPart {
|
||||
Text { text: String },
|
||||
Image,
|
||||
Video,
|
||||
Audio,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
@@ -437,6 +439,12 @@ fn to_template_openai_content(
|
||||
.ok_or(Error::UnsupportedMultimodalContent("video_url"))?;
|
||||
Ok(TemplateContentPart::Video)
|
||||
}
|
||||
ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => {
|
||||
multimodal
|
||||
.and_then(|multimodal| multimodal.audio_token.as_ref())
|
||||
.ok_or(Error::UnsupportedMultimodalContent("audio"))?;
|
||||
Ok(TemplateContentPart::Audio)
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
@@ -465,6 +473,12 @@ fn to_template_string_content(
|
||||
.ok_or(Error::UnsupportedMultimodalContent("video_url"))?;
|
||||
out.push_str(video_token);
|
||||
}
|
||||
ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => {
|
||||
let audio_token = multimodal
|
||||
.and_then(|multimodal| multimodal.audio_token.as_ref())
|
||||
.ok_or(Error::UnsupportedMultimodalContent("audio"))?;
|
||||
out.push_str(audio_token);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
@@ -492,7 +506,9 @@ fn append_continue_final_message_tag(message: &mut TemplateMessage) -> Result<St
|
||||
// Pick the last text part in the message.
|
||||
TemplateContent::OpenAi(parts) => parts.iter_mut().rev().find_map(|part| match part {
|
||||
TemplateContentPart::Text { text } => Some(text),
|
||||
TemplateContentPart::Image | TemplateContentPart::Video => None,
|
||||
TemplateContentPart::Image
|
||||
| TemplateContentPart::Video
|
||||
| TemplateContentPart::Audio => None,
|
||||
}),
|
||||
};
|
||||
let text = text.ok_or_else(|| {
|
||||
@@ -603,6 +619,7 @@ mod tests {
|
||||
.with_multimodal(Some(MultimodalRenderInfo {
|
||||
image_token: Some("<image>".to_string()),
|
||||
video_token: Some("<video>".to_string()),
|
||||
audio_token: Some("<audio>".to_string()),
|
||||
}))
|
||||
.render(request)
|
||||
}
|
||||
@@ -623,6 +640,15 @@ mod tests {
|
||||
])])
|
||||
}
|
||||
|
||||
fn audio_request() -> ChatRequest {
|
||||
sample_request(vec![ChatMessage::user(vec![
|
||||
ChatContentPart::text("a"),
|
||||
ChatContentPart::input_audio("dGVzdA==", Some("wav".to_string())),
|
||||
ChatContentPart::audio_url("https://example.com/demo.mp3"),
|
||||
ChatContentPart::text("b"),
|
||||
])])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn string_content_format_replaces_image_with_placeholder_text() {
|
||||
let rendered = render_mm(
|
||||
@@ -647,6 +673,21 @@ mod tests {
|
||||
assert_eq!(rendered.prompt, Prompt::Text("a<video>b".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn string_content_format_replaces_audio_with_placeholder_text() {
|
||||
let rendered = render_mm(
|
||||
"{{ messages[0].content }}",
|
||||
&audio_request(),
|
||||
ChatTemplateContentFormatOption::String,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
rendered.prompt,
|
||||
Prompt::Text("a<audio><audio>b".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_content_format_normalizes_image_url_for_template() {
|
||||
let rendered = render_mm(
|
||||
@@ -671,6 +712,21 @@ mod tests {
|
||||
assert_eq!(rendered.prompt, Prompt::Text("a<|video_pad|>b".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_content_format_normalizes_audio_for_template() {
|
||||
let rendered = render_mm(
|
||||
"{% for item in messages[0].content %}{% if item.type == 'audio' %}<|audio_pad|>{% else %}{{ item.text }}{% endif %}{% endfor %}",
|
||||
&audio_request(),
|
||||
ChatTemplateContentFormatOption::OpenAi,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
rendered.prompt,
|
||||
Prompt::Text("a<|audio_pad|><|audio_pad|>b".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_parts_are_rejected_when_model_lacks_video_support() {
|
||||
let error = HfChatRenderer::new(
|
||||
@@ -682,6 +738,7 @@ mod tests {
|
||||
.with_multimodal(Some(MultimodalRenderInfo {
|
||||
image_token: Some("<image>".to_string()),
|
||||
video_token: None,
|
||||
audio_token: None,
|
||||
}))
|
||||
.render(&video_request())
|
||||
.unwrap_err();
|
||||
|
||||
@@ -44,6 +44,17 @@ pub enum ChatContentPart {
|
||||
video_url: String,
|
||||
uuid: Option<String>,
|
||||
},
|
||||
/// One `input_audio` content block carrying base64-encoded audio bytes.
|
||||
InputAudio {
|
||||
data: String,
|
||||
format: Option<String>,
|
||||
uuid: Option<String>,
|
||||
},
|
||||
/// One audio URL/data URL content block.
|
||||
AudioUrl {
|
||||
audio_url: String,
|
||||
uuid: Option<String>,
|
||||
},
|
||||
// ImageData...
|
||||
// VideoData...
|
||||
// ImageEmbeds...
|
||||
@@ -72,6 +83,23 @@ impl ChatContentPart {
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct one base64-encoded input-audio content part.
|
||||
pub fn input_audio(data: impl Into<String>, format: Option<String>) -> Self {
|
||||
Self::InputAudio {
|
||||
data: data.into(),
|
||||
format,
|
||||
uuid: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct one audio URL content part with the given URL string.
|
||||
pub fn audio_url(audio_url: impl Into<String>) -> Self {
|
||||
Self::AudioUrl {
|
||||
audio_url: audio_url.into(),
|
||||
uuid: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the text content of this part when it's a text block, or an
|
||||
/// "unsupported multimodal content" error otherwise.
|
||||
pub(crate) fn as_text(&self) -> Result<&str> {
|
||||
@@ -79,6 +107,8 @@ impl ChatContentPart {
|
||||
Self::Text { text } => Ok(text),
|
||||
Self::ImageUrl { .. } => Err(Error::UnsupportedMultimodalContent("image_url")),
|
||||
Self::VideoUrl { .. } => Err(Error::UnsupportedMultimodalContent("video_url")),
|
||||
Self::InputAudio { .. } => Err(Error::UnsupportedMultimodalContent("input_audio")),
|
||||
Self::AudioUrl { .. } => Err(Error::UnsupportedMultimodalContent("audio_url")),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,7 +121,10 @@ impl ChatContentPart {
|
||||
pub(crate) fn is_multimodal(&self) -> bool {
|
||||
match self {
|
||||
Self::Text { .. } => false,
|
||||
Self::ImageUrl { .. } | Self::VideoUrl { .. } => true,
|
||||
Self::ImageUrl { .. }
|
||||
| Self::VideoUrl { .. }
|
||||
| Self::InputAudio { .. }
|
||||
| Self::AudioUrl { .. } => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -299,6 +299,15 @@ fn convert_content(content: MessageContent) -> Result<ChatContent, ApiError> {
|
||||
video_url: video_url.url,
|
||||
uuid,
|
||||
}),
|
||||
ContentPart::AudioUrl { audio_url, uuid } => Ok(ChatContentPart::AudioUrl {
|
||||
audio_url: audio_url.url,
|
||||
uuid,
|
||||
}),
|
||||
ContentPart::InputAudio { input_audio, uuid } => Ok(ChatContentPart::InputAudio {
|
||||
data: input_audio.data,
|
||||
format: input_audio.format,
|
||||
uuid,
|
||||
}),
|
||||
})
|
||||
.try_collect()
|
||||
.map(ChatContent::Parts),
|
||||
@@ -402,8 +411,8 @@ mod tests {
|
||||
AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
|
||||
};
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, MessageContent,
|
||||
StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
|
||||
AudioUrl, ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, InputAudio,
|
||||
MessageContent, StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
|
||||
};
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context};
|
||||
|
||||
@@ -810,6 +819,51 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_accepts_audio_content_parts() {
|
||||
let request = ChatCompletionRequest {
|
||||
model: "Qwen/Qwen3-ASR-1.7B".to_string(),
|
||||
messages: vec![ChatMessage::User {
|
||||
content: MessageContent::Parts(vec![
|
||||
ContentPart::InputAudio {
|
||||
input_audio: InputAudio {
|
||||
data: "dGVzdA==".to_string(),
|
||||
format: Some("wav".to_string()),
|
||||
},
|
||||
uuid: Some("audio-1".to_string()),
|
||||
},
|
||||
ContentPart::AudioUrl {
|
||||
audio_url: AudioUrl {
|
||||
url: "https://example.com/audio.mp3".to_string(),
|
||||
},
|
||||
uuid: None,
|
||||
},
|
||||
]),
|
||||
name: None,
|
||||
}],
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen3-ASR-1.7B"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(
|
||||
prepared.chat_request.messages,
|
||||
vec![VllmChatMessage::user(vec![
|
||||
ChatContentPart::InputAudio {
|
||||
data: "dGVzdA==".to_string(),
|
||||
format: Some("wav".to_string()),
|
||||
uuid: Some("audio-1".to_string()),
|
||||
},
|
||||
ChatContentPart::audio_url("https://example.com/audio.mp3"),
|
||||
])]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_rejects_assistant_image_url_content_parts() {
|
||||
let request = ChatCompletionRequest {
|
||||
|
||||
@@ -99,6 +99,18 @@ pub enum ContentPart {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
uuid: Option<String>,
|
||||
},
|
||||
#[serde(rename = "audio_url")]
|
||||
AudioUrl {
|
||||
audio_url: AudioUrl,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
uuid: Option<String>,
|
||||
},
|
||||
#[serde(rename = "input_audio")]
|
||||
InputAudio {
|
||||
input_audio: InputAudio,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
uuid: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
@@ -113,6 +125,19 @@ pub struct VideoUrl {
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct AudioUrl {
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
/// Base64-encoded audio bytes in OpenAI `input_audio` form.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct InputAudio {
|
||||
pub data: String,
|
||||
pub format: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Streaming
|
||||
// ============================================================================
|
||||
|
||||
@@ -551,9 +551,10 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R
|
||||
for part in parts {
|
||||
match part {
|
||||
ChatContentPart::Text { text } => out.push_str(text),
|
||||
ChatContentPart::ImageUrl { .. } | ChatContentPart::VideoUrl { .. } => {
|
||||
out.push_str(placeholder)
|
||||
}
|
||||
ChatContentPart::ImageUrl { .. }
|
||||
| ChatContentPart::VideoUrl { .. }
|
||||
| ChatContentPart::InputAudio { .. }
|
||||
| ChatContentPart::AudioUrl { .. } => out.push_str(placeholder),
|
||||
}
|
||||
}
|
||||
out
|
||||
|
||||
Reference in New Issue
Block a user