[Rust Frontend] Integrate MM audio support (#48554)

Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Bugen Zhao
2026-07-15 15:00:17 +08:00
committed by GitHub
co-authored by Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent adce068118
commit e281ac663a
18 changed files with 1275 additions and 258 deletions
+1 -1
View File
@@ -16,7 +16,7 @@ llm-multimodal.workspace = true
minijinja.workspace = true
minijinja-contrib.workspace = true
openai-harmony.workspace = true
reqwest.workspace = true
reqwest-0-13.workspace = true
serde.workspace = true
serde-json-fmt.workspace = true
serde_json.workspace = true
+1
View File
@@ -151,6 +151,7 @@ fn resolve_multimodal_render_info(
info.map(|info| MultimodalRenderInfo {
image_token: info.placeholder_token(Modality::Image).map(str::to_string),
video_token: info.placeholder_token(Modality::Video).map(str::to_string),
audio_token: info.placeholder_token(Modality::Audio).map(str::to_string),
})
}
+203 -92
View File
@@ -5,24 +5,24 @@
//!
//! This module owns the multimodal path for chat requests: it extracts media
//! parts from structured chat messages, fetches and preprocesses them through
//! `llm-multimodal` one modality at a time, expands rendered prompt
//! placeholders after tokenization, and builds the engine-facing
//! `MmFeatures` payload.
//! `llm-multimodal`, expands rendered prompt placeholders after tokenization,
//! and builds the engine-facing `MmFeatures` payload.
//!
//! Raw media stays above `vllm-text`; this module lowers it into token IDs and
//! opaque tensor payloads before the request is handed to text generation.
use std::collections::{HashMap, HashSet};
use std::collections::HashSet;
use std::fs;
use std::path::Path;
use std::sync::{Arc, LazyLock};
use itertools::izip;
use llm_multimodal::{
AsyncMultiModalTracker, FieldLayout, ImageFrame, MediaConnector, MediaConnectorConfig,
MediaContentPart, Modality, ModelMetadata, ModelProcessorSpec, ModelRegistry,
PreProcessorConfig, PreprocessedEncoderInputs, PromptReplacement, Tokenizer as TokenResolver,
TrackedMedia, VideoClip, VisionPreProcessor, VisionProcessorRegistry,
AsyncMultiModalTracker, AudioClip, AudioPreProcessor, EncoderFieldLayouts, FieldLayout,
ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata,
ModelProcessorSpec, ModelRegistry, PreProcessorConfig, PreprocessedEncoderInputs,
PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor,
VisionProcessorRegistry,
};
use thiserror_ext::AsReport as _;
use tracing::warn;
@@ -35,8 +35,10 @@ use crate::error::{Error, Result, bail_multimodal, multimodal};
use crate::renderer::RenderedPrompt;
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
mod audio;
mod expand;
mod image;
mod item;
mod tensor;
mod video;
@@ -46,9 +48,9 @@ use self::expand::expand_prompt_token_ids;
#[derive(Clone)]
pub struct MultimodalModelInfo {
context: MultimodalModelContext,
spec: ResolvedMultimodalSpec,
image: Option<ModalitySupport>,
video: Option<ModalitySupport>,
audio: Option<AudioModalitySupport>,
media_connector: Arc<MediaConnector>,
}
@@ -88,22 +90,50 @@ impl MultimodalModelContext {
LazyLock::new(VisionProcessorRegistry::with_defaults);
REGISTRY.find(&self.model_id, self.model_type.as_deref())
}
/// Resolve an audio preprocessor for one loaded model.
fn resolve_audio_processor(
&self,
model_spec: &'static dyn ModelProcessorSpec,
preprocessor_config: &PreProcessorConfig,
) -> Option<Arc<dyn AudioPreProcessor>> {
model_spec.audio_processor(&self.config, preprocessor_config).map(Arc::from)
}
}
/// Static model-specific tensor-layout behavior shared across modalities.
/// Static model-specific tensor-layout behavior for one modality.
#[derive(Clone)]
struct ResolvedMultimodalSpec {
raw: &'static dyn ModelProcessorSpec,
field_layouts: HashMap<String, FieldLayout>,
modality: Modality,
field_layouts: EncoderFieldLayouts,
keep_on_cpu_keys: HashSet<String>,
}
impl ResolvedMultimodalSpec {
fn new(raw: &'static dyn ModelProcessorSpec) -> Self {
fn new(raw: &'static dyn ModelProcessorSpec, modality: Modality) -> Self {
Self {
raw,
field_layouts: raw.field_layouts(),
keep_on_cpu_keys: raw.keep_on_cpu_keys().into_iter().collect(),
modality,
field_layouts: raw.encoder_field_layouts_for(modality),
keep_on_cpu_keys: raw.keep_on_cpu_keys_for(modality).into_iter().collect(),
}
}
fn primary_key(&self) -> &'static str {
match self.modality {
Modality::Image => image::IMAGE_PRIMARY_KEY,
Modality::Video => video::VIDEO_PRIMARY_KEY,
Modality::Audio => audio::AUDIO_PRIMARY_KEY,
Modality::ImageEmbeds => unreachable!("image embeds use no encoder preprocessor"),
}
}
fn field_layout_for(&self, key: &str) -> Option<&FieldLayout> {
if key == self.primary_key() {
Some(&self.field_layouts.encoder_input)
} else {
self.field_layouts.model_specific.get(key)
}
}
@@ -111,14 +141,15 @@ impl ResolvedMultimodalSpec {
&self,
context: &MultimodalModelContext,
preprocessed: &PreprocessedEncoderInputs,
modality: Modality,
) -> Result<Vec<PromptReplacement>> {
Ok(self.raw.prompt_replacements_for(&context.metadata(), preprocessed, modality)?)
Ok(self
.raw
.prompt_replacements_for(&context.metadata(), preprocessed, self.modality)?)
}
}
/// Resolved placeholder tokens for one modality.
#[derive(Clone)]
#[derive(Debug, Clone)]
struct ResolvedPlaceholder {
token: String,
/// The token ID emitted for `token` in the rendered prompt.
@@ -152,19 +183,30 @@ impl ResolvedPlaceholder {
}
}
/// Static per-modality vision preprocessor plus its loaded config and
/// resolved placeholder tokens.
/// Static per-modality vision preprocessor plus its loaded config, resolved
/// placeholder tokens, and the model's shared tensor-layout spec.
#[derive(Clone)]
struct ModalitySupport {
spec: ResolvedMultimodalSpec,
placeholder: ResolvedPlaceholder,
processor: &'static dyn VisionPreProcessor,
config: PreProcessorConfig,
}
/// Static audio preprocessor plus the resolved model contract for its output.
#[derive(Clone)]
struct AudioModalitySupport {
spec: ResolvedMultimodalSpec,
placeholder: ResolvedPlaceholder,
processor: Arc<dyn AudioPreProcessor>,
}
/// Model-repo config file locations consumed by multimodal support.
#[derive(Debug, Default, Clone, Copy)]
pub struct MultimodalConfigFiles<'a> {
pub config: Option<&'a Path>,
/// Shared preprocessor config (`preprocessor_config.json`), consumed by
/// image/audio processors and used as the video fallback.
pub preprocessor_config: Option<&'a Path>,
/// Video-specific preprocessor config (`video_preprocessor_config.json`).
pub video_preprocessor_config: Option<&'a Path>,
@@ -212,6 +254,8 @@ struct FetchedMedia {
image_uuids: Vec<Option<String>>,
videos: Vec<Arc<VideoClip>>,
video_uuids: Vec<Option<String>>,
audios: Vec<Arc<AudioClip>>,
audio_uuids: Vec<Option<String>>,
}
/// One modality's preprocessed output, ready for the shared expansion and
@@ -253,7 +297,7 @@ impl MultimodalModelInfo {
}
None => serde_json::Value::Object(Default::default()),
};
let image_preprocessor_config = load_preprocessor_config(
let preprocessor_config = load_preprocessor_config(
files.preprocessor_config,
"preprocessor_config.json",
files.processor_config,
@@ -266,7 +310,7 @@ impl MultimodalModelInfo {
files.processor_config,
"video_processor",
)?
.unwrap_or_else(|| image_preprocessor_config.clone());
.unwrap_or_else(|| preprocessor_config.clone());
let context = MultimodalModelContext {
model_id,
@@ -275,41 +319,76 @@ impl MultimodalModelInfo {
tokenizer: TokenizerResolver(tokenizer),
};
Self::from_loaded(
context,
image_preprocessor_config,
video_preprocessor_config,
)
Self::from_loaded(context, preprocessor_config, video_preprocessor_config)
}
/// Resolve multimodal support from an assembled context and parsed
/// preprocessor configs.
fn from_loaded(
context: MultimodalModelContext,
image_preprocessor_config: PreProcessorConfig,
preprocessor_config: PreProcessorConfig,
video_preprocessor_config: PreProcessorConfig,
) -> Result<Option<Self>> {
let (image, video) = Self::resolve_vision_lanes(
&context,
preprocessor_config.clone(),
video_preprocessor_config,
);
let audio = Self::resolve_audio_lane(&context, &preprocessor_config)?;
if image.is_none() && video.is_none() && audio.is_none() {
warn!(
model_id = context.model_id,
model_type = context.model_type,
"no multimodal modality resolved; disabling multimodal support for this model"
);
return Ok(None);
}
let media_connector = Arc::new(MediaConnector::new(
reqwest_0_13::Client::new(),
MediaConnectorConfig::default(),
)?);
Ok(Some(Self {
context,
image,
video,
audio,
media_connector,
}))
}
/// Resolve the spec-backed vision lanes (image and video).
///
/// A missing model spec or vision processor disables both lanes but is
/// not an error: unsupported model families are logged and skipped.
fn resolve_vision_lanes(
context: &MultimodalModelContext,
preprocessor_config: PreProcessorConfig,
video_preprocessor_config: PreProcessorConfig,
) -> (Option<ModalitySupport>, Option<ModalitySupport>) {
let Some(raw_spec) = context.resolve_model_spec() else {
warn!(
model_id = context.model_id,
model_type = context.model_type,
"multimodal model spec is not registered; disabling multimodal support for this model"
"multimodal model spec is not registered; disabling image/video support for this model"
);
return Ok(None);
return (None, None);
};
let Some(processor) = context.resolve_vision_processor() else {
warn!(
model_id = context.model_id,
model_type = context.model_type,
"vision processor is not registered; disabling multimodal support for this model"
"vision processor is not registered; disabling image/video support for this model"
);
return Ok(None);
return (None, None);
};
// Warn and disable the modality if the placeholder resolution fails.
let resolve_placeholder =
|modality: Modality| match ResolvedPlaceholder::resolve(raw_spec, &context, modality) {
|modality: Modality| match ResolvedPlaceholder::resolve(raw_spec, context, modality) {
Ok(placeholder) => Some(placeholder),
Err(error) => {
warn!(
@@ -323,9 +402,10 @@ impl MultimodalModelInfo {
};
let image = resolve_placeholder(Modality::Image).map(|placeholder| ModalitySupport {
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Image),
placeholder,
processor,
config: image_preprocessor_config,
config: preprocessor_config.clone(),
});
let video = resolve_placeholder(Modality::Video).and_then(|placeholder| {
@@ -341,6 +421,7 @@ impl MultimodalModelInfo {
None
} else {
Some(ModalitySupport {
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Video),
placeholder,
processor,
config: video_preprocessor_config,
@@ -348,26 +429,33 @@ impl MultimodalModelInfo {
}
});
if image.is_none() && video.is_none() {
warn!(
model_id = context.model_id,
model_type = context.model_type,
"no multimodal modality resolved; disabling multimodal support for this model"
);
(image, video)
}
/// Resolve a spec-backed audio lane when the model advertises audio support.
fn resolve_audio_lane(
context: &MultimodalModelContext,
preprocessor_config: &PreProcessorConfig,
) -> Result<Option<AudioModalitySupport>> {
let Some(raw_spec) = context.resolve_model_spec() else {
return Ok(None);
};
if !raw_spec.modality_limits(&context.metadata())?.contains_key(&Modality::Audio) {
return Ok(None);
}
let media_connector = Arc::new(MediaConnector::new(
reqwest::Client::new(),
MediaConnectorConfig::default(),
)?);
Ok(Some(Self {
context,
spec: ResolvedMultimodalSpec::new(raw_spec),
image,
video,
media_connector,
let Some(processor) = context.resolve_audio_processor(raw_spec, preprocessor_config) else {
warn!(
model_id = context.model_id,
model_spec = raw_spec.name(),
"model spec does not provide an audio processor; disabling audio support for this model"
);
return Ok(None);
};
Ok(Some(AudioModalitySupport {
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Audio),
placeholder: ResolvedPlaceholder::resolve(raw_spec, context, Modality::Audio)?,
processor,
}))
}
@@ -380,7 +468,8 @@ impl MultimodalModelInfo {
match modality {
Modality::Image => self.image.as_ref()?.placeholder.token.as_str().into(),
Modality::Video => self.video.as_ref()?.placeholder.token.as_str().into(),
_ => None,
Modality::Audio => self.audio.as_ref()?.placeholder.token.as_str().into(),
Modality::ImageEmbeds => None,
}
}
}
@@ -449,12 +538,35 @@ fn extract_media_parts(request: &ChatRequest) -> Result<Vec<MediaContentPart>> {
uuid: uuid.clone(),
})
}
ChatContentPart::InputAudio { data, format, uuid } => {
all_parts.push(MediaContentPart::AudioUrl {
url: input_audio_data_url(data, format.as_deref())?,
uuid: uuid.clone(),
})
}
ChatContentPart::AudioUrl { audio_url, uuid } => {
all_parts.push(MediaContentPart::AudioUrl {
url: audio_url.clone(),
uuid: uuid.clone(),
})
}
}
}
}
Ok(all_parts)
}
/// Wrap OpenAI base64 audio in a data URL consumed by `MediaConnector`.
fn input_audio_data_url(data: &str, format: Option<&str>) -> Result<String> {
let mime_type = match format {
Some(format) if format.eq_ignore_ascii_case("wav") => "audio/wav",
Some(format) if format.eq_ignore_ascii_case("mp3") => "audio/mpeg",
Some(format) => bail_multimodal!("unsupported input_audio format `{format}`"),
None => "application/octet-stream",
};
Ok(format!("data:{mime_type};base64,{data}"))
}
impl MultimodalModelInfo {
/// Run media fetch, per-modality preprocessing, prompt expansion, and
/// feature build.
@@ -468,11 +580,10 @@ impl MultimodalModelInfo {
prompt_token_ids: &mut Vec<u32>,
model_dtype: ModelDtype,
) -> Result<MmFeatures> {
if media_parts.is_empty() {
let media_parts_len = media_parts.len();
if media_parts_len == 0 {
return Ok(Vec::new());
}
let media_parts_len = media_parts.len();
// TODO: enforce per-modality item-count limits, aligned with the
// engine's `--limit-mm-per-prompt` semantics.
let fetched = self.fetch_media(media_parts).await?;
@@ -486,6 +597,9 @@ impl MultimodalModelInfo {
prepared
.push(self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype).await?);
}
if !fetched.audios.is_empty() {
prepared.push(self.prepare_audios(fetched.audios, fetched.audio_uuids).await?);
}
let mut ranges = expand_prompt_token_ids(prompt_token_ids, &prepared)?;
@@ -524,8 +638,8 @@ impl MultimodalModelInfo {
Ok(features)
}
/// Fetch all media parts and split them per modality, preserving their
/// request-order UUID metadata.
/// Fetch all connector-backed media parts and split them per modality,
/// preserving their request-order UUID metadata.
async fn fetch_media(&self, media_parts: Vec<MediaContentPart>) -> Result<FetchedMedia> {
let mut tracker = AsyncMultiModalTracker::new(Arc::clone(&self.media_connector));
for part in media_parts {
@@ -562,11 +676,27 @@ impl MultimodalModelInfo {
.collect::<Result<Vec<_>>>()?;
let video_uuids = tracker_output.uuids.remove(&Modality::Video).unwrap_or_default();
let audios = tracker_output
.data
.remove(&Modality::Audio)
.unwrap_or_default()
.into_iter()
.map(|media| match media {
TrackedMedia::Audio(clip) => Ok(clip),
_ => Err(multimodal!(
"tracker returned non-audio media for the audio modality"
)),
})
.collect::<Result<Vec<_>>>()?;
let audio_uuids = tracker_output.uuids.remove(&Modality::Audio).unwrap_or_default();
Ok(FetchedMedia {
images,
image_uuids,
videos,
video_uuids,
audios,
audio_uuids,
})
}
}
@@ -665,42 +795,6 @@ mod tests {
test_info("qwen3_vl", config, qwen3_vl_tokenizer())
}
#[test]
fn from_paths_resolves_image_config_from_processor_config() {
let dir = tempfile::tempdir().unwrap();
let config_path = dir.path().join("config.json");
std::fs::write(
&config_path,
serde_json::json!({
"model_type": "qwen3_vl",
"image_token_id": QWEN3_IMAGE_PAD_ID,
})
.to_string(),
)
.unwrap();
let processor_config_path = dir.path().join("processor_config.json");
std::fs::write(
&processor_config_path,
r#"{"image_processor":{"size":{"shortest_edge":64}}}"#,
)
.unwrap();
let info = MultimodalModelInfo::from_paths(
"qwen3-vl-test".to_string(),
Some("qwen3_vl".to_string()),
MultimodalConfigFiles {
config: Some(&config_path),
processor_config: Some(&processor_config_path),
..Default::default()
},
Arc::new(qwen3_vl_tokenizer()),
)
.unwrap()
.unwrap();
assert_eq!(info.image.unwrap().config.get_shortest_edge(), Some(64));
}
#[test]
fn qwen3_vl_resolves_image_and_video_support() {
let info = qwen3_vl_info();
@@ -742,4 +836,21 @@ mod tests {
assert_eq!(info.placeholder_token(Modality::Image), Some("<|image|>"));
assert_eq!(info.placeholder_token(Modality::Video), None);
}
#[test]
fn input_audio_uses_connector_data_urls() {
assert_eq!(
input_audio_data_url("AAAA", Some("wav")).unwrap(),
"data:audio/wav;base64,AAAA"
);
assert_eq!(
input_audio_data_url("BBBB", Some("MP3")).unwrap(),
"data:audio/mpeg;base64,BBBB"
);
assert_eq!(
input_audio_data_url("CCCC", None).unwrap(),
"data:application/octet-stream;base64,CCCC"
);
assert!(input_audio_data_url("AAAA", Some("flac")).is_err());
}
}
+213
View File
@@ -0,0 +1,213 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Audio-modality preparation through `llm-multimodal`.
use std::sync::Arc;
use llm_multimodal::{AudioClip, Modality, PreprocessedEncoderInputs};
use vllm_engine_core_client::protocol::dtype::ModelDtype;
use super::{AudioModalitySupport, MultimodalModelInfo, PreparedMedia, item};
use crate::error::{Error, Result, bail_multimodal, multimodal};
/// Forward-kwargs name of the primary audio encoder input.
pub(super) const AUDIO_PRIMARY_KEY: &str = "input_audio_features";
impl MultimodalModelInfo {
/// Preprocess fetched audio clips as one batch and build per-item features.
pub(super) async fn prepare_audios(
&self,
clips: Vec<Arc<AudioClip>>,
uuids: Vec<Option<String>>,
) -> Result<PreparedMedia> {
let support = self.audio.as_ref().ok_or_else(|| Error::UnsupportedModality {
modality: Modality::Audio.to_string(),
})?;
let preprocessed = self.preprocess_audios(support, &clips).await?;
let replacements = support.spec.prompt_replacements_for(&self.context, &preprocessed)?;
if replacements.len() != clips.len() {
bail_multimodal!(
"number of audio prompt replacements {} does not match number of audio clips {}",
replacements.len(),
clips.len()
);
}
let hashes = clips.iter().map(|clip| clip.hash.clone()).collect();
let items = item::build_batched_items(
&support.spec,
preprocessed,
hashes,
uuids,
ModelDtype::Float32,
)?;
Ok(PreparedMedia {
modality: Modality::Audio,
placeholder: support.placeholder.clone(),
replacements,
items,
})
}
/// Run CPU-heavy audio preprocessing in a blocking task.
async fn preprocess_audios(
&self,
support: &AudioModalitySupport,
clips: &[Arc<AudioClip>],
) -> Result<PreprocessedEncoderInputs> {
let processor = Arc::clone(&support.processor);
let clips = clips.to_vec();
tokio::task::spawn_blocking(move || Ok(processor.preprocess(&clips)?))
.await
.map_err(|error| multimodal!("audio preprocessing task failed: {error}"))?
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::sync::Arc;
use llm_multimodal::{MediaContentPart, ModelSpecificValue, PreProcessorConfig};
use ndarray::ArrayD;
use vllm_engine_core_client::protocol::multimodal::{MmField, MmKwargValue};
use vllm_tokenizer::test_utils::TestTokenizer;
use super::super::{MultimodalModelContext, TokenizerResolver};
use super::*;
const AUDIO_PAD_ID: u32 = 151_676;
fn qwen3_asr_info() -> MultimodalModelInfo {
let context = MultimodalModelContext {
model_id: "Qwen/Qwen3-ASR-1.7B".to_string(),
model_type: Some("qwen3_asr".to_string()),
config: serde_json::json!({"model_type": "qwen3_asr"}),
tokenizer: TokenizerResolver(Arc::new(
TestTokenizer::new().with_regular_token("<|audio_pad|>", AUDIO_PAD_ID),
)),
};
MultimodalModelInfo::from_loaded(
context,
PreProcessorConfig::default(),
PreProcessorConfig::default(),
)
.unwrap()
.expect("Qwen3-ASR multimodal support")
}
fn wav_i16_mono(sample_rate: u32, samples: &[i16]) -> Vec<u8> {
let data_bytes = samples.len() as u32 * 2;
let mut bytes = Vec::new();
bytes.extend_from_slice(b"RIFF");
bytes.extend_from_slice(&(36 + data_bytes).to_le_bytes());
bytes.extend_from_slice(b"WAVEfmt ");
bytes.extend_from_slice(&16_u32.to_le_bytes());
bytes.extend_from_slice(&1_u16.to_le_bytes());
bytes.extend_from_slice(&1_u16.to_le_bytes());
bytes.extend_from_slice(&sample_rate.to_le_bytes());
bytes.extend_from_slice(&(sample_rate * 2).to_le_bytes());
bytes.extend_from_slice(&2_u16.to_le_bytes());
bytes.extend_from_slice(&16_u16.to_le_bytes());
bytes.extend_from_slice(b"data");
bytes.extend_from_slice(&data_bytes.to_le_bytes());
for sample in samples {
bytes.extend_from_slice(&sample.to_le_bytes());
}
bytes
}
#[test]
fn resolves_qwen_audio_from_model_spec() {
let info = qwen3_asr_info();
let support = info.audio.as_ref().expect("audio support");
assert_eq!(
info.placeholder_token(Modality::Audio),
Some("<|audio_pad|>")
);
assert_eq!(support.placeholder.marker_token_id, AUDIO_PAD_ID);
assert_eq!(support.placeholder.embed_token_id, AUDIO_PAD_ID);
assert!(matches!(
support.spec.field_layouts.encoder_input,
llm_multimodal::FieldLayout::Batched
));
let preprocessed = PreprocessedEncoderInputs {
encoder_input: ArrayD::zeros(vec![1, 128, 4]),
feature_token_counts: vec![1],
item_sizes: vec![(128, 4)],
model_specific: HashMap::from([
(
"feature_attention_mask".to_string(),
ModelSpecificValue::int_2d(vec![1; 4], 1, 4),
),
(
"audio_feature_lengths".to_string(),
ModelSpecificValue::int_1d(vec![4]),
),
]),
};
let item = item::build_batched_items(
&support.spec,
preprocessed,
vec!["<hash>".to_string()],
vec![None],
ModelDtype::Float32,
)
.unwrap()
.pop()
.unwrap();
assert!(matches!(
&item.data[AUDIO_PRIMARY_KEY].field,
MmField::Batched(_)
));
}
#[tokio::test]
async fn tracker_processor_and_lowering_preserve_audio_contract() {
let info = qwen3_asr_info();
let wav = wav_i16_mono(16_000, &[0; 1_600]);
let expected_hash = llm_multimodal::hasher::hash_audio(&wav);
let fetched = info
.fetch_media(vec![MediaContentPart::AudioData {
data: wav,
mime_type: Some("audio/wav".to_string()),
uuid: Some("audio-1".to_string()),
}])
.await
.unwrap();
let prepared = info.prepare_audios(fetched.audios, fetched.audio_uuids).await.unwrap();
assert_eq!(prepared.replacements.len(), 1);
assert!(
prepared.replacements[0]
.tokens
.iter()
.all(|token| *token == AUDIO_PAD_ID as i32)
);
let item = &prepared.items[0];
assert_eq!(item.hash, expected_hash);
assert_eq!(item.uuid.as_deref(), Some("audio-1"));
let features = &item.data[AUDIO_PRIMARY_KEY];
assert!(matches!(&features.field, MmField::Batched(_)));
assert!(matches!(
features.data.as_ref(),
Some(MmKwargValue::Tensor(tensor))
if tensor.dtype == "float32" && tensor.shape.first() == Some(&128)
));
let lengths = &item.data["audio_feature_lengths"];
assert!(matches!(&lengths.field, MmField::Batched(_)));
assert!(matches!(
lengths.data.as_ref(),
Some(MmKwargValue::Tensor(tensor))
if tensor.dtype == "int64" && tensor.shape.is_empty()
));
}
}
+61
View File
@@ -418,6 +418,67 @@ mod tests {
assert_bool_mask(&video_ranges[0], &[true, true, true, true]);
}
const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
prepared_media(
Modality::Audio,
"<|audio_pad|>",
QWEN3_AUDIO_PAD_ID,
QWEN3_AUDIO_PAD_ID,
replacements,
)
}
fn qwen3_audio_replacement(tokens: usize) -> PromptReplacement {
PromptReplacement::repeated(
Modality::Audio,
"<|audio_pad|>",
QWEN3_AUDIO_PAD_ID as TokenId,
tokens,
)
}
#[test]
fn expand_prompt_tokens_interleaves_audio_and_image_prepared_media() {
// Audio marker precedes the image marker: the image range offset must
// account for the earlier audio expansion growth.
let mut prompt_token_ids = vec![1, QWEN3_AUDIO_PAD_ID, 2, LLAMA4_IMAGE_ID, 3];
let prepared = vec![
llama4_prepared(vec![llama4_single_tile_replacement()]),
qwen3_audio_prepared(vec![qwen3_audio_replacement(3)]),
];
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
assert_eq!(
prompt_token_ids,
vec![
1,
QWEN3_AUDIO_PAD_ID,
QWEN3_AUDIO_PAD_ID,
QWEN3_AUDIO_PAD_ID,
2,
LLAMA4_IMAGE_START_ID,
LLAMA4_IMAGE_ID,
LLAMA4_PATCH_ID,
LLAMA4_PATCH_ID,
LLAMA4_IMAGE_END_ID,
3,
]
);
let audio_ranges = &ranges[&Modality::Audio];
assert_eq!(audio_ranges[0].offset, 1);
assert_eq!(audio_ranges[0].length, 3);
assert_bool_mask(&audio_ranges[0], &[true, true, true]);
let image_ranges = &ranges[&Modality::Image];
assert_eq!(image_ranges[0].offset, 5);
assert_eq!(image_ranges[0].length, 5);
assert_bool_mask(&image_ranges[0], &[false, false, true, true, false]);
}
#[test]
fn expand_prompt_tokens_error_names_modality_with_leftover_replacements() {
let mut prompt_token_ids = vec![1, QWEN3_IMAGE_PAD_ID, 2];
+9 -82
View File
@@ -6,17 +6,15 @@
use std::sync::Arc;
use itertools::izip;
use llm_multimodal::{FieldLayout, ImageFrame, Modality, PreprocessedEncoderInputs};
use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs};
use vllm_engine_core_client::protocol::dtype::ModelDtype;
use vllm_engine_core_client::protocol::multimodal::{
MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice,
SliceSpec,
};
use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor};
use super::{ModalitySupport, MultimodalModelInfo, PreparedMedia, item};
use crate::error::{Error, Result, bail_multimodal, multimodal};
/// Forward-kwargs name of the primary image encoder input.
pub(super) const IMAGE_PRIMARY_KEY: &str = "pixel_values";
impl MultimodalModelInfo {
/// Preprocess all fetched image frames as one batch and build per-item
/// features.
@@ -30,9 +28,7 @@ impl MultimodalModelInfo {
modality: Modality::Image.to_string(),
})?;
let preprocessed = self.preprocess_images(support, &frames).await?;
let replacements =
self.spec
.prompt_replacements_for(&self.context, &preprocessed, Modality::Image)?;
let replacements = support.spec.prompt_replacements_for(&self.context, &preprocessed)?;
if replacements.len() != frames.len() {
bail_multimodal!(
"number of image prompt replacements {} does not match number of images {}",
@@ -40,7 +36,9 @@ impl MultimodalModelInfo {
frames.len()
);
}
let items = self.build_image_items(preprocessed, &frames, uuids, model_dtype)?;
let hashes = frames.iter().map(|frame| frame.hash.clone()).collect();
let items =
item::build_batched_items(&support.spec, preprocessed, hashes, uuids, model_dtype)?;
Ok(PreparedMedia {
modality: Modality::Image,
@@ -70,75 +68,4 @@ impl MultimodalModelInfo {
.await
.map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
}
/// Convert one batch of preprocessed image tensors into per-item engine
/// kwargs.
///
/// Tensor fields are sliced per item according to the model spec's field
/// layout declarations.
fn build_image_items(
&self,
preprocessed: PreprocessedEncoderInputs,
frames: &[Arc<ImageFrame>],
uuids: Vec<Option<String>>,
model_dtype: ModelDtype,
) -> Result<Vec<PreparedItem>> {
let len = frames.len();
let tensors = tensor::collect_tensors(preprocessed, "pixel_values", model_dtype)?;
let mut items = Vec::with_capacity(len);
for (index, (frame, uuid)) in izip!(frames, uuids).enumerate() {
let mut data = MmKwargsItem::new();
for (key, tensor) in &tensors {
let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(key);
let (value, field) = match self.spec.field_layouts.get(key) {
Some(FieldLayout::Batched) => (
tensor.batched_value_at(index)?,
MmField::Batched(MmBatchedField { keep_on_cpu }),
),
Some(FieldLayout::Flat { sizes_key }) => {
let sizes = tensors.get(sizes_key).ok_or_else(|| {
multimodal!("flat tensor sizes key `{sizes_key}` is missing")
})?;
let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?;
(
tensor.flat_value_range(start, end)?,
MmField::Flat(MmFlatField {
slices: vec![MmSlice::Slice(SliceSpec {
start: Some(0),
stop: Some((end - start) as isize),
step: None,
})],
dim: 0,
keep_on_cpu,
}),
)
}
None => (
tensor.clone(),
MmField::Shared(MmSharedField {
batch_size: len,
keep_on_cpu,
}),
),
};
data.insert(
key.clone(),
MmFieldElem {
data: Some(value.try_into()?),
field,
},
);
}
items.push(PreparedItem {
data,
hash: frame.hash.clone(),
uuid,
});
}
Ok(items)
}
}
+84
View File
@@ -0,0 +1,84 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Shared lowering from batched preprocessor output to per-item engine kwargs.
use itertools::izip;
use llm_multimodal::{FieldLayout, PreprocessedEncoderInputs};
use vllm_engine_core_client::protocol::dtype::ModelDtype;
use vllm_engine_core_client::protocol::multimodal::{
MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice,
SliceSpec,
};
use super::{PreparedItem, ResolvedMultimodalSpec, tensor};
use crate::error::{Result, bail_multimodal, multimodal};
/// Split one batch of preprocessed tensors into engine kwargs per media item.
pub(super) fn build_batched_items(
spec: &ResolvedMultimodalSpec,
preprocessed: PreprocessedEncoderInputs,
hashes: Vec<String>,
uuids: Vec<Option<String>>,
float_dtype: ModelDtype,
) -> Result<Vec<PreparedItem>> {
let len = hashes.len();
if uuids.len() != len {
bail_multimodal!(
"number of media UUIDs {} does not match number of media items {len}",
uuids.len()
);
}
let tensors = tensor::collect_tensors(preprocessed, spec.primary_key(), float_dtype)?;
let mut items = Vec::with_capacity(len);
for (index, (hash, uuid)) in izip!(hashes, uuids).enumerate() {
let mut data = MmKwargsItem::new();
for (key, tensor) in &tensors {
let keep_on_cpu = spec.keep_on_cpu_keys.contains(key);
let (value, field) = match spec.field_layout_for(key) {
Some(FieldLayout::Batched) => (
tensor.batched_value_at(index)?,
MmField::Batched(MmBatchedField { keep_on_cpu }),
),
Some(FieldLayout::Flat { sizes_key }) => {
let sizes = tensors.get(sizes_key).ok_or_else(|| {
multimodal!("flat tensor sizes key `{sizes_key}` is missing")
})?;
let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?;
(
tensor.flat_value_range(start, end)?,
MmField::Flat(MmFlatField {
slices: vec![MmSlice::Slice(SliceSpec {
start: Some(0),
stop: Some((end - start) as isize),
step: None,
})],
dim: 0,
keep_on_cpu,
}),
)
}
None => (
tensor.clone(),
MmField::Shared(MmSharedField {
batch_size: len,
keep_on_cpu,
}),
),
};
data.insert(
key.clone(),
MmFieldElem {
data: Some(value.try_into()?),
field,
},
);
}
items.push(PreparedItem { data, hash, uuid });
}
Ok(items)
}
+56 -59
View File
@@ -23,7 +23,7 @@ use crate::error::{Error, Result, bail_multimodal, multimodal};
///
/// Video-capable vLLM models read `pixel_values_videos` alongside
/// `video_grid_thw`, mirroring the HF processor output naming.
const VIDEO_PRIMARY_KEY: &str = "pixel_values_videos";
pub(super) const VIDEO_PRIMARY_KEY: &str = "pixel_values_videos";
impl MultimodalModelInfo {
/// Preprocess fetched video clips one at a time and build per-item
@@ -47,8 +47,7 @@ impl MultimodalModelInfo {
for (clip, uuid) in izip!(&clips, uuids) {
let preprocessed = self.preprocess_video_clip(support, Arc::clone(clip)).await?;
let mut clip_replacements =
self.spec
.prompt_replacements_for(&self.context, &preprocessed, Modality::Video)?;
support.spec.prompt_replacements_for(&self.context, &preprocessed)?;
if clip_replacements.len() != 1 {
bail_multimodal!(
"expected exactly one prompt replacement per video clip, got {}",
@@ -56,7 +55,8 @@ impl MultimodalModelInfo {
);
}
replacements.push(clip_replacements.pop().unwrap());
items.push(self.build_video_item(
items.push(build_video_item(
support,
preprocessed,
clip.hash.clone(),
uuid,
@@ -107,30 +107,36 @@ impl MultimodalModelInfo {
.await
.map_err(|error| multimodal!("video preprocessing task failed: {error}"))?
}
}
/// Convert one preprocessed video clip into engine kwargs.
///
/// The clip is a batch of one, so no per-item slicing is required: the
/// primary tensor ships as a full-range flat field (the engine re-batches
/// flat fields by concatenating along the declared dim, matching vLLM's
/// `flat_from_sizes` treatment of video patches), and batched metadata
/// tensors drop their singleton batch axis.
fn build_video_item(
&self,
preprocessed: PreprocessedEncoderInputs,
hash: String,
uuid: Option<String>,
model_dtype: ModelDtype,
) -> Result<PreparedItem> {
let tensors = tensor::collect_tensors(preprocessed, VIDEO_PRIMARY_KEY, model_dtype)?;
/// Convert one preprocessed video clip into engine kwargs.
///
/// The clip is a batch of one, so no per-item slicing is required: the
/// primary tensor ships as a full-range flat field (the engine re-batches
/// flat fields by concatenating along the declared dim, matching vLLM's
/// `flat_from_sizes` treatment of video patches), and batched metadata
/// tensors drop their singleton batch axis.
fn build_video_item(
support: &ModalitySupport,
preprocessed: PreprocessedEncoderInputs,
hash: String,
uuid: Option<String>,
model_dtype: ModelDtype,
) -> Result<PreparedItem> {
let tensors = tensor::collect_tensors(preprocessed, support.spec.primary_key(), model_dtype)?;
let mut data = MmKwargsItem::new();
for (key, tensor) in tensors {
let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(&key);
let (value, field) = if key == VIDEO_PRIMARY_KEY {
let mut data = MmKwargsItem::new();
for (key, tensor) in tensors {
let keep_on_cpu = support.spec.keep_on_cpu_keys.contains(&key);
let (value, field) = match support.spec.field_layout_for(&key) {
Some(FieldLayout::Batched) => (
tensor.batched_value_at(0)?,
MmField::Batched(MmBatchedField { keep_on_cpu }),
),
Some(FieldLayout::Flat { .. }) => {
let len = tensor
.first_dim()
.ok_or_else(|| multimodal!("video encoder input `{key}` is not a tensor"))?;
.ok_or_else(|| multimodal!("flat video input `{key}` is not a tensor"))?;
(
tensor,
MmField::Flat(MmFlatField {
@@ -143,35 +149,26 @@ impl MultimodalModelInfo {
keep_on_cpu,
}),
)
} else if matches!(
self.spec.field_layouts.get(&key),
Some(FieldLayout::Batched)
) {
(
tensor.batched_value_at(0)?,
MmField::Batched(MmBatchedField { keep_on_cpu }),
)
} else {
(
tensor,
MmField::Shared(MmSharedField {
batch_size: 1,
keep_on_cpu,
}),
)
};
}
None => (
tensor,
MmField::Shared(MmSharedField {
batch_size: 1,
keep_on_cpu,
}),
),
};
data.insert(
key,
MmFieldElem {
data: Some(value.try_into()?),
field,
},
);
}
Ok(PreparedItem { data, hash, uuid })
data.insert(
key,
MmFieldElem {
data: Some(value.try_into()?),
field,
},
);
}
Ok(PreparedItem { data, hash, uuid })
}
#[cfg(test)]
@@ -287,14 +284,14 @@ mod tests {
]),
};
let item = info
.build_video_item(
preprocessed,
"<hash>".to_string(),
None,
ModelDtype::Float32,
)
.unwrap();
let item = build_video_item(
info.video.as_ref().unwrap(),
preprocessed,
"<hash>".to_string(),
None,
ModelDtype::Float32,
)
.unwrap();
let primary = &item.data[VIDEO_PRIMARY_KEY];
assert!(matches!(
+58 -1
View File
@@ -42,6 +42,7 @@ pub use self::format::ChatTemplateContentFormatOption;
pub struct MultimodalRenderInfo {
pub image_token: Option<String>,
pub video_token: Option<String>,
pub audio_token: Option<String>,
}
/// Hugging Face chat-template renderer backed by the local Jinja chat-template
@@ -263,6 +264,7 @@ enum TemplateContentPart {
Text { text: String },
Image,
Video,
Audio,
}
#[derive(Debug, Serialize)]
@@ -437,6 +439,12 @@ fn to_template_openai_content(
.ok_or(Error::UnsupportedMultimodalContent("video_url"))?;
Ok(TemplateContentPart::Video)
}
ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => {
multimodal
.and_then(|multimodal| multimodal.audio_token.as_ref())
.ok_or(Error::UnsupportedMultimodalContent("audio"))?;
Ok(TemplateContentPart::Audio)
}
})
.collect(),
}
@@ -465,6 +473,12 @@ fn to_template_string_content(
.ok_or(Error::UnsupportedMultimodalContent("video_url"))?;
out.push_str(video_token);
}
ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => {
let audio_token = multimodal
.and_then(|multimodal| multimodal.audio_token.as_ref())
.ok_or(Error::UnsupportedMultimodalContent("audio"))?;
out.push_str(audio_token);
}
}
}
Ok(out)
@@ -492,7 +506,9 @@ fn append_continue_final_message_tag(message: &mut TemplateMessage) -> Result<St
// Pick the last text part in the message.
TemplateContent::OpenAi(parts) => parts.iter_mut().rev().find_map(|part| match part {
TemplateContentPart::Text { text } => Some(text),
TemplateContentPart::Image | TemplateContentPart::Video => None,
TemplateContentPart::Image
| TemplateContentPart::Video
| TemplateContentPart::Audio => None,
}),
};
let text = text.ok_or_else(|| {
@@ -603,6 +619,7 @@ mod tests {
.with_multimodal(Some(MultimodalRenderInfo {
image_token: Some("<image>".to_string()),
video_token: Some("<video>".to_string()),
audio_token: Some("<audio>".to_string()),
}))
.render(request)
}
@@ -623,6 +640,15 @@ mod tests {
])])
}
fn audio_request() -> ChatRequest {
sample_request(vec![ChatMessage::user(vec![
ChatContentPart::text("a"),
ChatContentPart::input_audio("dGVzdA==", Some("wav".to_string())),
ChatContentPart::audio_url("https://example.com/demo.mp3"),
ChatContentPart::text("b"),
])])
}
#[test]
fn string_content_format_replaces_image_with_placeholder_text() {
let rendered = render_mm(
@@ -647,6 +673,21 @@ mod tests {
assert_eq!(rendered.prompt, Prompt::Text("a<video>b".to_string()));
}
#[test]
fn string_content_format_replaces_audio_with_placeholder_text() {
let rendered = render_mm(
"{{ messages[0].content }}",
&audio_request(),
ChatTemplateContentFormatOption::String,
)
.unwrap();
assert_eq!(
rendered.prompt,
Prompt::Text("a<audio><audio>b".to_string())
);
}
#[test]
fn openai_content_format_normalizes_image_url_for_template() {
let rendered = render_mm(
@@ -671,6 +712,21 @@ mod tests {
assert_eq!(rendered.prompt, Prompt::Text("a<|video_pad|>b".to_string()));
}
#[test]
fn openai_content_format_normalizes_audio_for_template() {
let rendered = render_mm(
"{% for item in messages[0].content %}{% if item.type == 'audio' %}<|audio_pad|>{% else %}{{ item.text }}{% endif %}{% endfor %}",
&audio_request(),
ChatTemplateContentFormatOption::OpenAi,
)
.unwrap();
assert_eq!(
rendered.prompt,
Prompt::Text("a<|audio_pad|><|audio_pad|>b".to_string())
);
}
#[test]
fn video_parts_are_rejected_when_model_lacks_video_support() {
let error = HfChatRenderer::new(
@@ -682,6 +738,7 @@ mod tests {
.with_multimodal(Some(MultimodalRenderInfo {
image_token: Some("<image>".to_string()),
video_token: None,
audio_token: None,
}))
.render(&video_request())
.unwrap_err();
+34 -1
View File
@@ -44,6 +44,17 @@ pub enum ChatContentPart {
video_url: String,
uuid: Option<String>,
},
/// One `input_audio` content block carrying base64-encoded audio bytes.
InputAudio {
data: String,
format: Option<String>,
uuid: Option<String>,
},
/// One audio URL/data URL content block.
AudioUrl {
audio_url: String,
uuid: Option<String>,
},
// ImageData...
// VideoData...
// ImageEmbeds...
@@ -72,6 +83,23 @@ impl ChatContentPart {
}
}
/// Construct one base64-encoded input-audio content part.
pub fn input_audio(data: impl Into<String>, format: Option<String>) -> Self {
Self::InputAudio {
data: data.into(),
format,
uuid: None,
}
}
/// Construct one audio URL content part with the given URL string.
pub fn audio_url(audio_url: impl Into<String>) -> Self {
Self::AudioUrl {
audio_url: audio_url.into(),
uuid: None,
}
}
/// Return the text content of this part when it's a text block, or an
/// "unsupported multimodal content" error otherwise.
pub(crate) fn as_text(&self) -> Result<&str> {
@@ -79,6 +107,8 @@ impl ChatContentPart {
Self::Text { text } => Ok(text),
Self::ImageUrl { .. } => Err(Error::UnsupportedMultimodalContent("image_url")),
Self::VideoUrl { .. } => Err(Error::UnsupportedMultimodalContent("video_url")),
Self::InputAudio { .. } => Err(Error::UnsupportedMultimodalContent("input_audio")),
Self::AudioUrl { .. } => Err(Error::UnsupportedMultimodalContent("audio_url")),
}
}
@@ -91,7 +121,10 @@ impl ChatContentPart {
pub(crate) fn is_multimodal(&self) -> bool {
match self {
Self::Text { .. } => false,
Self::ImageUrl { .. } | Self::VideoUrl { .. } => true,
Self::ImageUrl { .. }
| Self::VideoUrl { .. }
| Self::InputAudio { .. }
| Self::AudioUrl { .. } => true,
}
}
}
@@ -299,6 +299,15 @@ fn convert_content(content: MessageContent) -> Result<ChatContent, ApiError> {
video_url: video_url.url,
uuid,
}),
ContentPart::AudioUrl { audio_url, uuid } => Ok(ChatContentPart::AudioUrl {
audio_url: audio_url.url,
uuid,
}),
ContentPart::InputAudio { input_audio, uuid } => Ok(ChatContentPart::InputAudio {
data: input_audio.data,
format: input_audio.format,
uuid,
}),
})
.try_collect()
.map(ChatContent::Parts),
@@ -402,8 +411,8 @@ mod tests {
AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
};
use crate::routes::openai::utils::types::{
ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, MessageContent,
StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
AudioUrl, ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, InputAudio,
MessageContent, StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
};
use crate::utils::{ResolvedRequestContext, resolve_request_context};
@@ -810,6 +819,51 @@ mod tests {
);
}
#[test]
fn prepare_chat_request_accepts_audio_content_parts() {
let request = ChatCompletionRequest {
model: "Qwen/Qwen3-ASR-1.7B".to_string(),
messages: vec![ChatMessage::User {
content: MessageContent::Parts(vec![
ContentPart::InputAudio {
input_audio: InputAudio {
data: "dGVzdA==".to_string(),
format: Some("wav".to_string()),
},
uuid: Some("audio-1".to_string()),
},
ContentPart::AudioUrl {
audio_url: AudioUrl {
url: "https://example.com/audio.mp3".to_string(),
},
uuid: None,
},
]),
name: None,
}],
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen3-ASR-1.7B"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.messages,
vec![VllmChatMessage::user(vec![
ChatContentPart::InputAudio {
data: "dGVzdA==".to_string(),
format: Some("wav".to_string()),
uuid: Some("audio-1".to_string()),
},
ChatContentPart::audio_url("https://example.com/audio.mp3"),
])]
);
}
#[test]
fn prepare_chat_request_rejects_assistant_image_url_content_parts() {
let request = ChatCompletionRequest {
@@ -99,6 +99,18 @@ pub enum ContentPart {
#[serde(skip_serializing_if = "Option::is_none")]
uuid: Option<String>,
},
#[serde(rename = "audio_url")]
AudioUrl {
audio_url: AudioUrl,
#[serde(skip_serializing_if = "Option::is_none")]
uuid: Option<String>,
},
#[serde(rename = "input_audio")]
InputAudio {
input_audio: InputAudio,
#[serde(skip_serializing_if = "Option::is_none")]
uuid: Option<String>,
},
}
#[serde_with::skip_serializing_none]
@@ -113,6 +125,19 @@ pub struct VideoUrl {
pub url: String,
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
pub struct AudioUrl {
pub url: String,
}
/// Base64-encoded audio bytes in OpenAI `input_audio` form.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
pub struct InputAudio {
pub data: String,
pub format: Option<String>,
}
// ============================================================================
// Streaming
// ============================================================================
+4 -3
View File
@@ -551,9 +551,10 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R
for part in parts {
match part {
ChatContentPart::Text { text } => out.push_str(text),
ChatContentPart::ImageUrl { .. } | ChatContentPart::VideoUrl { .. } => {
out.push_str(placeholder)
}
ChatContentPart::ImageUrl { .. }
| ChatContentPart::VideoUrl { .. }
| ChatContentPart::InputAudio { .. }
| ChatContentPart::AudioUrl { .. } => out.push_str(placeholder),
}
}
out