[Fix] Introduce audio channels spec (#31595)

Signed-off-by: Jeremy Teboul <jeremyte@meta.com>
This commit is contained in:
Jeremy Teboul
2026-01-09 19:34:51 +00:00
committed by GitHub
parent 308feab33f
commit 657e9c0e18
9 changed files with 717 additions and 189 deletions
+11 -1
View File
@@ -22,7 +22,7 @@ from typing_extensions import assert_never
from vllm.utils.collection_utils import is_list_of
from vllm.utils.import_utils import LazyLoader
from .audio import AudioResampler
from .audio import AudioResampler, AudioSpec, normalize_audio
from .base import MediaWithBytes
from .inputs import (
AudioItem,
@@ -456,6 +456,9 @@ class MultiModalDataParser:
Args:
target_sr (float, optional): Enables automatic resampling of audio
items to the model's expected sampling rate.
target_channels (int, optional): Target number of audio channels.
If provided, normalizes audio to this many channels (e.g., 1 for mono).
If None, audio channels are passed through unchanged.
expected_hidden_size (int, optional): Expected hidden dimension for
embedding inputs. If provided, validates that user-supplied
embeddings have the correct hidden size to prevent crashes
@@ -466,6 +469,7 @@ class MultiModalDataParser:
self,
*,
target_sr: float | None = None,
target_channels: int | None = None,
audio_resample_method: Literal["librosa", "scipy"] = "librosa",
video_needs_metadata: bool = False,
expected_hidden_size: int | None = None,
@@ -476,6 +480,7 @@ class MultiModalDataParser:
target_sr=target_sr,
method=audio_resample_method,
)
self.target_channels = target_channels
self.video_needs_metadata = video_needs_metadata
self.expected_hidden_size = expected_hidden_size
@@ -565,6 +570,11 @@ class MultiModalDataParser:
else:
new_audio = self.audio_resampler.resample(audio, orig_sr=orig_sr)
# Apply channel normalization if target_channels is set
if self.target_channels is not None:
spec = AudioSpec(target_channels=self.target_channels)
new_audio = normalize_audio(new_audio, spec)
new_audios.append(new_audio)
return AudioProcessorItems(new_audios)