[Fix] Introduce audio channels spec (#31595)

Signed-off-by: Jeremy Teboul <jeremyte@meta.com>
This commit is contained in:
Jeremy Teboul
2026-01-09 19:34:51 +00:00
committed by GitHub
parent 308feab33f
commit 657e9c0e18
9 changed files with 717 additions and 189 deletions
@@ -226,6 +226,10 @@ class Qwen2_5OmniThinkerProcessingInfo(
assert isinstance(feature_extractor, WhisperFeatureExtractor)
return feature_extractor
def get_target_channels(self) -> int:
"""Return target audio channels for Qwen2.5 Omni models (mono)."""
return 1
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": None, "image": None, "video": None}
@@ -310,6 +314,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
return Qwen2_5OmniThinkerMultiModalDataParser(
spatial_merge_size=self.info.get_hf_config().vision_config.spatial_merge_size,
target_sr=feature_extractor.sampling_rate,
target_channels=self.info.get_target_channels(),
)
def _call_hf_processor(
+8 -1
View File
@@ -140,6 +140,10 @@ class Qwen2AudioProcessingInfo(BaseProcessingInfo):
assert isinstance(feature_extractor, WhisperFeatureExtractor)
return feature_extractor
def get_target_channels(self) -> int:
"""Return target audio channels for Qwen2 Audio models (mono)."""
return 1
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": None}
@@ -201,7 +205,10 @@ class Qwen2AudioMultiModalDataParser(MultiModalDataParser):
class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
def _get_data_parser(self) -> MultiModalDataParser:
feature_extractor = self.info.get_feature_extractor()
return Qwen2AudioMultiModalDataParser(target_sr=feature_extractor.sampling_rate)
return Qwen2AudioMultiModalDataParser(
target_sr=feature_extractor.sampling_rate,
target_channels=self.info.get_target_channels(),
)
def _call_hf_processor(
self,
+8 -1
View File
@@ -133,6 +133,10 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
assert isinstance(feature_extractor, WhisperFeatureExtractor)
return feature_extractor
def get_target_channels(self) -> int:
"""Return target audio channels for Ultravox models (mono)."""
return 1
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": None}
@@ -169,7 +173,10 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):
def _get_data_parser(self) -> MultiModalDataParser:
feature_extractor = self.info.get_feature_extractor()
return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
return MultiModalDataParser(
target_sr=feature_extractor.sampling_rate,
target_channels=self.info.get_target_channels(),
)
def _call_hf_processor(
self,
+8 -1
View File
@@ -690,6 +690,10 @@ class WhisperProcessingInfo(BaseProcessingInfo):
assert isinstance(feature_extractor, WhisperFeatureExtractor)
return feature_extractor
def get_target_channels(self) -> int:
"""Return target audio channels for Whisper models (mono)."""
return 1
def get_num_audio_tokens(self) -> int:
return self.get_hf_config().max_source_positions
@@ -724,7 +728,10 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo]):
def _get_data_parser(self) -> MultiModalDataParser:
feature_extractor = self.info.get_feature_extractor()
return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
return MultiModalDataParser(
target_sr=feature_extractor.sampling_rate,
target_channels=self.info.get_target_channels(),
)
@property
def pad_dummy_encoder_prompt(self) -> bool:
+132
View File
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import base64
from dataclasses import dataclass
from enum import Enum
from io import BytesIO
from pathlib import Path
from typing import Literal
@@ -25,6 +27,136 @@ try:
except ImportError:
soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
# ============================================================
class ChannelReduction(str, Enum):
"""Method to reduce multi-channel audio to target channels."""
MEAN = "mean" # Average across channels (default, preserves energy balance)
FIRST = "first" # Take first channel only
MAX = "max" # Take max value across channels
SUM = "sum" # Sum across channels
@dataclass
class AudioSpec:
"""Specification for target audio format.
This dataclass defines the expected audio format for a model's feature
extractor. It is used to normalize audio data before processing.
Attributes:
target_channels: Number of output channels. None means passthrough
(no normalization). 1 = mono, 2 = stereo, etc.
channel_reduction: Method to reduce channels when input has more
channels than target. Only used when reducing channels.
"""
target_channels: int | None = 1
channel_reduction: ChannelReduction = ChannelReduction.MEAN
@property
def needs_normalization(self) -> bool:
"""Whether audio normalization is needed."""
return self.target_channels is not None
def __repr__(self) -> str:
if self.target_channels is None:
return "AudioSpec(passthrough)"
return (
f"AudioSpec(channels={self.target_channels}, "
f"reduction={self.channel_reduction.value})"
)
# Pre-defined specs for common use cases
MONO_AUDIO_SPEC = AudioSpec(target_channels=1, channel_reduction=ChannelReduction.MEAN)
PASSTHROUGH_AUDIO_SPEC = AudioSpec(target_channels=None)
def normalize_audio(
audio: npt.NDArray[np.floating] | torch.Tensor,
spec: AudioSpec,
) -> npt.NDArray[np.floating] | torch.Tensor:
"""Normalize audio to the specified format.
This function handles channel reduction for multi-channel audio,
supporting both numpy arrays and torch tensors.
Args:
audio: Input audio data. Can be:
- 1D array/tensor: (time,) - already mono
- 2D array/tensor: (channels, time) - standard format from torchaudio
- 2D array/tensor: (time, channels) - format from soundfile
(will be auto-detected and transposed if time > channels)
spec: AudioSpec defining the target format.
Returns:
Normalized audio in the same type as input (numpy or torch).
For mono output (target_channels=1), returns 1D array/tensor.
Raises:
ValueError: If audio has unsupported dimensions or channel expansion
is requested (e.g., mono to stereo).
"""
if not spec.needs_normalization:
return audio
# Handle 1D audio (already mono)
if audio.ndim == 1:
if spec.target_channels == 1:
return audio
raise ValueError(f"Cannot expand mono audio to {spec.target_channels} channels")
# Handle 2D audio
if audio.ndim != 2:
raise ValueError(f"Unsupported audio shape: {audio.shape}. Expected 1D or 2D.")
# Auto-detect format: if shape[0] > shape[1], assume (time, channels)
# This handles soundfile format where time dimension is typically much larger
if audio.shape[0] > audio.shape[1]:
# Transpose from (time, channels) to (channels, time)
audio = audio.T if isinstance(audio, np.ndarray) else audio.T
num_channels = audio.shape[0]
# No reduction needed if already at target
if num_channels == spec.target_channels:
return audio
# Cannot expand channels
if num_channels < spec.target_channels:
raise ValueError(
f"Cannot expand {num_channels} channels to {spec.target_channels}"
)
# Reduce channels
is_numpy = isinstance(audio, np.ndarray)
if spec.target_channels == 1:
# Reduce to mono
if spec.channel_reduction == ChannelReduction.MEAN:
result = np.mean(audio, axis=0) if is_numpy else audio.mean(dim=0)
elif spec.channel_reduction == ChannelReduction.FIRST:
result = audio[0]
elif spec.channel_reduction == ChannelReduction.MAX:
result = np.max(audio, axis=0) if is_numpy else audio.max(dim=0).values
elif spec.channel_reduction == ChannelReduction.SUM:
result = np.sum(audio, axis=0) if is_numpy else audio.sum(dim=0)
else:
raise ValueError(f"Unknown reduction method: {spec.channel_reduction}")
return result
else:
# Reduce to N channels (take first N and apply reduction if needed)
# For now, just take first N channels
return audio[: spec.target_channels]
# ============================================================
# Audio Resampling
# ============================================================
def resample_audio_librosa(
audio: npt.NDArray[np.floating],
+11 -1
View File
@@ -22,7 +22,7 @@ from typing_extensions import assert_never
from vllm.utils.collection_utils import is_list_of
from vllm.utils.import_utils import LazyLoader
from .audio import AudioResampler
from .audio import AudioResampler, AudioSpec, normalize_audio
from .base import MediaWithBytes
from .inputs import (
AudioItem,
@@ -456,6 +456,9 @@ class MultiModalDataParser:
Args:
target_sr (float, optional): Enables automatic resampling of audio
items to the model's expected sampling rate.
target_channels (int, optional): Target number of audio channels.
If provided, normalizes audio to this many channels (e.g., 1 for mono).
If None, audio channels are passed through unchanged.
expected_hidden_size (int, optional): Expected hidden dimension for
embedding inputs. If provided, validates that user-supplied
embeddings have the correct hidden size to prevent crashes
@@ -466,6 +469,7 @@ class MultiModalDataParser:
self,
*,
target_sr: float | None = None,
target_channels: int | None = None,
audio_resample_method: Literal["librosa", "scipy"] = "librosa",
video_needs_metadata: bool = False,
expected_hidden_size: int | None = None,
@@ -476,6 +480,7 @@ class MultiModalDataParser:
target_sr=target_sr,
method=audio_resample_method,
)
self.target_channels = target_channels
self.video_needs_metadata = video_needs_metadata
self.expected_hidden_size = expected_hidden_size
@@ -565,6 +570,11 @@ class MultiModalDataParser:
else:
new_audio = self.audio_resampler.resample(audio, orig_sr=orig_sr)
# Apply channel normalization if target_channels is set
if self.target_channels is not None:
spec = AudioSpec(target_channels=self.target_channels)
new_audio = normalize_audio(new_audio, spec)
new_audios.append(new_audio)
return AudioProcessorItems(new_audios)