From 912d2e79fe8571ded2fcc6ec669d26429f814d7b Mon Sep 17 00:00:00 2001 From: Jared Wen Date: Thu, 10 Sep 2026 14:42:38 +0800 Subject: [PATCH] [multimodal][feat] add torchaudio backend to AudioResampler (#52598) Signed-off-by: JaredforReal Signed-off-by: Isotr0py Co-authored-by: Isotr0py Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/multimodal/test_audio.py | 84 ++++++++++++++++++++++++++++++++-- vllm/multimodal/audio.py | 74 +++++++++++++++++++++++++++++- vllm/multimodal/parse.py | 7 ++- 3 files changed, 159 insertions(+), 6 deletions(-) diff --git a/tests/multimodal/test_audio.py b/tests/multimodal/test_audio.py index a4a4a94767..21b39710d4 100644 --- a/tests/multimodal/test_audio.py +++ b/tests/multimodal/test_audio.py @@ -14,9 +14,11 @@ from vllm.multimodal.audio import ( AudioResampler, AudioSpec, ChannelReduction, + _get_torchaudio_resampler, normalize_audio, resample_audio_pyav, resample_audio_scipy, + resample_audio_torchaudio, split_audio, ) @@ -76,6 +78,46 @@ def test_resample_audio_scipy_resamples_last_axis_for_multichannel(): assert np.isfinite(out).all() +def test_resample_audio_torchaudio(dummy_audio): + out_down = resample_audio_torchaudio(dummy_audio, orig_sr=4, target_sr=2) + out_up = resample_audio_torchaudio(dummy_audio, orig_sr=2, target_sr=4) + out_same = resample_audio_torchaudio(dummy_audio, orig_sr=4, target_sr=4) + + assert len(out_down) == 3 + assert len(out_up) == 10 + assert np.all(out_same == dummy_audio) + + +def test_resample_audio_torchaudio_non_divisible_sample_rates(): + audio = np.arange(441, dtype=float) + out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000) + + expected_len = math.ceil(len(audio) * 16000 / 44100) + assert len(out) == expected_len + + assert isinstance(out, np.ndarray) + assert np.isfinite(out).all() + + +def test_resample_audio_torchaudio_resamples_last_axis_for_multichannel(): + audio = np.arange(2 * 441, dtype=float).reshape(2, 441) + out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000) + + expected_len = math.ceil(audio.shape[-1] * 16000 / 44100) + assert out.shape == (2, expected_len) + assert np.isfinite(out).all() + + +def test_resample_audio_torchaudio_short_input_needs_no_padding(): + # Unlike the PyAV resampler, torchaudio produces output for inputs + # shorter than libswresample's minimum frame size. + audio = np.arange(8, dtype=float) + out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000) + + assert len(out) == math.ceil(len(audio) * 16000 / 44100) + assert np.isfinite(out).all() + + def test_audio_resampler_pyav_calls_resample(dummy_audio): resampler = AudioResampler(target_sr=22050, method="pyav") with patch("vllm.multimodal.audio.resample_audio_pyav") as mock_resample: @@ -98,10 +140,46 @@ def test_audio_resampler_scipy_calls_resample(dummy_audio): assert np.all(out == dummy_audio) +def test_audio_resampler_torchaudio(dummy_audio): + resampler = AudioResampler(target_sr=22050, method="torchaudio") + out_down = resampler.resample(dummy_audio, orig_sr=44100) + + expected_len = math.ceil(len(dummy_audio) * 22050 / 44100) + assert len(out_down) == expected_len + assert np.isfinite(out_down).all() + + +def test_resample_audio_torchaudio_caches_kernel(): + # The sinc kernel is rebuilt per (orig_sr, target_sr) pair; repeated + # requests at the same pair must reuse the cached one. + _get_torchaudio_resampler.cache_clear() + audio = np.arange(441, dtype=float) + + resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000) + resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000) + resample_audio_torchaudio(audio, orig_sr=48000, target_sr=16000) + + info = _get_torchaudio_resampler.cache_info() + assert info.misses == 2 + assert info.hits == 1 + + +def test_audio_resampler_default_is_torchaudio(): + # Deliberate behavior change: torchaudio is the default resampling backend + # for both AudioResampler and the data parser that wraps it. + assert AudioResampler(target_sr=16000).method == "torchaudio" + + from vllm.multimodal.parse import MultiModalDataParser + + parser = MultiModalDataParser(target_sr=16000) + assert parser.audio_resampler.method == "torchaudio" + + def test_audio_resampler_invalid_method(dummy_audio): - resampler = AudioResampler(target_sr=22050, method="invalid") - with pytest.raises(ValueError): - resampler.resample(dummy_audio, orig_sr=44100) + # Validated eagerly so a bad method fails at construction, not on the + # first audio request. + with pytest.raises(ValueError, match="Invalid resampling method"): + AudioResampler(target_sr=22050, method="invalid") def test_audio_resampler_no_target_sr(dummy_audio): diff --git a/vllm/multimodal/audio.py b/vllm/multimodal/audio.py index 50834efe5c..59f0202303 100644 --- a/vllm/multimodal/audio.py +++ b/vllm/multimodal/audio.py @@ -3,6 +3,7 @@ import math from dataclasses import dataclass from enum import Enum +from functools import lru_cache from typing import Literal import numpy as np @@ -10,6 +11,7 @@ import numpy.typing as npt import torch from vllm.utils.import_utils import PlaceholderModule +from vllm.utils.torch_utils import set_default_torch_num_threads try: import av as av @@ -26,6 +28,11 @@ try: except ImportError: soxr = PlaceholderModule("soxr") # type: ignore[assignment] +try: + import torchaudio +except ImportError: + torchaudio = PlaceholderModule("torchaudio") # type: ignore[assignment] + # ============================================================ # Aligned with `librosa.get_duration` function @@ -274,14 +281,73 @@ def resample_audio_soxr( return soxr.resample(audio, orig_sr_int, target_sr_int) +@lru_cache(maxsize=32) +def _get_torchaudio_resampler( + orig_sr: int, target_sr: int +) -> "torchaudio.transforms.Resample": + # `torchaudio.transforms.Resample` precomputes its kernel for a fixed + # (orig_sr, target_sr) pair; cache instances so repeated requests at a + # common input rate skip the kernel rebuild. + return torchaudio.transforms.Resample(orig_sr, target_sr) + + +def resample_audio_torchaudio( + audio: npt.NDArray[np.floating], + *, + orig_sr: float, + target_sr: float, +) -> npt.NDArray[np.floating]: + """Resample audio using torchaudio's bandlimited sinc interpolation. + + Unlike the PyAV resampler, this handles any input length without padding + and applies the kernel over the trailing axis, so 2D ``(channels, + samples)`` input needs no per-channel loop. + + Args: + audio: Input audio. Can be: + - 1D array ``(samples,)``: mono audio + - 2D array ``(channels, samples)``: stereo audio + orig_sr: Original sample rate in Hz. + target_sr: Target sample rate in Hz. + + Returns: + Resampled audio with the same shape as the input (1D → 1D, 2D → 2D). + """ + orig_sr_int = int(round(orig_sr)) + target_sr_int = int(round(target_sr)) + + if orig_sr_int == target_sr_int: + return audio + + # The kernel is float32; cast the input to match (same coercion as the + # PyAV path). + tensor = torch.as_tensor(audio, dtype=torch.float32) + # Resampling runs in the API/server parent process. Keep it from + # touching OpenMP or oneDNN thread state: both poison subsequently + # forked engine-core processes, which then segfault on their first + # parallel CPU op. + with set_default_torch_num_threads(1), torch.backends.mkldnn.flags(enabled=False): + resampler = _get_torchaudio_resampler(orig_sr_int, target_sr_int) + return resampler(tensor).numpy() + + class AudioResampler: """Resample audio data to a target sample rate.""" + _METHODS = ("pyav", "scipy", "soxr", "torchaudio") + def __init__( self, target_sr: float | None = None, - method: Literal["pyav", "scipy", "soxr"] = "pyav", + method: Literal["pyav", "scipy", "soxr", "torchaudio"] = "torchaudio", ): + # Eager validation so a bad method fails at construction rather than + # on the first audio request. + if method not in self._METHODS: + raise ValueError( + f"Invalid resampling method: {method!r}. " + f"Supported methods: {list(self._METHODS)}." + ) self.target_sr = target_sr self.method = method @@ -310,10 +376,14 @@ class AudioResampler: ) elif self.method == "soxr": return resample_audio_soxr(audio, orig_sr=orig_sr, target_sr=self.target_sr) + elif self.method == "torchaudio": + return resample_audio_torchaudio( + audio, orig_sr=orig_sr, target_sr=self.target_sr + ) else: raise ValueError( f"Invalid resampling method: {self.method}. " - "Supported methods are 'pyav', 'scipy', and 'soxr'." + f"Supported methods are {list(self._METHODS)}." ) diff --git a/vllm/multimodal/parse.py b/vllm/multimodal/parse.py index 7a4ffc3d6d..d96f19f39b 100644 --- a/vllm/multimodal/parse.py +++ b/vllm/multimodal/parse.py @@ -547,6 +547,9 @@ class MultiModalDataParser: Args: target_sr (float, optional): Enables automatic resampling of audio items to the model's expected sampling rate. + audio_resample_method (str): Backend used for the resampling above. + Defaults to torchaudio; models with specific needs may override + (e.g. phi4mm uses scipy). target_channels (int, optional): Target number of audio channels. If provided, normalizes audio to this many channels (e.g., 1 for mono). If None, audio channels are passed through unchanged. @@ -602,7 +605,9 @@ class MultiModalDataParser: *, target_sr: float | None = None, target_channels: int | None = None, - audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav", + audio_resample_method: Literal["pyav", "scipy", "soxr", "torchaudio"] = ( + "torchaudio" + ), video_needs_metadata: bool = False, expected_hidden_size: int | None = None, allow_missing_mm_embeddings: bool = False,