[multimodal][feat] add torchaudio backend to AudioResampler (#52598)

Signed-off-by: JaredforReal <w13431838023@gmail.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

tests/multimodal/test_audio.py

@@ -14,9 +14,11 @@ from vllm.multimodal.audio import (
     AudioResampler,
     AudioSpec,
     ChannelReduction,
+    _get_torchaudio_resampler,
     normalize_audio,
     resample_audio_pyav,
     resample_audio_scipy,
+    resample_audio_torchaudio,
     split_audio,
 )
 
@@ -76,6 +78,46 @@ def test_resample_audio_scipy_resamples_last_axis_for_multichannel():
     assert np.isfinite(out).all()
 
 
+def test_resample_audio_torchaudio(dummy_audio):
+    out_down = resample_audio_torchaudio(dummy_audio, orig_sr=4, target_sr=2)
+    out_up = resample_audio_torchaudio(dummy_audio, orig_sr=2, target_sr=4)
+    out_same = resample_audio_torchaudio(dummy_audio, orig_sr=4, target_sr=4)
+
+    assert len(out_down) == 3
+    assert len(out_up) == 10
+    assert np.all(out_same == dummy_audio)
+
+
+def test_resample_audio_torchaudio_non_divisible_sample_rates():
+    audio = np.arange(441, dtype=float)
+    out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
+
+    expected_len = math.ceil(len(audio) * 16000 / 44100)
+    assert len(out) == expected_len
+
+    assert isinstance(out, np.ndarray)
+    assert np.isfinite(out).all()
+
+
+def test_resample_audio_torchaudio_resamples_last_axis_for_multichannel():
+    audio = np.arange(2 * 441, dtype=float).reshape(2, 441)
+    out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
+
+    expected_len = math.ceil(audio.shape[-1] * 16000 / 44100)
+    assert out.shape == (2, expected_len)
+    assert np.isfinite(out).all()
+
+
+def test_resample_audio_torchaudio_short_input_needs_no_padding():
+    # Unlike the PyAV resampler, torchaudio produces output for inputs
+    # shorter than libswresample's minimum frame size.
+    audio = np.arange(8, dtype=float)
+    out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
+
+    assert len(out) == math.ceil(len(audio) * 16000 / 44100)
+    assert np.isfinite(out).all()
+
+
 def test_audio_resampler_pyav_calls_resample(dummy_audio):
     resampler = AudioResampler(target_sr=22050, method="pyav")
     with patch("vllm.multimodal.audio.resample_audio_pyav") as mock_resample:
@@ -98,10 +140,46 @@ def test_audio_resampler_scipy_calls_resample(dummy_audio):
         assert np.all(out == dummy_audio)
 
 
+def test_audio_resampler_torchaudio(dummy_audio):
+    resampler = AudioResampler(target_sr=22050, method="torchaudio")
+    out_down = resampler.resample(dummy_audio, orig_sr=44100)
+
+    expected_len = math.ceil(len(dummy_audio) * 22050 / 44100)
+    assert len(out_down) == expected_len
+    assert np.isfinite(out_down).all()
+
+
+def test_resample_audio_torchaudio_caches_kernel():
+    # The sinc kernel is rebuilt per (orig_sr, target_sr) pair; repeated
+    # requests at the same pair must reuse the cached one.
+    _get_torchaudio_resampler.cache_clear()
+    audio = np.arange(441, dtype=float)
+
+    resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
+    resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
+    resample_audio_torchaudio(audio, orig_sr=48000, target_sr=16000)
+
+    info = _get_torchaudio_resampler.cache_info()
+    assert info.misses == 2
+    assert info.hits == 1
+
+
+def test_audio_resampler_default_is_torchaudio():
+    # Deliberate behavior change: torchaudio is the default resampling backend
+    # for both AudioResampler and the data parser that wraps it.
+    assert AudioResampler(target_sr=16000).method == "torchaudio"
+
+    from vllm.multimodal.parse import MultiModalDataParser
+
+    parser = MultiModalDataParser(target_sr=16000)
+    assert parser.audio_resampler.method == "torchaudio"
+
+
 def test_audio_resampler_invalid_method(dummy_audio):
-    resampler = AudioResampler(target_sr=22050, method="invalid")
-    with pytest.raises(ValueError):
-        resampler.resample(dummy_audio, orig_sr=44100)
+    # Validated eagerly so a bad method fails at construction, not on the
+    # first audio request.
+    with pytest.raises(ValueError, match="Invalid resampling method"):
+        AudioResampler(target_sr=22050, method="invalid")
 
 
 def test_audio_resampler_no_target_sr(dummy_audio):

vllm/multimodal/audio.py

@@ -3,6 +3,7 @@
 import math
 from dataclasses import dataclass
 from enum import Enum
+from functools import lru_cache
 from typing import Literal
 
 import numpy as np
@@ -10,6 +11,7 @@ import numpy.typing as npt
 import torch
 
 from vllm.utils.import_utils import PlaceholderModule
+from vllm.utils.torch_utils import set_default_torch_num_threads
 
 try:
     import av as av
@@ -26,6 +28,11 @@ try:
 except ImportError:
     soxr = PlaceholderModule("soxr")  # type: ignore[assignment]
 
+try:
+    import torchaudio
+except ImportError:
+    torchaudio = PlaceholderModule("torchaudio")  # type: ignore[assignment]
+
 
 # ============================================================
 # Aligned with `librosa.get_duration` function
@@ -274,14 +281,73 @@ def resample_audio_soxr(
     return soxr.resample(audio, orig_sr_int, target_sr_int)
 
 
+@lru_cache(maxsize=32)
+def _get_torchaudio_resampler(
+    orig_sr: int, target_sr: int
+) -> "torchaudio.transforms.Resample":
+    # `torchaudio.transforms.Resample` precomputes its kernel for a fixed
+    # (orig_sr, target_sr) pair; cache instances so repeated requests at a
+    # common input rate skip the kernel rebuild.
+    return torchaudio.transforms.Resample(orig_sr, target_sr)
+
+
+def resample_audio_torchaudio(
+    audio: npt.NDArray[np.floating],
+    *,
+    orig_sr: float,
+    target_sr: float,
+) -> npt.NDArray[np.floating]:
+    """Resample audio using torchaudio's bandlimited sinc interpolation.
+
+    Unlike the PyAV resampler, this handles any input length without padding
+    and applies the kernel over the trailing axis, so 2D ``(channels,
+    samples)`` input needs no per-channel loop.
+
+    Args:
+        audio: Input audio. Can be:
+            - 1D array ``(samples,)``: mono audio
+            - 2D array ``(channels, samples)``: stereo audio
+        orig_sr: Original sample rate in Hz.
+        target_sr: Target sample rate in Hz.
+
+    Returns:
+        Resampled audio with the same shape as the input (1D → 1D, 2D → 2D).
+    """
+    orig_sr_int = int(round(orig_sr))
+    target_sr_int = int(round(target_sr))
+
+    if orig_sr_int == target_sr_int:
+        return audio
+
+    # The kernel is float32; cast the input to match (same coercion as the
+    # PyAV path).
+    tensor = torch.as_tensor(audio, dtype=torch.float32)
+    # Resampling runs in the API/server parent process. Keep it from
+    # touching OpenMP or oneDNN thread state: both poison subsequently
+    # forked engine-core processes, which then segfault on their first
+    # parallel CPU op.
+    with set_default_torch_num_threads(1), torch.backends.mkldnn.flags(enabled=False):
+        resampler = _get_torchaudio_resampler(orig_sr_int, target_sr_int)
+        return resampler(tensor).numpy()
+
+
 class AudioResampler:
     """Resample audio data to a target sample rate."""
 
+    _METHODS = ("pyav", "scipy", "soxr", "torchaudio")
+
     def __init__(
         self,
         target_sr: float | None = None,
-        method: Literal["pyav", "scipy", "soxr"] = "pyav",
+        method: Literal["pyav", "scipy", "soxr", "torchaudio"] = "torchaudio",
     ):
+        # Eager validation so a bad method fails at construction rather than
+        # on the first audio request.
+        if method not in self._METHODS:
+            raise ValueError(
+                f"Invalid resampling method: {method!r}. "
+                f"Supported methods: {list(self._METHODS)}."
+            )
         self.target_sr = target_sr
         self.method = method
 
@@ -310,10 +376,14 @@ class AudioResampler:
             )
         elif self.method == "soxr":
             return resample_audio_soxr(audio, orig_sr=orig_sr, target_sr=self.target_sr)
+        elif self.method == "torchaudio":
+            return resample_audio_torchaudio(
+                audio, orig_sr=orig_sr, target_sr=self.target_sr
+            )
         else:
             raise ValueError(
                 f"Invalid resampling method: {self.method}. "
-                "Supported methods are 'pyav', 'scipy', and 'soxr'."
+                f"Supported methods are {list(self._METHODS)}."
             )
 
 

vllm/multimodal/parse.py

@@ -547,6 +547,9 @@ class MultiModalDataParser:
     Args:
         target_sr (float, optional): Enables automatic resampling of audio
             items to the model's expected sampling rate.
+        audio_resample_method (str): Backend used for the resampling above.
+            Defaults to torchaudio; models with specific needs may override
+            (e.g. phi4mm uses scipy).
         target_channels (int, optional): Target number of audio channels.
             If provided, normalizes audio to this many channels (e.g., 1 for mono).
             If None, audio channels are passed through unchanged.
@@ -602,7 +605,9 @@ class MultiModalDataParser:
         *,
         target_sr: float | None = None,
         target_channels: int | None = None,
-        audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav",
+        audio_resample_method: Literal["pyav", "scipy", "soxr", "torchaudio"] = (
+            "torchaudio"
+        ),
         video_needs_metadata: bool = False,
         expected_hidden_size: int | None = None,
         allow_missing_mm_embeddings: bool = False,