[multimodal][feat] add torchaudio backend to AudioResampler (#52598)

Signed-off-by: JaredforReal <w13431838023@gmail.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jared Wen
2026-09-10 06:42:38 +00:00
committed by GitHub
co-authored by Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7e91760650
commit 912d2e79fe
3 changed files with 159 additions and 6 deletions
+72 -2
View File
@@ -3,6 +3,7 @@
import math
from dataclasses import dataclass
from enum import Enum
from functools import lru_cache
from typing import Literal
import numpy as np
@@ -10,6 +11,7 @@ import numpy.typing as npt
import torch
from vllm.utils.import_utils import PlaceholderModule
from vllm.utils.torch_utils import set_default_torch_num_threads
try:
import av as av
@@ -26,6 +28,11 @@ try:
except ImportError:
soxr = PlaceholderModule("soxr") # type: ignore[assignment]
try:
import torchaudio
except ImportError:
torchaudio = PlaceholderModule("torchaudio") # type: ignore[assignment]
# ============================================================
# Aligned with `librosa.get_duration` function
@@ -274,14 +281,73 @@ def resample_audio_soxr(
return soxr.resample(audio, orig_sr_int, target_sr_int)
@lru_cache(maxsize=32)
def _get_torchaudio_resampler(
orig_sr: int, target_sr: int
) -> "torchaudio.transforms.Resample":
# `torchaudio.transforms.Resample` precomputes its kernel for a fixed
# (orig_sr, target_sr) pair; cache instances so repeated requests at a
# common input rate skip the kernel rebuild.
return torchaudio.transforms.Resample(orig_sr, target_sr)
def resample_audio_torchaudio(
audio: npt.NDArray[np.floating],
*,
orig_sr: float,
target_sr: float,
) -> npt.NDArray[np.floating]:
"""Resample audio using torchaudio's bandlimited sinc interpolation.
Unlike the PyAV resampler, this handles any input length without padding
and applies the kernel over the trailing axis, so 2D ``(channels,
samples)`` input needs no per-channel loop.
Args:
audio: Input audio. Can be:
- 1D array ``(samples,)``: mono audio
- 2D array ``(channels, samples)``: stereo audio
orig_sr: Original sample rate in Hz.
target_sr: Target sample rate in Hz.
Returns:
Resampled audio with the same shape as the input (1D → 1D, 2D → 2D).
"""
orig_sr_int = int(round(orig_sr))
target_sr_int = int(round(target_sr))
if orig_sr_int == target_sr_int:
return audio
# The kernel is float32; cast the input to match (same coercion as the
# PyAV path).
tensor = torch.as_tensor(audio, dtype=torch.float32)
# Resampling runs in the API/server parent process. Keep it from
# touching OpenMP or oneDNN thread state: both poison subsequently
# forked engine-core processes, which then segfault on their first
# parallel CPU op.
with set_default_torch_num_threads(1), torch.backends.mkldnn.flags(enabled=False):
resampler = _get_torchaudio_resampler(orig_sr_int, target_sr_int)
return resampler(tensor).numpy()
class AudioResampler:
"""Resample audio data to a target sample rate."""
_METHODS = ("pyav", "scipy", "soxr", "torchaudio")
def __init__(
self,
target_sr: float | None = None,
method: Literal["pyav", "scipy", "soxr"] = "pyav",
method: Literal["pyav", "scipy", "soxr", "torchaudio"] = "torchaudio",
):
# Eager validation so a bad method fails at construction rather than
# on the first audio request.
if method not in self._METHODS:
raise ValueError(
f"Invalid resampling method: {method!r}. "
f"Supported methods: {list(self._METHODS)}."
)
self.target_sr = target_sr
self.method = method
@@ -310,10 +376,14 @@ class AudioResampler:
)
elif self.method == "soxr":
return resample_audio_soxr(audio, orig_sr=orig_sr, target_sr=self.target_sr)
elif self.method == "torchaudio":
return resample_audio_torchaudio(
audio, orig_sr=orig_sr, target_sr=self.target_sr
)
else:
raise ValueError(
f"Invalid resampling method: {self.method}. "
"Supported methods are 'pyav', 'scipy', and 'soxr'."
f"Supported methods are {list(self._METHODS)}."
)
+6 -1
View File
@@ -547,6 +547,9 @@ class MultiModalDataParser:
Args:
target_sr (float, optional): Enables automatic resampling of audio
items to the model's expected sampling rate.
audio_resample_method (str): Backend used for the resampling above.
Defaults to torchaudio; models with specific needs may override
(e.g. phi4mm uses scipy).
target_channels (int, optional): Target number of audio channels.
If provided, normalizes audio to this many channels (e.g., 1 for mono).
If None, audio channels are passed through unchanged.
@@ -602,7 +605,9 @@ class MultiModalDataParser:
*,
target_sr: float | None = None,
target_channels: int | None = None,
audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav",
audio_resample_method: Literal["pyav", "scipy", "soxr", "torchaudio"] = (
"torchaudio"
),
video_needs_metadata: bool = False,
expected_hidden_size: int | None = None,
allow_missing_mm_embeddings: bool = False,