[multimodal][feat] add torchaudio backend to AudioResampler (#52598)

Signed-off-by: JaredforReal <w13431838023@gmail.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jared Wen
2026-09-10 06:42:38 +00:00
committed by GitHub
co-authored by Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7e91760650
commit 912d2e79fe
3 changed files with 159 additions and 6 deletions
+81 -3
View File
@@ -14,9 +14,11 @@ from vllm.multimodal.audio import (
AudioResampler,
AudioSpec,
ChannelReduction,
_get_torchaudio_resampler,
normalize_audio,
resample_audio_pyav,
resample_audio_scipy,
resample_audio_torchaudio,
split_audio,
)
@@ -76,6 +78,46 @@ def test_resample_audio_scipy_resamples_last_axis_for_multichannel():
assert np.isfinite(out).all()
def test_resample_audio_torchaudio(dummy_audio):
out_down = resample_audio_torchaudio(dummy_audio, orig_sr=4, target_sr=2)
out_up = resample_audio_torchaudio(dummy_audio, orig_sr=2, target_sr=4)
out_same = resample_audio_torchaudio(dummy_audio, orig_sr=4, target_sr=4)
assert len(out_down) == 3
assert len(out_up) == 10
assert np.all(out_same == dummy_audio)
def test_resample_audio_torchaudio_non_divisible_sample_rates():
audio = np.arange(441, dtype=float)
out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
expected_len = math.ceil(len(audio) * 16000 / 44100)
assert len(out) == expected_len
assert isinstance(out, np.ndarray)
assert np.isfinite(out).all()
def test_resample_audio_torchaudio_resamples_last_axis_for_multichannel():
audio = np.arange(2 * 441, dtype=float).reshape(2, 441)
out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
expected_len = math.ceil(audio.shape[-1] * 16000 / 44100)
assert out.shape == (2, expected_len)
assert np.isfinite(out).all()
def test_resample_audio_torchaudio_short_input_needs_no_padding():
# Unlike the PyAV resampler, torchaudio produces output for inputs
# shorter than libswresample's minimum frame size.
audio = np.arange(8, dtype=float)
out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
assert len(out) == math.ceil(len(audio) * 16000 / 44100)
assert np.isfinite(out).all()
def test_audio_resampler_pyav_calls_resample(dummy_audio):
resampler = AudioResampler(target_sr=22050, method="pyav")
with patch("vllm.multimodal.audio.resample_audio_pyav") as mock_resample:
@@ -98,10 +140,46 @@ def test_audio_resampler_scipy_calls_resample(dummy_audio):
assert np.all(out == dummy_audio)
def test_audio_resampler_torchaudio(dummy_audio):
resampler = AudioResampler(target_sr=22050, method="torchaudio")
out_down = resampler.resample(dummy_audio, orig_sr=44100)
expected_len = math.ceil(len(dummy_audio) * 22050 / 44100)
assert len(out_down) == expected_len
assert np.isfinite(out_down).all()
def test_resample_audio_torchaudio_caches_kernel():
# The sinc kernel is rebuilt per (orig_sr, target_sr) pair; repeated
# requests at the same pair must reuse the cached one.
_get_torchaudio_resampler.cache_clear()
audio = np.arange(441, dtype=float)
resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
resample_audio_torchaudio(audio, orig_sr=48000, target_sr=16000)
info = _get_torchaudio_resampler.cache_info()
assert info.misses == 2
assert info.hits == 1
def test_audio_resampler_default_is_torchaudio():
# Deliberate behavior change: torchaudio is the default resampling backend
# for both AudioResampler and the data parser that wraps it.
assert AudioResampler(target_sr=16000).method == "torchaudio"
from vllm.multimodal.parse import MultiModalDataParser
parser = MultiModalDataParser(target_sr=16000)
assert parser.audio_resampler.method == "torchaudio"
def test_audio_resampler_invalid_method(dummy_audio):
resampler = AudioResampler(target_sr=22050, method="invalid")
with pytest.raises(ValueError):
resampler.resample(dummy_audio, orig_sr=44100)
# Validated eagerly so a bad method fails at construction, not on the
# first audio request.
with pytest.raises(ValueError, match="Invalid resampling method"):
AudioResampler(target_sr=22050, method="invalid")
def test_audio_resampler_no_target_sr(dummy_audio):