[feat] add torchcodec as audio loader and implement selective audio backend (#51826)

Signed-off-by: JaredforReal <w13431838023@gmail.com>
Co-authored-by: saltman155 <saltman155@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Jared Wen
2026-09-04 16:16:10 +00:00
committed by GitHub
co-authored by saltman155 Isotr0py
parent 131e0285f6
commit 6a039f465e
4 changed files with 444 additions and 65 deletions
@@ -172,7 +172,7 @@ async def test_invalid_audio_file(whisper_client):
)
assert exc_info.value.status_code == 400
assert "Invalid or unsupported audio file" in exc_info.value.message
assert "Invalid or corrupted audio data" in exc_info.value.message
@pytest.mark.asyncio
+100 -1
View File
@@ -10,7 +10,12 @@ import pytest
import soundfile as sf
from vllm.multimodal.media import AudioMediaIO
from vllm.multimodal.media.audio import load_audio, load_audio_soundfile
from vllm.multimodal.media import audio as audio_module
from vllm.multimodal.media.audio import (
load_audio,
load_audio_soundfile,
load_audio_torchcodec,
)
from ...conftest import AudioTestAssets
@@ -170,3 +175,97 @@ def test_load_audio_threads_max_decode_bytes():
max_duration_s=600,
max_decode_bytes=512 * 1024,
)
@pytest.mark.parametrize("backend", ["soundfile", "pyav", "torchcodec"])
def test_load_audio_backend_matches_default(backend, dummy_audio_bytes):
"""Every explicit backend must decode to the same samples as `auto`."""
if backend == "torchcodec":
pytest.importorskip("torchcodec")
ref_audio, ref_sr = load_audio(BytesIO(dummy_audio_bytes), sr=None)
audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend=backend)
assert sr == ref_sr
# Decoders disagree only on codec encoder-delay/padding, so torchcodec may
# emit a few extra trailing samples (e.g. ~192 for Ogg Vorbis). Compare the
# overlapping region, which must agree to float32 precision.
n = min(ref_audio.shape[-1], audio.shape[-1])
assert n > 0
np.testing.assert_allclose(ref_audio[:n], audio[:n], atol=1e-4)
def test_load_audio_unknown_backend_rejected(dummy_audio_bytes):
"""An unknown backend must fail loudly instead of silently degrading."""
with pytest.raises(ValueError, match="Unknown audio backend"):
load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="not_a_backend")
def test_load_audio_auto_falls_back_without_torchcodec(dummy_audio_bytes):
"""`auto` must fall back to the soundfile → PyAV chain when torchcodec
is not importable."""
ref_audio, ref_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None)
with patch.object(audio_module, "load_audio_torchcodec", side_effect=ImportError):
audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="auto")
assert sr == ref_sr
np.testing.assert_array_equal(ref_audio, audio)
def test_load_audio_auto_falls_back_without_ffmpeg(dummy_audio_bytes):
"""torchcodec installed but system ffmpeg missing (`AudioDecoder is None`)
must surface as ImportError so `auto` falls back to soundfile → PyAV."""
ref_audio, ref_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None)
with patch.object(audio_module, "AudioDecoder", None):
with pytest.raises(ImportError, match="torchcodec audio backend"):
load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None)
audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="auto")
assert sr == ref_sr
np.testing.assert_array_equal(ref_audio, audio)
def test_load_audio_auto_falls_back_when_libtorchcodec_unloadable(
dummy_audio_bytes,
):
"""torchcodec loads its ffmpeg-backed core lazily at decoder construction;
a "Could not load libtorchcodec" RuntimeError there (no system ffmpeg)
must also surface as ImportError so `auto` falls back."""
class _UnloadableDecoder:
def __init__(self, *args, **kwargs):
raise RuntimeError("Could not load libtorchcodec. Likely causes: ...")
ref_audio, ref_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None)
with patch.object(audio_module, "AudioDecoder", _UnloadableDecoder):
with pytest.raises(ImportError, match="torchcodec audio backend"):
load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None)
audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="auto")
assert sr == ref_sr
np.testing.assert_array_equal(ref_audio, audio)
def test_audio_media_io_audio_backend_kwarg(dummy_audio_bytes):
"""`audio_backend` selects the backend; unknown values fail at init."""
audio, sr = AudioMediaIO(audio_backend="pyav").load_bytes(dummy_audio_bytes)
assert isinstance(audio, np.ndarray)
assert sr == 16000
with pytest.raises(ValueError, match="Unknown audio_backend"):
AudioMediaIO(audio_backend="not_a_backend")
def test_torchcodec_max_duration_rejected(dummy_audio_bytes):
"""The decompression-bomb duration guard must hold for torchcodec too."""
pytest.importorskip("torchcodec")
with pytest.raises(ValueError, match="exceeds maximum allowed duration"):
load_audio_torchcodec(
BytesIO(dummy_audio_bytes), sr=None, max_duration_s=0.0001
)
def test_torchcodec_max_decode_bytes_rejected(dummy_audio_bytes):
"""The decompression-bomb memory guard must hold for torchcodec too."""
pytest.importorskip("torchcodec")
audio, _ = load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None)
with pytest.raises(ValueError, match="VLLM_MAX_AUDIO_DECODE_BYTES"):
load_audio_torchcodec(
BytesIO(dummy_audio_bytes),
sr=None,
max_decode_bytes=audio.nbytes - 1,
)