diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index 7eef6ce85c..64dca49d28 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -1210,7 +1210,7 @@ vLLM decodes audio bytes into waveforms using a selectable decoding backend: | Backend | Description | | --- | --- | -| `auto` (default) | torchcodec, falling back to soundfile, then PyAV | +| `auto` (default) | soundfile, falling back to torchcodec, then PyAV | | `soundfile` | libsndfile only, no fallback | | `pyav` | PyAV (FFmpeg) only, no fallback | | `torchcodec` | TorchCodec (PyTorch-native) only, no fallback | @@ -1226,15 +1226,17 @@ vllm serve mistralai/Voxtral-Mini-3B-2507 \ `pyav` drives FFmpeg through a per-frame Python generator, so under concurrency the Python/C crossings contend on the GIL. `torchcodec` decodes each stream in a single call that releases the GIL for its whole - duration, which is why `auto` prefers it when many requests decode audio - concurrently, such as when audio tracks are extracted from video. + duration. Select it explicitly for concurrent decoding workloads. + `auto` prefers soundfile for supported formats to preserve their existing + decoding behavior, including encoder padding. Audio extracted from formats + that soundfile cannot read, such as video containers, can use torchcodec + through the fallback chain. !!! note `torchcodec` ships as a requirement on CUDA, CPU and XPU builds. On other - platforms (e.g. ROCm, TPU) `auto` falls back to the soundfile → PyAV - chain unless you install it manually. torchcodec also links against a - system FFmpeg installation; if FFmpeg is missing, `auto` falls back the - same way. + platforms (e.g. ROCm, TPU), install it manually to enable that backend. + torchcodec also links against a system FFmpeg installation. If the package + or FFmpeg is unavailable, `auto` uses the soundfile → PyAV chain. ### Embedding Inputs diff --git a/tests/multimodal/media/test_audio.py b/tests/multimodal/media/test_audio.py index 89810f60d2..3473a911ea 100644 --- a/tests/multimodal/media/test_audio.py +++ b/tests/multimodal/media/test_audio.py @@ -193,6 +193,14 @@ def test_load_audio_backend_matches_default(backend, dummy_audio_bytes): np.testing.assert_allclose(ref_audio[:n], audio[:n], atol=1e-4) +def test_load_audio_default_preserves_vorbis_length(dummy_audio_bytes): + """Default decoding must not append Vorbis padding to the speech waveform.""" + expected, expected_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None) + audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None) + assert sr == expected_sr + assert audio.shape == expected.shape + + def test_load_audio_unknown_backend_rejected(dummy_audio_bytes): """An unknown backend must fail loudly instead of silently degrading.""" with pytest.raises(ValueError, match="Unknown audio backend"): diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py index 5187ffe1cc..e1e75dd8d7 100644 --- a/vllm/multimodal/media/audio.py +++ b/vllm/multimodal/media/audio.py @@ -60,11 +60,11 @@ _BAD_SF_CODES = {0, 1, 3, 4} # Audio decoding backends selectable via `load_audio(backend=...)` or # `--media-io-kwargs '{"audio": {"audio_backend": ...}}'`. -# "auto" tries torchcodec, then soundfile, then PyAV. +# "auto" tries soundfile, then torchcodec, then PyAV. AUDIO_BACKENDS = ("auto", "soundfile", "pyav", "torchcodec") # Raised as ImportError when the torchcodec backend cannot be used, so -# `load_audio(backend="auto")` falls back to the soundfile → PyAV chain. +# `load_audio(backend="auto")` can continue to PyAV. _TORCHCODEC_UNAVAILABLE_MSG = ( "torchcodec audio backend is unavailable (requires the torchcodec " "package and a system ffmpeg installation)" @@ -413,8 +413,8 @@ def load_audio( Args: backend: One of ``AUDIO_BACKENDS``. ``None`` (default) selects - ``"auto"``, which tries torchcodec, then falls back to the - soundfile → PyAV chain; the other values select a single + ``"auto"``, which tries soundfile, then torchcodec, then PyAV; + the other values select a single backend with no fallback. """ backend = backend or "auto" @@ -433,27 +433,11 @@ def load_audio( max_decode_bytes=max_decode_bytes, ) - # "auto": torchcodec → soundfile → PyAV. Each backend is called by its + # Keep soundfile first to preserve decoding and padding of supported formats. + # "auto": soundfile → torchcodec → PyAV. Each backend is called by its # bare name so tests that monkeypatch it take effect. Only an ImportError # (backend missing) or, for soundfile, an unsupported-format error defers # to the next; any other error (decode failure, guard ValueError) raises. - if isinstance(path, BytesIO): - path.seek(0) - try: - return load_audio_torchcodec( - path, - sr=sr, - mono=mono, - max_duration_s=max_duration_s, - max_decode_bytes=max_decode_bytes, - ) - except ImportError as exc: - # Decode errors don't defer: torchcodec is FFmpeg-based like PyAV, so - # retrying would just fail the same way. - logger.warning( - "torchcodec unavailable (%r); falling back to soundfile/PyAV.", exc - ) - if isinstance(path, BytesIO): path.seek(0) try: @@ -477,6 +461,21 @@ def load_audio( if exc.code not in _BAD_SF_CODES: raise + if isinstance(path, BytesIO): + path.seek(0) + try: + return load_audio_torchcodec( + path, + sr=sr, + mono=mono, + max_duration_s=max_duration_s, + max_decode_bytes=max_decode_bytes, + ) + except ImportError as exc: + # Decode errors don't defer: torchcodec is FFmpeg-based like PyAV, so + # retrying would just fail the same way. + logger.warning("torchcodec unavailable (%r); falling back to PyAV.", exc) + # PyAV is terminal: nothing left to fall back to. Normalize an FFmpeg # failure to ValueError; let a missing soundfile/PyAV surface its # "install vllm[audio]" ImportError.