diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index 64dca49d28..40ceb8158b 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -1210,7 +1210,7 @@ vLLM decodes audio bytes into waveforms using a selectable decoding backend: | Backend | Description | | --- | --- | -| `auto` (default) | soundfile, falling back to torchcodec, then PyAV | +| `auto` (default) | torchcodec, falling back to soundfile, then PyAV | | `soundfile` | libsndfile only, no fallback | | `pyav` | PyAV (FFmpeg) only, no fallback | | `torchcodec` | TorchCodec (PyTorch-native) only, no fallback | @@ -1226,17 +1226,18 @@ vllm serve mistralai/Voxtral-Mini-3B-2507 \ `pyav` drives FFmpeg through a per-frame Python generator, so under concurrency the Python/C crossings contend on the GIL. `torchcodec` decodes each stream in a single call that releases the GIL for its whole - duration. Select it explicitly for concurrent decoding workloads. - `auto` prefers soundfile for supported formats to preserve their existing - decoding behavior, including encoder padding. Audio extracted from formats - that soundfile cannot read, such as video containers, can use torchcodec - through the fallback chain. + duration, which is why `auto` prefers it when many requests decode audio + concurrently, such as when audio tracks are extracted from video. + vLLM normalizes codec padding on this path (e.g. it trims trailing Vorbis + padding that older system FFmpeg versions (< 5.0) fail to trim), so the + decoded waveform matches the soundfile reference length. !!! note `torchcodec` ships as a requirement on CUDA, CPU and XPU builds. On other - platforms (e.g. ROCm, TPU), install it manually to enable that backend. - torchcodec also links against a system FFmpeg installation. If the package - or FFmpeg is unavailable, `auto` uses the soundfile → PyAV chain. + platforms (e.g. ROCm, TPU) `auto` falls back to the soundfile → PyAV + chain unless you install it manually. torchcodec also links against a + system FFmpeg installation; if FFmpeg is missing, `auto` falls back the + same way. ### Embedding Inputs diff --git a/tests/multimodal/media/test_audio.py b/tests/multimodal/media/test_audio.py index 0404535e1b..a75c20979e 100644 --- a/tests/multimodal/media/test_audio.py +++ b/tests/multimodal/media/test_audio.py @@ -185,9 +185,15 @@ def test_load_audio_backend_matches_default(backend, dummy_audio_bytes): ref_audio, ref_sr = load_audio(BytesIO(dummy_audio_bytes), sr=None) audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend=backend) assert sr == ref_sr - # Decoders disagree only on codec encoder-delay/padding, so torchcodec may - # emit a few extra trailing samples (e.g. ~192 for Ogg Vorbis). Compare the - # overlapping region, which must agree to float32 precision. + if backend == "torchcodec": + # Vorbis encoder padding is trimmed, so torchcodec must match the + # soundfile reference exactly, including the waveform length. + assert audio.shape == ref_audio.shape + np.testing.assert_allclose(ref_audio, audio, atol=1e-4) + return + # Decoders disagree only on codec encoder-delay/padding, so PyAV may + # emit a few extra trailing samples (e.g. ~192 for Ogg Vorbis). Compare + # the overlapping region, which must agree to float32 precision. n = min(ref_audio.shape[-1], audio.shape[-1]) assert n > 0 np.testing.assert_allclose(ref_audio[:n], audio[:n], atol=1e-4) @@ -201,6 +207,17 @@ def test_load_audio_default_preserves_vorbis_length(dummy_audio_bytes): assert audio.shape == expected.shape +def test_load_audio_torchcodec_preserves_vorbis_length(dummy_audio_bytes): + """Torchcodec must trim Vorbis trailing padding even on FFmpeg < 5.0, + matching the soundfile reference exactly.""" + pytest.importorskip("torchcodec") + expected, expected_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None) + audio, sr = load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None) + assert sr == expected_sr + assert audio.shape == expected.shape + np.testing.assert_allclose(audio, expected, atol=1e-4) + + def test_load_audio_unknown_backend_rejected(dummy_audio_bytes): """An unknown backend must fail loudly instead of silently degrading.""" with pytest.raises(ValueError, match="Unknown audio backend"): diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py index 1f1cda8dcc..c05f4e4dcc 100644 --- a/vllm/multimodal/media/audio.py +++ b/vllm/multimodal/media/audio.py @@ -15,6 +15,7 @@ from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.multimodal.audio import resample_audio_pyav from vllm.utils.import_utils import PlaceholderModule +from vllm.utils.math_utils import cdiv from vllm.utils.mem_constants import MiB_bytes from vllm.utils.serial_utils import tensor2base64 from vllm.utils.sparse_utils import ( @@ -60,11 +61,11 @@ _BAD_SF_CODES = {0, 1, 3, 4} # Audio decoding backends selectable via `load_audio(backend=...)` or # `--media-io-kwargs '{"audio": {"audio_backend": ...}}'`. -# "auto" tries soundfile, then torchcodec, then PyAV. +# "auto" tries torchcodec, then soundfile, then PyAV. AUDIO_BACKENDS = ("auto", "soundfile", "pyav", "torchcodec") # Raised as ImportError when the torchcodec backend cannot be used, so -# `load_audio(backend="auto")` can continue to PyAV. +# `load_audio(backend="auto")` falls back to the soundfile → PyAV chain. _TORCHCODEC_UNAVAILABLE_MSG = ( "torchcodec audio backend is unavailable (requires the torchcodec " "package and a system ffmpeg installation)" @@ -74,6 +75,9 @@ _TORCHCODEC_UNAVAILABLE_MSG = ( # over-long stream is rejected rather than truncated to the limit. _DURATION_GUARD_MARGIN_S = 1.0 +# Largest Vorbis block size in native-rate samples +_VORBIS_MAX_BLOCK_SAMPLES = 8192 + def load_audio_pyav( path: BytesIO | Path | str, @@ -364,6 +368,16 @@ def load_audio_torchcodec( "audio or video file (e.g. a complete WAV, MP3, or MP4)." ) from e + # FFmpeg < 5.0 misses Ogg end_trimming, leaving up to one Vorbis block of + # padding at the tail; we need to trim for valid audio length. + if metadata.codec == "vorbis" and metadata.duration_seconds is not None: + expected_samples = round(metadata.duration_seconds * samples.sample_rate) + padding = samples.data.shape[-1] - expected_samples + native_sr = metadata.sample_rate or samples.sample_rate + max_padding = cdiv(_VORBIS_MAX_BLOCK_SAMPLES * samples.sample_rate, native_sr) + if 0 < padding <= max_padding: + samples.data = samples.data[..., :expected_samples] + # Authoritative post-decode checks: the metadata estimates above can be # bypassed by a container that lies about its duration or size. if max_duration_s is not None: @@ -425,8 +439,8 @@ def load_audio( max_decode_bytes: Reject audio that would decode to more than this many bytes. backend: One of ``AUDIO_BACKENDS``. ``None`` (default) selects - ``"auto"``, which tries soundfile, then torchcodec, then PyAV; - the other values select a single + ``"auto"``, which tries torchcodec, then falls back to the + soundfile → PyAV chain; the other values select a single backend with no fallback. """ @@ -446,11 +460,25 @@ def load_audio( max_decode_bytes=max_decode_bytes, ) - # Keep soundfile first to preserve decoding and padding of supported formats. - # "auto": soundfile → torchcodec → PyAV. Each backend is called by its - # bare name so tests that monkeypatch it take effect. Only an ImportError - # (backend missing) or, for soundfile, an unsupported-format error defers - # to the next; any other error (decode failure, guard ValueError) raises. + # "auto": torchcodec → soundfile → PyAV, each called by its bare name so + # monkeypatched tests take effect. Only ImportError (or, for soundfile, an + # unsupported-format error) defers to the next backend; anything else raises. + if isinstance(path, BytesIO): + path.seek(0) + try: + return load_audio_torchcodec( + path, + sr=sr, + mono=mono, + max_duration_s=max_duration_s, + max_decode_bytes=max_decode_bytes, + ) + except ImportError as exc: + # Decode errors don't defer: FFmpeg-based PyAV would fail the same way. + logger.warning( + "torchcodec unavailable (%r); falling back to soundfile/PyAV.", exc + ) + if isinstance(path, BytesIO): path.seek(0) try: @@ -474,21 +502,6 @@ def load_audio( if exc.code not in _BAD_SF_CODES: raise - if isinstance(path, BytesIO): - path.seek(0) - try: - return load_audio_torchcodec( - path, - sr=sr, - mono=mono, - max_duration_s=max_duration_s, - max_decode_bytes=max_decode_bytes, - ) - except ImportError as exc: - # Decode errors don't defer: torchcodec is FFmpeg-based like PyAV, so - # retrying would just fail the same way. - logger.warning("torchcodec unavailable (%r); falling back to PyAV.", exc) - # PyAV is terminal: nothing left to fall back to. Normalize an FFmpeg # failure to ValueError; let a missing soundfile/PyAV surface its # "install vllm[audio]" ImportError.