iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Bugfix] Fix TorchCodec audio IO correctness (#58364)

Browse Source 
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-09-29 11:34:48 -05:00
committed by GitHub
parent c3a4a36c20
commit 31876d8ea1
3 changed files with 67 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/features/multimodal_inputs.md
+10 -9
View File
@@ -1210,7 +1210,7 @@ vLLM decodes audio bytes into waveforms using a selectable decoding backend:
| Backend | Description |
| --- | --- |
| `auto` (default) | soundfile, falling back to torchcodec, then PyAV |
| `auto` (default) | torchcodec, falling back to soundfile, then PyAV |
| `soundfile` | libsndfile only, no fallback |
| `pyav` | PyAV (FFmpeg) only, no fallback |
| `torchcodec` | TorchCodec (PyTorch-native) only, no fallback |
@@ -1226,17 +1226,18 @@ vllm serve mistralai/Voxtral-Mini-3B-2507 \
`pyav` drives FFmpeg through a per-frame Python generator, so under
concurrency the Python/C crossings contend on the GIL. `torchcodec`
decodes each stream in a single call that releases the GIL for its whole
duration. Select it explicitly for concurrent decoding workloads.
`auto` prefers soundfile for supported formats to preserve their existing
decoding behavior, including encoder padding. Audio extracted from formats
that soundfile cannot read, such as video containers, can use torchcodec
through the fallback chain.
duration, which is why `auto` prefers it when many requests decode audio
concurrently, such as when audio tracks are extracted from video.
vLLM normalizes codec padding on this path (e.g. it trims trailing Vorbis
padding that older system FFmpeg versions (< 5.0) fail to trim), so the
decoded waveform matches the soundfile reference length.
!!! note
`torchcodec` ships as a requirement on CUDA, CPU and XPU builds. On other
platforms (e.g. ROCm, TPU), install it manually to enable that backend.
torchcodec also links against a system FFmpeg installation. If the package
or FFmpeg is unavailable, `auto` uses the soundfile → PyAV chain.
platforms (e.g. ROCm, TPU) `auto` falls back to the soundfile → PyAV
chain unless you install it manually. torchcodec also links against a
system FFmpeg installation; if FFmpeg is missing, `auto` falls back the
same way.
### Embedding Inputs
tests/multimodal/media/test_audio.py
+20 -3
View File
@@ -185,9 +185,15 @@ def test_load_audio_backend_matches_default(backend, dummy_audio_bytes):
ref_audio, ref_sr = load_audio(BytesIO(dummy_audio_bytes), sr=None)
audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend=backend)
assert sr == ref_sr
# Decoders disagree only on codec encoder-delay/padding, so torchcodec may
# emit a few extra trailing samples (e.g. ~192 for Ogg Vorbis). Compare the
# overlapping region, which must agree to float32 precision.
if backend == "torchcodec":
# Vorbis encoder padding is trimmed, so torchcodec must match the
# soundfile reference exactly, including the waveform length.
assert audio.shape == ref_audio.shape
np.testing.assert_allclose(ref_audio, audio, atol=1e-4)
return
# Decoders disagree only on codec encoder-delay/padding, so PyAV may
# emit a few extra trailing samples (e.g. ~192 for Ogg Vorbis). Compare
# the overlapping region, which must agree to float32 precision.
n = min(ref_audio.shape[-1], audio.shape[-1])
assert n > 0
np.testing.assert_allclose(ref_audio[:n], audio[:n], atol=1e-4)
@@ -201,6 +207,17 @@ def test_load_audio_default_preserves_vorbis_length(dummy_audio_bytes):
assert audio.shape == expected.shape
def test_load_audio_torchcodec_preserves_vorbis_length(dummy_audio_bytes):
"""Torchcodec must trim Vorbis trailing padding even on FFmpeg < 5.0,
matching the soundfile reference exactly."""
pytest.importorskip("torchcodec")
expected, expected_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None)
audio, sr = load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None)
assert sr == expected_sr
assert audio.shape == expected.shape
np.testing.assert_allclose(audio, expected, atol=1e-4)
def test_load_audio_unknown_backend_rejected(dummy_audio_bytes):
"""An unknown backend must fail loudly instead of silently degrading."""
with pytest.raises(ValueError, match="Unknown audio backend"):
vllm/multimodal/media/audio.py
+37 -24
View File
@@ -15,6 +15,7 @@ from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.multimodal.audio import resample_audio_pyav
from vllm.utils.import_utils import PlaceholderModule
from vllm.utils.math_utils import cdiv
from vllm.utils.mem_constants import MiB_bytes
from vllm.utils.serial_utils import tensor2base64
from vllm.utils.sparse_utils import (
@@ -60,11 +61,11 @@ _BAD_SF_CODES = {0, 1, 3, 4}
# Audio decoding backends selectable via `load_audio(backend=...)` or
# `--media-io-kwargs '{"audio": {"audio_backend": ...}}'`.
# "auto" tries soundfile, then torchcodec, then PyAV.
# "auto" tries torchcodec, then soundfile, then PyAV.
AUDIO_BACKENDS = ("auto", "soundfile", "pyav", "torchcodec")
# Raised as ImportError when the torchcodec backend cannot be used, so
# `load_audio(backend="auto")` can continue to PyAV.
# `load_audio(backend="auto")` falls back to the soundfile → PyAV chain.
_TORCHCODEC_UNAVAILABLE_MSG = (
"torchcodec audio backend is unavailable (requires the torchcodec "
"package and a system ffmpeg installation)"
@@ -74,6 +75,9 @@ _TORCHCODEC_UNAVAILABLE_MSG = (
# over-long stream is rejected rather than truncated to the limit.
_DURATION_GUARD_MARGIN_S = 1.0
# Largest Vorbis block size in native-rate samples
_VORBIS_MAX_BLOCK_SAMPLES = 8192
def load_audio_pyav(
path: BytesIO | Path | str,
@@ -364,6 +368,16 @@ def load_audio_torchcodec(
"audio or video file (e.g. a complete WAV, MP3, or MP4)."
) from e
# FFmpeg < 5.0 misses Ogg end_trimming, leaving up to one Vorbis block of
# padding at the tail; we need to trim for valid audio length.
if metadata.codec == "vorbis" and metadata.duration_seconds is not None:
expected_samples = round(metadata.duration_seconds * samples.sample_rate)
padding = samples.data.shape[-1] - expected_samples
native_sr = metadata.sample_rate or samples.sample_rate
max_padding = cdiv(_VORBIS_MAX_BLOCK_SAMPLES * samples.sample_rate, native_sr)
if 0 < padding <= max_padding:
samples.data = samples.data[..., :expected_samples]
# Authoritative post-decode checks: the metadata estimates above can be
# bypassed by a container that lies about its duration or size.
if max_duration_s is not None:
@@ -425,8 +439,8 @@ def load_audio(
max_decode_bytes: Reject audio that would decode to more than this
many bytes.
backend: One of ``AUDIO_BACKENDS``. ``None`` (default) selects
``"auto"``, which tries soundfile, then torchcodec, then PyAV;
the other values select a single
``"auto"``, which tries torchcodec, then falls back to the
soundfile → PyAV chain; the other values select a single
backend with no fallback.
"""
@@ -446,11 +460,25 @@ def load_audio(
max_decode_bytes=max_decode_bytes,
)
# Keep soundfile first to preserve decoding and padding of supported formats.
# "auto": soundfile → torchcodec → PyAV. Each backend is called by its
# bare name so tests that monkeypatch it take effect. Only an ImportError
# (backend missing) or, for soundfile, an unsupported-format error defers
# to the next; any other error (decode failure, guard ValueError) raises.
# "auto": torchcodec → soundfile → PyAV, each called by its bare name so
# monkeypatched tests take effect. Only ImportError (or, for soundfile, an
# unsupported-format error) defers to the next backend; anything else raises.
if isinstance(path, BytesIO):
path.seek(0)
try:
return load_audio_torchcodec(
path,
sr=sr,
mono=mono,
max_duration_s=max_duration_s,
max_decode_bytes=max_decode_bytes,
)
except ImportError as exc:
# Decode errors don't defer: FFmpeg-based PyAV would fail the same way.
logger.warning(
"torchcodec unavailable (%r); falling back to soundfile/PyAV.", exc
)
if isinstance(path, BytesIO):
path.seek(0)
try:
@@ -474,21 +502,6 @@ def load_audio(
if exc.code not in _BAD_SF_CODES:
raise
if isinstance(path, BytesIO):
path.seek(0)
try:
return load_audio_torchcodec(
path,
sr=sr,
mono=mono,
max_duration_s=max_duration_s,
max_decode_bytes=max_decode_bytes,
)
except ImportError as exc:
# Decode errors don't defer: torchcodec is FFmpeg-based like PyAV, so
# retrying would just fail the same way.
logger.warning("torchcodec unavailable (%r); falling back to PyAV.", exc)
# PyAV is terminal: nothing left to fall back to. Normalize an FFmpeg
# failure to ValueError; let a missing soundfile/PyAV surface its
# "install vllm[audio]" ImportError.