[Bugfix][Audio] Restore soundfile-first automatic decoding (#55642)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
This commit is contained in:
@@ -1210,7 +1210,7 @@ vLLM decodes audio bytes into waveforms using a selectable decoding backend:
|
|||||||
|
|
||||||
| Backend | Description |
|
| Backend | Description |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `auto` (default) | torchcodec, falling back to soundfile, then PyAV |
|
| `auto` (default) | soundfile, falling back to torchcodec, then PyAV |
|
||||||
| `soundfile` | libsndfile only, no fallback |
|
| `soundfile` | libsndfile only, no fallback |
|
||||||
| `pyav` | PyAV (FFmpeg) only, no fallback |
|
| `pyav` | PyAV (FFmpeg) only, no fallback |
|
||||||
| `torchcodec` | TorchCodec (PyTorch-native) only, no fallback |
|
| `torchcodec` | TorchCodec (PyTorch-native) only, no fallback |
|
||||||
@@ -1226,15 +1226,17 @@ vllm serve mistralai/Voxtral-Mini-3B-2507 \
|
|||||||
`pyav` drives FFmpeg through a per-frame Python generator, so under
|
`pyav` drives FFmpeg through a per-frame Python generator, so under
|
||||||
concurrency the Python/C crossings contend on the GIL. `torchcodec`
|
concurrency the Python/C crossings contend on the GIL. `torchcodec`
|
||||||
decodes each stream in a single call that releases the GIL for its whole
|
decodes each stream in a single call that releases the GIL for its whole
|
||||||
duration, which is why `auto` prefers it when many requests decode audio
|
duration. Select it explicitly for concurrent decoding workloads.
|
||||||
concurrently, such as when audio tracks are extracted from video.
|
`auto` prefers soundfile for supported formats to preserve their existing
|
||||||
|
decoding behavior, including encoder padding. Audio extracted from formats
|
||||||
|
that soundfile cannot read, such as video containers, can use torchcodec
|
||||||
|
through the fallback chain.
|
||||||
|
|
||||||
!!! note
|
!!! note
|
||||||
`torchcodec` ships as a requirement on CUDA, CPU and XPU builds. On other
|
`torchcodec` ships as a requirement on CUDA, CPU and XPU builds. On other
|
||||||
platforms (e.g. ROCm, TPU) `auto` falls back to the soundfile → PyAV
|
platforms (e.g. ROCm, TPU), install it manually to enable that backend.
|
||||||
chain unless you install it manually. torchcodec also links against a
|
torchcodec also links against a system FFmpeg installation. If the package
|
||||||
system FFmpeg installation; if FFmpeg is missing, `auto` falls back the
|
or FFmpeg is unavailable, `auto` uses the soundfile → PyAV chain.
|
||||||
same way.
|
|
||||||
|
|
||||||
### Embedding Inputs
|
### Embedding Inputs
|
||||||
|
|
||||||
|
|||||||
@@ -193,6 +193,14 @@ def test_load_audio_backend_matches_default(backend, dummy_audio_bytes):
|
|||||||
np.testing.assert_allclose(ref_audio[:n], audio[:n], atol=1e-4)
|
np.testing.assert_allclose(ref_audio[:n], audio[:n], atol=1e-4)
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_audio_default_preserves_vorbis_length(dummy_audio_bytes):
|
||||||
|
"""Default decoding must not append Vorbis padding to the speech waveform."""
|
||||||
|
expected, expected_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None)
|
||||||
|
audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None)
|
||||||
|
assert sr == expected_sr
|
||||||
|
assert audio.shape == expected.shape
|
||||||
|
|
||||||
|
|
||||||
def test_load_audio_unknown_backend_rejected(dummy_audio_bytes):
|
def test_load_audio_unknown_backend_rejected(dummy_audio_bytes):
|
||||||
"""An unknown backend must fail loudly instead of silently degrading."""
|
"""An unknown backend must fail loudly instead of silently degrading."""
|
||||||
with pytest.raises(ValueError, match="Unknown audio backend"):
|
with pytest.raises(ValueError, match="Unknown audio backend"):
|
||||||
|
|||||||
@@ -60,11 +60,11 @@ _BAD_SF_CODES = {0, 1, 3, 4}
|
|||||||
|
|
||||||
# Audio decoding backends selectable via `load_audio(backend=...)` or
|
# Audio decoding backends selectable via `load_audio(backend=...)` or
|
||||||
# `--media-io-kwargs '{"audio": {"audio_backend": ...}}'`.
|
# `--media-io-kwargs '{"audio": {"audio_backend": ...}}'`.
|
||||||
# "auto" tries torchcodec, then soundfile, then PyAV.
|
# "auto" tries soundfile, then torchcodec, then PyAV.
|
||||||
AUDIO_BACKENDS = ("auto", "soundfile", "pyav", "torchcodec")
|
AUDIO_BACKENDS = ("auto", "soundfile", "pyav", "torchcodec")
|
||||||
|
|
||||||
# Raised as ImportError when the torchcodec backend cannot be used, so
|
# Raised as ImportError when the torchcodec backend cannot be used, so
|
||||||
# `load_audio(backend="auto")` falls back to the soundfile → PyAV chain.
|
# `load_audio(backend="auto")` can continue to PyAV.
|
||||||
_TORCHCODEC_UNAVAILABLE_MSG = (
|
_TORCHCODEC_UNAVAILABLE_MSG = (
|
||||||
"torchcodec audio backend is unavailable (requires the torchcodec "
|
"torchcodec audio backend is unavailable (requires the torchcodec "
|
||||||
"package and a system ffmpeg installation)"
|
"package and a system ffmpeg installation)"
|
||||||
@@ -413,8 +413,8 @@ def load_audio(
|
|||||||
|
|
||||||
Args:
|
Args:
|
||||||
backend: One of ``AUDIO_BACKENDS``. ``None`` (default) selects
|
backend: One of ``AUDIO_BACKENDS``. ``None`` (default) selects
|
||||||
``"auto"``, which tries torchcodec, then falls back to the
|
``"auto"``, which tries soundfile, then torchcodec, then PyAV;
|
||||||
soundfile → PyAV chain; the other values select a single
|
the other values select a single
|
||||||
backend with no fallback.
|
backend with no fallback.
|
||||||
"""
|
"""
|
||||||
backend = backend or "auto"
|
backend = backend or "auto"
|
||||||
@@ -433,27 +433,11 @@ def load_audio(
|
|||||||
max_decode_bytes=max_decode_bytes,
|
max_decode_bytes=max_decode_bytes,
|
||||||
)
|
)
|
||||||
|
|
||||||
# "auto": torchcodec → soundfile → PyAV. Each backend is called by its
|
# Keep soundfile first to preserve decoding and padding of supported formats.
|
||||||
|
# "auto": soundfile → torchcodec → PyAV. Each backend is called by its
|
||||||
# bare name so tests that monkeypatch it take effect. Only an ImportError
|
# bare name so tests that monkeypatch it take effect. Only an ImportError
|
||||||
# (backend missing) or, for soundfile, an unsupported-format error defers
|
# (backend missing) or, for soundfile, an unsupported-format error defers
|
||||||
# to the next; any other error (decode failure, guard ValueError) raises.
|
# to the next; any other error (decode failure, guard ValueError) raises.
|
||||||
if isinstance(path, BytesIO):
|
|
||||||
path.seek(0)
|
|
||||||
try:
|
|
||||||
return load_audio_torchcodec(
|
|
||||||
path,
|
|
||||||
sr=sr,
|
|
||||||
mono=mono,
|
|
||||||
max_duration_s=max_duration_s,
|
|
||||||
max_decode_bytes=max_decode_bytes,
|
|
||||||
)
|
|
||||||
except ImportError as exc:
|
|
||||||
# Decode errors don't defer: torchcodec is FFmpeg-based like PyAV, so
|
|
||||||
# retrying would just fail the same way.
|
|
||||||
logger.warning(
|
|
||||||
"torchcodec unavailable (%r); falling back to soundfile/PyAV.", exc
|
|
||||||
)
|
|
||||||
|
|
||||||
if isinstance(path, BytesIO):
|
if isinstance(path, BytesIO):
|
||||||
path.seek(0)
|
path.seek(0)
|
||||||
try:
|
try:
|
||||||
@@ -477,6 +461,21 @@ def load_audio(
|
|||||||
if exc.code not in _BAD_SF_CODES:
|
if exc.code not in _BAD_SF_CODES:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
if isinstance(path, BytesIO):
|
||||||
|
path.seek(0)
|
||||||
|
try:
|
||||||
|
return load_audio_torchcodec(
|
||||||
|
path,
|
||||||
|
sr=sr,
|
||||||
|
mono=mono,
|
||||||
|
max_duration_s=max_duration_s,
|
||||||
|
max_decode_bytes=max_decode_bytes,
|
||||||
|
)
|
||||||
|
except ImportError as exc:
|
||||||
|
# Decode errors don't defer: torchcodec is FFmpeg-based like PyAV, so
|
||||||
|
# retrying would just fail the same way.
|
||||||
|
logger.warning("torchcodec unavailable (%r); falling back to PyAV.", exc)
|
||||||
|
|
||||||
# PyAV is terminal: nothing left to fall back to. Normalize an FFmpeg
|
# PyAV is terminal: nothing left to fall back to. Normalize an FFmpeg
|
||||||
# failure to ValueError; let a missing soundfile/PyAV surface its
|
# failure to ValueError; let a missing soundfile/PyAV surface its
|
||||||
# "install vllm[audio]" ImportError.
|
# "install vllm[audio]" ImportError.
|
||||||
|
|||||||
Reference in New Issue
Block a user