[feat] add torchcodec as audio loader and implement selective audio backend (#51826)
Signed-off-by: JaredforReal <w13431838023@gmail.com> Co-authored-by: saltman155 <saltman155@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
saltman155
Isotr0py
parent
131e0285f6
commit
6a039f465e
@@ -1204,6 +1204,38 @@ Full example: [examples/generate/multimodal/openai_chat_completion_client_for_mu
|
||||
export VLLM_AUDIO_FETCH_TIMEOUT=<timeout>
|
||||
```
|
||||
|
||||
#### Audio Decoding Backend
|
||||
|
||||
vLLM decodes audio bytes into waveforms using a selectable decoding backend:
|
||||
|
||||
| Backend | Description |
|
||||
| --- | --- |
|
||||
| `auto` (default) | torchcodec, falling back to soundfile, then PyAV |
|
||||
| `soundfile` | libsndfile only, no fallback |
|
||||
| `pyav` | PyAV (FFmpeg) only, no fallback |
|
||||
| `torchcodec` | TorchCodec (PyTorch-native) only, no fallback |
|
||||
|
||||
Select the backend per server via `--media-io-kwargs`:
|
||||
|
||||
```bash
|
||||
vllm serve mistralai/Voxtral-Mini-3B-2507 \
|
||||
--media-io-kwargs '{"audio": {"audio_backend": "soundfile"}}'
|
||||
```
|
||||
|
||||
!!! tip
|
||||
`pyav` drives FFmpeg through a per-frame Python generator, so under
|
||||
concurrency the Python/C crossings contend on the GIL. `torchcodec`
|
||||
decodes each stream in a single call that releases the GIL for its whole
|
||||
duration, which is why `auto` prefers it when many requests decode audio
|
||||
concurrently, such as when audio tracks are extracted from video.
|
||||
|
||||
!!! note
|
||||
`torchcodec` ships as a requirement on CUDA, CPU and XPU builds. On other
|
||||
platforms (e.g. ROCm, TPU) `auto` falls back to the soundfile → PyAV
|
||||
chain unless you install it manually. torchcodec also links against a
|
||||
system FFmpeg installation; if FFmpeg is missing, `auto` falls back the
|
||||
same way.
|
||||
|
||||
### Embedding Inputs
|
||||
|
||||
To input pre-computed embeddings belonging to a data type (i.e. image, video, or audio) directly to the language model,
|
||||
|
||||
+1
-1
@@ -172,7 +172,7 @@ async def test_invalid_audio_file(whisper_client):
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "Invalid or unsupported audio file" in exc_info.value.message
|
||||
assert "Invalid or corrupted audio data" in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -10,7 +10,12 @@ import pytest
|
||||
import soundfile as sf
|
||||
|
||||
from vllm.multimodal.media import AudioMediaIO
|
||||
from vllm.multimodal.media.audio import load_audio, load_audio_soundfile
|
||||
from vllm.multimodal.media import audio as audio_module
|
||||
from vllm.multimodal.media.audio import (
|
||||
load_audio,
|
||||
load_audio_soundfile,
|
||||
load_audio_torchcodec,
|
||||
)
|
||||
|
||||
from ...conftest import AudioTestAssets
|
||||
|
||||
@@ -170,3 +175,97 @@ def test_load_audio_threads_max_decode_bytes():
|
||||
max_duration_s=600,
|
||||
max_decode_bytes=512 * 1024,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["soundfile", "pyav", "torchcodec"])
|
||||
def test_load_audio_backend_matches_default(backend, dummy_audio_bytes):
|
||||
"""Every explicit backend must decode to the same samples as `auto`."""
|
||||
if backend == "torchcodec":
|
||||
pytest.importorskip("torchcodec")
|
||||
ref_audio, ref_sr = load_audio(BytesIO(dummy_audio_bytes), sr=None)
|
||||
audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend=backend)
|
||||
assert sr == ref_sr
|
||||
# Decoders disagree only on codec encoder-delay/padding, so torchcodec may
|
||||
# emit a few extra trailing samples (e.g. ~192 for Ogg Vorbis). Compare the
|
||||
# overlapping region, which must agree to float32 precision.
|
||||
n = min(ref_audio.shape[-1], audio.shape[-1])
|
||||
assert n > 0
|
||||
np.testing.assert_allclose(ref_audio[:n], audio[:n], atol=1e-4)
|
||||
|
||||
|
||||
def test_load_audio_unknown_backend_rejected(dummy_audio_bytes):
|
||||
"""An unknown backend must fail loudly instead of silently degrading."""
|
||||
with pytest.raises(ValueError, match="Unknown audio backend"):
|
||||
load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="not_a_backend")
|
||||
|
||||
|
||||
def test_load_audio_auto_falls_back_without_torchcodec(dummy_audio_bytes):
|
||||
"""`auto` must fall back to the soundfile → PyAV chain when torchcodec
|
||||
is not importable."""
|
||||
ref_audio, ref_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None)
|
||||
with patch.object(audio_module, "load_audio_torchcodec", side_effect=ImportError):
|
||||
audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="auto")
|
||||
assert sr == ref_sr
|
||||
np.testing.assert_array_equal(ref_audio, audio)
|
||||
|
||||
|
||||
def test_load_audio_auto_falls_back_without_ffmpeg(dummy_audio_bytes):
|
||||
"""torchcodec installed but system ffmpeg missing (`AudioDecoder is None`)
|
||||
must surface as ImportError so `auto` falls back to soundfile → PyAV."""
|
||||
ref_audio, ref_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None)
|
||||
with patch.object(audio_module, "AudioDecoder", None):
|
||||
with pytest.raises(ImportError, match="torchcodec audio backend"):
|
||||
load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None)
|
||||
audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="auto")
|
||||
assert sr == ref_sr
|
||||
np.testing.assert_array_equal(ref_audio, audio)
|
||||
|
||||
|
||||
def test_load_audio_auto_falls_back_when_libtorchcodec_unloadable(
|
||||
dummy_audio_bytes,
|
||||
):
|
||||
"""torchcodec loads its ffmpeg-backed core lazily at decoder construction;
|
||||
a "Could not load libtorchcodec" RuntimeError there (no system ffmpeg)
|
||||
must also surface as ImportError so `auto` falls back."""
|
||||
|
||||
class _UnloadableDecoder:
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise RuntimeError("Could not load libtorchcodec. Likely causes: ...")
|
||||
|
||||
ref_audio, ref_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None)
|
||||
with patch.object(audio_module, "AudioDecoder", _UnloadableDecoder):
|
||||
with pytest.raises(ImportError, match="torchcodec audio backend"):
|
||||
load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None)
|
||||
audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="auto")
|
||||
assert sr == ref_sr
|
||||
np.testing.assert_array_equal(ref_audio, audio)
|
||||
|
||||
|
||||
def test_audio_media_io_audio_backend_kwarg(dummy_audio_bytes):
|
||||
"""`audio_backend` selects the backend; unknown values fail at init."""
|
||||
audio, sr = AudioMediaIO(audio_backend="pyav").load_bytes(dummy_audio_bytes)
|
||||
assert isinstance(audio, np.ndarray)
|
||||
assert sr == 16000
|
||||
with pytest.raises(ValueError, match="Unknown audio_backend"):
|
||||
AudioMediaIO(audio_backend="not_a_backend")
|
||||
|
||||
|
||||
def test_torchcodec_max_duration_rejected(dummy_audio_bytes):
|
||||
"""The decompression-bomb duration guard must hold for torchcodec too."""
|
||||
pytest.importorskip("torchcodec")
|
||||
with pytest.raises(ValueError, match="exceeds maximum allowed duration"):
|
||||
load_audio_torchcodec(
|
||||
BytesIO(dummy_audio_bytes), sr=None, max_duration_s=0.0001
|
||||
)
|
||||
|
||||
|
||||
def test_torchcodec_max_decode_bytes_rejected(dummy_audio_bytes):
|
||||
"""The decompression-bomb memory guard must hold for torchcodec too."""
|
||||
pytest.importorskip("torchcodec")
|
||||
audio, _ = load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None)
|
||||
with pytest.raises(ValueError, match="VLLM_MAX_AUDIO_DECODE_BYTES"):
|
||||
load_audio_torchcodec(
|
||||
BytesIO(dummy_audio_bytes),
|
||||
sr=None,
|
||||
max_decode_bytes=audio.nbytes - 1,
|
||||
)
|
||||
|
||||
+311
-63
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
@@ -35,6 +36,15 @@ try:
|
||||
except ImportError:
|
||||
soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
|
||||
|
||||
try:
|
||||
from torchcodec.decoders import AudioDecoder
|
||||
except (ImportError, RuntimeError) as exc:
|
||||
# RuntimeError: torchcodec is installed but the system ffmpeg is missing.
|
||||
AudioDecoder = None # type: ignore[assignment]
|
||||
_torchcodec_import_exc: BaseException | None = exc
|
||||
else:
|
||||
_torchcodec_import_exc = None
|
||||
|
||||
|
||||
# Public libsndfile error codes exposed via `soundfile.LibsndfileError.code`,
|
||||
# soundfile being the main audio loading backend. Used to validate if an audio
|
||||
@@ -48,6 +58,22 @@ except ImportError:
|
||||
# 4 = unsupported encoding (codec not supported by this libsndfile build)
|
||||
_BAD_SF_CODES = {0, 1, 3, 4}
|
||||
|
||||
# Audio decoding backends selectable via `load_audio(backend=...)` or
|
||||
# `--media-io-kwargs '{"audio": {"audio_backend": ...}}'`.
|
||||
# "auto" tries torchcodec, then soundfile, then PyAV.
|
||||
AUDIO_BACKENDS = ("auto", "soundfile", "pyav", "torchcodec")
|
||||
|
||||
# Raised as ImportError when the torchcodec backend cannot be used, so
|
||||
# `load_audio(backend="auto")` falls back to the soundfile → PyAV chain.
|
||||
_TORCHCODEC_UNAVAILABLE_MSG = (
|
||||
"torchcodec audio backend is unavailable (requires the torchcodec "
|
||||
"package and a system ffmpeg installation)"
|
||||
)
|
||||
|
||||
# Slack on the torchcodec decode window when enforcing `max_duration_s`, so an
|
||||
# over-long stream is rejected rather than truncated to the limit.
|
||||
_DURATION_GUARD_MARGIN_S = 1.0
|
||||
|
||||
|
||||
def load_audio_pyav(
|
||||
path: BytesIO | Path | str,
|
||||
@@ -75,54 +101,56 @@ def load_audio_pyav(
|
||||
``(waveform, sample_rate)`` where *waveform* is a 1-D float32
|
||||
NumPy array and *sample_rate* is the native sample rate in Hz.
|
||||
"""
|
||||
native_sr = None
|
||||
try:
|
||||
with av.open(path) as container:
|
||||
if not container.streams.audio:
|
||||
raise ValueError("No audio stream found.")
|
||||
stream = container.streams.audio[0]
|
||||
stream.thread_type = "AUTO"
|
||||
native_sr = stream.rate
|
||||
sr = sr or native_sr
|
||||
container = av.open(path)
|
||||
except av.error.FFmpegError as e:
|
||||
raise ValueError(
|
||||
"Invalid or corrupted audio data. Ensure the input is a valid "
|
||||
"audio or video file (e.g. a complete WAV, MP3, or MP4)."
|
||||
) from e
|
||||
|
||||
# Early rejection from container/stream metadata to avoid
|
||||
# wasting resources on decoding decompression bombs.
|
||||
if max_duration_s is not None:
|
||||
metadata_duration_s = None
|
||||
if stream.duration and stream.time_base:
|
||||
metadata_duration_s = float(stream.duration * stream.time_base)
|
||||
elif container.duration:
|
||||
metadata_duration_s = container.duration / 1_000_000
|
||||
if (
|
||||
metadata_duration_s is not None
|
||||
and metadata_duration_s > max_duration_s
|
||||
):
|
||||
raise ValueError(
|
||||
f"Audio exceeds maximum allowed duration of "
|
||||
f"{max_duration_s}s (metadata reports "
|
||||
f"{metadata_duration_s:.1f}s). Set "
|
||||
f"VLLM_MAX_AUDIO_DECODE_DURATION_S to "
|
||||
f"increase this limit."
|
||||
)
|
||||
with container:
|
||||
if not container.streams.audio:
|
||||
raise ValueError("No audio stream found.")
|
||||
stream = container.streams.audio[0]
|
||||
stream.thread_type = "AUTO"
|
||||
native_sr = stream.rate
|
||||
sr = sr or native_sr
|
||||
|
||||
max_samples = (
|
||||
int(sr * max_duration_s) if max_duration_s is not None else None
|
||||
)
|
||||
total_samples = 0
|
||||
total_decode_bytes = 0
|
||||
# Early rejection from container/stream metadata to avoid
|
||||
# wasting resources on decoding decompression bombs.
|
||||
if max_duration_s is not None:
|
||||
metadata_duration_s = None
|
||||
if stream.duration and stream.time_base:
|
||||
metadata_duration_s = float(stream.duration * stream.time_base)
|
||||
elif container.duration:
|
||||
metadata_duration_s = container.duration / 1_000_000
|
||||
if metadata_duration_s is not None and metadata_duration_s > max_duration_s:
|
||||
raise ValueError(
|
||||
f"Audio exceeds maximum allowed duration of "
|
||||
f"{max_duration_s}s (metadata reports "
|
||||
f"{metadata_duration_s:.1f}s). Set "
|
||||
f"VLLM_MAX_AUDIO_DECODE_DURATION_S to "
|
||||
f"increase this limit."
|
||||
)
|
||||
|
||||
chunks: list[npt.NDArray] = []
|
||||
needs_resampling = not math.isclose(
|
||||
float(sr),
|
||||
float(native_sr),
|
||||
rel_tol=0.0,
|
||||
abs_tol=1e-6,
|
||||
)
|
||||
resampler = (
|
||||
av.AudioResampler(format="fltp", layout="mono", rate=sr)
|
||||
if needs_resampling
|
||||
else None
|
||||
)
|
||||
max_samples = int(sr * max_duration_s) if max_duration_s is not None else None
|
||||
total_samples = 0
|
||||
total_decode_bytes = 0
|
||||
|
||||
chunks: list[npt.NDArray] = []
|
||||
needs_resampling = not math.isclose(
|
||||
float(sr),
|
||||
float(native_sr),
|
||||
rel_tol=0.0,
|
||||
abs_tol=1e-6,
|
||||
)
|
||||
resampler = (
|
||||
av.AudioResampler(format="fltp", layout="mono", rate=sr)
|
||||
if needs_resampling
|
||||
else None
|
||||
)
|
||||
try:
|
||||
for frame in container.decode(stream):
|
||||
if needs_resampling:
|
||||
assert resampler is not None
|
||||
@@ -157,16 +185,14 @@ def load_audio_pyav(
|
||||
f"VLLM_MAX_AUDIO_DECODE_BYTES to increase this "
|
||||
f"limit."
|
||||
)
|
||||
except (ValueError, ImportError):
|
||||
raise
|
||||
except Exception as e:
|
||||
raise ValueError(
|
||||
"Invalid or corrupted video data when extracting audio. "
|
||||
"Ensure the input is valid video bytes (e.g. a complete MP4)."
|
||||
) from e
|
||||
except av.error.FFmpegError as e:
|
||||
raise ValueError(
|
||||
"Invalid or corrupted audio data. Ensure the input is a valid "
|
||||
"audio or video file (e.g. a complete WAV, MP3, or MP4)."
|
||||
) from e
|
||||
|
||||
if not chunks:
|
||||
raise ValueError("No audio found in the video.")
|
||||
raise ValueError("No audio found in the input.")
|
||||
|
||||
audio = np.concatenate(chunks, axis=-1).astype(np.float32)
|
||||
if mono and audio.ndim > 1:
|
||||
@@ -217,6 +243,163 @@ def load_audio_soundfile(
|
||||
return y, native_sr
|
||||
|
||||
|
||||
def load_audio_torchcodec(
|
||||
path: BytesIO | Path | str,
|
||||
*,
|
||||
sr: float | None = 22050,
|
||||
mono: bool = True,
|
||||
max_duration_s: float | None = None,
|
||||
max_decode_bytes: int | None = None,
|
||||
) -> tuple[npt.NDArray, float]:
|
||||
"""Load an audio file using torchcodec, returning a float32 waveform.
|
||||
|
||||
Unlike :func:`load_audio_pyav`, which drives FFmpeg through a per-frame
|
||||
Python generator, torchcodec decodes inside a single call that releases
|
||||
the GIL. Resampling also happens in C++ (``sample_rate=sr``), so this
|
||||
backend never falls back to PyAV or libsndfile.
|
||||
|
||||
Args:
|
||||
path: A :class:`~io.BytesIO` buffer, a filesystem
|
||||
:class:`~pathlib.Path`, or a string path.
|
||||
sr: Output sample rate; ``None`` keeps the native rate. Must be an
|
||||
integer when set.
|
||||
mono: Average channels down to a single channel.
|
||||
max_duration_s: Reject audio longer than this many seconds.
|
||||
Prevents decompression-bomb attacks where a small compressed
|
||||
file expands into gigabytes of PCM.
|
||||
max_decode_bytes: Reject audio whose decoded PCM exceeds this many
|
||||
bytes.
|
||||
|
||||
Returns:
|
||||
``(waveform, sample_rate)`` where *waveform* is a float32 NumPy
|
||||
array (1-D when ``mono=True``) and *sample_rate* is the output
|
||||
sample rate in Hz.
|
||||
"""
|
||||
if AudioDecoder is None:
|
||||
# Unify "torchcodec not installed" and "system ffmpeg missing" into
|
||||
# ImportError so `load_audio(backend="auto")` falls back cleanly.
|
||||
raise ImportError(_TORCHCODEC_UNAVAILABLE_MSG) from _torchcodec_import_exc
|
||||
if sr is not None and sr != int(sr):
|
||||
raise ValueError(f"torchcodec requires an integer sample rate, got {sr}")
|
||||
sample_rate = int(sr) if sr is not None else None
|
||||
|
||||
# Opening the source and reading container metadata both touch untrusted
|
||||
# input; torchcodec signals corrupt/unsupported data with RuntimeError.
|
||||
try:
|
||||
decoder = AudioDecoder(path, sample_rate=sample_rate)
|
||||
metadata = decoder.metadata
|
||||
except RuntimeError as e:
|
||||
# torchcodec loads its ffmpeg-backed core lazily at construction, so
|
||||
# an ffmpeg-less (or ABI-incompatible) install can fail here rather
|
||||
# than at import. Treat that as backend-unavailable, not bad input.
|
||||
if "Could not load libtorchcodec" in str(e):
|
||||
raise ImportError(_TORCHCODEC_UNAVAILABLE_MSG) from e
|
||||
raise ValueError(
|
||||
"Invalid or corrupted audio data. Ensure the input is a valid "
|
||||
"audio or video file (e.g. a complete WAV, MP3, or MP4)."
|
||||
) from e
|
||||
|
||||
# Pre-check the memory budget from container metadata so an obviously
|
||||
# oversized stream is rejected before allocating the PCM buffer. Metadata
|
||||
# is attacker-controlled, so this may only reject, never admit; the
|
||||
# post-decode check below is authoritative.
|
||||
if max_decode_bytes is not None:
|
||||
est_duration_s = metadata.duration_seconds
|
||||
# Estimate at the *output* rate: when `sr` is set torchcodec
|
||||
# resamples to it during decode.
|
||||
est_sample_rate = sr if sr is not None else metadata.sample_rate
|
||||
est_num_channels = metadata.num_channels
|
||||
if (
|
||||
est_duration_s is not None
|
||||
and est_sample_rate is not None
|
||||
and est_num_channels is not None
|
||||
):
|
||||
estimated_bytes = (
|
||||
int(est_duration_s * est_sample_rate)
|
||||
* est_num_channels
|
||||
* np.dtype(np.float32).itemsize
|
||||
)
|
||||
if estimated_bytes > max_decode_bytes:
|
||||
raise ValueError(
|
||||
f"Audio would allocate "
|
||||
f"{estimated_bytes / MiB_bytes:.0f} MiB of PCM "
|
||||
f"(~{est_duration_s:.1f}s at {est_sample_rate}Hz x "
|
||||
f"{est_num_channels}ch), exceeding the "
|
||||
f"{max_decode_bytes / MiB_bytes:.0f} MiB limit. Set "
|
||||
f"VLLM_MAX_AUDIO_DECODE_BYTES to increase this limit."
|
||||
)
|
||||
|
||||
# Duration pre-check, same attacker-controlled → reject-only semantics.
|
||||
if max_duration_s is not None:
|
||||
duration_s = metadata.duration_seconds
|
||||
if duration_s is not None and duration_s > max_duration_s:
|
||||
raise ValueError(
|
||||
f"Audio exceeds maximum allowed duration of "
|
||||
f"{max_duration_s}s (metadata reports "
|
||||
f"{duration_s:.1f}s). Set "
|
||||
f"VLLM_MAX_AUDIO_DECODE_DURATION_S to "
|
||||
f"increase this limit."
|
||||
)
|
||||
|
||||
# Decode the stream. RuntimeError here covers a corrupt stream or a
|
||||
# container that lies about its length. Bounding the range (when a
|
||||
# duration limit is set) keeps an under-reported duration from expanding
|
||||
# into unbounded PCM.
|
||||
try:
|
||||
if max_duration_s is None:
|
||||
samples = decoder.get_all_samples()
|
||||
else:
|
||||
samples = decoder.get_samples_played_in_range(
|
||||
0.0, max_duration_s + _DURATION_GUARD_MARGIN_S
|
||||
)
|
||||
except RuntimeError as e:
|
||||
raise ValueError(
|
||||
"Invalid or corrupted audio data. Ensure the input is a valid "
|
||||
"audio or video file (e.g. a complete WAV, MP3, or MP4)."
|
||||
) from e
|
||||
|
||||
# Authoritative post-decode checks: the metadata estimates above can be
|
||||
# bypassed by a container that lies about its duration or size.
|
||||
if max_duration_s is not None:
|
||||
# Same `int()` semantics as load_audio_pyav's
|
||||
# `total_samples > int(sr * max_duration_s)`.
|
||||
max_samples = int(samples.sample_rate * max_duration_s)
|
||||
if samples.data.shape[-1] > max_samples:
|
||||
raise ValueError(
|
||||
f"Audio exceeds maximum allowed duration of "
|
||||
f"{max_duration_s}s (decoded {samples.data.shape[-1]} "
|
||||
f"samples at {samples.sample_rate}Hz). Set "
|
||||
f"VLLM_MAX_AUDIO_DECODE_DURATION_S to "
|
||||
f"increase this limit."
|
||||
)
|
||||
if max_decode_bytes is not None and samples.data.nbytes > max_decode_bytes:
|
||||
raise ValueError(
|
||||
f"Audio decode exceeded "
|
||||
f"{max_decode_bytes / MiB_bytes:.0f} MiB memory "
|
||||
f"limit ({samples.data.nbytes / MiB_bytes:.0f} MiB decoded). "
|
||||
f"Set VLLM_MAX_AUDIO_DECODE_BYTES to increase this limit."
|
||||
)
|
||||
|
||||
out_sr = samples.sample_rate
|
||||
audio = samples.data.numpy() # (num_channels, num_samples), float32
|
||||
if audio.size == 0:
|
||||
raise ValueError("No audio found in the input.")
|
||||
|
||||
if mono and audio.ndim > 1:
|
||||
# Same reduction as load_audio_pyav, so both backends agree at the
|
||||
# native sample rate, which is what AudioMediaIO requests.
|
||||
audio = np.mean(audio, axis=0)
|
||||
|
||||
return audio, out_sr
|
||||
|
||||
|
||||
_AUDIO_LOADERS: dict[str, Callable[..., tuple[npt.NDArray, float]]] = {
|
||||
"torchcodec": load_audio_torchcodec,
|
||||
"pyav": load_audio_pyav,
|
||||
"soundfile": load_audio_soundfile,
|
||||
}
|
||||
|
||||
|
||||
def load_audio(
|
||||
path: BytesIO | Path | str,
|
||||
*,
|
||||
@@ -224,7 +407,55 @@ def load_audio(
|
||||
mono: bool = True,
|
||||
max_duration_s: float | None = None,
|
||||
max_decode_bytes: int | None = None,
|
||||
backend: str | None = None,
|
||||
):
|
||||
"""Load audio using the selected decoding backend.
|
||||
|
||||
Args:
|
||||
backend: One of ``AUDIO_BACKENDS``. ``None`` (default) selects
|
||||
``"auto"``, which tries torchcodec, then falls back to the
|
||||
soundfile → PyAV chain; the other values select a single
|
||||
backend with no fallback.
|
||||
"""
|
||||
backend = backend or "auto"
|
||||
if backend not in AUDIO_BACKENDS:
|
||||
raise ValueError(
|
||||
f"Unknown audio backend {backend!r}. "
|
||||
f"Available backends: {list(AUDIO_BACKENDS)}"
|
||||
)
|
||||
|
||||
if backend != "auto":
|
||||
return _AUDIO_LOADERS[backend](
|
||||
path,
|
||||
sr=sr,
|
||||
mono=mono,
|
||||
max_duration_s=max_duration_s,
|
||||
max_decode_bytes=max_decode_bytes,
|
||||
)
|
||||
|
||||
# "auto": torchcodec → soundfile → PyAV. Each backend is called by its
|
||||
# bare name so tests that monkeypatch it take effect. Only an ImportError
|
||||
# (backend missing) or, for soundfile, an unsupported-format error defers
|
||||
# to the next; any other error (decode failure, guard ValueError) raises.
|
||||
if isinstance(path, BytesIO):
|
||||
path.seek(0)
|
||||
try:
|
||||
return load_audio_torchcodec(
|
||||
path,
|
||||
sr=sr,
|
||||
mono=mono,
|
||||
max_duration_s=max_duration_s,
|
||||
max_decode_bytes=max_decode_bytes,
|
||||
)
|
||||
except ImportError as exc:
|
||||
# Decode errors don't defer: torchcodec is FFmpeg-based like PyAV, so
|
||||
# retrying would just fail the same way.
|
||||
logger.warning(
|
||||
"torchcodec unavailable (%r); falling back to soundfile/PyAV.", exc
|
||||
)
|
||||
|
||||
if isinstance(path, BytesIO):
|
||||
path.seek(0)
|
||||
try:
|
||||
return load_audio_soundfile(
|
||||
path,
|
||||
@@ -234,18 +465,21 @@ def load_audio(
|
||||
max_decode_bytes=max_decode_bytes,
|
||||
)
|
||||
except ImportError as exc:
|
||||
# soundfile (or resampy) is not installed — fall through to pyav.
|
||||
# NOTE: this clause must stay BEFORE ``soundfile.LibsndfileError``
|
||||
# because when soundfile is a PlaceholderModule, evaluating
|
||||
# ``soundfile.LibsndfileError`` itself raises ImportError.
|
||||
# Reach the LibsndfileError clause below only if exc is not an
|
||||
# ImportError — i.e. soundfile is a real module — so evaluating
|
||||
# ``soundfile.LibsndfileError`` here is safe even when soundfile is a
|
||||
# PlaceholderModule (which would make that attribute access raise).
|
||||
logger.error("Failed to load audio via soundfile: %r", exc)
|
||||
except soundfile.LibsndfileError as exc:
|
||||
# Only fall back for known format-detection failures.
|
||||
# Re-raise anything else (e.g. corrupt but recognised format).
|
||||
# libsndfile covers fewer containers than FFmpeg, so defer only on a
|
||||
# format-detection failure; re-raise anything else (e.g. corrupt but
|
||||
# recognised) since PyAV would hit the same corruption.
|
||||
if exc.code not in _BAD_SF_CODES:
|
||||
raise
|
||||
# soundfile may have advanced the BytesIO seek position before failing;
|
||||
# reset it so PyAV can read from the beginning.
|
||||
|
||||
# PyAV is terminal: nothing left to fall back to. Normalize an FFmpeg
|
||||
# failure to ValueError; let a missing soundfile/PyAV surface its
|
||||
# "install vllm[audio]" ImportError.
|
||||
if isinstance(path, BytesIO):
|
||||
path.seek(0)
|
||||
try:
|
||||
@@ -257,9 +491,9 @@ def load_audio(
|
||||
max_decode_bytes=max_decode_bytes,
|
||||
)
|
||||
except ImportError:
|
||||
raise # Let PlaceholderModule's message ("install vllm[audio]") propagate.
|
||||
except Exception as pyav_exc:
|
||||
raise ValueError("Invalid or unsupported audio file.") from pyav_exc
|
||||
raise
|
||||
except av.error.FFmpegError as exc:
|
||||
raise ValueError("Invalid or unsupported audio file.") from exc
|
||||
|
||||
|
||||
class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
||||
@@ -271,6 +505,18 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
||||
def __init__(self, **kwargs) -> None:
|
||||
super().__init__()
|
||||
|
||||
# Allow per-server override of the audio decoding backend, e.g.:
|
||||
# --media-io-kwargs '{"audio": {"audio_backend": "torchcodec"}}'
|
||||
# Validate eagerly so a misconfigured deployment fails at startup
|
||||
# rather than per request.
|
||||
backend = kwargs.pop("audio_backend", None)
|
||||
if backend is not None and backend not in AUDIO_BACKENDS:
|
||||
raise ValueError(
|
||||
f"Unknown audio_backend {backend!r}. "
|
||||
f"Available backends: {list(AUDIO_BACKENDS)}"
|
||||
)
|
||||
self.audio_backend = backend
|
||||
|
||||
# `kwargs` contains custom arguments from
|
||||
# --media-io-kwargs for this modality, merged with
|
||||
# per-request runtime media_io_kwargs via merge_kwargs().
|
||||
@@ -298,6 +544,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
||||
sr=None,
|
||||
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
|
||||
max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES,
|
||||
backend=self.audio_backend,
|
||||
)
|
||||
|
||||
def load_base64(
|
||||
@@ -321,6 +568,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
||||
sr=None,
|
||||
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
|
||||
max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES,
|
||||
backend=self.audio_backend,
|
||||
)
|
||||
|
||||
def encode_base64(
|
||||
|
||||
Reference in New Issue
Block a user