From 6a039f465e3770192085e551f642c7a218797d2d Mon Sep 17 00:00:00 2001 From: Jared Wen Date: Sat, 5 Sep 2026 00:16:10 +0800 Subject: [PATCH] [feat] add torchcodec as audio loader and implement selective audio backend (#51826) Signed-off-by: JaredforReal Co-authored-by: saltman155 Co-authored-by: Isotr0py --- docs/features/multimodal_inputs.md | 32 ++ .../test_transcription_validation_whisper.py | 2 +- tests/multimodal/media/test_audio.py | 101 ++++- vllm/multimodal/media/audio.py | 374 +++++++++++++++--- 4 files changed, 444 insertions(+), 65 deletions(-) diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index a36e40c3b9..7eef6ce85c 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -1204,6 +1204,38 @@ Full example: [examples/generate/multimodal/openai_chat_completion_client_for_mu export VLLM_AUDIO_FETCH_TIMEOUT= ``` +#### Audio Decoding Backend + +vLLM decodes audio bytes into waveforms using a selectable decoding backend: + +| Backend | Description | +| --- | --- | +| `auto` (default) | torchcodec, falling back to soundfile, then PyAV | +| `soundfile` | libsndfile only, no fallback | +| `pyav` | PyAV (FFmpeg) only, no fallback | +| `torchcodec` | TorchCodec (PyTorch-native) only, no fallback | + +Select the backend per server via `--media-io-kwargs`: + +```bash +vllm serve mistralai/Voxtral-Mini-3B-2507 \ + --media-io-kwargs '{"audio": {"audio_backend": "soundfile"}}' +``` + +!!! tip + `pyav` drives FFmpeg through a per-frame Python generator, so under + concurrency the Python/C crossings contend on the GIL. `torchcodec` + decodes each stream in a single call that releases the GIL for its whole + duration, which is why `auto` prefers it when many requests decode audio + concurrently, such as when audio tracks are extracted from video. + +!!! note + `torchcodec` ships as a requirement on CUDA, CPU and XPU builds. On other + platforms (e.g. ROCm, TPU) `auto` falls back to the soundfile → PyAV + chain unless you install it manually. torchcodec also links against a + system FFmpeg installation; if FFmpeg is missing, `auto` falls back the + same way. + ### Embedding Inputs To input pre-computed embeddings belonging to a data type (i.e. image, video, or audio) directly to the language model, diff --git a/tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py b/tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py index 63357195d0..51599d359e 100644 --- a/tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py +++ b/tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py @@ -172,7 +172,7 @@ async def test_invalid_audio_file(whisper_client): ) assert exc_info.value.status_code == 400 - assert "Invalid or unsupported audio file" in exc_info.value.message + assert "Invalid or corrupted audio data" in exc_info.value.message @pytest.mark.asyncio diff --git a/tests/multimodal/media/test_audio.py b/tests/multimodal/media/test_audio.py index 7a3c09701a..89810f60d2 100644 --- a/tests/multimodal/media/test_audio.py +++ b/tests/multimodal/media/test_audio.py @@ -10,7 +10,12 @@ import pytest import soundfile as sf from vllm.multimodal.media import AudioMediaIO -from vllm.multimodal.media.audio import load_audio, load_audio_soundfile +from vllm.multimodal.media import audio as audio_module +from vllm.multimodal.media.audio import ( + load_audio, + load_audio_soundfile, + load_audio_torchcodec, +) from ...conftest import AudioTestAssets @@ -170,3 +175,97 @@ def test_load_audio_threads_max_decode_bytes(): max_duration_s=600, max_decode_bytes=512 * 1024, ) + + +@pytest.mark.parametrize("backend", ["soundfile", "pyav", "torchcodec"]) +def test_load_audio_backend_matches_default(backend, dummy_audio_bytes): + """Every explicit backend must decode to the same samples as `auto`.""" + if backend == "torchcodec": + pytest.importorskip("torchcodec") + ref_audio, ref_sr = load_audio(BytesIO(dummy_audio_bytes), sr=None) + audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend=backend) + assert sr == ref_sr + # Decoders disagree only on codec encoder-delay/padding, so torchcodec may + # emit a few extra trailing samples (e.g. ~192 for Ogg Vorbis). Compare the + # overlapping region, which must agree to float32 precision. + n = min(ref_audio.shape[-1], audio.shape[-1]) + assert n > 0 + np.testing.assert_allclose(ref_audio[:n], audio[:n], atol=1e-4) + + +def test_load_audio_unknown_backend_rejected(dummy_audio_bytes): + """An unknown backend must fail loudly instead of silently degrading.""" + with pytest.raises(ValueError, match="Unknown audio backend"): + load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="not_a_backend") + + +def test_load_audio_auto_falls_back_without_torchcodec(dummy_audio_bytes): + """`auto` must fall back to the soundfile → PyAV chain when torchcodec + is not importable.""" + ref_audio, ref_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None) + with patch.object(audio_module, "load_audio_torchcodec", side_effect=ImportError): + audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="auto") + assert sr == ref_sr + np.testing.assert_array_equal(ref_audio, audio) + + +def test_load_audio_auto_falls_back_without_ffmpeg(dummy_audio_bytes): + """torchcodec installed but system ffmpeg missing (`AudioDecoder is None`) + must surface as ImportError so `auto` falls back to soundfile → PyAV.""" + ref_audio, ref_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None) + with patch.object(audio_module, "AudioDecoder", None): + with pytest.raises(ImportError, match="torchcodec audio backend"): + load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None) + audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="auto") + assert sr == ref_sr + np.testing.assert_array_equal(ref_audio, audio) + + +def test_load_audio_auto_falls_back_when_libtorchcodec_unloadable( + dummy_audio_bytes, +): + """torchcodec loads its ffmpeg-backed core lazily at decoder construction; + a "Could not load libtorchcodec" RuntimeError there (no system ffmpeg) + must also surface as ImportError so `auto` falls back.""" + + class _UnloadableDecoder: + def __init__(self, *args, **kwargs): + raise RuntimeError("Could not load libtorchcodec. Likely causes: ...") + + ref_audio, ref_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None) + with patch.object(audio_module, "AudioDecoder", _UnloadableDecoder): + with pytest.raises(ImportError, match="torchcodec audio backend"): + load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None) + audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="auto") + assert sr == ref_sr + np.testing.assert_array_equal(ref_audio, audio) + + +def test_audio_media_io_audio_backend_kwarg(dummy_audio_bytes): + """`audio_backend` selects the backend; unknown values fail at init.""" + audio, sr = AudioMediaIO(audio_backend="pyav").load_bytes(dummy_audio_bytes) + assert isinstance(audio, np.ndarray) + assert sr == 16000 + with pytest.raises(ValueError, match="Unknown audio_backend"): + AudioMediaIO(audio_backend="not_a_backend") + + +def test_torchcodec_max_duration_rejected(dummy_audio_bytes): + """The decompression-bomb duration guard must hold for torchcodec too.""" + pytest.importorskip("torchcodec") + with pytest.raises(ValueError, match="exceeds maximum allowed duration"): + load_audio_torchcodec( + BytesIO(dummy_audio_bytes), sr=None, max_duration_s=0.0001 + ) + + +def test_torchcodec_max_decode_bytes_rejected(dummy_audio_bytes): + """The decompression-bomb memory guard must hold for torchcodec too.""" + pytest.importorskip("torchcodec") + audio, _ = load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None) + with pytest.raises(ValueError, match="VLLM_MAX_AUDIO_DECODE_BYTES"): + load_audio_torchcodec( + BytesIO(dummy_audio_bytes), + sr=None, + max_decode_bytes=audio.nbytes - 1, + ) diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py index e1e27d469e..5187ffe1cc 100644 --- a/vllm/multimodal/media/audio.py +++ b/vllm/multimodal/media/audio.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import math +from collections.abc import Callable from io import BytesIO from pathlib import Path @@ -35,6 +36,15 @@ try: except ImportError: soundfile = PlaceholderModule("soundfile") # type: ignore[assignment] +try: + from torchcodec.decoders import AudioDecoder +except (ImportError, RuntimeError) as exc: + # RuntimeError: torchcodec is installed but the system ffmpeg is missing. + AudioDecoder = None # type: ignore[assignment] + _torchcodec_import_exc: BaseException | None = exc +else: + _torchcodec_import_exc = None + # Public libsndfile error codes exposed via `soundfile.LibsndfileError.code`, # soundfile being the main audio loading backend. Used to validate if an audio @@ -48,6 +58,22 @@ except ImportError: # 4 = unsupported encoding (codec not supported by this libsndfile build) _BAD_SF_CODES = {0, 1, 3, 4} +# Audio decoding backends selectable via `load_audio(backend=...)` or +# `--media-io-kwargs '{"audio": {"audio_backend": ...}}'`. +# "auto" tries torchcodec, then soundfile, then PyAV. +AUDIO_BACKENDS = ("auto", "soundfile", "pyav", "torchcodec") + +# Raised as ImportError when the torchcodec backend cannot be used, so +# `load_audio(backend="auto")` falls back to the soundfile → PyAV chain. +_TORCHCODEC_UNAVAILABLE_MSG = ( + "torchcodec audio backend is unavailable (requires the torchcodec " + "package and a system ffmpeg installation)" +) + +# Slack on the torchcodec decode window when enforcing `max_duration_s`, so an +# over-long stream is rejected rather than truncated to the limit. +_DURATION_GUARD_MARGIN_S = 1.0 + def load_audio_pyav( path: BytesIO | Path | str, @@ -75,54 +101,56 @@ def load_audio_pyav( ``(waveform, sample_rate)`` where *waveform* is a 1-D float32 NumPy array and *sample_rate* is the native sample rate in Hz. """ - native_sr = None try: - with av.open(path) as container: - if not container.streams.audio: - raise ValueError("No audio stream found.") - stream = container.streams.audio[0] - stream.thread_type = "AUTO" - native_sr = stream.rate - sr = sr or native_sr + container = av.open(path) + except av.error.FFmpegError as e: + raise ValueError( + "Invalid or corrupted audio data. Ensure the input is a valid " + "audio or video file (e.g. a complete WAV, MP3, or MP4)." + ) from e - # Early rejection from container/stream metadata to avoid - # wasting resources on decoding decompression bombs. - if max_duration_s is not None: - metadata_duration_s = None - if stream.duration and stream.time_base: - metadata_duration_s = float(stream.duration * stream.time_base) - elif container.duration: - metadata_duration_s = container.duration / 1_000_000 - if ( - metadata_duration_s is not None - and metadata_duration_s > max_duration_s - ): - raise ValueError( - f"Audio exceeds maximum allowed duration of " - f"{max_duration_s}s (metadata reports " - f"{metadata_duration_s:.1f}s). Set " - f"VLLM_MAX_AUDIO_DECODE_DURATION_S to " - f"increase this limit." - ) + with container: + if not container.streams.audio: + raise ValueError("No audio stream found.") + stream = container.streams.audio[0] + stream.thread_type = "AUTO" + native_sr = stream.rate + sr = sr or native_sr - max_samples = ( - int(sr * max_duration_s) if max_duration_s is not None else None - ) - total_samples = 0 - total_decode_bytes = 0 + # Early rejection from container/stream metadata to avoid + # wasting resources on decoding decompression bombs. + if max_duration_s is not None: + metadata_duration_s = None + if stream.duration and stream.time_base: + metadata_duration_s = float(stream.duration * stream.time_base) + elif container.duration: + metadata_duration_s = container.duration / 1_000_000 + if metadata_duration_s is not None and metadata_duration_s > max_duration_s: + raise ValueError( + f"Audio exceeds maximum allowed duration of " + f"{max_duration_s}s (metadata reports " + f"{metadata_duration_s:.1f}s). Set " + f"VLLM_MAX_AUDIO_DECODE_DURATION_S to " + f"increase this limit." + ) - chunks: list[npt.NDArray] = [] - needs_resampling = not math.isclose( - float(sr), - float(native_sr), - rel_tol=0.0, - abs_tol=1e-6, - ) - resampler = ( - av.AudioResampler(format="fltp", layout="mono", rate=sr) - if needs_resampling - else None - ) + max_samples = int(sr * max_duration_s) if max_duration_s is not None else None + total_samples = 0 + total_decode_bytes = 0 + + chunks: list[npt.NDArray] = [] + needs_resampling = not math.isclose( + float(sr), + float(native_sr), + rel_tol=0.0, + abs_tol=1e-6, + ) + resampler = ( + av.AudioResampler(format="fltp", layout="mono", rate=sr) + if needs_resampling + else None + ) + try: for frame in container.decode(stream): if needs_resampling: assert resampler is not None @@ -157,16 +185,14 @@ def load_audio_pyav( f"VLLM_MAX_AUDIO_DECODE_BYTES to increase this " f"limit." ) - except (ValueError, ImportError): - raise - except Exception as e: - raise ValueError( - "Invalid or corrupted video data when extracting audio. " - "Ensure the input is valid video bytes (e.g. a complete MP4)." - ) from e + except av.error.FFmpegError as e: + raise ValueError( + "Invalid or corrupted audio data. Ensure the input is a valid " + "audio or video file (e.g. a complete WAV, MP3, or MP4)." + ) from e if not chunks: - raise ValueError("No audio found in the video.") + raise ValueError("No audio found in the input.") audio = np.concatenate(chunks, axis=-1).astype(np.float32) if mono and audio.ndim > 1: @@ -217,6 +243,163 @@ def load_audio_soundfile( return y, native_sr +def load_audio_torchcodec( + path: BytesIO | Path | str, + *, + sr: float | None = 22050, + mono: bool = True, + max_duration_s: float | None = None, + max_decode_bytes: int | None = None, +) -> tuple[npt.NDArray, float]: + """Load an audio file using torchcodec, returning a float32 waveform. + + Unlike :func:`load_audio_pyav`, which drives FFmpeg through a per-frame + Python generator, torchcodec decodes inside a single call that releases + the GIL. Resampling also happens in C++ (``sample_rate=sr``), so this + backend never falls back to PyAV or libsndfile. + + Args: + path: A :class:`~io.BytesIO` buffer, a filesystem + :class:`~pathlib.Path`, or a string path. + sr: Output sample rate; ``None`` keeps the native rate. Must be an + integer when set. + mono: Average channels down to a single channel. + max_duration_s: Reject audio longer than this many seconds. + Prevents decompression-bomb attacks where a small compressed + file expands into gigabytes of PCM. + max_decode_bytes: Reject audio whose decoded PCM exceeds this many + bytes. + + Returns: + ``(waveform, sample_rate)`` where *waveform* is a float32 NumPy + array (1-D when ``mono=True``) and *sample_rate* is the output + sample rate in Hz. + """ + if AudioDecoder is None: + # Unify "torchcodec not installed" and "system ffmpeg missing" into + # ImportError so `load_audio(backend="auto")` falls back cleanly. + raise ImportError(_TORCHCODEC_UNAVAILABLE_MSG) from _torchcodec_import_exc + if sr is not None and sr != int(sr): + raise ValueError(f"torchcodec requires an integer sample rate, got {sr}") + sample_rate = int(sr) if sr is not None else None + + # Opening the source and reading container metadata both touch untrusted + # input; torchcodec signals corrupt/unsupported data with RuntimeError. + try: + decoder = AudioDecoder(path, sample_rate=sample_rate) + metadata = decoder.metadata + except RuntimeError as e: + # torchcodec loads its ffmpeg-backed core lazily at construction, so + # an ffmpeg-less (or ABI-incompatible) install can fail here rather + # than at import. Treat that as backend-unavailable, not bad input. + if "Could not load libtorchcodec" in str(e): + raise ImportError(_TORCHCODEC_UNAVAILABLE_MSG) from e + raise ValueError( + "Invalid or corrupted audio data. Ensure the input is a valid " + "audio or video file (e.g. a complete WAV, MP3, or MP4)." + ) from e + + # Pre-check the memory budget from container metadata so an obviously + # oversized stream is rejected before allocating the PCM buffer. Metadata + # is attacker-controlled, so this may only reject, never admit; the + # post-decode check below is authoritative. + if max_decode_bytes is not None: + est_duration_s = metadata.duration_seconds + # Estimate at the *output* rate: when `sr` is set torchcodec + # resamples to it during decode. + est_sample_rate = sr if sr is not None else metadata.sample_rate + est_num_channels = metadata.num_channels + if ( + est_duration_s is not None + and est_sample_rate is not None + and est_num_channels is not None + ): + estimated_bytes = ( + int(est_duration_s * est_sample_rate) + * est_num_channels + * np.dtype(np.float32).itemsize + ) + if estimated_bytes > max_decode_bytes: + raise ValueError( + f"Audio would allocate " + f"{estimated_bytes / MiB_bytes:.0f} MiB of PCM " + f"(~{est_duration_s:.1f}s at {est_sample_rate}Hz x " + f"{est_num_channels}ch), exceeding the " + f"{max_decode_bytes / MiB_bytes:.0f} MiB limit. Set " + f"VLLM_MAX_AUDIO_DECODE_BYTES to increase this limit." + ) + + # Duration pre-check, same attacker-controlled → reject-only semantics. + if max_duration_s is not None: + duration_s = metadata.duration_seconds + if duration_s is not None and duration_s > max_duration_s: + raise ValueError( + f"Audio exceeds maximum allowed duration of " + f"{max_duration_s}s (metadata reports " + f"{duration_s:.1f}s). Set " + f"VLLM_MAX_AUDIO_DECODE_DURATION_S to " + f"increase this limit." + ) + + # Decode the stream. RuntimeError here covers a corrupt stream or a + # container that lies about its length. Bounding the range (when a + # duration limit is set) keeps an under-reported duration from expanding + # into unbounded PCM. + try: + if max_duration_s is None: + samples = decoder.get_all_samples() + else: + samples = decoder.get_samples_played_in_range( + 0.0, max_duration_s + _DURATION_GUARD_MARGIN_S + ) + except RuntimeError as e: + raise ValueError( + "Invalid or corrupted audio data. Ensure the input is a valid " + "audio or video file (e.g. a complete WAV, MP3, or MP4)." + ) from e + + # Authoritative post-decode checks: the metadata estimates above can be + # bypassed by a container that lies about its duration or size. + if max_duration_s is not None: + # Same `int()` semantics as load_audio_pyav's + # `total_samples > int(sr * max_duration_s)`. + max_samples = int(samples.sample_rate * max_duration_s) + if samples.data.shape[-1] > max_samples: + raise ValueError( + f"Audio exceeds maximum allowed duration of " + f"{max_duration_s}s (decoded {samples.data.shape[-1]} " + f"samples at {samples.sample_rate}Hz). Set " + f"VLLM_MAX_AUDIO_DECODE_DURATION_S to " + f"increase this limit." + ) + if max_decode_bytes is not None and samples.data.nbytes > max_decode_bytes: + raise ValueError( + f"Audio decode exceeded " + f"{max_decode_bytes / MiB_bytes:.0f} MiB memory " + f"limit ({samples.data.nbytes / MiB_bytes:.0f} MiB decoded). " + f"Set VLLM_MAX_AUDIO_DECODE_BYTES to increase this limit." + ) + + out_sr = samples.sample_rate + audio = samples.data.numpy() # (num_channels, num_samples), float32 + if audio.size == 0: + raise ValueError("No audio found in the input.") + + if mono and audio.ndim > 1: + # Same reduction as load_audio_pyav, so both backends agree at the + # native sample rate, which is what AudioMediaIO requests. + audio = np.mean(audio, axis=0) + + return audio, out_sr + + +_AUDIO_LOADERS: dict[str, Callable[..., tuple[npt.NDArray, float]]] = { + "torchcodec": load_audio_torchcodec, + "pyav": load_audio_pyav, + "soundfile": load_audio_soundfile, +} + + def load_audio( path: BytesIO | Path | str, *, @@ -224,7 +407,55 @@ def load_audio( mono: bool = True, max_duration_s: float | None = None, max_decode_bytes: int | None = None, + backend: str | None = None, ): + """Load audio using the selected decoding backend. + + Args: + backend: One of ``AUDIO_BACKENDS``. ``None`` (default) selects + ``"auto"``, which tries torchcodec, then falls back to the + soundfile → PyAV chain; the other values select a single + backend with no fallback. + """ + backend = backend or "auto" + if backend not in AUDIO_BACKENDS: + raise ValueError( + f"Unknown audio backend {backend!r}. " + f"Available backends: {list(AUDIO_BACKENDS)}" + ) + + if backend != "auto": + return _AUDIO_LOADERS[backend]( + path, + sr=sr, + mono=mono, + max_duration_s=max_duration_s, + max_decode_bytes=max_decode_bytes, + ) + + # "auto": torchcodec → soundfile → PyAV. Each backend is called by its + # bare name so tests that monkeypatch it take effect. Only an ImportError + # (backend missing) or, for soundfile, an unsupported-format error defers + # to the next; any other error (decode failure, guard ValueError) raises. + if isinstance(path, BytesIO): + path.seek(0) + try: + return load_audio_torchcodec( + path, + sr=sr, + mono=mono, + max_duration_s=max_duration_s, + max_decode_bytes=max_decode_bytes, + ) + except ImportError as exc: + # Decode errors don't defer: torchcodec is FFmpeg-based like PyAV, so + # retrying would just fail the same way. + logger.warning( + "torchcodec unavailable (%r); falling back to soundfile/PyAV.", exc + ) + + if isinstance(path, BytesIO): + path.seek(0) try: return load_audio_soundfile( path, @@ -234,18 +465,21 @@ def load_audio( max_decode_bytes=max_decode_bytes, ) except ImportError as exc: - # soundfile (or resampy) is not installed — fall through to pyav. - # NOTE: this clause must stay BEFORE ``soundfile.LibsndfileError`` - # because when soundfile is a PlaceholderModule, evaluating - # ``soundfile.LibsndfileError`` itself raises ImportError. + # Reach the LibsndfileError clause below only if exc is not an + # ImportError — i.e. soundfile is a real module — so evaluating + # ``soundfile.LibsndfileError`` here is safe even when soundfile is a + # PlaceholderModule (which would make that attribute access raise). logger.error("Failed to load audio via soundfile: %r", exc) except soundfile.LibsndfileError as exc: - # Only fall back for known format-detection failures. - # Re-raise anything else (e.g. corrupt but recognised format). + # libsndfile covers fewer containers than FFmpeg, so defer only on a + # format-detection failure; re-raise anything else (e.g. corrupt but + # recognised) since PyAV would hit the same corruption. if exc.code not in _BAD_SF_CODES: raise - # soundfile may have advanced the BytesIO seek position before failing; - # reset it so PyAV can read from the beginning. + + # PyAV is terminal: nothing left to fall back to. Normalize an FFmpeg + # failure to ValueError; let a missing soundfile/PyAV surface its + # "install vllm[audio]" ImportError. if isinstance(path, BytesIO): path.seek(0) try: @@ -257,9 +491,9 @@ def load_audio( max_decode_bytes=max_decode_bytes, ) except ImportError: - raise # Let PlaceholderModule's message ("install vllm[audio]") propagate. - except Exception as pyav_exc: - raise ValueError("Invalid or unsupported audio file.") from pyav_exc + raise + except av.error.FFmpegError as exc: + raise ValueError("Invalid or unsupported audio file.") from exc class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]): @@ -271,6 +505,18 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]): def __init__(self, **kwargs) -> None: super().__init__() + # Allow per-server override of the audio decoding backend, e.g.: + # --media-io-kwargs '{"audio": {"audio_backend": "torchcodec"}}' + # Validate eagerly so a misconfigured deployment fails at startup + # rather than per request. + backend = kwargs.pop("audio_backend", None) + if backend is not None and backend not in AUDIO_BACKENDS: + raise ValueError( + f"Unknown audio_backend {backend!r}. " + f"Available backends: {list(AUDIO_BACKENDS)}" + ) + self.audio_backend = backend + # `kwargs` contains custom arguments from # --media-io-kwargs for this modality, merged with # per-request runtime media_io_kwargs via merge_kwargs(). @@ -298,6 +544,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]): sr=None, max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S, max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES, + backend=self.audio_backend, ) def load_base64( @@ -321,6 +568,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]): sr=None, max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S, max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES, + backend=self.audio_backend, ) def encode_base64(