[Frontend] Remove librosa from audio dependency (#37058)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2026-03-21 11:36:15 +08:00
committed by GitHub
parent 1c472f8fe1
commit c7f98b4d0a
18 changed files with 247 additions and 188 deletions
+87 -9
View File
@@ -12,17 +12,35 @@ import torch
from vllm.utils.import_utils import PlaceholderModule
try:
import librosa
import av as av
except ImportError:
librosa = PlaceholderModule("librosa") # type: ignore[assignment]
av = PlaceholderModule("av") # type: ignore[assignment]
try:
import resampy
except ImportError:
resampy = PlaceholderModule("resampy") # type: ignore[assignment]
try:
import scipy.signal as scipy_signal
except ImportError:
scipy_signal = PlaceholderModule("scipy").placeholder_attr("signal") # type: ignore[assignment]
# ============================================================
# Aligned with `librosa.get_duration` function
def get_audio_duration(*, y: npt.NDArray[np.floating], sr: float = 22050) -> float:
"""Get the duration of an audio array in seconds.
Args:
y: Audio time series. Can be 1D (samples,) or 2D (channels, samples).
sr: Sample rate of the audio in Hz.
Returns:
Duration of the audio in seconds.
"""
n_samples = y.shape[-1]
return float(n_samples) / sr
class ChannelReduction(str, Enum):
@@ -153,13 +171,71 @@ def normalize_audio(
# ============================================================
def resample_audio_librosa(
def resample_audio_pyav(
audio: npt.NDArray[np.floating],
*,
orig_sr: float,
target_sr: float,
) -> npt.NDArray[np.floating]:
return librosa.resample(audio, orig_sr=orig_sr, target_sr=target_sr)
"""Resample audio using PyAV (libswresample via FFmpeg).
Args:
audio: Input audio. Can be:
- 1D array ``(samples,)``: mono audio
- 2D array ``(channels, samples)``: stereo audio
orig_sr: Original sample rate in Hz.
target_sr: Target sample rate in Hz.
Returns:
Resampled audio with the same shape as the input (1D → 1D, 2D → 2D).
"""
orig_sr_int = int(round(orig_sr))
target_sr_int = int(round(target_sr))
if orig_sr_int == target_sr_int:
return audio
if audio.ndim == 2:
# Resample each channel independently and re-stack.
return np.stack(
[
resample_audio_pyav(ch, orig_sr=orig_sr, target_sr=target_sr)
for ch in audio
],
axis=0,
)
expected_len = int(math.ceil(audio.shape[-1] * target_sr_int / orig_sr_int))
# from_ndarray expects shape (channels, samples) for planar formats.
# libswresample requires a minimum number of input samples to produce
# output frames; pad short inputs with zeros so we always get output,
# then trim to the expected output length.
_MIN_SAMPLES = 1024
audio_f32 = np.asarray(audio, dtype=np.float32)
if len(audio_f32) < _MIN_SAMPLES:
audio_f32 = np.pad(audio_f32, (0, _MIN_SAMPLES - len(audio_f32)))
audio_f32 = audio_f32.reshape(1, -1)
resampler = av.AudioResampler(format="fltp", layout="mono", rate=target_sr_int)
frame = av.AudioFrame.from_ndarray(audio_f32, format="fltp", layout="mono")
frame.sample_rate = orig_sr_int
out_frames = resampler.resample(frame)
out_frames.extend(resampler.resample(None)) # flush buffered samples
result = np.concatenate([f.to_ndarray() for f in out_frames], axis=1).squeeze(0)
return result[:expected_len]
def resample_audio_resampy(
audio: npt.NDArray[np.floating],
*,
orig_sr: float,
target_sr: float,
) -> npt.NDArray[np.floating]:
return resampy.resample(audio, sr_orig=orig_sr, sr_new=target_sr)
def resample_audio_scipy(
@@ -167,7 +243,7 @@ def resample_audio_scipy(
*,
orig_sr: float,
target_sr: float,
):
) -> npt.NDArray[np.floating]:
if orig_sr > target_sr:
return scipy_signal.resample_poly(audio, 1, orig_sr // target_sr)
elif orig_sr < target_sr:
@@ -181,7 +257,7 @@ class AudioResampler:
def __init__(
self,
target_sr: float | None = None,
method: Literal["librosa", "scipy"] = "librosa",
method: Literal["pyav", "resampy", "scipy"] = "resampy",
):
self.target_sr = target_sr
self.method = method
@@ -203,8 +279,10 @@ class AudioResampler:
abs_tol=1e-6,
):
return audio
if self.method == "librosa":
return resample_audio_librosa(
if self.method == "pyav":
return resample_audio_pyav(audio, orig_sr=orig_sr, target_sr=self.target_sr)
if self.method == "resampy":
return resample_audio_resampy(
audio, orig_sr=orig_sr, target_sr=self.target_sr
)
elif self.method == "scipy":
@@ -214,7 +292,7 @@ class AudioResampler:
else:
raise ValueError(
f"Invalid resampling method: {self.method}. "
"Supported methods are 'librosa' and 'scipy'."
"Supported methods are 'pyav' and 'scipy'."
)
+96 -58
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from io import BytesIO
from pathlib import Path
@@ -14,58 +15,80 @@ from vllm.utils.serial_utils import tensor2base64
from .base import MediaIO
try:
import librosa
import av
except ImportError:
librosa = PlaceholderModule("librosa") # type: ignore[assignment]
av = PlaceholderModule("av") # type: ignore[assignment]
try:
import soundfile
except ImportError:
soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
try:
import av
import resampy
except ImportError:
av = PlaceholderModule("av") # type: ignore[assignment]
resampy = PlaceholderModule("resampy") # type: ignore[assignment]
def extract_audio_from_video_bytes(
data: bytes,
# Public libsndfile error codes exposed via `soundfile.LibsndfileError.code`, soundfile
# being librosa's main backend. Used to validate if an audio loading error is due to a
# server error vs a client error (invalid audio file).
# 1 = unrecognised format (file is not a supported audio container)
# 3 = malformed file (corrupt or structurally invalid audio)
# 4 = unsupported encoding (codec not supported by this libsndfile build)
_BAD_SF_CODES = {1, 3, 4}
def load_audio_pyav(
path: BytesIO | Path | str,
*,
sr: float | None = 22050,
mono: bool = True,
) -> tuple[npt.NDArray, float]:
"""Extract the audio track from raw video bytes using PyAV.
"""Load an audio file using PyAV (FFmpeg), returning float32 mono waveform.
PyAV wraps FFmpeg's C libraries in-process — no subprocess is
spawned, which is critical to avoid crashing CUDA-active vLLM
worker processes.
The returned waveform is at the native sample rate of the video's
audio stream. Resampling to a model-specific rate is left to the
downstream :class:`AudioResampler` in the parsing pipeline.
Decodes the audio stream at its native sample rate. Channel reduction to
mono is performed by averaging across channels. Resampling to a
model-specific rate is left to the downstream :class:`AudioResampler`.
Args:
data: Raw video file bytes (e.g. from an mp4 file).
path: A :class:`~io.BytesIO` buffer, a filesystem
:class:`~pathlib.Path`, or a string path.
Returns:
A tuple of ``(waveform, sample_rate)`` suitable for use as an
:class:`AudioItem`.
``(waveform, sample_rate)`` where *waveform* is a 1-D float32
NumPy array and *sample_rate* is the native sample rate in Hz.
"""
if data is None or len(data) == 0:
raise ValueError(
"Cannot extract audio: video bytes are missing or empty. "
"Ensure video was loaded with keep_video_bytes=True for "
"audio-in-video extraction."
)
native_sr = None
try:
with av.open(BytesIO(data)) as container:
with av.open(path) as container:
if not container.streams.audio:
raise ValueError("No audio stream found in the video.")
raise ValueError("No audio stream found.")
stream = container.streams.audio[0]
stream.thread_type = "AUTO"
native_sr = stream.rate
sr = sr or native_sr
chunks: list[npt.NDArray] = []
for frame in container.decode(audio=0):
arr = frame.to_ndarray()
chunks.append(arr.mean(axis=0) if arr.ndim > 1 else arr)
needs_resampling = not math.isclose(
float(sr),
float(native_sr),
rel_tol=0.0,
abs_tol=1e-6,
)
resampler = (
av.AudioResampler(format="fltp", layout="mono", rate=sr)
if needs_resampling
else None
)
for frame in container.decode(stream):
if needs_resampling:
assert resampler is not None
for out_frame in resampler.resample(frame):
chunks.append(out_frame.to_ndarray())
else:
chunks.append(frame.to_ndarray())
except ValueError:
raise
except Exception as e:
@@ -77,37 +100,54 @@ def extract_audio_from_video_bytes(
if not chunks:
raise ValueError("No audio found in the video.")
audio = np.concatenate(chunks).astype(np.float32)
return audio, float(native_sr)
audio = np.concatenate(chunks, axis=-1).astype(np.float32)
if mono and audio.ndim > 1:
audio = np.mean(audio, axis=0)
return audio, sr
def is_video(data: bytes) -> bool:
"""Check if the fetched bytes are video"""
if len(data) < 12:
return False
def load_audio_soundfile(
path: BytesIO | Path | str,
*,
sr: float | None = 22050,
mono: bool = True,
) -> tuple[np.ndarray, int]:
"""Load audio via soundfile"""
with soundfile.SoundFile(path) as f:
native_sr = f.samplerate
y = f.read(dtype="float32", always_2d=False).T
box_type = data[4:8]
major_brand = data[8:12]
if mono and y.ndim > 1:
y = np.mean(y, axis=tuple(range(y.ndim - 1)))
MP4_BRANDS = {
b"mp41",
b"mp42", # MP4
b"isom", # ISO Base Media
b"iso2",
b"iso4",
b"iso5",
b"iso6",
b"M4V ",
b"M4A ", # Apple
b"avc1", # H.264
b"dash", # DASH
b"mmp4",
b"MSNV",
}
if sr is not None and sr != native_sr:
y = resampy.resample(y, sr_orig=native_sr, sr_new=sr)
return y, int(sr)
return y, native_sr
is_avi = data[:4] == b"RIFF" and major_brand == b"AVI "
is_mp4 = box_type == b"ftyp" and major_brand in MP4_BRANDS
return is_mp4 or is_avi
def load_audio(
path: BytesIO | Path | str,
*,
sr: float | None = 22050,
mono: bool = True,
):
try:
return load_audio_soundfile(path, sr=sr, mono=mono)
except soundfile.LibsndfileError as exc:
# Only fall back for known format-detection failures.
# Re-raise anything else (e.g. corrupt but recognised format).
if exc.code not in _BAD_SF_CODES:
raise
# soundfile may have advanced the BytesIO seek position before failing;
# reset it so PyAV can read from the beginning.
if isinstance(path, BytesIO):
path.seek(0)
try:
return load_audio_pyav(path, sr=sr, mono=mono)
except Exception as pyav_exc:
raise ValueError("Invalid or unsupported audio file.") from pyav_exc
class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
@@ -128,9 +168,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
self.kwargs = kwargs
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
if is_video(data):
return extract_audio_from_video_bytes(data)
return librosa.load(BytesIO(data), sr=None)
return load_audio(BytesIO(data), sr=None)
def load_base64(
self,
@@ -140,7 +178,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
return self.load_bytes(pybase64.b64decode(data))
def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]:
return librosa.load(filepath, sr=None)
return load_audio(filepath, sr=None)
def encode_base64(
self,
+1 -1
View File
@@ -497,7 +497,7 @@ class MultiModalDataParser:
*,
target_sr: float | None = None,
target_channels: int | None = None,
audio_resample_method: Literal["librosa", "scipy"] = "librosa",
audio_resample_method: Literal["pyav", "scipy"] = "pyav",
video_needs_metadata: bool = False,
expected_hidden_size: int | None = None,
) -> None: