[Frontend] Remove librosa from audio dependency (#37058)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -12,17 +12,35 @@ import torch
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
|
||||
try:
|
||||
import librosa
|
||||
import av as av
|
||||
except ImportError:
|
||||
librosa = PlaceholderModule("librosa") # type: ignore[assignment]
|
||||
av = PlaceholderModule("av") # type: ignore[assignment]
|
||||
|
||||
try:
|
||||
import resampy
|
||||
except ImportError:
|
||||
resampy = PlaceholderModule("resampy") # type: ignore[assignment]
|
||||
|
||||
try:
|
||||
import scipy.signal as scipy_signal
|
||||
except ImportError:
|
||||
scipy_signal = PlaceholderModule("scipy").placeholder_attr("signal") # type: ignore[assignment]
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Aligned with `librosa.get_duration` function
|
||||
def get_audio_duration(*, y: npt.NDArray[np.floating], sr: float = 22050) -> float:
|
||||
"""Get the duration of an audio array in seconds.
|
||||
|
||||
Args:
|
||||
y: Audio time series. Can be 1D (samples,) or 2D (channels, samples).
|
||||
sr: Sample rate of the audio in Hz.
|
||||
|
||||
Returns:
|
||||
Duration of the audio in seconds.
|
||||
"""
|
||||
n_samples = y.shape[-1]
|
||||
return float(n_samples) / sr
|
||||
|
||||
|
||||
class ChannelReduction(str, Enum):
|
||||
@@ -153,13 +171,71 @@ def normalize_audio(
|
||||
# ============================================================
|
||||
|
||||
|
||||
def resample_audio_librosa(
|
||||
def resample_audio_pyav(
|
||||
audio: npt.NDArray[np.floating],
|
||||
*,
|
||||
orig_sr: float,
|
||||
target_sr: float,
|
||||
) -> npt.NDArray[np.floating]:
|
||||
return librosa.resample(audio, orig_sr=orig_sr, target_sr=target_sr)
|
||||
"""Resample audio using PyAV (libswresample via FFmpeg).
|
||||
|
||||
Args:
|
||||
audio: Input audio. Can be:
|
||||
- 1D array ``(samples,)``: mono audio
|
||||
- 2D array ``(channels, samples)``: stereo audio
|
||||
orig_sr: Original sample rate in Hz.
|
||||
target_sr: Target sample rate in Hz.
|
||||
|
||||
Returns:
|
||||
Resampled audio with the same shape as the input (1D → 1D, 2D → 2D).
|
||||
"""
|
||||
orig_sr_int = int(round(orig_sr))
|
||||
target_sr_int = int(round(target_sr))
|
||||
|
||||
if orig_sr_int == target_sr_int:
|
||||
return audio
|
||||
|
||||
if audio.ndim == 2:
|
||||
# Resample each channel independently and re-stack.
|
||||
return np.stack(
|
||||
[
|
||||
resample_audio_pyav(ch, orig_sr=orig_sr, target_sr=target_sr)
|
||||
for ch in audio
|
||||
],
|
||||
axis=0,
|
||||
)
|
||||
|
||||
expected_len = int(math.ceil(audio.shape[-1] * target_sr_int / orig_sr_int))
|
||||
|
||||
# from_ndarray expects shape (channels, samples) for planar formats.
|
||||
# libswresample requires a minimum number of input samples to produce
|
||||
# output frames; pad short inputs with zeros so we always get output,
|
||||
# then trim to the expected output length.
|
||||
_MIN_SAMPLES = 1024
|
||||
audio_f32 = np.asarray(audio, dtype=np.float32)
|
||||
if len(audio_f32) < _MIN_SAMPLES:
|
||||
audio_f32 = np.pad(audio_f32, (0, _MIN_SAMPLES - len(audio_f32)))
|
||||
audio_f32 = audio_f32.reshape(1, -1)
|
||||
|
||||
resampler = av.AudioResampler(format="fltp", layout="mono", rate=target_sr_int)
|
||||
|
||||
frame = av.AudioFrame.from_ndarray(audio_f32, format="fltp", layout="mono")
|
||||
frame.sample_rate = orig_sr_int
|
||||
|
||||
out_frames = resampler.resample(frame)
|
||||
out_frames.extend(resampler.resample(None)) # flush buffered samples
|
||||
|
||||
result = np.concatenate([f.to_ndarray() for f in out_frames], axis=1).squeeze(0)
|
||||
return result[:expected_len]
|
||||
|
||||
|
||||
def resample_audio_resampy(
|
||||
audio: npt.NDArray[np.floating],
|
||||
*,
|
||||
orig_sr: float,
|
||||
target_sr: float,
|
||||
) -> npt.NDArray[np.floating]:
|
||||
return resampy.resample(audio, sr_orig=orig_sr, sr_new=target_sr)
|
||||
|
||||
|
||||
def resample_audio_scipy(
|
||||
@@ -167,7 +243,7 @@ def resample_audio_scipy(
|
||||
*,
|
||||
orig_sr: float,
|
||||
target_sr: float,
|
||||
):
|
||||
) -> npt.NDArray[np.floating]:
|
||||
if orig_sr > target_sr:
|
||||
return scipy_signal.resample_poly(audio, 1, orig_sr // target_sr)
|
||||
elif orig_sr < target_sr:
|
||||
@@ -181,7 +257,7 @@ class AudioResampler:
|
||||
def __init__(
|
||||
self,
|
||||
target_sr: float | None = None,
|
||||
method: Literal["librosa", "scipy"] = "librosa",
|
||||
method: Literal["pyav", "resampy", "scipy"] = "resampy",
|
||||
):
|
||||
self.target_sr = target_sr
|
||||
self.method = method
|
||||
@@ -203,8 +279,10 @@ class AudioResampler:
|
||||
abs_tol=1e-6,
|
||||
):
|
||||
return audio
|
||||
if self.method == "librosa":
|
||||
return resample_audio_librosa(
|
||||
if self.method == "pyav":
|
||||
return resample_audio_pyav(audio, orig_sr=orig_sr, target_sr=self.target_sr)
|
||||
if self.method == "resampy":
|
||||
return resample_audio_resampy(
|
||||
audio, orig_sr=orig_sr, target_sr=self.target_sr
|
||||
)
|
||||
elif self.method == "scipy":
|
||||
@@ -214,7 +292,7 @@ class AudioResampler:
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid resampling method: {self.method}. "
|
||||
"Supported methods are 'librosa' and 'scipy'."
|
||||
"Supported methods are 'pyav' and 'scipy'."
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
@@ -14,58 +15,80 @@ from vllm.utils.serial_utils import tensor2base64
|
||||
from .base import MediaIO
|
||||
|
||||
try:
|
||||
import librosa
|
||||
import av
|
||||
except ImportError:
|
||||
librosa = PlaceholderModule("librosa") # type: ignore[assignment]
|
||||
av = PlaceholderModule("av") # type: ignore[assignment]
|
||||
|
||||
try:
|
||||
import soundfile
|
||||
except ImportError:
|
||||
soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
|
||||
|
||||
|
||||
try:
|
||||
import av
|
||||
import resampy
|
||||
except ImportError:
|
||||
av = PlaceholderModule("av") # type: ignore[assignment]
|
||||
resampy = PlaceholderModule("resampy") # type: ignore[assignment]
|
||||
|
||||
|
||||
def extract_audio_from_video_bytes(
|
||||
data: bytes,
|
||||
# Public libsndfile error codes exposed via `soundfile.LibsndfileError.code`, soundfile
|
||||
# being librosa's main backend. Used to validate if an audio loading error is due to a
|
||||
# server error vs a client error (invalid audio file).
|
||||
# 1 = unrecognised format (file is not a supported audio container)
|
||||
# 3 = malformed file (corrupt or structurally invalid audio)
|
||||
# 4 = unsupported encoding (codec not supported by this libsndfile build)
|
||||
_BAD_SF_CODES = {1, 3, 4}
|
||||
|
||||
|
||||
def load_audio_pyav(
|
||||
path: BytesIO | Path | str,
|
||||
*,
|
||||
sr: float | None = 22050,
|
||||
mono: bool = True,
|
||||
) -> tuple[npt.NDArray, float]:
|
||||
"""Extract the audio track from raw video bytes using PyAV.
|
||||
"""Load an audio file using PyAV (FFmpeg), returning float32 mono waveform.
|
||||
|
||||
PyAV wraps FFmpeg's C libraries in-process — no subprocess is
|
||||
spawned, which is critical to avoid crashing CUDA-active vLLM
|
||||
worker processes.
|
||||
|
||||
The returned waveform is at the native sample rate of the video's
|
||||
audio stream. Resampling to a model-specific rate is left to the
|
||||
downstream :class:`AudioResampler` in the parsing pipeline.
|
||||
Decodes the audio stream at its native sample rate. Channel reduction to
|
||||
mono is performed by averaging across channels. Resampling to a
|
||||
model-specific rate is left to the downstream :class:`AudioResampler`.
|
||||
|
||||
Args:
|
||||
data: Raw video file bytes (e.g. from an mp4 file).
|
||||
path: A :class:`~io.BytesIO` buffer, a filesystem
|
||||
:class:`~pathlib.Path`, or a string path.
|
||||
|
||||
Returns:
|
||||
A tuple of ``(waveform, sample_rate)`` suitable for use as an
|
||||
:class:`AudioItem`.
|
||||
``(waveform, sample_rate)`` where *waveform* is a 1-D float32
|
||||
NumPy array and *sample_rate* is the native sample rate in Hz.
|
||||
"""
|
||||
if data is None or len(data) == 0:
|
||||
raise ValueError(
|
||||
"Cannot extract audio: video bytes are missing or empty. "
|
||||
"Ensure video was loaded with keep_video_bytes=True for "
|
||||
"audio-in-video extraction."
|
||||
)
|
||||
native_sr = None
|
||||
try:
|
||||
with av.open(BytesIO(data)) as container:
|
||||
with av.open(path) as container:
|
||||
if not container.streams.audio:
|
||||
raise ValueError("No audio stream found in the video.")
|
||||
raise ValueError("No audio stream found.")
|
||||
stream = container.streams.audio[0]
|
||||
stream.thread_type = "AUTO"
|
||||
native_sr = stream.rate
|
||||
sr = sr or native_sr
|
||||
|
||||
chunks: list[npt.NDArray] = []
|
||||
for frame in container.decode(audio=0):
|
||||
arr = frame.to_ndarray()
|
||||
chunks.append(arr.mean(axis=0) if arr.ndim > 1 else arr)
|
||||
needs_resampling = not math.isclose(
|
||||
float(sr),
|
||||
float(native_sr),
|
||||
rel_tol=0.0,
|
||||
abs_tol=1e-6,
|
||||
)
|
||||
resampler = (
|
||||
av.AudioResampler(format="fltp", layout="mono", rate=sr)
|
||||
if needs_resampling
|
||||
else None
|
||||
)
|
||||
for frame in container.decode(stream):
|
||||
if needs_resampling:
|
||||
assert resampler is not None
|
||||
for out_frame in resampler.resample(frame):
|
||||
chunks.append(out_frame.to_ndarray())
|
||||
else:
|
||||
chunks.append(frame.to_ndarray())
|
||||
except ValueError:
|
||||
raise
|
||||
except Exception as e:
|
||||
@@ -77,37 +100,54 @@ def extract_audio_from_video_bytes(
|
||||
if not chunks:
|
||||
raise ValueError("No audio found in the video.")
|
||||
|
||||
audio = np.concatenate(chunks).astype(np.float32)
|
||||
return audio, float(native_sr)
|
||||
audio = np.concatenate(chunks, axis=-1).astype(np.float32)
|
||||
if mono and audio.ndim > 1:
|
||||
audio = np.mean(audio, axis=0)
|
||||
|
||||
return audio, sr
|
||||
|
||||
|
||||
def is_video(data: bytes) -> bool:
|
||||
"""Check if the fetched bytes are video"""
|
||||
if len(data) < 12:
|
||||
return False
|
||||
def load_audio_soundfile(
|
||||
path: BytesIO | Path | str,
|
||||
*,
|
||||
sr: float | None = 22050,
|
||||
mono: bool = True,
|
||||
) -> tuple[np.ndarray, int]:
|
||||
"""Load audio via soundfile"""
|
||||
with soundfile.SoundFile(path) as f:
|
||||
native_sr = f.samplerate
|
||||
y = f.read(dtype="float32", always_2d=False).T
|
||||
|
||||
box_type = data[4:8]
|
||||
major_brand = data[8:12]
|
||||
if mono and y.ndim > 1:
|
||||
y = np.mean(y, axis=tuple(range(y.ndim - 1)))
|
||||
|
||||
MP4_BRANDS = {
|
||||
b"mp41",
|
||||
b"mp42", # MP4
|
||||
b"isom", # ISO Base Media
|
||||
b"iso2",
|
||||
b"iso4",
|
||||
b"iso5",
|
||||
b"iso6",
|
||||
b"M4V ",
|
||||
b"M4A ", # Apple
|
||||
b"avc1", # H.264
|
||||
b"dash", # DASH
|
||||
b"mmp4",
|
||||
b"MSNV",
|
||||
}
|
||||
if sr is not None and sr != native_sr:
|
||||
y = resampy.resample(y, sr_orig=native_sr, sr_new=sr)
|
||||
return y, int(sr)
|
||||
return y, native_sr
|
||||
|
||||
is_avi = data[:4] == b"RIFF" and major_brand == b"AVI "
|
||||
is_mp4 = box_type == b"ftyp" and major_brand in MP4_BRANDS
|
||||
return is_mp4 or is_avi
|
||||
|
||||
def load_audio(
|
||||
path: BytesIO | Path | str,
|
||||
*,
|
||||
sr: float | None = 22050,
|
||||
mono: bool = True,
|
||||
):
|
||||
try:
|
||||
return load_audio_soundfile(path, sr=sr, mono=mono)
|
||||
except soundfile.LibsndfileError as exc:
|
||||
# Only fall back for known format-detection failures.
|
||||
# Re-raise anything else (e.g. corrupt but recognised format).
|
||||
if exc.code not in _BAD_SF_CODES:
|
||||
raise
|
||||
# soundfile may have advanced the BytesIO seek position before failing;
|
||||
# reset it so PyAV can read from the beginning.
|
||||
if isinstance(path, BytesIO):
|
||||
path.seek(0)
|
||||
try:
|
||||
return load_audio_pyav(path, sr=sr, mono=mono)
|
||||
except Exception as pyav_exc:
|
||||
raise ValueError("Invalid or unsupported audio file.") from pyav_exc
|
||||
|
||||
|
||||
class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
||||
@@ -128,9 +168,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
||||
self.kwargs = kwargs
|
||||
|
||||
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
|
||||
if is_video(data):
|
||||
return extract_audio_from_video_bytes(data)
|
||||
return librosa.load(BytesIO(data), sr=None)
|
||||
return load_audio(BytesIO(data), sr=None)
|
||||
|
||||
def load_base64(
|
||||
self,
|
||||
@@ -140,7 +178,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
||||
return self.load_bytes(pybase64.b64decode(data))
|
||||
|
||||
def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]:
|
||||
return librosa.load(filepath, sr=None)
|
||||
return load_audio(filepath, sr=None)
|
||||
|
||||
def encode_base64(
|
||||
self,
|
||||
|
||||
@@ -497,7 +497,7 @@ class MultiModalDataParser:
|
||||
*,
|
||||
target_sr: float | None = None,
|
||||
target_channels: int | None = None,
|
||||
audio_resample_method: Literal["librosa", "scipy"] = "librosa",
|
||||
audio_resample_method: Literal["pyav", "scipy"] = "pyav",
|
||||
video_needs_metadata: bool = False,
|
||||
expected_hidden_size: int | None = None,
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user