[Multimodal] Add PyAV video backend for concurrent video decoding (#39986)

Signed-off-by: Jaseel Muhammad <jaseel.muhammad@mbzuai.ac.ae>
Signed-off-by: Isotr0py <2037008807@qq.com>
Co-authored-by: Isotr0py <2037008807@qq.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Jaseel Muhammad
2026-04-21 20:14:57 -07:00
committed by GitHub
co-authored by Isotr0py Isotr0py
parent 2463f00fb6
commit 6f2c71be8f
4 changed files with 292 additions and 120 deletions
+176 -96
View File
@@ -3,7 +3,7 @@
import math
from abc import abstractmethod
from io import BytesIO
from typing import Any, NamedTuple, cast
from typing import Any, ClassVar, Literal, NamedTuple, cast
import numpy as np
import numpy.typing as npt
@@ -19,6 +19,11 @@ except ImportError:
cv2 = PlaceholderModule("cv2")
vr = PlaceholderModule("cv2").placeholder_attr("videoio_registry")
try:
import av
except ImportError:
av = PlaceholderModule("av") # type: ignore[assignment]
logger = init_logger(__name__)
@@ -355,8 +360,75 @@ class OpenCVVideoBackendMixin:
return frames, valid_frame_indices
class PyAVVideoBackendMixin:
"""PyAV (in-process FFmpeg bindings) codec utilities.
Reads stream metadata and decodes target frames via per-frame
``container.seek()``. The seek releases the GIL between frames and
scales with the number of sampled frames rather than the video
length, enabling concurrent decoding under serving load.
"""
@staticmethod
def get_metadata(
container: "av.container.InputContainer",
) -> VideoSourceMetadata:
if not container.streams.video:
raise ValueError("No video streams found in container")
stream = container.streams.video[0]
total_frames = stream.frames or 0
fps = float(stream.average_rate) if stream.average_rate else 0.0
duration = float(stream.duration * stream.time_base) if stream.duration else 0.0
if total_frames == 0 and duration > 0 and fps > 0:
total_frames = int(duration * fps)
return VideoSourceMetadata(total_frames, fps, duration)
@staticmethod
def decode_frames(
container: "av.container.InputContainer",
frame_indices: list[int],
fps: float,
duration: float,
) -> tuple[npt.NDArray, list[int]]:
"""Decode target frames via per-frame seek + keyframe decode."""
stream = container.streams.video[0]
# SLICE parallelizes within a single frame without the
# one-frame-per-thread latency penalty of FRAME threading.
stream.thread_type = "SLICE"
time_base = stream.time_base
frames_list: list[npt.NDArray] = []
valid_indices: list[int] = []
frame_interval = 1.0 / fps if fps > 0 else 0.1
max_ts = max(0.0, duration - frame_interval) if duration > 0 else float("inf")
for idx in frame_indices:
ts = min(idx / fps, max_ts) if fps > 0 else 0.0
pts = int(ts / time_base)
container.seek(pts, stream=stream)
frame = next(container.decode(video=0), None)
if frame is not None:
frames_list.append(frame.to_ndarray(format="rgb24"))
valid_indices.append(idx)
if not frames_list:
return np.empty((0,), dtype=np.uint8), valid_indices
return np.stack(frames_list), valid_indices
@VIDEO_LOADER_REGISTRY.register("opencv")
class OpenCVVideoBackend(VideoLoader, OpenCVVideoBackendMixin):
class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
"""Uniform-sampling video backend.
Samples ``num_frames`` uniformly across the video (or one frame every
``1/fps`` seconds, whichever produces fewer frames). The decoding codec
is selected via the ``backend`` kwarg (``"opencv"`` or ``"pyav"``),
which can be passed through ``--media-io-kwargs``. Defaults to
``"pyav"`` for concurrent decoding.
"""
_sampling_suffix: ClassVar[str] = ""
@classmethod
def compute_frames_index_to_sample(
cls,
@@ -366,7 +438,6 @@ class OpenCVVideoBackend(VideoLoader, OpenCVVideoBackendMixin):
) -> list[int]:
total_frames_num = source.total_frames_num
duration = source.duration
num_frames = target.num_frames
fps = target.fps
# resample video to target num_frames and fps
@@ -376,16 +447,18 @@ class OpenCVVideoBackend(VideoLoader, OpenCVVideoBackendMixin):
num_frames_to_sample = min(num_frames, total_frames_num)
if fps > 0:
num_frames_to_sample = min(num_frames_to_sample, math.floor(duration * fps))
num_frames_to_sample = max(1, num_frames_to_sample) # at least one sample
num_frames_to_sample = max(1, num_frames_to_sample)
if num_frames_to_sample == total_frames_num:
frame_idx = list(range(0, num_frames_to_sample))
else:
uniform_sampled_frames = np.linspace(
0, total_frames_num - 1, num_frames_to_sample, dtype=int
)
frame_idx = uniform_sampled_frames.tolist()
return frame_idx
return list(range(num_frames_to_sample))
return np.linspace(
0, total_frames_num - 1, num_frames_to_sample, dtype=int
).tolist()
@classmethod
def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata:
"""Sampling-algorithm-specific metadata adjustment hook."""
return source
@classmethod
def load_bytes(
@@ -395,55 +468,101 @@ class OpenCVVideoBackend(VideoLoader, OpenCVVideoBackendMixin):
fps: int = -1,
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal["opencv", "pyav"] = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
"""
Load video frames from bytes.
"""Load sampled frames from raw video bytes.
Args:
data: Raw video bytes
num_frames: Target number of frames to sample (-1 for all)
fps: Target FPS for sampling (-1 for original)
max_duration: Maximum duration (unused in base backend)
frame_recovery: Enable forward-scan recovery for failed frames
data: Raw video bytes.
num_frames: Target number of frames to sample (``-1`` for all).
fps: Target FPS for sampling (``-1`` for original).
max_duration: Maximum duration in seconds — only used by the
dynamic subclass; ignored here.
frame_recovery: Enable forward-scan recovery for failed frames.
Only honored by the OpenCV codec.
backend: Decoding codec — ``"opencv"`` or ``"pyav"`` .
Returns:
Tuple of (frames_array, metadata_dict)
Tuple of ``(frames_array, metadata_dict)``.
"""
cap = cls.open_video_capture(data)
source = OpenCVVideoBackendMixin.get_video_metadata(cap)
target = VideoTargetMetadata(
num_frames=num_frames,
fps=fps,
max_duration=max_duration,
num_frames=num_frames, fps=fps, max_duration=max_duration
)
# resample video to target num_frames and fps
# - the minimum of the two will be used
frame_idx = cls.compute_frames_index_to_sample(
if backend == "opencv":
cap = cls.open_video_capture(data)
source = cls._prepare_source(cls.get_video_metadata(cap))
frame_idx = cls.compute_frames_index_to_sample(
source=source, target=target, **kwargs
)
frames, valid = cls.read_frames(
cap,
frame_idx,
total_frames_num=source.total_frames_num,
frame_recovery=frame_recovery,
)
elif backend == "pyav":
assert not frame_recovery, (
"frame_recovery is only available for `opencv` backend"
)
with av.open(BytesIO(data)) as container:
source = cls._prepare_source(cls.get_metadata(container))
frame_idx = cls.compute_frames_index_to_sample(
source=source, target=target, **kwargs
)
frames, valid = cls.decode_frames(
container, frame_idx, source.original_fps, source.duration
)
else:
raise ValueError(
f"Unknown video codec backend {backend!r}; "
"valid options: 'opencv', 'pyav'."
)
if len(valid) < len(frame_idx):
logger.warning(
"%s video loading: expected %d frames but got %d.",
backend,
len(frame_idx),
len(valid),
)
return frames, cls.create_hf_metadata(
source=source,
target=target,
video_backend=f"{backend}{cls._sampling_suffix}",
valid_frame_indices=valid,
)
frames, valid_frame_indices = cls.read_frames(
cap,
frame_idx,
total_frames_num=source.total_frames_num,
frame_recovery=frame_recovery,
)
metadata = cls.create_hf_metadata(
source=source,
video_backend="opencv",
valid_frame_indices=valid_frame_indices,
)
return frames, metadata
@VIDEO_LOADER_REGISTRY.register("opencv_dynamic")
class OpenCVDynamicVideoBackend(VideoLoader, OpenCVVideoBackendMixin):
class DynamicVideoBackend(VideoBackend):
"""Duration-aware dynamic-sampling video backend.
Samples at ``fps`` up to ``max_duration`` seconds, falling back to
uniform sampling across the full duration when the video is longer
than ``max_duration``. Codec is selectable the same way as
:class:`VideoBackend`.
"""
_sampling_suffix: ClassVar[str] = "_dynamic"
@classmethod
def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata:
# Estimate duration from frame count and fps when the container
# does not report it (common for WebM/streaming inputs).
if source.duration:
return source
if source.original_fps > 0:
max_frame_idx = source.total_frames_num - 1
duration = round(max_frame_idx / source.original_fps) + 1
else:
duration = 0
return VideoSourceMetadata(
source.total_frames_num, source.original_fps, duration
)
@classmethod
def compute_frames_index_to_sample(
cls,
@@ -456,8 +575,8 @@ class OpenCVDynamicVideoBackend(VideoLoader, OpenCVVideoBackendMixin):
original_fps = source.original_fps
max_duration = target.max_duration
fps = target.fps
max_frame_idx = source.total_frames_num - 1
# Refer to:
# https://github.com/huggingface/transformers/blob/v4.55.4/src/transformers/models/glm4v/video_processing_glm4v.py#L103-L140
frame_indices_list: list[int]
@@ -491,62 +610,20 @@ class OpenCVDynamicVideoBackend(VideoLoader, OpenCVVideoBackendMixin):
fps: int = 2,
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal["opencv", "pyav"] = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
"""
Load video frames with dynamic sampling based on duration.
Args:
data: Raw video bytes
num_frames: Not used in dynamic backend
fps: Target FPS for sampling (default: 2)
max_duration: Maximum video duration to process (default: 300s)
frame_recovery: Enable forward-scan recovery for failed frames
Returns:
Tuple of (frames_array, metadata_dict)
"""
cap = cls.open_video_capture(data)
orig_source = OpenCVVideoBackendMixin.get_video_metadata(cap)
max_frame_idx = orig_source.total_frames_num - 1
duration = (
orig_source.duration or round(max_frame_idx / orig_source.original_fps) + 1
)
# recompute source metadata with adjusted duration to ensure correct
# sampling indices computation
source = VideoSourceMetadata(
total_frames_num=orig_source.total_frames_num,
original_fps=orig_source.original_fps,
duration=duration,
)
target = VideoTargetMetadata(
return super().load_bytes(
data,
num_frames=num_frames,
fps=fps,
max_duration=max_duration,
)
frame_indices_list = cls.compute_frames_index_to_sample(
source=source,
target=target,
)
frames, valid_frame_indices = cls.read_frames(
cap,
frame_indices_list,
total_frames_num=source.total_frames_num,
frame_recovery=frame_recovery,
backend=backend,
**kwargs,
)
metadata = cls.create_hf_metadata(
source=source,
video_backend="opencv_dynamic",
valid_frame_indices=valid_frame_indices,
)
return frames, metadata
@VIDEO_LOADER_REGISTRY.register("molmo2")
class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin):
@@ -835,7 +912,7 @@ class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin):
@VIDEO_LOADER_REGISTRY.register("nemotron_vl")
class NemotronVLVideoBackend(OpenCVVideoBackend):
class NemotronVLVideoBackend(VideoBackend):
@classmethod
def load_bytes(
cls,
@@ -844,14 +921,17 @@ class NemotronVLVideoBackend(OpenCVVideoBackend):
fps: int = -1,
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal["opencv", "pyav"] = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
frames, metadata = OpenCVVideoBackend.load_bytes(
frames, metadata = super().load_bytes(
data,
num_frames=num_frames,
fps=fps,
max_duration=max_duration,
frame_recovery=frame_recovery,
backend=backend,
**kwargs,
)