[Multimodal] Deprecate PyAV video decoder backend (#54231)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-08-29 15:13:28 +00:00
committed by GitHub
parent fd5d3aea94
commit 4fc943b867
9 changed files with 35 additions and 301 deletions
@@ -3,7 +3,7 @@
import math
from collections.abc import Mapping, Sequence
from typing import Any, Literal, cast
from typing import Any, cast
import numpy.typing as npt
import torch
@@ -35,6 +35,7 @@ from vllm.multimodal.processing import (
from vllm.multimodal.video import (
VIDEO_LOADER_REGISTRY,
VideoBackend,
VideoDecoderBackend,
VideoSourceMetadata,
VideoTargetMetadata,
)
@@ -485,13 +486,7 @@ class MiniMaxM3VideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv",
"pyav",
"torchcodec",
"pynvvideocodec",
"deepstream",
] = "opencv",
backend: VideoDecoderBackend = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
+12 -25
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from abc import abstractmethod
from typing import Any, ClassVar, Literal, cast
from typing import Any, ClassVar, cast
import numpy as np
import numpy.typing as npt
@@ -177,8 +177,8 @@ class VideoBackend(VideoLoader):
Samples ``num_frames`` uniformly across the video (or one frame every
``1/fps`` seconds, whichever produces fewer frames). The decoding codec
is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``,
``"torchcodec"``, ``"pynvvideocodec"``, or ``"deepstream"``),
is selected via the ``backend`` kwarg (``"opencv"``, ``"torchcodec"``,
``"pynvvideocodec"``, or ``"deepstream"``),
which can be passed through ``--media-io-kwargs``. Defaults to ``"opencv"``.
"""
@@ -232,16 +232,15 @@ class VideoBackend(VideoLoader):
dynamic subclass; ignored here.
frame_recovery: Enable forward-scan recovery for failed frames.
Only honored by the OpenCV codec.
backend: Decoding codec — ``"opencv"``, ``"pyav"``,
``"torchcodec"``, ``"pynvvideocodec"`` or ``"deepstream"``.
backend: Decoding codec — ``"opencv"``, ``"torchcodec"``,
``"pynvvideocodec"`` or ``"deepstream"``.
kwargs: Codec-specific options, validated against and forwarded to
``backend``:
- ``num_ffmpeg_threads`` (TorchCodec): number of FFmpeg
decoding threads; ``0`` (default) relies on the FFmpeg
default value which is ``min(cpu_count + 1, 16)``.
OpenCV will always use ``min(cpu_count, 16)`` while pyav
will always use ``min(cpu_count, (height + 15) / 16)``.
OpenCV will always use ``min(cpu_count, 16)``.
- ``seek_mode`` (TorchCodec): ``"exact"`` (default) guarantees
frame-accurate sampling by scanning the file on creation,
while ``"approximate"`` skips that scan for faster decoder
@@ -355,9 +354,7 @@ class Qwen3VLVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
] = "opencv",
backend: VideoDecoderBackend = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
@@ -436,9 +433,7 @@ class Qwen2VLVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
] = "opencv",
backend: VideoDecoderBackend = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
@@ -530,9 +525,7 @@ class DynamicVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
] = "opencv",
backend: VideoDecoderBackend = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
@@ -657,9 +650,7 @@ class GLM46VVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
] = "opencv",
backend: VideoDecoderBackend = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
@@ -757,9 +748,7 @@ class GLMGAVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
] = "opencv",
backend: VideoDecoderBackend = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
frames, metadata = super().load_bytes(
@@ -1073,9 +1062,7 @@ class NemotronVLVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
] = "opencv",
backend: VideoDecoderBackend = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
frames, metadata = super().load_bytes(
+1 -4
View File
@@ -12,13 +12,10 @@ from .base import (
check_frame_pixel_limit,
)
VideoDecoderBackend = Literal[
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
]
VideoDecoderBackend = Literal["opencv", "torchcodec", "pynvvideocodec", "deepstream"]
_BACKEND_OPTION_DEFAULTS: dict[str, dict[str, Any]] = {
"opencv": {},
"pyav": {},
"torchcodec": {
"num_ffmpeg_threads": 0,
"seek_mode": "exact",
+2 -2
View File
@@ -52,7 +52,7 @@ class DeepStreamVideoBackendMixin:
local file path is required — HTTP and base64 sources decode identically
to local files.
Like the OpenCV/PyAV mixins, this provides only the codec layer.
Like the OpenCV mixin, this provides only the codec layer.
Frame *selection* lives in the loader's
``compute_frames_index_to_sample`` and arrives here as an explicit
list of frame indices.
@@ -132,7 +132,7 @@ class DeepStreamVideoBackendMixin:
valid = frame_indices[: result.n_kept]
# GPU -> CPU NHWC uint8 at the codec boundary (one PCIe copy); keeps
# the array shape identical to the OpenCV/PyAV backends. Copy into
# the array shape identical to the OpenCV backend. Copy into
# PINNED host memory (reused across calls by PyTorch's pinned caching
# allocator) so the D2H runs at full PCIe bandwidth (~13 GB/s) rather
# than the ~1 GB/s pageable path that plain ``.cpu()`` takes — ~12x
-119
View File
@@ -1,119 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from io import BytesIO
import numpy as np
import numpy.typing as npt
from vllm.utils.import_utils import PlaceholderModule
from .base import (
VideoSourceMetadata,
VideoTargetMetadata,
check_frame_pixel_limit,
)
try:
import av
except ImportError:
av = PlaceholderModule("av") # type: ignore[assignment]
def decode_pyav(
loader_cls,
data: bytes,
target: VideoTargetMetadata,
sampling_kwargs: dict,
) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]:
with av.open(BytesIO(data)) as container:
stream = container.streams.video[0]
check_frame_pixel_limit(stream.width, stream.height)
source = loader_cls._prepare_source(
PyAVVideoBackendMixin.get_metadata(container)
)
frame_idx = loader_cls.compute_frames_index_to_sample(
source=source, target=target, **sampling_kwargs
)
frames, valid = PyAVVideoBackendMixin.decode_frames(
container, frame_idx, source.original_fps, source.duration
)
return frames, source, frame_idx, valid
class PyAVVideoBackendMixin:
"""PyAV (in-process FFmpeg bindings) codec utilities.
Reads stream metadata and decodes target frames via per-frame
``container.seek()``. The seek releases the GIL between frames and
scales with the number of sampled frames rather than the video
length, enabling concurrent decoding under serving load.
"""
@staticmethod
def get_metadata(
container: "av.container.InputContainer",
) -> VideoSourceMetadata:
if not container.streams.video:
raise ValueError("No video streams found in container")
stream = container.streams.video[0]
total_frames = stream.frames or 0
fps = float(stream.average_rate) if stream.average_rate else 0.0
duration = float(stream.duration * stream.time_base) if stream.duration else 0.0
if total_frames == 0 and duration > 0 and fps > 0:
total_frames = int(duration * fps)
elif duration > 0 and fps > 0 and total_frames > round(duration * fps) + 1:
# The header sample count can exceed the frames the presentation
# timeline actually holds — e.g. an mp4 edit list hides the decode
# lead-in of a stream-copied cut, but ``stream.frames`` still
# counts the hidden samples. Sampling indices past the visible
# range would silently collapse onto the last visible frame, so
# trust the (edit-list-aware) duration instead.
total_frames = round(duration * fps)
return VideoSourceMetadata(total_frames, fps, duration)
@staticmethod
def decode_frames(
container: "av.container.InputContainer",
frame_indices: list[int],
fps: float,
duration: float,
) -> tuple[npt.NDArray, list[int]]:
"""Decode target frames via per-frame seek + forward decode to PTS."""
stream = container.streams.video[0]
# SLICE parallelizes within a single frame without the
# one-frame-per-thread latency penalty of FRAME threading.
stream.thread_type = "SLICE"
time_base = stream.time_base
frames_list: list[npt.NDArray] = []
valid_indices: list[int] = []
frame_interval = 1.0 / fps if fps > 0 else 0.1
max_ts = max(0.0, duration - frame_interval) if duration > 0 else float("inf")
decoder = None
last_pts = None
for idx in frame_indices:
ts = min(idx / fps, max_ts) if fps > 0 else 0.0
pts = int(ts / time_base)
# seek() snaps backward to a keyframe; reuse the running decoder
# while targets advance monotonically to avoid re-decoding the
# GOP prefix once per requested frame.
if decoder is None or last_pts is None or pts <= last_pts:
container.seek(pts, stream=stream)
decoder = container.decode(video=0)
chosen = None
for frame in decoder:
if frame.pts is not None and frame.pts >= pts:
chosen = frame
last_pts = frame.pts
break
if chosen is not None:
frames_list.append(chosen.to_ndarray(format="rgb24"))
valid_indices.append(idx)
else:
decoder = None
if not frames_list:
return np.empty((0,), dtype=np.uint8), valid_indices
return np.stack(frames_list), valid_indices