[Multimodal] Deprecate PyAV video decoder backend (#54231)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Literal, cast
|
||||
from typing import Any, cast
|
||||
|
||||
import numpy.typing as npt
|
||||
import torch
|
||||
@@ -35,6 +35,7 @@ from vllm.multimodal.processing import (
|
||||
from vllm.multimodal.video import (
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
VideoBackend,
|
||||
VideoDecoderBackend,
|
||||
VideoSourceMetadata,
|
||||
VideoTargetMetadata,
|
||||
)
|
||||
@@ -485,13 +486,7 @@ class MiniMaxM3VideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal[
|
||||
"opencv",
|
||||
"pyav",
|
||||
"torchcodec",
|
||||
"pynvvideocodec",
|
||||
"deepstream",
|
||||
] = "opencv",
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
|
||||
+12
-25
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
from abc import abstractmethod
|
||||
from typing import Any, ClassVar, Literal, cast
|
||||
from typing import Any, ClassVar, cast
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
@@ -177,8 +177,8 @@ class VideoBackend(VideoLoader):
|
||||
|
||||
Samples ``num_frames`` uniformly across the video (or one frame every
|
||||
``1/fps`` seconds, whichever produces fewer frames). The decoding codec
|
||||
is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``,
|
||||
``"torchcodec"``, ``"pynvvideocodec"``, or ``"deepstream"``),
|
||||
is selected via the ``backend`` kwarg (``"opencv"``, ``"torchcodec"``,
|
||||
``"pynvvideocodec"``, or ``"deepstream"``),
|
||||
which can be passed through ``--media-io-kwargs``. Defaults to ``"opencv"``.
|
||||
"""
|
||||
|
||||
@@ -232,16 +232,15 @@ class VideoBackend(VideoLoader):
|
||||
dynamic subclass; ignored here.
|
||||
frame_recovery: Enable forward-scan recovery for failed frames.
|
||||
Only honored by the OpenCV codec.
|
||||
backend: Decoding codec — ``"opencv"``, ``"pyav"``,
|
||||
``"torchcodec"``, ``"pynvvideocodec"`` or ``"deepstream"``.
|
||||
backend: Decoding codec — ``"opencv"``, ``"torchcodec"``,
|
||||
``"pynvvideocodec"`` or ``"deepstream"``.
|
||||
kwargs: Codec-specific options, validated against and forwarded to
|
||||
``backend``:
|
||||
|
||||
- ``num_ffmpeg_threads`` (TorchCodec): number of FFmpeg
|
||||
decoding threads; ``0`` (default) relies on the FFmpeg
|
||||
default value which is ``min(cpu_count + 1, 16)``.
|
||||
OpenCV will always use ``min(cpu_count, 16)`` while pyav
|
||||
will always use ``min(cpu_count, (height + 15) / 16)``.
|
||||
OpenCV will always use ``min(cpu_count, 16)``.
|
||||
- ``seek_mode`` (TorchCodec): ``"exact"`` (default) guarantees
|
||||
frame-accurate sampling by scanning the file on creation,
|
||||
while ``"approximate"`` skips that scan for faster decoder
|
||||
@@ -355,9 +354,7 @@ class Qwen3VLVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal[
|
||||
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
|
||||
] = "opencv",
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
@@ -436,9 +433,7 @@ class Qwen2VLVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal[
|
||||
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
|
||||
] = "opencv",
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
@@ -530,9 +525,7 @@ class DynamicVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal[
|
||||
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
|
||||
] = "opencv",
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
@@ -657,9 +650,7 @@ class GLM46VVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal[
|
||||
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
|
||||
] = "opencv",
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
@@ -757,9 +748,7 @@ class GLMGAVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal[
|
||||
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
|
||||
] = "opencv",
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
frames, metadata = super().load_bytes(
|
||||
@@ -1073,9 +1062,7 @@ class NemotronVLVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal[
|
||||
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
|
||||
] = "opencv",
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
frames, metadata = super().load_bytes(
|
||||
|
||||
@@ -12,13 +12,10 @@ from .base import (
|
||||
check_frame_pixel_limit,
|
||||
)
|
||||
|
||||
VideoDecoderBackend = Literal[
|
||||
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
|
||||
]
|
||||
VideoDecoderBackend = Literal["opencv", "torchcodec", "pynvvideocodec", "deepstream"]
|
||||
|
||||
_BACKEND_OPTION_DEFAULTS: dict[str, dict[str, Any]] = {
|
||||
"opencv": {},
|
||||
"pyav": {},
|
||||
"torchcodec": {
|
||||
"num_ffmpeg_threads": 0,
|
||||
"seek_mode": "exact",
|
||||
|
||||
@@ -52,7 +52,7 @@ class DeepStreamVideoBackendMixin:
|
||||
local file path is required — HTTP and base64 sources decode identically
|
||||
to local files.
|
||||
|
||||
Like the OpenCV/PyAV mixins, this provides only the codec layer.
|
||||
Like the OpenCV mixin, this provides only the codec layer.
|
||||
Frame *selection* lives in the loader's
|
||||
``compute_frames_index_to_sample`` and arrives here as an explicit
|
||||
list of frame indices.
|
||||
@@ -132,7 +132,7 @@ class DeepStreamVideoBackendMixin:
|
||||
|
||||
valid = frame_indices[: result.n_kept]
|
||||
# GPU -> CPU NHWC uint8 at the codec boundary (one PCIe copy); keeps
|
||||
# the array shape identical to the OpenCV/PyAV backends. Copy into
|
||||
# the array shape identical to the OpenCV backend. Copy into
|
||||
# PINNED host memory (reused across calls by PyTorch's pinned caching
|
||||
# allocator) so the D2H runs at full PCIe bandwidth (~13 GB/s) rather
|
||||
# than the ~1 GB/s pageable path that plain ``.cpu()`` takes — ~12x
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
|
||||
from .base import (
|
||||
VideoSourceMetadata,
|
||||
VideoTargetMetadata,
|
||||
check_frame_pixel_limit,
|
||||
)
|
||||
|
||||
try:
|
||||
import av
|
||||
except ImportError:
|
||||
av = PlaceholderModule("av") # type: ignore[assignment]
|
||||
|
||||
|
||||
def decode_pyav(
|
||||
loader_cls,
|
||||
data: bytes,
|
||||
target: VideoTargetMetadata,
|
||||
sampling_kwargs: dict,
|
||||
) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]:
|
||||
with av.open(BytesIO(data)) as container:
|
||||
stream = container.streams.video[0]
|
||||
check_frame_pixel_limit(stream.width, stream.height)
|
||||
source = loader_cls._prepare_source(
|
||||
PyAVVideoBackendMixin.get_metadata(container)
|
||||
)
|
||||
frame_idx = loader_cls.compute_frames_index_to_sample(
|
||||
source=source, target=target, **sampling_kwargs
|
||||
)
|
||||
frames, valid = PyAVVideoBackendMixin.decode_frames(
|
||||
container, frame_idx, source.original_fps, source.duration
|
||||
)
|
||||
return frames, source, frame_idx, valid
|
||||
|
||||
|
||||
class PyAVVideoBackendMixin:
|
||||
"""PyAV (in-process FFmpeg bindings) codec utilities.
|
||||
|
||||
Reads stream metadata and decodes target frames via per-frame
|
||||
``container.seek()``. The seek releases the GIL between frames and
|
||||
scales with the number of sampled frames rather than the video
|
||||
length, enabling concurrent decoding under serving load.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_metadata(
|
||||
container: "av.container.InputContainer",
|
||||
) -> VideoSourceMetadata:
|
||||
if not container.streams.video:
|
||||
raise ValueError("No video streams found in container")
|
||||
stream = container.streams.video[0]
|
||||
total_frames = stream.frames or 0
|
||||
fps = float(stream.average_rate) if stream.average_rate else 0.0
|
||||
duration = float(stream.duration * stream.time_base) if stream.duration else 0.0
|
||||
if total_frames == 0 and duration > 0 and fps > 0:
|
||||
total_frames = int(duration * fps)
|
||||
elif duration > 0 and fps > 0 and total_frames > round(duration * fps) + 1:
|
||||
# The header sample count can exceed the frames the presentation
|
||||
# timeline actually holds — e.g. an mp4 edit list hides the decode
|
||||
# lead-in of a stream-copied cut, but ``stream.frames`` still
|
||||
# counts the hidden samples. Sampling indices past the visible
|
||||
# range would silently collapse onto the last visible frame, so
|
||||
# trust the (edit-list-aware) duration instead.
|
||||
total_frames = round(duration * fps)
|
||||
return VideoSourceMetadata(total_frames, fps, duration)
|
||||
|
||||
@staticmethod
|
||||
def decode_frames(
|
||||
container: "av.container.InputContainer",
|
||||
frame_indices: list[int],
|
||||
fps: float,
|
||||
duration: float,
|
||||
) -> tuple[npt.NDArray, list[int]]:
|
||||
"""Decode target frames via per-frame seek + forward decode to PTS."""
|
||||
stream = container.streams.video[0]
|
||||
# SLICE parallelizes within a single frame without the
|
||||
# one-frame-per-thread latency penalty of FRAME threading.
|
||||
stream.thread_type = "SLICE"
|
||||
time_base = stream.time_base
|
||||
|
||||
frames_list: list[npt.NDArray] = []
|
||||
valid_indices: list[int] = []
|
||||
frame_interval = 1.0 / fps if fps > 0 else 0.1
|
||||
max_ts = max(0.0, duration - frame_interval) if duration > 0 else float("inf")
|
||||
|
||||
decoder = None
|
||||
last_pts = None
|
||||
for idx in frame_indices:
|
||||
ts = min(idx / fps, max_ts) if fps > 0 else 0.0
|
||||
pts = int(ts / time_base)
|
||||
# seek() snaps backward to a keyframe; reuse the running decoder
|
||||
# while targets advance monotonically to avoid re-decoding the
|
||||
# GOP prefix once per requested frame.
|
||||
if decoder is None or last_pts is None or pts <= last_pts:
|
||||
container.seek(pts, stream=stream)
|
||||
decoder = container.decode(video=0)
|
||||
chosen = None
|
||||
for frame in decoder:
|
||||
if frame.pts is not None and frame.pts >= pts:
|
||||
chosen = frame
|
||||
last_pts = frame.pts
|
||||
break
|
||||
if chosen is not None:
|
||||
frames_list.append(chosen.to_ndarray(format="rgb24"))
|
||||
valid_indices.append(idx)
|
||||
else:
|
||||
decoder = None
|
||||
|
||||
if not frames_list:
|
||||
return np.empty((0,), dtype=np.uint8), valid_indices
|
||||
return np.stack(frames_list), valid_indices
|
||||
Reference in New Issue
Block a user