[Multimodal] Deprecate PyAV video decoder backend (#54231)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-08-29 15:13:28 +00:00
committed by GitHub
parent fd5d3aea94
commit 4fc943b867
9 changed files with 35 additions and 301 deletions
+4 -5
View File
@@ -844,20 +844,19 @@ Full example: [examples/generate/multimodal/openai_chat_completion_client_for_mu
#### Video Decoding Backend
vLLM decodes video bytes into frames using a selectable decoding backend. Five
vLLM decodes video bytes into frames using a selectable decoding backend. Four
backends are supported:
| Backend | Device | Description |
| --- | --- | --- |
| `opencv` (default) | CPU | OpenCV-based decoder |
| `pyav` | CPU | PyAV decoder |
| `torchcodec` | CPU | TorchCodec (PyTorch-native) decoder |
| `pynvvideocodec` | GPU | NVIDIA PyNvVideoCodec decoder |
| `deepstream` | GPU | NVIDIA DeepStream decoder |
The three CPU backends are ultimately backed by FFmpeg. `torchcodec` lets you
choose which FFmpeg version is used while `opencv` and `pyav` rely on whichever
FFmpeg build they were linked against.
The two CPU backends are ultimately backed by FFmpeg. `torchcodec` lets you
choose which FFmpeg version is used while `opencv` relies on whichever
FFmpeg build it was linked against.
Select the backend by passing the `backend` parameter via `--media-io-kwargs`:
@@ -146,7 +146,7 @@ def test_processor_override(
@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
@pytest.mark.parametrize("fps", [2])
@pytest.mark.parametrize("backend", ["opencv", "pyav"])
@pytest.mark.parametrize("backend", ["opencv"])
def test_video_loader_consistency(
model_id: str,
fps: int,
+2 -2
View File
@@ -454,7 +454,7 @@ class TestMergeKwargsGpuBackendPolicy:
)
assert result["hw_decoders"] == 2
@pytest.mark.parametrize("backend", ["opencv", "pyav", "torchcodec"])
@pytest.mark.parametrize("backend", ["opencv", "torchcodec"])
def test_software_video_backend_passes_through(self, backend: str):
result = VideoMediaIO.merge_kwargs(
default_kwargs=None,
@@ -462,7 +462,7 @@ class TestMergeKwargsGpuBackendPolicy:
)
assert result["video_backend"] == backend
@pytest.mark.parametrize("backend", ["opencv", "pyav"])
@pytest.mark.parametrize("backend", ["opencv"])
def test_software_codec_backend_passes_through(self, backend: str):
result = VideoMediaIO.merge_kwargs(
default_kwargs=None,
+10 -135
View File
@@ -113,7 +113,7 @@ import vllm.multimodal.video # noqa: F401
backend_modules = {
f"vllm.multimodal.video_decoders.{backend}"
for backend in ("opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream")
for backend in ("opencv", "torchcodec", "pynvvideocodec", "deepstream")
}
loaded = sorted(backend_modules & sys.modules.keys())
assert not loaded, loaded
@@ -128,7 +128,7 @@ assert not loaded, loaded
@pytest.mark.parametrize(
"backend",
["opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"],
["opencv", "torchcodec", "pynvvideocodec", "deepstream"],
)
def test_decode_video_imports_only_selected_backend(
backend: str,
@@ -199,15 +199,15 @@ def test_video_backend_kwargs_are_separated_from_sampling_kwargs(
def test_video_backend_rejects_options_for_another_decoder():
with pytest.raises(
ValueError, match="num_ffmpeg_threads is not supported by the 'pyav' backend"
ValueError, match="num_ffmpeg_threads is not supported by the 'opencv' backend"
):
resolve_video_backend_kwargs("pyav", {"num_ffmpeg_threads": 2})
resolve_video_backend_kwargs("opencv", {"num_ffmpeg_threads": 2})
@pytest.mark.parametrize(
("backend", "error"),
[
("pyav", AssertionError),
("torchcodec", AssertionError),
(PYNVVIDEOCODEC_VIDEO_BACKEND, ValueError),
],
)
@@ -885,7 +885,7 @@ def test_video_backend_handles_edit_list_trimmed_video(
)
loader = VIDEO_LOADER_REGISTRY.load("opencv")
for backend in ["opencv", "pyav"]:
for backend in ["opencv"]:
frames, metadata = loader.load_bytes(
video_data, num_frames=-1, backend=backend
)
@@ -1136,97 +1136,6 @@ def dummy_video_path(tmp_path):
return video_path
# ============================================================================
# PyAV Backend Tests
# ============================================================================
def test_pyav_backend_loads_frames(dummy_video_path, monkeypatch: pytest.MonkeyPatch):
"""Test that the pyav codec backend can load frames from a valid video."""
with monkeypatch.context() as m:
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
with open(dummy_video_path, "rb") as f:
video_data = f.read()
loader = VIDEO_LOADER_REGISTRY.load("opencv")
frames, metadata = loader.load_bytes(video_data, num_frames=8, backend="pyav")
assert frames.ndim == 4
assert frames.shape[3] == 3 # RGB
assert frames.shape[0] == 8
assert frames.shape[0] == len(metadata["frames_indices"])
assert metadata["video_backend"] == "pyav"
assert "total_num_frames" in metadata
assert "fps" in metadata
assert "duration" in metadata
def test_pyav_dynamic_backend_loads_frames(
dummy_video_path, monkeypatch: pytest.MonkeyPatch
):
"""Test that the pyav codec with dynamic sampling can load frames."""
with monkeypatch.context() as m:
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv_dynamic")
with open(dummy_video_path, "rb") as f:
video_data = f.read()
loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic")
frames, metadata = loader.load_bytes(
video_data, fps=2, max_duration=10, backend="pyav"
)
assert frames.ndim == 4
assert frames.shape[3] == 3 # RGB
assert frames.shape[0] > 0
assert frames.shape[0] == len(metadata["frames_indices"])
assert metadata["video_backend"] == "pyav_dynamic"
def test_pyav_backend_returns_target_frames_not_keyframes():
"""Regression test: PyAV must decode forward past the seek keyframe.
container.seek() snaps backward to the nearest keyframe. With a long GOP
(here: one keyframe at frame 0), a decoder that does not advance forward
to the target PTS collapses every sampled slot onto the keyframe. This
test encodes a per-frame marker on the green channel and verifies the
returned frames are distinct, ordered, and match the requested indices.
"""
num_frames = 50
num_sampled = 4
height, width = 64, 64
video_bytes = create_long_gop_video(
num_frames=num_frames, width=width, height=height
)
loader = VIDEO_LOADER_REGISTRY.load("opencv")
frames, metadata = loader.load_bytes(
video_bytes, num_frames=num_sampled, backend="pyav"
)
assert frames.shape == (num_sampled, height, width, 3)
requested = list(metadata["frames_indices"])
assert len(requested) == num_sampled
actual = [int(f[height // 2, width // 2, 1]) for f in frames]
assert len(set(actual)) == num_sampled, (
f"PyAV returned only {len(set(actual))} distinct frames for "
f"{num_sampled} requested indices: markers={actual}, "
f"requested={requested}. Keyframe-snap regression."
)
assert actual == sorted(actual), f"Returned frames out of order: markers={actual}"
for marker, want_idx in zip(actual, requested):
assert abs(marker - want_idx) <= 10, (
f"Frame mismatch: requested index {want_idx}, "
f"got marker {marker} (tolerance ±10)"
)
# ============================================================================
# TorchCodec Backend Tests
# ============================================================================
@@ -1298,11 +1207,10 @@ def test_torchcodec_backend_returns_target_frames_not_keyframes():
"""Regression test: torchcodec must return the requested frames, not the
GOP keyframe they seek back to.
Mirrors ``test_pyav_backend_returns_target_frames_not_keyframes``: a long
GOP (single keyframe at frame 0) with a per-frame green-channel marker.
With ``seek_mode="exact"`` torchcodec resolves each index to the exact
frame, so the returned markers must be distinct, ordered, and match the
requested indices.
A long GOP (single keyframe at frame 0) with a per-frame green-channel
marker. With ``seek_mode="exact"`` torchcodec resolves each index to the
exact frame, so the returned markers must be distinct, ordered, and match
the requested indices.
"""
pytest.importorskip("torchcodec")
num_frames = 50
@@ -1384,33 +1292,6 @@ def test_torchcodec_backend_returns_target_frames_not_keyframes():
119,
id="molmo2-fps",
),
# uniform sampling + pyav codec (same frame counts as opencv)
pytest.param(
"opencv",
{"num_frames": 32, "backend": "pyav"},
32,
id="pyav-num_frames",
),
pytest.param("opencv", {"fps": 2, "backend": "pyav"}, 120, id="pyav-fps"),
pytest.param(
"opencv",
{"num_frames": 500, "fps": 2, "backend": "pyav"},
120,
id="pyav-num_frames_wins_fps",
),
# dynamic sampling + pyav codec
pytest.param(
"opencv_dynamic",
{"fps": 1, "max_duration": 60, "backend": "pyav"},
60,
id="pyav_dynamic-within_max_duration",
),
pytest.param(
"opencv_dynamic",
{"fps": 2, "max_duration": 30, "backend": "pyav"},
60,
id="pyav_dynamic-exceeds_max_duration",
),
# glm46v dynamic FPS (1800 frames @ 30fps = 60s)
# 60s falls in (30, 300] → target_fps=1.0, extract_t = 60*1.0*2 = 120
pytest.param(
@@ -1419,12 +1300,6 @@ def test_torchcodec_backend_returns_target_frames_not_keyframes():
120,
id="glm46v-60s",
),
pytest.param(
"glm46v",
{"backend": "pyav"},
120,
id="glm46v-pyav-60s",
),
# uniform sampling + torchcodec codec (same frame counts as opencv)
pytest.param(
"opencv",
@@ -3,7 +3,7 @@
import math
from collections.abc import Mapping, Sequence
from typing import Any, Literal, cast
from typing import Any, cast
import numpy.typing as npt
import torch
@@ -35,6 +35,7 @@ from vllm.multimodal.processing import (
from vllm.multimodal.video import (
VIDEO_LOADER_REGISTRY,
VideoBackend,
VideoDecoderBackend,
VideoSourceMetadata,
VideoTargetMetadata,
)
@@ -485,13 +486,7 @@ class MiniMaxM3VideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv",
"pyav",
"torchcodec",
"pynvvideocodec",
"deepstream",
] = "opencv",
backend: VideoDecoderBackend = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
+12 -25
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from abc import abstractmethod
from typing import Any, ClassVar, Literal, cast
from typing import Any, ClassVar, cast
import numpy as np
import numpy.typing as npt
@@ -177,8 +177,8 @@ class VideoBackend(VideoLoader):
Samples ``num_frames`` uniformly across the video (or one frame every
``1/fps`` seconds, whichever produces fewer frames). The decoding codec
is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``,
``"torchcodec"``, ``"pynvvideocodec"``, or ``"deepstream"``),
is selected via the ``backend`` kwarg (``"opencv"``, ``"torchcodec"``,
``"pynvvideocodec"``, or ``"deepstream"``),
which can be passed through ``--media-io-kwargs``. Defaults to ``"opencv"``.
"""
@@ -232,16 +232,15 @@ class VideoBackend(VideoLoader):
dynamic subclass; ignored here.
frame_recovery: Enable forward-scan recovery for failed frames.
Only honored by the OpenCV codec.
backend: Decoding codec — ``"opencv"``, ``"pyav"``,
``"torchcodec"``, ``"pynvvideocodec"`` or ``"deepstream"``.
backend: Decoding codec — ``"opencv"``, ``"torchcodec"``,
``"pynvvideocodec"`` or ``"deepstream"``.
kwargs: Codec-specific options, validated against and forwarded to
``backend``:
- ``num_ffmpeg_threads`` (TorchCodec): number of FFmpeg
decoding threads; ``0`` (default) relies on the FFmpeg
default value which is ``min(cpu_count + 1, 16)``.
OpenCV will always use ``min(cpu_count, 16)`` while pyav
will always use ``min(cpu_count, (height + 15) / 16)``.
OpenCV will always use ``min(cpu_count, 16)``.
- ``seek_mode`` (TorchCodec): ``"exact"`` (default) guarantees
frame-accurate sampling by scanning the file on creation,
while ``"approximate"`` skips that scan for faster decoder
@@ -355,9 +354,7 @@ class Qwen3VLVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
] = "opencv",
backend: VideoDecoderBackend = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
@@ -436,9 +433,7 @@ class Qwen2VLVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
] = "opencv",
backend: VideoDecoderBackend = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
@@ -530,9 +525,7 @@ class DynamicVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
] = "opencv",
backend: VideoDecoderBackend = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
@@ -657,9 +650,7 @@ class GLM46VVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
] = "opencv",
backend: VideoDecoderBackend = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
@@ -757,9 +748,7 @@ class GLMGAVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
] = "opencv",
backend: VideoDecoderBackend = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
frames, metadata = super().load_bytes(
@@ -1073,9 +1062,7 @@ class NemotronVLVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
] = "opencv",
backend: VideoDecoderBackend = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
frames, metadata = super().load_bytes(
+1 -4
View File
@@ -12,13 +12,10 @@ from .base import (
check_frame_pixel_limit,
)
VideoDecoderBackend = Literal[
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
]
VideoDecoderBackend = Literal["opencv", "torchcodec", "pynvvideocodec", "deepstream"]
_BACKEND_OPTION_DEFAULTS: dict[str, dict[str, Any]] = {
"opencv": {},
"pyav": {},
"torchcodec": {
"num_ffmpeg_threads": 0,
"seek_mode": "exact",
+2 -2
View File
@@ -52,7 +52,7 @@ class DeepStreamVideoBackendMixin:
local file path is required — HTTP and base64 sources decode identically
to local files.
Like the OpenCV/PyAV mixins, this provides only the codec layer.
Like the OpenCV mixin, this provides only the codec layer.
Frame *selection* lives in the loader's
``compute_frames_index_to_sample`` and arrives here as an explicit
list of frame indices.
@@ -132,7 +132,7 @@ class DeepStreamVideoBackendMixin:
valid = frame_indices[: result.n_kept]
# GPU -> CPU NHWC uint8 at the codec boundary (one PCIe copy); keeps
# the array shape identical to the OpenCV/PyAV backends. Copy into
# the array shape identical to the OpenCV backend. Copy into
# PINNED host memory (reused across calls by PyTorch's pinned caching
# allocator) so the D2H runs at full PCIe bandwidth (~13 GB/s) rather
# than the ~1 GB/s pageable path that plain ``.cpu()`` takes — ~12x
-119
View File
@@ -1,119 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from io import BytesIO
import numpy as np
import numpy.typing as npt
from vllm.utils.import_utils import PlaceholderModule
from .base import (
VideoSourceMetadata,
VideoTargetMetadata,
check_frame_pixel_limit,
)
try:
import av
except ImportError:
av = PlaceholderModule("av") # type: ignore[assignment]
def decode_pyav(
loader_cls,
data: bytes,
target: VideoTargetMetadata,
sampling_kwargs: dict,
) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]:
with av.open(BytesIO(data)) as container:
stream = container.streams.video[0]
check_frame_pixel_limit(stream.width, stream.height)
source = loader_cls._prepare_source(
PyAVVideoBackendMixin.get_metadata(container)
)
frame_idx = loader_cls.compute_frames_index_to_sample(
source=source, target=target, **sampling_kwargs
)
frames, valid = PyAVVideoBackendMixin.decode_frames(
container, frame_idx, source.original_fps, source.duration
)
return frames, source, frame_idx, valid
class PyAVVideoBackendMixin:
"""PyAV (in-process FFmpeg bindings) codec utilities.
Reads stream metadata and decodes target frames via per-frame
``container.seek()``. The seek releases the GIL between frames and
scales with the number of sampled frames rather than the video
length, enabling concurrent decoding under serving load.
"""
@staticmethod
def get_metadata(
container: "av.container.InputContainer",
) -> VideoSourceMetadata:
if not container.streams.video:
raise ValueError("No video streams found in container")
stream = container.streams.video[0]
total_frames = stream.frames or 0
fps = float(stream.average_rate) if stream.average_rate else 0.0
duration = float(stream.duration * stream.time_base) if stream.duration else 0.0
if total_frames == 0 and duration > 0 and fps > 0:
total_frames = int(duration * fps)
elif duration > 0 and fps > 0 and total_frames > round(duration * fps) + 1:
# The header sample count can exceed the frames the presentation
# timeline actually holds — e.g. an mp4 edit list hides the decode
# lead-in of a stream-copied cut, but ``stream.frames`` still
# counts the hidden samples. Sampling indices past the visible
# range would silently collapse onto the last visible frame, so
# trust the (edit-list-aware) duration instead.
total_frames = round(duration * fps)
return VideoSourceMetadata(total_frames, fps, duration)
@staticmethod
def decode_frames(
container: "av.container.InputContainer",
frame_indices: list[int],
fps: float,
duration: float,
) -> tuple[npt.NDArray, list[int]]:
"""Decode target frames via per-frame seek + forward decode to PTS."""
stream = container.streams.video[0]
# SLICE parallelizes within a single frame without the
# one-frame-per-thread latency penalty of FRAME threading.
stream.thread_type = "SLICE"
time_base = stream.time_base
frames_list: list[npt.NDArray] = []
valid_indices: list[int] = []
frame_interval = 1.0 / fps if fps > 0 else 0.1
max_ts = max(0.0, duration - frame_interval) if duration > 0 else float("inf")
decoder = None
last_pts = None
for idx in frame_indices:
ts = min(idx / fps, max_ts) if fps > 0 else 0.0
pts = int(ts / time_base)
# seek() snaps backward to a keyframe; reuse the running decoder
# while targets advance monotonically to avoid re-decoding the
# GOP prefix once per requested frame.
if decoder is None or last_pts is None or pts <= last_pts:
container.seek(pts, stream=stream)
decoder = container.decode(video=0)
chosen = None
for frame in decoder:
if frame.pts is not None and frame.pts >= pts:
chosen = frame
last_pts = frame.pts
break
if chosen is not None:
frames_list.append(chosen.to_ndarray(format="rgb24"))
valid_indices.append(idx)
else:
decoder = None
if not frames_list:
return np.empty((0,), dtype=np.uint8), valid_indices
return np.stack(frames_list), valid_indices