diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index 387049f7cc..dd7ffec50c 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -844,20 +844,19 @@ Full example: [examples/generate/multimodal/openai_chat_completion_client_for_mu #### Video Decoding Backend -vLLM decodes video bytes into frames using a selectable decoding backend. Five +vLLM decodes video bytes into frames using a selectable decoding backend. Four backends are supported: | Backend | Device | Description | | --- | --- | --- | | `opencv` (default) | CPU | OpenCV-based decoder | -| `pyav` | CPU | PyAV decoder | | `torchcodec` | CPU | TorchCodec (PyTorch-native) decoder | | `pynvvideocodec` | GPU | NVIDIA PyNvVideoCodec decoder | | `deepstream` | GPU | NVIDIA DeepStream decoder | -The three CPU backends are ultimately backed by FFmpeg. `torchcodec` lets you -choose which FFmpeg version is used while `opencv` and `pyav` rely on whichever -FFmpeg build they were linked against. +The two CPU backends are ultimately backed by FFmpeg. `torchcodec` lets you +choose which FFmpeg version is used while `opencv` relies on whichever +FFmpeg build it was linked against. Select the backend by passing the `backend` parameter via `--media-io-kwargs`: diff --git a/tests/models/multimodal/processing/test_glm4_1v.py b/tests/models/multimodal/processing/test_glm4_1v.py index e45e741c5b..ce2ab2adfb 100644 --- a/tests/models/multimodal/processing/test_glm4_1v.py +++ b/tests/models/multimodal/processing/test_glm4_1v.py @@ -146,7 +146,7 @@ def test_processor_override( @pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"]) @pytest.mark.parametrize("fps", [2]) -@pytest.mark.parametrize("backend", ["opencv", "pyav"]) +@pytest.mark.parametrize("backend", ["opencv"]) def test_video_loader_consistency( model_id: str, fps: int, diff --git a/tests/multimodal/media/test_video.py b/tests/multimodal/media/test_video.py index d0402a2dfa..a3ea0890a2 100644 --- a/tests/multimodal/media/test_video.py +++ b/tests/multimodal/media/test_video.py @@ -454,7 +454,7 @@ class TestMergeKwargsGpuBackendPolicy: ) assert result["hw_decoders"] == 2 - @pytest.mark.parametrize("backend", ["opencv", "pyav", "torchcodec"]) + @pytest.mark.parametrize("backend", ["opencv", "torchcodec"]) def test_software_video_backend_passes_through(self, backend: str): result = VideoMediaIO.merge_kwargs( default_kwargs=None, @@ -462,7 +462,7 @@ class TestMergeKwargsGpuBackendPolicy: ) assert result["video_backend"] == backend - @pytest.mark.parametrize("backend", ["opencv", "pyav"]) + @pytest.mark.parametrize("backend", ["opencv"]) def test_software_codec_backend_passes_through(self, backend: str): result = VideoMediaIO.merge_kwargs( default_kwargs=None, diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index 4020900696..547e6dde23 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -113,7 +113,7 @@ import vllm.multimodal.video # noqa: F401 backend_modules = { f"vllm.multimodal.video_decoders.{backend}" - for backend in ("opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream") + for backend in ("opencv", "torchcodec", "pynvvideocodec", "deepstream") } loaded = sorted(backend_modules & sys.modules.keys()) assert not loaded, loaded @@ -128,7 +128,7 @@ assert not loaded, loaded @pytest.mark.parametrize( "backend", - ["opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"], + ["opencv", "torchcodec", "pynvvideocodec", "deepstream"], ) def test_decode_video_imports_only_selected_backend( backend: str, @@ -199,15 +199,15 @@ def test_video_backend_kwargs_are_separated_from_sampling_kwargs( def test_video_backend_rejects_options_for_another_decoder(): with pytest.raises( - ValueError, match="num_ffmpeg_threads is not supported by the 'pyav' backend" + ValueError, match="num_ffmpeg_threads is not supported by the 'opencv' backend" ): - resolve_video_backend_kwargs("pyav", {"num_ffmpeg_threads": 2}) + resolve_video_backend_kwargs("opencv", {"num_ffmpeg_threads": 2}) @pytest.mark.parametrize( ("backend", "error"), [ - ("pyav", AssertionError), + ("torchcodec", AssertionError), (PYNVVIDEOCODEC_VIDEO_BACKEND, ValueError), ], ) @@ -885,7 +885,7 @@ def test_video_backend_handles_edit_list_trimmed_video( ) loader = VIDEO_LOADER_REGISTRY.load("opencv") - for backend in ["opencv", "pyav"]: + for backend in ["opencv"]: frames, metadata = loader.load_bytes( video_data, num_frames=-1, backend=backend ) @@ -1136,97 +1136,6 @@ def dummy_video_path(tmp_path): return video_path -# ============================================================================ -# PyAV Backend Tests -# ============================================================================ - - -def test_pyav_backend_loads_frames(dummy_video_path, monkeypatch: pytest.MonkeyPatch): - """Test that the pyav codec backend can load frames from a valid video.""" - with monkeypatch.context() as m: - m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv") - - with open(dummy_video_path, "rb") as f: - video_data = f.read() - - loader = VIDEO_LOADER_REGISTRY.load("opencv") - frames, metadata = loader.load_bytes(video_data, num_frames=8, backend="pyav") - - assert frames.ndim == 4 - assert frames.shape[3] == 3 # RGB - assert frames.shape[0] == 8 - assert frames.shape[0] == len(metadata["frames_indices"]) - assert metadata["video_backend"] == "pyav" - assert "total_num_frames" in metadata - assert "fps" in metadata - assert "duration" in metadata - - -def test_pyav_dynamic_backend_loads_frames( - dummy_video_path, monkeypatch: pytest.MonkeyPatch -): - """Test that the pyav codec with dynamic sampling can load frames.""" - with monkeypatch.context() as m: - m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv_dynamic") - - with open(dummy_video_path, "rb") as f: - video_data = f.read() - - loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic") - frames, metadata = loader.load_bytes( - video_data, fps=2, max_duration=10, backend="pyav" - ) - - assert frames.ndim == 4 - assert frames.shape[3] == 3 # RGB - assert frames.shape[0] > 0 - assert frames.shape[0] == len(metadata["frames_indices"]) - assert metadata["video_backend"] == "pyav_dynamic" - - -def test_pyav_backend_returns_target_frames_not_keyframes(): - """Regression test: PyAV must decode forward past the seek keyframe. - - container.seek() snaps backward to the nearest keyframe. With a long GOP - (here: one keyframe at frame 0), a decoder that does not advance forward - to the target PTS collapses every sampled slot onto the keyframe. This - test encodes a per-frame marker on the green channel and verifies the - returned frames are distinct, ordered, and match the requested indices. - """ - num_frames = 50 - num_sampled = 4 - height, width = 64, 64 - - video_bytes = create_long_gop_video( - num_frames=num_frames, width=width, height=height - ) - - loader = VIDEO_LOADER_REGISTRY.load("opencv") - frames, metadata = loader.load_bytes( - video_bytes, num_frames=num_sampled, backend="pyav" - ) - assert frames.shape == (num_sampled, height, width, 3) - - requested = list(metadata["frames_indices"]) - assert len(requested) == num_sampled - - actual = [int(f[height // 2, width // 2, 1]) for f in frames] - - assert len(set(actual)) == num_sampled, ( - f"PyAV returned only {len(set(actual))} distinct frames for " - f"{num_sampled} requested indices: markers={actual}, " - f"requested={requested}. Keyframe-snap regression." - ) - - assert actual == sorted(actual), f"Returned frames out of order: markers={actual}" - - for marker, want_idx in zip(actual, requested): - assert abs(marker - want_idx) <= 10, ( - f"Frame mismatch: requested index {want_idx}, " - f"got marker {marker} (tolerance ±10)" - ) - - # ============================================================================ # TorchCodec Backend Tests # ============================================================================ @@ -1298,11 +1207,10 @@ def test_torchcodec_backend_returns_target_frames_not_keyframes(): """Regression test: torchcodec must return the requested frames, not the GOP keyframe they seek back to. - Mirrors ``test_pyav_backend_returns_target_frames_not_keyframes``: a long - GOP (single keyframe at frame 0) with a per-frame green-channel marker. - With ``seek_mode="exact"`` torchcodec resolves each index to the exact - frame, so the returned markers must be distinct, ordered, and match the - requested indices. + A long GOP (single keyframe at frame 0) with a per-frame green-channel + marker. With ``seek_mode="exact"`` torchcodec resolves each index to the + exact frame, so the returned markers must be distinct, ordered, and match + the requested indices. """ pytest.importorskip("torchcodec") num_frames = 50 @@ -1384,33 +1292,6 @@ def test_torchcodec_backend_returns_target_frames_not_keyframes(): 119, id="molmo2-fps", ), - # uniform sampling + pyav codec (same frame counts as opencv) - pytest.param( - "opencv", - {"num_frames": 32, "backend": "pyav"}, - 32, - id="pyav-num_frames", - ), - pytest.param("opencv", {"fps": 2, "backend": "pyav"}, 120, id="pyav-fps"), - pytest.param( - "opencv", - {"num_frames": 500, "fps": 2, "backend": "pyav"}, - 120, - id="pyav-num_frames_wins_fps", - ), - # dynamic sampling + pyav codec - pytest.param( - "opencv_dynamic", - {"fps": 1, "max_duration": 60, "backend": "pyav"}, - 60, - id="pyav_dynamic-within_max_duration", - ), - pytest.param( - "opencv_dynamic", - {"fps": 2, "max_duration": 30, "backend": "pyav"}, - 60, - id="pyav_dynamic-exceeds_max_duration", - ), # glm46v dynamic FPS (1800 frames @ 30fps = 60s) # 60s falls in (30, 300] → target_fps=1.0, extract_t = 60*1.0*2 = 120 pytest.param( @@ -1419,12 +1300,6 @@ def test_torchcodec_backend_returns_target_frames_not_keyframes(): 120, id="glm46v-60s", ), - pytest.param( - "glm46v", - {"backend": "pyav"}, - 120, - id="glm46v-pyav-60s", - ), # uniform sampling + torchcodec codec (same frame counts as opencv) pytest.param( "opencv", diff --git a/vllm/models/minimax_m3/common/mm_preprocess.py b/vllm/models/minimax_m3/common/mm_preprocess.py index 86b702585f..9f1d08011a 100644 --- a/vllm/models/minimax_m3/common/mm_preprocess.py +++ b/vllm/models/minimax_m3/common/mm_preprocess.py @@ -3,7 +3,7 @@ import math from collections.abc import Mapping, Sequence -from typing import Any, Literal, cast +from typing import Any, cast import numpy.typing as npt import torch @@ -35,6 +35,7 @@ from vllm.multimodal.processing import ( from vllm.multimodal.video import ( VIDEO_LOADER_REGISTRY, VideoBackend, + VideoDecoderBackend, VideoSourceMetadata, VideoTargetMetadata, ) @@ -485,13 +486,7 @@ class MiniMaxM3VideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal[ - "opencv", - "pyav", - "torchcodec", - "pynvvideocodec", - "deepstream", - ] = "opencv", + backend: VideoDecoderBackend = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: return super().load_bytes( diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index f1a605311a..698afd397a 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import math from abc import abstractmethod -from typing import Any, ClassVar, Literal, cast +from typing import Any, ClassVar, cast import numpy as np import numpy.typing as npt @@ -177,8 +177,8 @@ class VideoBackend(VideoLoader): Samples ``num_frames`` uniformly across the video (or one frame every ``1/fps`` seconds, whichever produces fewer frames). The decoding codec - is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``, - ``"torchcodec"``, ``"pynvvideocodec"``, or ``"deepstream"``), + is selected via the ``backend`` kwarg (``"opencv"``, ``"torchcodec"``, + ``"pynvvideocodec"``, or ``"deepstream"``), which can be passed through ``--media-io-kwargs``. Defaults to ``"opencv"``. """ @@ -232,16 +232,15 @@ class VideoBackend(VideoLoader): dynamic subclass; ignored here. frame_recovery: Enable forward-scan recovery for failed frames. Only honored by the OpenCV codec. - backend: Decoding codec — ``"opencv"``, ``"pyav"``, - ``"torchcodec"``, ``"pynvvideocodec"`` or ``"deepstream"``. + backend: Decoding codec — ``"opencv"``, ``"torchcodec"``, + ``"pynvvideocodec"`` or ``"deepstream"``. kwargs: Codec-specific options, validated against and forwarded to ``backend``: - ``num_ffmpeg_threads`` (TorchCodec): number of FFmpeg decoding threads; ``0`` (default) relies on the FFmpeg default value which is ``min(cpu_count + 1, 16)``. - OpenCV will always use ``min(cpu_count, 16)`` while pyav - will always use ``min(cpu_count, (height + 15) / 16)``. + OpenCV will always use ``min(cpu_count, 16)``. - ``seek_mode`` (TorchCodec): ``"exact"`` (default) guarantees frame-accurate sampling by scanning the file on creation, while ``"approximate"`` skips that scan for faster decoder @@ -355,9 +354,7 @@ class Qwen3VLVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal[ - "opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream" - ] = "opencv", + backend: VideoDecoderBackend = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: return super().load_bytes( @@ -436,9 +433,7 @@ class Qwen2VLVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal[ - "opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream" - ] = "opencv", + backend: VideoDecoderBackend = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: return super().load_bytes( @@ -530,9 +525,7 @@ class DynamicVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal[ - "opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream" - ] = "opencv", + backend: VideoDecoderBackend = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: return super().load_bytes( @@ -657,9 +650,7 @@ class GLM46VVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal[ - "opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream" - ] = "opencv", + backend: VideoDecoderBackend = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: return super().load_bytes( @@ -757,9 +748,7 @@ class GLMGAVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal[ - "opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream" - ] = "opencv", + backend: VideoDecoderBackend = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: frames, metadata = super().load_bytes( @@ -1073,9 +1062,7 @@ class NemotronVLVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal[ - "opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream" - ] = "opencv", + backend: VideoDecoderBackend = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: frames, metadata = super().load_bytes( diff --git a/vllm/multimodal/video_decoders/__init__.py b/vllm/multimodal/video_decoders/__init__.py index c623104316..1e6106e199 100644 --- a/vllm/multimodal/video_decoders/__init__.py +++ b/vllm/multimodal/video_decoders/__init__.py @@ -12,13 +12,10 @@ from .base import ( check_frame_pixel_limit, ) -VideoDecoderBackend = Literal[ - "opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream" -] +VideoDecoderBackend = Literal["opencv", "torchcodec", "pynvvideocodec", "deepstream"] _BACKEND_OPTION_DEFAULTS: dict[str, dict[str, Any]] = { "opencv": {}, - "pyav": {}, "torchcodec": { "num_ffmpeg_threads": 0, "seek_mode": "exact", diff --git a/vllm/multimodal/video_decoders/deepstream.py b/vllm/multimodal/video_decoders/deepstream.py index 05262f5586..b0fc0f7c42 100644 --- a/vllm/multimodal/video_decoders/deepstream.py +++ b/vllm/multimodal/video_decoders/deepstream.py @@ -52,7 +52,7 @@ class DeepStreamVideoBackendMixin: local file path is required — HTTP and base64 sources decode identically to local files. - Like the OpenCV/PyAV mixins, this provides only the codec layer. + Like the OpenCV mixin, this provides only the codec layer. Frame *selection* lives in the loader's ``compute_frames_index_to_sample`` and arrives here as an explicit list of frame indices. @@ -132,7 +132,7 @@ class DeepStreamVideoBackendMixin: valid = frame_indices[: result.n_kept] # GPU -> CPU NHWC uint8 at the codec boundary (one PCIe copy); keeps - # the array shape identical to the OpenCV/PyAV backends. Copy into + # the array shape identical to the OpenCV backend. Copy into # PINNED host memory (reused across calls by PyTorch's pinned caching # allocator) so the D2H runs at full PCIe bandwidth (~13 GB/s) rather # than the ~1 GB/s pageable path that plain ``.cpu()`` takes — ~12x diff --git a/vllm/multimodal/video_decoders/pyav.py b/vllm/multimodal/video_decoders/pyav.py deleted file mode 100644 index 80c2829666..0000000000 --- a/vllm/multimodal/video_decoders/pyav.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from io import BytesIO - -import numpy as np -import numpy.typing as npt - -from vllm.utils.import_utils import PlaceholderModule - -from .base import ( - VideoSourceMetadata, - VideoTargetMetadata, - check_frame_pixel_limit, -) - -try: - import av -except ImportError: - av = PlaceholderModule("av") # type: ignore[assignment] - - -def decode_pyav( - loader_cls, - data: bytes, - target: VideoTargetMetadata, - sampling_kwargs: dict, -) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]: - with av.open(BytesIO(data)) as container: - stream = container.streams.video[0] - check_frame_pixel_limit(stream.width, stream.height) - source = loader_cls._prepare_source( - PyAVVideoBackendMixin.get_metadata(container) - ) - frame_idx = loader_cls.compute_frames_index_to_sample( - source=source, target=target, **sampling_kwargs - ) - frames, valid = PyAVVideoBackendMixin.decode_frames( - container, frame_idx, source.original_fps, source.duration - ) - return frames, source, frame_idx, valid - - -class PyAVVideoBackendMixin: - """PyAV (in-process FFmpeg bindings) codec utilities. - - Reads stream metadata and decodes target frames via per-frame - ``container.seek()``. The seek releases the GIL between frames and - scales with the number of sampled frames rather than the video - length, enabling concurrent decoding under serving load. - """ - - @staticmethod - def get_metadata( - container: "av.container.InputContainer", - ) -> VideoSourceMetadata: - if not container.streams.video: - raise ValueError("No video streams found in container") - stream = container.streams.video[0] - total_frames = stream.frames or 0 - fps = float(stream.average_rate) if stream.average_rate else 0.0 - duration = float(stream.duration * stream.time_base) if stream.duration else 0.0 - if total_frames == 0 and duration > 0 and fps > 0: - total_frames = int(duration * fps) - elif duration > 0 and fps > 0 and total_frames > round(duration * fps) + 1: - # The header sample count can exceed the frames the presentation - # timeline actually holds — e.g. an mp4 edit list hides the decode - # lead-in of a stream-copied cut, but ``stream.frames`` still - # counts the hidden samples. Sampling indices past the visible - # range would silently collapse onto the last visible frame, so - # trust the (edit-list-aware) duration instead. - total_frames = round(duration * fps) - return VideoSourceMetadata(total_frames, fps, duration) - - @staticmethod - def decode_frames( - container: "av.container.InputContainer", - frame_indices: list[int], - fps: float, - duration: float, - ) -> tuple[npt.NDArray, list[int]]: - """Decode target frames via per-frame seek + forward decode to PTS.""" - stream = container.streams.video[0] - # SLICE parallelizes within a single frame without the - # one-frame-per-thread latency penalty of FRAME threading. - stream.thread_type = "SLICE" - time_base = stream.time_base - - frames_list: list[npt.NDArray] = [] - valid_indices: list[int] = [] - frame_interval = 1.0 / fps if fps > 0 else 0.1 - max_ts = max(0.0, duration - frame_interval) if duration > 0 else float("inf") - - decoder = None - last_pts = None - for idx in frame_indices: - ts = min(idx / fps, max_ts) if fps > 0 else 0.0 - pts = int(ts / time_base) - # seek() snaps backward to a keyframe; reuse the running decoder - # while targets advance monotonically to avoid re-decoding the - # GOP prefix once per requested frame. - if decoder is None or last_pts is None or pts <= last_pts: - container.seek(pts, stream=stream) - decoder = container.decode(video=0) - chosen = None - for frame in decoder: - if frame.pts is not None and frame.pts >= pts: - chosen = frame - last_pts = frame.pts - break - if chosen is not None: - frames_list.append(chosen.to_ndarray(format="rgb24")) - valid_indices.append(idx) - else: - decoder = None - - if not frames_list: - return np.empty((0,), dtype=np.uint8), valid_indices - return np.stack(frames_list), valid_indices