[Multimodal] Deprecate PyAV video decoder backend (#54231)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -844,20 +844,19 @@ Full example: [examples/generate/multimodal/openai_chat_completion_client_for_mu
|
||||
|
||||
#### Video Decoding Backend
|
||||
|
||||
vLLM decodes video bytes into frames using a selectable decoding backend. Five
|
||||
vLLM decodes video bytes into frames using a selectable decoding backend. Four
|
||||
backends are supported:
|
||||
|
||||
| Backend | Device | Description |
|
||||
| --- | --- | --- |
|
||||
| `opencv` (default) | CPU | OpenCV-based decoder |
|
||||
| `pyav` | CPU | PyAV decoder |
|
||||
| `torchcodec` | CPU | TorchCodec (PyTorch-native) decoder |
|
||||
| `pynvvideocodec` | GPU | NVIDIA PyNvVideoCodec decoder |
|
||||
| `deepstream` | GPU | NVIDIA DeepStream decoder |
|
||||
|
||||
The three CPU backends are ultimately backed by FFmpeg. `torchcodec` lets you
|
||||
choose which FFmpeg version is used while `opencv` and `pyav` rely on whichever
|
||||
FFmpeg build they were linked against.
|
||||
The two CPU backends are ultimately backed by FFmpeg. `torchcodec` lets you
|
||||
choose which FFmpeg version is used while `opencv` relies on whichever
|
||||
FFmpeg build it was linked against.
|
||||
|
||||
Select the backend by passing the `backend` parameter via `--media-io-kwargs`:
|
||||
|
||||
|
||||
@@ -146,7 +146,7 @@ def test_processor_override(
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
|
||||
@pytest.mark.parametrize("fps", [2])
|
||||
@pytest.mark.parametrize("backend", ["opencv", "pyav"])
|
||||
@pytest.mark.parametrize("backend", ["opencv"])
|
||||
def test_video_loader_consistency(
|
||||
model_id: str,
|
||||
fps: int,
|
||||
|
||||
@@ -454,7 +454,7 @@ class TestMergeKwargsGpuBackendPolicy:
|
||||
)
|
||||
assert result["hw_decoders"] == 2
|
||||
|
||||
@pytest.mark.parametrize("backend", ["opencv", "pyav", "torchcodec"])
|
||||
@pytest.mark.parametrize("backend", ["opencv", "torchcodec"])
|
||||
def test_software_video_backend_passes_through(self, backend: str):
|
||||
result = VideoMediaIO.merge_kwargs(
|
||||
default_kwargs=None,
|
||||
@@ -462,7 +462,7 @@ class TestMergeKwargsGpuBackendPolicy:
|
||||
)
|
||||
assert result["video_backend"] == backend
|
||||
|
||||
@pytest.mark.parametrize("backend", ["opencv", "pyav"])
|
||||
@pytest.mark.parametrize("backend", ["opencv"])
|
||||
def test_software_codec_backend_passes_through(self, backend: str):
|
||||
result = VideoMediaIO.merge_kwargs(
|
||||
default_kwargs=None,
|
||||
|
||||
+10
-135
@@ -113,7 +113,7 @@ import vllm.multimodal.video # noqa: F401
|
||||
|
||||
backend_modules = {
|
||||
f"vllm.multimodal.video_decoders.{backend}"
|
||||
for backend in ("opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream")
|
||||
for backend in ("opencv", "torchcodec", "pynvvideocodec", "deepstream")
|
||||
}
|
||||
loaded = sorted(backend_modules & sys.modules.keys())
|
||||
assert not loaded, loaded
|
||||
@@ -128,7 +128,7 @@ assert not loaded, loaded
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend",
|
||||
["opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"],
|
||||
["opencv", "torchcodec", "pynvvideocodec", "deepstream"],
|
||||
)
|
||||
def test_decode_video_imports_only_selected_backend(
|
||||
backend: str,
|
||||
@@ -199,15 +199,15 @@ def test_video_backend_kwargs_are_separated_from_sampling_kwargs(
|
||||
|
||||
def test_video_backend_rejects_options_for_another_decoder():
|
||||
with pytest.raises(
|
||||
ValueError, match="num_ffmpeg_threads is not supported by the 'pyav' backend"
|
||||
ValueError, match="num_ffmpeg_threads is not supported by the 'opencv' backend"
|
||||
):
|
||||
resolve_video_backend_kwargs("pyav", {"num_ffmpeg_threads": 2})
|
||||
resolve_video_backend_kwargs("opencv", {"num_ffmpeg_threads": 2})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "error"),
|
||||
[
|
||||
("pyav", AssertionError),
|
||||
("torchcodec", AssertionError),
|
||||
(PYNVVIDEOCODEC_VIDEO_BACKEND, ValueError),
|
||||
],
|
||||
)
|
||||
@@ -885,7 +885,7 @@ def test_video_backend_handles_edit_list_trimmed_video(
|
||||
)
|
||||
|
||||
loader = VIDEO_LOADER_REGISTRY.load("opencv")
|
||||
for backend in ["opencv", "pyav"]:
|
||||
for backend in ["opencv"]:
|
||||
frames, metadata = loader.load_bytes(
|
||||
video_data, num_frames=-1, backend=backend
|
||||
)
|
||||
@@ -1136,97 +1136,6 @@ def dummy_video_path(tmp_path):
|
||||
return video_path
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# PyAV Backend Tests
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def test_pyav_backend_loads_frames(dummy_video_path, monkeypatch: pytest.MonkeyPatch):
|
||||
"""Test that the pyav codec backend can load frames from a valid video."""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
|
||||
|
||||
with open(dummy_video_path, "rb") as f:
|
||||
video_data = f.read()
|
||||
|
||||
loader = VIDEO_LOADER_REGISTRY.load("opencv")
|
||||
frames, metadata = loader.load_bytes(video_data, num_frames=8, backend="pyav")
|
||||
|
||||
assert frames.ndim == 4
|
||||
assert frames.shape[3] == 3 # RGB
|
||||
assert frames.shape[0] == 8
|
||||
assert frames.shape[0] == len(metadata["frames_indices"])
|
||||
assert metadata["video_backend"] == "pyav"
|
||||
assert "total_num_frames" in metadata
|
||||
assert "fps" in metadata
|
||||
assert "duration" in metadata
|
||||
|
||||
|
||||
def test_pyav_dynamic_backend_loads_frames(
|
||||
dummy_video_path, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
"""Test that the pyav codec with dynamic sampling can load frames."""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv_dynamic")
|
||||
|
||||
with open(dummy_video_path, "rb") as f:
|
||||
video_data = f.read()
|
||||
|
||||
loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic")
|
||||
frames, metadata = loader.load_bytes(
|
||||
video_data, fps=2, max_duration=10, backend="pyav"
|
||||
)
|
||||
|
||||
assert frames.ndim == 4
|
||||
assert frames.shape[3] == 3 # RGB
|
||||
assert frames.shape[0] > 0
|
||||
assert frames.shape[0] == len(metadata["frames_indices"])
|
||||
assert metadata["video_backend"] == "pyav_dynamic"
|
||||
|
||||
|
||||
def test_pyav_backend_returns_target_frames_not_keyframes():
|
||||
"""Regression test: PyAV must decode forward past the seek keyframe.
|
||||
|
||||
container.seek() snaps backward to the nearest keyframe. With a long GOP
|
||||
(here: one keyframe at frame 0), a decoder that does not advance forward
|
||||
to the target PTS collapses every sampled slot onto the keyframe. This
|
||||
test encodes a per-frame marker on the green channel and verifies the
|
||||
returned frames are distinct, ordered, and match the requested indices.
|
||||
"""
|
||||
num_frames = 50
|
||||
num_sampled = 4
|
||||
height, width = 64, 64
|
||||
|
||||
video_bytes = create_long_gop_video(
|
||||
num_frames=num_frames, width=width, height=height
|
||||
)
|
||||
|
||||
loader = VIDEO_LOADER_REGISTRY.load("opencv")
|
||||
frames, metadata = loader.load_bytes(
|
||||
video_bytes, num_frames=num_sampled, backend="pyav"
|
||||
)
|
||||
assert frames.shape == (num_sampled, height, width, 3)
|
||||
|
||||
requested = list(metadata["frames_indices"])
|
||||
assert len(requested) == num_sampled
|
||||
|
||||
actual = [int(f[height // 2, width // 2, 1]) for f in frames]
|
||||
|
||||
assert len(set(actual)) == num_sampled, (
|
||||
f"PyAV returned only {len(set(actual))} distinct frames for "
|
||||
f"{num_sampled} requested indices: markers={actual}, "
|
||||
f"requested={requested}. Keyframe-snap regression."
|
||||
)
|
||||
|
||||
assert actual == sorted(actual), f"Returned frames out of order: markers={actual}"
|
||||
|
||||
for marker, want_idx in zip(actual, requested):
|
||||
assert abs(marker - want_idx) <= 10, (
|
||||
f"Frame mismatch: requested index {want_idx}, "
|
||||
f"got marker {marker} (tolerance ±10)"
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# TorchCodec Backend Tests
|
||||
# ============================================================================
|
||||
@@ -1298,11 +1207,10 @@ def test_torchcodec_backend_returns_target_frames_not_keyframes():
|
||||
"""Regression test: torchcodec must return the requested frames, not the
|
||||
GOP keyframe they seek back to.
|
||||
|
||||
Mirrors ``test_pyav_backend_returns_target_frames_not_keyframes``: a long
|
||||
GOP (single keyframe at frame 0) with a per-frame green-channel marker.
|
||||
With ``seek_mode="exact"`` torchcodec resolves each index to the exact
|
||||
frame, so the returned markers must be distinct, ordered, and match the
|
||||
requested indices.
|
||||
A long GOP (single keyframe at frame 0) with a per-frame green-channel
|
||||
marker. With ``seek_mode="exact"`` torchcodec resolves each index to the
|
||||
exact frame, so the returned markers must be distinct, ordered, and match
|
||||
the requested indices.
|
||||
"""
|
||||
pytest.importorskip("torchcodec")
|
||||
num_frames = 50
|
||||
@@ -1384,33 +1292,6 @@ def test_torchcodec_backend_returns_target_frames_not_keyframes():
|
||||
119,
|
||||
id="molmo2-fps",
|
||||
),
|
||||
# uniform sampling + pyav codec (same frame counts as opencv)
|
||||
pytest.param(
|
||||
"opencv",
|
||||
{"num_frames": 32, "backend": "pyav"},
|
||||
32,
|
||||
id="pyav-num_frames",
|
||||
),
|
||||
pytest.param("opencv", {"fps": 2, "backend": "pyav"}, 120, id="pyav-fps"),
|
||||
pytest.param(
|
||||
"opencv",
|
||||
{"num_frames": 500, "fps": 2, "backend": "pyav"},
|
||||
120,
|
||||
id="pyav-num_frames_wins_fps",
|
||||
),
|
||||
# dynamic sampling + pyav codec
|
||||
pytest.param(
|
||||
"opencv_dynamic",
|
||||
{"fps": 1, "max_duration": 60, "backend": "pyav"},
|
||||
60,
|
||||
id="pyav_dynamic-within_max_duration",
|
||||
),
|
||||
pytest.param(
|
||||
"opencv_dynamic",
|
||||
{"fps": 2, "max_duration": 30, "backend": "pyav"},
|
||||
60,
|
||||
id="pyav_dynamic-exceeds_max_duration",
|
||||
),
|
||||
# glm46v dynamic FPS (1800 frames @ 30fps = 60s)
|
||||
# 60s falls in (30, 300] → target_fps=1.0, extract_t = 60*1.0*2 = 120
|
||||
pytest.param(
|
||||
@@ -1419,12 +1300,6 @@ def test_torchcodec_backend_returns_target_frames_not_keyframes():
|
||||
120,
|
||||
id="glm46v-60s",
|
||||
),
|
||||
pytest.param(
|
||||
"glm46v",
|
||||
{"backend": "pyav"},
|
||||
120,
|
||||
id="glm46v-pyav-60s",
|
||||
),
|
||||
# uniform sampling + torchcodec codec (same frame counts as opencv)
|
||||
pytest.param(
|
||||
"opencv",
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Literal, cast
|
||||
from typing import Any, cast
|
||||
|
||||
import numpy.typing as npt
|
||||
import torch
|
||||
@@ -35,6 +35,7 @@ from vllm.multimodal.processing import (
|
||||
from vllm.multimodal.video import (
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
VideoBackend,
|
||||
VideoDecoderBackend,
|
||||
VideoSourceMetadata,
|
||||
VideoTargetMetadata,
|
||||
)
|
||||
@@ -485,13 +486,7 @@ class MiniMaxM3VideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal[
|
||||
"opencv",
|
||||
"pyav",
|
||||
"torchcodec",
|
||||
"pynvvideocodec",
|
||||
"deepstream",
|
||||
] = "opencv",
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
|
||||
+12
-25
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
from abc import abstractmethod
|
||||
from typing import Any, ClassVar, Literal, cast
|
||||
from typing import Any, ClassVar, cast
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
@@ -177,8 +177,8 @@ class VideoBackend(VideoLoader):
|
||||
|
||||
Samples ``num_frames`` uniformly across the video (or one frame every
|
||||
``1/fps`` seconds, whichever produces fewer frames). The decoding codec
|
||||
is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``,
|
||||
``"torchcodec"``, ``"pynvvideocodec"``, or ``"deepstream"``),
|
||||
is selected via the ``backend`` kwarg (``"opencv"``, ``"torchcodec"``,
|
||||
``"pynvvideocodec"``, or ``"deepstream"``),
|
||||
which can be passed through ``--media-io-kwargs``. Defaults to ``"opencv"``.
|
||||
"""
|
||||
|
||||
@@ -232,16 +232,15 @@ class VideoBackend(VideoLoader):
|
||||
dynamic subclass; ignored here.
|
||||
frame_recovery: Enable forward-scan recovery for failed frames.
|
||||
Only honored by the OpenCV codec.
|
||||
backend: Decoding codec — ``"opencv"``, ``"pyav"``,
|
||||
``"torchcodec"``, ``"pynvvideocodec"`` or ``"deepstream"``.
|
||||
backend: Decoding codec — ``"opencv"``, ``"torchcodec"``,
|
||||
``"pynvvideocodec"`` or ``"deepstream"``.
|
||||
kwargs: Codec-specific options, validated against and forwarded to
|
||||
``backend``:
|
||||
|
||||
- ``num_ffmpeg_threads`` (TorchCodec): number of FFmpeg
|
||||
decoding threads; ``0`` (default) relies on the FFmpeg
|
||||
default value which is ``min(cpu_count + 1, 16)``.
|
||||
OpenCV will always use ``min(cpu_count, 16)`` while pyav
|
||||
will always use ``min(cpu_count, (height + 15) / 16)``.
|
||||
OpenCV will always use ``min(cpu_count, 16)``.
|
||||
- ``seek_mode`` (TorchCodec): ``"exact"`` (default) guarantees
|
||||
frame-accurate sampling by scanning the file on creation,
|
||||
while ``"approximate"`` skips that scan for faster decoder
|
||||
@@ -355,9 +354,7 @@ class Qwen3VLVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal[
|
||||
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
|
||||
] = "opencv",
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
@@ -436,9 +433,7 @@ class Qwen2VLVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal[
|
||||
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
|
||||
] = "opencv",
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
@@ -530,9 +525,7 @@ class DynamicVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal[
|
||||
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
|
||||
] = "opencv",
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
@@ -657,9 +650,7 @@ class GLM46VVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal[
|
||||
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
|
||||
] = "opencv",
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
@@ -757,9 +748,7 @@ class GLMGAVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal[
|
||||
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
|
||||
] = "opencv",
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
frames, metadata = super().load_bytes(
|
||||
@@ -1073,9 +1062,7 @@ class NemotronVLVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal[
|
||||
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
|
||||
] = "opencv",
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
frames, metadata = super().load_bytes(
|
||||
|
||||
@@ -12,13 +12,10 @@ from .base import (
|
||||
check_frame_pixel_limit,
|
||||
)
|
||||
|
||||
VideoDecoderBackend = Literal[
|
||||
"opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"
|
||||
]
|
||||
VideoDecoderBackend = Literal["opencv", "torchcodec", "pynvvideocodec", "deepstream"]
|
||||
|
||||
_BACKEND_OPTION_DEFAULTS: dict[str, dict[str, Any]] = {
|
||||
"opencv": {},
|
||||
"pyav": {},
|
||||
"torchcodec": {
|
||||
"num_ffmpeg_threads": 0,
|
||||
"seek_mode": "exact",
|
||||
|
||||
@@ -52,7 +52,7 @@ class DeepStreamVideoBackendMixin:
|
||||
local file path is required — HTTP and base64 sources decode identically
|
||||
to local files.
|
||||
|
||||
Like the OpenCV/PyAV mixins, this provides only the codec layer.
|
||||
Like the OpenCV mixin, this provides only the codec layer.
|
||||
Frame *selection* lives in the loader's
|
||||
``compute_frames_index_to_sample`` and arrives here as an explicit
|
||||
list of frame indices.
|
||||
@@ -132,7 +132,7 @@ class DeepStreamVideoBackendMixin:
|
||||
|
||||
valid = frame_indices[: result.n_kept]
|
||||
# GPU -> CPU NHWC uint8 at the codec boundary (one PCIe copy); keeps
|
||||
# the array shape identical to the OpenCV/PyAV backends. Copy into
|
||||
# the array shape identical to the OpenCV backend. Copy into
|
||||
# PINNED host memory (reused across calls by PyTorch's pinned caching
|
||||
# allocator) so the D2H runs at full PCIe bandwidth (~13 GB/s) rather
|
||||
# than the ~1 GB/s pageable path that plain ``.cpu()`` takes — ~12x
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
|
||||
from .base import (
|
||||
VideoSourceMetadata,
|
||||
VideoTargetMetadata,
|
||||
check_frame_pixel_limit,
|
||||
)
|
||||
|
||||
try:
|
||||
import av
|
||||
except ImportError:
|
||||
av = PlaceholderModule("av") # type: ignore[assignment]
|
||||
|
||||
|
||||
def decode_pyav(
|
||||
loader_cls,
|
||||
data: bytes,
|
||||
target: VideoTargetMetadata,
|
||||
sampling_kwargs: dict,
|
||||
) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]:
|
||||
with av.open(BytesIO(data)) as container:
|
||||
stream = container.streams.video[0]
|
||||
check_frame_pixel_limit(stream.width, stream.height)
|
||||
source = loader_cls._prepare_source(
|
||||
PyAVVideoBackendMixin.get_metadata(container)
|
||||
)
|
||||
frame_idx = loader_cls.compute_frames_index_to_sample(
|
||||
source=source, target=target, **sampling_kwargs
|
||||
)
|
||||
frames, valid = PyAVVideoBackendMixin.decode_frames(
|
||||
container, frame_idx, source.original_fps, source.duration
|
||||
)
|
||||
return frames, source, frame_idx, valid
|
||||
|
||||
|
||||
class PyAVVideoBackendMixin:
|
||||
"""PyAV (in-process FFmpeg bindings) codec utilities.
|
||||
|
||||
Reads stream metadata and decodes target frames via per-frame
|
||||
``container.seek()``. The seek releases the GIL between frames and
|
||||
scales with the number of sampled frames rather than the video
|
||||
length, enabling concurrent decoding under serving load.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_metadata(
|
||||
container: "av.container.InputContainer",
|
||||
) -> VideoSourceMetadata:
|
||||
if not container.streams.video:
|
||||
raise ValueError("No video streams found in container")
|
||||
stream = container.streams.video[0]
|
||||
total_frames = stream.frames or 0
|
||||
fps = float(stream.average_rate) if stream.average_rate else 0.0
|
||||
duration = float(stream.duration * stream.time_base) if stream.duration else 0.0
|
||||
if total_frames == 0 and duration > 0 and fps > 0:
|
||||
total_frames = int(duration * fps)
|
||||
elif duration > 0 and fps > 0 and total_frames > round(duration * fps) + 1:
|
||||
# The header sample count can exceed the frames the presentation
|
||||
# timeline actually holds — e.g. an mp4 edit list hides the decode
|
||||
# lead-in of a stream-copied cut, but ``stream.frames`` still
|
||||
# counts the hidden samples. Sampling indices past the visible
|
||||
# range would silently collapse onto the last visible frame, so
|
||||
# trust the (edit-list-aware) duration instead.
|
||||
total_frames = round(duration * fps)
|
||||
return VideoSourceMetadata(total_frames, fps, duration)
|
||||
|
||||
@staticmethod
|
||||
def decode_frames(
|
||||
container: "av.container.InputContainer",
|
||||
frame_indices: list[int],
|
||||
fps: float,
|
||||
duration: float,
|
||||
) -> tuple[npt.NDArray, list[int]]:
|
||||
"""Decode target frames via per-frame seek + forward decode to PTS."""
|
||||
stream = container.streams.video[0]
|
||||
# SLICE parallelizes within a single frame without the
|
||||
# one-frame-per-thread latency penalty of FRAME threading.
|
||||
stream.thread_type = "SLICE"
|
||||
time_base = stream.time_base
|
||||
|
||||
frames_list: list[npt.NDArray] = []
|
||||
valid_indices: list[int] = []
|
||||
frame_interval = 1.0 / fps if fps > 0 else 0.1
|
||||
max_ts = max(0.0, duration - frame_interval) if duration > 0 else float("inf")
|
||||
|
||||
decoder = None
|
||||
last_pts = None
|
||||
for idx in frame_indices:
|
||||
ts = min(idx / fps, max_ts) if fps > 0 else 0.0
|
||||
pts = int(ts / time_base)
|
||||
# seek() snaps backward to a keyframe; reuse the running decoder
|
||||
# while targets advance monotonically to avoid re-decoding the
|
||||
# GOP prefix once per requested frame.
|
||||
if decoder is None or last_pts is None or pts <= last_pts:
|
||||
container.seek(pts, stream=stream)
|
||||
decoder = container.decode(video=0)
|
||||
chosen = None
|
||||
for frame in decoder:
|
||||
if frame.pts is not None and frame.pts >= pts:
|
||||
chosen = frame
|
||||
last_pts = frame.pts
|
||||
break
|
||||
if chosen is not None:
|
||||
frames_list.append(chosen.to_ndarray(format="rgb24"))
|
||||
valid_indices.append(idx)
|
||||
else:
|
||||
decoder = None
|
||||
|
||||
if not frames_list:
|
||||
return np.empty((0,), dtype=np.uint8), valid_indices
|
||||
return np.stack(frames_list), valid_indices
|
||||
Reference in New Issue
Block a user