[Multimodal] Deprecate PyAV video decoder backend (#54231)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-08-29 15:13:28 +00:00
committed by GitHub
parent fd5d3aea94
commit 4fc943b867
9 changed files with 35 additions and 301 deletions
@@ -146,7 +146,7 @@ def test_processor_override(
@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
@pytest.mark.parametrize("fps", [2])
@pytest.mark.parametrize("backend", ["opencv", "pyav"])
@pytest.mark.parametrize("backend", ["opencv"])
def test_video_loader_consistency(
model_id: str,
fps: int,
+2 -2
View File
@@ -454,7 +454,7 @@ class TestMergeKwargsGpuBackendPolicy:
)
assert result["hw_decoders"] == 2
@pytest.mark.parametrize("backend", ["opencv", "pyav", "torchcodec"])
@pytest.mark.parametrize("backend", ["opencv", "torchcodec"])
def test_software_video_backend_passes_through(self, backend: str):
result = VideoMediaIO.merge_kwargs(
default_kwargs=None,
@@ -462,7 +462,7 @@ class TestMergeKwargsGpuBackendPolicy:
)
assert result["video_backend"] == backend
@pytest.mark.parametrize("backend", ["opencv", "pyav"])
@pytest.mark.parametrize("backend", ["opencv"])
def test_software_codec_backend_passes_through(self, backend: str):
result = VideoMediaIO.merge_kwargs(
default_kwargs=None,
+10 -135
View File
@@ -113,7 +113,7 @@ import vllm.multimodal.video # noqa: F401
backend_modules = {
f"vllm.multimodal.video_decoders.{backend}"
for backend in ("opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream")
for backend in ("opencv", "torchcodec", "pynvvideocodec", "deepstream")
}
loaded = sorted(backend_modules & sys.modules.keys())
assert not loaded, loaded
@@ -128,7 +128,7 @@ assert not loaded, loaded
@pytest.mark.parametrize(
"backend",
["opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"],
["opencv", "torchcodec", "pynvvideocodec", "deepstream"],
)
def test_decode_video_imports_only_selected_backend(
backend: str,
@@ -199,15 +199,15 @@ def test_video_backend_kwargs_are_separated_from_sampling_kwargs(
def test_video_backend_rejects_options_for_another_decoder():
with pytest.raises(
ValueError, match="num_ffmpeg_threads is not supported by the 'pyav' backend"
ValueError, match="num_ffmpeg_threads is not supported by the 'opencv' backend"
):
resolve_video_backend_kwargs("pyav", {"num_ffmpeg_threads": 2})
resolve_video_backend_kwargs("opencv", {"num_ffmpeg_threads": 2})
@pytest.mark.parametrize(
("backend", "error"),
[
("pyav", AssertionError),
("torchcodec", AssertionError),
(PYNVVIDEOCODEC_VIDEO_BACKEND, ValueError),
],
)
@@ -885,7 +885,7 @@ def test_video_backend_handles_edit_list_trimmed_video(
)
loader = VIDEO_LOADER_REGISTRY.load("opencv")
for backend in ["opencv", "pyav"]:
for backend in ["opencv"]:
frames, metadata = loader.load_bytes(
video_data, num_frames=-1, backend=backend
)
@@ -1136,97 +1136,6 @@ def dummy_video_path(tmp_path):
return video_path
# ============================================================================
# PyAV Backend Tests
# ============================================================================
def test_pyav_backend_loads_frames(dummy_video_path, monkeypatch: pytest.MonkeyPatch):
"""Test that the pyav codec backend can load frames from a valid video."""
with monkeypatch.context() as m:
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
with open(dummy_video_path, "rb") as f:
video_data = f.read()
loader = VIDEO_LOADER_REGISTRY.load("opencv")
frames, metadata = loader.load_bytes(video_data, num_frames=8, backend="pyav")
assert frames.ndim == 4
assert frames.shape[3] == 3 # RGB
assert frames.shape[0] == 8
assert frames.shape[0] == len(metadata["frames_indices"])
assert metadata["video_backend"] == "pyav"
assert "total_num_frames" in metadata
assert "fps" in metadata
assert "duration" in metadata
def test_pyav_dynamic_backend_loads_frames(
dummy_video_path, monkeypatch: pytest.MonkeyPatch
):
"""Test that the pyav codec with dynamic sampling can load frames."""
with monkeypatch.context() as m:
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv_dynamic")
with open(dummy_video_path, "rb") as f:
video_data = f.read()
loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic")
frames, metadata = loader.load_bytes(
video_data, fps=2, max_duration=10, backend="pyav"
)
assert frames.ndim == 4
assert frames.shape[3] == 3 # RGB
assert frames.shape[0] > 0
assert frames.shape[0] == len(metadata["frames_indices"])
assert metadata["video_backend"] == "pyav_dynamic"
def test_pyav_backend_returns_target_frames_not_keyframes():
"""Regression test: PyAV must decode forward past the seek keyframe.
container.seek() snaps backward to the nearest keyframe. With a long GOP
(here: one keyframe at frame 0), a decoder that does not advance forward
to the target PTS collapses every sampled slot onto the keyframe. This
test encodes a per-frame marker on the green channel and verifies the
returned frames are distinct, ordered, and match the requested indices.
"""
num_frames = 50
num_sampled = 4
height, width = 64, 64
video_bytes = create_long_gop_video(
num_frames=num_frames, width=width, height=height
)
loader = VIDEO_LOADER_REGISTRY.load("opencv")
frames, metadata = loader.load_bytes(
video_bytes, num_frames=num_sampled, backend="pyav"
)
assert frames.shape == (num_sampled, height, width, 3)
requested = list(metadata["frames_indices"])
assert len(requested) == num_sampled
actual = [int(f[height // 2, width // 2, 1]) for f in frames]
assert len(set(actual)) == num_sampled, (
f"PyAV returned only {len(set(actual))} distinct frames for "
f"{num_sampled} requested indices: markers={actual}, "
f"requested={requested}. Keyframe-snap regression."
)
assert actual == sorted(actual), f"Returned frames out of order: markers={actual}"
for marker, want_idx in zip(actual, requested):
assert abs(marker - want_idx) <= 10, (
f"Frame mismatch: requested index {want_idx}, "
f"got marker {marker} (tolerance ±10)"
)
# ============================================================================
# TorchCodec Backend Tests
# ============================================================================
@@ -1298,11 +1207,10 @@ def test_torchcodec_backend_returns_target_frames_not_keyframes():
"""Regression test: torchcodec must return the requested frames, not the
GOP keyframe they seek back to.
Mirrors ``test_pyav_backend_returns_target_frames_not_keyframes``: a long
GOP (single keyframe at frame 0) with a per-frame green-channel marker.
With ``seek_mode="exact"`` torchcodec resolves each index to the exact
frame, so the returned markers must be distinct, ordered, and match the
requested indices.
A long GOP (single keyframe at frame 0) with a per-frame green-channel
marker. With ``seek_mode="exact"`` torchcodec resolves each index to the
exact frame, so the returned markers must be distinct, ordered, and match
the requested indices.
"""
pytest.importorskip("torchcodec")
num_frames = 50
@@ -1384,33 +1292,6 @@ def test_torchcodec_backend_returns_target_frames_not_keyframes():
119,
id="molmo2-fps",
),
# uniform sampling + pyav codec (same frame counts as opencv)
pytest.param(
"opencv",
{"num_frames": 32, "backend": "pyav"},
32,
id="pyav-num_frames",
),
pytest.param("opencv", {"fps": 2, "backend": "pyav"}, 120, id="pyav-fps"),
pytest.param(
"opencv",
{"num_frames": 500, "fps": 2, "backend": "pyav"},
120,
id="pyav-num_frames_wins_fps",
),
# dynamic sampling + pyav codec
pytest.param(
"opencv_dynamic",
{"fps": 1, "max_duration": 60, "backend": "pyav"},
60,
id="pyav_dynamic-within_max_duration",
),
pytest.param(
"opencv_dynamic",
{"fps": 2, "max_duration": 30, "backend": "pyav"},
60,
id="pyav_dynamic-exceeds_max_duration",
),
# glm46v dynamic FPS (1800 frames @ 30fps = 60s)
# 60s falls in (30, 300] → target_fps=1.0, extract_t = 60*1.0*2 = 120
pytest.param(
@@ -1419,12 +1300,6 @@ def test_torchcodec_backend_returns_target_frames_not_keyframes():
120,
id="glm46v-60s",
),
pytest.param(
"glm46v",
{"backend": "pyav"},
120,
id="glm46v-pyav-60s",
),
# uniform sampling + torchcodec codec (same frame counts as opencv)
pytest.param(
"opencv",