[Bugfix] Fix Opencv video backend frames count missing for WebM video inputs (#61759)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-10-18 10:37:39 +00:00
committed by GitHub
parent 2365177ddd
commit 3f1b04e045
3 changed files with 122 additions and 9 deletions
+34 -2
View File
@@ -49,7 +49,9 @@ from vllm.transformers_utils.processor import get_video_processor_cls_name_from_
from .utils import (
create_edit_list_trimmed_video,
create_live_webm_video,
create_long_gop_video,
create_raw_h264_video,
create_video_from_image,
)
@@ -1261,8 +1263,8 @@ def test_qwen3_vl_unknown_source_fps_matches_hf(
the metadata carries the fps HF assumes, which the frame timestamps use."""
get_video_metadata = OpenCVVideoBackendMixin.get_video_metadata
def unknown_fps(cap) -> VideoSourceMetadata:
return get_video_metadata(cap)._replace(original_fps=0.0, duration=0.0)
def unknown_fps(data) -> VideoSourceMetadata:
return get_video_metadata(data)._replace(original_fps=0.0, duration=0.0)
monkeypatch.setattr(
OpenCVVideoBackendMixin, "get_video_metadata", staticmethod(unknown_fps)
@@ -1476,6 +1478,36 @@ def test_video_backend_handles_edit_list_trimmed_video(
assert qwen_frames.shape[0] >= 4
def test_opencv_demux_counts_frames_when_video_header_lacks_metadata(
monkeypatch: pytest.MonkeyPatch,
):
"""Streamed WebM and raw H.264 carry no frame count/duration in their
headers, so OpenCV reports a bogus negative ``CAP_PROP_FRAME_COUNT``.
The loader must count the frames via a demux-only pass, without
decoding.
"""
with monkeypatch.context() as m:
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
loader = VIDEO_LOADER_REGISTRY.load("opencv")
webm_data = create_live_webm_video(num_frames=20, fps=10)
frames, metadata = loader.load_bytes(webm_data, backend="opencv")
assert frames.shape[0] == 20
assert metadata["total_num_frames"] == 20
assert metadata["duration"] == pytest.approx(2.0)
h264_data = create_raw_h264_video(num_frames=20, fps=25)
frames, metadata = loader.load_bytes(h264_data, backend="opencv")
assert frames.shape[0] == 20
assert metadata["total_num_frames"] == 20
assert metadata["duration"] == pytest.approx(0.8)
# The Qwen samplers used to raise on the bogus negative frame count.
qwen_frames, qwen_metadata = Qwen2VLVideoBackend.load_bytes(webm_data)
assert qwen_metadata["total_num_frames"] == 20
assert qwen_frames.shape[0] >= 4
# ============================================================================
# Frame Recovery Tests
# ============================================================================
+59 -4
View File
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import cv2
import numpy as np
import numpy.typing as npt
@@ -77,8 +79,6 @@ def create_long_gop_video(
The marker lets a test recover which frame the decoder actually returned,
independent of any metadata label.
"""
import io
import av
buf = io.BytesIO()
@@ -103,6 +103,63 @@ def create_long_gop_video(
return buf.getvalue()
def create_live_webm_video(
num_frames: int = 20,
fps: int = 10,
width: int = 64,
height: int = 64,
) -> bytes:
"""Encode a WebM stream without container duration or frame-count metadata.
The ``live`` muxing option mimics writing to a pipe: the Segment size is
unknown and no Duration or Cues elements are written, so decoder backends
cannot learn the frame count from the header.
"""
import av
buf = io.BytesIO()
with av.open(buf, mode="w", format="webm", options={"live": "1"}) as container:
stream = container.add_stream("libvpx", rate=fps)
stream.width = width
stream.height = height
stream.pix_fmt = "yuv420p"
for i in range(num_frames):
img = np.zeros((height, width, 3), dtype=np.uint8)
img[:, :, 1] = i % 256
frame = av.VideoFrame.from_ndarray(img, format="rgb24")
for packet in stream.encode(frame):
container.mux(packet)
for packet in stream.encode():
container.mux(packet)
return buf.getvalue()
def create_raw_h264_video(
num_frames: int = 20,
fps: int = 25,
width: int = 64,
height: int = 64,
) -> bytes:
"""Encode a raw H.264 Annex B elementary stream (no container metadata)."""
import av
buf = io.BytesIO()
with av.open(buf, mode="w", format="h264") as container:
stream = container.add_stream("h264", rate=fps)
stream.width = width
stream.height = height
stream.pix_fmt = "yuv420p"
for i in range(num_frames):
img = np.zeros((height, width, 3), dtype=np.uint8)
img[:, :, 1] = i % 256
frame = av.VideoFrame.from_ndarray(img, format="rgb24")
for packet in stream.encode(frame):
container.mux(packet)
for packet in stream.encode():
container.mux(packet)
return buf.getvalue()
def create_edit_list_trimmed_video(
num_frames: int = 90,
trim_start_frame: int = 60,
@@ -117,8 +174,6 @@ def create_edit_list_trimmed_video(
trailing frames are presentable — the shape produced by lossless trims
(e.g. ``ffmpeg -ss ... -c copy``). Returns (video_bytes, visible_frames).
"""
import io
import av
src = create_long_gop_video(num_frames=num_frames, fps=fps)