[Bugfix] Fix Opencv video backend frames count missing for WebM video inputs (#61759)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -49,7 +49,9 @@ from vllm.transformers_utils.processor import get_video_processor_cls_name_from_
|
||||
|
||||
from .utils import (
|
||||
create_edit_list_trimmed_video,
|
||||
create_live_webm_video,
|
||||
create_long_gop_video,
|
||||
create_raw_h264_video,
|
||||
create_video_from_image,
|
||||
)
|
||||
|
||||
@@ -1261,8 +1263,8 @@ def test_qwen3_vl_unknown_source_fps_matches_hf(
|
||||
the metadata carries the fps HF assumes, which the frame timestamps use."""
|
||||
get_video_metadata = OpenCVVideoBackendMixin.get_video_metadata
|
||||
|
||||
def unknown_fps(cap) -> VideoSourceMetadata:
|
||||
return get_video_metadata(cap)._replace(original_fps=0.0, duration=0.0)
|
||||
def unknown_fps(data) -> VideoSourceMetadata:
|
||||
return get_video_metadata(data)._replace(original_fps=0.0, duration=0.0)
|
||||
|
||||
monkeypatch.setattr(
|
||||
OpenCVVideoBackendMixin, "get_video_metadata", staticmethod(unknown_fps)
|
||||
@@ -1476,6 +1478,36 @@ def test_video_backend_handles_edit_list_trimmed_video(
|
||||
assert qwen_frames.shape[0] >= 4
|
||||
|
||||
|
||||
def test_opencv_demux_counts_frames_when_video_header_lacks_metadata(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Streamed WebM and raw H.264 carry no frame count/duration in their
|
||||
headers, so OpenCV reports a bogus negative ``CAP_PROP_FRAME_COUNT``.
|
||||
The loader must count the frames via a demux-only pass, without
|
||||
decoding.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
|
||||
loader = VIDEO_LOADER_REGISTRY.load("opencv")
|
||||
|
||||
webm_data = create_live_webm_video(num_frames=20, fps=10)
|
||||
frames, metadata = loader.load_bytes(webm_data, backend="opencv")
|
||||
assert frames.shape[0] == 20
|
||||
assert metadata["total_num_frames"] == 20
|
||||
assert metadata["duration"] == pytest.approx(2.0)
|
||||
|
||||
h264_data = create_raw_h264_video(num_frames=20, fps=25)
|
||||
frames, metadata = loader.load_bytes(h264_data, backend="opencv")
|
||||
assert frames.shape[0] == 20
|
||||
assert metadata["total_num_frames"] == 20
|
||||
assert metadata["duration"] == pytest.approx(0.8)
|
||||
|
||||
# The Qwen samplers used to raise on the bogus negative frame count.
|
||||
qwen_frames, qwen_metadata = Qwen2VLVideoBackend.load_bytes(webm_data)
|
||||
assert qwen_metadata["total_num_frames"] == 20
|
||||
assert qwen_frames.shape[0] >= 4
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Frame Recovery Tests
|
||||
# ============================================================================
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
@@ -77,8 +79,6 @@ def create_long_gop_video(
|
||||
The marker lets a test recover which frame the decoder actually returned,
|
||||
independent of any metadata label.
|
||||
"""
|
||||
import io
|
||||
|
||||
import av
|
||||
|
||||
buf = io.BytesIO()
|
||||
@@ -103,6 +103,63 @@ def create_long_gop_video(
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def create_live_webm_video(
|
||||
num_frames: int = 20,
|
||||
fps: int = 10,
|
||||
width: int = 64,
|
||||
height: int = 64,
|
||||
) -> bytes:
|
||||
"""Encode a WebM stream without container duration or frame-count metadata.
|
||||
|
||||
The ``live`` muxing option mimics writing to a pipe: the Segment size is
|
||||
unknown and no Duration or Cues elements are written, so decoder backends
|
||||
cannot learn the frame count from the header.
|
||||
"""
|
||||
import av
|
||||
|
||||
buf = io.BytesIO()
|
||||
with av.open(buf, mode="w", format="webm", options={"live": "1"}) as container:
|
||||
stream = container.add_stream("libvpx", rate=fps)
|
||||
stream.width = width
|
||||
stream.height = height
|
||||
stream.pix_fmt = "yuv420p"
|
||||
for i in range(num_frames):
|
||||
img = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
img[:, :, 1] = i % 256
|
||||
frame = av.VideoFrame.from_ndarray(img, format="rgb24")
|
||||
for packet in stream.encode(frame):
|
||||
container.mux(packet)
|
||||
for packet in stream.encode():
|
||||
container.mux(packet)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def create_raw_h264_video(
|
||||
num_frames: int = 20,
|
||||
fps: int = 25,
|
||||
width: int = 64,
|
||||
height: int = 64,
|
||||
) -> bytes:
|
||||
"""Encode a raw H.264 Annex B elementary stream (no container metadata)."""
|
||||
import av
|
||||
|
||||
buf = io.BytesIO()
|
||||
with av.open(buf, mode="w", format="h264") as container:
|
||||
stream = container.add_stream("h264", rate=fps)
|
||||
stream.width = width
|
||||
stream.height = height
|
||||
stream.pix_fmt = "yuv420p"
|
||||
for i in range(num_frames):
|
||||
img = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
img[:, :, 1] = i % 256
|
||||
frame = av.VideoFrame.from_ndarray(img, format="rgb24")
|
||||
for packet in stream.encode(frame):
|
||||
container.mux(packet)
|
||||
for packet in stream.encode():
|
||||
container.mux(packet)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def create_edit_list_trimmed_video(
|
||||
num_frames: int = 90,
|
||||
trim_start_frame: int = 60,
|
||||
@@ -117,8 +174,6 @@ def create_edit_list_trimmed_video(
|
||||
trailing frames are presentable — the shape produced by lossless trims
|
||||
(e.g. ``ffmpeg -ss ... -c copy``). Returns (video_bytes, visible_frames).
|
||||
"""
|
||||
import io
|
||||
|
||||
import av
|
||||
|
||||
src = create_long_gop_video(num_frames=num_frames, fps=fps)
|
||||
|
||||
Reference in New Issue
Block a user