diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index 1c7730a37e..16334fdb0d 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -49,7 +49,9 @@ from vllm.transformers_utils.processor import get_video_processor_cls_name_from_ from .utils import ( create_edit_list_trimmed_video, + create_live_webm_video, create_long_gop_video, + create_raw_h264_video, create_video_from_image, ) @@ -1261,8 +1263,8 @@ def test_qwen3_vl_unknown_source_fps_matches_hf( the metadata carries the fps HF assumes, which the frame timestamps use.""" get_video_metadata = OpenCVVideoBackendMixin.get_video_metadata - def unknown_fps(cap) -> VideoSourceMetadata: - return get_video_metadata(cap)._replace(original_fps=0.0, duration=0.0) + def unknown_fps(data) -> VideoSourceMetadata: + return get_video_metadata(data)._replace(original_fps=0.0, duration=0.0) monkeypatch.setattr( OpenCVVideoBackendMixin, "get_video_metadata", staticmethod(unknown_fps) @@ -1476,6 +1478,36 @@ def test_video_backend_handles_edit_list_trimmed_video( assert qwen_frames.shape[0] >= 4 +def test_opencv_demux_counts_frames_when_video_header_lacks_metadata( + monkeypatch: pytest.MonkeyPatch, +): + """Streamed WebM and raw H.264 carry no frame count/duration in their + headers, so OpenCV reports a bogus negative ``CAP_PROP_FRAME_COUNT``. + The loader must count the frames via a demux-only pass, without + decoding. + """ + with monkeypatch.context() as m: + m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv") + loader = VIDEO_LOADER_REGISTRY.load("opencv") + + webm_data = create_live_webm_video(num_frames=20, fps=10) + frames, metadata = loader.load_bytes(webm_data, backend="opencv") + assert frames.shape[0] == 20 + assert metadata["total_num_frames"] == 20 + assert metadata["duration"] == pytest.approx(2.0) + + h264_data = create_raw_h264_video(num_frames=20, fps=25) + frames, metadata = loader.load_bytes(h264_data, backend="opencv") + assert frames.shape[0] == 20 + assert metadata["total_num_frames"] == 20 + assert metadata["duration"] == pytest.approx(0.8) + + # The Qwen samplers used to raise on the bogus negative frame count. + qwen_frames, qwen_metadata = Qwen2VLVideoBackend.load_bytes(webm_data) + assert qwen_metadata["total_num_frames"] == 20 + assert qwen_frames.shape[0] >= 4 + + # ============================================================================ # Frame Recovery Tests # ============================================================================ diff --git a/tests/multimodal/utils.py b/tests/multimodal/utils.py index 23a05a0944..d74f684531 100644 --- a/tests/multimodal/utils.py +++ b/tests/multimodal/utils.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import io + import cv2 import numpy as np import numpy.typing as npt @@ -77,8 +79,6 @@ def create_long_gop_video( The marker lets a test recover which frame the decoder actually returned, independent of any metadata label. """ - import io - import av buf = io.BytesIO() @@ -103,6 +103,63 @@ def create_long_gop_video( return buf.getvalue() +def create_live_webm_video( + num_frames: int = 20, + fps: int = 10, + width: int = 64, + height: int = 64, +) -> bytes: + """Encode a WebM stream without container duration or frame-count metadata. + + The ``live`` muxing option mimics writing to a pipe: the Segment size is + unknown and no Duration or Cues elements are written, so decoder backends + cannot learn the frame count from the header. + """ + import av + + buf = io.BytesIO() + with av.open(buf, mode="w", format="webm", options={"live": "1"}) as container: + stream = container.add_stream("libvpx", rate=fps) + stream.width = width + stream.height = height + stream.pix_fmt = "yuv420p" + for i in range(num_frames): + img = np.zeros((height, width, 3), dtype=np.uint8) + img[:, :, 1] = i % 256 + frame = av.VideoFrame.from_ndarray(img, format="rgb24") + for packet in stream.encode(frame): + container.mux(packet) + for packet in stream.encode(): + container.mux(packet) + return buf.getvalue() + + +def create_raw_h264_video( + num_frames: int = 20, + fps: int = 25, + width: int = 64, + height: int = 64, +) -> bytes: + """Encode a raw H.264 Annex B elementary stream (no container metadata).""" + import av + + buf = io.BytesIO() + with av.open(buf, mode="w", format="h264") as container: + stream = container.add_stream("h264", rate=fps) + stream.width = width + stream.height = height + stream.pix_fmt = "yuv420p" + for i in range(num_frames): + img = np.zeros((height, width, 3), dtype=np.uint8) + img[:, :, 1] = i % 256 + frame = av.VideoFrame.from_ndarray(img, format="rgb24") + for packet in stream.encode(frame): + container.mux(packet) + for packet in stream.encode(): + container.mux(packet) + return buf.getvalue() + + def create_edit_list_trimmed_video( num_frames: int = 90, trim_start_frame: int = 60, @@ -117,8 +174,6 @@ def create_edit_list_trimmed_video( trailing frames are presentable — the shape produced by lossless trims (e.g. ``ffmpeg -ss ... -c copy``). Returns (video_bytes, visible_frames). """ - import io - import av src = create_long_gop_video(num_frames=num_frames, fps=fps) diff --git a/vllm/multimodal/video_decoders/opencv.py b/vllm/multimodal/video_decoders/opencv.py index 3ca141c738..88d7fb8375 100644 --- a/vllm/multimodal/video_decoders/opencv.py +++ b/vllm/multimodal/video_decoders/opencv.py @@ -38,7 +38,9 @@ def decode_opencv( int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)), ) - source = loader_cls._prepare_source(OpenCVVideoBackendMixin.get_video_metadata(cap)) + source = loader_cls._prepare_source( + OpenCVVideoBackendMixin.get_video_metadata(data) + ) frame_idx = loader_cls.compute_frames_index_to_sample( source=source, target=target, **sampling_kwargs ) @@ -74,10 +76,34 @@ class OpenCVVideoBackendMixin: raise ValueError("Could not open video stream") return cap + @classmethod + def demux_count_frames(cls, data: bytes) -> int: + """Count frames with OpenCV's demuxer-only mode.""" + # NOTE: buf must stay referenced until after release(). + # OpenCV's Python stream binding does not safely own the stream object + buf = BytesIO(data) + demux_cap = cv2.VideoCapture( + buf, cls.get_cv2_video_api(), [cv2.CAP_PROP_FORMAT, -1] + ) + count = 0 + while demux_cap.grab(): + count += 1 + demux_cap.release() + if count == 0: + raise ValueError("Could not determine the number of video frames") + return count + @staticmethod - def get_video_metadata(cap: "cv2.VideoCapture") -> VideoSourceMetadata: + def get_video_metadata(data: bytes) -> VideoSourceMetadata: + cap = OpenCVVideoBackendMixin.open_video_capture(data) total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) original_fps = cap.get(cv2.CAP_PROP_FPS) + if total_frames_num <= 0: + # Streamed WebM/MKV and raw H.264 miss frame count in headers; + # We need to use demux frames count for retrieval + total_frames_num = OpenCVVideoBackendMixin.demux_count_frames(data) + duration = total_frames_num / original_fps if original_fps > 0 else 0 + return VideoSourceMetadata(total_frames_num, original_fps, duration) # CAP_PROP_FRAME_COUNT counts every physical sample in the container, # overstating the presentable frames when an mp4 edit list hides the # decode lead-in (e.g. a lossless `ffmpeg -ss ... -c copy` trim). @@ -87,7 +113,7 @@ class OpenCVVideoBackendMixin: # may land mid-stream on corrupt files, so only trust it when the # stream really ends there. A one-frame tolerance keeps mere header # miscounts from being "corrected". - if total_frames_num > 0 and cap.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0): + if cap.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0): visible_frames = int(cap.get(cv2.CAP_PROP_POS_FRAMES)) at_stream_end = not cap.grab() cap.set(cv2.CAP_PROP_POS_FRAMES, 0)