[Bugfix] Fix Opencv video backend frames count missing for WebM video inputs (#61759)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>

tests/multimodal/test_video.py

@@ -49,7 +49,9 @@ from vllm.transformers_utils.processor import get_video_processor_cls_name_from_
 
 from .utils import (
     create_edit_list_trimmed_video,
+    create_live_webm_video,
     create_long_gop_video,
+    create_raw_h264_video,
     create_video_from_image,
 )
 
@@ -1261,8 +1263,8 @@ def test_qwen3_vl_unknown_source_fps_matches_hf(
     the metadata carries the fps HF assumes, which the frame timestamps use."""
     get_video_metadata = OpenCVVideoBackendMixin.get_video_metadata
 
-    def unknown_fps(cap) -> VideoSourceMetadata:
-        return get_video_metadata(cap)._replace(original_fps=0.0, duration=0.0)
+    def unknown_fps(data) -> VideoSourceMetadata:
+        return get_video_metadata(data)._replace(original_fps=0.0, duration=0.0)
 
     monkeypatch.setattr(
         OpenCVVideoBackendMixin, "get_video_metadata", staticmethod(unknown_fps)
@@ -1476,6 +1478,36 @@ def test_video_backend_handles_edit_list_trimmed_video(
         assert qwen_frames.shape[0] >= 4
 
 
+def test_opencv_demux_counts_frames_when_video_header_lacks_metadata(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    """Streamed WebM and raw H.264 carry no frame count/duration in their
+    headers, so OpenCV reports a bogus negative ``CAP_PROP_FRAME_COUNT``.
+    The loader must count the frames via a demux-only pass, without
+    decoding.
+    """
+    with monkeypatch.context() as m:
+        m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
+        loader = VIDEO_LOADER_REGISTRY.load("opencv")
+
+        webm_data = create_live_webm_video(num_frames=20, fps=10)
+        frames, metadata = loader.load_bytes(webm_data, backend="opencv")
+        assert frames.shape[0] == 20
+        assert metadata["total_num_frames"] == 20
+        assert metadata["duration"] == pytest.approx(2.0)
+
+        h264_data = create_raw_h264_video(num_frames=20, fps=25)
+        frames, metadata = loader.load_bytes(h264_data, backend="opencv")
+        assert frames.shape[0] == 20
+        assert metadata["total_num_frames"] == 20
+        assert metadata["duration"] == pytest.approx(0.8)
+
+        # The Qwen samplers used to raise on the bogus negative frame count.
+        qwen_frames, qwen_metadata = Qwen2VLVideoBackend.load_bytes(webm_data)
+        assert qwen_metadata["total_num_frames"] == 20
+        assert qwen_frames.shape[0] >= 4
+
+
 # ============================================================================
 # Frame Recovery Tests
 # ============================================================================

tests/multimodal/utils.py

@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
+import io
+
 import cv2
 import numpy as np
 import numpy.typing as npt
@@ -77,8 +79,6 @@ def create_long_gop_video(
     The marker lets a test recover which frame the decoder actually returned,
     independent of any metadata label.
     """
-    import io
-
     import av
 
     buf = io.BytesIO()
@@ -103,6 +103,63 @@ def create_long_gop_video(
     return buf.getvalue()
 
 
+def create_live_webm_video(
+    num_frames: int = 20,
+    fps: int = 10,
+    width: int = 64,
+    height: int = 64,
+) -> bytes:
+    """Encode a WebM stream without container duration or frame-count metadata.
+
+    The ``live`` muxing option mimics writing to a pipe: the Segment size is
+    unknown and no Duration or Cues elements are written, so decoder backends
+    cannot learn the frame count from the header.
+    """
+    import av
+
+    buf = io.BytesIO()
+    with av.open(buf, mode="w", format="webm", options={"live": "1"}) as container:
+        stream = container.add_stream("libvpx", rate=fps)
+        stream.width = width
+        stream.height = height
+        stream.pix_fmt = "yuv420p"
+        for i in range(num_frames):
+            img = np.zeros((height, width, 3), dtype=np.uint8)
+            img[:, :, 1] = i % 256
+            frame = av.VideoFrame.from_ndarray(img, format="rgb24")
+            for packet in stream.encode(frame):
+                container.mux(packet)
+        for packet in stream.encode():
+            container.mux(packet)
+    return buf.getvalue()
+
+
+def create_raw_h264_video(
+    num_frames: int = 20,
+    fps: int = 25,
+    width: int = 64,
+    height: int = 64,
+) -> bytes:
+    """Encode a raw H.264 Annex B elementary stream (no container metadata)."""
+    import av
+
+    buf = io.BytesIO()
+    with av.open(buf, mode="w", format="h264") as container:
+        stream = container.add_stream("h264", rate=fps)
+        stream.width = width
+        stream.height = height
+        stream.pix_fmt = "yuv420p"
+        for i in range(num_frames):
+            img = np.zeros((height, width, 3), dtype=np.uint8)
+            img[:, :, 1] = i % 256
+            frame = av.VideoFrame.from_ndarray(img, format="rgb24")
+            for packet in stream.encode(frame):
+                container.mux(packet)
+        for packet in stream.encode():
+            container.mux(packet)
+    return buf.getvalue()
+
+
 def create_edit_list_trimmed_video(
     num_frames: int = 90,
     trim_start_frame: int = 60,
@@ -117,8 +174,6 @@ def create_edit_list_trimmed_video(
     trailing frames are presentable — the shape produced by lossless trims
     (e.g. ``ffmpeg -ss ... -c copy``). Returns (video_bytes, visible_frames).
     """
-    import io
-
     import av
 
     src = create_long_gop_video(num_frames=num_frames, fps=fps)

vllm/multimodal/video_decoders/opencv.py

@@ -38,7 +38,9 @@ def decode_opencv(
         int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)),
         int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)),
     )
-    source = loader_cls._prepare_source(OpenCVVideoBackendMixin.get_video_metadata(cap))
+    source = loader_cls._prepare_source(
+        OpenCVVideoBackendMixin.get_video_metadata(data)
+    )
     frame_idx = loader_cls.compute_frames_index_to_sample(
         source=source, target=target, **sampling_kwargs
     )
@@ -74,10 +76,34 @@ class OpenCVVideoBackendMixin:
             raise ValueError("Could not open video stream")
         return cap
 
+    @classmethod
+    def demux_count_frames(cls, data: bytes) -> int:
+        """Count frames with OpenCV's demuxer-only mode."""
+        # NOTE: buf must stay referenced until after release().
+        # OpenCV's Python stream binding does not safely own the stream object
+        buf = BytesIO(data)
+        demux_cap = cv2.VideoCapture(
+            buf, cls.get_cv2_video_api(), [cv2.CAP_PROP_FORMAT, -1]
+        )
+        count = 0
+        while demux_cap.grab():
+            count += 1
+        demux_cap.release()
+        if count == 0:
+            raise ValueError("Could not determine the number of video frames")
+        return count
+
     @staticmethod
-    def get_video_metadata(cap: "cv2.VideoCapture") -> VideoSourceMetadata:
+    def get_video_metadata(data: bytes) -> VideoSourceMetadata:
+        cap = OpenCVVideoBackendMixin.open_video_capture(data)
         total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
         original_fps = cap.get(cv2.CAP_PROP_FPS)
+        if total_frames_num <= 0:
+            # Streamed WebM/MKV and raw H.264 miss frame count in headers;
+            # We need to use demux frames count for retrieval
+            total_frames_num = OpenCVVideoBackendMixin.demux_count_frames(data)
+            duration = total_frames_num / original_fps if original_fps > 0 else 0
+            return VideoSourceMetadata(total_frames_num, original_fps, duration)
         # CAP_PROP_FRAME_COUNT counts every physical sample in the container,
         # overstating the presentable frames when an mp4 edit list hides the
         # decode lead-in (e.g. a lossless `ffmpeg -ss ... -c copy` trim).
@@ -87,7 +113,7 @@ class OpenCVVideoBackendMixin:
         # may land mid-stream on corrupt files, so only trust it when the
         # stream really ends there. A one-frame tolerance keeps mere header
         # miscounts from being "corrected".
-        if total_frames_num > 0 and cap.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0):
+        if cap.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0):
             visible_frames = int(cap.get(cv2.CAP_PROP_POS_FRAMES))
             at_stream_end = not cap.grab()
             cap.set(cv2.CAP_PROP_POS_FRAMES, 0)