From 9b2aef512440f024fc98cd0ac954cd452613ad5b Mon Sep 17 00:00:00 2001 From: Amit Moryossef Date: Thu, 20 Aug 2026 06:23:09 +0200 Subject: [PATCH] [Bugfix] Video loading: sample over presentable frames, not header sample count (MP4 edit-list trims) (#48608) Signed-off-by: AmitMY Signed-off-by: Isotr0py Co-authored-by: Claude Fable 5 Co-authored-by: Isotr0py --- tests/multimodal/test_video.py | 52 +++++++++++++++++++++++- tests/multimodal/utils.py | 37 +++++++++++++++++ vllm/multimodal/video_decoders/opencv.py | 21 ++++++++++ vllm/multimodal/video_decoders/pyav.py | 8 ++++ 4 files changed, 117 insertions(+), 1 deletion(-) diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index 9ab718d945..4020900696 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -40,7 +40,11 @@ from vllm.multimodal.video_decoders.pynvvideocodec import ( from vllm.platforms import current_platform from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config -from .utils import create_long_gop_video, create_video_from_image +from .utils import ( + create_edit_list_trimmed_video, + create_long_gop_video, + create_video_from_image, +) pytestmark = pytest.mark.cpu_test @@ -862,6 +866,44 @@ def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch): ) +def test_video_backend_handles_edit_list_trimmed_video( + monkeypatch: pytest.MonkeyPatch, +): + """ + An mp4 edit list (e.g. from a lossless ``ffmpeg -ss ... -c copy`` cut) + hides the decode lead-in: the header still counts every physical sample + while sequential decode only yields the visible frames. Sampling over the + header count used to collapse such videos to the few indices below the + visible count (a single frame for the Qwen loaders) — the loader must + resample over the true stream length instead. + """ + with monkeypatch.context() as m: + m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv") + + video_data, num_visible = create_edit_list_trimmed_video( + num_frames=90, trim_start_frame=60 + ) + + loader = VIDEO_LOADER_REGISTRY.load("opencv") + for backend in ["opencv", "pyav"]: + frames, metadata = loader.load_bytes( + video_data, num_frames=-1, backend=backend + ) + assert metadata["total_num_frames"] == num_visible, backend + assert frames.shape[0] == num_visible, backend + assert len(metadata["frames_indices"]) == num_visible, backend + # The green channel encodes the source frame index: the visible + # frames must be the trailing ones (60..89), not the lead-in. + mean_green = frames[..., 1].reshape(frames.shape[0], -1).mean(axis=1) + assert abs(mean_green[0] - 60) <= 5, backend + assert abs(mean_green[-1] - 89) <= 5, backend + + # The Qwen samplers used to degenerate to a single frame here. + qwen_frames, qwen_metadata = Qwen2VLVideoBackend.load_bytes(video_data) + assert qwen_metadata["total_num_frames"] == num_visible + assert qwen_frames.shape[0] >= 4 + + # ============================================================================ # Frame Recovery Tests # ============================================================================ @@ -912,6 +954,14 @@ def test_video_recovery_simulated_failures(monkeypatch: pytest.MonkeyPatch): def get(self, prop): return self._cap.get(prop) + def set(self, prop, value): + # get_video_metadata probes the stream end with a seek; + # keep the simulated frame counter aligned with rewinds. + result = self._cap.set(prop, value) + if prop == cv2.CAP_PROP_POS_FRAMES: + self._current_frame = int(value) - 1 + return result + def isOpened(self): return self._cap.isOpened() diff --git a/tests/multimodal/utils.py b/tests/multimodal/utils.py index bae0a9d294..23a05a0944 100644 --- a/tests/multimodal/utils.py +++ b/tests/multimodal/utils.py @@ -103,6 +103,43 @@ def create_long_gop_video( return buf.getvalue() +def create_edit_list_trimmed_video( + num_frames: int = 90, + trim_start_frame: int = 60, + fps: int = 30, +) -> tuple[bytes, int]: + """Stream-copy-cut a long-GOP clip so an mp4 edit list hides the lead-in. + + Remuxes the tail of a single-keyframe clip without re-encoding, rebasing + timestamps at ``trim_start_frame``: the lead-in packets are still needed + to decode, so they stay in the file at negative pts and the muxer records + an edit list. The header sample count stays ``num_frames`` while only the + trailing frames are presentable — the shape produced by lossless trims + (e.g. ``ffmpeg -ss ... -c copy``). Returns (video_bytes, visible_frames). + """ + import io + + import av + + src = create_long_gop_video(num_frames=num_frames, fps=fps) + buf = io.BytesIO() + with ( + av.open(io.BytesIO(src)) as source, + av.open(buf, mode="w", format="mp4") as out, + ): + in_stream = source.streams.video[0] + out_stream = out.add_stream_from_template(in_stream) + start_pts = round(trim_start_frame / fps / in_stream.time_base) + for packet in source.demux(in_stream): + if packet.pts is None or packet.dts is None: + continue + packet.pts -= start_pts + packet.dts -= start_pts + packet.stream = out_stream + out.mux(packet) + return buf.getvalue(), num_frames - trim_start_frame + + def cosine_similarity(A: npt.NDArray, B: npt.NDArray, axis: int = -1) -> npt.NDArray: """Compute cosine similarity between two vectors.""" return np.sum(A * B, axis=axis) / ( diff --git a/vllm/multimodal/video_decoders/opencv.py b/vllm/multimodal/video_decoders/opencv.py index e5e474eab6..e46979dd1d 100644 --- a/vllm/multimodal/video_decoders/opencv.py +++ b/vllm/multimodal/video_decoders/opencv.py @@ -78,6 +78,27 @@ class OpenCVVideoBackendMixin: def get_video_metadata(cap: "cv2.VideoCapture") -> VideoSourceMetadata: total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) original_fps = cap.get(cv2.CAP_PROP_FPS) + # CAP_PROP_FRAME_COUNT counts every physical sample in the container, + # overstating the presentable frames when an mp4 edit list hides the + # decode lead-in (e.g. a lossless `ffmpeg -ss ... -c copy` trim). + # Cross-check with an end-of-stream seek: the FFMPEG backend reports + # the resulting position against the edit-list-aware timeline, which + # exposes the true frame count without decoding anything. The seek + # may land mid-stream on corrupt files, so only trust it when the + # stream really ends there. A one-frame tolerance keeps mere header + # miscounts from being "corrected". + if total_frames_num > 0 and cap.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0): + visible_frames = int(cap.get(cv2.CAP_PROP_POS_FRAMES)) + at_stream_end = not cap.grab() + cap.set(cv2.CAP_PROP_POS_FRAMES, 0) + if at_stream_end and 0 < visible_frames < total_frames_num - 1: + logger.warning( + "Video header claims %d frames but only %d are " + "presentable; sampling over the true frame count.", + total_frames_num, + visible_frames, + ) + total_frames_num = visible_frames duration = total_frames_num / original_fps if original_fps > 0 else 0 return VideoSourceMetadata( total_frames_num=total_frames_num, diff --git a/vllm/multimodal/video_decoders/pyav.py b/vllm/multimodal/video_decoders/pyav.py index dc82b01230..80c2829666 100644 --- a/vllm/multimodal/video_decoders/pyav.py +++ b/vllm/multimodal/video_decoders/pyav.py @@ -62,6 +62,14 @@ class PyAVVideoBackendMixin: duration = float(stream.duration * stream.time_base) if stream.duration else 0.0 if total_frames == 0 and duration > 0 and fps > 0: total_frames = int(duration * fps) + elif duration > 0 and fps > 0 and total_frames > round(duration * fps) + 1: + # The header sample count can exceed the frames the presentation + # timeline actually holds — e.g. an mp4 edit list hides the decode + # lead-in of a stream-copied cut, but ``stream.frames`` still + # counts the hidden samples. Sampling indices past the visible + # range would silently collapse onto the last visible frame, so + # trust the (edit-list-aware) duration instead. + total_frames = round(duration * fps) return VideoSourceMetadata(total_frames, fps, duration) @staticmethod