[Bugfix] Video loading: sample over presentable frames, not header sample count (MP4 edit-list trims) (#48608)

Signed-off-by: AmitMY <amitmoryossef@gmail.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Amit Moryossef
2026-08-20 04:23:09 +00:00
committed by GitHub
co-authored by Claude Fable 5 Isotr0py
parent 11baa0ebd8
commit 9b2aef5124
4 changed files with 117 additions and 1 deletions
+51 -1
View File
@@ -40,7 +40,11 @@ from vllm.multimodal.video_decoders.pynvvideocodec import (
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
from .utils import create_long_gop_video, create_video_from_image from .utils import (
create_edit_list_trimmed_video,
create_long_gop_video,
create_video_from_image,
)
pytestmark = pytest.mark.cpu_test pytestmark = pytest.mark.cpu_test
@@ -862,6 +866,44 @@ def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
) )
def test_video_backend_handles_edit_list_trimmed_video(
monkeypatch: pytest.MonkeyPatch,
):
"""
An mp4 edit list (e.g. from a lossless ``ffmpeg -ss ... -c copy`` cut)
hides the decode lead-in: the header still counts every physical sample
while sequential decode only yields the visible frames. Sampling over the
header count used to collapse such videos to the few indices below the
visible count (a single frame for the Qwen loaders) — the loader must
resample over the true stream length instead.
"""
with monkeypatch.context() as m:
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
video_data, num_visible = create_edit_list_trimmed_video(
num_frames=90, trim_start_frame=60
)
loader = VIDEO_LOADER_REGISTRY.load("opencv")
for backend in ["opencv", "pyav"]:
frames, metadata = loader.load_bytes(
video_data, num_frames=-1, backend=backend
)
assert metadata["total_num_frames"] == num_visible, backend
assert frames.shape[0] == num_visible, backend
assert len(metadata["frames_indices"]) == num_visible, backend
# The green channel encodes the source frame index: the visible
# frames must be the trailing ones (60..89), not the lead-in.
mean_green = frames[..., 1].reshape(frames.shape[0], -1).mean(axis=1)
assert abs(mean_green[0] - 60) <= 5, backend
assert abs(mean_green[-1] - 89) <= 5, backend
# The Qwen samplers used to degenerate to a single frame here.
qwen_frames, qwen_metadata = Qwen2VLVideoBackend.load_bytes(video_data)
assert qwen_metadata["total_num_frames"] == num_visible
assert qwen_frames.shape[0] >= 4
# ============================================================================ # ============================================================================
# Frame Recovery Tests # Frame Recovery Tests
# ============================================================================ # ============================================================================
@@ -912,6 +954,14 @@ def test_video_recovery_simulated_failures(monkeypatch: pytest.MonkeyPatch):
def get(self, prop): def get(self, prop):
return self._cap.get(prop) return self._cap.get(prop)
def set(self, prop, value):
# get_video_metadata probes the stream end with a seek;
# keep the simulated frame counter aligned with rewinds.
result = self._cap.set(prop, value)
if prop == cv2.CAP_PROP_POS_FRAMES:
self._current_frame = int(value) - 1
return result
def isOpened(self): def isOpened(self):
return self._cap.isOpened() return self._cap.isOpened()
+37
View File
@@ -103,6 +103,43 @@ def create_long_gop_video(
return buf.getvalue() return buf.getvalue()
def create_edit_list_trimmed_video(
num_frames: int = 90,
trim_start_frame: int = 60,
fps: int = 30,
) -> tuple[bytes, int]:
"""Stream-copy-cut a long-GOP clip so an mp4 edit list hides the lead-in.
Remuxes the tail of a single-keyframe clip without re-encoding, rebasing
timestamps at ``trim_start_frame``: the lead-in packets are still needed
to decode, so they stay in the file at negative pts and the muxer records
an edit list. The header sample count stays ``num_frames`` while only the
trailing frames are presentable — the shape produced by lossless trims
(e.g. ``ffmpeg -ss ... -c copy``). Returns (video_bytes, visible_frames).
"""
import io
import av
src = create_long_gop_video(num_frames=num_frames, fps=fps)
buf = io.BytesIO()
with (
av.open(io.BytesIO(src)) as source,
av.open(buf, mode="w", format="mp4") as out,
):
in_stream = source.streams.video[0]
out_stream = out.add_stream_from_template(in_stream)
start_pts = round(trim_start_frame / fps / in_stream.time_base)
for packet in source.demux(in_stream):
if packet.pts is None or packet.dts is None:
continue
packet.pts -= start_pts
packet.dts -= start_pts
packet.stream = out_stream
out.mux(packet)
return buf.getvalue(), num_frames - trim_start_frame
def cosine_similarity(A: npt.NDArray, B: npt.NDArray, axis: int = -1) -> npt.NDArray: def cosine_similarity(A: npt.NDArray, B: npt.NDArray, axis: int = -1) -> npt.NDArray:
"""Compute cosine similarity between two vectors.""" """Compute cosine similarity between two vectors."""
return np.sum(A * B, axis=axis) / ( return np.sum(A * B, axis=axis) / (
+21
View File
@@ -78,6 +78,27 @@ class OpenCVVideoBackendMixin:
def get_video_metadata(cap: "cv2.VideoCapture") -> VideoSourceMetadata: def get_video_metadata(cap: "cv2.VideoCapture") -> VideoSourceMetadata:
total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
original_fps = cap.get(cv2.CAP_PROP_FPS) original_fps = cap.get(cv2.CAP_PROP_FPS)
# CAP_PROP_FRAME_COUNT counts every physical sample in the container,
# overstating the presentable frames when an mp4 edit list hides the
# decode lead-in (e.g. a lossless `ffmpeg -ss ... -c copy` trim).
# Cross-check with an end-of-stream seek: the FFMPEG backend reports
# the resulting position against the edit-list-aware timeline, which
# exposes the true frame count without decoding anything. The seek
# may land mid-stream on corrupt files, so only trust it when the
# stream really ends there. A one-frame tolerance keeps mere header
# miscounts from being "corrected".
if total_frames_num > 0 and cap.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0):
visible_frames = int(cap.get(cv2.CAP_PROP_POS_FRAMES))
at_stream_end = not cap.grab()
cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
if at_stream_end and 0 < visible_frames < total_frames_num - 1:
logger.warning(
"Video header claims %d frames but only %d are "
"presentable; sampling over the true frame count.",
total_frames_num,
visible_frames,
)
total_frames_num = visible_frames
duration = total_frames_num / original_fps if original_fps > 0 else 0 duration = total_frames_num / original_fps if original_fps > 0 else 0
return VideoSourceMetadata( return VideoSourceMetadata(
total_frames_num=total_frames_num, total_frames_num=total_frames_num,
+8
View File
@@ -62,6 +62,14 @@ class PyAVVideoBackendMixin:
duration = float(stream.duration * stream.time_base) if stream.duration else 0.0 duration = float(stream.duration * stream.time_base) if stream.duration else 0.0
if total_frames == 0 and duration > 0 and fps > 0: if total_frames == 0 and duration > 0 and fps > 0:
total_frames = int(duration * fps) total_frames = int(duration * fps)
elif duration > 0 and fps > 0 and total_frames > round(duration * fps) + 1:
# The header sample count can exceed the frames the presentation
# timeline actually holds — e.g. an mp4 edit list hides the decode
# lead-in of a stream-copied cut, but ``stream.frames`` still
# counts the hidden samples. Sampling indices past the visible
# range would silently collapse onto the last visible frame, so
# trust the (edit-list-aware) duration instead.
total_frames = round(duration * fps)
return VideoSourceMetadata(total_frames, fps, duration) return VideoSourceMetadata(total_frames, fps, duration)
@staticmethod @staticmethod