[Bugfix] Video loading: sample over presentable frames, not header sample count (MP4 edit-list trims) (#48608)

Signed-off-by: AmitMY <amitmoryossef@gmail.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Amit Moryossef
2026-08-20 04:23:09 +00:00
committed by GitHub
co-authored by Claude Fable 5 Isotr0py
parent 11baa0ebd8
commit 9b2aef5124
4 changed files with 117 additions and 1 deletions
+51 -1
View File
@@ -40,7 +40,11 @@ from vllm.multimodal.video_decoders.pynvvideocodec import (
from vllm.platforms import current_platform
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
from .utils import create_long_gop_video, create_video_from_image
from .utils import (
create_edit_list_trimmed_video,
create_long_gop_video,
create_video_from_image,
)
pytestmark = pytest.mark.cpu_test
@@ -862,6 +866,44 @@ def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
)
def test_video_backend_handles_edit_list_trimmed_video(
monkeypatch: pytest.MonkeyPatch,
):
"""
An mp4 edit list (e.g. from a lossless ``ffmpeg -ss ... -c copy`` cut)
hides the decode lead-in: the header still counts every physical sample
while sequential decode only yields the visible frames. Sampling over the
header count used to collapse such videos to the few indices below the
visible count (a single frame for the Qwen loaders) — the loader must
resample over the true stream length instead.
"""
with monkeypatch.context() as m:
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
video_data, num_visible = create_edit_list_trimmed_video(
num_frames=90, trim_start_frame=60
)
loader = VIDEO_LOADER_REGISTRY.load("opencv")
for backend in ["opencv", "pyav"]:
frames, metadata = loader.load_bytes(
video_data, num_frames=-1, backend=backend
)
assert metadata["total_num_frames"] == num_visible, backend
assert frames.shape[0] == num_visible, backend
assert len(metadata["frames_indices"]) == num_visible, backend
# The green channel encodes the source frame index: the visible
# frames must be the trailing ones (60..89), not the lead-in.
mean_green = frames[..., 1].reshape(frames.shape[0], -1).mean(axis=1)
assert abs(mean_green[0] - 60) <= 5, backend
assert abs(mean_green[-1] - 89) <= 5, backend
# The Qwen samplers used to degenerate to a single frame here.
qwen_frames, qwen_metadata = Qwen2VLVideoBackend.load_bytes(video_data)
assert qwen_metadata["total_num_frames"] == num_visible
assert qwen_frames.shape[0] >= 4
# ============================================================================
# Frame Recovery Tests
# ============================================================================
@@ -912,6 +954,14 @@ def test_video_recovery_simulated_failures(monkeypatch: pytest.MonkeyPatch):
def get(self, prop):
return self._cap.get(prop)
def set(self, prop, value):
# get_video_metadata probes the stream end with a seek;
# keep the simulated frame counter aligned with rewinds.
result = self._cap.set(prop, value)
if prop == cv2.CAP_PROP_POS_FRAMES:
self._current_frame = int(value) - 1
return result
def isOpened(self):
return self._cap.isOpened()
+37
View File
@@ -103,6 +103,43 @@ def create_long_gop_video(
return buf.getvalue()
def create_edit_list_trimmed_video(
num_frames: int = 90,
trim_start_frame: int = 60,
fps: int = 30,
) -> tuple[bytes, int]:
"""Stream-copy-cut a long-GOP clip so an mp4 edit list hides the lead-in.
Remuxes the tail of a single-keyframe clip without re-encoding, rebasing
timestamps at ``trim_start_frame``: the lead-in packets are still needed
to decode, so they stay in the file at negative pts and the muxer records
an edit list. The header sample count stays ``num_frames`` while only the
trailing frames are presentable — the shape produced by lossless trims
(e.g. ``ffmpeg -ss ... -c copy``). Returns (video_bytes, visible_frames).
"""
import io
import av
src = create_long_gop_video(num_frames=num_frames, fps=fps)
buf = io.BytesIO()
with (
av.open(io.BytesIO(src)) as source,
av.open(buf, mode="w", format="mp4") as out,
):
in_stream = source.streams.video[0]
out_stream = out.add_stream_from_template(in_stream)
start_pts = round(trim_start_frame / fps / in_stream.time_base)
for packet in source.demux(in_stream):
if packet.pts is None or packet.dts is None:
continue
packet.pts -= start_pts
packet.dts -= start_pts
packet.stream = out_stream
out.mux(packet)
return buf.getvalue(), num_frames - trim_start_frame
def cosine_similarity(A: npt.NDArray, B: npt.NDArray, axis: int = -1) -> npt.NDArray:
"""Compute cosine similarity between two vectors."""
return np.sum(A * B, axis=axis) / (