[Bugfix] Video loading: sample over presentable frames, not header sample count (MP4 edit-list trims) (#48608)
Signed-off-by: AmitMY <amitmoryossef@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Claude Fable 5
Isotr0py
parent
11baa0ebd8
commit
9b2aef5124
@@ -40,7 +40,11 @@ from vllm.multimodal.video_decoders.pynvvideocodec import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
|
||||
|
||||
from .utils import create_long_gop_video, create_video_from_image
|
||||
from .utils import (
|
||||
create_edit_list_trimmed_video,
|
||||
create_long_gop_video,
|
||||
create_video_from_image,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -862,6 +866,44 @@ def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
|
||||
)
|
||||
|
||||
|
||||
def test_video_backend_handles_edit_list_trimmed_video(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""
|
||||
An mp4 edit list (e.g. from a lossless ``ffmpeg -ss ... -c copy`` cut)
|
||||
hides the decode lead-in: the header still counts every physical sample
|
||||
while sequential decode only yields the visible frames. Sampling over the
|
||||
header count used to collapse such videos to the few indices below the
|
||||
visible count (a single frame for the Qwen loaders) — the loader must
|
||||
resample over the true stream length instead.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
|
||||
|
||||
video_data, num_visible = create_edit_list_trimmed_video(
|
||||
num_frames=90, trim_start_frame=60
|
||||
)
|
||||
|
||||
loader = VIDEO_LOADER_REGISTRY.load("opencv")
|
||||
for backend in ["opencv", "pyav"]:
|
||||
frames, metadata = loader.load_bytes(
|
||||
video_data, num_frames=-1, backend=backend
|
||||
)
|
||||
assert metadata["total_num_frames"] == num_visible, backend
|
||||
assert frames.shape[0] == num_visible, backend
|
||||
assert len(metadata["frames_indices"]) == num_visible, backend
|
||||
# The green channel encodes the source frame index: the visible
|
||||
# frames must be the trailing ones (60..89), not the lead-in.
|
||||
mean_green = frames[..., 1].reshape(frames.shape[0], -1).mean(axis=1)
|
||||
assert abs(mean_green[0] - 60) <= 5, backend
|
||||
assert abs(mean_green[-1] - 89) <= 5, backend
|
||||
|
||||
# The Qwen samplers used to degenerate to a single frame here.
|
||||
qwen_frames, qwen_metadata = Qwen2VLVideoBackend.load_bytes(video_data)
|
||||
assert qwen_metadata["total_num_frames"] == num_visible
|
||||
assert qwen_frames.shape[0] >= 4
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Frame Recovery Tests
|
||||
# ============================================================================
|
||||
@@ -912,6 +954,14 @@ def test_video_recovery_simulated_failures(monkeypatch: pytest.MonkeyPatch):
|
||||
def get(self, prop):
|
||||
return self._cap.get(prop)
|
||||
|
||||
def set(self, prop, value):
|
||||
# get_video_metadata probes the stream end with a seek;
|
||||
# keep the simulated frame counter aligned with rewinds.
|
||||
result = self._cap.set(prop, value)
|
||||
if prop == cv2.CAP_PROP_POS_FRAMES:
|
||||
self._current_frame = int(value) - 1
|
||||
return result
|
||||
|
||||
def isOpened(self):
|
||||
return self._cap.isOpened()
|
||||
|
||||
|
||||
@@ -103,6 +103,43 @@ def create_long_gop_video(
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def create_edit_list_trimmed_video(
|
||||
num_frames: int = 90,
|
||||
trim_start_frame: int = 60,
|
||||
fps: int = 30,
|
||||
) -> tuple[bytes, int]:
|
||||
"""Stream-copy-cut a long-GOP clip so an mp4 edit list hides the lead-in.
|
||||
|
||||
Remuxes the tail of a single-keyframe clip without re-encoding, rebasing
|
||||
timestamps at ``trim_start_frame``: the lead-in packets are still needed
|
||||
to decode, so they stay in the file at negative pts and the muxer records
|
||||
an edit list. The header sample count stays ``num_frames`` while only the
|
||||
trailing frames are presentable — the shape produced by lossless trims
|
||||
(e.g. ``ffmpeg -ss ... -c copy``). Returns (video_bytes, visible_frames).
|
||||
"""
|
||||
import io
|
||||
|
||||
import av
|
||||
|
||||
src = create_long_gop_video(num_frames=num_frames, fps=fps)
|
||||
buf = io.BytesIO()
|
||||
with (
|
||||
av.open(io.BytesIO(src)) as source,
|
||||
av.open(buf, mode="w", format="mp4") as out,
|
||||
):
|
||||
in_stream = source.streams.video[0]
|
||||
out_stream = out.add_stream_from_template(in_stream)
|
||||
start_pts = round(trim_start_frame / fps / in_stream.time_base)
|
||||
for packet in source.demux(in_stream):
|
||||
if packet.pts is None or packet.dts is None:
|
||||
continue
|
||||
packet.pts -= start_pts
|
||||
packet.dts -= start_pts
|
||||
packet.stream = out_stream
|
||||
out.mux(packet)
|
||||
return buf.getvalue(), num_frames - trim_start_frame
|
||||
|
||||
|
||||
def cosine_similarity(A: npt.NDArray, B: npt.NDArray, axis: int = -1) -> npt.NDArray:
|
||||
"""Compute cosine similarity between two vectors."""
|
||||
return np.sum(A * B, axis=axis) / (
|
||||
|
||||
Reference in New Issue
Block a user