[Bugfix] Video loading: sample over presentable frames, not header sample count (MP4 edit-list trims) (#48608)
Signed-off-by: AmitMY <amitmoryossef@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Claude Fable 5
Isotr0py
parent
11baa0ebd8
commit
9b2aef5124
@@ -40,7 +40,11 @@ from vllm.multimodal.video_decoders.pynvvideocodec import (
|
|||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
|
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
|
||||||
|
|
||||||
from .utils import create_long_gop_video, create_video_from_image
|
from .utils import (
|
||||||
|
create_edit_list_trimmed_video,
|
||||||
|
create_long_gop_video,
|
||||||
|
create_video_from_image,
|
||||||
|
)
|
||||||
|
|
||||||
pytestmark = pytest.mark.cpu_test
|
pytestmark = pytest.mark.cpu_test
|
||||||
|
|
||||||
@@ -862,6 +866,44 @@ def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_backend_handles_edit_list_trimmed_video(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
An mp4 edit list (e.g. from a lossless ``ffmpeg -ss ... -c copy`` cut)
|
||||||
|
hides the decode lead-in: the header still counts every physical sample
|
||||||
|
while sequential decode only yields the visible frames. Sampling over the
|
||||||
|
header count used to collapse such videos to the few indices below the
|
||||||
|
visible count (a single frame for the Qwen loaders) — the loader must
|
||||||
|
resample over the true stream length instead.
|
||||||
|
"""
|
||||||
|
with monkeypatch.context() as m:
|
||||||
|
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
|
||||||
|
|
||||||
|
video_data, num_visible = create_edit_list_trimmed_video(
|
||||||
|
num_frames=90, trim_start_frame=60
|
||||||
|
)
|
||||||
|
|
||||||
|
loader = VIDEO_LOADER_REGISTRY.load("opencv")
|
||||||
|
for backend in ["opencv", "pyav"]:
|
||||||
|
frames, metadata = loader.load_bytes(
|
||||||
|
video_data, num_frames=-1, backend=backend
|
||||||
|
)
|
||||||
|
assert metadata["total_num_frames"] == num_visible, backend
|
||||||
|
assert frames.shape[0] == num_visible, backend
|
||||||
|
assert len(metadata["frames_indices"]) == num_visible, backend
|
||||||
|
# The green channel encodes the source frame index: the visible
|
||||||
|
# frames must be the trailing ones (60..89), not the lead-in.
|
||||||
|
mean_green = frames[..., 1].reshape(frames.shape[0], -1).mean(axis=1)
|
||||||
|
assert abs(mean_green[0] - 60) <= 5, backend
|
||||||
|
assert abs(mean_green[-1] - 89) <= 5, backend
|
||||||
|
|
||||||
|
# The Qwen samplers used to degenerate to a single frame here.
|
||||||
|
qwen_frames, qwen_metadata = Qwen2VLVideoBackend.load_bytes(video_data)
|
||||||
|
assert qwen_metadata["total_num_frames"] == num_visible
|
||||||
|
assert qwen_frames.shape[0] >= 4
|
||||||
|
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# Frame Recovery Tests
|
# Frame Recovery Tests
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
@@ -912,6 +954,14 @@ def test_video_recovery_simulated_failures(monkeypatch: pytest.MonkeyPatch):
|
|||||||
def get(self, prop):
|
def get(self, prop):
|
||||||
return self._cap.get(prop)
|
return self._cap.get(prop)
|
||||||
|
|
||||||
|
def set(self, prop, value):
|
||||||
|
# get_video_metadata probes the stream end with a seek;
|
||||||
|
# keep the simulated frame counter aligned with rewinds.
|
||||||
|
result = self._cap.set(prop, value)
|
||||||
|
if prop == cv2.CAP_PROP_POS_FRAMES:
|
||||||
|
self._current_frame = int(value) - 1
|
||||||
|
return result
|
||||||
|
|
||||||
def isOpened(self):
|
def isOpened(self):
|
||||||
return self._cap.isOpened()
|
return self._cap.isOpened()
|
||||||
|
|
||||||
|
|||||||
@@ -103,6 +103,43 @@ def create_long_gop_video(
|
|||||||
return buf.getvalue()
|
return buf.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def create_edit_list_trimmed_video(
|
||||||
|
num_frames: int = 90,
|
||||||
|
trim_start_frame: int = 60,
|
||||||
|
fps: int = 30,
|
||||||
|
) -> tuple[bytes, int]:
|
||||||
|
"""Stream-copy-cut a long-GOP clip so an mp4 edit list hides the lead-in.
|
||||||
|
|
||||||
|
Remuxes the tail of a single-keyframe clip without re-encoding, rebasing
|
||||||
|
timestamps at ``trim_start_frame``: the lead-in packets are still needed
|
||||||
|
to decode, so they stay in the file at negative pts and the muxer records
|
||||||
|
an edit list. The header sample count stays ``num_frames`` while only the
|
||||||
|
trailing frames are presentable — the shape produced by lossless trims
|
||||||
|
(e.g. ``ffmpeg -ss ... -c copy``). Returns (video_bytes, visible_frames).
|
||||||
|
"""
|
||||||
|
import io
|
||||||
|
|
||||||
|
import av
|
||||||
|
|
||||||
|
src = create_long_gop_video(num_frames=num_frames, fps=fps)
|
||||||
|
buf = io.BytesIO()
|
||||||
|
with (
|
||||||
|
av.open(io.BytesIO(src)) as source,
|
||||||
|
av.open(buf, mode="w", format="mp4") as out,
|
||||||
|
):
|
||||||
|
in_stream = source.streams.video[0]
|
||||||
|
out_stream = out.add_stream_from_template(in_stream)
|
||||||
|
start_pts = round(trim_start_frame / fps / in_stream.time_base)
|
||||||
|
for packet in source.demux(in_stream):
|
||||||
|
if packet.pts is None or packet.dts is None:
|
||||||
|
continue
|
||||||
|
packet.pts -= start_pts
|
||||||
|
packet.dts -= start_pts
|
||||||
|
packet.stream = out_stream
|
||||||
|
out.mux(packet)
|
||||||
|
return buf.getvalue(), num_frames - trim_start_frame
|
||||||
|
|
||||||
|
|
||||||
def cosine_similarity(A: npt.NDArray, B: npt.NDArray, axis: int = -1) -> npt.NDArray:
|
def cosine_similarity(A: npt.NDArray, B: npt.NDArray, axis: int = -1) -> npt.NDArray:
|
||||||
"""Compute cosine similarity between two vectors."""
|
"""Compute cosine similarity between two vectors."""
|
||||||
return np.sum(A * B, axis=axis) / (
|
return np.sum(A * B, axis=axis) / (
|
||||||
|
|||||||
@@ -78,6 +78,27 @@ class OpenCVVideoBackendMixin:
|
|||||||
def get_video_metadata(cap: "cv2.VideoCapture") -> VideoSourceMetadata:
|
def get_video_metadata(cap: "cv2.VideoCapture") -> VideoSourceMetadata:
|
||||||
total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||||
original_fps = cap.get(cv2.CAP_PROP_FPS)
|
original_fps = cap.get(cv2.CAP_PROP_FPS)
|
||||||
|
# CAP_PROP_FRAME_COUNT counts every physical sample in the container,
|
||||||
|
# overstating the presentable frames when an mp4 edit list hides the
|
||||||
|
# decode lead-in (e.g. a lossless `ffmpeg -ss ... -c copy` trim).
|
||||||
|
# Cross-check with an end-of-stream seek: the FFMPEG backend reports
|
||||||
|
# the resulting position against the edit-list-aware timeline, which
|
||||||
|
# exposes the true frame count without decoding anything. The seek
|
||||||
|
# may land mid-stream on corrupt files, so only trust it when the
|
||||||
|
# stream really ends there. A one-frame tolerance keeps mere header
|
||||||
|
# miscounts from being "corrected".
|
||||||
|
if total_frames_num > 0 and cap.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0):
|
||||||
|
visible_frames = int(cap.get(cv2.CAP_PROP_POS_FRAMES))
|
||||||
|
at_stream_end = not cap.grab()
|
||||||
|
cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
|
||||||
|
if at_stream_end and 0 < visible_frames < total_frames_num - 1:
|
||||||
|
logger.warning(
|
||||||
|
"Video header claims %d frames but only %d are "
|
||||||
|
"presentable; sampling over the true frame count.",
|
||||||
|
total_frames_num,
|
||||||
|
visible_frames,
|
||||||
|
)
|
||||||
|
total_frames_num = visible_frames
|
||||||
duration = total_frames_num / original_fps if original_fps > 0 else 0
|
duration = total_frames_num / original_fps if original_fps > 0 else 0
|
||||||
return VideoSourceMetadata(
|
return VideoSourceMetadata(
|
||||||
total_frames_num=total_frames_num,
|
total_frames_num=total_frames_num,
|
||||||
|
|||||||
@@ -62,6 +62,14 @@ class PyAVVideoBackendMixin:
|
|||||||
duration = float(stream.duration * stream.time_base) if stream.duration else 0.0
|
duration = float(stream.duration * stream.time_base) if stream.duration else 0.0
|
||||||
if total_frames == 0 and duration > 0 and fps > 0:
|
if total_frames == 0 and duration > 0 and fps > 0:
|
||||||
total_frames = int(duration * fps)
|
total_frames = int(duration * fps)
|
||||||
|
elif duration > 0 and fps > 0 and total_frames > round(duration * fps) + 1:
|
||||||
|
# The header sample count can exceed the frames the presentation
|
||||||
|
# timeline actually holds — e.g. an mp4 edit list hides the decode
|
||||||
|
# lead-in of a stream-copied cut, but ``stream.frames`` still
|
||||||
|
# counts the hidden samples. Sampling indices past the visible
|
||||||
|
# range would silently collapse onto the last visible frame, so
|
||||||
|
# trust the (edit-list-aware) duration instead.
|
||||||
|
total_frames = round(duration * fps)
|
||||||
return VideoSourceMetadata(total_frames, fps, duration)
|
return VideoSourceMetadata(total_frames, fps, duration)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
Reference in New Issue
Block a user