[Frontend] Implement robust video frame recovery for corrupted videos (#29197)
Signed-off-by: cmartinez <cmartinez@roblox.com> Signed-off-by: vSeamar <cmartinez@roblox.com>
This commit is contained in:
@@ -299,3 +299,212 @@ def test_video_media_io_backend_env_var_fallback(monkeypatch: pytest.MonkeyPatch
|
||||
frames_missing, metadata_missing = videoio_missing.load_bytes(b"test")
|
||||
np.testing.assert_array_equal(frames_missing, FAKE_OUTPUT_2)
|
||||
assert metadata_missing["video_backend"] == "test_video_backend_override_2"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Frame Recovery Tests
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def test_video_recovery_simulated_failures(monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
Test that frame recovery correctly uses the next valid frame when
|
||||
target frames fail to load.
|
||||
|
||||
Uses corrupted.mp4 and mocks VideoCapture.grab() to fail on specific
|
||||
frame indices (in addition to the real corruption at frame 17), then
|
||||
verifies recovery produces more frames.
|
||||
"""
|
||||
import cv2
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
|
||||
|
||||
# Load corrupted.mp4 (26 frames, frame 17 is genuinely corrupted)
|
||||
video_path = ASSETS_DIR / "corrupted.mp4"
|
||||
with open(video_path, "rb") as f:
|
||||
video_data = f.read()
|
||||
|
||||
# Simulate additional failures on frames 3 and 10
|
||||
# (in addition to the real corruption at frame 17)
|
||||
fail_on_frames = {3, 10}
|
||||
|
||||
# Store original VideoCapture class
|
||||
original_video_capture = cv2.VideoCapture
|
||||
|
||||
class MockVideoCapture:
|
||||
"""Wrapper that simulates grab() failures on specific frames."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._cap = original_video_capture(*args, **kwargs)
|
||||
self._current_frame = -1
|
||||
|
||||
def grab(self):
|
||||
self._current_frame += 1
|
||||
if self._current_frame in fail_on_frames:
|
||||
return False # Simulate failure
|
||||
return self._cap.grab()
|
||||
|
||||
def retrieve(self):
|
||||
return self._cap.retrieve()
|
||||
|
||||
def get(self, prop):
|
||||
return self._cap.get(prop)
|
||||
|
||||
def isOpened(self):
|
||||
return self._cap.isOpened()
|
||||
|
||||
def release(self):
|
||||
return self._cap.release()
|
||||
|
||||
# Patch cv2.VideoCapture
|
||||
m.setattr(cv2, "VideoCapture", MockVideoCapture)
|
||||
|
||||
loader = VIDEO_LOADER_REGISTRY.load("opencv")
|
||||
|
||||
# Use num_frames=8 which samples: [0, 3, 7, 10, 14, 17, 21, 25]
|
||||
# Frame 3: mocked failure, recovery window [3, 7) -> use frame 4
|
||||
# Frame 10: mocked failure, recovery window [10, 14) -> use frame 11
|
||||
# Frame 17: real corruption, recovery window [17, 21) -> use frame 18
|
||||
|
||||
# Test WITHOUT recovery - should have fewer frames due to failures
|
||||
frames_no_recovery, meta_no = loader.load_bytes(
|
||||
video_data, num_frames=8, frame_recovery=False
|
||||
)
|
||||
|
||||
# Test WITH recovery - should recover using next valid frames
|
||||
frames_with_recovery, meta_yes = loader.load_bytes(
|
||||
video_data, num_frames=8, frame_recovery=True
|
||||
)
|
||||
|
||||
# With recovery should have MORE frames than without
|
||||
# Without: 5 frames (3, 10, 17 all fail)
|
||||
# With: 8 frames (all recovered)
|
||||
assert frames_with_recovery.shape[0] > frames_no_recovery.shape[0], (
|
||||
f"Recovery should produce more frames. "
|
||||
f"Without: {frames_no_recovery.shape[0]}, "
|
||||
f"With: {frames_with_recovery.shape[0]}"
|
||||
)
|
||||
|
||||
# Verify metadata consistency
|
||||
assert frames_no_recovery.shape[0] == len(meta_no["frames_indices"])
|
||||
assert frames_with_recovery.shape[0] == len(meta_yes["frames_indices"])
|
||||
|
||||
# Verify temporal order is preserved
|
||||
assert meta_yes["frames_indices"] == sorted(meta_yes["frames_indices"])
|
||||
|
||||
|
||||
def test_video_recovery_with_corrupted_file(monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
Test frame recovery with an actual corrupted video file using sparse sampling.
|
||||
|
||||
This test uses corrupted.mp4 which has genuine H.264 codec errors on
|
||||
frame 17. With num_frames=8, the target frames are [0, 3, 7, 10, 14, 17, 21, 25].
|
||||
Frame 17 is corrupted but frames 18-20 are readable, so recovery can use
|
||||
frame 18 to fill in for the failed frame 17.
|
||||
|
||||
This test verifies:
|
||||
1. Without recovery: frame 17 is skipped (7 frames loaded)
|
||||
2. With recovery: frame 18 fills in for frame 17 (8 frames loaded)
|
||||
3. Recovery produces MORE frames than without recovery
|
||||
4. Metadata is consistent with loaded frames
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
|
||||
|
||||
corrupted_video_path = ASSETS_DIR / "corrupted.mp4"
|
||||
|
||||
with open(corrupted_video_path, "rb") as f:
|
||||
video_data = f.read()
|
||||
|
||||
loader = VIDEO_LOADER_REGISTRY.load("opencv")
|
||||
|
||||
# Use num_frames=8 which makes frame 17 a target with recovery window [17, 21)
|
||||
# Target frames: [0, 3, 7, 10, 14, 17, 21, 25]
|
||||
# Frame 17 is corrupted, but frames 18-20 are readable for recovery
|
||||
|
||||
# Test without recovery - frame 17 will be skipped
|
||||
frames_no_recovery, meta_no_recovery = loader.load_bytes(
|
||||
video_data, num_frames=8, frame_recovery=False
|
||||
)
|
||||
|
||||
# Test with recovery - frame 18 should fill in for frame 17
|
||||
frames_with_recovery, meta_with_recovery = loader.load_bytes(
|
||||
video_data, num_frames=8, frame_recovery=True
|
||||
)
|
||||
|
||||
# Verify metadata consistency for both modes
|
||||
assert frames_no_recovery.shape[0] == len(meta_no_recovery["frames_indices"]), (
|
||||
"Frame count must match indices without recovery"
|
||||
)
|
||||
assert frames_with_recovery.shape[0] == len(
|
||||
meta_with_recovery["frames_indices"]
|
||||
), "Frame count must match indices with recovery"
|
||||
|
||||
# KEY ASSERTION: Recovery should produce MORE frames than without recovery
|
||||
# Without recovery: 7 frames (frame 17 skipped)
|
||||
# With recovery: 8 frames (frame 18 used for frame 17)
|
||||
assert frames_with_recovery.shape[0] > frames_no_recovery.shape[0], (
|
||||
f"Recovery should produce more frames with sparse sampling. "
|
||||
f"Got {frames_with_recovery.shape[0]} with recovery vs "
|
||||
f"{frames_no_recovery.shape[0]} without"
|
||||
)
|
||||
|
||||
# Verify we got all 8 requested frames with recovery
|
||||
assert frames_with_recovery.shape[0] == 8, (
|
||||
f"With recovery, should load all 8 requested frames. "
|
||||
f"Got {frames_with_recovery.shape[0]}"
|
||||
)
|
||||
|
||||
# Verify the video metadata is correct
|
||||
expected_total_frames = 26
|
||||
assert meta_with_recovery["total_num_frames"] == expected_total_frames, (
|
||||
f"Expected {expected_total_frames} total frames in metadata"
|
||||
)
|
||||
|
||||
|
||||
def test_video_recovery_dynamic_backend(monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
Test that frame_recovery works with the dynamic video backend.
|
||||
|
||||
The dynamic backend samples frames based on fps/duration rather than
|
||||
loading all frames. This test verifies recovery works in that context.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv_dynamic")
|
||||
|
||||
corrupted_video_path = ASSETS_DIR / "corrupted.mp4"
|
||||
|
||||
with open(corrupted_video_path, "rb") as f:
|
||||
video_data = f.read()
|
||||
|
||||
loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic")
|
||||
|
||||
# Test without recovery
|
||||
frames_no_recovery, meta_no = loader.load_bytes(
|
||||
video_data, fps=2, max_duration=10, frame_recovery=False
|
||||
)
|
||||
|
||||
# Test with frame_recovery enabled
|
||||
frames_with_recovery, meta_with = loader.load_bytes(
|
||||
video_data, fps=2, max_duration=10, frame_recovery=True
|
||||
)
|
||||
|
||||
# Verify basic properties
|
||||
assert frames_no_recovery.shape[0] > 0, (
|
||||
"Should load some frames without recovery"
|
||||
)
|
||||
assert frames_with_recovery.shape[0] > 0, (
|
||||
"Should load some frames with recovery"
|
||||
)
|
||||
assert "do_sample_frames" in meta_with
|
||||
assert meta_with["do_sample_frames"] is False # Dynamic backend always False
|
||||
assert frames_with_recovery.shape[0] == len(meta_with["frames_indices"])
|
||||
|
||||
# Key assertion: recovery should help when corrupted frames are sampled
|
||||
# We expect recovery to produce >= frames than without recovery
|
||||
assert frames_with_recovery.shape[0] >= frames_no_recovery.shape[0], (
|
||||
f"Recovery should produce at least as many frames. "
|
||||
f"Got {frames_with_recovery.shape[0]} with recovery vs "
|
||||
f"{frames_no_recovery.shape[0]} without"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user