diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index b82bcec9dc..15a6373932 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -11,8 +11,11 @@ from vllm.assets.base import get_vllm_public_assets from vllm.multimodal.video import ( VIDEO_LOADER_REGISTRY, DynamicVideoBackend, + GLM46VVideoBackend, Molmo2VideoBackend, VideoLoader, + VideoSourceMetadata, + VideoTargetMetadata, get_video_loader_backend_for_processor, ) from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config @@ -76,6 +79,11 @@ def test_video_loader_type_doesnt_exist(): DynamicVideoBackend, id="glm4v", ), + pytest.param( + "zai-org/GLM-4.6V-Flash", + GLM46VVideoBackend, + id="glm46v", + ), ], ) def test_video_processor_from_model_repo( @@ -527,6 +535,20 @@ def test_pyav_backend_returns_target_frames_not_keyframes(): 60, id="pyav_dynamic-exceeds_max_duration", ), + # glm46v dynamic FPS (1800 frames @ 30fps = 60s) + # 60s falls in (30, 300] → target_fps=1.0, extract_t = 60*1.0*2 = 120 + pytest.param( + "glm46v", + {"backend": "opencv"}, + 120, + id="glm46v-60s", + ), + pytest.param( + "glm46v", + {"backend": "pyav"}, + 120, + id="glm46v-pyav-60s", + ), ], ) def test_video_loader_frames_sampling( @@ -548,3 +570,114 @@ def test_video_loader_frames_sampling( assert frames.ndim == 4 assert frames.shape[3] == 3 # RGB assert frames.shape[0] == expected_num_frames + + +# ============================================================================ +# GLM-4.6V Dynamic FPS Threshold Tests +# ============================================================================ + + +@pytest.mark.parametrize( + "duration, original_fps, total_frames, temporal_patch_size, expected_extract_t", + [ + # Short video ≤30s → target_fps=3.0 + # extract_t = 10 * 3.0 * 2 = 60 + pytest.param(10, 30, 300, 2, 60, id="short-10s"), + # Exactly at boundary → target_fps=3.0 + # extract_t = 30 * 3.0 * 2 = 180 + pytest.param(30, 30, 900, 2, 180, id="boundary-30s"), + # Medium video → target_fps=1.0 + # extract_t = 60 * 1.0 * 2 = 120 + pytest.param(60, 30, 1800, 2, 120, id="medium-60s"), + # Medium boundary → target_fps=1.0 + # extract_t = 300 * 1.0 * 2 = 600 + pytest.param(300, 30, 9000, 2, 600, id="boundary-300s"), + # Long video → target_fps=0.5 + # extract_t = 600 * 0.5 * 2 = 600 + pytest.param(600, 30, 18000, 2, 600, id="long-600s"), + # Very long video, capped by _MAX_FRAME_COUNT_DYNAMIC=640 + # extract_t = min(2400 * 0.5 * 2, 640) = min(2400, 640) = 640 + pytest.param(2400, 30, 72000, 2, 640, id="long-capped-640"), + # Duration exceeds _MAX_DURATION=2400 + # effective_duration = min(5000, 2400) = 2400, target_fps=0.5 + # extract_t = min(2400 * 0.5 * 2, 640) = 640 + pytest.param(5000, 30, 150000, 2, 640, id="exceeds-max-duration"), + # temporal_patch_size=4 + # extract_t = 60 * 1.0 * 4 = 240 + pytest.param(60, 30, 1800, 4, 240, id="medium-patch-size-4"), + # temporal_patch_size=1 + # extract_t = 60 * 1.0 * 1 = 60 + pytest.param(60, 30, 1800, 1, 60, id="medium-patch-size-1"), + ], +) +def test_glm46v_dynamic_fps_thresholds( + duration: int, + original_fps: int, + total_frames: int, + temporal_patch_size: int, + expected_extract_t: int, +): + """Test GLM-4.6V dynamic FPS threshold selection and frame count.""" + source = VideoSourceMetadata( + total_frames_num=total_frames, + original_fps=original_fps, + duration=duration, + ) + target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1) + + indices = GLM46VVideoBackend.compute_frames_index_to_sample( + source, target, temporal_patch_size=temporal_patch_size + ) + + # Frame count should match expected (may be +1 from even padding) + assert len(indices) in (expected_extract_t, expected_extract_t + 1), ( + f"Expected ~{expected_extract_t} frames, got {len(indices)}" + ) + + # Frame count must be even + assert len(indices) % 2 == 0, f"Frame count must be even, got {len(indices)}" + + # All indices must be valid + assert all(0 <= idx < total_frames for idx in indices), ( + f"Indices out of range [0, {total_frames})" + ) + + # Indices must be sorted and deduplicated + assert indices == sorted(set(indices)), "Indices must be sorted and deduplicated" + + +def test_glm46v_even_frame_count_enforcement(): + """Test that GLM-4.6V always returns an even number of frames.""" + target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1) + # 5-second video at 30fps → 150 frames + # extract_t = 5 * 3.0 * 2 = 30 (even, no padding needed) + source_even = VideoSourceMetadata(total_frames_num=150, original_fps=30, duration=5) + indices_even = GLM46VVideoBackend.compute_frames_index_to_sample( + source_even, target + ) + assert len(indices_even) % 2 == 0 + + # 3-second video at 30fps → 90 frames + # extract_t = 3 * 3.0 * 2 = 18 (even, no padding needed) + source_even2 = VideoSourceMetadata(total_frames_num=90, original_fps=30, duration=3) + indices_even2 = GLM46VVideoBackend.compute_frames_index_to_sample( + source_even2, target + ) + assert len(indices_even2) % 2 == 0 + + +def test_glm46v_duration_estimation_from_fps(): + """Test GLM-4.6V handles missing duration by estimating from fps.""" + target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1) + # duration=0 → estimated from total_frames / fps + # (89 / 30) + 1 ≈ 4s → target_fps=3.0, extract_t = 4 * 3.0 * 2 = 24 + source_no_duration = VideoSourceMetadata( + total_frames_num=90, original_fps=30, duration=0 + ) + indices = GLM46VVideoBackend.compute_frames_index_to_sample( + source_no_duration, target + ) + + assert len(indices) > 0 + assert len(indices) % 2 == 0 + assert all(0 <= idx < 90 for idx in indices) diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index 1324e79c5c..03e2b0a85c 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -696,6 +696,132 @@ class DynamicVideoBackend(VideoBackend): ) +@VIDEO_LOADER_REGISTRY.register( + "glm46v", + video_processor="Glm46VVideoProcessor", +) +class GLM46VVideoBackend(VideoBackend): + """GLM-4.6V dynamic FPS video backend. + + Faithfully replicates the frame sampling logic from transformers' + ``Glm46VVideoProcessor.sample_frames``: + + - Dynamic FPS thresholds based on effective video duration: + ``{≤30s: 3fps, ≤300s: 1fps, >300s: 0.5fps}`` + - ``temporal_patch_size`` multiplier (default 2) applied to extract count + - Duration capped at 2400s, frame count capped at 640 + - Even frame count enforced (append last frame if odd) + """ + + # Match transformers defaults + _DYNAMIC_FPS_THRESHOLDS: ClassVar[dict[int, float]] = { + 30: 3.0, + 300: 1.0, + 2400: 0.5, + } + _MAX_FRAME_COUNT_DYNAMIC: ClassVar[int] = 640 + _MAX_DURATION: ClassVar[int] = 2400 + + @classmethod + def compute_frames_index_to_sample( + cls, + source: VideoSourceMetadata, + target: VideoTargetMetadata, + **kwargs, + ) -> list[int]: + # Refer to: + # https://github.com/huggingface/transformers/blob/v5.9.0/src/transformers/models/glm46v/video_processing_glm46v.py#L97-L102 + total_frames_num = source.total_frames_num + original_fps = source.original_fps + duration = source.duration + temporal_patch_size = kwargs.get("temporal_patch_size", 2) + + max_frame_idx = total_frames_num - 1 + + # Estimate duration from frame count and fps when not reported + if not duration and original_fps > 0: + duration = round(max_frame_idx / original_fps) + 1 + + effective_duration = min(duration, cls._MAX_DURATION) + + # Select target_fps from dynamic thresholds + if effective_duration <= 30: + target_fps = cls._DYNAMIC_FPS_THRESHOLDS[30] + elif effective_duration <= 300: + target_fps = cls._DYNAMIC_FPS_THRESHOLDS[300] + else: + target_fps = cls._DYNAMIC_FPS_THRESHOLDS[2400] + + extract_t = int(effective_duration * target_fps * temporal_patch_size) + extract_t = min(extract_t, cls._MAX_FRAME_COUNT_DYNAMIC) + + duration_per_frame = 1 / original_fps if original_fps > 0 else 0 + timestamps = [i * duration_per_frame for i in range(total_frames_num)] + max_second = int(duration) if duration else 0 + + if total_frames_num < extract_t: + frame_indices = np.linspace( + 0, total_frames_num - 1, extract_t, dtype=int + ).tolist() + else: + frame_indices = [] + current_second = 0.0 + inv_fps = 1 / (temporal_patch_size * target_fps) + for frame_index in range(total_frames_num): + if timestamps[frame_index] >= current_second: + current_second += inv_fps + frame_indices.append(frame_index) + if current_second >= max_second: + break + + if len(frame_indices) < extract_t: + if len(frame_indices) == 0: + start, end = 0, max(total_frames_num - 1, 0) + else: + start, end = frame_indices[0], frame_indices[-1] + frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist() + elif len(frame_indices) > extract_t: + frame_indices = np.linspace( + 0, total_frames_num - 1, extract_t, dtype=int + ).tolist() + + # Deduplicate + seen: set[int] = set() + uniq: list[int] = [] + for idx in frame_indices: + if idx not in seen: + seen.add(idx) + uniq.append(idx) + + # Ensure even frame count + if len(uniq) & 1: + uniq.append(uniq[-1]) + + return uniq + + @classmethod + def load_bytes( + cls, + data: bytes, + num_frames: int = -1, + fps: int = -1, + max_duration: int = 300, + frame_recovery: bool = False, + *, + backend: Literal["opencv", "pyav"] = "opencv", + **kwargs, + ) -> tuple[npt.NDArray, dict[str, Any]]: + return super().load_bytes( + data, + num_frames=num_frames, + fps=fps, + max_duration=max_duration, + frame_recovery=frame_recovery, + backend=backend, + **kwargs, + ) + + @VIDEO_LOADER_REGISTRY.register( "glmga", video_processor="GlmgaVideoProcessor",