[videoloader] implement glm46v video loader (#44417)
Signed-off-by: JaredforReal <w13431838023@gmail.com>
This commit is contained in:
@@ -11,8 +11,11 @@ from vllm.assets.base import get_vllm_public_assets
|
||||
from vllm.multimodal.video import (
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
DynamicVideoBackend,
|
||||
GLM46VVideoBackend,
|
||||
Molmo2VideoBackend,
|
||||
VideoLoader,
|
||||
VideoSourceMetadata,
|
||||
VideoTargetMetadata,
|
||||
get_video_loader_backend_for_processor,
|
||||
)
|
||||
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
|
||||
@@ -76,6 +79,11 @@ def test_video_loader_type_doesnt_exist():
|
||||
DynamicVideoBackend,
|
||||
id="glm4v",
|
||||
),
|
||||
pytest.param(
|
||||
"zai-org/GLM-4.6V-Flash",
|
||||
GLM46VVideoBackend,
|
||||
id="glm46v",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_video_processor_from_model_repo(
|
||||
@@ -527,6 +535,20 @@ def test_pyav_backend_returns_target_frames_not_keyframes():
|
||||
60,
|
||||
id="pyav_dynamic-exceeds_max_duration",
|
||||
),
|
||||
# glm46v dynamic FPS (1800 frames @ 30fps = 60s)
|
||||
# 60s falls in (30, 300] → target_fps=1.0, extract_t = 60*1.0*2 = 120
|
||||
pytest.param(
|
||||
"glm46v",
|
||||
{"backend": "opencv"},
|
||||
120,
|
||||
id="glm46v-60s",
|
||||
),
|
||||
pytest.param(
|
||||
"glm46v",
|
||||
{"backend": "pyav"},
|
||||
120,
|
||||
id="glm46v-pyav-60s",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_video_loader_frames_sampling(
|
||||
@@ -548,3 +570,114 @@ def test_video_loader_frames_sampling(
|
||||
assert frames.ndim == 4
|
||||
assert frames.shape[3] == 3 # RGB
|
||||
assert frames.shape[0] == expected_num_frames
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# GLM-4.6V Dynamic FPS Threshold Tests
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"duration, original_fps, total_frames, temporal_patch_size, expected_extract_t",
|
||||
[
|
||||
# Short video ≤30s → target_fps=3.0
|
||||
# extract_t = 10 * 3.0 * 2 = 60
|
||||
pytest.param(10, 30, 300, 2, 60, id="short-10s"),
|
||||
# Exactly at boundary → target_fps=3.0
|
||||
# extract_t = 30 * 3.0 * 2 = 180
|
||||
pytest.param(30, 30, 900, 2, 180, id="boundary-30s"),
|
||||
# Medium video → target_fps=1.0
|
||||
# extract_t = 60 * 1.0 * 2 = 120
|
||||
pytest.param(60, 30, 1800, 2, 120, id="medium-60s"),
|
||||
# Medium boundary → target_fps=1.0
|
||||
# extract_t = 300 * 1.0 * 2 = 600
|
||||
pytest.param(300, 30, 9000, 2, 600, id="boundary-300s"),
|
||||
# Long video → target_fps=0.5
|
||||
# extract_t = 600 * 0.5 * 2 = 600
|
||||
pytest.param(600, 30, 18000, 2, 600, id="long-600s"),
|
||||
# Very long video, capped by _MAX_FRAME_COUNT_DYNAMIC=640
|
||||
# extract_t = min(2400 * 0.5 * 2, 640) = min(2400, 640) = 640
|
||||
pytest.param(2400, 30, 72000, 2, 640, id="long-capped-640"),
|
||||
# Duration exceeds _MAX_DURATION=2400
|
||||
# effective_duration = min(5000, 2400) = 2400, target_fps=0.5
|
||||
# extract_t = min(2400 * 0.5 * 2, 640) = 640
|
||||
pytest.param(5000, 30, 150000, 2, 640, id="exceeds-max-duration"),
|
||||
# temporal_patch_size=4
|
||||
# extract_t = 60 * 1.0 * 4 = 240
|
||||
pytest.param(60, 30, 1800, 4, 240, id="medium-patch-size-4"),
|
||||
# temporal_patch_size=1
|
||||
# extract_t = 60 * 1.0 * 1 = 60
|
||||
pytest.param(60, 30, 1800, 1, 60, id="medium-patch-size-1"),
|
||||
],
|
||||
)
|
||||
def test_glm46v_dynamic_fps_thresholds(
|
||||
duration: int,
|
||||
original_fps: int,
|
||||
total_frames: int,
|
||||
temporal_patch_size: int,
|
||||
expected_extract_t: int,
|
||||
):
|
||||
"""Test GLM-4.6V dynamic FPS threshold selection and frame count."""
|
||||
source = VideoSourceMetadata(
|
||||
total_frames_num=total_frames,
|
||||
original_fps=original_fps,
|
||||
duration=duration,
|
||||
)
|
||||
target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1)
|
||||
|
||||
indices = GLM46VVideoBackend.compute_frames_index_to_sample(
|
||||
source, target, temporal_patch_size=temporal_patch_size
|
||||
)
|
||||
|
||||
# Frame count should match expected (may be +1 from even padding)
|
||||
assert len(indices) in (expected_extract_t, expected_extract_t + 1), (
|
||||
f"Expected ~{expected_extract_t} frames, got {len(indices)}"
|
||||
)
|
||||
|
||||
# Frame count must be even
|
||||
assert len(indices) % 2 == 0, f"Frame count must be even, got {len(indices)}"
|
||||
|
||||
# All indices must be valid
|
||||
assert all(0 <= idx < total_frames for idx in indices), (
|
||||
f"Indices out of range [0, {total_frames})"
|
||||
)
|
||||
|
||||
# Indices must be sorted and deduplicated
|
||||
assert indices == sorted(set(indices)), "Indices must be sorted and deduplicated"
|
||||
|
||||
|
||||
def test_glm46v_even_frame_count_enforcement():
|
||||
"""Test that GLM-4.6V always returns an even number of frames."""
|
||||
target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1)
|
||||
# 5-second video at 30fps → 150 frames
|
||||
# extract_t = 5 * 3.0 * 2 = 30 (even, no padding needed)
|
||||
source_even = VideoSourceMetadata(total_frames_num=150, original_fps=30, duration=5)
|
||||
indices_even = GLM46VVideoBackend.compute_frames_index_to_sample(
|
||||
source_even, target
|
||||
)
|
||||
assert len(indices_even) % 2 == 0
|
||||
|
||||
# 3-second video at 30fps → 90 frames
|
||||
# extract_t = 3 * 3.0 * 2 = 18 (even, no padding needed)
|
||||
source_even2 = VideoSourceMetadata(total_frames_num=90, original_fps=30, duration=3)
|
||||
indices_even2 = GLM46VVideoBackend.compute_frames_index_to_sample(
|
||||
source_even2, target
|
||||
)
|
||||
assert len(indices_even2) % 2 == 0
|
||||
|
||||
|
||||
def test_glm46v_duration_estimation_from_fps():
|
||||
"""Test GLM-4.6V handles missing duration by estimating from fps."""
|
||||
target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1)
|
||||
# duration=0 → estimated from total_frames / fps
|
||||
# (89 / 30) + 1 ≈ 4s → target_fps=3.0, extract_t = 4 * 3.0 * 2 = 24
|
||||
source_no_duration = VideoSourceMetadata(
|
||||
total_frames_num=90, original_fps=30, duration=0
|
||||
)
|
||||
indices = GLM46VVideoBackend.compute_frames_index_to_sample(
|
||||
source_no_duration, target
|
||||
)
|
||||
|
||||
assert len(indices) > 0
|
||||
assert len(indices) % 2 == 0
|
||||
assert all(0 <= idx < 90 for idx in indices)
|
||||
|
||||
@@ -696,6 +696,132 @@ class DynamicVideoBackend(VideoBackend):
|
||||
)
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register(
|
||||
"glm46v",
|
||||
video_processor="Glm46VVideoProcessor",
|
||||
)
|
||||
class GLM46VVideoBackend(VideoBackend):
|
||||
"""GLM-4.6V dynamic FPS video backend.
|
||||
|
||||
Faithfully replicates the frame sampling logic from transformers'
|
||||
``Glm46VVideoProcessor.sample_frames``:
|
||||
|
||||
- Dynamic FPS thresholds based on effective video duration:
|
||||
``{≤30s: 3fps, ≤300s: 1fps, >300s: 0.5fps}``
|
||||
- ``temporal_patch_size`` multiplier (default 2) applied to extract count
|
||||
- Duration capped at 2400s, frame count capped at 640
|
||||
- Even frame count enforced (append last frame if odd)
|
||||
"""
|
||||
|
||||
# Match transformers defaults
|
||||
_DYNAMIC_FPS_THRESHOLDS: ClassVar[dict[int, float]] = {
|
||||
30: 3.0,
|
||||
300: 1.0,
|
||||
2400: 0.5,
|
||||
}
|
||||
_MAX_FRAME_COUNT_DYNAMIC: ClassVar[int] = 640
|
||||
_MAX_DURATION: ClassVar[int] = 2400
|
||||
|
||||
@classmethod
|
||||
def compute_frames_index_to_sample(
|
||||
cls,
|
||||
source: VideoSourceMetadata,
|
||||
target: VideoTargetMetadata,
|
||||
**kwargs,
|
||||
) -> list[int]:
|
||||
# Refer to:
|
||||
# https://github.com/huggingface/transformers/blob/v5.9.0/src/transformers/models/glm46v/video_processing_glm46v.py#L97-L102
|
||||
total_frames_num = source.total_frames_num
|
||||
original_fps = source.original_fps
|
||||
duration = source.duration
|
||||
temporal_patch_size = kwargs.get("temporal_patch_size", 2)
|
||||
|
||||
max_frame_idx = total_frames_num - 1
|
||||
|
||||
# Estimate duration from frame count and fps when not reported
|
||||
if not duration and original_fps > 0:
|
||||
duration = round(max_frame_idx / original_fps) + 1
|
||||
|
||||
effective_duration = min(duration, cls._MAX_DURATION)
|
||||
|
||||
# Select target_fps from dynamic thresholds
|
||||
if effective_duration <= 30:
|
||||
target_fps = cls._DYNAMIC_FPS_THRESHOLDS[30]
|
||||
elif effective_duration <= 300:
|
||||
target_fps = cls._DYNAMIC_FPS_THRESHOLDS[300]
|
||||
else:
|
||||
target_fps = cls._DYNAMIC_FPS_THRESHOLDS[2400]
|
||||
|
||||
extract_t = int(effective_duration * target_fps * temporal_patch_size)
|
||||
extract_t = min(extract_t, cls._MAX_FRAME_COUNT_DYNAMIC)
|
||||
|
||||
duration_per_frame = 1 / original_fps if original_fps > 0 else 0
|
||||
timestamps = [i * duration_per_frame for i in range(total_frames_num)]
|
||||
max_second = int(duration) if duration else 0
|
||||
|
||||
if total_frames_num < extract_t:
|
||||
frame_indices = np.linspace(
|
||||
0, total_frames_num - 1, extract_t, dtype=int
|
||||
).tolist()
|
||||
else:
|
||||
frame_indices = []
|
||||
current_second = 0.0
|
||||
inv_fps = 1 / (temporal_patch_size * target_fps)
|
||||
for frame_index in range(total_frames_num):
|
||||
if timestamps[frame_index] >= current_second:
|
||||
current_second += inv_fps
|
||||
frame_indices.append(frame_index)
|
||||
if current_second >= max_second:
|
||||
break
|
||||
|
||||
if len(frame_indices) < extract_t:
|
||||
if len(frame_indices) == 0:
|
||||
start, end = 0, max(total_frames_num - 1, 0)
|
||||
else:
|
||||
start, end = frame_indices[0], frame_indices[-1]
|
||||
frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
|
||||
elif len(frame_indices) > extract_t:
|
||||
frame_indices = np.linspace(
|
||||
0, total_frames_num - 1, extract_t, dtype=int
|
||||
).tolist()
|
||||
|
||||
# Deduplicate
|
||||
seen: set[int] = set()
|
||||
uniq: list[int] = []
|
||||
for idx in frame_indices:
|
||||
if idx not in seen:
|
||||
seen.add(idx)
|
||||
uniq.append(idx)
|
||||
|
||||
# Ensure even frame count
|
||||
if len(uniq) & 1:
|
||||
uniq.append(uniq[-1])
|
||||
|
||||
return uniq
|
||||
|
||||
@classmethod
|
||||
def load_bytes(
|
||||
cls,
|
||||
data: bytes,
|
||||
num_frames: int = -1,
|
||||
fps: int = -1,
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal["opencv", "pyav"] = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
data,
|
||||
num_frames=num_frames,
|
||||
fps=fps,
|
||||
max_duration=max_duration,
|
||||
frame_recovery=frame_recovery,
|
||||
backend=backend,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register(
|
||||
"glmga",
|
||||
video_processor="GlmgaVideoProcessor",
|
||||
|
||||
Reference in New Issue
Block a user