[Model] add GLM-5.3-Flash support (#53906)
This commit is contained in:
@@ -20,6 +20,7 @@ from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
DynamicVideoBackend,
|
||||
Glm5NextVideoBackend,
|
||||
GLM46VVideoBackend,
|
||||
Molmo2VideoBackend,
|
||||
Qwen2VLVideoBackend,
|
||||
@@ -1470,3 +1471,233 @@ def test_glm46v_duration_estimation_from_fps():
|
||||
assert len(indices) > 0
|
||||
assert len(indices) % 2 == 0
|
||||
assert all(0 <= idx < 90 for idx in indices)
|
||||
|
||||
|
||||
def test_glm5next_backend_selected_for_processor():
|
||||
"""Glm5NextVideoProcessor maps to the glm5next loader so only the
|
||||
sampled frames are decoded instead of the whole container. Both the
|
||||
borrowed-config spelling and the dedicated Glm5next class name (landing
|
||||
with the new checkpoint) must resolve."""
|
||||
for name in ("Glm5NextVideoProcessor", "Glm5nextVideoProcessor"):
|
||||
assert VIDEO_LOADER_REGISTRY.get_backend_for_video_processor(name) == "glm5next"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("total_frames", "original_fps", "duration", "fps", "max_frames"),
|
||||
[
|
||||
(900, 30.0, 30.0, -1, None), # 30s at flat 2.0 raw fps -> 60 frames
|
||||
(3000, 30.0, 100.0, -1, None),
|
||||
(72000, 30.0, 2400.0, -1, None), # 2048 cap
|
||||
(48, 2.0, 24.0, -1, None),
|
||||
(7, 30.0, 10.0, -1, None), # short video -> uniform spread + dedup
|
||||
(300, 25.0, 0, -1, None), # duration derived from frame count
|
||||
(900, 30.0, 30.0, 4, None), # request fps override (raw fps)
|
||||
(900, 30.0, 30.0, -1, 16), # request max_frames override
|
||||
],
|
||||
)
|
||||
def test_glm5next_backend_indices_match_sampler(
|
||||
total_frames, original_fps, duration, fps, max_frames
|
||||
):
|
||||
"""The loader must select exactly the frames the processor's sampler
|
||||
would, with target.fps mapping onto the raw-fps override."""
|
||||
from vllm.transformers_utils.processors.glm5next import (
|
||||
glm_sample_frame_indices,
|
||||
)
|
||||
|
||||
source = VideoSourceMetadata(
|
||||
total_frames_num=total_frames, original_fps=original_fps, duration=duration
|
||||
)
|
||||
target = VideoTargetMetadata(num_frames=-1, fps=fps, max_duration=-1)
|
||||
|
||||
indices = Glm5NextVideoBackend.compute_frames_index_to_sample(
|
||||
source, target, max_frames=max_frames
|
||||
)
|
||||
|
||||
assert indices == glm_sample_frame_indices(
|
||||
total_frames,
|
||||
original_fps,
|
||||
duration,
|
||||
target_fps=fps if fps > 0 else None,
|
||||
max_frame_count=max_frames,
|
||||
)
|
||||
assert len(indices) % 2 == 0
|
||||
assert indices == sorted(indices) # pair padding may repeat the last frame
|
||||
assert all(0 <= idx < total_frames for idx in indices)
|
||||
|
||||
|
||||
def test_glm5next_backend_metadata_contract():
|
||||
"""create_hf_metadata reports the subset so the processor skips
|
||||
re-sampling (do_sample_frames=False) and keeps the original totals."""
|
||||
source = VideoSourceMetadata(total_frames_num=900, original_fps=30.0, duration=30.0)
|
||||
target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1)
|
||||
indices = Glm5NextVideoBackend.compute_frames_index_to_sample(source, target)
|
||||
|
||||
metadata = Glm5NextVideoBackend.create_hf_metadata(
|
||||
source, indices, video_backend="glm5next"
|
||||
)
|
||||
assert metadata["do_sample_frames"] is False
|
||||
assert metadata["frames_indices"] == indices
|
||||
assert metadata["total_num_frames"] == 900
|
||||
assert metadata["fps"] == 30.0
|
||||
assert metadata["duration"] == 30.0
|
||||
|
||||
# A fully-selected source keeps do_sample_frames=True so the processor's
|
||||
# sampler takes over on the complete frame set.
|
||||
full = list(range(48))
|
||||
assert (
|
||||
Glm5NextVideoBackend.create_hf_metadata(
|
||||
VideoSourceMetadata(total_frames_num=48, original_fps=2.0, duration=24.0),
|
||||
full,
|
||||
video_backend="glm5next",
|
||||
)["do_sample_frames"]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
def _write_gray_video(tmp_path, total_frames, fps, size=(32, 32)):
|
||||
"""Synthetic clip whose frame i is flat gray level i (near-lossless under
|
||||
mp4v), so a decoded frame's level maps back to its source index."""
|
||||
cv2 = pytest.importorskip("cv2")
|
||||
|
||||
path = tmp_path / f"gray_{total_frames}_{fps}.mp4"
|
||||
writer = cv2.VideoWriter(
|
||||
str(path), cv2.VideoWriter_fourcc(*"mp4v"), fps, size, isColor=False
|
||||
)
|
||||
assert writer.isOpened()
|
||||
for i in range(total_frames):
|
||||
writer.write(np.full((*size, 1), i, dtype=np.uint8))
|
||||
writer.release()
|
||||
return path
|
||||
|
||||
|
||||
class _CountingCap:
|
||||
"""Proxy over a real capture that counts grab/seek decoding work."""
|
||||
|
||||
def __init__(self, cap):
|
||||
self._cap = cap
|
||||
self.grabs = 0
|
||||
self.seeks = 0
|
||||
|
||||
def get(self, prop):
|
||||
return self._cap.get(prop)
|
||||
|
||||
def grab(self):
|
||||
self.grabs += 1
|
||||
return self._cap.grab()
|
||||
|
||||
def set(self, prop, value):
|
||||
self.seeks += 1
|
||||
return self._cap.set(prop, value)
|
||||
|
||||
def read(self):
|
||||
return self._cap.read()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["opencv", "torchcodec"])
|
||||
def test_glm5next_backend_codec_parity(tmp_path, backend):
|
||||
"""Every codec samples the same GLM indices and decodes the same
|
||||
frames; the OpenCV seek reader and torchcodec batched index-exact decode
|
||||
must agree."""
|
||||
if backend == "torchcodec":
|
||||
pytest.importorskip("torchcodec")
|
||||
|
||||
from vllm.transformers_utils.processors.glm5next import (
|
||||
glm_sample_frame_indices,
|
||||
)
|
||||
|
||||
total_frames, fps = 120, 10
|
||||
path = _write_gray_video(tmp_path, total_frames, fps)
|
||||
# Dense default sampling (gap 5) and a sparse max_frames cap (gap 20).
|
||||
for max_frames in (None, 6):
|
||||
kwargs = {} if max_frames is None else {"max_frames": max_frames}
|
||||
expected = glm_sample_frame_indices(
|
||||
total_frames, float(fps), 12.0, max_frame_count=max_frames
|
||||
)
|
||||
|
||||
frames, metadata = Glm5NextVideoBackend.load_bytes(
|
||||
path.read_bytes(), backend=backend, **kwargs
|
||||
)
|
||||
|
||||
assert metadata["frames_indices"] == expected
|
||||
assert metadata["video_backend"].startswith(backend)
|
||||
assert len(frames) == len(expected)
|
||||
for i, idx in enumerate(expected):
|
||||
assert abs(round(float(np.asarray(frames[i]).mean())) - idx) <= 1
|
||||
|
||||
|
||||
def test_glm5next_backend_decodes_only_sampled_frames(tmp_path):
|
||||
"""End to end over a synthetic clip: load_bytes returns exactly the
|
||||
sampler's frame count, with the right frame content at each index."""
|
||||
pytest.importorskip("cv2")
|
||||
|
||||
total_frames, fps = 60, 10
|
||||
path = _write_gray_video(tmp_path, total_frames, fps)
|
||||
|
||||
from vllm.transformers_utils.processors.glm5next import (
|
||||
glm_sample_frame_indices,
|
||||
)
|
||||
|
||||
expected = glm_sample_frame_indices(total_frames, float(fps), 6.0)
|
||||
|
||||
frames, metadata = Glm5NextVideoBackend.load_bytes(path.read_bytes())
|
||||
|
||||
assert len(frames) == len(expected)
|
||||
assert metadata["frames_indices"] == expected
|
||||
assert metadata["do_sample_frames"] is (len(expected) == total_frames)
|
||||
# Flat gray frames survive mp4v near-losslessly: each decoded frame's
|
||||
# level maps back to its source frame index.
|
||||
for frame, idx in zip(frames, expected):
|
||||
decoded_idx = round(float(np.asarray(frame).mean()))
|
||||
assert abs(decoded_idx - idx) <= 1
|
||||
|
||||
|
||||
def test_glm5next_read_frames_seeks_past_large_gaps(tmp_path):
|
||||
"""Sparse targets must not walk the container: the stock reader grabs
|
||||
every frame up to the last index; the GLM reader seeks instead."""
|
||||
cv2 = pytest.importorskip("cv2")
|
||||
|
||||
total_frames, fps = 200, 10
|
||||
path = _write_gray_video(tmp_path, total_frames, fps)
|
||||
targets = [0, 80, 160, 190] # gaps of 80/80/30 -> only 30 <= threshold
|
||||
|
||||
stock = cv2.VideoCapture(str(path))
|
||||
_, stock_indices = VideoBackend.read_frames(stock, targets, total_frames)
|
||||
stock.release()
|
||||
assert stock_indices == targets
|
||||
|
||||
cap = _CountingCap(cv2.VideoCapture(str(path)))
|
||||
frames, indices = Glm5NextVideoBackend.read_frames(cap, targets, total_frames)
|
||||
cap._cap.release()
|
||||
|
||||
assert indices == targets == stock_indices
|
||||
for frame, idx in zip(frames, targets):
|
||||
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
|
||||
# Walks only the sub-threshold 30-frame hop; the two 80-frame gaps are
|
||||
# seeks. The stock reader grabs all 190 preceding frames.
|
||||
assert cap.grabs <= 29
|
||||
assert cap.seeks == 3
|
||||
|
||||
|
||||
def test_glm5next_read_frames_dense_walk_matches_stock(tmp_path):
|
||||
"""Dense targets keep the sequential walk (seeking would be slower) and
|
||||
return the same frames as the stock reader."""
|
||||
cv2 = pytest.importorskip("cv2")
|
||||
|
||||
total_frames, fps = 120, 10
|
||||
path = _write_gray_video(tmp_path, total_frames, fps)
|
||||
targets = list(range(0, total_frames, 15)) # gaps of 15 -> all walking
|
||||
|
||||
stock = cv2.VideoCapture(str(path))
|
||||
stock_frames, stock_indices = VideoBackend.read_frames(stock, targets, total_frames)
|
||||
stock.release()
|
||||
|
||||
cap = _CountingCap(cv2.VideoCapture(str(path)))
|
||||
frames, indices = Glm5NextVideoBackend.read_frames(cap, targets, total_frames)
|
||||
cap._cap.release()
|
||||
|
||||
assert indices == stock_indices
|
||||
for frame, stock_frame, idx in zip(frames, stock_frames, targets):
|
||||
assert abs(float(np.asarray(frame).mean()) - float(stock_frame.mean())) <= 2.0
|
||||
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
|
||||
# One initial seek, then pure walking -- no re-seek churn.
|
||||
assert cap.seeks == 1
|
||||
|
||||
Reference in New Issue
Block a user