[Model] add GLM-5.3-Flash support (#53906)

This commit is contained in:
Jiangyun Zhu
2026-09-03 09:40:35 -07:00
committed by GitHub
parent da8ec28268
commit 98ed0856f3
94 changed files with 17927 additions and 266 deletions
+231
View File
@@ -20,6 +20,7 @@ from vllm.multimodal.video import (
PYNVVIDEOCODEC_VIDEO_BACKEND,
VIDEO_LOADER_REGISTRY,
DynamicVideoBackend,
Glm5NextVideoBackend,
GLM46VVideoBackend,
Molmo2VideoBackend,
Qwen2VLVideoBackend,
@@ -1470,3 +1471,233 @@ def test_glm46v_duration_estimation_from_fps():
assert len(indices) > 0
assert len(indices) % 2 == 0
assert all(0 <= idx < 90 for idx in indices)
def test_glm5next_backend_selected_for_processor():
"""Glm5NextVideoProcessor maps to the glm5next loader so only the
sampled frames are decoded instead of the whole container. Both the
borrowed-config spelling and the dedicated Glm5next class name (landing
with the new checkpoint) must resolve."""
for name in ("Glm5NextVideoProcessor", "Glm5nextVideoProcessor"):
assert VIDEO_LOADER_REGISTRY.get_backend_for_video_processor(name) == "glm5next"
@pytest.mark.parametrize(
("total_frames", "original_fps", "duration", "fps", "max_frames"),
[
(900, 30.0, 30.0, -1, None), # 30s at flat 2.0 raw fps -> 60 frames
(3000, 30.0, 100.0, -1, None),
(72000, 30.0, 2400.0, -1, None), # 2048 cap
(48, 2.0, 24.0, -1, None),
(7, 30.0, 10.0, -1, None), # short video -> uniform spread + dedup
(300, 25.0, 0, -1, None), # duration derived from frame count
(900, 30.0, 30.0, 4, None), # request fps override (raw fps)
(900, 30.0, 30.0, -1, 16), # request max_frames override
],
)
def test_glm5next_backend_indices_match_sampler(
total_frames, original_fps, duration, fps, max_frames
):
"""The loader must select exactly the frames the processor's sampler
would, with target.fps mapping onto the raw-fps override."""
from vllm.transformers_utils.processors.glm5next import (
glm_sample_frame_indices,
)
source = VideoSourceMetadata(
total_frames_num=total_frames, original_fps=original_fps, duration=duration
)
target = VideoTargetMetadata(num_frames=-1, fps=fps, max_duration=-1)
indices = Glm5NextVideoBackend.compute_frames_index_to_sample(
source, target, max_frames=max_frames
)
assert indices == glm_sample_frame_indices(
total_frames,
original_fps,
duration,
target_fps=fps if fps > 0 else None,
max_frame_count=max_frames,
)
assert len(indices) % 2 == 0
assert indices == sorted(indices) # pair padding may repeat the last frame
assert all(0 <= idx < total_frames for idx in indices)
def test_glm5next_backend_metadata_contract():
"""create_hf_metadata reports the subset so the processor skips
re-sampling (do_sample_frames=False) and keeps the original totals."""
source = VideoSourceMetadata(total_frames_num=900, original_fps=30.0, duration=30.0)
target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1)
indices = Glm5NextVideoBackend.compute_frames_index_to_sample(source, target)
metadata = Glm5NextVideoBackend.create_hf_metadata(
source, indices, video_backend="glm5next"
)
assert metadata["do_sample_frames"] is False
assert metadata["frames_indices"] == indices
assert metadata["total_num_frames"] == 900
assert metadata["fps"] == 30.0
assert metadata["duration"] == 30.0
# A fully-selected source keeps do_sample_frames=True so the processor's
# sampler takes over on the complete frame set.
full = list(range(48))
assert (
Glm5NextVideoBackend.create_hf_metadata(
VideoSourceMetadata(total_frames_num=48, original_fps=2.0, duration=24.0),
full,
video_backend="glm5next",
)["do_sample_frames"]
is True
)
def _write_gray_video(tmp_path, total_frames, fps, size=(32, 32)):
"""Synthetic clip whose frame i is flat gray level i (near-lossless under
mp4v), so a decoded frame's level maps back to its source index."""
cv2 = pytest.importorskip("cv2")
path = tmp_path / f"gray_{total_frames}_{fps}.mp4"
writer = cv2.VideoWriter(
str(path), cv2.VideoWriter_fourcc(*"mp4v"), fps, size, isColor=False
)
assert writer.isOpened()
for i in range(total_frames):
writer.write(np.full((*size, 1), i, dtype=np.uint8))
writer.release()
return path
class _CountingCap:
"""Proxy over a real capture that counts grab/seek decoding work."""
def __init__(self, cap):
self._cap = cap
self.grabs = 0
self.seeks = 0
def get(self, prop):
return self._cap.get(prop)
def grab(self):
self.grabs += 1
return self._cap.grab()
def set(self, prop, value):
self.seeks += 1
return self._cap.set(prop, value)
def read(self):
return self._cap.read()
@pytest.mark.parametrize("backend", ["opencv", "torchcodec"])
def test_glm5next_backend_codec_parity(tmp_path, backend):
"""Every codec samples the same GLM indices and decodes the same
frames; the OpenCV seek reader and torchcodec batched index-exact decode
must agree."""
if backend == "torchcodec":
pytest.importorskip("torchcodec")
from vllm.transformers_utils.processors.glm5next import (
glm_sample_frame_indices,
)
total_frames, fps = 120, 10
path = _write_gray_video(tmp_path, total_frames, fps)
# Dense default sampling (gap 5) and a sparse max_frames cap (gap 20).
for max_frames in (None, 6):
kwargs = {} if max_frames is None else {"max_frames": max_frames}
expected = glm_sample_frame_indices(
total_frames, float(fps), 12.0, max_frame_count=max_frames
)
frames, metadata = Glm5NextVideoBackend.load_bytes(
path.read_bytes(), backend=backend, **kwargs
)
assert metadata["frames_indices"] == expected
assert metadata["video_backend"].startswith(backend)
assert len(frames) == len(expected)
for i, idx in enumerate(expected):
assert abs(round(float(np.asarray(frames[i]).mean())) - idx) <= 1
def test_glm5next_backend_decodes_only_sampled_frames(tmp_path):
"""End to end over a synthetic clip: load_bytes returns exactly the
sampler's frame count, with the right frame content at each index."""
pytest.importorskip("cv2")
total_frames, fps = 60, 10
path = _write_gray_video(tmp_path, total_frames, fps)
from vllm.transformers_utils.processors.glm5next import (
glm_sample_frame_indices,
)
expected = glm_sample_frame_indices(total_frames, float(fps), 6.0)
frames, metadata = Glm5NextVideoBackend.load_bytes(path.read_bytes())
assert len(frames) == len(expected)
assert metadata["frames_indices"] == expected
assert metadata["do_sample_frames"] is (len(expected) == total_frames)
# Flat gray frames survive mp4v near-losslessly: each decoded frame's
# level maps back to its source frame index.
for frame, idx in zip(frames, expected):
decoded_idx = round(float(np.asarray(frame).mean()))
assert abs(decoded_idx - idx) <= 1
def test_glm5next_read_frames_seeks_past_large_gaps(tmp_path):
"""Sparse targets must not walk the container: the stock reader grabs
every frame up to the last index; the GLM reader seeks instead."""
cv2 = pytest.importorskip("cv2")
total_frames, fps = 200, 10
path = _write_gray_video(tmp_path, total_frames, fps)
targets = [0, 80, 160, 190] # gaps of 80/80/30 -> only 30 <= threshold
stock = cv2.VideoCapture(str(path))
_, stock_indices = VideoBackend.read_frames(stock, targets, total_frames)
stock.release()
assert stock_indices == targets
cap = _CountingCap(cv2.VideoCapture(str(path)))
frames, indices = Glm5NextVideoBackend.read_frames(cap, targets, total_frames)
cap._cap.release()
assert indices == targets == stock_indices
for frame, idx in zip(frames, targets):
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
# Walks only the sub-threshold 30-frame hop; the two 80-frame gaps are
# seeks. The stock reader grabs all 190 preceding frames.
assert cap.grabs <= 29
assert cap.seeks == 3
def test_glm5next_read_frames_dense_walk_matches_stock(tmp_path):
"""Dense targets keep the sequential walk (seeking would be slower) and
return the same frames as the stock reader."""
cv2 = pytest.importorskip("cv2")
total_frames, fps = 120, 10
path = _write_gray_video(tmp_path, total_frames, fps)
targets = list(range(0, total_frames, 15)) # gaps of 15 -> all walking
stock = cv2.VideoCapture(str(path))
stock_frames, stock_indices = VideoBackend.read_frames(stock, targets, total_frames)
stock.release()
cap = _CountingCap(cv2.VideoCapture(str(path)))
frames, indices = Glm5NextVideoBackend.read_frames(cap, targets, total_frames)
cap._cap.release()
assert indices == stock_indices
for frame, stock_frame, idx in zip(frames, stock_frames, targets):
assert abs(float(np.asarray(frame).mean()) - float(stock_frame.mean())) <= 2.0
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
# One initial seek, then pure walking -- no re-seek churn.
assert cap.seeks == 1