From 65ed0a2642d2fb1d897408ec39f64b1cb74f0369 Mon Sep 17 00:00:00 2001 From: burak Date: Tue, 29 Sep 2026 09:45:29 +0300 Subject: [PATCH] [Bugfix][GLM-5.3-Flash] Take video placeholder timestamps from the pixel path's frame sampler (#55647) Signed-off-by: NNNtrance <232268405+NNNtrance@users.noreply.github.com> Signed-off-by: Isotr0py Co-authored-by: NNNtrance <232268405+NNNtrance@users.noreply.github.com> Co-authored-by: Isotr0py --- .../multimodal/processing/test_glm5next.py | 170 ++++++++++++++++++ vllm/models/glm5next/common/multimodal.py | 59 ++++++ 2 files changed, 229 insertions(+) create mode 100644 tests/models/multimodal/processing/test_glm5next.py diff --git a/tests/models/multimodal/processing/test_glm5next.py b/tests/models/multimodal/processing/test_glm5next.py new file mode 100644 index 0000000000..6fb0e397c2 --- /dev/null +++ b/tests/models/multimodal/processing/test_glm5next.py @@ -0,0 +1,170 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Video placeholder accounting for GLM-5.3-Flash. + +``Glm4vProcessingInfo._construct_video_placeholder`` emits one frame of +placeholders per timestamp, so the timestamps returned by +``_get_video_second_idx_glm46v`` decide how many placeholders the prompt gets +while ``video_grid_thw`` decides how many rows the vision tower produces. If +the two disagree they collide in ``_merge_multimodal_embeddings``, which raises +inside a worker and takes the engine down with it. + +The checks below are arithmetic: no weights, no GPU. +""" + +import pytest + +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.transformers_utils.processors.glm5next import ( + Glm5NextVideoProcessor, + _pixel_budget, + glm_sample_frame_indices, + smart_resize, +) + +from ...utils import build_model_context + + +@pytest.fixture(scope="module") +def processor(): + ctx = build_model_context( + "zai-org/GLM-5.3-Flash", + limit_mm_per_prompt={"video": 1}, + ) + return MULTIMODAL_REGISTRY.create_processor( + ctx.model_config, + tokenizer=ctx.tokenizer, + ) + + +def _pixel_path_grid( + video_processor: Glm5NextVideoProcessor, + num_frames: int, + height: int, + width: int, +) -> tuple[int, int, int]: + """The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds.""" + min_pixels, max_pixels = _pixel_budget( + video_processor.min_image_tokens, + video_processor.max_image_tokens, + video_processor.patch_size, + video_processor.merge_size, + video_processor.temporal_patch_size, + ) + factor = ( + video_processor.patch_size + * video_processor.merge_size + * video_processor.patch_expand_factor + ) + resized_height, resized_width = smart_resize( + t=num_frames, + h=height, + w=width, + t_factor=video_processor.temporal_patch_size, + h_factor=factor, + w_factor=factor, + min_pixels=min_pixels, + max_pixels=max_pixels, + ) + padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size) + return ( + padded_frames // video_processor.temporal_patch_size, + resized_height // video_processor.patch_size, + resized_width // video_processor.patch_size, + ) + + +@pytest.mark.parametrize( + ("total_num_frames", "fps", "duration", "height", "width", "expected_grid"), + [ + # 4 s at 8 fps: the GLM-4.6V sampler asks for 3x as many timestamps. + (32, 8.0, 4.0, 480, 640, (4, 36, 46)), + # 1080p, 20 s: same factor of 3 at a full-size canvas. + (600, 30.0, 20.0, 1080, 1920, (20, 58, 102)), + # 1080p, 60 s: the one duration window where the two samplers agree + # anyway -- a regression guard, the count must not move. + (1800, 30.0, 60.0, 1080, 1920, (60, 34, 58)), + # Past 300 s the GLM-4.6V sampler asks for half as many instead. + (9030, 30.0, 301.0, 720, 1280, (301, 14, 26)), + ], +) +def test_video_placeholders_match_encoder_rows( + processor, + total_num_frames: int, + fps: float, + duration: float, + height: int, + width: int, + expected_grid: tuple[int, int, int], +): + info = processor.info + video_processor = info.get_video_processor() + + frame_indices = glm_sample_frame_indices( + total_num_frames, + fps, + duration, + target_fps=video_processor.fps_interval, + max_frame_count=video_processor.max_frame_count_dynamic, + temporal_patch_size=video_processor.temporal_patch_size, + ) + grid_t, grid_h, grid_w = _pixel_path_grid( + video_processor, len(frame_indices), height, width + ) + assert (grid_t, grid_h, grid_w) == expected_grid + + timestamps = info._get_video_second_idx_glm46v( + { + "total_num_frames": total_num_frames, + "fps": fps, + "duration": duration, + "do_sample_frames": True, + }, + total_num_frames, + ) + + merge_length = video_processor.merge_size**2 + tokens_per_frame = grid_h * grid_w // merge_length + encoder_rows = grid_t * grid_h * grid_w // merge_length + + assert len(timestamps) == grid_t + assert len(timestamps) * tokens_per_frame == encoder_rows + assert timestamps == sorted(timestamps) + assert timestamps[0] == 0 + assert timestamps[-1] <= duration + + +def test_video_placeholders_match_encoder_rows_when_presampled(processor): + """The loader may pre-sample and hand the frames over as they are.""" + info = processor.info + video_processor = info.get_video_processor() + + num_frames = 32 + grid_t, _, _ = _pixel_path_grid(video_processor, num_frames, 480, 640) + + timestamps = info._get_video_second_idx_glm46v( + { + "total_num_frames": 256, + "fps": 8.0, + "duration": 32.0, + "do_sample_frames": False, + "frames_indices": list(range(0, 256, 256 // num_frames)), + }, + num_frames, + ) + + assert len(timestamps) == grid_t + + +def test_video_shorter_than_one_sampling_interval_is_rejected(processor): + """A clip the sampler cannot pick a single frame from is a bad request.""" + with pytest.raises(ValueError, match="selected no frames"): + processor.info._get_video_second_idx_glm46v( + { + "total_num_frames": 2, + "fps": 8.0, + "duration": 0.25, + "do_sample_frames": True, + }, + 2, + ) diff --git a/vllm/models/glm5next/common/multimodal.py b/vllm/models/glm5next/common/multimodal.py index d2b21345d0..03ac3fa664 100644 --- a/vllm/models/glm5next/common/multimodal.py +++ b/vllm/models/glm5next/common/multimodal.py @@ -4,6 +4,7 @@ from collections.abc import Mapping from functools import cached_property, partial +from typing import Any import numpy as np import torch @@ -42,6 +43,7 @@ from vllm.model_executor.models.vision import ( ) from vllm.models.common.ops import fused_q_kv_rmsnorm from vllm.multimodal.parse import ImageSize, MultiModalDataItems +from vllm.transformers_utils.processors.glm5next import glm_sample_frame_indices from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -656,6 +658,63 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo): return int(override) return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1] + def _get_video_second_idx_glm46v( + self, metadata: dict[str, Any], total_frames: int + ) -> list[int]: + """Timestamps for the video placeholder, from the pixel path's sampler. + + The inherited GLM-4.6V timestamp sampling uses different constants + than this checkpoint's ``Glm5NextVideoProcessor`` pixel sampling, so + placeholder count and ``video_grid_thw`` disagree outside 30-300s + durations. Sampling with the pixel path's own policy makes one + timestamp per temporal patch == ``grid_t``. + """ + video_processor = self.get_video_processor() + temporal_patch_size = int( + getattr(video_processor, "temporal_patch_size", 1) or 1 + ) + video_fps = float(metadata["fps"]) + + if not metadata.get("do_sample_frames", True): + # The loader already picked the frames and the processor keeps them + # as they are, so the frame count is what was handed in and + # ``frames_indices`` only supplies the timestamp values. + frame_indices = [int(idx) for idx in metadata["frames_indices"]] + num_frames = int(total_frames) + else: + frame_indices = [ + int(idx) + for idx in glm_sample_frame_indices( + int(metadata.get("total_num_frames", total_frames)), + video_fps, + float(metadata.get("duration") or 0), + target_fps=video_processor.fps_interval, + max_frame_count=video_processor.max_frame_count_dynamic, + temporal_patch_size=temporal_patch_size, + ) + ] + num_frames = len(frame_indices) + + if not frame_indices or num_frames <= 0: + # ``duration * fps_interval < 1`` selects no frames at all; the + # pixel path raises an opaque IndexError on the same input a moment + # later, so say what happened while there is still a request. + raise ValueError( + "The GLM-5-Next frame sampler selected no frames for this " + f"video (fps={video_fps}, duration={metadata.get('duration')!r}," + f" total_num_frames={metadata.get('total_num_frames')!r}). " + "Clips shorter than one frame sampling interval cannot be " + "processed." + ) + + num_padded_frames = num_frames + (-num_frames % temporal_patch_size) + grid_t = max(num_padded_frames // temporal_patch_size, 1) + last = len(frame_indices) - 1 + return [ + int(frame_indices[min(i * temporal_patch_size, last)] / video_fps) + for i in range(grid_t) + ] + def _get_vision_info( self, *,