[Bugfix][GLM-5.3-Flash] Take video placeholder timestamps from the pixel path's frame sampler (#55647)
Signed-off-by: NNNtrance <232268405+NNNtrance@users.noreply.github.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: NNNtrance <232268405+NNNtrance@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
NNNtrance
Isotr0py
parent
63b9931b73
commit
65ed0a2642
@@ -0,0 +1,170 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Video placeholder accounting for GLM-5.3-Flash.
|
||||
|
||||
``Glm4vProcessingInfo._construct_video_placeholder`` emits one frame of
|
||||
placeholders per timestamp, so the timestamps returned by
|
||||
``_get_video_second_idx_glm46v`` decide how many placeholders the prompt gets
|
||||
while ``video_grid_thw`` decides how many rows the vision tower produces. If
|
||||
the two disagree they collide in ``_merge_multimodal_embeddings``, which raises
|
||||
inside a worker and takes the engine down with it.
|
||||
|
||||
The checks below are arithmetic: no weights, no GPU.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.transformers_utils.processors.glm5next import (
|
||||
Glm5NextVideoProcessor,
|
||||
_pixel_budget,
|
||||
glm_sample_frame_indices,
|
||||
smart_resize,
|
||||
)
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def processor():
|
||||
ctx = build_model_context(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
)
|
||||
return MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
|
||||
def _pixel_path_grid(
|
||||
video_processor: Glm5NextVideoProcessor,
|
||||
num_frames: int,
|
||||
height: int,
|
||||
width: int,
|
||||
) -> tuple[int, int, int]:
|
||||
"""The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds."""
|
||||
min_pixels, max_pixels = _pixel_budget(
|
||||
video_processor.min_image_tokens,
|
||||
video_processor.max_image_tokens,
|
||||
video_processor.patch_size,
|
||||
video_processor.merge_size,
|
||||
video_processor.temporal_patch_size,
|
||||
)
|
||||
factor = (
|
||||
video_processor.patch_size
|
||||
* video_processor.merge_size
|
||||
* video_processor.patch_expand_factor
|
||||
)
|
||||
resized_height, resized_width = smart_resize(
|
||||
t=num_frames,
|
||||
h=height,
|
||||
w=width,
|
||||
t_factor=video_processor.temporal_patch_size,
|
||||
h_factor=factor,
|
||||
w_factor=factor,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size)
|
||||
return (
|
||||
padded_frames // video_processor.temporal_patch_size,
|
||||
resized_height // video_processor.patch_size,
|
||||
resized_width // video_processor.patch_size,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("total_num_frames", "fps", "duration", "height", "width", "expected_grid"),
|
||||
[
|
||||
# 4 s at 8 fps: the GLM-4.6V sampler asks for 3x as many timestamps.
|
||||
(32, 8.0, 4.0, 480, 640, (4, 36, 46)),
|
||||
# 1080p, 20 s: same factor of 3 at a full-size canvas.
|
||||
(600, 30.0, 20.0, 1080, 1920, (20, 58, 102)),
|
||||
# 1080p, 60 s: the one duration window where the two samplers agree
|
||||
# anyway -- a regression guard, the count must not move.
|
||||
(1800, 30.0, 60.0, 1080, 1920, (60, 34, 58)),
|
||||
# Past 300 s the GLM-4.6V sampler asks for half as many instead.
|
||||
(9030, 30.0, 301.0, 720, 1280, (301, 14, 26)),
|
||||
],
|
||||
)
|
||||
def test_video_placeholders_match_encoder_rows(
|
||||
processor,
|
||||
total_num_frames: int,
|
||||
fps: float,
|
||||
duration: float,
|
||||
height: int,
|
||||
width: int,
|
||||
expected_grid: tuple[int, int, int],
|
||||
):
|
||||
info = processor.info
|
||||
video_processor = info.get_video_processor()
|
||||
|
||||
frame_indices = glm_sample_frame_indices(
|
||||
total_num_frames,
|
||||
fps,
|
||||
duration,
|
||||
target_fps=video_processor.fps_interval,
|
||||
max_frame_count=video_processor.max_frame_count_dynamic,
|
||||
temporal_patch_size=video_processor.temporal_patch_size,
|
||||
)
|
||||
grid_t, grid_h, grid_w = _pixel_path_grid(
|
||||
video_processor, len(frame_indices), height, width
|
||||
)
|
||||
assert (grid_t, grid_h, grid_w) == expected_grid
|
||||
|
||||
timestamps = info._get_video_second_idx_glm46v(
|
||||
{
|
||||
"total_num_frames": total_num_frames,
|
||||
"fps": fps,
|
||||
"duration": duration,
|
||||
"do_sample_frames": True,
|
||||
},
|
||||
total_num_frames,
|
||||
)
|
||||
|
||||
merge_length = video_processor.merge_size**2
|
||||
tokens_per_frame = grid_h * grid_w // merge_length
|
||||
encoder_rows = grid_t * grid_h * grid_w // merge_length
|
||||
|
||||
assert len(timestamps) == grid_t
|
||||
assert len(timestamps) * tokens_per_frame == encoder_rows
|
||||
assert timestamps == sorted(timestamps)
|
||||
assert timestamps[0] == 0
|
||||
assert timestamps[-1] <= duration
|
||||
|
||||
|
||||
def test_video_placeholders_match_encoder_rows_when_presampled(processor):
|
||||
"""The loader may pre-sample and hand the frames over as they are."""
|
||||
info = processor.info
|
||||
video_processor = info.get_video_processor()
|
||||
|
||||
num_frames = 32
|
||||
grid_t, _, _ = _pixel_path_grid(video_processor, num_frames, 480, 640)
|
||||
|
||||
timestamps = info._get_video_second_idx_glm46v(
|
||||
{
|
||||
"total_num_frames": 256,
|
||||
"fps": 8.0,
|
||||
"duration": 32.0,
|
||||
"do_sample_frames": False,
|
||||
"frames_indices": list(range(0, 256, 256 // num_frames)),
|
||||
},
|
||||
num_frames,
|
||||
)
|
||||
|
||||
assert len(timestamps) == grid_t
|
||||
|
||||
|
||||
def test_video_shorter_than_one_sampling_interval_is_rejected(processor):
|
||||
"""A clip the sampler cannot pick a single frame from is a bad request."""
|
||||
with pytest.raises(ValueError, match="selected no frames"):
|
||||
processor.info._get_video_second_idx_glm46v(
|
||||
{
|
||||
"total_num_frames": 2,
|
||||
"fps": 8.0,
|
||||
"duration": 0.25,
|
||||
"do_sample_frames": True,
|
||||
},
|
||||
2,
|
||||
)
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
from collections.abc import Mapping
|
||||
from functools import cached_property, partial
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -42,6 +43,7 @@ from vllm.model_executor.models.vision import (
|
||||
)
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
|
||||
from vllm.transformers_utils.processors.glm5next import glm_sample_frame_indices
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
@@ -656,6 +658,63 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
return int(override)
|
||||
return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1]
|
||||
|
||||
def _get_video_second_idx_glm46v(
|
||||
self, metadata: dict[str, Any], total_frames: int
|
||||
) -> list[int]:
|
||||
"""Timestamps for the video placeholder, from the pixel path's sampler.
|
||||
|
||||
The inherited GLM-4.6V timestamp sampling uses different constants
|
||||
than this checkpoint's ``Glm5NextVideoProcessor`` pixel sampling, so
|
||||
placeholder count and ``video_grid_thw`` disagree outside 30-300s
|
||||
durations. Sampling with the pixel path's own policy makes one
|
||||
timestamp per temporal patch == ``grid_t``.
|
||||
"""
|
||||
video_processor = self.get_video_processor()
|
||||
temporal_patch_size = int(
|
||||
getattr(video_processor, "temporal_patch_size", 1) or 1
|
||||
)
|
||||
video_fps = float(metadata["fps"])
|
||||
|
||||
if not metadata.get("do_sample_frames", True):
|
||||
# The loader already picked the frames and the processor keeps them
|
||||
# as they are, so the frame count is what was handed in and
|
||||
# ``frames_indices`` only supplies the timestamp values.
|
||||
frame_indices = [int(idx) for idx in metadata["frames_indices"]]
|
||||
num_frames = int(total_frames)
|
||||
else:
|
||||
frame_indices = [
|
||||
int(idx)
|
||||
for idx in glm_sample_frame_indices(
|
||||
int(metadata.get("total_num_frames", total_frames)),
|
||||
video_fps,
|
||||
float(metadata.get("duration") or 0),
|
||||
target_fps=video_processor.fps_interval,
|
||||
max_frame_count=video_processor.max_frame_count_dynamic,
|
||||
temporal_patch_size=temporal_patch_size,
|
||||
)
|
||||
]
|
||||
num_frames = len(frame_indices)
|
||||
|
||||
if not frame_indices or num_frames <= 0:
|
||||
# ``duration * fps_interval < 1`` selects no frames at all; the
|
||||
# pixel path raises an opaque IndexError on the same input a moment
|
||||
# later, so say what happened while there is still a request.
|
||||
raise ValueError(
|
||||
"The GLM-5-Next frame sampler selected no frames for this "
|
||||
f"video (fps={video_fps}, duration={metadata.get('duration')!r},"
|
||||
f" total_num_frames={metadata.get('total_num_frames')!r}). "
|
||||
"Clips shorter than one frame sampling interval cannot be "
|
||||
"processed."
|
||||
)
|
||||
|
||||
num_padded_frames = num_frames + (-num_frames % temporal_patch_size)
|
||||
grid_t = max(num_padded_frames // temporal_patch_size, 1)
|
||||
last = len(frame_indices) - 1
|
||||
return [
|
||||
int(frame_indices[min(i * temporal_patch_size, last)] / video_fps)
|
||||
for i in range(grid_t)
|
||||
]
|
||||
|
||||
def _get_vision_info(
|
||||
self,
|
||||
*,
|
||||
|
||||
Reference in New Issue
Block a user