[Bugfix][GLM-5.3-Flash] Take video placeholder timestamps from the pixel path's frame sampler (#55647)

Signed-off-by: NNNtrance <232268405+NNNtrance@users.noreply.github.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: NNNtrance <232268405+NNNtrance@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
burak
2026-09-29 06:45:29 +00:00
committed by GitHub
co-authored by NNNtrance Isotr0py
parent 63b9931b73
commit 65ed0a2642
2 changed files with 229 additions and 0 deletions
+59
View File
@@ -4,6 +4,7 @@
from collections.abc import Mapping
from functools import cached_property, partial
from typing import Any
import numpy as np
import torch
@@ -42,6 +43,7 @@ from vllm.model_executor.models.vision import (
)
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
from vllm.transformers_utils.processors.glm5next import glm_sample_frame_indices
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -656,6 +658,63 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
return int(override)
return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1]
def _get_video_second_idx_glm46v(
self, metadata: dict[str, Any], total_frames: int
) -> list[int]:
"""Timestamps for the video placeholder, from the pixel path's sampler.
The inherited GLM-4.6V timestamp sampling uses different constants
than this checkpoint's ``Glm5NextVideoProcessor`` pixel sampling, so
placeholder count and ``video_grid_thw`` disagree outside 30-300s
durations. Sampling with the pixel path's own policy makes one
timestamp per temporal patch == ``grid_t``.
"""
video_processor = self.get_video_processor()
temporal_patch_size = int(
getattr(video_processor, "temporal_patch_size", 1) or 1
)
video_fps = float(metadata["fps"])
if not metadata.get("do_sample_frames", True):
# The loader already picked the frames and the processor keeps them
# as they are, so the frame count is what was handed in and
# ``frames_indices`` only supplies the timestamp values.
frame_indices = [int(idx) for idx in metadata["frames_indices"]]
num_frames = int(total_frames)
else:
frame_indices = [
int(idx)
for idx in glm_sample_frame_indices(
int(metadata.get("total_num_frames", total_frames)),
video_fps,
float(metadata.get("duration") or 0),
target_fps=video_processor.fps_interval,
max_frame_count=video_processor.max_frame_count_dynamic,
temporal_patch_size=temporal_patch_size,
)
]
num_frames = len(frame_indices)
if not frame_indices or num_frames <= 0:
# ``duration * fps_interval < 1`` selects no frames at all; the
# pixel path raises an opaque IndexError on the same input a moment
# later, so say what happened while there is still a request.
raise ValueError(
"The GLM-5-Next frame sampler selected no frames for this "
f"video (fps={video_fps}, duration={metadata.get('duration')!r},"
f" total_num_frames={metadata.get('total_num_frames')!r}). "
"Clips shorter than one frame sampling interval cannot be "
"processed."
)
num_padded_frames = num_frames + (-num_frames % temporal_patch_size)
grid_t = max(num_padded_frames // temporal_patch_size, 1)
last = len(frame_indices) - 1
return [
int(frame_indices[min(i * temporal_patch_size, last)] / video_fps)
for i in range(grid_t)
]
def _get_vision_info(
self,
*,