[Bugfix][GLM-5.3-Flash] Take video placeholder timestamps from the pixel path's frame sampler (#55647)
Signed-off-by: NNNtrance <232268405+NNNtrance@users.noreply.github.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: NNNtrance <232268405+NNNtrance@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
NNNtrance
Isotr0py
parent
63b9931b73
commit
65ed0a2642
@@ -4,6 +4,7 @@
|
||||
|
||||
from collections.abc import Mapping
|
||||
from functools import cached_property, partial
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -42,6 +43,7 @@ from vllm.model_executor.models.vision import (
|
||||
)
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
|
||||
from vllm.transformers_utils.processors.glm5next import glm_sample_frame_indices
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
@@ -656,6 +658,63 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
return int(override)
|
||||
return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1]
|
||||
|
||||
def _get_video_second_idx_glm46v(
|
||||
self, metadata: dict[str, Any], total_frames: int
|
||||
) -> list[int]:
|
||||
"""Timestamps for the video placeholder, from the pixel path's sampler.
|
||||
|
||||
The inherited GLM-4.6V timestamp sampling uses different constants
|
||||
than this checkpoint's ``Glm5NextVideoProcessor`` pixel sampling, so
|
||||
placeholder count and ``video_grid_thw`` disagree outside 30-300s
|
||||
durations. Sampling with the pixel path's own policy makes one
|
||||
timestamp per temporal patch == ``grid_t``.
|
||||
"""
|
||||
video_processor = self.get_video_processor()
|
||||
temporal_patch_size = int(
|
||||
getattr(video_processor, "temporal_patch_size", 1) or 1
|
||||
)
|
||||
video_fps = float(metadata["fps"])
|
||||
|
||||
if not metadata.get("do_sample_frames", True):
|
||||
# The loader already picked the frames and the processor keeps them
|
||||
# as they are, so the frame count is what was handed in and
|
||||
# ``frames_indices`` only supplies the timestamp values.
|
||||
frame_indices = [int(idx) for idx in metadata["frames_indices"]]
|
||||
num_frames = int(total_frames)
|
||||
else:
|
||||
frame_indices = [
|
||||
int(idx)
|
||||
for idx in glm_sample_frame_indices(
|
||||
int(metadata.get("total_num_frames", total_frames)),
|
||||
video_fps,
|
||||
float(metadata.get("duration") or 0),
|
||||
target_fps=video_processor.fps_interval,
|
||||
max_frame_count=video_processor.max_frame_count_dynamic,
|
||||
temporal_patch_size=temporal_patch_size,
|
||||
)
|
||||
]
|
||||
num_frames = len(frame_indices)
|
||||
|
||||
if not frame_indices or num_frames <= 0:
|
||||
# ``duration * fps_interval < 1`` selects no frames at all; the
|
||||
# pixel path raises an opaque IndexError on the same input a moment
|
||||
# later, so say what happened while there is still a request.
|
||||
raise ValueError(
|
||||
"The GLM-5-Next frame sampler selected no frames for this "
|
||||
f"video (fps={video_fps}, duration={metadata.get('duration')!r},"
|
||||
f" total_num_frames={metadata.get('total_num_frames')!r}). "
|
||||
"Clips shorter than one frame sampling interval cannot be "
|
||||
"processed."
|
||||
)
|
||||
|
||||
num_padded_frames = num_frames + (-num_frames % temporal_patch_size)
|
||||
grid_t = max(num_padded_frames // temporal_patch_size, 1)
|
||||
last = len(frame_indices) - 1
|
||||
return [
|
||||
int(frame_indices[min(i * temporal_patch_size, last)] / video_fps)
|
||||
for i in range(grid_t)
|
||||
]
|
||||
|
||||
def _get_vision_info(
|
||||
self,
|
||||
*,
|
||||
|
||||
Reference in New Issue
Block a user