954 lines
34 KiB
Python
954 lines
34 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""vLLM-native multimodal processor for GLM-5.3-Flash."""
|
|
|
|
import math
|
|
|
|
import numpy as np
|
|
import torch
|
|
from torchvision.transforms.v2 import functional as tvF
|
|
from transformers.image_processing_utils import BatchFeature
|
|
from transformers.image_processing_utils_fast import (
|
|
BaseImageProcessorFast,
|
|
group_images_by_shape,
|
|
reorder_images,
|
|
)
|
|
from transformers.image_utils import (
|
|
OPENAI_CLIP_MEAN,
|
|
OPENAI_CLIP_STD,
|
|
ChannelDimension,
|
|
ImageInput,
|
|
PILImageResampling,
|
|
SizeDict,
|
|
get_image_size,
|
|
)
|
|
from transformers.models.auto.image_processing_auto import get_image_processor_config
|
|
from transformers.processing_utils import (
|
|
ImagesKwargs,
|
|
MultiModalData,
|
|
ProcessingKwargs,
|
|
ProcessorMixin,
|
|
Unpack,
|
|
VideosKwargs,
|
|
)
|
|
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
|
from transformers.utils import TensorType, logging
|
|
from transformers.video_processing_utils import BaseVideoProcessor
|
|
from transformers.video_utils import (
|
|
VideoInput,
|
|
VideoMetadata,
|
|
group_videos_by_shape,
|
|
reorder_videos,
|
|
)
|
|
|
|
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
|
|
|
logger = logging.get_logger(__name__)
|
|
|
|
# Cap video inputs at 30,000 vision tokens to keep encoder profiling from
|
|
# starving the KV cache. Image inputs retain their checkpoint-defined budget.
|
|
_MAX_VIDEO_TOKENS = 30000
|
|
|
|
# Frame-sampler fallbacks (mirror the checkpoint's fps_interval=2 /
|
|
# max_frame_count_dynamic=2048); used when neither the request nor the
|
|
# processor config overrides them.
|
|
GLM_VIDEO_DEFAULT_FPS = 2.0
|
|
GLM_VIDEO_DEFAULT_MAX_FRAMES = 2048
|
|
|
|
|
|
def glm_sample_frame_indices(
|
|
total_frames: int,
|
|
fps: float,
|
|
duration: float,
|
|
*,
|
|
target_fps: float | None = None,
|
|
max_frame_count: int | None = None,
|
|
temporal_patch_size: int = 2,
|
|
) -> list[int]:
|
|
"""GLM video frame sampling (training-reference parity).
|
|
|
|
``target_fps`` is the ``fps_interval`` request knob. The greedy walk
|
|
advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on
|
|
frame-dense sources it collects more candidates than ``extract_t`` and
|
|
the ``> extract_t`` fixup re-spreads the picks uniformly with
|
|
``np.linspace`` -- that fallback is the intended reference behavior, not
|
|
an accident. Short clips (fewer frames than ``extract_t``) are spread at
|
|
evenly spaced timestamps (``floor`` sampling; the linspace variant
|
|
samples frames unevenly and cost 4 points on video grounding evals).
|
|
Request overrides: ``target_fps`` -> fps interval, ``max_frame_count``
|
|
-> frame cap.
|
|
"""
|
|
max_frame_idx = total_frames - 1
|
|
if not duration:
|
|
duration = (round(max_frame_idx / fps) + 1) if fps else 0
|
|
if max_frame_count is None:
|
|
max_frame_count = GLM_VIDEO_DEFAULT_MAX_FRAMES
|
|
if target_fps is None:
|
|
target_fps = GLM_VIDEO_DEFAULT_FPS
|
|
|
|
extract_t = int(duration * target_fps)
|
|
extract_t = min(extract_t, int(max_frame_count))
|
|
|
|
duration_per_frame = 1 / fps
|
|
timestamps = [i * duration_per_frame for i in range(total_frames)]
|
|
max_second = int(duration)
|
|
|
|
if total_frames < extract_t:
|
|
frame_indices = [
|
|
math.floor(_i * total_frames / extract_t) for _i in range(extract_t)
|
|
]
|
|
else:
|
|
frame_indices = []
|
|
current_second = 0.0
|
|
inv_fps = 1 / (temporal_patch_size * target_fps)
|
|
for frame_index in range(total_frames):
|
|
if timestamps[frame_index] >= current_second:
|
|
current_second += inv_fps
|
|
frame_indices.append(frame_index)
|
|
if current_second >= max_second:
|
|
break
|
|
|
|
if len(frame_indices) < extract_t:
|
|
if len(frame_indices) == 0:
|
|
start, end = 0, max(total_frames - 1, 0)
|
|
else:
|
|
start, end = frame_indices[0], frame_indices[-1]
|
|
frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
|
|
elif len(frame_indices) > extract_t:
|
|
frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist()
|
|
|
|
seen, uniq = set(), []
|
|
for idx in frame_indices:
|
|
if idx not in seen:
|
|
seen.add(idx)
|
|
uniq.append(int(idx))
|
|
|
|
if len(uniq) & 1:
|
|
uniq.append(uniq[-1])
|
|
|
|
return uniq
|
|
|
|
|
|
def _ceil_to_factor(value: int, factor: int) -> int:
|
|
"""Round a positive integer upward to the nearest multiple of factor."""
|
|
return math.ceil(value / factor) * factor
|
|
|
|
|
|
def _fit_aligned_size_within_budget(
|
|
t: int,
|
|
h: int,
|
|
w: int,
|
|
h_factor: int,
|
|
w_factor: int,
|
|
max_pixels: int,
|
|
) -> tuple[int, int]:
|
|
"""Largest proportional size whose upward-aligned canvas fits the budget.
|
|
|
|
Binary search on the unaligned content height; each candidate is rounded
|
|
upward to h_factor/w_factor, so the returned canvas always satisfies
|
|
``t * aligned_h * aligned_w <= max_pixels``.
|
|
"""
|
|
minimum_pixels = t * h_factor * w_factor
|
|
if max_pixels < minimum_pixels:
|
|
raise ValueError(
|
|
f"max_pixels={max_pixels} is too small. At least "
|
|
f"{minimum_pixels} pixels are required for one aligned patch."
|
|
)
|
|
|
|
low, high = 1, h
|
|
best_h, best_w = h_factor, w_factor
|
|
while low <= high:
|
|
content_h = (low + high) // 2
|
|
content_w = max(1, math.floor(w * content_h / h))
|
|
aligned_h = _ceil_to_factor(content_h, h_factor)
|
|
aligned_w = _ceil_to_factor(content_w, w_factor)
|
|
if t * aligned_h * aligned_w <= max_pixels:
|
|
best_h, best_w = aligned_h, aligned_w
|
|
low = content_h + 1
|
|
else:
|
|
high = content_h - 1
|
|
return best_h, best_w
|
|
|
|
|
|
def smart_resize(
|
|
t: int,
|
|
h: int,
|
|
w: int,
|
|
t_factor: int = 1,
|
|
h_factor: int = 28,
|
|
w_factor: int = 28,
|
|
min_pixels: int = 56 * 56,
|
|
max_pixels: int = 14 * 14 * 4 * 1280,
|
|
) -> tuple[int, int]:
|
|
"""GLM-5.3-Flash ``smart_resize``: upward-aligned canvas under a
|
|
``t_bar * h_bar * w_bar`` pixel budget.
|
|
|
|
Height/width always round UP to their factors (content is then padded,
|
|
never cropped or distorted); an over-budget canvas is refit by binary
|
|
search instead of one-shot square-root scaling. ``h_factor`` /
|
|
``w_factor`` carry ``patch_expand_factor`` on top of
|
|
``patch_size * merge_size``; ``t_factor`` is ``temporal_patch_size``. For
|
|
a still image ``t = t_factor = temporal_patch_size`` so ``t_bar =
|
|
temporal_patch_size``.
|
|
"""
|
|
if min(t, h, w, t_factor, h_factor, w_factor) <= 0:
|
|
raise ValueError("Image dimensions and alignment factors must be positive.")
|
|
if min_pixels <= 0 or max_pixels <= 0:
|
|
raise ValueError("min_pixels and max_pixels must be positive.")
|
|
if min_pixels > max_pixels:
|
|
raise ValueError("min_pixels must be less than or equal to max_pixels.")
|
|
|
|
t_bar = max(t_factor, round(t / t_factor) * t_factor)
|
|
h_bar = _ceil_to_factor(h, h_factor)
|
|
w_bar = _ceil_to_factor(w, w_factor)
|
|
|
|
if t_bar * h_bar * w_bar > max_pixels:
|
|
h_bar, w_bar = _fit_aligned_size_within_budget(
|
|
t=t_bar,
|
|
h=h,
|
|
w=w,
|
|
h_factor=h_factor,
|
|
w_factor=w_factor,
|
|
max_pixels=max_pixels,
|
|
)
|
|
elif t_bar * h_bar * w_bar < min_pixels:
|
|
beta = math.sqrt(min_pixels / (t * h * w))
|
|
h_bar = _ceil_to_factor(max(1, math.ceil(h * beta)), h_factor)
|
|
w_bar = _ceil_to_factor(max(1, math.ceil(w * beta)), w_factor)
|
|
|
|
# Alignment can push a candidate slightly over a tight max_pixels
|
|
# budget. Refit it when that happens.
|
|
if t_bar * h_bar * w_bar > max_pixels:
|
|
h_bar, w_bar = _fit_aligned_size_within_budget(
|
|
t=t_bar,
|
|
h=h,
|
|
w=w,
|
|
h_factor=h_factor,
|
|
w_factor=w_factor,
|
|
max_pixels=max_pixels,
|
|
)
|
|
|
|
return h_bar, w_bar
|
|
|
|
|
|
def _get_pad_content_size(
|
|
image_height: int,
|
|
image_width: int,
|
|
canvas_height: int,
|
|
canvas_width: int,
|
|
allow_upscale: bool = False,
|
|
) -> tuple[int, int]:
|
|
"""Aspect-ratio-preserving content size that fits the canvas.
|
|
|
|
Oversized images are shrunk proportionally. Small images are enlarged
|
|
only when ``allow_upscale``. Padding is applied after the resize.
|
|
"""
|
|
scale = min(canvas_height / image_height, canvas_width / image_width)
|
|
if not allow_upscale:
|
|
scale = min(1.0, scale)
|
|
content_height = max(1, min(canvas_height, math.floor(image_height * scale)))
|
|
content_width = max(1, min(canvas_width, math.floor(image_width * scale)))
|
|
return content_height, content_width
|
|
|
|
|
|
def _resize_or_pad(
|
|
stacked_images: torch.Tensor,
|
|
target_height: int,
|
|
target_width: int,
|
|
resize_mode: str,
|
|
resample: "PILImageResampling | tvF.InterpolationMode | int | None",
|
|
resize,
|
|
allow_upscale: bool = False,
|
|
) -> torch.Tensor:
|
|
"""Resize onto the aligned canvas, or keep the aspect ratio and
|
|
zero-pad the right/bottom sides (``resize_mode="pad"``)."""
|
|
height, width = stacked_images.shape[-2:]
|
|
|
|
if resize_mode == "resize":
|
|
return resize(
|
|
stacked_images,
|
|
size=SizeDict(height=target_height, width=target_width),
|
|
resample=resample,
|
|
)
|
|
|
|
if resize_mode != "pad":
|
|
raise ValueError("resize_mode must be either 'resize' or 'pad'.")
|
|
|
|
content_height, content_width = _get_pad_content_size(
|
|
image_height=height,
|
|
image_width=width,
|
|
canvas_height=target_height,
|
|
canvas_width=target_width,
|
|
allow_upscale=allow_upscale,
|
|
)
|
|
|
|
if (content_height, content_width) != (height, width):
|
|
stacked_images = resize(
|
|
stacked_images,
|
|
size=SizeDict(height=content_height, width=content_width),
|
|
resample=resample,
|
|
)
|
|
|
|
# torchvision padding order: [left, top, right, bottom] -> pad only the
|
|
# right and bottom sides.
|
|
return tvF.pad(
|
|
stacked_images,
|
|
padding=[0, 0, target_width - content_width, target_height - content_height],
|
|
fill=0,
|
|
)
|
|
|
|
|
|
def _pixel_budget(
|
|
min_image_tokens: int | None,
|
|
max_image_tokens: int | None,
|
|
patch_size: int,
|
|
merge_size: int,
|
|
temporal_patch_size: int,
|
|
) -> tuple[int, int]:
|
|
"""(min_pixels, max_pixels) from the token bounds of
|
|
``processor_config.json``; one vision token covers
|
|
``temporal_patch_size * (patch_size * merge_size) ** 2`` pixels."""
|
|
if min_image_tokens is None or max_image_tokens is None:
|
|
raise ValueError(
|
|
"min_image_tokens and max_image_tokens must be provided by "
|
|
"processor_config.json (or per-call kwargs)."
|
|
)
|
|
factor = temporal_patch_size * (patch_size * merge_size) ** 2
|
|
return min_image_tokens * factor, max_image_tokens * factor
|
|
|
|
|
|
class Glm5NextImageProcessorKwargs(ImagesKwargs, total=False): # type: ignore[call-arg]
|
|
patch_size: int | None
|
|
temporal_patch_size: int | None
|
|
merge_size: int | None
|
|
patch_expand_factor: int | None
|
|
resize_mode: str | None
|
|
min_image_tokens: int | None
|
|
max_image_tokens: int | None
|
|
|
|
|
|
class Glm5NextImageProcessor(BaseImageProcessorFast):
|
|
"""Fast torchvision image processor for GLM-5.3-Flash.
|
|
|
|
``patch_expand_factor`` multiplies into the ``smart_resize`` spatial
|
|
factor ``patch_size * merge_size``. ``resize_mode`` picks the geometry:
|
|
``"pad"`` (default) preserves the aspect ratio and zero-pads the
|
|
right/bottom of the upward-aligned canvas, ``"resize"`` stretches onto
|
|
it. Defaults mirror the checkpoint's ``image_processor`` config.
|
|
"""
|
|
|
|
do_resize = True
|
|
resample = PILImageResampling.BICUBIC
|
|
size = {"longest_edge": 1} # unused: budgets come from the token bounds
|
|
do_rescale = True
|
|
do_normalize = True
|
|
image_mean = OPENAI_CLIP_MEAN
|
|
image_std = OPENAI_CLIP_STD
|
|
do_convert_rgb = True
|
|
patch_size = 14
|
|
temporal_patch_size = 2
|
|
merge_size = 2
|
|
patch_expand_factor = 1
|
|
resize_mode = "pad"
|
|
min_image_tokens = 16
|
|
max_image_tokens = 8000
|
|
valid_kwargs = Glm5NextImageProcessorKwargs
|
|
model_input_names = ["pixel_values", "image_grid_thw"]
|
|
|
|
def _preprocess(
|
|
self,
|
|
images: list[torch.Tensor],
|
|
do_resize: bool,
|
|
size: SizeDict,
|
|
resample: "PILImageResampling | tvF.InterpolationMode | int | None",
|
|
do_rescale: bool,
|
|
rescale_factor: float,
|
|
do_normalize: bool,
|
|
image_mean: float | list[float] | None,
|
|
image_std: float | list[float] | None,
|
|
patch_size: int,
|
|
temporal_patch_size: int,
|
|
merge_size: int,
|
|
patch_expand_factor: int,
|
|
resize_mode: str | None,
|
|
min_image_tokens: int | None,
|
|
max_image_tokens: int | None,
|
|
disable_grouping: bool | None,
|
|
return_tensors: str | TensorType | None,
|
|
**kwargs,
|
|
) -> BatchFeature:
|
|
resize_mode = resize_mode if resize_mode is not None else self.resize_mode
|
|
min_pixels, max_pixels = _pixel_budget(
|
|
min_image_tokens if min_image_tokens is not None else self.min_image_tokens,
|
|
max_image_tokens if max_image_tokens is not None else self.max_image_tokens,
|
|
patch_size,
|
|
merge_size,
|
|
temporal_patch_size,
|
|
)
|
|
grouped_images, grouped_images_index = group_images_by_shape(
|
|
images, disable_grouping=disable_grouping
|
|
)
|
|
resized_images_grouped = {}
|
|
for shape, stacked_images in grouped_images.items():
|
|
height, width = stacked_images.shape[-2:]
|
|
if do_resize:
|
|
resized_height, resized_width = smart_resize(
|
|
t=temporal_patch_size,
|
|
h=height,
|
|
w=width,
|
|
t_factor=temporal_patch_size,
|
|
h_factor=patch_size * merge_size * patch_expand_factor,
|
|
w_factor=patch_size * merge_size * patch_expand_factor,
|
|
min_pixels=min_pixels,
|
|
max_pixels=max_pixels,
|
|
)
|
|
stacked_images = _resize_or_pad(
|
|
stacked_images,
|
|
target_height=resized_height,
|
|
target_width=resized_width,
|
|
resize_mode=resize_mode,
|
|
resample=resample,
|
|
resize=self.resize,
|
|
allow_upscale=(temporal_patch_size * height * width < min_pixels),
|
|
)
|
|
resized_images_grouped[shape] = stacked_images
|
|
|
|
resized_images = reorder_images(resized_images_grouped, grouped_images_index)
|
|
|
|
grouped_images, grouped_images_index = group_images_by_shape(
|
|
resized_images, disable_grouping=disable_grouping
|
|
)
|
|
processed_images_grouped = {}
|
|
processed_grids = {}
|
|
|
|
for shape, stacked_images in grouped_images.items():
|
|
resized_height, resized_width = stacked_images.shape[-2:]
|
|
|
|
patches = self.rescale_and_normalize(
|
|
stacked_images,
|
|
do_rescale,
|
|
rescale_factor,
|
|
do_normalize,
|
|
image_mean,
|
|
image_std,
|
|
)
|
|
if patches.ndim == 4: # (B, C, H, W)
|
|
patches = patches.unsqueeze(1) # (B, T=1, C, H, W)
|
|
|
|
if patches.shape[1] % temporal_patch_size != 0:
|
|
repeats = patches[:, -1:].repeat(
|
|
1,
|
|
temporal_patch_size - (patches.shape[1] % temporal_patch_size),
|
|
1,
|
|
1,
|
|
1,
|
|
)
|
|
patches = torch.cat([patches, repeats], dim=1)
|
|
|
|
batch_size, t_len, channel = patches.shape[:3]
|
|
grid_t = t_len // temporal_patch_size
|
|
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
|
|
|
patches = patches.view(
|
|
batch_size,
|
|
grid_t,
|
|
temporal_patch_size,
|
|
channel,
|
|
grid_h // merge_size,
|
|
merge_size,
|
|
patch_size,
|
|
grid_w // merge_size,
|
|
merge_size,
|
|
patch_size,
|
|
)
|
|
# (B, grid_t, gh, gw, mh, mw, C, tp, ph, pw)
|
|
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
|
|
|
|
flatten_patches = patches.reshape(
|
|
batch_size,
|
|
grid_t * grid_h * grid_w,
|
|
channel * temporal_patch_size * patch_size * patch_size,
|
|
)
|
|
|
|
processed_images_grouped[shape] = flatten_patches
|
|
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
|
|
|
|
processed_images = reorder_images(
|
|
processed_images_grouped, grouped_images_index
|
|
)
|
|
processed_grids = reorder_images(processed_grids, grouped_images_index)
|
|
|
|
pixel_values = torch.cat(processed_images, dim=0)
|
|
image_grid_thw = torch.tensor(processed_grids)
|
|
|
|
return BatchFeature(
|
|
data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw},
|
|
tensor_type=return_tensors,
|
|
)
|
|
|
|
def preprocess(
|
|
self, images: ImageInput, **kwargs: Unpack[Glm5NextImageProcessorKwargs]
|
|
) -> BatchFeature:
|
|
return super().preprocess(images, **kwargs)
|
|
|
|
def get_number_of_image_patches(
|
|
self, height: int, width: int, images_kwargs: dict | None = None
|
|
) -> int:
|
|
"""Number of image patches (pre-merge) for a given (height, width)."""
|
|
images_kwargs = images_kwargs or {}
|
|
patch_size = images_kwargs.get("patch_size", self.patch_size)
|
|
merge_size = images_kwargs.get("merge_size", self.merge_size)
|
|
patch_expand_factor = images_kwargs.get(
|
|
"patch_expand_factor", self.patch_expand_factor
|
|
)
|
|
min_pixels, max_pixels = _pixel_budget(
|
|
images_kwargs.get("min_image_tokens", self.min_image_tokens),
|
|
images_kwargs.get("max_image_tokens", self.max_image_tokens),
|
|
patch_size,
|
|
merge_size,
|
|
self.temporal_patch_size,
|
|
)
|
|
resized_height, resized_width = smart_resize(
|
|
t=self.temporal_patch_size,
|
|
h=height,
|
|
w=width,
|
|
t_factor=self.temporal_patch_size,
|
|
h_factor=patch_size * merge_size * patch_expand_factor,
|
|
w_factor=patch_size * merge_size * patch_expand_factor,
|
|
min_pixels=min_pixels,
|
|
max_pixels=max_pixels,
|
|
)
|
|
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
|
return grid_h * grid_w
|
|
|
|
|
|
class Glm5NextVideoProcessorKwargs(VideosKwargs, total=False): # type: ignore[call-arg]
|
|
fps: list[float] | float
|
|
patch_size: int
|
|
temporal_patch_size: int
|
|
merge_size: int
|
|
patch_expand_factor: int
|
|
resize_mode: str | None
|
|
target_fps: float | None
|
|
max_frames: int | None
|
|
fps_interval: int | None
|
|
max_frame_count_dynamic: int | None
|
|
min_image_tokens: int | None
|
|
max_image_tokens: int | None
|
|
|
|
|
|
class Glm5NextVideoProcessor(BaseVideoProcessor):
|
|
"""Fast video processor for GLM-5.3-Flash.
|
|
|
|
Shares ``smart_resize`` / the pad-mode geometry / the patchify with the
|
|
image processor, and adds GLM-5.3-Flash frame sampling
|
|
(``glm_sample_frame_indices``: ``fps_interval`` semantics with a
|
|
temporal-patch-scaled greedy walk). Defaults mirror the checkpoint's
|
|
``video_processor`` config.
|
|
"""
|
|
|
|
resample = PILImageResampling.BICUBIC
|
|
size = {"longest_edge": 1} # unused: budgets come from the token bounds
|
|
image_mean = OPENAI_CLIP_MEAN
|
|
image_std = OPENAI_CLIP_STD
|
|
do_resize = True
|
|
do_rescale = True
|
|
do_normalize = True
|
|
do_convert_rgb = True
|
|
do_sample_frames = True
|
|
patch_size = 14
|
|
temporal_patch_size = 2
|
|
patch_expand_factor = 1
|
|
merge_size = 2
|
|
valid_kwargs = Glm5NextVideoProcessorKwargs
|
|
num_frames = 16
|
|
fps = 2
|
|
fps_interval = 2.0
|
|
max_frame_count_dynamic = 2048
|
|
resize_mode = "pad"
|
|
min_image_tokens = 16
|
|
max_image_tokens = 240000
|
|
model_input_names = ["pixel_values_videos", "video_grid_thw"]
|
|
|
|
def sample_frames(
|
|
self,
|
|
metadata: VideoMetadata,
|
|
fps: int | float | None = None,
|
|
**kwargs,
|
|
) -> np.ndarray:
|
|
"""Sample frame indices with GLM's fps-interval policy.
|
|
|
|
``fps`` / ``target_fps``, ``max_frames`` and ``fps_interval`` /
|
|
``max_frame_count_dynamic`` are the overrides described in
|
|
:func:`glm_sample_frame_indices`.
|
|
"""
|
|
if metadata is None or getattr(metadata, "fps", None) is None:
|
|
raise ValueError(
|
|
"Asked to sample frames per second but no video metadata was "
|
|
"provided which is required when sampling in GLM-5.3-Flash. Please "
|
|
"pass in `VideoMetadata` object or set `do_sample_frames=False`."
|
|
)
|
|
|
|
target_fps = fps if fps is not None else kwargs.get("target_fps")
|
|
if target_fps is None:
|
|
target_fps = self.fps_interval
|
|
indices = glm_sample_frame_indices(
|
|
metadata.total_num_frames,
|
|
metadata.fps,
|
|
metadata.duration or 0,
|
|
target_fps=target_fps,
|
|
max_frame_count=kwargs.get("max_frames") or self.max_frame_count_dynamic,
|
|
temporal_patch_size=self.temporal_patch_size,
|
|
)
|
|
return np.array(indices)
|
|
|
|
def _preprocess(
|
|
self,
|
|
videos: list[torch.Tensor],
|
|
do_convert_rgb: bool = True,
|
|
do_resize: bool = True,
|
|
size: SizeDict | None = None,
|
|
resample: "PILImageResampling | int | None" = PILImageResampling.BICUBIC,
|
|
do_rescale: bool = True,
|
|
rescale_factor: float = 1 / 255.0,
|
|
do_normalize: bool = True,
|
|
image_mean: float | list[float] | None = None,
|
|
image_std: float | list[float] | None = None,
|
|
patch_size: int | None = None,
|
|
temporal_patch_size: int | None = None,
|
|
patch_expand_factor: int | None = None,
|
|
merge_size: int | None = None,
|
|
resize_mode: str | None = None,
|
|
min_image_tokens: int | None = None,
|
|
max_image_tokens: int | None = None,
|
|
return_tensors: str | TensorType | None = None,
|
|
**kwargs,
|
|
) -> BatchFeature:
|
|
patch_expand_factor = self.patch_expand_factor
|
|
patch_size = patch_size if patch_size is not None else self.patch_size
|
|
temporal_patch_size = (
|
|
temporal_patch_size
|
|
if temporal_patch_size is not None
|
|
else self.temporal_patch_size
|
|
)
|
|
merge_size = merge_size if merge_size is not None else self.merge_size
|
|
resize_mode = resize_mode if resize_mode is not None else self.resize_mode
|
|
min_pixels, max_pixels = _pixel_budget(
|
|
min_image_tokens if min_image_tokens is not None else self.min_image_tokens,
|
|
max_image_tokens if max_image_tokens is not None else self.max_image_tokens,
|
|
patch_size,
|
|
merge_size,
|
|
temporal_patch_size,
|
|
)
|
|
grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
|
|
resized_videos_grouped = {}
|
|
for shape, stacked_videos in grouped_videos.items():
|
|
if do_convert_rgb:
|
|
stacked_videos = self.convert_to_rgb(stacked_videos)
|
|
b, t_len, c, h, w = stacked_videos.shape
|
|
num_frames, height, width = t_len, h, w
|
|
if do_resize:
|
|
resized_height, resized_width = smart_resize(
|
|
t=num_frames,
|
|
h=height,
|
|
w=width,
|
|
t_factor=temporal_patch_size,
|
|
h_factor=patch_size * merge_size * patch_expand_factor,
|
|
w_factor=patch_size * merge_size * patch_expand_factor,
|
|
min_pixels=min_pixels,
|
|
max_pixels=max_pixels,
|
|
)
|
|
stacked_videos = stacked_videos.view(b * t_len, c, h, w)
|
|
stacked_videos = _resize_or_pad(
|
|
stacked_videos,
|
|
target_height=resized_height,
|
|
target_width=resized_width,
|
|
resize_mode=resize_mode,
|
|
resample=resample,
|
|
resize=self.resize,
|
|
allow_upscale=(num_frames * height * width < min_pixels),
|
|
)
|
|
stacked_videos = stacked_videos.view(
|
|
b, t_len, c, resized_height, resized_width
|
|
)
|
|
resized_videos_grouped[shape] = stacked_videos
|
|
resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
|
|
|
|
grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
|
|
processed_videos_grouped = {}
|
|
processed_grids = {}
|
|
for shape, stacked_videos in grouped_videos.items():
|
|
resized_height, resized_width = get_image_size(
|
|
stacked_videos[0], channel_dim=ChannelDimension.FIRST
|
|
)
|
|
stacked_videos = self.rescale_and_normalize(
|
|
stacked_videos,
|
|
do_rescale,
|
|
rescale_factor,
|
|
do_normalize,
|
|
image_mean,
|
|
image_std,
|
|
)
|
|
patches = stacked_videos
|
|
|
|
if pad := -patches.shape[1] % temporal_patch_size:
|
|
repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1)
|
|
patches = torch.cat((patches, repeats), dim=1)
|
|
batch_size, grid_t, channel = patches.shape[:3]
|
|
grid_t = grid_t // temporal_patch_size
|
|
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
|
|
|
patches = patches.view(
|
|
batch_size,
|
|
grid_t,
|
|
temporal_patch_size,
|
|
channel,
|
|
grid_h // merge_size,
|
|
merge_size,
|
|
patch_size,
|
|
grid_w // merge_size,
|
|
merge_size,
|
|
patch_size,
|
|
)
|
|
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
|
|
flatten_patches = patches.reshape(
|
|
batch_size,
|
|
grid_t * grid_h * grid_w,
|
|
channel * temporal_patch_size * patch_size * patch_size,
|
|
)
|
|
|
|
processed_videos_grouped[shape] = flatten_patches
|
|
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
|
|
|
|
processed_videos = reorder_videos(
|
|
processed_videos_grouped, grouped_videos_index
|
|
)
|
|
processed_grids = reorder_videos(processed_grids, grouped_videos_index)
|
|
pixel_values_videos = torch.cat(processed_videos, dim=0)
|
|
video_grid_thw = torch.tensor(processed_grids)
|
|
return BatchFeature(
|
|
data={
|
|
"pixel_values_videos": pixel_values_videos,
|
|
"video_grid_thw": video_grid_thw,
|
|
},
|
|
tensor_type=return_tensors,
|
|
)
|
|
|
|
|
|
class Glm5NextProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
|
|
images_kwargs: Glm5NextImageProcessorKwargs
|
|
videos_kwargs: Glm5NextVideoProcessorKwargs
|
|
_defaults = {
|
|
"text_kwargs": {
|
|
"padding": False,
|
|
"return_token_type_ids": False,
|
|
"return_mm_token_type_ids": False,
|
|
},
|
|
"videos_kwargs": {"return_metadata": True},
|
|
}
|
|
|
|
|
|
class Glm5NextProcessor(ProcessorMixin):
|
|
"""Wraps a GLM-5.3-Flash image processor, video processor and tokenizer.
|
|
|
|
Token expansion per image = ``prod(image_grid_thw) // merge_size**2``; video
|
|
frames are expanded with ``<|begin_of_image|>...<|end_of_image|>{ts} seconds``
|
|
structure (mrope timestamps).
|
|
"""
|
|
|
|
attributes = ["image_processor", "tokenizer", "video_processor"]
|
|
image_processor_class = "AutoImageProcessor"
|
|
video_processor_class = "AutoVideoProcessor"
|
|
tokenizer_class = ("PreTrainedTokenizer", "PreTrainedTokenizerFast")
|
|
|
|
def __init__(
|
|
self,
|
|
image_processor=None,
|
|
tokenizer=None,
|
|
video_processor=None,
|
|
chat_template=None,
|
|
**kwargs,
|
|
) -> None:
|
|
super().__init__(
|
|
image_processor, tokenizer, video_processor, chat_template=chat_template
|
|
)
|
|
self.image_token = (
|
|
"<|image|>"
|
|
if not hasattr(tokenizer, "image_token")
|
|
else tokenizer.image_token
|
|
)
|
|
self.video_token = (
|
|
"<|video|>"
|
|
if not hasattr(tokenizer, "video_token")
|
|
else tokenizer.video_token
|
|
)
|
|
self.image_token_id = (
|
|
tokenizer.image_token_id
|
|
if getattr(tokenizer, "image_token_id", None)
|
|
else tokenizer.convert_tokens_to_ids(self.image_token)
|
|
)
|
|
self.video_token_id = (
|
|
tokenizer.video_token_id
|
|
if getattr(tokenizer, "video_token_id", None)
|
|
else tokenizer.convert_tokens_to_ids(self.video_token)
|
|
)
|
|
|
|
@classmethod
|
|
def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
|
|
"""Build the processor directly from the checkpoint config.
|
|
|
|
GLM-5.3-Flash stores nested image/video configs in
|
|
``processor_config.json`` and declares a custom processor class. This
|
|
method reads those configs directly and caps only the video token budget.
|
|
"""
|
|
from transformers import AutoTokenizer
|
|
|
|
model_path = pretrained_model_name_or_path
|
|
tokenizer = AutoTokenizer.from_pretrained(model_path, **kwargs)
|
|
|
|
def _cap_cfg(cfg: dict, *, is_video: bool) -> dict:
|
|
# Video keeps a serving token cap (the checkpoint's 240k-token
|
|
# budget would starve the KV cache at startup profiling); images
|
|
# follow the checkpoint budget verbatim so preprocessing matches
|
|
# the HF reference exactly.
|
|
if is_video and cfg.get("max_image_tokens") is not None:
|
|
cfg["max_image_tokens"] = min(
|
|
cfg["max_image_tokens"], _MAX_VIDEO_TOKENS
|
|
)
|
|
return cfg
|
|
|
|
ip_cfg = _cap_cfg(
|
|
dict(get_image_processor_config(model_path, **kwargs)), is_video=False
|
|
)
|
|
image_processor = Glm5NextImageProcessor(
|
|
**{k: v for k, v in ip_cfg.items() if k != "image_processor_type"}
|
|
)
|
|
|
|
processor_config = get_hf_file_to_dict(
|
|
"processor_config.json",
|
|
model_path,
|
|
revision=kwargs.get("revision", "main"),
|
|
)
|
|
if processor_config is None:
|
|
raise ValueError(f"Missing processor_config.json for {model_path}")
|
|
vp_cfg = _cap_cfg(dict(processor_config["video_processor"]), is_video=True)
|
|
video_processor = Glm5NextVideoProcessor(
|
|
**{k: v for k, v in vp_cfg.items() if k != "video_processor_type"}
|
|
)
|
|
|
|
return cls(
|
|
image_processor=image_processor,
|
|
tokenizer=tokenizer,
|
|
video_processor=video_processor,
|
|
)
|
|
|
|
def __call__(
|
|
self,
|
|
images: ImageInput | None = None,
|
|
text: TextInput
|
|
| PreTokenizedInput
|
|
| list[TextInput]
|
|
| list[PreTokenizedInput] = None,
|
|
videos: VideoInput | None = None,
|
|
**kwargs: Unpack[Glm5NextProcessorKwargs],
|
|
) -> BatchFeature:
|
|
output_kwargs = self._merge_kwargs(
|
|
Glm5NextProcessorKwargs,
|
|
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
|
**kwargs,
|
|
)
|
|
if images is not None:
|
|
image_inputs = self.image_processor(
|
|
images=images, **output_kwargs["images_kwargs"]
|
|
)
|
|
else:
|
|
image_inputs = {}
|
|
|
|
if videos is not None:
|
|
videos_inputs = self.video_processor(
|
|
videos=videos, **output_kwargs["videos_kwargs"]
|
|
)
|
|
if "return_metadata" not in kwargs:
|
|
videos_inputs.pop("video_metadata")
|
|
else:
|
|
videos_inputs = {}
|
|
|
|
if not isinstance(text, list):
|
|
text = [text]
|
|
|
|
# Prompt updates expand the unchanged image/video markers after this call.
|
|
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
|
|
return_mm_token_type_ids = output_kwargs["text_kwargs"].pop(
|
|
"return_mm_token_type_ids", False
|
|
)
|
|
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
|
|
|
|
if return_mm_token_type_ids:
|
|
array_ids = np.array(text_inputs["input_ids"])
|
|
mm_token_type_ids = np.zeros_like(text_inputs["input_ids"])
|
|
mm_token_type_ids[array_ids == self.image_token_id] = 1
|
|
text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist()
|
|
return BatchFeature(
|
|
data={**text_inputs, **image_inputs, **videos_inputs},
|
|
tensor_type=return_tensors,
|
|
)
|
|
|
|
def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs):
|
|
vision_data = {}
|
|
if image_sizes is not None:
|
|
images_kwargs = Glm5NextProcessorKwargs._defaults.get("images_kwargs", {})
|
|
images_kwargs.update(kwargs)
|
|
merge_size = (
|
|
images_kwargs.get("merge_size", None) or self.image_processor.merge_size
|
|
)
|
|
|
|
num_image_patches = [
|
|
self.image_processor.get_number_of_image_patches(
|
|
*image_size, images_kwargs
|
|
)
|
|
for image_size in image_sizes
|
|
]
|
|
num_image_tokens = [(n // merge_size**2) for n in num_image_patches]
|
|
vision_data.update(
|
|
{
|
|
"num_image_tokens": num_image_tokens,
|
|
"num_image_patches": num_image_patches,
|
|
}
|
|
)
|
|
|
|
if video_sizes is not None:
|
|
videos_kwargs = Glm5NextProcessorKwargs._defaults.get("videos_kwargs", {})
|
|
videos_kwargs.update(kwargs)
|
|
num_video_patches = [
|
|
self.video_processor.get_number_of_video_patches(
|
|
*video_size, videos_kwargs
|
|
)
|
|
for video_size in video_sizes
|
|
]
|
|
num_video_tokens = [(n // merge_size**2) for n in num_video_patches]
|
|
vision_data["num_video_tokens"] = num_video_tokens
|
|
|
|
return MultiModalData(**vision_data)
|
|
|
|
def post_process_image_text_to_text(
|
|
self,
|
|
generated_outputs,
|
|
skip_special_tokens=True,
|
|
clean_up_tokenization_spaces=False,
|
|
**kwargs,
|
|
):
|
|
return self.tokenizer.batch_decode(
|
|
generated_outputs,
|
|
skip_special_tokens=skip_special_tokens,
|
|
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
|
**kwargs,
|
|
)
|
|
|
|
|
|
__all__ = [
|
|
"Glm5NextImageProcessor",
|
|
"Glm5NextVideoProcessor",
|
|
"Glm5NextProcessor",
|
|
"smart_resize",
|
|
]
|