[Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
866f02db8a
commit
e3cae8d2ac
@@ -17,7 +17,6 @@ __all__ = [
|
||||
"FireRedASR2Processor",
|
||||
"FunASRProcessor",
|
||||
"GLM4VProcessor",
|
||||
"Glm5NextProcessor",
|
||||
"Granite4VisionProcessor",
|
||||
"H2OVLProcessor",
|
||||
"Moondream3Processor",
|
||||
@@ -56,7 +55,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
|
||||
"FunASRProcessor": "vllm.transformers_utils.processors.funasr",
|
||||
"GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
|
||||
"Glm5NextProcessor": "vllm.transformers_utils.processors.glm5next",
|
||||
"Granite4VisionProcessor": "vllm.transformers_utils.processors.granite4_vision",
|
||||
"H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl",
|
||||
"InternVLProcessor": "vllm.transformers_utils.processors.internvl",
|
||||
|
||||
@@ -1,953 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vLLM-native multimodal processor for GLM-5.3-Flash."""
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torchvision.transforms.v2 import functional as tvF
|
||||
from transformers.image_processing_backends import (
|
||||
TorchvisionBackend,
|
||||
group_images_by_shape,
|
||||
reorder_images,
|
||||
)
|
||||
from transformers.image_processing_utils import BatchFeature
|
||||
from transformers.image_utils import (
|
||||
OPENAI_CLIP_MEAN,
|
||||
OPENAI_CLIP_STD,
|
||||
ChannelDimension,
|
||||
ImageInput,
|
||||
PILImageResampling,
|
||||
SizeDict,
|
||||
get_image_size,
|
||||
)
|
||||
from transformers.models.auto.image_processing_auto import get_image_processor_config
|
||||
from transformers.processing_utils import (
|
||||
ImagesKwargs,
|
||||
MultiModalData,
|
||||
ProcessingKwargs,
|
||||
ProcessorMixin,
|
||||
Unpack,
|
||||
VideosKwargs,
|
||||
)
|
||||
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
||||
from transformers.utils import TensorType, logging
|
||||
from transformers.video_processing_utils import BaseVideoProcessor
|
||||
from transformers.video_utils import (
|
||||
VideoInput,
|
||||
VideoMetadata,
|
||||
group_videos_by_shape,
|
||||
reorder_videos,
|
||||
)
|
||||
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
# Cap video inputs at 30,000 vision tokens to keep encoder profiling from
|
||||
# starving the KV cache. Image inputs retain their checkpoint-defined budget.
|
||||
_MAX_VIDEO_TOKENS = 30000
|
||||
|
||||
# Frame-sampler fallbacks (mirror the checkpoint's fps_interval=2 /
|
||||
# max_frame_count_dynamic=2048); used when neither the request nor the
|
||||
# processor config overrides them.
|
||||
GLM_VIDEO_DEFAULT_FPS = 2.0
|
||||
GLM_VIDEO_DEFAULT_MAX_FRAMES = 2048
|
||||
|
||||
|
||||
def glm_sample_frame_indices(
|
||||
total_frames: int,
|
||||
fps: float,
|
||||
duration: float,
|
||||
*,
|
||||
target_fps: float | None = None,
|
||||
max_frame_count: int | None = None,
|
||||
temporal_patch_size: int = 2,
|
||||
) -> list[int]:
|
||||
"""GLM video frame sampling (training-reference parity).
|
||||
|
||||
``target_fps`` is the ``fps_interval`` request knob. The greedy walk
|
||||
advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on
|
||||
frame-dense sources it collects more candidates than ``extract_t`` and
|
||||
the ``> extract_t`` fixup re-spreads the picks uniformly with
|
||||
``np.linspace`` -- that fallback is the intended reference behavior, not
|
||||
an accident. Short clips (fewer frames than ``extract_t``) are spread at
|
||||
evenly spaced timestamps (``floor`` sampling; the linspace variant
|
||||
samples frames unevenly and cost 4 points on video grounding evals).
|
||||
Request overrides: ``target_fps`` -> fps interval, ``max_frame_count``
|
||||
-> frame cap.
|
||||
"""
|
||||
max_frame_idx = total_frames - 1
|
||||
if not duration:
|
||||
duration = (round(max_frame_idx / fps) + 1) if fps else 0
|
||||
if max_frame_count is None:
|
||||
max_frame_count = GLM_VIDEO_DEFAULT_MAX_FRAMES
|
||||
if target_fps is None:
|
||||
target_fps = GLM_VIDEO_DEFAULT_FPS
|
||||
|
||||
extract_t = int(duration * target_fps)
|
||||
extract_t = min(extract_t, int(max_frame_count))
|
||||
|
||||
duration_per_frame = 1 / fps
|
||||
timestamps = [i * duration_per_frame for i in range(total_frames)]
|
||||
max_second = int(duration)
|
||||
|
||||
if total_frames < extract_t:
|
||||
frame_indices = [
|
||||
math.floor(_i * total_frames / extract_t) for _i in range(extract_t)
|
||||
]
|
||||
else:
|
||||
frame_indices = []
|
||||
current_second = 0.0
|
||||
inv_fps = 1 / (temporal_patch_size * target_fps)
|
||||
for frame_index in range(total_frames):
|
||||
if timestamps[frame_index] >= current_second:
|
||||
current_second += inv_fps
|
||||
frame_indices.append(frame_index)
|
||||
if current_second >= max_second:
|
||||
break
|
||||
|
||||
if len(frame_indices) < extract_t:
|
||||
if len(frame_indices) == 0:
|
||||
start, end = 0, max(total_frames - 1, 0)
|
||||
else:
|
||||
start, end = frame_indices[0], frame_indices[-1]
|
||||
frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
|
||||
elif len(frame_indices) > extract_t:
|
||||
frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist()
|
||||
|
||||
seen, uniq = set(), []
|
||||
for idx in frame_indices:
|
||||
if idx not in seen:
|
||||
seen.add(idx)
|
||||
uniq.append(int(idx))
|
||||
|
||||
if len(uniq) & 1:
|
||||
uniq.append(uniq[-1])
|
||||
|
||||
return uniq
|
||||
|
||||
|
||||
def _ceil_to_factor(value: int, factor: int) -> int:
|
||||
"""Round a positive integer upward to the nearest multiple of factor."""
|
||||
return math.ceil(value / factor) * factor
|
||||
|
||||
|
||||
def _fit_aligned_size_within_budget(
|
||||
t: int,
|
||||
h: int,
|
||||
w: int,
|
||||
h_factor: int,
|
||||
w_factor: int,
|
||||
max_pixels: int,
|
||||
) -> tuple[int, int]:
|
||||
"""Largest proportional size whose upward-aligned canvas fits the budget.
|
||||
|
||||
Binary search on the unaligned content height; each candidate is rounded
|
||||
upward to h_factor/w_factor, so the returned canvas always satisfies
|
||||
``t * aligned_h * aligned_w <= max_pixels``.
|
||||
"""
|
||||
minimum_pixels = t * h_factor * w_factor
|
||||
if max_pixels < minimum_pixels:
|
||||
raise ValueError(
|
||||
f"max_pixels={max_pixels} is too small. At least "
|
||||
f"{minimum_pixels} pixels are required for one aligned patch."
|
||||
)
|
||||
|
||||
low, high = 1, h
|
||||
best_h, best_w = h_factor, w_factor
|
||||
while low <= high:
|
||||
content_h = (low + high) // 2
|
||||
content_w = max(1, math.floor(w * content_h / h))
|
||||
aligned_h = _ceil_to_factor(content_h, h_factor)
|
||||
aligned_w = _ceil_to_factor(content_w, w_factor)
|
||||
if t * aligned_h * aligned_w <= max_pixels:
|
||||
best_h, best_w = aligned_h, aligned_w
|
||||
low = content_h + 1
|
||||
else:
|
||||
high = content_h - 1
|
||||
return best_h, best_w
|
||||
|
||||
|
||||
def smart_resize(
|
||||
t: int,
|
||||
h: int,
|
||||
w: int,
|
||||
t_factor: int = 1,
|
||||
h_factor: int = 28,
|
||||
w_factor: int = 28,
|
||||
min_pixels: int = 56 * 56,
|
||||
max_pixels: int = 14 * 14 * 4 * 1280,
|
||||
) -> tuple[int, int]:
|
||||
"""GLM-5.3-Flash ``smart_resize``: upward-aligned canvas under a
|
||||
``t_bar * h_bar * w_bar`` pixel budget.
|
||||
|
||||
Height/width always round UP to their factors (content is then padded,
|
||||
never cropped or distorted); an over-budget canvas is refit by binary
|
||||
search instead of one-shot square-root scaling. ``h_factor`` /
|
||||
``w_factor`` carry ``patch_expand_factor`` on top of
|
||||
``patch_size * merge_size``; ``t_factor`` is ``temporal_patch_size``. For
|
||||
a still image ``t = t_factor = temporal_patch_size`` so ``t_bar =
|
||||
temporal_patch_size``.
|
||||
"""
|
||||
if min(t, h, w, t_factor, h_factor, w_factor) <= 0:
|
||||
raise ValueError("Image dimensions and alignment factors must be positive.")
|
||||
if min_pixels <= 0 or max_pixels <= 0:
|
||||
raise ValueError("min_pixels and max_pixels must be positive.")
|
||||
if min_pixels > max_pixels:
|
||||
raise ValueError("min_pixels must be less than or equal to max_pixels.")
|
||||
|
||||
t_bar = max(t_factor, round(t / t_factor) * t_factor)
|
||||
h_bar = _ceil_to_factor(h, h_factor)
|
||||
w_bar = _ceil_to_factor(w, w_factor)
|
||||
|
||||
if t_bar * h_bar * w_bar > max_pixels:
|
||||
h_bar, w_bar = _fit_aligned_size_within_budget(
|
||||
t=t_bar,
|
||||
h=h,
|
||||
w=w,
|
||||
h_factor=h_factor,
|
||||
w_factor=w_factor,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
elif t_bar * h_bar * w_bar < min_pixels:
|
||||
beta = math.sqrt(min_pixels / (t * h * w))
|
||||
h_bar = _ceil_to_factor(max(1, math.ceil(h * beta)), h_factor)
|
||||
w_bar = _ceil_to_factor(max(1, math.ceil(w * beta)), w_factor)
|
||||
|
||||
# Alignment can push a candidate slightly over a tight max_pixels
|
||||
# budget. Refit it when that happens.
|
||||
if t_bar * h_bar * w_bar > max_pixels:
|
||||
h_bar, w_bar = _fit_aligned_size_within_budget(
|
||||
t=t_bar,
|
||||
h=h,
|
||||
w=w,
|
||||
h_factor=h_factor,
|
||||
w_factor=w_factor,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
|
||||
return h_bar, w_bar
|
||||
|
||||
|
||||
def _get_pad_content_size(
|
||||
image_height: int,
|
||||
image_width: int,
|
||||
canvas_height: int,
|
||||
canvas_width: int,
|
||||
allow_upscale: bool = False,
|
||||
) -> tuple[int, int]:
|
||||
"""Aspect-ratio-preserving content size that fits the canvas.
|
||||
|
||||
Oversized images are shrunk proportionally. Small images are enlarged
|
||||
only when ``allow_upscale``. Padding is applied after the resize.
|
||||
"""
|
||||
scale = min(canvas_height / image_height, canvas_width / image_width)
|
||||
if not allow_upscale:
|
||||
scale = min(1.0, scale)
|
||||
content_height = max(1, min(canvas_height, math.floor(image_height * scale)))
|
||||
content_width = max(1, min(canvas_width, math.floor(image_width * scale)))
|
||||
return content_height, content_width
|
||||
|
||||
|
||||
def _resize_or_pad(
|
||||
stacked_images: torch.Tensor,
|
||||
target_height: int,
|
||||
target_width: int,
|
||||
resize_mode: str,
|
||||
resample: "PILImageResampling | tvF.InterpolationMode | int | None",
|
||||
resize,
|
||||
allow_upscale: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Resize onto the aligned canvas, or keep the aspect ratio and
|
||||
zero-pad the right/bottom sides (``resize_mode="pad"``)."""
|
||||
height, width = stacked_images.shape[-2:]
|
||||
|
||||
if resize_mode == "resize":
|
||||
return resize(
|
||||
stacked_images,
|
||||
size=SizeDict(height=target_height, width=target_width),
|
||||
resample=resample,
|
||||
)
|
||||
|
||||
if resize_mode != "pad":
|
||||
raise ValueError("resize_mode must be either 'resize' or 'pad'.")
|
||||
|
||||
content_height, content_width = _get_pad_content_size(
|
||||
image_height=height,
|
||||
image_width=width,
|
||||
canvas_height=target_height,
|
||||
canvas_width=target_width,
|
||||
allow_upscale=allow_upscale,
|
||||
)
|
||||
|
||||
if (content_height, content_width) != (height, width):
|
||||
stacked_images = resize(
|
||||
stacked_images,
|
||||
size=SizeDict(height=content_height, width=content_width),
|
||||
resample=resample,
|
||||
)
|
||||
|
||||
# torchvision padding order: [left, top, right, bottom] -> pad only the
|
||||
# right and bottom sides.
|
||||
return tvF.pad(
|
||||
stacked_images,
|
||||
padding=[0, 0, target_width - content_width, target_height - content_height],
|
||||
fill=0,
|
||||
)
|
||||
|
||||
|
||||
def _pixel_budget(
|
||||
min_image_tokens: int | None,
|
||||
max_image_tokens: int | None,
|
||||
patch_size: int,
|
||||
merge_size: int,
|
||||
temporal_patch_size: int,
|
||||
) -> tuple[int, int]:
|
||||
"""(min_pixels, max_pixels) from the token bounds of
|
||||
``processor_config.json``; one vision token covers
|
||||
``temporal_patch_size * (patch_size * merge_size) ** 2`` pixels."""
|
||||
if min_image_tokens is None or max_image_tokens is None:
|
||||
raise ValueError(
|
||||
"min_image_tokens and max_image_tokens must be provided by "
|
||||
"processor_config.json (or per-call kwargs)."
|
||||
)
|
||||
factor = temporal_patch_size * (patch_size * merge_size) ** 2
|
||||
return min_image_tokens * factor, max_image_tokens * factor
|
||||
|
||||
|
||||
class Glm5NextImageProcessorKwargs(ImagesKwargs, total=False): # type: ignore[call-arg]
|
||||
patch_size: int | None
|
||||
temporal_patch_size: int | None
|
||||
merge_size: int | None
|
||||
patch_expand_factor: int | None
|
||||
resize_mode: str | None
|
||||
min_image_tokens: int | None
|
||||
max_image_tokens: int | None
|
||||
|
||||
|
||||
class Glm5NextImageProcessor(TorchvisionBackend):
|
||||
"""Fast torchvision image processor for GLM-5.3-Flash.
|
||||
|
||||
``patch_expand_factor`` multiplies into the ``smart_resize`` spatial
|
||||
factor ``patch_size * merge_size``. ``resize_mode`` picks the geometry:
|
||||
``"pad"`` (default) preserves the aspect ratio and zero-pads the
|
||||
right/bottom of the upward-aligned canvas, ``"resize"`` stretches onto
|
||||
it. Defaults mirror the checkpoint's ``image_processor`` config.
|
||||
"""
|
||||
|
||||
do_resize = True
|
||||
resample = PILImageResampling.BICUBIC
|
||||
size = {"longest_edge": 1} # unused: budgets come from the token bounds
|
||||
do_rescale = True
|
||||
do_normalize = True
|
||||
image_mean = OPENAI_CLIP_MEAN
|
||||
image_std = OPENAI_CLIP_STD
|
||||
do_convert_rgb = True
|
||||
patch_size = 14
|
||||
temporal_patch_size = 2
|
||||
merge_size = 2
|
||||
patch_expand_factor = 1
|
||||
resize_mode = "pad"
|
||||
min_image_tokens = 16
|
||||
max_image_tokens = 8000
|
||||
valid_kwargs = Glm5NextImageProcessorKwargs
|
||||
model_input_names = ["pixel_values", "image_grid_thw"]
|
||||
|
||||
def _preprocess(
|
||||
self,
|
||||
images: list[torch.Tensor],
|
||||
do_resize: bool,
|
||||
size: SizeDict,
|
||||
resample: "PILImageResampling | tvF.InterpolationMode | int | None",
|
||||
do_rescale: bool,
|
||||
rescale_factor: float,
|
||||
do_normalize: bool,
|
||||
image_mean: float | list[float] | None,
|
||||
image_std: float | list[float] | None,
|
||||
patch_size: int,
|
||||
temporal_patch_size: int,
|
||||
merge_size: int,
|
||||
patch_expand_factor: int,
|
||||
resize_mode: str | None,
|
||||
min_image_tokens: int | None,
|
||||
max_image_tokens: int | None,
|
||||
disable_grouping: bool | None,
|
||||
return_tensors: str | TensorType | None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
resize_mode = resize_mode if resize_mode is not None else self.resize_mode
|
||||
min_pixels, max_pixels = _pixel_budget(
|
||||
min_image_tokens if min_image_tokens is not None else self.min_image_tokens,
|
||||
max_image_tokens if max_image_tokens is not None else self.max_image_tokens,
|
||||
patch_size,
|
||||
merge_size,
|
||||
temporal_patch_size,
|
||||
)
|
||||
grouped_images, grouped_images_index = group_images_by_shape(
|
||||
images, disable_grouping=disable_grouping
|
||||
)
|
||||
resized_images_grouped = {}
|
||||
for shape, stacked_images in grouped_images.items():
|
||||
height, width = stacked_images.shape[-2:]
|
||||
if do_resize:
|
||||
resized_height, resized_width = smart_resize(
|
||||
t=temporal_patch_size,
|
||||
h=height,
|
||||
w=width,
|
||||
t_factor=temporal_patch_size,
|
||||
h_factor=patch_size * merge_size * patch_expand_factor,
|
||||
w_factor=patch_size * merge_size * patch_expand_factor,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
stacked_images = _resize_or_pad(
|
||||
stacked_images,
|
||||
target_height=resized_height,
|
||||
target_width=resized_width,
|
||||
resize_mode=resize_mode,
|
||||
resample=resample,
|
||||
resize=self.resize,
|
||||
allow_upscale=(temporal_patch_size * height * width < min_pixels),
|
||||
)
|
||||
resized_images_grouped[shape] = stacked_images
|
||||
|
||||
resized_images = reorder_images(resized_images_grouped, grouped_images_index)
|
||||
|
||||
grouped_images, grouped_images_index = group_images_by_shape(
|
||||
resized_images, disable_grouping=disable_grouping
|
||||
)
|
||||
processed_images_grouped = {}
|
||||
processed_grids = {}
|
||||
|
||||
for shape, stacked_images in grouped_images.items():
|
||||
resized_height, resized_width = stacked_images.shape[-2:]
|
||||
|
||||
patches = self.rescale_and_normalize(
|
||||
stacked_images,
|
||||
do_rescale,
|
||||
rescale_factor,
|
||||
do_normalize,
|
||||
image_mean,
|
||||
image_std,
|
||||
)
|
||||
if patches.ndim == 4: # (B, C, H, W)
|
||||
patches = patches.unsqueeze(1) # (B, T=1, C, H, W)
|
||||
|
||||
if patches.shape[1] % temporal_patch_size != 0:
|
||||
repeats = patches[:, -1:].repeat(
|
||||
1,
|
||||
temporal_patch_size - (patches.shape[1] % temporal_patch_size),
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
)
|
||||
patches = torch.cat([patches, repeats], dim=1)
|
||||
|
||||
batch_size, t_len, channel = patches.shape[:3]
|
||||
grid_t = t_len // temporal_patch_size
|
||||
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
||||
|
||||
patches = patches.view(
|
||||
batch_size,
|
||||
grid_t,
|
||||
temporal_patch_size,
|
||||
channel,
|
||||
grid_h // merge_size,
|
||||
merge_size,
|
||||
patch_size,
|
||||
grid_w // merge_size,
|
||||
merge_size,
|
||||
patch_size,
|
||||
)
|
||||
# (B, grid_t, gh, gw, mh, mw, C, tp, ph, pw)
|
||||
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
|
||||
|
||||
flatten_patches = patches.reshape(
|
||||
batch_size,
|
||||
grid_t * grid_h * grid_w,
|
||||
channel * temporal_patch_size * patch_size * patch_size,
|
||||
)
|
||||
|
||||
processed_images_grouped[shape] = flatten_patches
|
||||
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
|
||||
|
||||
processed_images = reorder_images(
|
||||
processed_images_grouped, grouped_images_index
|
||||
)
|
||||
processed_grids = reorder_images(processed_grids, grouped_images_index)
|
||||
|
||||
pixel_values = torch.cat(processed_images, dim=0)
|
||||
image_grid_thw = torch.tensor(processed_grids)
|
||||
|
||||
return BatchFeature(
|
||||
data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
|
||||
def preprocess(
|
||||
self, images: ImageInput, **kwargs: Unpack[Glm5NextImageProcessorKwargs]
|
||||
) -> BatchFeature:
|
||||
return super().preprocess(images, **kwargs)
|
||||
|
||||
def get_number_of_image_patches(
|
||||
self, height: int, width: int, images_kwargs: dict | None = None
|
||||
) -> int:
|
||||
"""Number of image patches (pre-merge) for a given (height, width)."""
|
||||
images_kwargs = images_kwargs or {}
|
||||
patch_size = images_kwargs.get("patch_size", self.patch_size)
|
||||
merge_size = images_kwargs.get("merge_size", self.merge_size)
|
||||
patch_expand_factor = images_kwargs.get(
|
||||
"patch_expand_factor", self.patch_expand_factor
|
||||
)
|
||||
min_pixels, max_pixels = _pixel_budget(
|
||||
images_kwargs.get("min_image_tokens", self.min_image_tokens),
|
||||
images_kwargs.get("max_image_tokens", self.max_image_tokens),
|
||||
patch_size,
|
||||
merge_size,
|
||||
self.temporal_patch_size,
|
||||
)
|
||||
resized_height, resized_width = smart_resize(
|
||||
t=self.temporal_patch_size,
|
||||
h=height,
|
||||
w=width,
|
||||
t_factor=self.temporal_patch_size,
|
||||
h_factor=patch_size * merge_size * patch_expand_factor,
|
||||
w_factor=patch_size * merge_size * patch_expand_factor,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
||||
return grid_h * grid_w
|
||||
|
||||
|
||||
class Glm5NextVideoProcessorKwargs(VideosKwargs, total=False): # type: ignore[call-arg]
|
||||
fps: list[float] | float
|
||||
patch_size: int
|
||||
temporal_patch_size: int
|
||||
merge_size: int
|
||||
patch_expand_factor: int
|
||||
resize_mode: str | None
|
||||
target_fps: float | None
|
||||
max_frames: int | None
|
||||
fps_interval: int | None
|
||||
max_frame_count_dynamic: int | None
|
||||
min_image_tokens: int | None
|
||||
max_image_tokens: int | None
|
||||
|
||||
|
||||
class Glm5NextVideoProcessor(BaseVideoProcessor):
|
||||
"""Fast video processor for GLM-5.3-Flash.
|
||||
|
||||
Shares ``smart_resize`` / the pad-mode geometry / the patchify with the
|
||||
image processor, and adds GLM-5.3-Flash frame sampling
|
||||
(``glm_sample_frame_indices``: ``fps_interval`` semantics with a
|
||||
temporal-patch-scaled greedy walk). Defaults mirror the checkpoint's
|
||||
``video_processor`` config.
|
||||
"""
|
||||
|
||||
resample = PILImageResampling.BICUBIC
|
||||
size = {"longest_edge": 1} # unused: budgets come from the token bounds
|
||||
image_mean = OPENAI_CLIP_MEAN
|
||||
image_std = OPENAI_CLIP_STD
|
||||
do_resize = True
|
||||
do_rescale = True
|
||||
do_normalize = True
|
||||
do_convert_rgb = True
|
||||
do_sample_frames = True
|
||||
patch_size = 14
|
||||
temporal_patch_size = 2
|
||||
patch_expand_factor = 1
|
||||
merge_size = 2
|
||||
valid_kwargs = Glm5NextVideoProcessorKwargs
|
||||
num_frames = 16
|
||||
fps = 2
|
||||
fps_interval = 2.0
|
||||
max_frame_count_dynamic = 2048
|
||||
resize_mode = "pad"
|
||||
min_image_tokens = 16
|
||||
max_image_tokens = 240000
|
||||
model_input_names = ["pixel_values_videos", "video_grid_thw"]
|
||||
|
||||
def sample_frames(
|
||||
self,
|
||||
metadata: VideoMetadata,
|
||||
fps: int | float | None = None,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
"""Sample frame indices with GLM's fps-interval policy.
|
||||
|
||||
``fps`` / ``target_fps``, ``max_frames`` and ``fps_interval`` /
|
||||
``max_frame_count_dynamic`` are the overrides described in
|
||||
:func:`glm_sample_frame_indices`.
|
||||
"""
|
||||
if metadata is None or getattr(metadata, "fps", None) is None:
|
||||
raise ValueError(
|
||||
"Asked to sample frames per second but no video metadata was "
|
||||
"provided which is required when sampling in GLM-5.3-Flash. Please "
|
||||
"pass in `VideoMetadata` object or set `do_sample_frames=False`."
|
||||
)
|
||||
|
||||
target_fps = fps if fps is not None else kwargs.get("target_fps")
|
||||
if target_fps is None:
|
||||
target_fps = self.fps_interval
|
||||
indices = glm_sample_frame_indices(
|
||||
metadata.total_num_frames,
|
||||
metadata.fps,
|
||||
metadata.duration or 0,
|
||||
target_fps=target_fps,
|
||||
max_frame_count=kwargs.get("max_frames") or self.max_frame_count_dynamic,
|
||||
temporal_patch_size=self.temporal_patch_size,
|
||||
)
|
||||
return np.array(indices)
|
||||
|
||||
def _preprocess(
|
||||
self,
|
||||
videos: list[torch.Tensor],
|
||||
do_convert_rgb: bool = True,
|
||||
do_resize: bool = True,
|
||||
size: SizeDict | None = None,
|
||||
resample: "PILImageResampling | int | None" = PILImageResampling.BICUBIC,
|
||||
do_rescale: bool = True,
|
||||
rescale_factor: float = 1 / 255.0,
|
||||
do_normalize: bool = True,
|
||||
image_mean: float | list[float] | None = None,
|
||||
image_std: float | list[float] | None = None,
|
||||
patch_size: int | None = None,
|
||||
temporal_patch_size: int | None = None,
|
||||
patch_expand_factor: int | None = None,
|
||||
merge_size: int | None = None,
|
||||
resize_mode: str | None = None,
|
||||
min_image_tokens: int | None = None,
|
||||
max_image_tokens: int | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
patch_expand_factor = self.patch_expand_factor
|
||||
patch_size = patch_size if patch_size is not None else self.patch_size
|
||||
temporal_patch_size = (
|
||||
temporal_patch_size
|
||||
if temporal_patch_size is not None
|
||||
else self.temporal_patch_size
|
||||
)
|
||||
merge_size = merge_size if merge_size is not None else self.merge_size
|
||||
resize_mode = resize_mode if resize_mode is not None else self.resize_mode
|
||||
min_pixels, max_pixels = _pixel_budget(
|
||||
min_image_tokens if min_image_tokens is not None else self.min_image_tokens,
|
||||
max_image_tokens if max_image_tokens is not None else self.max_image_tokens,
|
||||
patch_size,
|
||||
merge_size,
|
||||
temporal_patch_size,
|
||||
)
|
||||
grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
|
||||
resized_videos_grouped = {}
|
||||
for shape, stacked_videos in grouped_videos.items():
|
||||
if do_convert_rgb:
|
||||
stacked_videos = self.convert_to_rgb(stacked_videos)
|
||||
b, t_len, c, h, w = stacked_videos.shape
|
||||
num_frames, height, width = t_len, h, w
|
||||
if do_resize:
|
||||
resized_height, resized_width = smart_resize(
|
||||
t=num_frames,
|
||||
h=height,
|
||||
w=width,
|
||||
t_factor=temporal_patch_size,
|
||||
h_factor=patch_size * merge_size * patch_expand_factor,
|
||||
w_factor=patch_size * merge_size * patch_expand_factor,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
stacked_videos = stacked_videos.view(b * t_len, c, h, w)
|
||||
stacked_videos = _resize_or_pad(
|
||||
stacked_videos,
|
||||
target_height=resized_height,
|
||||
target_width=resized_width,
|
||||
resize_mode=resize_mode,
|
||||
resample=resample,
|
||||
resize=self.resize,
|
||||
allow_upscale=(num_frames * height * width < min_pixels),
|
||||
)
|
||||
stacked_videos = stacked_videos.view(
|
||||
b, t_len, c, resized_height, resized_width
|
||||
)
|
||||
resized_videos_grouped[shape] = stacked_videos
|
||||
resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
|
||||
|
||||
grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
|
||||
processed_videos_grouped = {}
|
||||
processed_grids = {}
|
||||
for shape, stacked_videos in grouped_videos.items():
|
||||
resized_height, resized_width = get_image_size(
|
||||
stacked_videos[0], channel_dim=ChannelDimension.FIRST
|
||||
)
|
||||
stacked_videos = self.rescale_and_normalize(
|
||||
stacked_videos,
|
||||
do_rescale,
|
||||
rescale_factor,
|
||||
do_normalize,
|
||||
image_mean,
|
||||
image_std,
|
||||
)
|
||||
patches = stacked_videos
|
||||
|
||||
if pad := -patches.shape[1] % temporal_patch_size:
|
||||
repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1)
|
||||
patches = torch.cat((patches, repeats), dim=1)
|
||||
batch_size, grid_t, channel = patches.shape[:3]
|
||||
grid_t = grid_t // temporal_patch_size
|
||||
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
||||
|
||||
patches = patches.view(
|
||||
batch_size,
|
||||
grid_t,
|
||||
temporal_patch_size,
|
||||
channel,
|
||||
grid_h // merge_size,
|
||||
merge_size,
|
||||
patch_size,
|
||||
grid_w // merge_size,
|
||||
merge_size,
|
||||
patch_size,
|
||||
)
|
||||
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
|
||||
flatten_patches = patches.reshape(
|
||||
batch_size,
|
||||
grid_t * grid_h * grid_w,
|
||||
channel * temporal_patch_size * patch_size * patch_size,
|
||||
)
|
||||
|
||||
processed_videos_grouped[shape] = flatten_patches
|
||||
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
|
||||
|
||||
processed_videos = reorder_videos(
|
||||
processed_videos_grouped, grouped_videos_index
|
||||
)
|
||||
processed_grids = reorder_videos(processed_grids, grouped_videos_index)
|
||||
pixel_values_videos = torch.cat(processed_videos, dim=0)
|
||||
video_grid_thw = torch.tensor(processed_grids)
|
||||
return BatchFeature(
|
||||
data={
|
||||
"pixel_values_videos": pixel_values_videos,
|
||||
"video_grid_thw": video_grid_thw,
|
||||
},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
|
||||
|
||||
class Glm5NextProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
|
||||
images_kwargs: Glm5NextImageProcessorKwargs
|
||||
videos_kwargs: Glm5NextVideoProcessorKwargs
|
||||
_defaults = {
|
||||
"text_kwargs": {
|
||||
"padding": False,
|
||||
"return_token_type_ids": False,
|
||||
"return_mm_token_type_ids": False,
|
||||
},
|
||||
"videos_kwargs": {"return_metadata": True},
|
||||
}
|
||||
|
||||
|
||||
class Glm5NextProcessor(ProcessorMixin):
|
||||
"""Wraps a GLM-5.3-Flash image processor, video processor and tokenizer.
|
||||
|
||||
Token expansion per image = ``prod(image_grid_thw) // merge_size**2``; video
|
||||
frames are expanded with ``<|begin_of_image|>...<|end_of_image|>{ts} seconds``
|
||||
structure (mrope timestamps).
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer", "video_processor"]
|
||||
image_processor_class = "AutoImageProcessor"
|
||||
video_processor_class = "AutoVideoProcessor"
|
||||
tokenizer_class = ("PythonBackend", "TokenizersBackend")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor=None,
|
||||
tokenizer=None,
|
||||
video_processor=None,
|
||||
chat_template=None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
image_processor, tokenizer, video_processor, chat_template=chat_template
|
||||
)
|
||||
self.image_token = (
|
||||
"<|image|>"
|
||||
if not hasattr(tokenizer, "image_token")
|
||||
else tokenizer.image_token
|
||||
)
|
||||
self.video_token = (
|
||||
"<|video|>"
|
||||
if not hasattr(tokenizer, "video_token")
|
||||
else tokenizer.video_token
|
||||
)
|
||||
self.image_token_id = (
|
||||
tokenizer.image_token_id
|
||||
if getattr(tokenizer, "image_token_id", None)
|
||||
else tokenizer.convert_tokens_to_ids(self.image_token)
|
||||
)
|
||||
self.video_token_id = (
|
||||
tokenizer.video_token_id
|
||||
if getattr(tokenizer, "video_token_id", None)
|
||||
else tokenizer.convert_tokens_to_ids(self.video_token)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
|
||||
"""Build the processor directly from the checkpoint config.
|
||||
|
||||
GLM-5.3-Flash stores nested image/video configs in
|
||||
``processor_config.json`` and declares a custom processor class. This
|
||||
method reads those configs directly and caps only the video token budget.
|
||||
"""
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
model_path = pretrained_model_name_or_path
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_path, **kwargs)
|
||||
|
||||
def _cap_cfg(cfg: dict, *, is_video: bool) -> dict:
|
||||
# Video keeps a serving token cap (the checkpoint's 240k-token
|
||||
# budget would starve the KV cache at startup profiling); images
|
||||
# follow the checkpoint budget verbatim so preprocessing matches
|
||||
# the HF reference exactly.
|
||||
if is_video and cfg.get("max_image_tokens") is not None:
|
||||
cfg["max_image_tokens"] = min(
|
||||
cfg["max_image_tokens"], _MAX_VIDEO_TOKENS
|
||||
)
|
||||
return cfg
|
||||
|
||||
ip_cfg = _cap_cfg(
|
||||
dict(get_image_processor_config(model_path, **kwargs)), is_video=False
|
||||
)
|
||||
image_processor = Glm5NextImageProcessor(
|
||||
**{k: v for k, v in ip_cfg.items() if k != "image_processor_type"}
|
||||
)
|
||||
|
||||
processor_config = get_hf_file_to_dict(
|
||||
"processor_config.json",
|
||||
model_path,
|
||||
revision=kwargs.get("revision", "main"),
|
||||
)
|
||||
if processor_config is None:
|
||||
raise ValueError(f"Missing processor_config.json for {model_path}")
|
||||
vp_cfg = _cap_cfg(dict(processor_config["video_processor"]), is_video=True)
|
||||
video_processor = Glm5NextVideoProcessor(
|
||||
**{k: v for k, v in vp_cfg.items() if k != "video_processor_type"}
|
||||
)
|
||||
|
||||
return cls(
|
||||
image_processor=image_processor,
|
||||
tokenizer=tokenizer,
|
||||
video_processor=video_processor,
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: ImageInput | None = None,
|
||||
text: TextInput
|
||||
| PreTokenizedInput
|
||||
| list[TextInput]
|
||||
| list[PreTokenizedInput] = None,
|
||||
videos: VideoInput | None = None,
|
||||
**kwargs: Unpack[Glm5NextProcessorKwargs],
|
||||
) -> BatchFeature:
|
||||
output_kwargs = self._merge_kwargs(
|
||||
Glm5NextProcessorKwargs,
|
||||
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
if images is not None:
|
||||
image_inputs = self.image_processor(
|
||||
images=images, **output_kwargs["images_kwargs"]
|
||||
)
|
||||
else:
|
||||
image_inputs = {}
|
||||
|
||||
if videos is not None:
|
||||
videos_inputs = self.video_processor(
|
||||
videos=videos, **output_kwargs["videos_kwargs"]
|
||||
)
|
||||
if "return_metadata" not in kwargs:
|
||||
videos_inputs.pop("video_metadata")
|
||||
else:
|
||||
videos_inputs = {}
|
||||
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
|
||||
# Prompt updates expand the unchanged image/video markers after this call.
|
||||
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
|
||||
return_mm_token_type_ids = output_kwargs["text_kwargs"].pop(
|
||||
"return_mm_token_type_ids", False
|
||||
)
|
||||
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
|
||||
|
||||
if return_mm_token_type_ids:
|
||||
array_ids = np.array(text_inputs["input_ids"])
|
||||
mm_token_type_ids = np.zeros_like(text_inputs["input_ids"])
|
||||
mm_token_type_ids[array_ids == self.image_token_id] = 1
|
||||
text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist()
|
||||
return BatchFeature(
|
||||
data={**text_inputs, **image_inputs, **videos_inputs},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
|
||||
def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs):
|
||||
vision_data = {}
|
||||
if image_sizes is not None:
|
||||
images_kwargs = Glm5NextProcessorKwargs._defaults.get("images_kwargs", {})
|
||||
images_kwargs.update(kwargs)
|
||||
merge_size = (
|
||||
images_kwargs.get("merge_size", None) or self.image_processor.merge_size
|
||||
)
|
||||
|
||||
num_image_patches = [
|
||||
self.image_processor.get_number_of_image_patches(
|
||||
*image_size, images_kwargs
|
||||
)
|
||||
for image_size in image_sizes
|
||||
]
|
||||
num_image_tokens = [(n // merge_size**2) for n in num_image_patches]
|
||||
vision_data.update(
|
||||
{
|
||||
"num_image_tokens": num_image_tokens,
|
||||
"num_image_patches": num_image_patches,
|
||||
}
|
||||
)
|
||||
|
||||
if video_sizes is not None:
|
||||
videos_kwargs = Glm5NextProcessorKwargs._defaults.get("videos_kwargs", {})
|
||||
videos_kwargs.update(kwargs)
|
||||
num_video_patches = [
|
||||
self.video_processor.get_number_of_video_patches(
|
||||
*video_size, videos_kwargs
|
||||
)
|
||||
for video_size in video_sizes
|
||||
]
|
||||
num_video_tokens = [(n // merge_size**2) for n in num_video_patches]
|
||||
vision_data["num_video_tokens"] = num_video_tokens
|
||||
|
||||
return MultiModalData(**vision_data)
|
||||
|
||||
def post_process_image_text_to_text(
|
||||
self,
|
||||
generated_outputs,
|
||||
skip_special_tokens=True,
|
||||
clean_up_tokenization_spaces=False,
|
||||
**kwargs,
|
||||
):
|
||||
return self.tokenizer.batch_decode(
|
||||
generated_outputs,
|
||||
skip_special_tokens=skip_special_tokens,
|
||||
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Glm5NextImageProcessor",
|
||||
"Glm5NextVideoProcessor",
|
||||
"Glm5NextProcessor",
|
||||
"smart_resize",
|
||||
]
|
||||
Reference in New Issue
Block a user