[Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-10-02 02:25:18 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 866f02db8a
commit e3cae8d2ac
36 changed files with 366 additions and 2385 deletions
-5
View File
@@ -107,9 +107,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
k3_dspark="K3DSparkConfig",
funaudiochat="FunAudioChatConfig",
granite4_vision="Granite4VisionConfig",
glm5_next="Glm5NextConfig",
glm5_next_text="Glm5NextTextConfig",
glm5_next_vision="Glm5NextVisionConfig",
hyperclovax="HyperCLOVAXConfig",
hy_v3="HYV3Config",
hy_v4="HYV4Config",
@@ -145,8 +142,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
qianfan_ocr="QianfanOCRConfig",
qwen3_asr="Qwen3ASRConfig",
qwen3_next="Qwen3NextConfig",
qwen4_exp="Qwen4ExpConfig",
qwen4_exp_text="Qwen4ExpTextConfig",
qwen3_5="Qwen3_5Config",
qwen3_5_text="Qwen3_5TextConfig",
qwen3_5_moe="Qwen3_5MoeConfig",
@@ -43,9 +43,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
"FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat",
"FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
"Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision",
"Glm5NextConfig": "vllm.transformers_utils.configs.glm5_next",
"Glm5NextTextConfig": "vllm.transformers_utils.configs.glm5_next",
"Glm5NextVisionConfig": "vllm.transformers_utils.configs.glm5_next",
"HYV3Config": "vllm.transformers_utils.configs.hy_v3",
"HYV4Config": "vllm.transformers_utils.configs.hy_v4",
"HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
@@ -95,9 +92,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
"QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr",
"Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr",
"Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next",
"Qwen4ExpConfig": "vllm.transformers_utils.configs.qwen4_exp",
"Qwen4ExpTextConfig": "vllm.transformers_utils.configs.qwen4_exp",
"Qwen4ExpVisionConfig": "vllm.transformers_utils.configs.qwen4_exp",
"Qwen3_5Config": "vllm.transformers_utils.configs.qwen3_5",
"Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5",
"Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
@@ -140,9 +134,6 @@ __all__ = [
"FunAudioChatConfig",
"FunAudioChatAudioEncoderConfig",
"Granite4VisionConfig",
"Glm5NextConfig",
"Glm5NextTextConfig",
"Glm5NextVisionConfig",
"HYV3Config",
"HYV4Config",
"HyperCLOVAXConfig",
@@ -187,9 +178,6 @@ __all__ = [
"QianfanOCRVisionConfig",
"Qwen3ASRConfig",
"Qwen3NextConfig",
"Qwen4ExpConfig",
"Qwen4ExpTextConfig",
"Qwen4ExpVisionConfig",
"Qwen3_5Config",
"Qwen3_5TextConfig",
"Qwen3_5MoeConfig",
@@ -1,420 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers.configuration_utils import PreTrainedConfig
from vllm.logger import init_logger
logger = init_logger(__name__)
class Glm5NextTextConfig(PreTrainedConfig):
model_type = "glm5_next_text"
base_config_key = "text_config"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
model_type="glm5_next_text",
vocab_size: int = 154880,
hidden_size: int = 4096,
head_dim: int | None = None,
intermediate_size: int = 12288,
num_hidden_layers: int = 45,
num_attention_heads: int = 64,
num_key_value_heads: int | None = None,
hidden_act: str = "silu",
rms_norm_eps: float = 1e-5,
pad_token_id: int | None = 151329,
bos_token_id: int | None = None,
eos_token_id: int | list[int] | None = None,
rope_parameters: dict | None = None,
max_position_embeddings: int = 1048576,
tie_word_embeddings: bool = False,
moe_intermediate_size: int = 2048,
moe_renormalize: bool = True,
scoring_func: str = "sigmoid",
n_routed_experts: int | None = 288,
num_experts_per_token: int = 7,
n_shared_experts: int = 1,
routed_scaling_factor: float = 2.5,
topk_method: str | None = None,
first_k_dense_replace: int = 0,
moe_layer_freq: int = 1,
use_grouped_topk: bool = True,
n_group: int = 1,
topk_group: int = 1,
mla: bool = True,
q_lora_rank: int | None = 1536,
kv_lora_rank: int | None = 512,
qk_nope_head_dim: int = 256,
qk_rope_head_dim: int = 0,
v_head_dim: int | None = 256,
mla_nope: bool | None = True,
num_nextn_predict_layers: int = 1,
# Per-layer layout: "linear_attention" | "deepseek_sparse_attention"
layer_types: list[str] | None = None,
# Per-layer MLP: "dense" | "sparse"
mlp_layer_types: list[str] | None = None,
# Linear-attention (KDA) head config (flattened from the old
# linear_attn_config dict).
linear_head_dim: int = 128,
linear_num_heads: int = 64,
linear_conv_kernel_dim: int = 4,
linear_lower_bound: float = -5.0,
index_head_dim: int | None = None,
index_topk: int | None = None,
index_n_heads: int | None = None,
index_dsa_use_layernorm: bool = True,
index_kpool_compress: bool = True,
# Every ``index_kpool`` indexer K entries pool into one stored entry
# (compress_ratio). The 300B checkpoint ships 4; topk runs at pool
# granularity (select_k = index_topk // index_kpool).
index_kpool: int | None = 4,
index_kpool_always_select_tail: bool = True,
indexer_rope_interleave: bool = False,
mhc: bool | None = True,
mhc_num_residual_streams: int = 4,
hc_eps: float | None = 1e-06,
mhc_tau: float = 0.05,
hres_vwnstyle: bool | None = True,
mhc_no_norm_weight: bool | None = False,
mhc_sinkhorn_iterations: int | None = 20,
mhc_post_mult_value: float | None = 2.0,
swiglu_limit: float | None = None,
logit_scale: float = 1.0,
**kwargs,
):
# Preserve checkpoint field names and local aliases because their
# consumers use different spellings.
num_experts_per_token = kwargs.get("num_experts_per_tok", num_experts_per_token)
moe_renormalize = kwargs.get("norm_topk_prob", moe_renormalize)
mhc_num_residual_streams = kwargs.get("hc_mult", mhc_num_residual_streams)
mhc_sinkhorn_iterations = kwargs.get(
"hc_sinkhorn_iters", mhc_sinkhorn_iterations
)
# Checkpoint ships ``mla_use_nope`` (not ``mla_nope``); without this
# alias self.mla_nope silently stays at the param default.
mla_nope = kwargs.get("mla_use_nope", mla_nope)
# Checkpoints ship the KDA head config as the ``linear_attn_config``
# dict (head_dim / num_heads / short_conv_kernel_size /
# gate_lower_bound) rather than the flattened top-level fields; fold it
# in so the trained values are read instead of the param defaults
# (which only match this checkpoint by coincidence).
linear_cfg = kwargs.get("linear_attn_config") or {}
if linear_cfg:
linear_head_dim = linear_cfg.get("head_dim", linear_head_dim)
linear_num_heads = linear_cfg.get("num_heads", linear_num_heads)
linear_conv_kernel_dim = linear_cfg.get(
"short_conv_kernel_size", linear_conv_kernel_dim
)
linear_lower_bound = linear_cfg.get("gate_lower_bound", linear_lower_bound)
if index_topk is not None:
if index_dsa_use_layernorm is not True:
raise NotImplementedError(
"GLM-5.3 sparse indexer requires index_dsa_use_layernorm=True"
)
if index_kpool_compress is not True:
raise NotImplementedError(
"GLM-5.3 sparse indexer requires index_kpool_compress=True"
)
if index_kpool_always_select_tail is not True:
raise NotImplementedError(
"GLM-5.3 sparse indexer requires "
"index_kpool_always_select_tail=True"
)
if mhc:
if hres_vwnstyle is not True:
raise NotImplementedError("GLM-5.3 mHC requires hres_vwnstyle=True")
if mhc_no_norm_weight not in (False, None):
raise NotImplementedError(
"GLM-5.3 mHC requires mhc_no_norm_weight=False"
)
self.model_type = model_type
self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.head_dim = (
head_dim if head_dim is not None else hidden_size // num_attention_heads
)
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
# for backward compatibility
if num_key_value_heads is None:
num_key_value_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.hidden_act = hidden_act
self.rms_norm_eps = rms_norm_eps
self.max_position_embeddings = max_position_embeddings
self.rope_parameters = rope_parameters
# mla config
self.mla = mla
self.q_lora_rank = q_lora_rank
self.kv_lora_rank = kv_lora_rank
self.qk_nope_head_dim = qk_nope_head_dim
self.qk_rope_head_dim = qk_rope_head_dim
self.v_head_dim = v_head_dim
self.mla_nope = mla_nope
# moe config
self.n_routed_experts = n_routed_experts
self.num_experts_per_token = num_experts_per_token
self.moe_renormalize = moe_renormalize
self.n_shared_experts = n_shared_experts
self.routed_scaling_factor = routed_scaling_factor
self.topk_method = topk_method
self.scoring_func = scoring_func
assert self.scoring_func in ("softmax", "sigmoid")
self.moe_intermediate_size = moe_intermediate_size
self.first_k_dense_replace = first_k_dense_replace
self.moe_layer_freq = moe_layer_freq
self.use_grouped_topk = use_grouped_topk
self.n_group = n_group
self.topk_group = topk_group
self.num_nextn_predict_layers = num_nextn_predict_layers
# Per-layer attention / MLP layout. Normalize mlp_layer_types from
# first_k_dense_replace when the new-schema field is absent so layer
# construction sees a consistent layout (mirrors cohere2_moe).
self.layer_types = layer_types
if mlp_layer_types is None:
n = self.num_hidden_layers
if first_k_dense_replace is not None:
mlp_layer_types = ["dense"] * first_k_dense_replace + ["sparse"] * (
n - first_k_dense_replace
)
else:
mlp_layer_types = ["sparse"] * n
self.mlp_layer_types = mlp_layer_types
# Linear-attention (KDA) head config.
self.linear_head_dim = linear_head_dim
self.linear_num_heads = linear_num_heads
self.linear_conv_kernel_dim = linear_conv_kernel_dim
self.linear_lower_bound = linear_lower_bound
# dsa index config
self.index_head_dim = index_head_dim
self.index_topk = index_topk
self.index_n_heads = index_n_heads
self.index_dsa_use_layernorm = index_dsa_use_layernorm
self.index_kpool_compress = index_kpool_compress
self.index_kpool = index_kpool
self.index_kpool_always_select_tail = index_kpool_always_select_tail
self.indexer_rope_interleave = indexer_rope_interleave
# mhc config
self.mhc = mhc
self.mhc_num_residual_streams = mhc_num_residual_streams
self.mhc_tau = mhc_tau
self.hres_vwnstyle = hres_vwnstyle
self.hc_eps = hc_eps
self.mhc_no_norm_weight = mhc_no_norm_weight
self.mhc_sinkhorn_iterations = mhc_sinkhorn_iterations
self.mhc_post_mult_value = mhc_post_mult_value
self.swiglu_limit = swiglu_limit
self.logit_scale = logit_scale
super().__init__(
pad_token_id=pad_token_id,
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
@property
def is_mla(self):
return (
self.q_lora_rank is not None
or self.kv_lora_rank is not None
or self.qk_nope_head_dim is not None
or self.qk_rope_head_dim is not None
or self.v_head_dim is not None
or self.mla_nope is True
)
@property
def is_moe(self):
return self.n_routed_experts is not None
@property
def is_linear_attn(self) -> bool:
return self.layer_types is not None and any(
t == "linear_attention" for t in self.layer_types
)
def is_kda_layer(self, layer_idx: int):
return (
self.layer_types is not None
and layer_idx < len(self.layer_types)
and self.layer_types[layer_idx] == "linear_attention"
)
@property
def layers_block_type(self):
# Map the schema's per-layer types onto the block strings vLLM's hybrid
# accounting (get_num_layers_by_block_type) recognizes: linear-attention
# layers stay "linear_attention"; every other attention variant collapses
# to "attention".
if self.layer_types is None:
return ["attention"] * self.num_hidden_layers
return [
"linear_attention" if t == "linear_attention" else "attention"
for t in self.layer_types
]
class Glm5NextVisionConfig(PreTrainedConfig):
model_type = "glm5_next_vision"
base_config_key = "vision_config"
def __init__(
self,
depth: int = 24,
hidden_size: int = 1024,
hidden_act: str = "silu",
image_size: int = 448,
intermediate_size: int = 4096,
num_heads: int = 16,
out_hidden_size: int = 4096,
projection_intermediate_size: int = 10240,
in_channels: int = 3,
initializer_range: float = 0.02,
patch_size: int = 14,
rms_norm_eps: float = 1e-5,
spatial_merge_size: int = 2,
temporal_patch_size: int = 2,
attention_dropout: float = 0.0,
attention_bias: bool = True,
swiglu_limit: float | None = None,
**kwargs,
):
super().__init__(**kwargs)
self.depth = depth
self.hidden_size = hidden_size
self.hidden_act = hidden_act
self.image_size = image_size
self.intermediate_size = intermediate_size
self.num_heads = num_heads
self.out_hidden_size = out_hidden_size
# GLM-5.3-Flash merger bottleneck width (absent from the generic
# GLM-OCR vision config); the tower uses it as the PatchMerger
# context_dim instead of text_config.intermediate_size.
self.projection_intermediate_size = projection_intermediate_size
self.in_channels = in_channels
self.initializer_range = initializer_range
self.patch_size = patch_size
# GLM-5.3-Flash checkpoints ship vision_config.rms_norm_eps = 1e-5,
# but the vision tower was trained with 1e-6. Serving with 1e-5 drifts
# the RMSNorm and produces repetitive/degraded image descriptions, so
# force the trained value regardless of the checkpoint field.
self.rms_norm_eps = 1e-6
self.spatial_merge_size = spatial_merge_size
self.temporal_patch_size = temporal_patch_size
self.attention_dropout = attention_dropout
self.attention_bias = attention_bias
self.swiglu_limit = swiglu_limit
class Glm5NextConfig(PreTrainedConfig):
model_type = "glm5_next"
sub_configs = {
"vision_config": Glm5NextVisionConfig,
"text_config": Glm5NextTextConfig,
}
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
text_config=None,
vision_config=None,
image_token_id: int = 154854,
video_token_id: int = 154855,
image_start_token_id: int = 154830,
image_end_token_id: int = 154831,
video_start_token_id: int = 154832,
video_end_token_id: int = 154833,
**kwargs,
):
# Init super() first so base-class defaults don't clobber text-config
# values set below (PreTrainedConfig has many text-related defaults
# that differ from Glm5NextTextConfig).
super().__init__(**kwargs)
if isinstance(vision_config, dict):
self.vision_config = self.sub_configs["vision_config"](**vision_config)
elif vision_config is None:
self.vision_config = self.sub_configs["vision_config"]()
else:
self.vision_config = vision_config
if isinstance(text_config, dict):
self.text_config = self.sub_configs["text_config"](**text_config)
elif text_config is None:
# Backward compatibility: a flat top-level checkpoint (no nested
# text_config) folds its text fields into Glm5NextTextConfig.
self.text_config = self.sub_configs["text_config"](**kwargs)
else:
self.text_config = text_config
self.image_token_id = image_token_id
self.video_token_id = video_token_id
self.image_start_token_id = image_start_token_id
self.image_end_token_id = image_end_token_id
self.video_start_token_id = video_start_token_id
self.video_end_token_id = video_end_token_id
# Mirror attention implementation recursively onto sub-configs.
self._attn_implementation = kwargs.pop("attn_implementation", None)
# Config-metadata fields that belong to the top-level (multimodal) config
# and must NOT be mirrored onto text_config: ``architectures`` /
# ``torch_dtype`` differ between the top-level config and the text
# sub-config, and mirroring them makes the top-level ``architectures``
# silently read back as None (PreTrainedConfig initializes both to None),
# which then fails model-class resolution ("No model architectures are
# specified").
_UNMIRRORED_KEYS = [
"_name_or_path",
"model_type",
"dtype",
"torch_dtype",
"architectures",
"_attn_implementation_internal",
]
def __setattr__(self, key, value):
unmirrored = type(self)._UNMIRRORED_KEYS
if (
(text_config := super().__getattribute__("__dict__").get("text_config"))
is not None
and key not in unmirrored
and key in text_config.__dict__
):
setattr(text_config, key, value)
else:
super().__setattr__(key, value)
def __getattribute__(self, key):
unmirrored = type(self)._UNMIRRORED_KEYS
if (
"text_config" in super().__getattribute__("__dict__")
and key not in unmirrored
):
text_config = super().__getattribute__("text_config")
# Forward both instance attributes AND class-defined properties/
# methods of the text config, so a flat text-only checkpoint
# (model_type "glm5_next", no nested text_config) sees is_moe /
# is_kda_layer / layers_block_type like a Glm5NextTextConfig.
if key in text_config.__dict__ or key in type(text_config).__dict__:
return getattr(text_config, key)
return super().__getattribute__(key)
@@ -1,15 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Exports for the package-owned Qwen4Exp configs."""
from vllm.models.qwen4_exp.config import (
Qwen4ExpConfig,
Qwen4ExpTextConfig,
Qwen4ExpVisionConfig,
)
__all__ = [
"Qwen4ExpConfig",
"Qwen4ExpTextConfig",
"Qwen4ExpVisionConfig",
]
@@ -17,7 +17,6 @@ __all__ = [
"FireRedASR2Processor",
"FunASRProcessor",
"GLM4VProcessor",
"Glm5NextProcessor",
"Granite4VisionProcessor",
"H2OVLProcessor",
"Moondream3Processor",
@@ -56,7 +55,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
"FunASRProcessor": "vllm.transformers_utils.processors.funasr",
"GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
"Glm5NextProcessor": "vllm.transformers_utils.processors.glm5next",
"Granite4VisionProcessor": "vllm.transformers_utils.processors.granite4_vision",
"H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl",
"InternVLProcessor": "vllm.transformers_utils.processors.internvl",
@@ -1,953 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM-native multimodal processor for GLM-5.3-Flash."""
import math
import numpy as np
import torch
from torchvision.transforms.v2 import functional as tvF
from transformers.image_processing_backends import (
TorchvisionBackend,
group_images_by_shape,
reorder_images,
)
from transformers.image_processing_utils import BatchFeature
from transformers.image_utils import (
OPENAI_CLIP_MEAN,
OPENAI_CLIP_STD,
ChannelDimension,
ImageInput,
PILImageResampling,
SizeDict,
get_image_size,
)
from transformers.models.auto.image_processing_auto import get_image_processor_config
from transformers.processing_utils import (
ImagesKwargs,
MultiModalData,
ProcessingKwargs,
ProcessorMixin,
Unpack,
VideosKwargs,
)
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
from transformers.utils import TensorType, logging
from transformers.video_processing_utils import BaseVideoProcessor
from transformers.video_utils import (
VideoInput,
VideoMetadata,
group_videos_by_shape,
reorder_videos,
)
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
logger = logging.get_logger(__name__)
# Cap video inputs at 30,000 vision tokens to keep encoder profiling from
# starving the KV cache. Image inputs retain their checkpoint-defined budget.
_MAX_VIDEO_TOKENS = 30000
# Frame-sampler fallbacks (mirror the checkpoint's fps_interval=2 /
# max_frame_count_dynamic=2048); used when neither the request nor the
# processor config overrides them.
GLM_VIDEO_DEFAULT_FPS = 2.0
GLM_VIDEO_DEFAULT_MAX_FRAMES = 2048
def glm_sample_frame_indices(
total_frames: int,
fps: float,
duration: float,
*,
target_fps: float | None = None,
max_frame_count: int | None = None,
temporal_patch_size: int = 2,
) -> list[int]:
"""GLM video frame sampling (training-reference parity).
``target_fps`` is the ``fps_interval`` request knob. The greedy walk
advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on
frame-dense sources it collects more candidates than ``extract_t`` and
the ``> extract_t`` fixup re-spreads the picks uniformly with
``np.linspace`` -- that fallback is the intended reference behavior, not
an accident. Short clips (fewer frames than ``extract_t``) are spread at
evenly spaced timestamps (``floor`` sampling; the linspace variant
samples frames unevenly and cost 4 points on video grounding evals).
Request overrides: ``target_fps`` -> fps interval, ``max_frame_count``
-> frame cap.
"""
max_frame_idx = total_frames - 1
if not duration:
duration = (round(max_frame_idx / fps) + 1) if fps else 0
if max_frame_count is None:
max_frame_count = GLM_VIDEO_DEFAULT_MAX_FRAMES
if target_fps is None:
target_fps = GLM_VIDEO_DEFAULT_FPS
extract_t = int(duration * target_fps)
extract_t = min(extract_t, int(max_frame_count))
duration_per_frame = 1 / fps
timestamps = [i * duration_per_frame for i in range(total_frames)]
max_second = int(duration)
if total_frames < extract_t:
frame_indices = [
math.floor(_i * total_frames / extract_t) for _i in range(extract_t)
]
else:
frame_indices = []
current_second = 0.0
inv_fps = 1 / (temporal_patch_size * target_fps)
for frame_index in range(total_frames):
if timestamps[frame_index] >= current_second:
current_second += inv_fps
frame_indices.append(frame_index)
if current_second >= max_second:
break
if len(frame_indices) < extract_t:
if len(frame_indices) == 0:
start, end = 0, max(total_frames - 1, 0)
else:
start, end = frame_indices[0], frame_indices[-1]
frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
elif len(frame_indices) > extract_t:
frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist()
seen, uniq = set(), []
for idx in frame_indices:
if idx not in seen:
seen.add(idx)
uniq.append(int(idx))
if len(uniq) & 1:
uniq.append(uniq[-1])
return uniq
def _ceil_to_factor(value: int, factor: int) -> int:
"""Round a positive integer upward to the nearest multiple of factor."""
return math.ceil(value / factor) * factor
def _fit_aligned_size_within_budget(
t: int,
h: int,
w: int,
h_factor: int,
w_factor: int,
max_pixels: int,
) -> tuple[int, int]:
"""Largest proportional size whose upward-aligned canvas fits the budget.
Binary search on the unaligned content height; each candidate is rounded
upward to h_factor/w_factor, so the returned canvas always satisfies
``t * aligned_h * aligned_w <= max_pixels``.
"""
minimum_pixels = t * h_factor * w_factor
if max_pixels < minimum_pixels:
raise ValueError(
f"max_pixels={max_pixels} is too small. At least "
f"{minimum_pixels} pixels are required for one aligned patch."
)
low, high = 1, h
best_h, best_w = h_factor, w_factor
while low <= high:
content_h = (low + high) // 2
content_w = max(1, math.floor(w * content_h / h))
aligned_h = _ceil_to_factor(content_h, h_factor)
aligned_w = _ceil_to_factor(content_w, w_factor)
if t * aligned_h * aligned_w <= max_pixels:
best_h, best_w = aligned_h, aligned_w
low = content_h + 1
else:
high = content_h - 1
return best_h, best_w
def smart_resize(
t: int,
h: int,
w: int,
t_factor: int = 1,
h_factor: int = 28,
w_factor: int = 28,
min_pixels: int = 56 * 56,
max_pixels: int = 14 * 14 * 4 * 1280,
) -> tuple[int, int]:
"""GLM-5.3-Flash ``smart_resize``: upward-aligned canvas under a
``t_bar * h_bar * w_bar`` pixel budget.
Height/width always round UP to their factors (content is then padded,
never cropped or distorted); an over-budget canvas is refit by binary
search instead of one-shot square-root scaling. ``h_factor`` /
``w_factor`` carry ``patch_expand_factor`` on top of
``patch_size * merge_size``; ``t_factor`` is ``temporal_patch_size``. For
a still image ``t = t_factor = temporal_patch_size`` so ``t_bar =
temporal_patch_size``.
"""
if min(t, h, w, t_factor, h_factor, w_factor) <= 0:
raise ValueError("Image dimensions and alignment factors must be positive.")
if min_pixels <= 0 or max_pixels <= 0:
raise ValueError("min_pixels and max_pixels must be positive.")
if min_pixels > max_pixels:
raise ValueError("min_pixels must be less than or equal to max_pixels.")
t_bar = max(t_factor, round(t / t_factor) * t_factor)
h_bar = _ceil_to_factor(h, h_factor)
w_bar = _ceil_to_factor(w, w_factor)
if t_bar * h_bar * w_bar > max_pixels:
h_bar, w_bar = _fit_aligned_size_within_budget(
t=t_bar,
h=h,
w=w,
h_factor=h_factor,
w_factor=w_factor,
max_pixels=max_pixels,
)
elif t_bar * h_bar * w_bar < min_pixels:
beta = math.sqrt(min_pixels / (t * h * w))
h_bar = _ceil_to_factor(max(1, math.ceil(h * beta)), h_factor)
w_bar = _ceil_to_factor(max(1, math.ceil(w * beta)), w_factor)
# Alignment can push a candidate slightly over a tight max_pixels
# budget. Refit it when that happens.
if t_bar * h_bar * w_bar > max_pixels:
h_bar, w_bar = _fit_aligned_size_within_budget(
t=t_bar,
h=h,
w=w,
h_factor=h_factor,
w_factor=w_factor,
max_pixels=max_pixels,
)
return h_bar, w_bar
def _get_pad_content_size(
image_height: int,
image_width: int,
canvas_height: int,
canvas_width: int,
allow_upscale: bool = False,
) -> tuple[int, int]:
"""Aspect-ratio-preserving content size that fits the canvas.
Oversized images are shrunk proportionally. Small images are enlarged
only when ``allow_upscale``. Padding is applied after the resize.
"""
scale = min(canvas_height / image_height, canvas_width / image_width)
if not allow_upscale:
scale = min(1.0, scale)
content_height = max(1, min(canvas_height, math.floor(image_height * scale)))
content_width = max(1, min(canvas_width, math.floor(image_width * scale)))
return content_height, content_width
def _resize_or_pad(
stacked_images: torch.Tensor,
target_height: int,
target_width: int,
resize_mode: str,
resample: "PILImageResampling | tvF.InterpolationMode | int | None",
resize,
allow_upscale: bool = False,
) -> torch.Tensor:
"""Resize onto the aligned canvas, or keep the aspect ratio and
zero-pad the right/bottom sides (``resize_mode="pad"``)."""
height, width = stacked_images.shape[-2:]
if resize_mode == "resize":
return resize(
stacked_images,
size=SizeDict(height=target_height, width=target_width),
resample=resample,
)
if resize_mode != "pad":
raise ValueError("resize_mode must be either 'resize' or 'pad'.")
content_height, content_width = _get_pad_content_size(
image_height=height,
image_width=width,
canvas_height=target_height,
canvas_width=target_width,
allow_upscale=allow_upscale,
)
if (content_height, content_width) != (height, width):
stacked_images = resize(
stacked_images,
size=SizeDict(height=content_height, width=content_width),
resample=resample,
)
# torchvision padding order: [left, top, right, bottom] -> pad only the
# right and bottom sides.
return tvF.pad(
stacked_images,
padding=[0, 0, target_width - content_width, target_height - content_height],
fill=0,
)
def _pixel_budget(
min_image_tokens: int | None,
max_image_tokens: int | None,
patch_size: int,
merge_size: int,
temporal_patch_size: int,
) -> tuple[int, int]:
"""(min_pixels, max_pixels) from the token bounds of
``processor_config.json``; one vision token covers
``temporal_patch_size * (patch_size * merge_size) ** 2`` pixels."""
if min_image_tokens is None or max_image_tokens is None:
raise ValueError(
"min_image_tokens and max_image_tokens must be provided by "
"processor_config.json (or per-call kwargs)."
)
factor = temporal_patch_size * (patch_size * merge_size) ** 2
return min_image_tokens * factor, max_image_tokens * factor
class Glm5NextImageProcessorKwargs(ImagesKwargs, total=False): # type: ignore[call-arg]
patch_size: int | None
temporal_patch_size: int | None
merge_size: int | None
patch_expand_factor: int | None
resize_mode: str | None
min_image_tokens: int | None
max_image_tokens: int | None
class Glm5NextImageProcessor(TorchvisionBackend):
"""Fast torchvision image processor for GLM-5.3-Flash.
``patch_expand_factor`` multiplies into the ``smart_resize`` spatial
factor ``patch_size * merge_size``. ``resize_mode`` picks the geometry:
``"pad"`` (default) preserves the aspect ratio and zero-pads the
right/bottom of the upward-aligned canvas, ``"resize"`` stretches onto
it. Defaults mirror the checkpoint's ``image_processor`` config.
"""
do_resize = True
resample = PILImageResampling.BICUBIC
size = {"longest_edge": 1} # unused: budgets come from the token bounds
do_rescale = True
do_normalize = True
image_mean = OPENAI_CLIP_MEAN
image_std = OPENAI_CLIP_STD
do_convert_rgb = True
patch_size = 14
temporal_patch_size = 2
merge_size = 2
patch_expand_factor = 1
resize_mode = "pad"
min_image_tokens = 16
max_image_tokens = 8000
valid_kwargs = Glm5NextImageProcessorKwargs
model_input_names = ["pixel_values", "image_grid_thw"]
def _preprocess(
self,
images: list[torch.Tensor],
do_resize: bool,
size: SizeDict,
resample: "PILImageResampling | tvF.InterpolationMode | int | None",
do_rescale: bool,
rescale_factor: float,
do_normalize: bool,
image_mean: float | list[float] | None,
image_std: float | list[float] | None,
patch_size: int,
temporal_patch_size: int,
merge_size: int,
patch_expand_factor: int,
resize_mode: str | None,
min_image_tokens: int | None,
max_image_tokens: int | None,
disable_grouping: bool | None,
return_tensors: str | TensorType | None,
**kwargs,
) -> BatchFeature:
resize_mode = resize_mode if resize_mode is not None else self.resize_mode
min_pixels, max_pixels = _pixel_budget(
min_image_tokens if min_image_tokens is not None else self.min_image_tokens,
max_image_tokens if max_image_tokens is not None else self.max_image_tokens,
patch_size,
merge_size,
temporal_patch_size,
)
grouped_images, grouped_images_index = group_images_by_shape(
images, disable_grouping=disable_grouping
)
resized_images_grouped = {}
for shape, stacked_images in grouped_images.items():
height, width = stacked_images.shape[-2:]
if do_resize:
resized_height, resized_width = smart_resize(
t=temporal_patch_size,
h=height,
w=width,
t_factor=temporal_patch_size,
h_factor=patch_size * merge_size * patch_expand_factor,
w_factor=patch_size * merge_size * patch_expand_factor,
min_pixels=min_pixels,
max_pixels=max_pixels,
)
stacked_images = _resize_or_pad(
stacked_images,
target_height=resized_height,
target_width=resized_width,
resize_mode=resize_mode,
resample=resample,
resize=self.resize,
allow_upscale=(temporal_patch_size * height * width < min_pixels),
)
resized_images_grouped[shape] = stacked_images
resized_images = reorder_images(resized_images_grouped, grouped_images_index)
grouped_images, grouped_images_index = group_images_by_shape(
resized_images, disable_grouping=disable_grouping
)
processed_images_grouped = {}
processed_grids = {}
for shape, stacked_images in grouped_images.items():
resized_height, resized_width = stacked_images.shape[-2:]
patches = self.rescale_and_normalize(
stacked_images,
do_rescale,
rescale_factor,
do_normalize,
image_mean,
image_std,
)
if patches.ndim == 4: # (B, C, H, W)
patches = patches.unsqueeze(1) # (B, T=1, C, H, W)
if patches.shape[1] % temporal_patch_size != 0:
repeats = patches[:, -1:].repeat(
1,
temporal_patch_size - (patches.shape[1] % temporal_patch_size),
1,
1,
1,
)
patches = torch.cat([patches, repeats], dim=1)
batch_size, t_len, channel = patches.shape[:3]
grid_t = t_len // temporal_patch_size
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
patches = patches.view(
batch_size,
grid_t,
temporal_patch_size,
channel,
grid_h // merge_size,
merge_size,
patch_size,
grid_w // merge_size,
merge_size,
patch_size,
)
# (B, grid_t, gh, gw, mh, mw, C, tp, ph, pw)
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
flatten_patches = patches.reshape(
batch_size,
grid_t * grid_h * grid_w,
channel * temporal_patch_size * patch_size * patch_size,
)
processed_images_grouped[shape] = flatten_patches
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
processed_images = reorder_images(
processed_images_grouped, grouped_images_index
)
processed_grids = reorder_images(processed_grids, grouped_images_index)
pixel_values = torch.cat(processed_images, dim=0)
image_grid_thw = torch.tensor(processed_grids)
return BatchFeature(
data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw},
tensor_type=return_tensors,
)
def preprocess(
self, images: ImageInput, **kwargs: Unpack[Glm5NextImageProcessorKwargs]
) -> BatchFeature:
return super().preprocess(images, **kwargs)
def get_number_of_image_patches(
self, height: int, width: int, images_kwargs: dict | None = None
) -> int:
"""Number of image patches (pre-merge) for a given (height, width)."""
images_kwargs = images_kwargs or {}
patch_size = images_kwargs.get("patch_size", self.patch_size)
merge_size = images_kwargs.get("merge_size", self.merge_size)
patch_expand_factor = images_kwargs.get(
"patch_expand_factor", self.patch_expand_factor
)
min_pixels, max_pixels = _pixel_budget(
images_kwargs.get("min_image_tokens", self.min_image_tokens),
images_kwargs.get("max_image_tokens", self.max_image_tokens),
patch_size,
merge_size,
self.temporal_patch_size,
)
resized_height, resized_width = smart_resize(
t=self.temporal_patch_size,
h=height,
w=width,
t_factor=self.temporal_patch_size,
h_factor=patch_size * merge_size * patch_expand_factor,
w_factor=patch_size * merge_size * patch_expand_factor,
min_pixels=min_pixels,
max_pixels=max_pixels,
)
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
return grid_h * grid_w
class Glm5NextVideoProcessorKwargs(VideosKwargs, total=False): # type: ignore[call-arg]
fps: list[float] | float
patch_size: int
temporal_patch_size: int
merge_size: int
patch_expand_factor: int
resize_mode: str | None
target_fps: float | None
max_frames: int | None
fps_interval: int | None
max_frame_count_dynamic: int | None
min_image_tokens: int | None
max_image_tokens: int | None
class Glm5NextVideoProcessor(BaseVideoProcessor):
"""Fast video processor for GLM-5.3-Flash.
Shares ``smart_resize`` / the pad-mode geometry / the patchify with the
image processor, and adds GLM-5.3-Flash frame sampling
(``glm_sample_frame_indices``: ``fps_interval`` semantics with a
temporal-patch-scaled greedy walk). Defaults mirror the checkpoint's
``video_processor`` config.
"""
resample = PILImageResampling.BICUBIC
size = {"longest_edge": 1} # unused: budgets come from the token bounds
image_mean = OPENAI_CLIP_MEAN
image_std = OPENAI_CLIP_STD
do_resize = True
do_rescale = True
do_normalize = True
do_convert_rgb = True
do_sample_frames = True
patch_size = 14
temporal_patch_size = 2
patch_expand_factor = 1
merge_size = 2
valid_kwargs = Glm5NextVideoProcessorKwargs
num_frames = 16
fps = 2
fps_interval = 2.0
max_frame_count_dynamic = 2048
resize_mode = "pad"
min_image_tokens = 16
max_image_tokens = 240000
model_input_names = ["pixel_values_videos", "video_grid_thw"]
def sample_frames(
self,
metadata: VideoMetadata,
fps: int | float | None = None,
**kwargs,
) -> np.ndarray:
"""Sample frame indices with GLM's fps-interval policy.
``fps`` / ``target_fps``, ``max_frames`` and ``fps_interval`` /
``max_frame_count_dynamic`` are the overrides described in
:func:`glm_sample_frame_indices`.
"""
if metadata is None or getattr(metadata, "fps", None) is None:
raise ValueError(
"Asked to sample frames per second but no video metadata was "
"provided which is required when sampling in GLM-5.3-Flash. Please "
"pass in `VideoMetadata` object or set `do_sample_frames=False`."
)
target_fps = fps if fps is not None else kwargs.get("target_fps")
if target_fps is None:
target_fps = self.fps_interval
indices = glm_sample_frame_indices(
metadata.total_num_frames,
metadata.fps,
metadata.duration or 0,
target_fps=target_fps,
max_frame_count=kwargs.get("max_frames") or self.max_frame_count_dynamic,
temporal_patch_size=self.temporal_patch_size,
)
return np.array(indices)
def _preprocess(
self,
videos: list[torch.Tensor],
do_convert_rgb: bool = True,
do_resize: bool = True,
size: SizeDict | None = None,
resample: "PILImageResampling | int | None" = PILImageResampling.BICUBIC,
do_rescale: bool = True,
rescale_factor: float = 1 / 255.0,
do_normalize: bool = True,
image_mean: float | list[float] | None = None,
image_std: float | list[float] | None = None,
patch_size: int | None = None,
temporal_patch_size: int | None = None,
patch_expand_factor: int | None = None,
merge_size: int | None = None,
resize_mode: str | None = None,
min_image_tokens: int | None = None,
max_image_tokens: int | None = None,
return_tensors: str | TensorType | None = None,
**kwargs,
) -> BatchFeature:
patch_expand_factor = self.patch_expand_factor
patch_size = patch_size if patch_size is not None else self.patch_size
temporal_patch_size = (
temporal_patch_size
if temporal_patch_size is not None
else self.temporal_patch_size
)
merge_size = merge_size if merge_size is not None else self.merge_size
resize_mode = resize_mode if resize_mode is not None else self.resize_mode
min_pixels, max_pixels = _pixel_budget(
min_image_tokens if min_image_tokens is not None else self.min_image_tokens,
max_image_tokens if max_image_tokens is not None else self.max_image_tokens,
patch_size,
merge_size,
temporal_patch_size,
)
grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
resized_videos_grouped = {}
for shape, stacked_videos in grouped_videos.items():
if do_convert_rgb:
stacked_videos = self.convert_to_rgb(stacked_videos)
b, t_len, c, h, w = stacked_videos.shape
num_frames, height, width = t_len, h, w
if do_resize:
resized_height, resized_width = smart_resize(
t=num_frames,
h=height,
w=width,
t_factor=temporal_patch_size,
h_factor=patch_size * merge_size * patch_expand_factor,
w_factor=patch_size * merge_size * patch_expand_factor,
min_pixels=min_pixels,
max_pixels=max_pixels,
)
stacked_videos = stacked_videos.view(b * t_len, c, h, w)
stacked_videos = _resize_or_pad(
stacked_videos,
target_height=resized_height,
target_width=resized_width,
resize_mode=resize_mode,
resample=resample,
resize=self.resize,
allow_upscale=(num_frames * height * width < min_pixels),
)
stacked_videos = stacked_videos.view(
b, t_len, c, resized_height, resized_width
)
resized_videos_grouped[shape] = stacked_videos
resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
processed_videos_grouped = {}
processed_grids = {}
for shape, stacked_videos in grouped_videos.items():
resized_height, resized_width = get_image_size(
stacked_videos[0], channel_dim=ChannelDimension.FIRST
)
stacked_videos = self.rescale_and_normalize(
stacked_videos,
do_rescale,
rescale_factor,
do_normalize,
image_mean,
image_std,
)
patches = stacked_videos
if pad := -patches.shape[1] % temporal_patch_size:
repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1)
patches = torch.cat((patches, repeats), dim=1)
batch_size, grid_t, channel = patches.shape[:3]
grid_t = grid_t // temporal_patch_size
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
patches = patches.view(
batch_size,
grid_t,
temporal_patch_size,
channel,
grid_h // merge_size,
merge_size,
patch_size,
grid_w // merge_size,
merge_size,
patch_size,
)
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
flatten_patches = patches.reshape(
batch_size,
grid_t * grid_h * grid_w,
channel * temporal_patch_size * patch_size * patch_size,
)
processed_videos_grouped[shape] = flatten_patches
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
processed_videos = reorder_videos(
processed_videos_grouped, grouped_videos_index
)
processed_grids = reorder_videos(processed_grids, grouped_videos_index)
pixel_values_videos = torch.cat(processed_videos, dim=0)
video_grid_thw = torch.tensor(processed_grids)
return BatchFeature(
data={
"pixel_values_videos": pixel_values_videos,
"video_grid_thw": video_grid_thw,
},
tensor_type=return_tensors,
)
class Glm5NextProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
images_kwargs: Glm5NextImageProcessorKwargs
videos_kwargs: Glm5NextVideoProcessorKwargs
_defaults = {
"text_kwargs": {
"padding": False,
"return_token_type_ids": False,
"return_mm_token_type_ids": False,
},
"videos_kwargs": {"return_metadata": True},
}
class Glm5NextProcessor(ProcessorMixin):
"""Wraps a GLM-5.3-Flash image processor, video processor and tokenizer.
Token expansion per image = ``prod(image_grid_thw) // merge_size**2``; video
frames are expanded with ``<|begin_of_image|>...<|end_of_image|>{ts} seconds``
structure (mrope timestamps).
"""
attributes = ["image_processor", "tokenizer", "video_processor"]
image_processor_class = "AutoImageProcessor"
video_processor_class = "AutoVideoProcessor"
tokenizer_class = ("PythonBackend", "TokenizersBackend")
def __init__(
self,
image_processor=None,
tokenizer=None,
video_processor=None,
chat_template=None,
**kwargs,
) -> None:
super().__init__(
image_processor, tokenizer, video_processor, chat_template=chat_template
)
self.image_token = (
"<|image|>"
if not hasattr(tokenizer, "image_token")
else tokenizer.image_token
)
self.video_token = (
"<|video|>"
if not hasattr(tokenizer, "video_token")
else tokenizer.video_token
)
self.image_token_id = (
tokenizer.image_token_id
if getattr(tokenizer, "image_token_id", None)
else tokenizer.convert_tokens_to_ids(self.image_token)
)
self.video_token_id = (
tokenizer.video_token_id
if getattr(tokenizer, "video_token_id", None)
else tokenizer.convert_tokens_to_ids(self.video_token)
)
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
"""Build the processor directly from the checkpoint config.
GLM-5.3-Flash stores nested image/video configs in
``processor_config.json`` and declares a custom processor class. This
method reads those configs directly and caps only the video token budget.
"""
from transformers import AutoTokenizer
model_path = pretrained_model_name_or_path
tokenizer = AutoTokenizer.from_pretrained(model_path, **kwargs)
def _cap_cfg(cfg: dict, *, is_video: bool) -> dict:
# Video keeps a serving token cap (the checkpoint's 240k-token
# budget would starve the KV cache at startup profiling); images
# follow the checkpoint budget verbatim so preprocessing matches
# the HF reference exactly.
if is_video and cfg.get("max_image_tokens") is not None:
cfg["max_image_tokens"] = min(
cfg["max_image_tokens"], _MAX_VIDEO_TOKENS
)
return cfg
ip_cfg = _cap_cfg(
dict(get_image_processor_config(model_path, **kwargs)), is_video=False
)
image_processor = Glm5NextImageProcessor(
**{k: v for k, v in ip_cfg.items() if k != "image_processor_type"}
)
processor_config = get_hf_file_to_dict(
"processor_config.json",
model_path,
revision=kwargs.get("revision", "main"),
)
if processor_config is None:
raise ValueError(f"Missing processor_config.json for {model_path}")
vp_cfg = _cap_cfg(dict(processor_config["video_processor"]), is_video=True)
video_processor = Glm5NextVideoProcessor(
**{k: v for k, v in vp_cfg.items() if k != "video_processor_type"}
)
return cls(
image_processor=image_processor,
tokenizer=tokenizer,
video_processor=video_processor,
)
def __call__(
self,
images: ImageInput | None = None,
text: TextInput
| PreTokenizedInput
| list[TextInput]
| list[PreTokenizedInput] = None,
videos: VideoInput | None = None,
**kwargs: Unpack[Glm5NextProcessorKwargs],
) -> BatchFeature:
output_kwargs = self._merge_kwargs(
Glm5NextProcessorKwargs,
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
**kwargs,
)
if images is not None:
image_inputs = self.image_processor(
images=images, **output_kwargs["images_kwargs"]
)
else:
image_inputs = {}
if videos is not None:
videos_inputs = self.video_processor(
videos=videos, **output_kwargs["videos_kwargs"]
)
if "return_metadata" not in kwargs:
videos_inputs.pop("video_metadata")
else:
videos_inputs = {}
if not isinstance(text, list):
text = [text]
# Prompt updates expand the unchanged image/video markers after this call.
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
return_mm_token_type_ids = output_kwargs["text_kwargs"].pop(
"return_mm_token_type_ids", False
)
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
if return_mm_token_type_ids:
array_ids = np.array(text_inputs["input_ids"])
mm_token_type_ids = np.zeros_like(text_inputs["input_ids"])
mm_token_type_ids[array_ids == self.image_token_id] = 1
text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist()
return BatchFeature(
data={**text_inputs, **image_inputs, **videos_inputs},
tensor_type=return_tensors,
)
def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs):
vision_data = {}
if image_sizes is not None:
images_kwargs = Glm5NextProcessorKwargs._defaults.get("images_kwargs", {})
images_kwargs.update(kwargs)
merge_size = (
images_kwargs.get("merge_size", None) or self.image_processor.merge_size
)
num_image_patches = [
self.image_processor.get_number_of_image_patches(
*image_size, images_kwargs
)
for image_size in image_sizes
]
num_image_tokens = [(n // merge_size**2) for n in num_image_patches]
vision_data.update(
{
"num_image_tokens": num_image_tokens,
"num_image_patches": num_image_patches,
}
)
if video_sizes is not None:
videos_kwargs = Glm5NextProcessorKwargs._defaults.get("videos_kwargs", {})
videos_kwargs.update(kwargs)
num_video_patches = [
self.video_processor.get_number_of_video_patches(
*video_size, videos_kwargs
)
for video_size in video_sizes
]
num_video_tokens = [(n // merge_size**2) for n in num_video_patches]
vision_data["num_video_tokens"] = num_video_tokens
return MultiModalData(**vision_data)
def post_process_image_text_to_text(
self,
generated_outputs,
skip_special_tokens=True,
clean_up_tokenization_spaces=False,
**kwargs,
):
return self.tokenizer.batch_decode(
generated_outputs,
skip_special_tokens=skip_special_tokens,
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
**kwargs,
)
__all__ = [
"Glm5NextImageProcessor",
"Glm5NextVideoProcessor",
"Glm5NextProcessor",
"smart_resize",
]