Files
2026-10-02 15:25:37 +00:00

836 lines
30 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLM-5.3-Flash vision tower and multimodal processor."""
import math
from collections.abc import Mapping
from functools import partial
from typing import Any
import numpy as np
import torch
import torch.nn as nn
from einops import rearrange
from transformers.video_utils import VideoMetadata
from vllm.distributed import (
get_tensor_model_parallel_world_size,
parallel_state,
)
from vllm.distributed import utils as dist_utils
from vllm.model_executor.layers.activation import SiluAndMulWithClamp
from vllm.model_executor.layers.attention import MMEncoderAttention
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import IdentityInputNorm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
from vllm.model_executor.models.glm4_1v import (
Glm4vMultiModalProcessor,
Glm4vProcessingInfo,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
WeightsMapper,
)
from vllm.model_executor.models.vision import (
get_vit_attn_backend,
is_vit_use_data_parallel,
)
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.registry import AttentionBackendEnum
class Glm5NextVisionPatchEmbed(nn.Module):
def __init__(
self,
patch_size: int = 14,
temporal_patch_size: int = 1,
in_channels: int = 3,
hidden_size: int = 1536,
) -> None:
super().__init__()
self.patch_size = patch_size
self.temporal_patch_size = temporal_patch_size
self.hidden_size = hidden_size
kernel_size = (temporal_patch_size, patch_size, patch_size)
self.proj = Conv3dLayer(
in_channels,
hidden_size,
kernel_size=kernel_size,
stride=kernel_size,
bias=True,
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
L, C = x.shape
x = x.view(L, -1, self.temporal_patch_size, self.patch_size, self.patch_size)
x = self.proj(x).view(L, self.hidden_size)
return x
class Glm5NextVisionMLP(nn.Module):
def __init__(
self,
in_features: int,
hidden_features: int,
swiglu_limit: float,
bias: bool = True,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
use_data_parallel = is_vit_use_data_parallel()
self.gate_up_proj = MergedColumnParallelLinear(
input_size=in_features,
output_sizes=[hidden_features] * 2,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.gate_up_proj",
disable_tp=use_data_parallel,
)
self.down_proj = RowParallelLinear(
hidden_features,
in_features,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.down_proj",
disable_tp=use_data_parallel,
)
# GLM-5.3-Flash clamps the vision SwiGLU gate/up unlike GLM-OCR/GLM-4V.
self.act_fn = SiluAndMulWithClamp(swiglu_limit=swiglu_limit)
def forward(self, x: torch.Tensor):
x, _ = self.gate_up_proj(x)
x = self.act_fn(x)
x, _ = self.down_proj(x)
return x
class Glm5NextVisionAttention(nn.Module):
def __init__(
self,
embed_dim: int,
num_heads: int,
projection_size: int,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
use_data_parallel = is_vit_use_data_parallel()
self.tp_size = (
1 if use_data_parallel else get_tensor_model_parallel_world_size()
)
self.tp_rank = (
0 if use_data_parallel else parallel_state.get_tensor_model_parallel_rank()
)
self.hidden_size_per_attention_head = dist_utils.divide(
projection_size, num_heads
)
self.num_attention_heads_per_partition = dist_utils.divide(
num_heads, self.tp_size
)
self.head_dim = embed_dim // num_heads
# q/k norm eps hard-coded 1e-5 — distinct from block/post norm eps.
self.q_norm = RMSNorm(self.head_dim, eps=1e-5)
self.k_norm = RMSNorm(self.head_dim, eps=1e-5)
self.qkv = QKVParallelLinear(
hidden_size=embed_dim,
head_size=self.hidden_size_per_attention_head,
total_num_heads=num_heads,
total_num_kv_heads=num_heads,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj" if quant_config else f"{prefix}.qkv",
disable_tp=use_data_parallel,
)
self.proj = RowParallelLinear(
input_size=projection_size,
output_size=embed_dim,
quant_config=quant_config,
prefix=f"{prefix}.proj",
bias=True,
disable_tp=use_data_parallel,
)
self.attn = MMEncoderAttention(
num_heads=self.num_attention_heads_per_partition,
head_size=self.hidden_size_per_attention_head,
scale=self.hidden_size_per_attention_head**-0.5,
prefix=f"{prefix}.attn",
)
self.apply_rotary_emb = ApplyRotaryEmb(enforce_enable=True)
def split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, ...]:
seq_len, bs, _ = qkv.shape
q, k, v = qkv.chunk(3, dim=2)
new_shape = (
seq_len,
bs,
self.num_attention_heads_per_partition,
self.hidden_size_per_attention_head,
)
q, k, v = (x.view(*new_shape) for x in (q, k, v))
return q, k, v
def forward(
self,
x: torch.Tensor,
cu_seqlens: torch.Tensor,
rotary_pos_emb_cos: torch.Tensor,
rotary_pos_emb_sin: torch.Tensor,
max_seqlen: torch.Tensor | None = None,
) -> torch.Tensor:
x, _ = self.qkv(x)
q, k, v = self.split_qkv(x)
# P1: fused q/k RMSNorm (two distinct weights, one launch; fp32, bit-identical).
q_shape, k_shape = q.shape, k.shape
q_flat = q.reshape(-1, self.head_dim)
k_flat = k.reshape(-1, self.head_dim)
q, k = fused_q_kv_rmsnorm(
q_flat,
k_flat,
self.q_norm.weight,
self.k_norm.weight,
self.q_norm.variance_epsilon,
)
q = q.view(q_shape)
k = k.view(k_shape)
q, k, v = (rearrange(t, "s b ... -> b s ...").contiguous() for t in (q, k, v))
if rotary_pos_emb_cos is not None and rotary_pos_emb_sin is not None:
qk_concat = torch.cat([q, k], dim=0)
qk_rotated = self.apply_rotary_emb(
qk_concat,
rotary_pos_emb_cos,
rotary_pos_emb_sin,
)
q, k = torch.chunk(qk_rotated, 2, dim=0)
context_layer = self.attn(
query=q,
key=k,
value=v,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
)
context_layer = rearrange(context_layer, "b s h d -> s b (h d)").contiguous()
output, _ = self.proj(context_layer)
return output
class Glm5NextVisionBlock(nn.Module):
def __init__(
self,
dim: int,
num_heads: int,
mlp_hidden_dim: int,
swiglu_limit: float,
norm_layer: partial[nn.Module] | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
if norm_layer is None:
norm_layer = partial(nn.LayerNorm, eps=1e-6)
self.norm1 = norm_layer(dim)
self.norm2 = norm_layer(dim)
self.attn = Glm5NextVisionAttention(
embed_dim=dim,
num_heads=num_heads,
projection_size=dim,
quant_config=quant_config,
prefix=f"{prefix}.attn",
)
self.mlp = Glm5NextVisionMLP(
dim,
mlp_hidden_dim,
swiglu_limit=swiglu_limit,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
def forward(
self,
x: torch.Tensor,
cu_seqlens: torch.Tensor,
rotary_pos_emb_cos: torch.Tensor,
rotary_pos_emb_sin: torch.Tensor,
max_seqlen: int | None = None,
) -> torch.Tensor:
x_attn = self.attn(
self.norm1(x),
cu_seqlens=cu_seqlens,
rotary_pos_emb_cos=rotary_pos_emb_cos,
rotary_pos_emb_sin=rotary_pos_emb_sin,
max_seqlen=max_seqlen,
)
x_fused_norm, residual = self.norm2(x, residual=x_attn)
x = residual + self.mlp(x_fused_norm)
return x
class Glm5NextPatchMerger(nn.Module):
def __init__(
self,
d_model: int,
context_dim: int,
swiglu_limit: float,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
prefix: str = "",
) -> None:
super().__init__()
use_data_parallel = is_vit_use_data_parallel()
self.hidden_size = d_model
self.proj = ColumnParallelLinear(
self.hidden_size,
self.hidden_size,
bias=bias,
gather_output=True,
quant_config=quant_config,
prefix=f"{prefix}.proj",
disable_tp=use_data_parallel,
)
self.post_projection_norm = nn.LayerNorm(self.hidden_size)
self.gate_up_proj = MergedColumnParallelLinear(
input_size=self.hidden_size,
output_sizes=[context_dim] * 2,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.gate_up_proj",
disable_tp=use_data_parallel,
)
self.down_proj = RowParallelLinear(
context_dim,
self.hidden_size,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.down_proj",
disable_tp=use_data_parallel,
)
# GLM-5.3-Flash also clamps the merger SwiGLU.
self.act_fn = SiluAndMulWithClamp(swiglu_limit=swiglu_limit)
self.extra_activation_func = nn.GELU()
def forward(self, x: torch.Tensor):
x, _ = self.proj(x)
x = self.extra_activation_func(self.post_projection_norm(x))
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
return x
class Glm5NextVisionTransformer(nn.Module):
# Stacked-weight remap for the GLM-OCR/GLM-4V vision checkpoint layout.
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".attn.q.": (".attn.qkv.", "q"),
".attn.k.": (".attn.qkv.", "k"),
".attn.v.": (".attn.qkv.", "v"),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
}
)
def __init__(
self,
text_config, # noqa: ANN001
vision_config,
norm_eps: float = 1e-6,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
) -> None:
super().__init__()
use_data_parallel = is_vit_use_data_parallel()
self.tp_size = (
1 if use_data_parallel else get_tensor_model_parallel_world_size()
)
patch_size = vision_config.patch_size
temporal_patch_size = vision_config.temporal_patch_size
in_channels = vision_config.in_channels
depth = vision_config.depth
self.hidden_size = vision_config.hidden_size
self.num_heads = vision_config.num_heads
self.patch_size = vision_config.patch_size
self.spatial_merge_size = vision_config.spatial_merge_size
self.out_hidden_size = vision_config.out_hidden_size
swiglu_limit = vision_config.swiglu_limit
if swiglu_limit is None:
swiglu_limit = text_config.swiglu_limit
assert swiglu_limit is not None, (
"GLM-5.3-Flash vision requires swiglu_limit (vision_config or text_config)"
)
# Single construction pass — no abs-pos embeddings / post-conv norm (OCR delta).
self.patch_embed = Glm5NextVisionPatchEmbed(
patch_size=patch_size,
temporal_patch_size=temporal_patch_size,
in_channels=in_channels,
hidden_size=self.hidden_size,
)
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
norm_layer = partial(RMSNorm, eps=norm_eps)
head_dim = self.hidden_size // self.num_heads
self.rotary_pos_emb = get_rope(
head_size=head_dim,
max_position=text_config.max_position_embeddings,
is_neox_style=True,
rope_parameters={"partial_rotary_factor": 0.5},
)
self.blocks = nn.ModuleList(
[
Glm5NextVisionBlock(
dim=self.hidden_size,
num_heads=self.num_heads,
mlp_hidden_dim=vision_config.intermediate_size,
swiglu_limit=swiglu_limit,
norm_layer=norm_layer,
quant_config=quant_config,
prefix=f"{prefix}.blocks.{layer_idx}",
)
for layer_idx in range(depth)
]
)
# GLM-5.3-Flash merger bottleneck width.
self.merger = Glm5NextPatchMerger(
d_model=vision_config.out_hidden_size,
context_dim=vision_config.projection_intermediate_size,
swiglu_limit=swiglu_limit,
quant_config=quant_config,
bias=False,
prefix=f"{prefix}.merger",
)
self.downsample = Conv2dLayer(
in_channels=vision_config.hidden_size,
out_channels=vision_config.out_hidden_size,
kernel_size=vision_config.spatial_merge_size,
stride=vision_config.spatial_merge_size,
)
self.post_layernorm = RMSNorm(
vision_config.hidden_size, eps=vision_config.rms_norm_eps
)
self.attn_backend = get_vit_attn_backend(
head_size=head_dim,
dtype=torch.get_default_dtype(),
)
@property
def dtype(self) -> torch.dtype:
return self.patch_embed.proj.weight.dtype
@property
def device(self) -> torch.device:
return self.patch_embed.proj.weight.device
def rot_pos_emb(
self, grid_thw: list[list[int]]
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
pos_ids = []
for t, h, w in grid_thw:
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
hpos_ids = (
hpos_ids.reshape(
h // self.spatial_merge_size,
self.spatial_merge_size,
w // self.spatial_merge_size,
self.spatial_merge_size,
)
.permute(0, 2, 1, 3)
.flatten()
)
wpos_ids = (
wpos_ids.reshape(
h // self.spatial_merge_size,
self.spatial_merge_size,
w // self.spatial_merge_size,
self.spatial_merge_size,
)
.permute(0, 2, 1, 3)
.flatten()
)
pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
pos_ids = torch.cat(pos_ids, dim=0)
max_grid_size = max(max(h, w) for _, h, w in grid_thw)
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
pos_ids = async_tensor_h2d(pos_ids, device=cos.device)
cos_combined = cos[pos_ids].flatten(1)
sin_combined = sin[pos_ids].flatten(1)
return cos_combined, sin_combined, pos_ids
def compute_attn_mask_seqlen(
self,
cu_seqlens: torch.Tensor,
) -> torch.Tensor | None:
max_seqlen = None
if self.attn_backend in {
AttentionBackendEnum.FLASH_ATTN,
AttentionBackendEnum.ROCM_AITER_FA,
AttentionBackendEnum.TRITON_ATTN,
}:
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
return max_seqlen
def prepare_encoder_metadata(
self,
grid_thw_list: list[list[int]],
*,
max_batch_size: int | None = None,
max_frames_per_batch: int | None = None,
max_seqlen_override: int | None = None,
device: torch.device | None = None,
) -> dict[str, torch.Tensor | None]:
"""Compute encoder metadata for eager and CUDA graph execution."""
if device is None:
device = self.device
metadata: dict[str, torch.Tensor | None] = {}
rotary_cos, rotary_sin, _ = self.rot_pos_emb(grid_thw_list)
metadata["rotary_pos_emb_cos"] = rotary_cos
metadata["rotary_pos_emb_sin"] = rotary_sin
grid_thw_np = np.array(grid_thw_list, dtype=np.int32)
patches_per_frame = grid_thw_np[:, 1] * grid_thw_np[:, 2]
cu_seqlens = np.repeat(patches_per_frame, grid_thw_np[:, 0]).cumsum(
dtype=np.int32
)
cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])
pad_to = (
max_frames_per_batch if max_frames_per_batch is not None else max_batch_size
)
if pad_to is not None:
num_seqs = len(cu_seqlens) - 1
if num_seqs < pad_to:
cu_seqlens = np.concatenate(
[
cu_seqlens,
np.full(
pad_to - num_seqs,
cu_seqlens[-1],
dtype=np.int32,
),
]
)
metadata["sequence_lengths"] = MMEncoderAttention.maybe_compute_seq_lens(
self.attn_backend, cu_seqlens, device
)
if max_seqlen_override is not None:
max_seqlen_val = max_seqlen_override
else:
max_seqlen_val = MMEncoderAttention.compute_max_seqlen(
self.attn_backend, cu_seqlens
)
metadata["max_seqlen"] = torch.tensor(max_seqlen_val, dtype=torch.int32)
metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
self.attn_backend,
cu_seqlens,
self.hidden_size,
self.tp_size,
device,
)
return metadata
def forward(
self,
x: torch.Tensor,
grid_thw: torch.Tensor | list[list[int]],
*,
encoder_metadata: dict[str, torch.Tensor] | None = None,
) -> torch.Tensor:
# patchify
x = self.input_norm(x.to(device=self.device), self.dtype)
x = self.patch_embed(x)
if encoder_metadata is not None:
# Encoder CUDA-graph path (PR #49852): rotary/cu_seqlens/max_seqlen are
# precomputed by prepare_encoder_metadata (which uses rot_pos_emb exactly
# as the eager rebuild does), so reuse them and skip the per-call CPU
# rebuild (the low-GPU-util culprit on multimodal workloads).
rotary_pos_emb_cos = encoder_metadata["rotary_pos_emb_cos"]
rotary_pos_emb_sin = encoder_metadata["rotary_pos_emb_sin"]
cu_seqlens = encoder_metadata["cu_seqlens"]
max_seqlen = encoder_metadata["max_seqlen"]
else:
if isinstance(grid_thw, list):
grid_thw = torch.tensor(grid_thw, dtype=torch.int32)
rotary_pos_emb_cos, rotary_pos_emb_sin, _ = self.rot_pos_emb(grid_thw)
cu_seqlens = torch.repeat_interleave(
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
).cumsum(dim=0, dtype=torch.int32)
cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
cu_seqlens = async_tensor_h2d(cu_seqlens, device=self.device)
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
# transformers
x = x.unsqueeze(1)
for blk in self.blocks:
x = blk(
x,
cu_seqlens=cu_seqlens,
rotary_pos_emb_cos=rotary_pos_emb_cos,
rotary_pos_emb_sin=rotary_pos_emb_sin,
max_seqlen=max_seqlen,
)
# adapter
x = self.post_layernorm(x)
x = x.view(-1, self.spatial_merge_size, self.spatial_merge_size, x.shape[-1])
x = x.permute(0, 3, 1, 2)
x = self.downsample(x).view(-1, self.out_hidden_size)
x = self.merger(x)
return x
def load_weights(self, weights) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
_MAX_VIDEO_TOKENS = 30000
class Glm5NextProcessingInfo(Glm4vProcessingInfo):
"""Token-budget geometry for the multimodal checkpoint.
This checkpoint's ``processor_config.json`` ships the token-budget style
(``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, so the
inherited Glm4v ``size.longest_edge`` path does not apply.
"""
def get_hf_processor(self, **kwargs: object):
processor = super().get_hf_processor(**kwargs)
# Cap video inputs to keep encoder profiling from starving the KV cache.
video_processor = processor.video_processor
video_processor.max_image_tokens = min(
video_processor.max_image_tokens, _MAX_VIDEO_TOKENS
)
return processor
@staticmethod
def _alignment_factor(proc) -> int:
"""Spatial factor canvases align to, as ``smart_resize`` takes it."""
return proc.patch_size * proc.merge_size * proc.patch_expand_factor
@classmethod
def _pixels_per_token(cls, proc) -> int:
"""Pixels one vision token covers, matching ``smart_resize``'s own
``temporal_factor * factor ** 2``."""
return proc.temporal_patch_size * cls._alignment_factor(proc) ** 2
def _processor_pixel_budget(self, proc) -> tuple[int, int]:
"""(min_pixels, max_pixels) from the processor's token bounds."""
if proc.min_image_tokens is None or proc.max_image_tokens is None:
raise ValueError(
"min_image_tokens and max_image_tokens must be provided by "
"processor_config.json (or per-call kwargs)."
)
pixels_per_token = self._pixels_per_token(proc)
return (
proc.min_image_tokens * pixels_per_token,
proc.max_image_tokens * pixels_per_token,
)
def _get_image_max_pixels(self) -> int:
mm_kwargs = self.ctx.get_merged_mm_kwargs({})
if (override := mm_kwargs.get("max_pixels")) is not None:
return int(override)
return self._processor_pixel_budget(self.get_hf_processor().image_processor)[1]
def _get_video_max_pixels(self) -> int:
mm_kwargs = self.ctx.get_merged_mm_kwargs({})
if (override := mm_kwargs.get("max_pixels")) is not None:
return int(override)
return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1]
def get_image_size_with_most_features(self) -> ImageSize:
# The inherited square probe strands budget whenever the token
# ceiling is not a perfect square: with max_image_tokens=8000 the
# square refits to 2492x2492 (89x89 = 7921 tokens) while a
# 2240x2800 canvas reaches 80x100 = 8000, so the encoder cache came
# up short of the processor's own maximum and refused valid images
# (#59539). Factor the ceiling exactly instead of probing a square.
vision_config = self.get_hf_config().vision_config
factor = (
vision_config.patch_size
* vision_config.spatial_merge_size
* self.get_hf_processor().image_processor.patch_expand_factor
)
pixels_per_token = vision_config.temporal_patch_size * factor * factor
max_tokens = max(1, self._get_image_max_pixels() // pixels_per_token)
short_side = math.isqrt(max_tokens)
while max_tokens % short_side:
short_side -= 1
return ImageSize(
width=(max_tokens // short_side) * factor,
height=short_side * factor,
)
def _get_video_second_idx_glm46v(
self, metadata: dict[str, Any], total_frames: int
) -> list[int]:
"""Timestamps for the video placeholder, from the pixel path's sampler.
The inherited GLM-4.6V timestamp sampling uses different constants
than this checkpoint's ``Glm5NextVideoProcessor`` pixel sampling, so
placeholder count and ``video_grid_thw`` disagree outside 30-300s
durations. Sampling with the pixel path's own policy makes one
timestamp per temporal patch == ``grid_t``.
"""
video_processor = self.get_video_processor()
temporal_patch_size = int(
getattr(video_processor, "temporal_patch_size", 1) or 1
)
video_fps = float(metadata["fps"])
if not metadata.get("do_sample_frames", True):
# The loader already picked the frames and the processor keeps them
# as they are, so the frame count is what was handed in and
# ``frames_indices`` only supplies the timestamp values.
frame_indices = [int(idx) for idx in metadata["frames_indices"]]
num_frames = int(total_frames)
else:
frame_indices = [
int(idx)
for idx in video_processor.sample_frames(
VideoMetadata(
total_num_frames=int(
metadata.get("total_num_frames", total_frames)
),
fps=video_fps,
duration=metadata.get("duration"),
)
)
]
num_frames = len(frame_indices)
if not frame_indices or num_frames <= 0:
# ``duration * fps < 1`` selects no frames at all; the
# pixel path raises an opaque IndexError on the same input a moment
# later, so say what happened while there is still a request.
raise ValueError(
"The GLM-5-Next frame sampler selected no frames for this "
f"video (fps={video_fps}, duration={metadata.get('duration')!r},"
f" total_num_frames={metadata.get('total_num_frames')!r}). "
"Clips shorter than one frame sampling interval cannot be "
"processed."
)
num_padded_frames = num_frames + (-num_frames % temporal_patch_size)
grid_t = max(num_padded_frames // temporal_patch_size, 1)
last = len(frame_indices) - 1
return [
int(frame_indices[min(i * temporal_patch_size, last)] / video_fps)
for i in range(grid_t)
]
def _get_vision_info(
self,
*,
image_width: int,
image_height: int,
num_frames: int = 16,
do_resize: bool = True,
max_image_pixels: int = 28 * 28 * 2 * 30000,
) -> tuple[ImageSize, int]:
"""GLM-5.3-Flash canvas geometry for token budgeting and dummy inputs.
The inherited Glm4v path resolves the pixel budget from
``size.longest_edge`` and resizes with GLM-4V's ``smart_resize``. This
checkpoint ships token bounds instead, and the alignment factor carries
``patch_expand_factor`` — resolve both from the processor so profiling
matches runtime geometry.
"""
from transformers.models.glm5_next.image_processing_glm5_next import (
smart_resize,
)
vision_config = self.get_hf_config().vision_config
patch_size = vision_config.patch_size
merge_size = vision_config.spatial_merge_size
temporal_patch_size = vision_config.temporal_patch_size
image_processor = self.get_hf_processor().image_processor
factor = self._alignment_factor(image_processor)
temporal_factor = image_processor.temporal_patch_size
# `smart_resize` denominates its bounds in vision tokens. Round down,
# but keep the profiling search viable when the caller's budget is
# below one aligned canvas of the requested duration.
max_image_tokens = max(
max_image_pixels // self._pixels_per_token(image_processor), 1
)
if do_resize:
t = num_frames if num_frames > temporal_factor else temporal_factor
resized_height, resized_width = smart_resize(
num_frames=t,
height=image_height,
width=image_width,
temporal_factor=temporal_factor,
factor=factor,
min_pixels=1,
max_pixels=max_image_tokens,
)
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
else:
preprocessed_size = ImageSize(width=image_width, height=image_height)
padded_num_frames = num_frames + (-num_frames % temporal_patch_size)
grid_t = max(padded_num_frames // temporal_patch_size, 1)
grid_h = preprocessed_size.height // patch_size
grid_w = preprocessed_size.width // patch_size
num_patches = grid_t * grid_h * grid_w
num_vision_tokens = num_patches // (merge_size**2)
return preprocessed_size, num_vision_tokens
class Glm5NextMultiModalProcessor(Glm4vMultiModalProcessor):
"""``Glm5NextProcessor`` extracts image/video features
only and passes the prompt text through unchanged, so prompt expansion
(image token repeat, video frame/timestamp structure) is owned by vLLM's
prompt-update machinery — the inherited ``_get_prompt_updates`` builds
the replacement content and the placeholder scan validates against
exactly that."""
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False