[Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
866f02db8a
commit
e3cae8d2ac
@@ -3,13 +3,14 @@
|
||||
"""GLM-5.3-Flash vision tower and multimodal processor."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from functools import cached_property, partial
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -44,7 +45,6 @@ from vllm.model_executor.models.vision import (
|
||||
)
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
|
||||
from vllm.transformers_utils.processors.glm5next import glm_sample_frame_indices
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
@@ -616,37 +616,48 @@ class Glm5NextVisionTransformer(nn.Module):
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
"""Wires up the vLLM-native processor for the multimodal checkpoint.
|
||||
_MAX_VIDEO_TOKENS = 30000
|
||||
|
||||
The checkpoint's ``processor_config.json`` declares a custom ``processor_class``
|
||||
and stores its image/video processor configs inline (no standalone
|
||||
``preprocessor_config.json``), so ``AutoProcessor`` cannot resolve the
|
||||
config. We bypass it and build our own ``Glm5NextProcessor``
|
||||
(``vllm/transformers_utils/processors/glm5next.py``), a port of the
|
||||
training-side pipeline that no longer imports transformers' GLM processor
|
||||
classes. The port applies ``patch_expand_factor`` (checkpoint ships 1)
|
||||
inside ``smart_resize``'s spatial factor.
|
||||
|
||||
class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
"""Token-budget geometry for the multimodal checkpoint.
|
||||
|
||||
This checkpoint's ``processor_config.json`` ships the token-budget style
|
||||
(``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, so the
|
||||
inherited Glm4v ``size.longest_edge`` path does not apply.
|
||||
"""
|
||||
|
||||
@cached_property
|
||||
def _glm5_hf_processor(self):
|
||||
from vllm.transformers_utils.processors.glm5next import Glm5NextProcessor
|
||||
|
||||
return Glm5NextProcessor.from_pretrained(self.ctx.model_config.model)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
return self._glm5_hf_processor
|
||||
processor = super().get_hf_processor(**kwargs)
|
||||
# Cap video inputs to keep encoder profiling from starving the KV cache.
|
||||
video_processor = processor.video_processor
|
||||
video_processor.max_image_tokens = min(
|
||||
video_processor.max_image_tokens, _MAX_VIDEO_TOKENS
|
||||
)
|
||||
return processor
|
||||
|
||||
@staticmethod
|
||||
def _alignment_factor(proc) -> int:
|
||||
"""Spatial factor canvases align to, as ``smart_resize`` takes it."""
|
||||
return proc.patch_size * proc.merge_size * proc.patch_expand_factor
|
||||
|
||||
@classmethod
|
||||
def _pixels_per_token(cls, proc) -> int:
|
||||
"""Pixels one vision token covers, matching ``smart_resize``'s own
|
||||
``temporal_factor * factor ** 2``."""
|
||||
return proc.temporal_patch_size * cls._alignment_factor(proc) ** 2
|
||||
|
||||
def _processor_pixel_budget(self, proc) -> tuple[int, int]:
|
||||
from vllm.transformers_utils.processors.glm5next import _pixel_budget
|
||||
|
||||
return _pixel_budget(
|
||||
proc.min_image_tokens,
|
||||
proc.max_image_tokens,
|
||||
proc.patch_size,
|
||||
proc.merge_size,
|
||||
proc.temporal_patch_size,
|
||||
"""(min_pixels, max_pixels) from the processor's token bounds."""
|
||||
if proc.min_image_tokens is None or proc.max_image_tokens is None:
|
||||
raise ValueError(
|
||||
"min_image_tokens and max_image_tokens must be provided by "
|
||||
"processor_config.json (or per-call kwargs)."
|
||||
)
|
||||
pixels_per_token = self._pixels_per_token(proc)
|
||||
return (
|
||||
proc.min_image_tokens * pixels_per_token,
|
||||
proc.max_image_tokens * pixels_per_token,
|
||||
)
|
||||
|
||||
def _get_image_max_pixels(self) -> int:
|
||||
@@ -687,19 +698,20 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
else:
|
||||
frame_indices = [
|
||||
int(idx)
|
||||
for idx in glm_sample_frame_indices(
|
||||
int(metadata.get("total_num_frames", total_frames)),
|
||||
video_fps,
|
||||
float(metadata.get("duration") or 0),
|
||||
target_fps=video_processor.fps_interval,
|
||||
max_frame_count=video_processor.max_frame_count_dynamic,
|
||||
temporal_patch_size=temporal_patch_size,
|
||||
for idx in video_processor.sample_frames(
|
||||
VideoMetadata(
|
||||
total_num_frames=int(
|
||||
metadata.get("total_num_frames", total_frames)
|
||||
),
|
||||
fps=video_fps,
|
||||
duration=metadata.get("duration"),
|
||||
)
|
||||
)
|
||||
]
|
||||
num_frames = len(frame_indices)
|
||||
|
||||
if not frame_indices or num_frames <= 0:
|
||||
# ``duration * fps_interval < 1`` selects no frames at all; the
|
||||
# ``duration * fps < 1`` selects no frames at all; the
|
||||
# pixel path raises an opaque IndexError on the same input a moment
|
||||
# later, so say what happened while there is still a request.
|
||||
raise ValueError(
|
||||
@@ -731,12 +743,13 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
|
||||
The inherited Glm4v path resolves the pixel budget from
|
||||
``size.longest_edge`` and resizes with GLM-4V's ``smart_resize``. This
|
||||
checkpoint's ``processor_config.json`` ships the token-budget style
|
||||
(``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, and
|
||||
the alignment factor carries ``patch_expand_factor`` — resolve both
|
||||
from the vLLM-native processor so profiling matches runtime geometry.
|
||||
checkpoint ships token bounds instead, and the alignment factor carries
|
||||
``patch_expand_factor`` — resolve both from the processor so profiling
|
||||
matches runtime geometry.
|
||||
"""
|
||||
from vllm.transformers_utils.processors.glm5next import smart_resize
|
||||
from transformers.models.glm5_next.image_processing_glm5_next import (
|
||||
smart_resize,
|
||||
)
|
||||
|
||||
vision_config = self.get_hf_config().vision_config
|
||||
patch_size = vision_config.patch_size
|
||||
@@ -744,22 +757,25 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
temporal_patch_size = vision_config.temporal_patch_size
|
||||
|
||||
image_processor = self.get_hf_processor().image_processor
|
||||
factor = patch_size * merge_size * image_processor.patch_expand_factor
|
||||
# Keep the profiling search viable when the caller's budget is below
|
||||
# one aligned canvas of the requested duration.
|
||||
max_image_pixels = max(max_image_pixels, temporal_patch_size * factor * factor)
|
||||
factor = self._alignment_factor(image_processor)
|
||||
temporal_factor = image_processor.temporal_patch_size
|
||||
# `smart_resize` denominates its bounds in vision tokens. Round down,
|
||||
# but keep the profiling search viable when the caller's budget is
|
||||
# below one aligned canvas of the requested duration.
|
||||
max_image_tokens = max(
|
||||
max_image_pixels // self._pixels_per_token(image_processor), 1
|
||||
)
|
||||
|
||||
if do_resize:
|
||||
t = num_frames if num_frames > temporal_patch_size else temporal_patch_size
|
||||
t = num_frames if num_frames > temporal_factor else temporal_factor
|
||||
resized_height, resized_width = smart_resize(
|
||||
t=t,
|
||||
h=image_height,
|
||||
w=image_width,
|
||||
t_factor=temporal_patch_size,
|
||||
h_factor=factor,
|
||||
w_factor=factor,
|
||||
num_frames=t,
|
||||
height=image_height,
|
||||
width=image_width,
|
||||
temporal_factor=temporal_factor,
|
||||
factor=factor,
|
||||
min_pixels=1,
|
||||
max_pixels=max_image_pixels,
|
||||
max_pixels=max_image_tokens,
|
||||
)
|
||||
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
|
||||
else:
|
||||
@@ -777,7 +793,7 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
|
||||
|
||||
class Glm5NextMultiModalProcessor(Glm4vMultiModalProcessor):
|
||||
"""The vLLM-native ``Glm5NextProcessor`` extracts image/video features
|
||||
"""``Glm5NextProcessor`` extracts image/video features
|
||||
only and passes the prompt text through unchanged, so prompt expansion
|
||||
(image token repeat, video frame/timestamp structure) is owned by vLLM's
|
||||
prompt-update machinery — the inherited ``_get_prompt_updates`` builds
|
||||
|
||||
Reference in New Issue
Block a user