[Bugfix][GLM-5.3] Size the image encoder cache from the exact token ceiling (#59565)

Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com>
Co-authored-by: Kimi Code <kimi-code@moonshot.cn>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Yufeng He
2026-10-02 10:43:29 +00:00
committed by GitHub
co-authored by Kimi Code Isotr0py
parent af2a582d5c
commit 5688a4dd4a
2 changed files with 75 additions and 0 deletions
@@ -188,3 +188,54 @@ def test_mm_device_do_normalize():
torch.testing.assert_close( torch.testing.assert_close(
output, normalized_values.to(device), rtol=1e-5, atol=1e-6 output, normalized_values.to(device), rtol=1e-5, atol=1e-6
) )
def _image_info(**kwargs):
ctx = build_model_context(
"zai-org/GLM-5.3-Flash",
limit_mm_per_prompt={"image": 1},
**kwargs,
)
return MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
).info
def test_image_encoder_cache_covers_full_token_budget():
"""The most-features probe must reach the processor's token ceiling.
The inherited square probe refits to 89x89 = 7921 tokens under the
max_image_tokens=8000 budget, so the encoder cache came up short and
ordinary non-square images in 7922-8000 tokens were refused with
HTTP 400 (#59539).
"""
info = _image_info()
assert info.get_max_image_tokens() == 8000
# The shapes from the issue, with the token counts the processor
# actually produces; every one must fit the cache.
for width, height, expected_tokens in [
(4032, 3024, 7931), # phone photo, 4:3
(3840, 2160, 7973), # 4K frame, 16:9
(3508, 2480, 7950), # A4 at 300 dpi
(2600, 2400, 7998), # 13:12
(3000, 3000, 7921), # square worst case before the fix
]:
num_tokens = info.get_num_image_tokens(image_width=width, image_height=height)
assert num_tokens == expected_tokens
assert num_tokens <= info.get_max_image_tokens()
# The profiling dummy covers the real worst case, not the square one.
size = info.get_image_size_with_most_features()
assert size.width * size.height == 2240 * 2800
def test_image_encoder_cache_follows_max_pixels_override():
info = _image_info(mm_processor_kwargs={"max_pixels": 1568 * 100})
assert info.get_max_image_tokens() == 100
size = info.get_image_size_with_most_features()
assert (
info.get_num_image_tokens(image_width=size.width, image_height=size.height)
== 100
)
+24
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLM-5.3-Flash vision tower and multimodal processor.""" """GLM-5.3-Flash vision tower and multimodal processor."""
import math
from collections.abc import Mapping from collections.abc import Mapping
from functools import partial from functools import partial
from typing import Any from typing import Any
@@ -672,6 +673,29 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
return int(override) return int(override)
return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1] return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1]
def get_image_size_with_most_features(self) -> ImageSize:
# The inherited square probe strands budget whenever the token
# ceiling is not a perfect square: with max_image_tokens=8000 the
# square refits to 2492x2492 (89x89 = 7921 tokens) while a
# 2240x2800 canvas reaches 80x100 = 8000, so the encoder cache came
# up short of the processor's own maximum and refused valid images
# (#59539). Factor the ceiling exactly instead of probing a square.
vision_config = self.get_hf_config().vision_config
factor = (
vision_config.patch_size
* vision_config.spatial_merge_size
* self.get_hf_processor().image_processor.patch_expand_factor
)
pixels_per_token = vision_config.temporal_patch_size * factor * factor
max_tokens = max(1, self._get_image_max_pixels() // pixels_per_token)
short_side = math.isqrt(max_tokens)
while max_tokens % short_side:
short_side -= 1
return ImageSize(
width=(max_tokens // short_side) * factor,
height=short_side * factor,
)
def _get_video_second_idx_glm46v( def _get_video_second_idx_glm46v(
self, metadata: dict[str, Any], total_frames: int self, metadata: dict[str, Any], total_frames: int
) -> list[int]: ) -> list[int]: