diff --git a/tests/models/multimodal/processing/test_glm5next.py b/tests/models/multimodal/processing/test_glm5next.py index 719154a2b4..111fb982cf 100644 --- a/tests/models/multimodal/processing/test_glm5next.py +++ b/tests/models/multimodal/processing/test_glm5next.py @@ -188,3 +188,54 @@ def test_mm_device_do_normalize(): torch.testing.assert_close( output, normalized_values.to(device), rtol=1e-5, atol=1e-6 ) + + +def _image_info(**kwargs): + ctx = build_model_context( + "zai-org/GLM-5.3-Flash", + limit_mm_per_prompt={"image": 1}, + **kwargs, + ) + return MULTIMODAL_REGISTRY.create_processor( + ctx.model_config, + tokenizer=ctx.tokenizer, + ).info + + +def test_image_encoder_cache_covers_full_token_budget(): + """The most-features probe must reach the processor's token ceiling. + + The inherited square probe refits to 89x89 = 7921 tokens under the + max_image_tokens=8000 budget, so the encoder cache came up short and + ordinary non-square images in 7922-8000 tokens were refused with + HTTP 400 (#59539). + """ + info = _image_info() + assert info.get_max_image_tokens() == 8000 + + # The shapes from the issue, with the token counts the processor + # actually produces; every one must fit the cache. + for width, height, expected_tokens in [ + (4032, 3024, 7931), # phone photo, 4:3 + (3840, 2160, 7973), # 4K frame, 16:9 + (3508, 2480, 7950), # A4 at 300 dpi + (2600, 2400, 7998), # 13:12 + (3000, 3000, 7921), # square worst case before the fix + ]: + num_tokens = info.get_num_image_tokens(image_width=width, image_height=height) + assert num_tokens == expected_tokens + assert num_tokens <= info.get_max_image_tokens() + + # The profiling dummy covers the real worst case, not the square one. + size = info.get_image_size_with_most_features() + assert size.width * size.height == 2240 * 2800 + + +def test_image_encoder_cache_follows_max_pixels_override(): + info = _image_info(mm_processor_kwargs={"max_pixels": 1568 * 100}) + assert info.get_max_image_tokens() == 100 + size = info.get_image_size_with_most_features() + assert ( + info.get_num_image_tokens(image_width=size.width, image_height=size.height) + == 100 + ) diff --git a/vllm/models/glm5next/common/multimodal.py b/vllm/models/glm5next/common/multimodal.py index 75921b697f..7384c8c316 100644 --- a/vllm/models/glm5next/common/multimodal.py +++ b/vllm/models/glm5next/common/multimodal.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """GLM-5.3-Flash vision tower and multimodal processor.""" +import math from collections.abc import Mapping from functools import partial from typing import Any @@ -672,6 +673,29 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo): return int(override) return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1] + def get_image_size_with_most_features(self) -> ImageSize: + # The inherited square probe strands budget whenever the token + # ceiling is not a perfect square: with max_image_tokens=8000 the + # square refits to 2492x2492 (89x89 = 7921 tokens) while a + # 2240x2800 canvas reaches 80x100 = 8000, so the encoder cache came + # up short of the processor's own maximum and refused valid images + # (#59539). Factor the ceiling exactly instead of probing a square. + vision_config = self.get_hf_config().vision_config + factor = ( + vision_config.patch_size + * vision_config.spatial_merge_size + * self.get_hf_processor().image_processor.patch_expand_factor + ) + pixels_per_token = vision_config.temporal_patch_size * factor * factor + max_tokens = max(1, self._get_image_max_pixels() // pixels_per_token) + short_side = math.isqrt(max_tokens) + while max_tokens % short_side: + short_side -= 1 + return ImageSize( + width=(max_tokens // short_side) * factor, + height=short_side * factor, + ) + def _get_video_second_idx_glm46v( self, metadata: dict[str, Any], total_frames: int ) -> list[int]: