[Bugfix][GLM-5.3] Size the image encoder cache from the exact token ceiling (#59565)
Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Co-authored-by: Kimi Code <kimi-code@moonshot.cn> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Kimi Code
Isotr0py
parent
af2a582d5c
commit
5688a4dd4a
@@ -188,3 +188,54 @@ def test_mm_device_do_normalize():
|
|||||||
torch.testing.assert_close(
|
torch.testing.assert_close(
|
||||||
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
|
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _image_info(**kwargs):
|
||||||
|
ctx = build_model_context(
|
||||||
|
"zai-org/GLM-5.3-Flash",
|
||||||
|
limit_mm_per_prompt={"image": 1},
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
return MULTIMODAL_REGISTRY.create_processor(
|
||||||
|
ctx.model_config,
|
||||||
|
tokenizer=ctx.tokenizer,
|
||||||
|
).info
|
||||||
|
|
||||||
|
|
||||||
|
def test_image_encoder_cache_covers_full_token_budget():
|
||||||
|
"""The most-features probe must reach the processor's token ceiling.
|
||||||
|
|
||||||
|
The inherited square probe refits to 89x89 = 7921 tokens under the
|
||||||
|
max_image_tokens=8000 budget, so the encoder cache came up short and
|
||||||
|
ordinary non-square images in 7922-8000 tokens were refused with
|
||||||
|
HTTP 400 (#59539).
|
||||||
|
"""
|
||||||
|
info = _image_info()
|
||||||
|
assert info.get_max_image_tokens() == 8000
|
||||||
|
|
||||||
|
# The shapes from the issue, with the token counts the processor
|
||||||
|
# actually produces; every one must fit the cache.
|
||||||
|
for width, height, expected_tokens in [
|
||||||
|
(4032, 3024, 7931), # phone photo, 4:3
|
||||||
|
(3840, 2160, 7973), # 4K frame, 16:9
|
||||||
|
(3508, 2480, 7950), # A4 at 300 dpi
|
||||||
|
(2600, 2400, 7998), # 13:12
|
||||||
|
(3000, 3000, 7921), # square worst case before the fix
|
||||||
|
]:
|
||||||
|
num_tokens = info.get_num_image_tokens(image_width=width, image_height=height)
|
||||||
|
assert num_tokens == expected_tokens
|
||||||
|
assert num_tokens <= info.get_max_image_tokens()
|
||||||
|
|
||||||
|
# The profiling dummy covers the real worst case, not the square one.
|
||||||
|
size = info.get_image_size_with_most_features()
|
||||||
|
assert size.width * size.height == 2240 * 2800
|
||||||
|
|
||||||
|
|
||||||
|
def test_image_encoder_cache_follows_max_pixels_override():
|
||||||
|
info = _image_info(mm_processor_kwargs={"max_pixels": 1568 * 100})
|
||||||
|
assert info.get_max_image_tokens() == 100
|
||||||
|
size = info.get_image_size_with_most_features()
|
||||||
|
assert (
|
||||||
|
info.get_num_image_tokens(image_width=size.width, image_height=size.height)
|
||||||
|
== 100
|
||||||
|
)
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""GLM-5.3-Flash vision tower and multimodal processor."""
|
"""GLM-5.3-Flash vision tower and multimodal processor."""
|
||||||
|
|
||||||
|
import math
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -672,6 +673,29 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
|||||||
return int(override)
|
return int(override)
|
||||||
return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1]
|
return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1]
|
||||||
|
|
||||||
|
def get_image_size_with_most_features(self) -> ImageSize:
|
||||||
|
# The inherited square probe strands budget whenever the token
|
||||||
|
# ceiling is not a perfect square: with max_image_tokens=8000 the
|
||||||
|
# square refits to 2492x2492 (89x89 = 7921 tokens) while a
|
||||||
|
# 2240x2800 canvas reaches 80x100 = 8000, so the encoder cache came
|
||||||
|
# up short of the processor's own maximum and refused valid images
|
||||||
|
# (#59539). Factor the ceiling exactly instead of probing a square.
|
||||||
|
vision_config = self.get_hf_config().vision_config
|
||||||
|
factor = (
|
||||||
|
vision_config.patch_size
|
||||||
|
* vision_config.spatial_merge_size
|
||||||
|
* self.get_hf_processor().image_processor.patch_expand_factor
|
||||||
|
)
|
||||||
|
pixels_per_token = vision_config.temporal_patch_size * factor * factor
|
||||||
|
max_tokens = max(1, self._get_image_max_pixels() // pixels_per_token)
|
||||||
|
short_side = math.isqrt(max_tokens)
|
||||||
|
while max_tokens % short_side:
|
||||||
|
short_side -= 1
|
||||||
|
return ImageSize(
|
||||||
|
width=(max_tokens // short_side) * factor,
|
||||||
|
height=short_side * factor,
|
||||||
|
)
|
||||||
|
|
||||||
def _get_video_second_idx_glm46v(
|
def _get_video_second_idx_glm46v(
|
||||||
self, metadata: dict[str, Any], total_frames: int
|
self, metadata: dict[str, Any], total_frames: int
|
||||||
) -> list[int]:
|
) -> list[int]:
|
||||||
|
|||||||
Reference in New Issue
Block a user