[Model] Extend device-side mm normalization to GLM4V/GLM5Next (#55389)

Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
This commit is contained in:
cjackal
2026-09-30 15:44:46 +08:00
committed by GitHub
parent 8ee4069033
commit 0a30bc3f9a
7 changed files with 114 additions and 10 deletions
@@ -13,8 +13,12 @@ The checks below are arithmetic: no weights, no GPU.
"""
import pytest
import torch
from PIL import Image
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from vllm.transformers_utils.processors.glm5next import (
Glm5NextVideoProcessor,
_pixel_budget,
@@ -168,3 +172,36 @@ def test_video_shorter_than_one_sampling_interval_is_rejected(processor):
},
2,
)
@pytest.mark.usefixtures("default_vllm_config")
def test_mm_device_do_normalize():
device = current_platform.device_type
ctx = build_model_context(
"zai-org/GLM-5.3-Flash",
limit_mm_per_prompt={"image": 2},
)
assert ctx.model_config.multimodal_config.mm_device_do_normalize
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
images = [
Image.new("RGB", (310, 470), color=(17, 89, 231)),
Image.new("RGB", (480, 320), color=(201, 13, 127)),
]
prompt = "<|begin_of_image|><|image|><|end_of_image|>" * len(images)
mm_items = processor.info.parse_mm_data({"image": images})
normalized_inputs = processor(prompt, mm_items=mm_items)
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
raw_inputs = processor(prompt, mm_items=mm_items)
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
assert raw_values.dtype == torch.uint8
input_norm = build_mm_input_norm(ctx.model_config).to(device)
output = input_norm(raw_values.to(device), normalized_values.dtype)
torch.testing.assert_close(
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
)