From 0a30bc3f9ac3cc1a9339e115377a99d32252aed0 Mon Sep 17 00:00:00 2001 From: cjackal <44624812+cjackal@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:44:46 +0900 Subject: [PATCH] [Model] Extend device-side mm normalization to GLM4V/GLM5Next (#55389) Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com> --- docs/design/mm_processing.md | 3 ++ .../generation_ppl_test/test_glm.py | 40 +++++++++++++++++++ .../multimodal/processing/test_glm5next.py | 37 +++++++++++++++++ vllm/model_executor/models/glm4_1v.py | 23 +++++++---- vllm/model_executor/models/glm_ocr.py | 14 ++++++- vllm/models/glm5next/common/model.py | 2 + vllm/models/glm5next/common/multimodal.py | 5 ++- 7 files changed, 114 insertions(+), 10 deletions(-) create mode 100644 tests/models/multimodal/generation_ppl_test/test_glm.py diff --git a/docs/design/mm_processing.md b/docs/design/mm_processing.md index 3a37a2d8a4..67b1a7742b 100644 --- a/docs/design/mm_processing.md +++ b/docs/design/mm_processing.md @@ -80,6 +80,9 @@ GPU-side fusion is controlled by `multimodal_config.mm_device_do_normalize`: | name | Architecture | Example HF Models | | -------------------------- | ------------------------------------------ | ----------------------------------------------------- | +| `glm-4v` | `Glm4vForConditionalGeneration` | `zai-org/GLM-4.1V-9B-Thinking`, etc. | +| `glm-5-next` | `Glm5NextForConditionalGeneration` | `zai-org/GLM-5.3-Flash` | +| `glm-ocr` | `GlmOcrForConditionalGeneration` | `zai-org/GLM-OCR` | | `intern-s1-pro` | `InternS1ProForConditionalGeneration` | `internlm/Intern-S1-Pro` | | `intern-s2-mobius` | `InternS2MobiusForConditionalGeneration` | `internlm/Intern-S2-Mobius` | | `llama-nemotron-vl-embed` | `LlamaNemotronVLForEmbedding` | `nvidia/llama-nemotron-embed-vl-1b-v2` | diff --git a/tests/models/multimodal/generation_ppl_test/test_glm.py b/tests/models/multimodal/generation_ppl_test/test_glm.py new file mode 100644 index 0000000000..e724ace657 --- /dev/null +++ b/tests/models/multimodal/generation_ppl_test/test_glm.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from tests.models.utils import GenerateModelInfo + +from .ppl_utils import vqa_ppl_test + +MODELS = [ + (GenerateModelInfo("zai-org/GLM-OCR", hf_ppl=13.885052680969238), 28), +] + + +def _pixel_size(factor: int): + # Match max image token size with qwen's PPL test + return { + "size": { + "shortest_edge": 2 * factor**2, + "longest_edge": 2 * 1280 * factor**2, + } + } + + +@pytest.mark.parametrize("model_info,factor", MODELS) +@pytest.mark.parametrize("mm_device_do_normalize", [True, False]) +def test_ppl( + hf_runner, + vllm_runner, + model_info: GenerateModelInfo, + factor: int, + mm_device_do_normalize: bool, +): + vqa_ppl_test( + hf_runner, + vllm_runner, + model_info, + vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize}, + mm_processor_kwargs=_pixel_size(factor), + ) diff --git a/tests/models/multimodal/processing/test_glm5next.py b/tests/models/multimodal/processing/test_glm5next.py index 6fb0e397c2..0e1e7b9876 100644 --- a/tests/models/multimodal/processing/test_glm5next.py +++ b/tests/models/multimodal/processing/test_glm5next.py @@ -13,8 +13,12 @@ The checks below are arithmetic: no weights, no GPU. """ import pytest +import torch +from PIL import Image +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.platforms import current_platform from vllm.transformers_utils.processors.glm5next import ( Glm5NextVideoProcessor, _pixel_budget, @@ -168,3 +172,36 @@ def test_video_shorter_than_one_sampling_interval_is_rejected(processor): }, 2, ) + + +@pytest.mark.usefixtures("default_vllm_config") +def test_mm_device_do_normalize(): + device = current_platform.device_type + ctx = build_model_context( + "zai-org/GLM-5.3-Flash", + limit_mm_per_prompt={"image": 2}, + ) + assert ctx.model_config.multimodal_config.mm_device_do_normalize + + ctx.model_config.multimodal_config.mm_device_do_normalize = False + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + images = [ + Image.new("RGB", (310, 470), color=(17, 89, 231)), + Image.new("RGB", (480, 320), color=(201, 13, 127)), + ] + prompt = "<|begin_of_image|><|image|><|end_of_image|>" * len(images) + mm_items = processor.info.parse_mm_data({"image": images}) + + normalized_inputs = processor(prompt, mm_items=mm_items) + normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"] + + ctx.model_config.multimodal_config.mm_device_do_normalize = True + raw_inputs = processor(prompt, mm_items=mm_items) + raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"] + assert raw_values.dtype == torch.uint8 + + input_norm = build_mm_input_norm(ctx.model_config).to(device) + output = input_norm(raw_values.to(device), normalized_values.dtype) + torch.testing.assert_close( + output, normalized_values.to(device), rtol=1e-5, atol=1e-6 + ) diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 95cdcc5a20..be456a6a58 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -61,6 +61,10 @@ from vllm.model_executor.layers.attention import ( MMEncoderAttention, ) from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer +from vllm.model_executor.layers.fusion.mm_input_norm import ( + IdentityInputNorm, + build_mm_input_norm, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -616,6 +620,7 @@ class Glm4vVisionTransformer(nn.Module): vision_config: Glm4vVisionConfig, norm_eps: float = 1e-6, quant_config: QuantizationConfig | None = None, + input_norm: nn.Module | None = None, prefix: str = "", ) -> None: super().__init__() @@ -642,6 +647,7 @@ class Glm4vVisionTransformer(nn.Module): in_channels=in_channels, hidden_size=self.hidden_size, ) + self.input_norm = input_norm if input_norm is not None else IdentityInputNorm() norm_layer = partial(RMSNorm, eps=norm_eps) head_dim = self.hidden_size // self.num_heads @@ -924,7 +930,7 @@ class Glm4vVisionTransformer(nn.Module): encoder_metadata = self.prepare_encoder_metadata(grid_thw) # patchify - x = x.to(device=self.device, dtype=self.dtype) + x = self.input_norm(x.to(device=self.device), self.dtype) x = self.patch_embed(x) x = self.post_conv_layernorm(x) @@ -1755,6 +1761,7 @@ class Glm4vForConditionalGeneration( supports_encoder_tp_data = True supports_tower_connector_lora = True + supports_mm_device_do_normalize = True @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: @@ -1786,6 +1793,7 @@ class Glm4vForConditionalGeneration( config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-5), quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) @@ -1867,7 +1875,7 @@ class Glm4vForConditionalGeneration( if image_input["type"] == "image_embeds": image_embeds = image_input["image_embeds"].type(self.visual.dtype) else: - pixel_values = image_input["pixel_values"].type(self.visual.dtype) + pixel_values = image_input["pixel_values"] if self.use_data_parallel: return run_dp_sharded_mrope_vision_model( self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d" @@ -1888,9 +1896,7 @@ class Glm4vForConditionalGeneration( if video_input["type"] == "video_embeds": video_embeds = video_input["video_embeds"].type(self.visual.dtype) else: - pixel_values_videos = video_input["pixel_values_videos"].type( - self.visual.dtype - ) + pixel_values_videos = video_input["pixel_values_videos"] if self.use_data_parallel: return run_dp_sharded_mrope_vision_model( self.visual, @@ -2098,8 +2104,11 @@ class Glm4vForConditionalGeneration( flattened_patch_size = ( in_channels * temporal_patch_size * patch_size * patch_size ) - dummy_pixel_values = torch.randn( - total_patches, flattened_patch_size, device=device, dtype=dtype + dummy_pixel_values = torch.zeros( + total_patches, + flattened_patch_size, + device=device, + dtype=self.visual.input_norm.input_dtype or dtype, ) # Override max_seqlen with a safe upper bound for capture. diff --git a/vllm/model_executor/models/glm_ocr.py b/vllm/model_executor/models/glm_ocr.py index 407b08e8a1..ea773f658d 100644 --- a/vllm/model_executor/models/glm_ocr.py +++ b/vllm/model_executor/models/glm_ocr.py @@ -48,6 +48,7 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import ( MMEncoderAttention, ) from vllm.model_executor.layers.conv import Conv2dLayer +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -254,9 +255,17 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer): vision_config: "GlmOcrVisionConfig", norm_eps: float = 1e-5, quant_config: QuantizationConfig | None = None, + input_norm: nn.Module | None = None, prefix: str = "", ) -> None: - super().__init__(text_config, vision_config, norm_eps, quant_config, prefix) + super().__init__( + text_config, + vision_config, + norm_eps, + quant_config, + input_norm, + prefix, + ) del self.post_conv_layernorm del self.embeddings @@ -334,7 +343,7 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer): grid_thw = torch.tensor(grid_thw, dtype=torch.int32) # patchify - x = x.to(device=self.device, dtype=self.dtype) + x = self.input_norm(x.to(device=self.device), self.dtype) x = self.patch_embed(x) # compute position embedding @@ -393,5 +402,6 @@ class GlmOcrForConditionalGeneration(Glm4vForConditionalGeneration): config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-5), quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/models/glm5next/common/model.py b/vllm/models/glm5next/common/model.py index 1c826f58be..3ff3476375 100644 --- a/vllm/models/glm5next/common/model.py +++ b/vllm/models/glm5next/common/model.py @@ -22,6 +22,7 @@ from vllm.model_executor.layers.fused_moe import ( GateLinear, fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -1103,6 +1104,7 @@ class Glm5NextForConditionalGeneration( # and yields NaN image features. Mirrors the MLA/KDA proj # pattern (quant_config=None for BF16 submodules). quant_config=None, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/models/glm5next/common/multimodal.py b/vllm/models/glm5next/common/multimodal.py index 03ac3fa664..60b5b47a60 100644 --- a/vllm/models/glm5next/common/multimodal.py +++ b/vllm/models/glm5next/common/multimodal.py @@ -19,6 +19,7 @@ from vllm.distributed import utils as dist_utils from vllm.model_executor.layers.activation import SiluAndMulWithClamp from vllm.model_executor.layers.attention import MMEncoderAttention from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer +from vllm.model_executor.layers.fusion.mm_input_norm import IdentityInputNorm from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -353,6 +354,7 @@ class Glm5NextVisionTransformer(nn.Module): vision_config, norm_eps: float = 1e-6, quant_config: QuantizationConfig | None = None, + input_norm: nn.Module | None = None, prefix: str = "", ) -> None: super().__init__() @@ -386,6 +388,7 @@ class Glm5NextVisionTransformer(nn.Module): in_channels=in_channels, hidden_size=self.hidden_size, ) + self.input_norm = input_norm if input_norm is not None else IdentityInputNorm() norm_layer = partial(RMSNorm, eps=norm_eps) head_dim = self.hidden_size // self.num_heads @@ -566,7 +569,7 @@ class Glm5NextVisionTransformer(nn.Module): encoder_metadata: dict[str, torch.Tensor] | None = None, ) -> torch.Tensor: # patchify - x = x.to(device=self.device, dtype=self.dtype) + x = self.input_norm(x.to(device=self.device), self.dtype) x = self.patch_embed(x) if encoder_metadata is not None: