[Model] Extend device-side mm normalization to GLM4V/GLM5Next (#55389)

Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
This commit is contained in:
cjackal
2026-09-30 15:44:46 +08:00
committed by GitHub
parent 8ee4069033
commit 0a30bc3f9a
7 changed files with 114 additions and 10 deletions
+3
View File
@@ -80,6 +80,9 @@ GPU-side fusion is controlled by `multimodal_config.mm_device_do_normalize`:
| name | Architecture | Example HF Models |
| -------------------------- | ------------------------------------------ | ----------------------------------------------------- |
| `glm-4v` | `Glm4vForConditionalGeneration` | `zai-org/GLM-4.1V-9B-Thinking`, etc. |
| `glm-5-next` | `Glm5NextForConditionalGeneration` | `zai-org/GLM-5.3-Flash` |
| `glm-ocr` | `GlmOcrForConditionalGeneration` | `zai-org/GLM-OCR` |
| `intern-s1-pro` | `InternS1ProForConditionalGeneration` | `internlm/Intern-S1-Pro` |
| `intern-s2-mobius` | `InternS2MobiusForConditionalGeneration` | `internlm/Intern-S2-Mobius` |
| `llama-nemotron-vl-embed` | `LlamaNemotronVLForEmbedding` | `nvidia/llama-nemotron-embed-vl-1b-v2` |
@@ -0,0 +1,40 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.utils import GenerateModelInfo
from .ppl_utils import vqa_ppl_test
MODELS = [
(GenerateModelInfo("zai-org/GLM-OCR", hf_ppl=13.885052680969238), 28),
]
def _pixel_size(factor: int):
# Match max image token size with qwen's PPL test
return {
"size": {
"shortest_edge": 2 * factor**2,
"longest_edge": 2 * 1280 * factor**2,
}
}
@pytest.mark.parametrize("model_info,factor", MODELS)
@pytest.mark.parametrize("mm_device_do_normalize", [True, False])
def test_ppl(
hf_runner,
vllm_runner,
model_info: GenerateModelInfo,
factor: int,
mm_device_do_normalize: bool,
):
vqa_ppl_test(
hf_runner,
vllm_runner,
model_info,
vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize},
mm_processor_kwargs=_pixel_size(factor),
)
@@ -13,8 +13,12 @@ The checks below are arithmetic: no weights, no GPU.
"""
import pytest
import torch
from PIL import Image
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from vllm.transformers_utils.processors.glm5next import (
Glm5NextVideoProcessor,
_pixel_budget,
@@ -168,3 +172,36 @@ def test_video_shorter_than_one_sampling_interval_is_rejected(processor):
},
2,
)
@pytest.mark.usefixtures("default_vllm_config")
def test_mm_device_do_normalize():
device = current_platform.device_type
ctx = build_model_context(
"zai-org/GLM-5.3-Flash",
limit_mm_per_prompt={"image": 2},
)
assert ctx.model_config.multimodal_config.mm_device_do_normalize
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
images = [
Image.new("RGB", (310, 470), color=(17, 89, 231)),
Image.new("RGB", (480, 320), color=(201, 13, 127)),
]
prompt = "<|begin_of_image|><|image|><|end_of_image|>" * len(images)
mm_items = processor.info.parse_mm_data({"image": images})
normalized_inputs = processor(prompt, mm_items=mm_items)
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
raw_inputs = processor(prompt, mm_items=mm_items)
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
assert raw_values.dtype == torch.uint8
input_norm = build_mm_input_norm(ctx.model_config).to(device)
output = input_norm(raw_values.to(device), normalized_values.dtype)
torch.testing.assert_close(
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
)
+16 -7
View File
@@ -61,6 +61,10 @@ from vllm.model_executor.layers.attention import (
MMEncoderAttention,
)
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import (
IdentityInputNorm,
build_mm_input_norm,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -616,6 +620,7 @@ class Glm4vVisionTransformer(nn.Module):
vision_config: Glm4vVisionConfig,
norm_eps: float = 1e-6,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -642,6 +647,7 @@ class Glm4vVisionTransformer(nn.Module):
in_channels=in_channels,
hidden_size=self.hidden_size,
)
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
norm_layer = partial(RMSNorm, eps=norm_eps)
head_dim = self.hidden_size // self.num_heads
@@ -924,7 +930,7 @@ class Glm4vVisionTransformer(nn.Module):
encoder_metadata = self.prepare_encoder_metadata(grid_thw)
# patchify
x = x.to(device=self.device, dtype=self.dtype)
x = self.input_norm(x.to(device=self.device), self.dtype)
x = self.patch_embed(x)
x = self.post_conv_layernorm(x)
@@ -1755,6 +1761,7 @@ class Glm4vForConditionalGeneration(
supports_encoder_tp_data = True
supports_tower_connector_lora = True
supports_mm_device_do_normalize = True
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
@@ -1786,6 +1793,7 @@ class Glm4vForConditionalGeneration(
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-5),
quant_config=quant_config,
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
@@ -1867,7 +1875,7 @@ class Glm4vForConditionalGeneration(
if image_input["type"] == "image_embeds":
image_embeds = image_input["image_embeds"].type(self.visual.dtype)
else:
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
pixel_values = image_input["pixel_values"]
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d"
@@ -1888,9 +1896,7 @@ class Glm4vForConditionalGeneration(
if video_input["type"] == "video_embeds":
video_embeds = video_input["video_embeds"].type(self.visual.dtype)
else:
pixel_values_videos = video_input["pixel_values_videos"].type(
self.visual.dtype
)
pixel_values_videos = video_input["pixel_values_videos"]
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
self.visual,
@@ -2098,8 +2104,11 @@ class Glm4vForConditionalGeneration(
flattened_patch_size = (
in_channels * temporal_patch_size * patch_size * patch_size
)
dummy_pixel_values = torch.randn(
total_patches, flattened_patch_size, device=device, dtype=dtype
dummy_pixel_values = torch.zeros(
total_patches,
flattened_patch_size,
device=device,
dtype=self.visual.input_norm.input_dtype or dtype,
)
# Override max_seqlen with a safe upper bound for capture.
+12 -2
View File
@@ -48,6 +48,7 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -254,9 +255,17 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
vision_config: "GlmOcrVisionConfig",
norm_eps: float = 1e-5,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
) -> None:
super().__init__(text_config, vision_config, norm_eps, quant_config, prefix)
super().__init__(
text_config,
vision_config,
norm_eps,
quant_config,
input_norm,
prefix,
)
del self.post_conv_layernorm
del self.embeddings
@@ -334,7 +343,7 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
grid_thw = torch.tensor(grid_thw, dtype=torch.int32)
# patchify
x = x.to(device=self.device, dtype=self.dtype)
x = self.input_norm(x.to(device=self.device), self.dtype)
x = self.patch_embed(x)
# compute position embedding
@@ -393,5 +402,6 @@ class GlmOcrForConditionalGeneration(Glm4vForConditionalGeneration):
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-5),
quant_config=quant_config,
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
+2
View File
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.fused_moe import (
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -1103,6 +1104,7 @@ class Glm5NextForConditionalGeneration(
# and yields NaN image features. Mirrors the MLA/KDA proj
# pattern (quant_config=None for BF16 submodules).
quant_config=None,
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
+4 -1
View File
@@ -19,6 +19,7 @@ from vllm.distributed import utils as dist_utils
from vllm.model_executor.layers.activation import SiluAndMulWithClamp
from vllm.model_executor.layers.attention import MMEncoderAttention
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import IdentityInputNorm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -353,6 +354,7 @@ class Glm5NextVisionTransformer(nn.Module):
vision_config,
norm_eps: float = 1e-6,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -386,6 +388,7 @@ class Glm5NextVisionTransformer(nn.Module):
in_channels=in_channels,
hidden_size=self.hidden_size,
)
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
norm_layer = partial(RMSNorm, eps=norm_eps)
head_dim = self.hidden_size // self.num_heads
@@ -566,7 +569,7 @@ class Glm5NextVisionTransformer(nn.Module):
encoder_metadata: dict[str, torch.Tensor] | None = None,
) -> torch.Tensor:
# patchify
x = x.to(device=self.device, dtype=self.dtype)
x = self.input_norm(x.to(device=self.device), self.dtype)
x = self.patch_embed(x)
if encoder_metadata is not None: