[Model] Fused mm preprocess normalisation on the Device (#50411)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: wang.yuqi <noooop@126.com>
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
wang.yuqi
2026-08-06 03:11:14 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent 9c22668436
commit 2fa490470d
13 changed files with 359 additions and 11 deletions
@@ -2,9 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from packaging.version import Version
from transformers import __version__ as TRANSFORMERS_VERSION
from vllm.model_executor.models.vision import FusedInputNorm
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
@@ -128,3 +130,52 @@ def test_get_image_size_with_most_features(
t, h, w = grid_thw[0]
tokens = (t * h * w) // (merge_size**2)
assert tokens < max_tokens
@pytest.mark.parametrize(
"model_id", ["Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]
)
@pytest.mark.parametrize("num_imgs", [1, 2])
def test_mm_device_do_normalize(
image_assets: ImageTestAssets,
model_id: str,
num_imgs: int,
):
"""Ensure that enable mm_device_do_normalize yields the correct result."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": num_imgs},
)
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Build the image str / prompt based on the number of images we pass
prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
processed_inputs_with_normalize = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
)
pixel_values_with_normalize = processed_inputs_with_normalize[
"mm_kwargs"
].get_data()["pixel_values"]
dtype = pixel_values_with_normalize.dtype
processed_inputs_without_normalize = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={"do_normalize": False, "do_rescale": False},
)
pixel_values_without_normalize = processed_inputs_without_normalize[
"mm_kwargs"
].get_data()["pixel_values"]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
input_norm = FusedInputNorm.from_model_config(ctx.model_config)
pixel_values_do_input_norm = input_norm(
pixel_values_without_normalize.to(dtype), dtype
)
torch.testing.assert_close(pixel_values_with_normalize, pixel_values_do_input_norm)