[MM] Add Triton kernel for mm_input_normal. (#56798)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: wang.yuqi <noooop@126.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <2037008807@qq.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
wang.yuqi
2026-09-25 04:31:20 +00:00
committed by GitHub
co-authored by Isotr0py Isotr0py
parent 267eee54bf
commit 8a269919f4
9 changed files with 749 additions and 304 deletions
@@ -8,10 +8,11 @@ import pytest
import torch
from PIL import Image
from vllm.model_executor.models.vision import FusedInputNorm
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.platforms import current_platform
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@@ -65,45 +66,6 @@ def test_jina_vl_processing_order() -> None:
]
@pytest.mark.parametrize(
("image_mean", "image_std", "rescale_factor", "is_identity"),
[
([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0, True),
([0.5, 0.5, 0.5], [0.25, 0.25, 0.25], 1 / 255, False),
],
)
def test_fused_input_norm_initialization_on_device(
monkeypatch: pytest.MonkeyPatch,
image_mean: list[float],
image_std: list[float],
rescale_factor: float,
is_identity: bool,
):
"""Identity detection must not synchronize the default device."""
original_allclose = torch.allclose
def cpu_allclose(input: torch.Tensor, other: torch.Tensor, *args, **kwargs):
assert input.device.type == "cpu"
assert other.device.type == "cpu"
return original_allclose(input, other, *args, **kwargs)
monkeypatch.setattr(torch, "allclose", cpu_allclose)
# Exercise the real accelerator when available. The meta device gives the
# CPU-only test shard the same non-CPU default-device semantics without
# requiring a CUDA-enabled PyTorch build.
default_device = "cuda" if torch.cuda.is_available() else "meta"
with torch.device(default_device):
input_norm = FusedInputNorm(image_mean, image_std, rescale_factor)
assert input_norm.is_identity is is_identity
if is_identity:
assert input_norm.weight is None
assert input_norm.bias is None
else:
assert input_norm.weight.device.type == default_device
assert input_norm.bias.device.type == default_device
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_toks_per_img", "expected_pixels_shape"),
@@ -240,16 +202,20 @@ def test_qwen2_5_vl_video_fps_to_second_per_grid_ts(model_id: str) -> None:
assert float(spg) == pytest.approx(temporal_patch_size / fps, rel=2e-2)
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize(
"model_id", ["Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]
)
@pytest.mark.parametrize("num_imgs", [1, 2])
def test_mm_device_do_normalize(
image_assets: ImageTestAssets,
model_id: str,
num_imgs: int,
):
"""Ensure that enable mm_device_do_normalize yields the correct result."""
image_assets: ImageTestAssets, model_id: str, num_imgs: int
) -> None:
"""Device-side normalisation must reproduce the on-CPU processor result.
Runs on any platform: the CPU platform exercises the ``forward_native``
fallback, accelerators exercise the fused kernel.
"""
device = current_platform.device_type
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": num_imgs},
@@ -280,9 +246,15 @@ def test_mm_device_do_normalize(
].get_data()["pixel_values"]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
input_norm = FusedInputNorm.from_model_config(ctx.model_config)
input_norm = build_mm_input_norm(ctx.model_config).to(device)
# With normalisation disabled, the processor emits raw uint8 pixels,
# matching the production mm_device_do_normalize path.
assert pixel_values_without_normalize.dtype == torch.uint8
pixel_values_do_input_norm = input_norm(
pixel_values_without_normalize.to(dtype), dtype
pixel_values_without_normalize.to(device), dtype
)
torch.testing.assert_close(pixel_values_with_normalize, pixel_values_do_input_norm)
torch.testing.assert_close(
pixel_values_with_normalize.to(device), pixel_values_do_input_norm
)