[MM] Add Triton kernel for mm_input_normal. (#56798)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Isotr0py
Isotr0py
parent
267eee54bf
commit
8a269919f4
@@ -8,10 +8,11 @@ import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm.model_executor.models.vision import FusedInputNorm
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import batched_tensors_equal
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
@@ -65,45 +66,6 @@ def test_jina_vl_processing_order() -> None:
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("image_mean", "image_std", "rescale_factor", "is_identity"),
|
||||
[
|
||||
([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0, True),
|
||||
([0.5, 0.5, 0.5], [0.25, 0.25, 0.25], 1 / 255, False),
|
||||
],
|
||||
)
|
||||
def test_fused_input_norm_initialization_on_device(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
image_mean: list[float],
|
||||
image_std: list[float],
|
||||
rescale_factor: float,
|
||||
is_identity: bool,
|
||||
):
|
||||
"""Identity detection must not synchronize the default device."""
|
||||
original_allclose = torch.allclose
|
||||
|
||||
def cpu_allclose(input: torch.Tensor, other: torch.Tensor, *args, **kwargs):
|
||||
assert input.device.type == "cpu"
|
||||
assert other.device.type == "cpu"
|
||||
return original_allclose(input, other, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(torch, "allclose", cpu_allclose)
|
||||
# Exercise the real accelerator when available. The meta device gives the
|
||||
# CPU-only test shard the same non-CPU default-device semantics without
|
||||
# requiring a CUDA-enabled PyTorch build.
|
||||
default_device = "cuda" if torch.cuda.is_available() else "meta"
|
||||
with torch.device(default_device):
|
||||
input_norm = FusedInputNorm(image_mean, image_std, rescale_factor)
|
||||
|
||||
assert input_norm.is_identity is is_identity
|
||||
if is_identity:
|
||||
assert input_norm.weight is None
|
||||
assert input_norm.bias is None
|
||||
else:
|
||||
assert input_norm.weight.device.type == default_device
|
||||
assert input_norm.bias.device.type == default_device
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_toks_per_img", "expected_pixels_shape"),
|
||||
@@ -240,16 +202,20 @@ def test_qwen2_5_vl_video_fps_to_second_per_grid_ts(model_id: str) -> None:
|
||||
assert float(spg) == pytest.approx(temporal_patch_size / fps, rel=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize(
|
||||
"model_id", ["Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]
|
||||
)
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
def test_mm_device_do_normalize(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
num_imgs: int,
|
||||
):
|
||||
"""Ensure that enable mm_device_do_normalize yields the correct result."""
|
||||
image_assets: ImageTestAssets, model_id: str, num_imgs: int
|
||||
) -> None:
|
||||
"""Device-side normalisation must reproduce the on-CPU processor result.
|
||||
|
||||
Runs on any platform: the CPU platform exercises the ``forward_native``
|
||||
fallback, accelerators exercise the fused kernel.
|
||||
"""
|
||||
device = current_platform.device_type
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
@@ -280,9 +246,15 @@ def test_mm_device_do_normalize(
|
||||
].get_data()["pixel_values"]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
input_norm = FusedInputNorm.from_model_config(ctx.model_config)
|
||||
input_norm = build_mm_input_norm(ctx.model_config).to(device)
|
||||
|
||||
# With normalisation disabled, the processor emits raw uint8 pixels,
|
||||
# matching the production mm_device_do_normalize path.
|
||||
assert pixel_values_without_normalize.dtype == torch.uint8
|
||||
pixel_values_do_input_norm = input_norm(
|
||||
pixel_values_without_normalize.to(dtype), dtype
|
||||
pixel_values_without_normalize.to(device), dtype
|
||||
)
|
||||
|
||||
torch.testing.assert_close(pixel_values_with_normalize, pixel_values_do_input_norm)
|
||||
torch.testing.assert_close(
|
||||
pixel_values_with_normalize.to(device), pixel_values_do_input_norm
|
||||
)
|
||||
|
||||
@@ -13,7 +13,6 @@ from vllm.distributed.parallel_state import (
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.models.vision import (
|
||||
FusedInputNorm,
|
||||
get_load_balance_assignment,
|
||||
resolve_visual_encoder_outputs,
|
||||
run_dp_sharded_mrope_vision_model,
|
||||
@@ -488,62 +487,3 @@ def test_simple_mrope_vision_model_spatial_merge(spatial_merge_size: int):
|
||||
|
||||
assert output.shape[0] == expected_output_patches
|
||||
assert output.shape[1] == vision_model.out_hidden_size
|
||||
|
||||
|
||||
def _reference_input_norm(
|
||||
pixel_values: torch.Tensor,
|
||||
image_mean: list[float],
|
||||
image_std: list[float],
|
||||
rescale_factor: float,
|
||||
channel: int,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Straightforward per-channel affine: (x * rescale - mean) / std."""
|
||||
patches, size = pixel_values.shape
|
||||
patch_size = size // channel
|
||||
mean = torch.tensor(image_mean, dtype=torch.float32).view(1, channel, 1)
|
||||
std = torch.tensor(image_std, dtype=torch.float32).view(1, channel, 1)
|
||||
x = pixel_values.to(torch.float32).view(patches, channel, patch_size)
|
||||
x = (x * rescale_factor - mean) / std
|
||||
return x.view(patches, size).to(out_dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_patches", [1, 37, 70000])
|
||||
def test_fused_input_norm_matches_reference(num_patches: int):
|
||||
"""FusedInputNorm must equal the plain affine, including for num_patches
|
||||
above the cuDNN batch-norm grid limit (~65535) that previously raised
|
||||
CUDNN_STATUS_INTERNAL_ERROR (issue #51717)."""
|
||||
channel = 3
|
||||
patch_size = 14 * 14
|
||||
image_mean = [0.48145466, 0.4578275, 0.40821073]
|
||||
image_std = [0.26862954, 0.26130258, 0.27577711]
|
||||
rescale_factor = 1.0 / 255.0
|
||||
|
||||
set_random_seed(0)
|
||||
pixel_values = torch.randint(
|
||||
0, 256, (num_patches, channel * patch_size), dtype=torch.float32
|
||||
)
|
||||
|
||||
norm = FusedInputNorm(
|
||||
image_mean=image_mean,
|
||||
image_std=image_std,
|
||||
rescale_factor=rescale_factor,
|
||||
channel=channel,
|
||||
)
|
||||
assert not norm.is_identity
|
||||
|
||||
out = norm(pixel_values, visual_dtype=torch.float32)
|
||||
expected = _reference_input_norm(
|
||||
pixel_values, image_mean, image_std, rescale_factor, channel, torch.float32
|
||||
)
|
||||
torch.testing.assert_close(out, expected)
|
||||
|
||||
|
||||
def test_fused_input_norm_identity_passthrough():
|
||||
"""The identity configuration returns the input unchanged (cast only)."""
|
||||
norm = FusedInputNorm.identity()
|
||||
assert norm.is_identity
|
||||
|
||||
pixel_values = torch.randn(8, 3 * 196, dtype=torch.float32)
|
||||
out = norm(pixel_values, visual_dtype=torch.bfloat16)
|
||||
torch.testing.assert_close(out, pixel_values.to(torch.bfloat16))
|
||||
|
||||
Reference in New Issue
Block a user