[MM] Add Triton kernel for mm_input_normal. (#56798)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: wang.yuqi <noooop@126.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <2037008807@qq.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
wang.yuqi
2026-09-25 04:31:20 +00:00
committed by GitHub
co-authored by Isotr0py Isotr0py
parent 267eee54bf
commit 8a269919f4
9 changed files with 749 additions and 304 deletions
+6 -5
View File
@@ -55,6 +55,10 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import get_act_and_mul_fn
from vllm.model_executor.layers.attention import MMEncoderAttention
from vllm.model_executor.layers.conv import Conv3dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import (
IdentityInputNorm,
build_mm_input_norm,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -120,7 +124,6 @@ from .utils import (
maybe_prefix,
)
from .vision import (
FusedInputNorm,
get_fp8_padded_hidden_size,
get_vit_attn_backend,
is_vit_use_data_parallel,
@@ -664,9 +667,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
in_channels=in_channels,
hidden_size=self.hidden_size,
)
self.input_norm = (
input_norm if input_norm is not None else FusedInputNorm.identity()
)
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
norm_layer = partial(RMSNorm, eps=norm_eps)
head_dim = self.hidden_size // self.num_heads
@@ -1425,7 +1426,7 @@ class Qwen2_5_VLForConditionalGeneration(
vision_config=config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
quant_config=self.quant_config,
input_norm=FusedInputNorm.from_model_config(self.model_config),
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
+6 -5
View File
@@ -51,6 +51,10 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import QuickGELU
from vllm.model_executor.layers.attention import MMEncoderAttention
from vllm.model_executor.layers.conv import Conv3dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import (
IdentityInputNorm,
build_mm_input_norm,
)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
RowParallelLinear,
@@ -105,7 +109,6 @@ from .utils import (
maybe_prefix,
)
from .vision import (
FusedInputNorm,
get_vit_attn_backend,
is_vit_use_data_parallel,
run_dp_sharded_mrope_vision_model,
@@ -562,9 +565,7 @@ class Qwen2VisionTransformer(nn.Module):
in_channels=in_channels,
embed_dim=embed_dim,
)
self.input_norm = (
input_norm if input_norm is not None else FusedInputNorm.identity()
)
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
norm_layer = partial(nn.LayerNorm, eps=norm_eps)
head_dim = embed_dim // num_heads
@@ -1295,7 +1296,7 @@ class Qwen2VLForConditionalGeneration(
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
quant_config=quant_config,
input_norm=FusedInputNorm.from_model_config(self.model_config),
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
+1 -165
View File
@@ -8,10 +8,9 @@ from collections.abc import Callable
from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar
import torch
import torch.nn as nn
from transformers import PreTrainedConfig
from vllm.config import ModelConfig, MultiModalConfig, get_current_vllm_config_or_none
from vllm.config import MultiModalConfig, get_current_vllm_config_or_none
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -19,7 +18,6 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.transformers_utils.processor import get_processor, get_processor_config
from vllm.utils.math_utils import round_up
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -578,165 +576,3 @@ def run_dp_sharded_mrope_vision_model(
"Found unassigned embeddings"
)
return out_embeddings
class FusedInputNorm(nn.Module):
"""Module that applies rescaling and normalization to input images.
Equivalent to: output = (input * rescale_factor - mean) / std
"""
def __init__(
self,
image_mean: list[float],
image_std: list[float],
rescale_factor: float,
channel: int = 3,
dtype: torch.dtype = torch.float32,
):
super().__init__()
self.channel = channel
# Model construction can set the accelerator as PyTorch's default
# device. Determine whether the normalization is an identity on CPU
# so torch.allclose does not introduce a device synchronization while
# the model is being initialized. The actual buffers below still use
# the caller's default device.
image_mean_cpu = torch.tensor(image_mean, dtype=dtype, device="cpu") * (
1.0 / rescale_factor
)
image_std_cpu = torch.tensor(image_std, dtype=dtype, device="cpu") * (
1.0 / rescale_factor
)
weight_cpu = 1.0 / image_std_cpu
bias_cpu = -image_mean_cpu / image_std_cpu
self.is_identity = bool(
torch.allclose(weight_cpu, torch.ones_like(weight_cpu))
and torch.allclose(bias_cpu, torch.zeros_like(bias_cpu))
)
if not self.is_identity:
image_mean_tensor = torch.tensor(image_mean, dtype=dtype) * (
1.0 / rescale_factor
)
image_std_tensor = torch.tensor(image_std, dtype=dtype) * (
1.0 / rescale_factor
)
weight = 1.0 / image_std_tensor
bias = -image_mean_tensor / image_std_tensor
self.register_buffer("weight", weight)
self.register_buffer("bias", bias)
else:
self.register_buffer("weight", None)
self.register_buffer("bias", None)
@property
def dtype(self) -> torch.dtype:
return self.weight.dtype
@classmethod
def identity(
cls, channel: int = 3, dtype: torch.dtype = torch.float32
) -> "FusedInputNorm":
return cls(
image_mean=[0.0, 0.0, 0.0],
image_std=[1.0, 1.0, 1.0],
rescale_factor=1.0,
channel=channel,
dtype=dtype,
)
@classmethod
def from_model_config(cls, model_config: "ModelConfig") -> nn.Module:
if not model_config.multimodal_config.mm_device_do_normalize:
return cls.identity()
model = model_config.model
revision = model_config.revision
# Try to read parameters from the processor config
config = get_processor_config(model, revision=revision)
do_rescale = config.get("do_rescale", None)
do_normalize = config.get("do_normalize", None)
image_mean = config.get("image_mean", None)
image_std = config.get("image_std", None)
rescale_factor = config.get("rescale_factor", None)
# Fallback to the image_processor object if any parameter is missing
if None in [do_rescale, do_normalize, image_mean, image_std, rescale_factor]:
image_processor = get_processor(model, revision=revision).image_processor
if do_rescale is None:
do_rescale = getattr(image_processor, "do_rescale", None)
if do_normalize is None:
do_normalize = getattr(image_processor, "do_normalize", None)
if image_mean is None:
image_mean = getattr(image_processor, "image_mean", None)
if image_std is None:
image_std = getattr(image_processor, "image_std", None)
if rescale_factor is None:
rescale_factor = getattr(image_processor, "rescale_factor", None)
# Apply defaults based on flags
if not do_rescale:
rescale_factor = 1.0
if not do_normalize:
image_mean = [0.0, 0.0, 0.0]
image_std = [1.0, 1.0, 1.0]
# Ensure all required parameters are resolved
assert None not in [
do_rescale,
do_normalize,
image_mean,
image_std,
rescale_factor,
], "Some normalization parameters are still None after resolution."
# If no processing is needed, return an identity module
if not do_rescale and not do_normalize:
return cls.identity()
return cls(
image_mean=image_mean, image_std=image_std, rescale_factor=rescale_factor
)
def forward(
self,
grid_thw: torch.Tensor,
visual_dtype: torch.dtype,
) -> torch.Tensor:
if self.is_identity:
return grid_thw.to(visual_dtype)
assert grid_thw.ndim == 2
patches, size = grid_thw.shape
patch_size = size // self.channel
# On XPU, fuse the whole rescale + normalize into a single custom
# kernel. The eager path below materializes an fp32 intermediate and
# then casts back, which adds device-side compute that cancels the
# bandwidth saving of transferring uint8 pixel_values. The fused
# kernel reads uint8 directly and writes ``visual_dtype`` in one pass.
if (
current_platform.is_xpu()
and grid_thw.dtype == torch.uint8
and self.weight.dtype == torch.float32
):
return torch.ops.vllm.xpu_fused_input_norm(
grid_thw, self.weight, self.bias, visual_dtype
)
# Apply the per-channel affine transform directly instead of via
# F.batch_norm. batch_norm dispatches to cuDNN, whose batch-norm
# kernels cap the batch dimension near the CUDA grid limit (~65535);
# here that dimension is the number of patches, which grows unbounded
# with image resolution and batch size and overflows the cap on large
# image-heavy requests (CUDNN_STATUS_INTERNAL_ERROR). The plain
# broadcasted multiply-add is numerically identical and has no such
# limit.
x = grid_thw.to(self.dtype).view(patches, self.channel, patch_size)
x = x * self.weight.view(1, self.channel, 1) + self.bias.view(
1, self.channel, 1
)
return x.view(patches, size).to(visual_dtype)