[MM] Add Triton kernel for mm_input_normal. (#56798)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Isotr0py
Isotr0py
parent
267eee54bf
commit
8a269919f4
@@ -0,0 +1,361 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Fused Normalisation on the Device.
|
||||
|
||||
Equivalent to::
|
||||
|
||||
output = (input * rescale_factor - image_mean) / image_std
|
||||
|
||||
This is implemented as a single per-channel affine transform::
|
||||
|
||||
output = input * weight[c] + bias[c]
|
||||
|
||||
where::
|
||||
|
||||
weight = rescale_factor / image_std
|
||||
bias = -image_mean / image_std
|
||||
"""
|
||||
|
||||
import math
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.transformers_utils.processor import cached_get_processor, get_processor_config
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_mm_input_norm_kernel(
|
||||
x_ptr,
|
||||
y_ptr,
|
||||
w_ptr,
|
||||
b_ptr,
|
||||
numel,
|
||||
L,
|
||||
C: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
# 1D grid over the flattened (N, C, L) tensor. Each program processes
|
||||
# BLOCK contiguous elements; the channel index is recovered from the
|
||||
# flat offset via `(offs // L) % C`. int32 offsets: int64 division is
|
||||
# ~2x slower here (measured on GB200); the wrapper asserts
|
||||
# numel < 2**31 so the flat index never overflows.
|
||||
pid = tl.program_id(0)
|
||||
offs = pid * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offs < numel
|
||||
c = (offs // L) % C
|
||||
|
||||
x = tl.load(
|
||||
x_ptr + offs,
|
||||
mask=mask,
|
||||
other=0,
|
||||
eviction_policy="evict_first",
|
||||
).to(tl.float32)
|
||||
w = tl.load(w_ptr + c, mask=mask, other=0).to(tl.float32)
|
||||
b = tl.load(b_ptr + c, mask=mask, other=0).to(tl.float32)
|
||||
tl.store(
|
||||
y_ptr + offs,
|
||||
x * w + b,
|
||||
mask=mask,
|
||||
eviction_policy="evict_first",
|
||||
)
|
||||
|
||||
|
||||
def fused_mm_input_norm_triton(
|
||||
inputs: torch.Tensor,
|
||||
outputs: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
):
|
||||
"""Fused per-channel affine transform for normalisation.
|
||||
|
||||
Args:
|
||||
inputs: Input tensor, shape ``(N, C, L)``. A contiguous copy is
|
||||
materialized internally if needed.
|
||||
outputs: Output tensor, contiguous and shaped exactly ``(N, C, L)``.
|
||||
weight: Per-channel scale, shape ``(C,)``.
|
||||
bias: Per-channel shift, shape ``(C,)``.
|
||||
|
||||
Returns:
|
||||
``outputs``, for chaining.
|
||||
|
||||
"""
|
||||
N, C, L = inputs.shape
|
||||
assert outputs.shape == (N, C, L), (
|
||||
f"outputs shape {tuple(outputs.shape)} != inputs shape {(N, C, L)}"
|
||||
)
|
||||
assert weight.numel() == C and bias.numel() == C, (
|
||||
f"weight/bias must have {C} elements, got {weight.numel()} / {bias.numel()}"
|
||||
)
|
||||
# The flat 1D kernel writes outputs by raw offset; a non-contiguous
|
||||
# buffer would be written at the wrong locations and cannot be
|
||||
# auto-materialized without breaking the caller's aliasing.
|
||||
assert outputs.is_contiguous(), "outputs must be contiguous"
|
||||
|
||||
# Tile size along the flattened element axis. 1024 with 4 warps gives
|
||||
# 8 elements per lane and benchmarks fastest across realistic image
|
||||
# sizes; larger tiles starve the SMs on small inputs.
|
||||
BLOCK_SIZE = 1024
|
||||
|
||||
numel = N * C * L
|
||||
# The kernel indexes with int32 offsets (int64 division costs ~2x
|
||||
# throughput); guard the flat index range here.
|
||||
assert numel < 2**31, f"numel={numel} exceeds the int32 index range"
|
||||
|
||||
grid = (triton.cdiv(numel, BLOCK_SIZE),)
|
||||
|
||||
_fused_mm_input_norm_kernel[grid](
|
||||
inputs.contiguous(),
|
||||
outputs,
|
||||
weight.contiguous(),
|
||||
bias.contiguous(),
|
||||
numel,
|
||||
L,
|
||||
C=C,
|
||||
BLOCK=BLOCK_SIZE,
|
||||
num_warps=4, # 8 elements per lane at BLOCK_SIZE=1024
|
||||
)
|
||||
return outputs
|
||||
|
||||
|
||||
class NormParams(NamedTuple):
|
||||
"""Resolved per-channel affine parameters (flags already folded in)."""
|
||||
|
||||
image_mean: list[float]
|
||||
image_std: list[float]
|
||||
rescale_factor: float
|
||||
|
||||
@property
|
||||
def is_identity(self) -> bool:
|
||||
"""Whether ``weight = rescale/std`` and ``bias = -mean/std`` are
|
||||
numerically the identity transform (mirrors ``torch.allclose``'s
|
||||
default fp32 tolerances)."""
|
||||
return all(
|
||||
math.isclose(self.rescale_factor / s, 1.0, rel_tol=1e-5, abs_tol=1e-8)
|
||||
and math.isclose(m / s, 0.0, abs_tol=1e-8)
|
||||
for m, s in zip(self.image_mean, self.image_std)
|
||||
)
|
||||
|
||||
|
||||
def _load_norm_params(model_config: ModelConfig) -> NormParams:
|
||||
"""Resolve the per-channel affine parameters ``(image_mean, image_std,
|
||||
rescale_factor)`` from the processor config, falling back to the image
|
||||
processor object."""
|
||||
model = model_config.model
|
||||
revision = model_config.revision
|
||||
|
||||
config = get_processor_config(model, revision=revision)
|
||||
# NOTE: do not use cached_image_processor_from_config here — it merges
|
||||
# mm_processor_kwargs, which mm_device_do_normalize poisons with
|
||||
# do_normalize=False.
|
||||
image_processor = cached_get_processor(
|
||||
model, revision=revision, trust_remote_code=model_config.trust_remote_code
|
||||
).image_processor
|
||||
|
||||
def resolve(key: str) -> Any:
|
||||
"""Processor config value, falling back to the image_processor."""
|
||||
if (value := config.get(key)) is not None:
|
||||
return value
|
||||
return getattr(image_processor, key, None)
|
||||
|
||||
do_rescale = bool(resolve("do_rescale"))
|
||||
do_normalize = bool(resolve("do_normalize"))
|
||||
|
||||
# Parameters whose flag is off are unused; default them to no-ops
|
||||
# without resolving them.
|
||||
rescale_factor = resolve("rescale_factor") if do_rescale else 1.0
|
||||
image_mean = resolve("image_mean") if do_normalize else [0.0] * 3
|
||||
image_std = resolve("image_std") if do_normalize else [1.0] * 3
|
||||
|
||||
assert rescale_factor is not None, "rescale_factor is still None after resolution."
|
||||
assert image_mean is not None, "image_mean is still None after resolution."
|
||||
assert image_std is not None, "image_std is still None after resolution."
|
||||
assert len(image_mean) == len(image_std), (
|
||||
f"image_mean and image_std have different lengths: "
|
||||
f"{len(image_mean)} vs {len(image_std)}"
|
||||
)
|
||||
|
||||
return NormParams(
|
||||
image_mean=[float(v) for v in image_mean],
|
||||
image_std=[float(v) for v in image_std],
|
||||
rescale_factor=float(rescale_factor),
|
||||
)
|
||||
|
||||
|
||||
class IdentityInputNorm(nn.Module):
|
||||
"""Stand-in used when the processor requires no rescale/normalise.
|
||||
|
||||
Not a no-op: with ``mm_device_do_normalize`` enabled, raw ``uint8``
|
||||
pixels travel to the device unprocessed and must be cast to
|
||||
``visual_dtype`` here.
|
||||
"""
|
||||
|
||||
def forward(
|
||||
self, pixel_values: torch.Tensor, visual_dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
return pixel_values.to(visual_dtype, copy=False)
|
||||
|
||||
|
||||
@CustomOp.register("fused_mm_input_norm")
|
||||
class FusedMMInputNorm(CustomOp):
|
||||
"""Module that applies rescaling and normalisation to input images.
|
||||
Equivalent to: output = (input * rescale_factor - mean) / std
|
||||
|
||||
Dtype semantics:
|
||||
|
||||
* Input dtype — the dtype of the ``pixel_values`` argument to ``forward_*``.
|
||||
It is ``uint8`` when ``mm_device_do_normalize`` is enabled (raw bytes
|
||||
travel to the device unprocessed) and equals ``visual_dtype``
|
||||
otherwise.
|
||||
* Output dtype — the ``visual_dtype`` argument of ``forward_*``. The
|
||||
computation itself is always fp32, independent of the output dtype
|
||||
(e.g. compute fp32, emit bf16).
|
||||
|
||||
Platform dispatch:
|
||||
|
||||
* ``forward_native`` — pure PyTorch eager path, used as the semantic
|
||||
reference and default fallback on all platforms.
|
||||
* ``forward_cuda`` — Triton kernel path for CUDA devices.
|
||||
* ``forward_xpu`` — custom XPU kernel path.
|
||||
* ``forward_oot`` — out-of-tree platform override entry point; falls back
|
||||
to ``forward_native`` unless a plugin overrides it.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_mean: list[float],
|
||||
image_std: list[float],
|
||||
rescale_factor: float,
|
||||
channel: int = 3,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
assert len(image_mean) == len(image_std) == channel, (
|
||||
f"image_mean/image_std must have {channel} entries, "
|
||||
f"got {len(image_mean)} / {len(image_std)}"
|
||||
)
|
||||
assert rescale_factor != 0.0, "rescale_factor must be non-zero"
|
||||
|
||||
self.channel = channel
|
||||
|
||||
# Model construction can set the accelerator as PyTorch's default
|
||||
# device; build the buffers on CPU first, then move them over.
|
||||
mean = torch.tensor(image_mean, dtype=torch.float32, device="cpu")
|
||||
std = torch.tensor(image_std, dtype=torch.float32, device="cpu")
|
||||
device = torch.get_default_device()
|
||||
self.register_buffer("weight", (rescale_factor / std).to(device))
|
||||
self.register_buffer("bias", (-mean / std).to(device))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal helpers shared by the platform-specific forward_* methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _unpack_2d(pixel_values: torch.Tensor) -> tuple[int, int]:
|
||||
assert pixel_values.ndim == 2, (
|
||||
f"pixel_values must be 2D (patches, size), got {pixel_values.dim()}D "
|
||||
f"with shape {tuple(pixel_values.shape)}"
|
||||
)
|
||||
patches, size = pixel_values.shape
|
||||
return patches, size
|
||||
|
||||
def _patch_size(self, size: int) -> int:
|
||||
assert size % self.channel == 0, (
|
||||
f"size={size} is not divisible by channel={self.channel}"
|
||||
)
|
||||
return size // self.channel
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Platform-specific implementations
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def forward_native(
|
||||
self, pixel_values: torch.Tensor, visual_dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
"""Pure PyTorch eager implementation.
|
||||
|
||||
This is the semantic reference implementation and the fallback used
|
||||
on any platform without a specialised kernel.
|
||||
"""
|
||||
patches, size = self._unpack_2d(pixel_values)
|
||||
patch_size = self._patch_size(size)
|
||||
|
||||
# weight/bias are fp32, so type promotion makes the arithmetic fp32
|
||||
x = pixel_values.reshape(patches, self.channel, patch_size)
|
||||
x = x * self.weight.view(1, self.channel, 1) + self.bias.view(
|
||||
1, self.channel, 1
|
||||
)
|
||||
return x.view(patches, size).to(visual_dtype)
|
||||
|
||||
def forward_cuda(
|
||||
self, pixel_values: torch.Tensor, visual_dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
"""Triton kernel path for CUDA devices."""
|
||||
patches, size = self._unpack_2d(pixel_values)
|
||||
patch_size = self._patch_size(size)
|
||||
|
||||
x3 = pixel_values.reshape(patches, self.channel, patch_size)
|
||||
y = torch.empty((patches, size), dtype=visual_dtype, device=pixel_values.device)
|
||||
y3 = y.view(patches, self.channel, patch_size)
|
||||
|
||||
fused_mm_input_norm_triton(x3, y3, self.weight, self.bias)
|
||||
return y
|
||||
|
||||
def forward_xpu(
|
||||
self, pixel_values: torch.Tensor, visual_dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
"""XPU fused custom kernel path.
|
||||
|
||||
On XPU, fuse the whole rescale + normalise into a single custom
|
||||
kernel. The eager path materializes an fp32 intermediate and then
|
||||
casts back, which adds device-side compute that cancels the
|
||||
bandwidth saving of transferring uint8 pixel_values. The fused
|
||||
kernel reads uint8 directly and writes ``visual_dtype`` in one pass.
|
||||
"""
|
||||
# The out-of-tree XPU kernel only supports the uint8 input that
|
||||
# device-side normalisation guarantees; fail loudly otherwise.
|
||||
assert pixel_values.dtype == torch.uint8, (
|
||||
f"xpu_fused_input_norm requires uint8 input, got {pixel_values.dtype}"
|
||||
)
|
||||
return torch.ops.vllm.xpu_fused_input_norm(
|
||||
pixel_values, self.weight, self.bias, visual_dtype
|
||||
)
|
||||
|
||||
def forward_oot(
|
||||
self, pixel_values: torch.Tensor, visual_dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
"""Out-of-tree platform override entrypoint."""
|
||||
return self.forward_native(pixel_values, visual_dtype)
|
||||
|
||||
|
||||
def build_mm_input_norm(model_config: ModelConfig) -> nn.Module:
|
||||
"""Build the input normalisation module for a model.
|
||||
|
||||
Returns an ``IdentityInputNorm`` when device-side normalisation is
|
||||
disabled or the processor's rescale/normalise is numerically the identity
|
||||
transform; otherwise a ``FusedMMInputNorm`` built from the processor's
|
||||
parameters.
|
||||
"""
|
||||
mm_config = getattr(model_config, "multimodal_config", None)
|
||||
if not getattr(mm_config, "mm_device_do_normalize", False):
|
||||
return IdentityInputNorm()
|
||||
|
||||
params = _load_norm_params(model_config)
|
||||
|
||||
# Flags off are already defaulted to identity values by
|
||||
# ``_load_norm_params``; this also catches configs that request
|
||||
# rescale/normalise but are numerically the identity transform.
|
||||
if params.is_identity:
|
||||
return IdentityInputNorm()
|
||||
|
||||
return FusedMMInputNorm(
|
||||
image_mean=params.image_mean,
|
||||
image_std=params.image_std,
|
||||
rescale_factor=params.rescale_factor,
|
||||
channel=len(params.image_mean),
|
||||
)
|
||||
@@ -55,6 +55,10 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import get_act_and_mul_fn
|
||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.conv import Conv3dLayer
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import (
|
||||
IdentityInputNorm,
|
||||
build_mm_input_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -120,7 +124,6 @@ from .utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from .vision import (
|
||||
FusedInputNorm,
|
||||
get_fp8_padded_hidden_size,
|
||||
get_vit_attn_backend,
|
||||
is_vit_use_data_parallel,
|
||||
@@ -664,9 +667,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
in_channels=in_channels,
|
||||
hidden_size=self.hidden_size,
|
||||
)
|
||||
self.input_norm = (
|
||||
input_norm if input_norm is not None else FusedInputNorm.identity()
|
||||
)
|
||||
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
|
||||
|
||||
norm_layer = partial(RMSNorm, eps=norm_eps)
|
||||
head_dim = self.hidden_size // self.num_heads
|
||||
@@ -1425,7 +1426,7 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
vision_config=config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=self.quant_config,
|
||||
input_norm=FusedInputNorm.from_model_config(self.model_config),
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
|
||||
@@ -51,6 +51,10 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import QuickGELU
|
||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.conv import Conv3dLayer
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import (
|
||||
IdentityInputNorm,
|
||||
build_mm_input_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -105,7 +109,6 @@ from .utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from .vision import (
|
||||
FusedInputNorm,
|
||||
get_vit_attn_backend,
|
||||
is_vit_use_data_parallel,
|
||||
run_dp_sharded_mrope_vision_model,
|
||||
@@ -562,9 +565,7 @@ class Qwen2VisionTransformer(nn.Module):
|
||||
in_channels=in_channels,
|
||||
embed_dim=embed_dim,
|
||||
)
|
||||
self.input_norm = (
|
||||
input_norm if input_norm is not None else FusedInputNorm.identity()
|
||||
)
|
||||
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
|
||||
|
||||
norm_layer = partial(nn.LayerNorm, eps=norm_eps)
|
||||
head_dim = embed_dim // num_heads
|
||||
@@ -1295,7 +1296,7 @@ class Qwen2VLForConditionalGeneration(
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=quant_config,
|
||||
input_norm=FusedInputNorm.from_model_config(self.model_config),
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
|
||||
@@ -8,10 +8,9 @@ from collections.abc import Callable
|
||||
from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import ModelConfig, MultiModalConfig, get_current_vllm_config_or_none
|
||||
from vllm.config import MultiModalConfig, get_current_vllm_config_or_none
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -19,7 +18,6 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.processor import get_processor, get_processor_config
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
@@ -578,165 +576,3 @@ def run_dp_sharded_mrope_vision_model(
|
||||
"Found unassigned embeddings"
|
||||
)
|
||||
return out_embeddings
|
||||
|
||||
|
||||
class FusedInputNorm(nn.Module):
|
||||
"""Module that applies rescaling and normalization to input images.
|
||||
Equivalent to: output = (input * rescale_factor - mean) / std
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_mean: list[float],
|
||||
image_std: list[float],
|
||||
rescale_factor: float,
|
||||
channel: int = 3,
|
||||
dtype: torch.dtype = torch.float32,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.channel = channel
|
||||
|
||||
# Model construction can set the accelerator as PyTorch's default
|
||||
# device. Determine whether the normalization is an identity on CPU
|
||||
# so torch.allclose does not introduce a device synchronization while
|
||||
# the model is being initialized. The actual buffers below still use
|
||||
# the caller's default device.
|
||||
image_mean_cpu = torch.tensor(image_mean, dtype=dtype, device="cpu") * (
|
||||
1.0 / rescale_factor
|
||||
)
|
||||
image_std_cpu = torch.tensor(image_std, dtype=dtype, device="cpu") * (
|
||||
1.0 / rescale_factor
|
||||
)
|
||||
weight_cpu = 1.0 / image_std_cpu
|
||||
bias_cpu = -image_mean_cpu / image_std_cpu
|
||||
self.is_identity = bool(
|
||||
torch.allclose(weight_cpu, torch.ones_like(weight_cpu))
|
||||
and torch.allclose(bias_cpu, torch.zeros_like(bias_cpu))
|
||||
)
|
||||
|
||||
if not self.is_identity:
|
||||
image_mean_tensor = torch.tensor(image_mean, dtype=dtype) * (
|
||||
1.0 / rescale_factor
|
||||
)
|
||||
image_std_tensor = torch.tensor(image_std, dtype=dtype) * (
|
||||
1.0 / rescale_factor
|
||||
)
|
||||
weight = 1.0 / image_std_tensor
|
||||
bias = -image_mean_tensor / image_std_tensor
|
||||
self.register_buffer("weight", weight)
|
||||
self.register_buffer("bias", bias)
|
||||
else:
|
||||
self.register_buffer("weight", None)
|
||||
self.register_buffer("bias", None)
|
||||
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
return self.weight.dtype
|
||||
|
||||
@classmethod
|
||||
def identity(
|
||||
cls, channel: int = 3, dtype: torch.dtype = torch.float32
|
||||
) -> "FusedInputNorm":
|
||||
return cls(
|
||||
image_mean=[0.0, 0.0, 0.0],
|
||||
image_std=[1.0, 1.0, 1.0],
|
||||
rescale_factor=1.0,
|
||||
channel=channel,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_model_config(cls, model_config: "ModelConfig") -> nn.Module:
|
||||
if not model_config.multimodal_config.mm_device_do_normalize:
|
||||
return cls.identity()
|
||||
|
||||
model = model_config.model
|
||||
revision = model_config.revision
|
||||
|
||||
# Try to read parameters from the processor config
|
||||
config = get_processor_config(model, revision=revision)
|
||||
do_rescale = config.get("do_rescale", None)
|
||||
do_normalize = config.get("do_normalize", None)
|
||||
image_mean = config.get("image_mean", None)
|
||||
image_std = config.get("image_std", None)
|
||||
rescale_factor = config.get("rescale_factor", None)
|
||||
|
||||
# Fallback to the image_processor object if any parameter is missing
|
||||
if None in [do_rescale, do_normalize, image_mean, image_std, rescale_factor]:
|
||||
image_processor = get_processor(model, revision=revision).image_processor
|
||||
|
||||
if do_rescale is None:
|
||||
do_rescale = getattr(image_processor, "do_rescale", None)
|
||||
if do_normalize is None:
|
||||
do_normalize = getattr(image_processor, "do_normalize", None)
|
||||
if image_mean is None:
|
||||
image_mean = getattr(image_processor, "image_mean", None)
|
||||
if image_std is None:
|
||||
image_std = getattr(image_processor, "image_std", None)
|
||||
if rescale_factor is None:
|
||||
rescale_factor = getattr(image_processor, "rescale_factor", None)
|
||||
|
||||
# Apply defaults based on flags
|
||||
if not do_rescale:
|
||||
rescale_factor = 1.0
|
||||
if not do_normalize:
|
||||
image_mean = [0.0, 0.0, 0.0]
|
||||
image_std = [1.0, 1.0, 1.0]
|
||||
|
||||
# Ensure all required parameters are resolved
|
||||
assert None not in [
|
||||
do_rescale,
|
||||
do_normalize,
|
||||
image_mean,
|
||||
image_std,
|
||||
rescale_factor,
|
||||
], "Some normalization parameters are still None after resolution."
|
||||
|
||||
# If no processing is needed, return an identity module
|
||||
if not do_rescale and not do_normalize:
|
||||
return cls.identity()
|
||||
|
||||
return cls(
|
||||
image_mean=image_mean, image_std=image_std, rescale_factor=rescale_factor
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
grid_thw: torch.Tensor,
|
||||
visual_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
if self.is_identity:
|
||||
return grid_thw.to(visual_dtype)
|
||||
|
||||
assert grid_thw.ndim == 2
|
||||
patches, size = grid_thw.shape
|
||||
patch_size = size // self.channel
|
||||
|
||||
# On XPU, fuse the whole rescale + normalize into a single custom
|
||||
# kernel. The eager path below materializes an fp32 intermediate and
|
||||
# then casts back, which adds device-side compute that cancels the
|
||||
# bandwidth saving of transferring uint8 pixel_values. The fused
|
||||
# kernel reads uint8 directly and writes ``visual_dtype`` in one pass.
|
||||
if (
|
||||
current_platform.is_xpu()
|
||||
and grid_thw.dtype == torch.uint8
|
||||
and self.weight.dtype == torch.float32
|
||||
):
|
||||
return torch.ops.vllm.xpu_fused_input_norm(
|
||||
grid_thw, self.weight, self.bias, visual_dtype
|
||||
)
|
||||
|
||||
# Apply the per-channel affine transform directly instead of via
|
||||
# F.batch_norm. batch_norm dispatches to cuDNN, whose batch-norm
|
||||
# kernels cap the batch dimension near the CUDA grid limit (~65535);
|
||||
# here that dimension is the number of patches, which grows unbounded
|
||||
# with image resolution and batch size and overflows the cap on large
|
||||
# image-heavy requests (CUDNN_STATUS_INTERNAL_ERROR). The plain
|
||||
# broadcasted multiply-add is numerically identical and has no such
|
||||
# limit.
|
||||
x = grid_thw.to(self.dtype).view(patches, self.channel, patch_size)
|
||||
x = x * self.weight.view(1, self.channel, 1) + self.bias.view(
|
||||
1, self.channel, 1
|
||||
)
|
||||
return x.view(patches, size).to(visual_dtype)
|
||||
|
||||
Reference in New Issue
Block a user