diff --git a/docs/design/mm_processing.md b/docs/design/mm_processing.md index 424a8593a8..b2b1ac307e 100644 --- a/docs/design/mm_processing.md +++ b/docs/design/mm_processing.md @@ -72,10 +72,14 @@ To accelerate the multi‑modal data pipeline (decoding, resizing, normalisation Traditionally, the CPU would divide pixel values by 255, then subtract the mean and divide by the standard deviation. We fuse these steps into one operation and run it entirely on the GPU. -- **How it works**: We use a dedicated `FusedInputNorm` module — essentially a frozen `BatchNorm1d` with the rescale factor baked into its statistics — and bake the rescale factor (typically 1/255) directly into the effective mean and standard deviation: - - Effective mean = `image_mean * (1/rescale_factor)` - - Effective std = `image_std * (1/rescale_factor)` -- **At runtime**: The `FusedInputNorm` layer takes raw `uint8` pixel values (0–255) and performs the full normalised mapping in a single GPU kernel—no CPU involvement. +**How it works**: We use a dedicated `FusedInputNorm` module that bakes the rescale factor (1/255) directly into the layer's `weight` and `bias` parameters. Instead of performing three separate steps (scale, subtract, divide), the module does everything in a single affine transformation: `y = x * weight + bias`. + +The parameters are set as follows: + +- `weight` controls both the standard deviation and the rescale factor +- `bias` centers the data using the mean and the same rescale factor + +This means the module takes raw pixel values (0–255) and outputs properly normalised values without ever explicitly dividing by 255 as a separate step. #### Optimized Data Path for Fused Normalisation diff --git a/tests/models/test_vision.py b/tests/models/test_vision.py index 7d03de1aba..987324eb86 100644 --- a/tests/models/test_vision.py +++ b/tests/models/test_vision.py @@ -13,6 +13,7 @@ from vllm.distributed.parallel_state import ( initialize_model_parallel, ) from vllm.model_executor.models.vision import ( + FusedInputNorm, get_load_balance_assignment, resolve_visual_encoder_outputs, run_dp_sharded_mrope_vision_model, @@ -492,3 +493,62 @@ def test_simple_mrope_vision_model_spatial_merge(spatial_merge_size: int): assert output.shape[0] == expected_output_patches assert output.shape[1] == vision_model.out_hidden_size + + +def _reference_input_norm( + pixel_values: torch.Tensor, + image_mean: list[float], + image_std: list[float], + rescale_factor: float, + channel: int, + out_dtype: torch.dtype, +) -> torch.Tensor: + """Straightforward per-channel affine: (x * rescale - mean) / std.""" + patches, size = pixel_values.shape + patch_size = size // channel + mean = torch.tensor(image_mean, dtype=torch.float32).view(1, channel, 1) + std = torch.tensor(image_std, dtype=torch.float32).view(1, channel, 1) + x = pixel_values.to(torch.float32).view(patches, channel, patch_size) + x = (x * rescale_factor - mean) / std + return x.view(patches, size).to(out_dtype) + + +@pytest.mark.parametrize("num_patches", [1, 37, 70000]) +def test_fused_input_norm_matches_reference(num_patches: int): + """FusedInputNorm must equal the plain affine, including for num_patches + above the cuDNN batch-norm grid limit (~65535) that previously raised + CUDNN_STATUS_INTERNAL_ERROR (issue #51717).""" + channel = 3 + patch_size = 14 * 14 + image_mean = [0.48145466, 0.4578275, 0.40821073] + image_std = [0.26862954, 0.26130258, 0.27577711] + rescale_factor = 1.0 / 255.0 + + set_random_seed(0) + pixel_values = torch.randint( + 0, 256, (num_patches, channel * patch_size), dtype=torch.float32 + ) + + norm = FusedInputNorm( + image_mean=image_mean, + image_std=image_std, + rescale_factor=rescale_factor, + channel=channel, + ) + assert not norm.is_identity + + out = norm(pixel_values, visual_dtype=torch.float32) + expected = _reference_input_norm( + pixel_values, image_mean, image_std, rescale_factor, channel, torch.float32 + ) + torch.testing.assert_close(out, expected) + + +def test_fused_input_norm_identity_passthrough(): + """The identity configuration returns the input unchanged (cast only).""" + norm = FusedInputNorm.identity() + assert norm.is_identity + + pixel_values = torch.randn(8, 3 * 196, dtype=torch.float32) + out = norm(pixel_values, visual_dtype=torch.bfloat16) + torch.testing.assert_close(out, pixel_values.to(torch.bfloat16)) diff --git a/vllm/model_executor/models/vision.py b/vllm/model_executor/models/vision.py index ff62b34ec7..0ccf410ed5 100644 --- a/vllm/model_executor/models/vision.py +++ b/vllm/model_executor/models/vision.py @@ -9,7 +9,6 @@ from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar import torch import torch.nn as nn -import torch.nn.functional as F from transformers import PretrainedConfig from vllm.config import ModelConfig, MultiModalConfig, get_current_vllm_config_or_none @@ -632,13 +631,9 @@ class FusedInputNorm(nn.Module): bias = -image_mean_tensor / image_std_tensor self.register_buffer("weight", weight) self.register_buffer("bias", bias) - self.register_buffer("running_mean", torch.zeros_like(image_mean_tensor)) - self.register_buffer("running_var", torch.ones_like(image_mean_tensor)) else: self.register_buffer("weight", None) self.register_buffer("bias", None) - self.register_buffer("running_mean", None) - self.register_buffer("running_var", None) @property def dtype(self) -> torch.dtype: @@ -723,14 +718,16 @@ class FusedInputNorm(nn.Module): patches, size = grid_thw.shape patch_size = size // self.channel - grid_thw = grid_thw.view(patches, self.channel, patch_size) - grid_thw = F.batch_norm( - grid_thw.to(self.dtype), - running_mean=self.running_mean, - running_var=self.running_var, - weight=self.weight, - bias=self.bias, - training=False, - eps=0.0, + # Apply the per-channel affine transform directly instead of via + # F.batch_norm. batch_norm dispatches to cuDNN, whose batch-norm + # kernels cap the batch dimension near the CUDA grid limit (~65535); + # here that dimension is the number of patches, which grows unbounded + # with image resolution and batch size and overflows the cap on large + # image-heavy requests (CUDNN_STATUS_INTERNAL_ERROR). The plain + # broadcasted multiply-add is numerically identical and has no such + # limit. + x = grid_thw.to(self.dtype).view(patches, self.channel, patch_size) + x = x * self.weight.view(1, self.channel, 1) + self.bias.view( + 1, self.channel, 1 ) - return grid_thw.view(patches, size).to(visual_dtype) + return x.view(patches, size).to(visual_dtype)