[Model] Fused mm preprocess normalisation on the Device (#50411)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: wang.yuqi <noooop@126.com>
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
wang.yuqi
2026-08-06 03:11:14 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent 9c22668436
commit 2fa490470d
13 changed files with 359 additions and 11 deletions
+6
View File
@@ -149,6 +149,12 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
`multimodal_config.mm_encoder_tp_mode="data"`.
"""
supports_mm_device_do_normalize: ClassVar[bool] = False
"""
A flag that indicates whether this model supports
`multimodal_config.mm_device_do_normalize`.
"""
requires_raw_input_tokens: ClassVar[bool] = False
"""
A flag that indicates this model processes input id tokens
+9
View File
@@ -112,6 +112,7 @@ from .utils import (
maybe_prefix,
)
from .vision import (
FusedInputNorm,
get_fp8_padded_hidden_size,
get_vit_attn_backend,
is_vit_use_data_parallel,
@@ -1254,6 +1255,7 @@ class Qwen2_5_VLForConditionalGeneration(
)
supports_encoder_tp_data = True
supports_mm_device_do_normalize = True
def iter_mm_grid_thw(
self, mm_features: list[MultiModalFeatureSpec]
@@ -1363,6 +1365,7 @@ class Qwen2_5_VLForConditionalGeneration(
quant_config=self.quant_config,
prefix=maybe_prefix(prefix, "visual"),
)
self.input_norm = FusedInputNorm.from_model_config(self.model_config)
with self._mark_language_model(vllm_config):
self.language_model = init_vllm_registered_model(
@@ -1437,6 +1440,8 @@ class Qwen2_5_VLForConditionalGeneration(
image_embeds = image_input["image_embeds"].type(self.visual.dtype)
else:
pixel_values = image_input["pixel_values"]
pixel_values = self.input_norm(pixel_values, self.visual.dtype)
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
self.visual, pixel_values, grid_thw_list, rope_type="rope_3d"
@@ -1493,6 +1498,10 @@ class Qwen2_5_VLForConditionalGeneration(
video_embeds = video_input["video_embeds"].type(self.visual.dtype)
else:
pixel_values_videos = video_input["pixel_values_videos"]
pixel_values_videos = self.input_norm(
pixel_values_videos, self.visual.dtype
)
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
self.visual,
+9 -2
View File
@@ -103,6 +103,7 @@ from .utils import (
maybe_prefix,
)
from .vision import (
FusedInputNorm,
get_vit_attn_backend,
is_vit_use_data_parallel,
run_dp_sharded_mrope_vision_model,
@@ -1192,6 +1193,7 @@ class Qwen2VLForConditionalGeneration(
)
supports_encoder_tp_data = True
supports_mm_device_do_normalize = True
def iter_mm_grid_thw(
self, mm_features: list[MultiModalFeatureSpec]
@@ -1296,6 +1298,7 @@ class Qwen2VLForConditionalGeneration(
quant_config=quant_config,
prefix=maybe_prefix(prefix, "visual"),
)
self.input_norm = FusedInputNorm.from_model_config(self.model_config)
with self._mark_language_model(vllm_config):
self.language_model = init_vllm_registered_model(
@@ -1363,9 +1366,10 @@ class Qwen2VLForConditionalGeneration(
assert grid_thw.ndim == 2
if image_input["type"] == "image_embeds":
image_embeds = image_input["image_embeds"]
image_embeds = image_input["image_embeds"].type(self.visual.dtype)
else:
pixel_values = image_input["pixel_values"]
pixel_values = self.input_norm(pixel_values, self.visual.dtype)
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
@@ -1386,9 +1390,12 @@ class Qwen2VLForConditionalGeneration(
assert grid_thw.ndim == 2
if video_input["type"] == "video_embeds":
video_embeds = video_input["video_embeds"]
video_embeds = video_input["video_embeds"].type(self.visual.dtype)
else:
pixel_values_videos = video_input["pixel_values_videos"]
pixel_values_videos = self.input_norm(
pixel_values_videos, self.visual.dtype
)
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
self.visual,
+4
View File
@@ -814,6 +814,7 @@ class _ModelInfo:
supports_transcription: bool
supports_transcription_only: bool
supported_video_pruning_methods: tuple[str, ...]
supports_mm_device_do_normalize: bool
@staticmethod
def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
@@ -847,6 +848,9 @@ class _ModelInfo:
supported_video_pruning_methods=getattr(
model, "supported_video_pruning_methods", ()
),
supports_mm_device_do_normalize=getattr(
model, "supports_mm_device_do_normalize", False
),
)
+141 -1
View File
@@ -8,9 +8,11 @@ from collections.abc import Callable
from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import PretrainedConfig
from vllm.config import MultiModalConfig, get_current_vllm_config_or_none
from vllm.config import ModelConfig, MultiModalConfig, get_current_vllm_config_or_none
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -18,6 +20,7 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.transformers_utils.processor import get_processor, get_processor_config
from vllm.utils.math_utils import round_up
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -580,3 +583,140 @@ def run_dp_sharded_mrope_vision_model(
"Found unassigned embeddings"
)
return out_embeddings
class FusedInputNorm(nn.Module):
"""
Module that applies rescaling and normalization to input images.
Equivalent to: output = (input * rescale_factor - mean) / std
"""
def __init__(
self,
image_mean: list[float],
image_std: list[float],
rescale_factor: float,
channel: int = 3,
dtype: torch.dtype = torch.float32,
):
super().__init__()
self.channel = channel
image_mean_tensor = torch.tensor(image_mean, dtype=dtype) * (
1.0 / rescale_factor
)
image_std_tensor = torch.tensor(image_std, dtype=dtype) * (1.0 / rescale_factor)
weight = 1.0 / image_std_tensor
bias = -image_mean_tensor / image_std_tensor
self.is_identity = bool(
torch.allclose(weight, torch.ones_like(weight))
and torch.allclose(bias, torch.zeros_like(bias))
)
if not self.is_identity:
self.register_buffer("weight", weight)
self.register_buffer("bias", bias)
self.register_buffer("running_mean", torch.zeros_like(image_mean_tensor))
self.register_buffer("running_var", torch.ones_like(image_mean_tensor))
else:
self.register_buffer("weight", None)
self.register_buffer("bias", None)
self.register_buffer("running_mean", None)
self.register_buffer("running_var", None)
@property
def dtype(self) -> torch.dtype:
return self.weight.dtype
@classmethod
def identity(
cls, channel: int = 3, dtype: torch.dtype = torch.float32
) -> "FusedInputNorm":
return cls(
image_mean=[0.0, 0.0, 0.0],
image_std=[1.0, 1.0, 1.0],
rescale_factor=1.0,
channel=channel,
dtype=dtype,
)
@classmethod
def from_model_config(cls, model_config: "ModelConfig") -> nn.Module:
if not model_config.multimodal_config.mm_device_do_normalize:
return cls.identity()
model = model_config.model
revision = model_config.revision
# Try to read parameters from the processor config
config = get_processor_config(model, revision=revision)
do_rescale = config.get("do_rescale", None)
do_normalize = config.get("do_normalize", None)
image_mean = config.get("image_mean", None)
image_std = config.get("image_std", None)
rescale_factor = config.get("rescale_factor", None)
# Fallback to the image_processor object if any parameter is missing
if None in [do_rescale, do_normalize, image_mean, image_std, rescale_factor]:
image_processor = get_processor(model, revision=revision).image_processor
if do_rescale is None:
do_rescale = getattr(image_processor, "do_rescale", None)
if do_normalize is None:
do_normalize = getattr(image_processor, "do_normalize", None)
if image_mean is None:
image_mean = getattr(image_processor, "image_mean", None)
if image_std is None:
image_std = getattr(image_processor, "image_std", None)
if rescale_factor is None:
rescale_factor = getattr(image_processor, "rescale_factor", None)
# Apply defaults based on flags
if not do_rescale:
rescale_factor = 1.0
if not do_normalize:
image_mean = [0.0, 0.0, 0.0]
image_std = [1.0, 1.0, 1.0]
# Ensure all required parameters are resolved
assert None not in [
do_rescale,
do_normalize,
image_mean,
image_std,
rescale_factor,
], "Some normalization parameters are still None after resolution."
# If no processing is needed, return an identity module
if not do_rescale and not do_normalize:
return cls.identity()
return cls(
image_mean=image_mean, image_std=image_std, rescale_factor=rescale_factor
)
def forward(
self,
grid_thw: torch.Tensor,
visual_dtype: torch.dtype,
) -> torch.Tensor:
if self.is_identity:
return grid_thw.to(visual_dtype)
assert grid_thw.ndim == 2
patches, size = grid_thw.shape
patch_size = size // self.channel
grid_thw = grid_thw.view(patches, self.channel, patch_size)
grid_thw = F.batch_norm(
grid_thw.to(self.dtype),
running_mean=self.running_mean,
running_var=self.running_var,
weight=self.weight,
bias=self.bias,
training=False,
eps=0.0,
)
return grid_thw.view(patches, size).to(visual_dtype)