From 6e18901195fc3726546484fa86254217a747e9fe Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sat, 8 Aug 2026 00:32:23 -0500 Subject: [PATCH] [Bugfix][MM] Avoid device sync in FusedInputNorm initialization (#51435) Signed-off-by: Andreas Karatzas --- .../multimodal/processing/test_qwen2_vl.py | 39 +++++++++++++++++++ vllm/model_executor/models/vision.py | 28 +++++++++---- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/tests/models/multimodal/processing/test_qwen2_vl.py b/tests/models/multimodal/processing/test_qwen2_vl.py index ee4c133bc6..84f4288b09 100644 --- a/tests/models/multimodal/processing/test_qwen2_vl.py +++ b/tests/models/multimodal/processing/test_qwen2_vl.py @@ -13,6 +13,45 @@ from ....conftest import ImageTestAssets from ...utils import build_model_context +@pytest.mark.parametrize( + ("image_mean", "image_std", "rescale_factor", "is_identity"), + [ + ([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0, True), + ([0.5, 0.5, 0.5], [0.25, 0.25, 0.25], 1 / 255, False), + ], +) +def test_fused_input_norm_initialization_on_device( + monkeypatch: pytest.MonkeyPatch, + image_mean: list[float], + image_std: list[float], + rescale_factor: float, + is_identity: bool, +): + """Identity detection must not synchronize the default device.""" + original_allclose = torch.allclose + + def cpu_allclose(input: torch.Tensor, other: torch.Tensor, *args, **kwargs): + assert input.device.type == "cpu" + assert other.device.type == "cpu" + return original_allclose(input, other, *args, **kwargs) + + monkeypatch.setattr(torch, "allclose", cpu_allclose) + # Exercise the real accelerator when available. The meta device gives the + # CPU-only test shard the same non-CPU default-device semantics without + # requiring a CUDA-enabled PyTorch build. + default_device = "cuda" if torch.cuda.is_available() else "meta" + with torch.device(default_device): + input_norm = FusedInputNorm(image_mean, image_std, rescale_factor) + + assert input_norm.is_identity is is_identity + if is_identity: + assert input_norm.weight is None + assert input_norm.bias is None + else: + assert input_norm.weight.device.type == default_device + assert input_norm.bias.device.type == default_device + + @pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"]) @pytest.mark.parametrize( ("mm_processor_kwargs", "expected_toks_per_img", "expected_pixels_shape"), diff --git a/vllm/model_executor/models/vision.py b/vllm/model_executor/models/vision.py index 380faee423..ff62b34ec7 100644 --- a/vllm/model_executor/models/vision.py +++ b/vllm/model_executor/models/vision.py @@ -603,19 +603,33 @@ class FusedInputNorm(nn.Module): self.channel = channel - image_mean_tensor = torch.tensor(image_mean, dtype=dtype) * ( + # Model construction can set the accelerator as PyTorch's default + # device. Determine whether the normalization is an identity on CPU + # so torch.allclose does not introduce a device synchronization while + # the model is being initialized. The actual buffers below still use + # the caller's default device. + image_mean_cpu = torch.tensor(image_mean, dtype=dtype, device="cpu") * ( 1.0 / rescale_factor ) - image_std_tensor = torch.tensor(image_std, dtype=dtype) * (1.0 / rescale_factor) - weight = 1.0 / image_std_tensor - bias = -image_mean_tensor / image_std_tensor - + image_std_cpu = torch.tensor(image_std, dtype=dtype, device="cpu") * ( + 1.0 / rescale_factor + ) + weight_cpu = 1.0 / image_std_cpu + bias_cpu = -image_mean_cpu / image_std_cpu self.is_identity = bool( - torch.allclose(weight, torch.ones_like(weight)) - and torch.allclose(bias, torch.zeros_like(bias)) + torch.allclose(weight_cpu, torch.ones_like(weight_cpu)) + and torch.allclose(bias_cpu, torch.zeros_like(bias_cpu)) ) if not self.is_identity: + image_mean_tensor = torch.tensor(image_mean, dtype=dtype) * ( + 1.0 / rescale_factor + ) + image_std_tensor = torch.tensor(image_std, dtype=dtype) * ( + 1.0 / rescale_factor + ) + weight = 1.0 / image_std_tensor + bias = -image_mean_tensor / image_std_tensor self.register_buffer("weight", weight) self.register_buffer("bias", bias) self.register_buffer("running_mean", torch.zeros_like(image_mean_tensor))