From cf1c90672404548aa3bc51f92c4745576a65ee26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20P=C3=A9rez=20de=20Algaba?= <124347725+jperezdealgaba@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:34:44 +0200 Subject: [PATCH] [Security] Fix image EXIF orientation and tRNS transparency handling (#44974) Signed-off-by: jperezde --- tests/multimodal/test_image.py | 87 +++++++++++++++++++++++++++++++++- vllm/multimodal/image.py | 36 +++++++++++--- vllm/multimodal/media/image.py | 7 ++- 3 files changed, 121 insertions(+), 9 deletions(-) diff --git a/tests/multimodal/test_image.py b/tests/multimodal/test_image.py index f8779d6f4b..ea7f18611c 100644 --- a/tests/multimodal/test_image.py +++ b/tests/multimodal/test_image.py @@ -1,12 +1,18 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import struct +from io import BytesIO from pathlib import Path import numpy as np import pytest from PIL import Image, ImageChops -from vllm.multimodal.image import convert_image_mode +from vllm.multimodal.image import ( + _has_transparency, + convert_image_mode, + normalize_image, +) pytestmark = pytest.mark.cpu_test @@ -38,3 +44,82 @@ def test_rgba_to_rgb(): assert converted_image_numpy[i][j][0] == 255 assert converted_image_numpy[i][j][1] == 255 assert converted_image_numpy[i][j][2] == 255 + + +def test_palette_with_trns_to_rgb(): + """P-mode PNG with tRNS: transparent index should become white.""" + # Built synthetically because the existing assets are RGBA PNGs; + # this vulnerability only affects P/L/RGB images carrying a tRNS + # chunk, which uses a different transparency mechanism. + img = Image.new("P", (4, 4)) + palette = [0] * 768 + palette[0:3] = [255, 0, 0] + palette[3:6] = [0, 0, 255] + img.putpalette(palette) + img.putpixel((0, 0), 0) + img.putpixel((1, 0), 1) + img.info["transparency"] = 1 + + assert _has_transparency(img) + converted = convert_image_mode(img, "RGB") + assert converted.mode == "RGB" + r, g, b = converted.getpixel((1, 0)) + assert (r, g, b) == (255, 255, 255) + r, g, b = converted.getpixel((0, 0)) + assert (r, g, b) == (255, 0, 0) + + +def test_l_mode_no_trns_to_rgb(): + """L-mode without transparency should convert directly.""" + img = Image.new("L", (4, 4), 128) + assert not _has_transparency(img) + converted = convert_image_mode(img, "RGB") + assert converted.mode == "RGB" + assert converted.getpixel((0, 0)) == (128, 128, 128) + + +def test_exif_transpose_normalizes_orientation(): + """Image with EXIF orientation 3 (180-degree rotation) should be + normalized so pixel data matches visual display.""" + # Built synthetically because the existing assets are PNGs which + # don't carry EXIF orientation metadata; we need a JPEG with an + # injected EXIF orientation=3 tag. + img = Image.new("RGB", (2, 1)) + img.putpixel((0, 0), (255, 0, 0)) + img.putpixel((1, 0), (0, 0, 255)) + + buf = BytesIO() + img.save(buf, format="JPEG") + jpeg_bytes = buf.getvalue() + + exif_orientation_3 = ( + b"\xff\xe1" + + struct.pack(">H", 26) + + b"Exif\x00\x00" + + b"MM" + + b"\x00\x2a" + + b"\x00\x00\x00\x08" + + b"\x00\x01" + + b"\x01\x12" + + b"\x00\x03" + + b"\x00\x00\x00\x01" + + b"\x00\x03" + + b"\x00\x00" + + b"\x00\x00\x00\x00" + ) + + soi = jpeg_bytes[:2] + rest = jpeg_bytes[2:] + patched = soi + exif_orientation_3 + rest + + rotated = Image.open(BytesIO(patched)) + normalized = normalize_image(rotated) + assert normalized.size == (2, 1) + + +def test_normalize_image_no_exif(): + """Images without EXIF should pass through unchanged.""" + img = Image.new("RGB", (4, 4), (100, 100, 100)) + result = normalize_image(img) + assert result.size == img.size + assert result.mode == img.mode diff --git a/vllm/multimodal/image.py b/vllm/multimodal/image.py index 1b0f70efcb..ff2da044a6 100644 --- a/vllm/multimodal/image.py +++ b/vllm/multimodal/image.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from PIL import Image +import contextlib + +from PIL import Image, ImageOps def rescale_image_size( @@ -16,6 +18,13 @@ def rescale_image_size( return image +def normalize_image(image: Image.Image) -> Image.Image: + """Normalize EXIF orientation so the pixel data matches visual display.""" + with contextlib.suppress(Exception): + image = ImageOps.exif_transpose(image) + return image + + def rgba_to_rgb( image: Image.Image, background_color: tuple[int, int, int] | list[int] = (255, 255, 255), @@ -27,10 +36,25 @@ def rgba_to_rgb( return converted -def convert_image_mode(image: Image.Image, to_mode: str): +def _has_transparency(image: Image.Image) -> bool: + """Detect whether an image carries transparency data (RGBA, LA, PA, + or tRNS chunk in P/L/RGB PNGs).""" + if image.mode in ("RGBA", "LA", "PA"): + return True + return "transparency" in getattr(image, "info", {}) + + +def convert_image_mode( + image: Image.Image, + to_mode: str, + background_color: tuple[int, int, int] | list[int] = (255, 255, 255), +) -> Image.Image: if image.mode == to_mode: return image - elif image.mode == "RGBA" and to_mode == "RGB": - return rgba_to_rgb(image) - else: - return image.convert(to_mode) + + if to_mode == "RGB" and _has_transparency(image): + if image.mode != "RGBA": + image = image.convert("RGBA") + return rgba_to_rgb(image, background_color) + + return image.convert(to_mode) diff --git a/vllm/multimodal/media/image.py b/vllm/multimodal/media/image.py index ea816b760f..c1a01d555b 100644 --- a/vllm/multimodal/media/image.py +++ b/vllm/multimodal/media/image.py @@ -11,7 +11,7 @@ from PIL import Image from vllm.utils.serial_utils import tensor2base64 -from ..image import convert_image_mode, rgba_to_rgb +from ..image import convert_image_mode, normalize_image, rgba_to_rgb from .base import MediaIO, MediaWithBytes MAGIC_NUMPY_PREFIX = b"\x93NUMPY" # https://numpy.org/devdocs/reference/generated/numpy.lib.format.html#format-version-1-0 @@ -65,11 +65,14 @@ class ImageMediaIO(MediaIO[Image.Image]): elif image.mode == "RGBA" and self.image_mode == "RGB": return rgba_to_rgb(image, self.rgba_background_color) else: - return convert_image_mode(image, self.image_mode) + return convert_image_mode( + image, self.image_mode, self.rgba_background_color + ) def load_bytes(self, data: bytes) -> MediaWithBytes[Image.Image]: try: image = Image.open(BytesIO(data)) + image = normalize_image(image) image.load() image = self._convert_image_mode(image) except (OSError, Image.UnidentifiedImageError) as e: