[Security] Fix image EXIF orientation and tRNS transparency handling (#44974)
Signed-off-by: jperezde <jperezde@redhat.com>
This commit is contained in:
@@ -1,12 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import struct
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image, ImageChops
|
||||
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.multimodal.image import (
|
||||
_has_transparency,
|
||||
convert_image_mode,
|
||||
normalize_image,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -38,3 +44,82 @@ def test_rgba_to_rgb():
|
||||
assert converted_image_numpy[i][j][0] == 255
|
||||
assert converted_image_numpy[i][j][1] == 255
|
||||
assert converted_image_numpy[i][j][2] == 255
|
||||
|
||||
|
||||
def test_palette_with_trns_to_rgb():
|
||||
"""P-mode PNG with tRNS: transparent index should become white."""
|
||||
# Built synthetically because the existing assets are RGBA PNGs;
|
||||
# this vulnerability only affects P/L/RGB images carrying a tRNS
|
||||
# chunk, which uses a different transparency mechanism.
|
||||
img = Image.new("P", (4, 4))
|
||||
palette = [0] * 768
|
||||
palette[0:3] = [255, 0, 0]
|
||||
palette[3:6] = [0, 0, 255]
|
||||
img.putpalette(palette)
|
||||
img.putpixel((0, 0), 0)
|
||||
img.putpixel((1, 0), 1)
|
||||
img.info["transparency"] = 1
|
||||
|
||||
assert _has_transparency(img)
|
||||
converted = convert_image_mode(img, "RGB")
|
||||
assert converted.mode == "RGB"
|
||||
r, g, b = converted.getpixel((1, 0))
|
||||
assert (r, g, b) == (255, 255, 255)
|
||||
r, g, b = converted.getpixel((0, 0))
|
||||
assert (r, g, b) == (255, 0, 0)
|
||||
|
||||
|
||||
def test_l_mode_no_trns_to_rgb():
|
||||
"""L-mode without transparency should convert directly."""
|
||||
img = Image.new("L", (4, 4), 128)
|
||||
assert not _has_transparency(img)
|
||||
converted = convert_image_mode(img, "RGB")
|
||||
assert converted.mode == "RGB"
|
||||
assert converted.getpixel((0, 0)) == (128, 128, 128)
|
||||
|
||||
|
||||
def test_exif_transpose_normalizes_orientation():
|
||||
"""Image with EXIF orientation 3 (180-degree rotation) should be
|
||||
normalized so pixel data matches visual display."""
|
||||
# Built synthetically because the existing assets are PNGs which
|
||||
# don't carry EXIF orientation metadata; we need a JPEG with an
|
||||
# injected EXIF orientation=3 tag.
|
||||
img = Image.new("RGB", (2, 1))
|
||||
img.putpixel((0, 0), (255, 0, 0))
|
||||
img.putpixel((1, 0), (0, 0, 255))
|
||||
|
||||
buf = BytesIO()
|
||||
img.save(buf, format="JPEG")
|
||||
jpeg_bytes = buf.getvalue()
|
||||
|
||||
exif_orientation_3 = (
|
||||
b"\xff\xe1"
|
||||
+ struct.pack(">H", 26)
|
||||
+ b"Exif\x00\x00"
|
||||
+ b"MM"
|
||||
+ b"\x00\x2a"
|
||||
+ b"\x00\x00\x00\x08"
|
||||
+ b"\x00\x01"
|
||||
+ b"\x01\x12"
|
||||
+ b"\x00\x03"
|
||||
+ b"\x00\x00\x00\x01"
|
||||
+ b"\x00\x03"
|
||||
+ b"\x00\x00"
|
||||
+ b"\x00\x00\x00\x00"
|
||||
)
|
||||
|
||||
soi = jpeg_bytes[:2]
|
||||
rest = jpeg_bytes[2:]
|
||||
patched = soi + exif_orientation_3 + rest
|
||||
|
||||
rotated = Image.open(BytesIO(patched))
|
||||
normalized = normalize_image(rotated)
|
||||
assert normalized.size == (2, 1)
|
||||
|
||||
|
||||
def test_normalize_image_no_exif():
|
||||
"""Images without EXIF should pass through unchanged."""
|
||||
img = Image.new("RGB", (4, 4), (100, 100, 100))
|
||||
result = normalize_image(img)
|
||||
assert result.size == img.size
|
||||
assert result.mode == img.mode
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from PIL import Image
|
||||
import contextlib
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
|
||||
def rescale_image_size(
|
||||
@@ -16,6 +18,13 @@ def rescale_image_size(
|
||||
return image
|
||||
|
||||
|
||||
def normalize_image(image: Image.Image) -> Image.Image:
|
||||
"""Normalize EXIF orientation so the pixel data matches visual display."""
|
||||
with contextlib.suppress(Exception):
|
||||
image = ImageOps.exif_transpose(image)
|
||||
return image
|
||||
|
||||
|
||||
def rgba_to_rgb(
|
||||
image: Image.Image,
|
||||
background_color: tuple[int, int, int] | list[int] = (255, 255, 255),
|
||||
@@ -27,10 +36,25 @@ def rgba_to_rgb(
|
||||
return converted
|
||||
|
||||
|
||||
def convert_image_mode(image: Image.Image, to_mode: str):
|
||||
def _has_transparency(image: Image.Image) -> bool:
|
||||
"""Detect whether an image carries transparency data (RGBA, LA, PA,
|
||||
or tRNS chunk in P/L/RGB PNGs)."""
|
||||
if image.mode in ("RGBA", "LA", "PA"):
|
||||
return True
|
||||
return "transparency" in getattr(image, "info", {})
|
||||
|
||||
|
||||
def convert_image_mode(
|
||||
image: Image.Image,
|
||||
to_mode: str,
|
||||
background_color: tuple[int, int, int] | list[int] = (255, 255, 255),
|
||||
) -> Image.Image:
|
||||
if image.mode == to_mode:
|
||||
return image
|
||||
elif image.mode == "RGBA" and to_mode == "RGB":
|
||||
return rgba_to_rgb(image)
|
||||
else:
|
||||
return image.convert(to_mode)
|
||||
|
||||
if to_mode == "RGB" and _has_transparency(image):
|
||||
if image.mode != "RGBA":
|
||||
image = image.convert("RGBA")
|
||||
return rgba_to_rgb(image, background_color)
|
||||
|
||||
return image.convert(to_mode)
|
||||
|
||||
@@ -11,7 +11,7 @@ from PIL import Image
|
||||
|
||||
from vllm.utils.serial_utils import tensor2base64
|
||||
|
||||
from ..image import convert_image_mode, rgba_to_rgb
|
||||
from ..image import convert_image_mode, normalize_image, rgba_to_rgb
|
||||
from .base import MediaIO, MediaWithBytes
|
||||
|
||||
MAGIC_NUMPY_PREFIX = b"\x93NUMPY" # https://numpy.org/devdocs/reference/generated/numpy.lib.format.html#format-version-1-0
|
||||
@@ -65,11 +65,14 @@ class ImageMediaIO(MediaIO[Image.Image]):
|
||||
elif image.mode == "RGBA" and self.image_mode == "RGB":
|
||||
return rgba_to_rgb(image, self.rgba_background_color)
|
||||
else:
|
||||
return convert_image_mode(image, self.image_mode)
|
||||
return convert_image_mode(
|
||||
image, self.image_mode, self.rgba_background_color
|
||||
)
|
||||
|
||||
def load_bytes(self, data: bytes) -> MediaWithBytes[Image.Image]:
|
||||
try:
|
||||
image = Image.open(BytesIO(data))
|
||||
image = normalize_image(image)
|
||||
image.load()
|
||||
image = self._convert_image_mode(image)
|
||||
except (OSError, Image.UnidentifiedImageError) as e:
|
||||
|
||||
Reference in New Issue
Block a user