diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index cb091c1c79..e4cd5f56a2 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -187,7 +187,7 @@ Multi-image input can be extended to perform video captioning. We show this with #### Custom RGBA Background Color -When loading RGBA images (images with transparency), vLLM converts them to RGB format. By default, transparent pixels are replaced with white background. You can customize this background color using the `rgba_background_color` parameter in `media_io_kwargs`. +When loading images with transparency (including RGBA images and PNGs with a tRNS chunk), vLLM converts them to RGB format. By default, transparent pixels are replaced with white background. You can customize this background color using the `rgba_background_color` parameter in `media_io_kwargs`. ??? code @@ -212,7 +212,7 @@ When loading RGBA images (images with transparency), vLLM converts them to RGB f !!! note - The `rgba_background_color` accepts RGB values as a list `[R, G, B]` or tuple `(R, G, B)` where each value is 0-255 - - This setting only affects RGBA images with transparency; RGB images are unchanged + - This setting affects images with transparency, including RGB PNGs with a transparent color; opaque RGB images are unchanged - If not specified, the default white background `(255, 255, 255)` is used for backward compatibility #### Moondream3 Prompt Recipes { #moondream3-prompt-recipes } @@ -1089,7 +1089,7 @@ By passing this metadata, the model can better understand the temporal distribut #### Custom RGBA Background Color -To use a custom background color for RGBA images, pass the `rgba_background_color` parameter via `--media-io-kwargs`: +To use a custom background color for images with transparency, pass the `rgba_background_color` parameter via `--media-io-kwargs`: ```bash # Example: Black background for dark theme diff --git a/tests/multimodal/media/test_image.py b/tests/multimodal/media/test_image.py index 60cfb86cc0..1980adf197 100644 --- a/tests/multimodal/media/test_image.py +++ b/tests/multimodal/media/test_image.py @@ -85,6 +85,24 @@ def test_image_media_io_rgba_custom_background(tmp_path): assert green_numpy[0][0][2] == 0 # B +@pytest.mark.parametrize("background_color", [(255, 255, 255), (12, 34, 56)]) +def test_image_media_io_rgb_png_transparency(tmp_path, background_color): + image = Image.new("RGB", (2, 1), (255, 0, 0)) + image.putpixel((1, 0), (0, 0, 255)) + path = tmp_path / "rgb_transparency.png" + image.save(path, transparency=(255, 0, 0)) + + with Image.open(path) as source: + reference = Image.alpha_composite( + Image.new("RGBA", source.size, (*background_color, 255)), + source.convert("RGBA"), + ).convert("RGB") + + image_io = ImageMediaIO(rgba_background_color=background_color) + converted = image_io.load_file(path) + np.testing.assert_array_equal(np.asarray(converted), np.asarray(reference)) + + def test_image_media_io_no_mode_conversion(tmp_path): """image_mode=None skips conversion and preserves the original mode.""" # RGBA image: opaque black pixel on a fully transparent background diff --git a/tests/multimodal/test_hasher.py b/tests/multimodal/test_hasher.py index b8d9dc7c9d..bdc1e98421 100644 --- a/tests/multimodal/test_hasher.py +++ b/tests/multimodal/test_hasher.py @@ -237,8 +237,16 @@ def _rgba_png_bytes() -> bytes: return buf.getvalue() -def test_hash_collision_media_io_config(): - data = _rgba_png_bytes() +def _rgb_trns_png_bytes() -> bytes: + image = Image.new("RGB", (8, 8), (255, 0, 0)) + buf = BytesIO() + image.save(buf, format="PNG", transparency=(255, 0, 0)) + return buf.getvalue() + + +@pytest.mark.parametrize("png_bytes", [_rgba_png_bytes, _rgb_trns_png_bytes]) +def test_hash_collision_media_io_config(png_bytes): + data = png_bytes() white = ImageMediaIO(rgba_background_color=(255, 255, 255)).load_bytes(data) black = ImageMediaIO(rgba_background_color=(0, 0, 0)).load_bytes(data) white2 = ImageMediaIO(rgba_background_color=(255, 255, 255)).load_bytes(data) diff --git a/tests/multimodal/test_image.py b/tests/multimodal/test_image.py index ea7f18611c..4365321cf4 100644 --- a/tests/multimodal/test_image.py +++ b/tests/multimodal/test_image.py @@ -5,6 +5,7 @@ from io import BytesIO from pathlib import Path import numpy as np +import pybase64 import pytest from PIL import Image, ImageChops @@ -13,6 +14,8 @@ from vllm.multimodal.image import ( convert_image_mode, normalize_image, ) +from vllm.multimodal.media import MediaConnector +from vllm.multimodal.parse import MultiModalDataParser pytestmark = pytest.mark.cpu_test @@ -69,6 +72,68 @@ def test_palette_with_trns_to_rgb(): assert (r, g, b) == (255, 0, 0) +def _reopen(image: Image.Image, image_format: str, **save_kwargs) -> Image.Image: + buffer = BytesIO() + image.save(buffer, image_format, **save_kwargs) + return Image.open(buffer) + + +def _rgb_png() -> Image.Image: + image = Image.new("RGB", (2, 1), (255, 0, 0)) + image.putpixel((1, 0), (0, 0, 255)) + return _reopen(image, "PNG", transparency=(255, 0, 0)) + + +def _gif() -> Image.Image: + # The transparent index 0 and the opaque index 1 are both black. + image = Image.new("P", (2, 1)) + image.putpalette([0] * 768) + image.putpixel((1, 0), 1) + return _reopen(image, "GIF", transparency=0) + + +def _rgba_png() -> Image.Image: + image = Image.new("RGBA", (2, 1), (255, 0, 0, 0)) + image.putpixel((1, 0), (0, 0, 255, 255)) + return _reopen(image, "PNG") + + +@pytest.mark.parametrize( + "make_image", + [ + _rgb_png, + lambda: _rgb_png().convert("RGB"), + _gif, + lambda: _gif().convert("RGB"), + _rgba_png, + ], + ids=["rgb_png", "rgb_png_to_rgb", "gif", "gif_to_rgb", "rgba_png"], +) +def test_transparency_offline_matches_online(make_image): + """A PIL image passed offline and the same image sent online as a PNG + end up with the same pixels, matching Pillow's own RGBA conversion. + + After ``convert("RGB")``, Pillow keeps the transparent color in ``info`` + and saves it as a tRNS chunk, so every pixel of that color is + transparent in both pipelines, including opaque GIF pixels that share it. + """ + image = make_image() + buffer = BytesIO() + image.save(buffer, "PNG") + url = f"data:image/png;base64,{pybase64.b64encode(buffer.getvalue()).decode()}" + + parser = MultiModalDataParser() + offline = parser.parse_mm_data({"image": image})["image"].get(0) + online = parser.parse_mm_data({"image": MediaConnector().fetch_image(url)}) + online = online["image"].get(0) + np.testing.assert_array_equal(np.asarray(offline), np.asarray(online)) + + expected = Image.alpha_composite( + Image.new("RGBA", image.size, (255, 255, 255, 255)), image.convert("RGBA") + ).convert("RGB") + np.testing.assert_array_equal(np.asarray(online), np.asarray(expected)) + + def test_l_mode_no_trns_to_rgb(): """L-mode without transparency should convert directly.""" img = Image.new("L", (4, 4), 128) diff --git a/vllm/multimodal/image.py b/vllm/multimodal/image.py index 13df7f3c9a..96eb606e60 100644 --- a/vllm/multimodal/image.py +++ b/vllm/multimodal/image.py @@ -104,12 +104,12 @@ def convert_image_mode( to_mode: str, background_color: tuple[int, int, int] | list[int] = (255, 255, 255), ) -> Image.Image: - if image.mode == to_mode: - return image - if to_mode == "RGB" and _has_transparency(image): if image.mode != "RGBA": image = image.convert("RGBA") return rgba_to_rgb(image, background_color) + if image.mode == to_mode: + return image + return image.convert(to_mode) diff --git a/vllm/multimodal/media/image.py b/vllm/multimodal/media/image.py index 1e2e106189..5dbd6e9381 100644 --- a/vllm/multimodal/media/image.py +++ b/vllm/multimodal/media/image.py @@ -16,7 +16,7 @@ from vllm.utils.sparse_utils import ( safe_to_dense, ) -from ..image import convert_image_mode, normalize_image, open_image, rgba_to_rgb +from ..image import convert_image_mode, normalize_image, open_image from .base import MediaDecodedBudget, MediaIO, MediaWithBytes MAGIC_NUMPY_PREFIX = b"\x93NUMPY" # https://numpy.org/devdocs/reference/generated/numpy.lib.format.html#format-version-1-0 @@ -74,14 +74,9 @@ class ImageMediaIO(MediaIO[Image.Image]): """Convert image mode with custom background color.""" if isinstance(image, MediaWithBytes): image = image.media - if self.image_mode is None or image.mode == self.image_mode: + if self.image_mode is None: return image - elif image.mode == "RGBA" and self.image_mode == "RGB": - return rgba_to_rgb(image, self.rgba_background_color) - else: - return convert_image_mode( - image, self.image_mode, self.rgba_background_color - ) + return convert_image_mode(image, self.image_mode, self.rgba_background_color) def load_bytes(self, data: bytes) -> MediaWithBytes[Image.Image]: try: