[Bugfix][Multimodal] Honor RGB PNG transparency when loading images (#61986)

Signed-off-by: mottopanikeiku <176798723+mottopanikeiku@users.noreply.github.com>
Co-authored-by: mottopanikeiku <176798723+mottopanikeiku@users.noreply.github.com>

docs/features/multimodal_inputs.md

@@ -187,7 +187,7 @@ Multi-image input can be extended to perform video captioning. We show this with
 
 #### Custom RGBA Background Color
 
-When loading RGBA images (images with transparency), vLLM converts them to RGB format. By default, transparent pixels are replaced with white background. You can customize this background color using the `rgba_background_color` parameter in `media_io_kwargs`.
+When loading images with transparency (including RGBA images and PNGs with a tRNS chunk), vLLM converts them to RGB format. By default, transparent pixels are replaced with white background. You can customize this background color using the `rgba_background_color` parameter in `media_io_kwargs`.
 
 ??? code
 
@@ -212,7 +212,7 @@ When loading RGBA images (images with transparency), vLLM converts them to RGB f
 
 !!! note
     - The `rgba_background_color` accepts RGB values as a list `[R, G, B]` or tuple `(R, G, B)` where each value is 0-255
-    - This setting only affects RGBA images with transparency; RGB images are unchanged
+    - This setting affects images with transparency, including RGB PNGs with a transparent color; opaque RGB images are unchanged
     - If not specified, the default white background `(255, 255, 255)` is used for backward compatibility
 
 #### Moondream3 Prompt Recipes { #moondream3-prompt-recipes }
@@ -1089,7 +1089,7 @@ By passing this metadata, the model can better understand the temporal distribut
 
 #### Custom RGBA Background Color
 
-To use a custom background color for RGBA images, pass the `rgba_background_color` parameter via `--media-io-kwargs`:
+To use a custom background color for images with transparency, pass the `rgba_background_color` parameter via `--media-io-kwargs`:
 
 ```bash
 # Example: Black background for dark theme

tests/multimodal/media/test_image.py

@@ -85,6 +85,24 @@ def test_image_media_io_rgba_custom_background(tmp_path):
     assert green_numpy[0][0][2] == 0  # B
 
 
+@pytest.mark.parametrize("background_color", [(255, 255, 255), (12, 34, 56)])
+def test_image_media_io_rgb_png_transparency(tmp_path, background_color):
+    image = Image.new("RGB", (2, 1), (255, 0, 0))
+    image.putpixel((1, 0), (0, 0, 255))
+    path = tmp_path / "rgb_transparency.png"
+    image.save(path, transparency=(255, 0, 0))
+
+    with Image.open(path) as source:
+        reference = Image.alpha_composite(
+            Image.new("RGBA", source.size, (*background_color, 255)),
+            source.convert("RGBA"),
+        ).convert("RGB")
+
+    image_io = ImageMediaIO(rgba_background_color=background_color)
+    converted = image_io.load_file(path)
+    np.testing.assert_array_equal(np.asarray(converted), np.asarray(reference))
+
+
 def test_image_media_io_no_mode_conversion(tmp_path):
     """image_mode=None skips conversion and preserves the original mode."""
     # RGBA image: opaque black pixel on a fully transparent background

tests/multimodal/test_hasher.py

@@ -237,8 +237,16 @@ def _rgba_png_bytes() -> bytes:
     return buf.getvalue()
 
 
-def test_hash_collision_media_io_config():
-    data = _rgba_png_bytes()
+def _rgb_trns_png_bytes() -> bytes:
+    image = Image.new("RGB", (8, 8), (255, 0, 0))
+    buf = BytesIO()
+    image.save(buf, format="PNG", transparency=(255, 0, 0))
+    return buf.getvalue()
+
+
+@pytest.mark.parametrize("png_bytes", [_rgba_png_bytes, _rgb_trns_png_bytes])
+def test_hash_collision_media_io_config(png_bytes):
+    data = png_bytes()
     white = ImageMediaIO(rgba_background_color=(255, 255, 255)).load_bytes(data)
     black = ImageMediaIO(rgba_background_color=(0, 0, 0)).load_bytes(data)
     white2 = ImageMediaIO(rgba_background_color=(255, 255, 255)).load_bytes(data)

tests/multimodal/test_image.py

@@ -5,6 +5,7 @@ from io import BytesIO
 from pathlib import Path
 
 import numpy as np
+import pybase64
 import pytest
 from PIL import Image, ImageChops
 
@@ -13,6 +14,8 @@ from vllm.multimodal.image import (
     convert_image_mode,
     normalize_image,
 )
+from vllm.multimodal.media import MediaConnector
+from vllm.multimodal.parse import MultiModalDataParser
 
 pytestmark = pytest.mark.cpu_test
 
@@ -69,6 +72,68 @@ def test_palette_with_trns_to_rgb():
     assert (r, g, b) == (255, 0, 0)
 
 
+def _reopen(image: Image.Image, image_format: str, **save_kwargs) -> Image.Image:
+    buffer = BytesIO()
+    image.save(buffer, image_format, **save_kwargs)
+    return Image.open(buffer)
+
+
+def _rgb_png() -> Image.Image:
+    image = Image.new("RGB", (2, 1), (255, 0, 0))
+    image.putpixel((1, 0), (0, 0, 255))
+    return _reopen(image, "PNG", transparency=(255, 0, 0))
+
+
+def _gif() -> Image.Image:
+    # The transparent index 0 and the opaque index 1 are both black.
+    image = Image.new("P", (2, 1))
+    image.putpalette([0] * 768)
+    image.putpixel((1, 0), 1)
+    return _reopen(image, "GIF", transparency=0)
+
+
+def _rgba_png() -> Image.Image:
+    image = Image.new("RGBA", (2, 1), (255, 0, 0, 0))
+    image.putpixel((1, 0), (0, 0, 255, 255))
+    return _reopen(image, "PNG")
+
+
+@pytest.mark.parametrize(
+    "make_image",
+    [
+        _rgb_png,
+        lambda: _rgb_png().convert("RGB"),
+        _gif,
+        lambda: _gif().convert("RGB"),
+        _rgba_png,
+    ],
+    ids=["rgb_png", "rgb_png_to_rgb", "gif", "gif_to_rgb", "rgba_png"],
+)
+def test_transparency_offline_matches_online(make_image):
+    """A PIL image passed offline and the same image sent online as a PNG
+    end up with the same pixels, matching Pillow's own RGBA conversion.
+
+    After ``convert("RGB")``, Pillow keeps the transparent color in ``info``
+    and saves it as a tRNS chunk, so every pixel of that color is
+    transparent in both pipelines, including opaque GIF pixels that share it.
+    """
+    image = make_image()
+    buffer = BytesIO()
+    image.save(buffer, "PNG")
+    url = f"data:image/png;base64,{pybase64.b64encode(buffer.getvalue()).decode()}"
+
+    parser = MultiModalDataParser()
+    offline = parser.parse_mm_data({"image": image})["image"].get(0)
+    online = parser.parse_mm_data({"image": MediaConnector().fetch_image(url)})
+    online = online["image"].get(0)
+    np.testing.assert_array_equal(np.asarray(offline), np.asarray(online))
+
+    expected = Image.alpha_composite(
+        Image.new("RGBA", image.size, (255, 255, 255, 255)), image.convert("RGBA")
+    ).convert("RGB")
+    np.testing.assert_array_equal(np.asarray(online), np.asarray(expected))
+
+
 def test_l_mode_no_trns_to_rgb():
     """L-mode without transparency should convert directly."""
     img = Image.new("L", (4, 4), 128)

vllm/multimodal/image.py

@@ -104,12 +104,12 @@ def convert_image_mode(
     to_mode: str,
     background_color: tuple[int, int, int] | list[int] = (255, 255, 255),
 ) -> Image.Image:
-    if image.mode == to_mode:
-        return image
-
     if to_mode == "RGB" and _has_transparency(image):
         if image.mode != "RGBA":
             image = image.convert("RGBA")
         return rgba_to_rgb(image, background_color)
 
+    if image.mode == to_mode:
+        return image
+
     return image.convert(to_mode)

vllm/multimodal/media/image.py

@@ -16,7 +16,7 @@ from vllm.utils.sparse_utils import (
     safe_to_dense,
 )
 
-from ..image import convert_image_mode, normalize_image, open_image, rgba_to_rgb
+from ..image import convert_image_mode, normalize_image, open_image
 from .base import MediaDecodedBudget, MediaIO, MediaWithBytes
 
 MAGIC_NUMPY_PREFIX = b"\x93NUMPY"  # https://numpy.org/devdocs/reference/generated/numpy.lib.format.html#format-version-1-0
@@ -74,14 +74,9 @@ class ImageMediaIO(MediaIO[Image.Image]):
         """Convert image mode with custom background color."""
         if isinstance(image, MediaWithBytes):
             image = image.media
-        if self.image_mode is None or image.mode == self.image_mode:
+        if self.image_mode is None:
             return image
-        elif image.mode == "RGBA" and self.image_mode == "RGB":
-            return rgba_to_rgb(image, self.rgba_background_color)
-        else:
-            return convert_image_mode(
-                image, self.image_mode, self.rgba_background_color
-            )
+        return convert_image_mode(image, self.image_mode, self.rgba_background_color)
 
     def load_bytes(self, data: bytes) -> MediaWithBytes[Image.Image]:
         try: