From 15cb8e140dffd9d2a33737c083fcf1d696e1d1f6 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Mon, 20 Jul 2026 21:42:45 +0800 Subject: [PATCH] [Multimodal] Allow keeping original image mode for ImageIO (#49159) Signed-off-by: Isotr0py --- tests/multimodal/media/test_connector.py | 29 ++++++++++++++++++++++++ tests/multimodal/media/test_image.py | 23 +++++++++++++++++++ vllm/multimodal/media/connector.py | 28 ++++++++++++++--------- vllm/multimodal/media/image.py | 6 +++-- vllm/multimodal/utils.py | 6 +++-- 5 files changed, 77 insertions(+), 15 deletions(-) diff --git a/tests/multimodal/media/test_connector.py b/tests/multimodal/media/test_connector.py index bee9d50ac1..5cd22a8b18 100644 --- a/tests/multimodal/media/test_connector.py +++ b/tests/multimodal/media/test_connector.py @@ -6,6 +6,7 @@ import mimetypes import os import shutil import time +from io import BytesIO from tempfile import NamedTemporaryFile, TemporaryDirectory import aiohttp @@ -111,6 +112,34 @@ async def test_fetch_image_base64( assert _image_equals(data_image_sync, data_image_async) +@pytest.mark.asyncio +async def test_fetch_image_keep_original_mode(): + """media_io_kwargs can disable the default RGB conversion.""" + # RGBA image: opaque black pixel on a fully transparent background + rgba_image = Image.new("RGBA", (4, 4), (0, 0, 0, 0)) + rgba_image.putpixel((2, 2), (0, 0, 0, 255)) + buffer = BytesIO() + rgba_image.save(buffer, "PNG") + data_url = ( + f"data:image/png;base64,{base64.b64encode(buffer.getvalue()).decode('utf-8')}" + ) + + # Default behavior: RGBA is composited onto a white background + default_image = MediaConnector().fetch_image(data_url) + assert default_image.mode == "RGB" + assert default_image.getpixel((0, 0)) == (255, 255, 255) + assert default_image.getpixel((2, 2)) == (0, 0, 0) + + # image_mode=None via media_io_kwargs: original mode is preserved + connector = MediaConnector(media_io_kwargs={"image": {"image_mode": None}}) + image_sync = connector.fetch_image(data_url) + image_async = await connector.fetch_image_async(data_url) + for image in (image_sync, image_async): + assert image.mode == "RGBA" + assert image.getpixel((0, 0)) == (0, 0, 0, 0) + assert image.getpixel((2, 2)) == (0, 0, 0, 255) + + @pytest.mark.asyncio @pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True) async def test_fetch_image_local_files(image_url: str): diff --git a/tests/multimodal/media/test_image.py b/tests/multimodal/media/test_image.py index c84343a378..5bbf54b787 100644 --- a/tests/multimodal/media/test_image.py +++ b/tests/multimodal/media/test_image.py @@ -80,6 +80,29 @@ def test_image_media_io_rgba_custom_background(tmp_path): assert green_numpy[0][0][2] == 0 # B +def test_image_media_io_no_mode_conversion(tmp_path): + """image_mode=None skips conversion and preserves the original mode.""" + # RGBA image: opaque black pixel on a fully transparent background + rgba_image = Image.new("RGBA", (10, 10), (0, 0, 0, 0)) + rgba_image.putpixel((5, 5), (0, 0, 0, 255)) + test_image_path = tmp_path / "test_rgba.png" + rgba_image.save(test_image_path) + + # Default behavior: RGBA is composited onto a white background + image_io_default = ImageMediaIO() + converted_default = image_io_default.load_file(test_image_path) + assert converted_default.media.mode == "RGB" + assert converted_default.media.getpixel((0, 0)) == (255, 255, 255) + assert converted_default.media.getpixel((5, 5)) == (0, 0, 0) + + # image_mode=None: original mode and alpha channel are preserved + image_io_keep = ImageMediaIO(image_mode=None) + converted_keep = image_io_keep.load_file(test_image_path) + assert converted_keep.media.mode == "RGBA" + assert converted_keep.media.getpixel((0, 0)) == (0, 0, 0, 0) + assert converted_keep.media.getpixel((5, 5)) == (0, 0, 0, 255) + + def test_image_media_io_rgba_background_color_validation(): """Test that invalid rgba_background_color values are properly rejected.""" diff --git a/vllm/multimodal/media/connector.py b/vllm/multimodal/media/connector.py index a440e69ef1..cda0575e13 100644 --- a/vllm/multimodal/media/connector.py +++ b/vllm/multimodal/media/connector.py @@ -476,15 +476,17 @@ class MediaConnector: self, image_url: str, *, - image_mode: str = "RGB", + image_mode: str | None = "RGB", ) -> Image.Image: """ Load a PIL image from an HTTP or base64 data URL. - By default, the image is converted into RGB format. + By default, the image is converted into RGB format. Set + `media_io_kwargs={"image": {"image_mode": None}}` to keep the + original image mode (e.g. preserving the alpha channel). """ image_io = ImageMediaIO( - image_mode=image_mode, **self.media_io_kwargs.get("image", {}) + **({"image_mode": image_mode} | self.media_io_kwargs.get("image", {})) ) try: @@ -501,15 +503,17 @@ class MediaConnector: self, image_url: str, *, - image_mode: str = "RGB", + image_mode: str | None = "RGB", ) -> Image.Image: """ Asynchronously load a PIL image from an HTTP or base64 data URL. - By default, the image is converted into RGB format. + By default, the image is converted into RGB format. Set + `media_io_kwargs={"image": {"image_mode": None}}` to keep the + original image mode (e.g. preserving the alpha channel). """ image_io = ImageMediaIO( - image_mode=image_mode, **self.media_io_kwargs.get("image", {}) + **({"image_mode": image_mode} | self.media_io_kwargs.get("image", {})) ) try: @@ -526,14 +530,14 @@ class MediaConnector: self, video_url: str, *, - image_mode: str = "RGB", + image_mode: str | None = "RGB", video_processor: str | None = None, ) -> tuple[npt.NDArray, dict[str, Any]]: """ Load video from an HTTP or base64 data URL. """ image_io = ImageMediaIO( - image_mode=image_mode, **self.media_io_kwargs.get("image", {}) + **({"image_mode": image_mode} | self.media_io_kwargs.get("image", {})) ) video_io_kwargs = dict(self.media_io_kwargs.get("video", {})) if "video_backend" not in video_io_kwargs and ( @@ -552,16 +556,18 @@ class MediaConnector: self, video_url: str, *, - image_mode: str = "RGB", + image_mode: str | None = "RGB", video_processor: str | None = None, ) -> tuple[npt.NDArray, dict[str, Any]]: """ Asynchronously load video from an HTTP or base64 data URL. - By default, the image is converted into RGB format. + By default, the image is converted into RGB format. Set + `media_io_kwargs={"image": {"image_mode": None}}` to keep the + original image mode (e.g. preserving the alpha channel). """ image_io = ImageMediaIO( - image_mode=image_mode, **self.media_io_kwargs.get("image", {}) + **({"image_mode": image_mode} | self.media_io_kwargs.get("image", {})) ) video_io_kwargs = dict(self.media_io_kwargs.get("video", {})) if "video_backend" not in video_io_kwargs and ( diff --git a/vllm/multimodal/media/image.py b/vllm/multimodal/media/image.py index 8a49574612..c98504fdec 100644 --- a/vllm/multimodal/media/image.py +++ b/vllm/multimodal/media/image.py @@ -25,9 +25,11 @@ class ImageMediaIO(MediaIO[Image.Image]): error handling. """ - def __init__(self, image_mode: str = "RGB", **kwargs) -> None: + def __init__(self, image_mode: str | None = "RGB", **kwargs) -> None: super().__init__() + # Target mode for loaded images; `None` keeps the original mode + # (i.e. no conversion, alpha channel is preserved as-is). self.image_mode = image_mode # `kwargs` contains custom arguments from # --media-io-kwargs for this modality, merged with @@ -62,7 +64,7 @@ class ImageMediaIO(MediaIO[Image.Image]): """Convert image mode with custom background color.""" if isinstance(image, MediaWithBytes): image = image.media - if image.mode == self.image_mode: + if self.image_mode is None or image.mode == self.image_mode: return image elif image.mode == "RGBA" and self.image_mode == "RGB": return rgba_to_rgb(image, self.rgba_background_color) diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py index f34769d838..401e6e517b 100644 --- a/vllm/multimodal/utils.py +++ b/vllm/multimodal/utils.py @@ -58,13 +58,14 @@ def encode_audio_url( def encode_image_base64( image: Image.Image, *, - image_mode: str = "RGB", + image_mode: str | None = "RGB", format: str = "PNG", ) -> str: """ Encode a pillow image to base64 format. By default, the image is converted into RGB format before being encoded. + Pass `image_mode=None` to keep the original image mode. """ image_io = ImageMediaIO(image_mode=image_mode) return image_io.encode_base64(image, image_format=format) @@ -73,13 +74,14 @@ def encode_image_base64( def encode_image_url( image: Image.Image, *, - image_mode: str = "RGB", + image_mode: str | None = "RGB", format: str = "PNG", ) -> str: """ Encode a pillow image as a data URL. By default, the image is converted into RGB format before being encoded. + Pass `image_mode=None` to keep the original image mode. """ image_b64 = encode_image_base64(image, image_mode=image_mode, format=format) mimetype = mimetypes.types_map.get("." + format.lower(), "image")