[Multimodal] Allow keeping original image mode for ImageIO (#49159)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-07-20 13:42:45 +00:00
committed by GitHub
parent f007cceb42
commit 15cb8e140d
5 changed files with 77 additions and 15 deletions
+29
View File
@@ -6,6 +6,7 @@ import mimetypes
import os import os
import shutil import shutil
import time import time
from io import BytesIO
from tempfile import NamedTemporaryFile, TemporaryDirectory from tempfile import NamedTemporaryFile, TemporaryDirectory
import aiohttp import aiohttp
@@ -111,6 +112,34 @@ async def test_fetch_image_base64(
assert _image_equals(data_image_sync, data_image_async) assert _image_equals(data_image_sync, data_image_async)
@pytest.mark.asyncio
async def test_fetch_image_keep_original_mode():
"""media_io_kwargs can disable the default RGB conversion."""
# RGBA image: opaque black pixel on a fully transparent background
rgba_image = Image.new("RGBA", (4, 4), (0, 0, 0, 0))
rgba_image.putpixel((2, 2), (0, 0, 0, 255))
buffer = BytesIO()
rgba_image.save(buffer, "PNG")
data_url = (
f"data:image/png;base64,{base64.b64encode(buffer.getvalue()).decode('utf-8')}"
)
# Default behavior: RGBA is composited onto a white background
default_image = MediaConnector().fetch_image(data_url)
assert default_image.mode == "RGB"
assert default_image.getpixel((0, 0)) == (255, 255, 255)
assert default_image.getpixel((2, 2)) == (0, 0, 0)
# image_mode=None via media_io_kwargs: original mode is preserved
connector = MediaConnector(media_io_kwargs={"image": {"image_mode": None}})
image_sync = connector.fetch_image(data_url)
image_async = await connector.fetch_image_async(data_url)
for image in (image_sync, image_async):
assert image.mode == "RGBA"
assert image.getpixel((0, 0)) == (0, 0, 0, 0)
assert image.getpixel((2, 2)) == (0, 0, 0, 255)
@pytest.mark.asyncio @pytest.mark.asyncio
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True) @pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
async def test_fetch_image_local_files(image_url: str): async def test_fetch_image_local_files(image_url: str):
+23
View File
@@ -80,6 +80,29 @@ def test_image_media_io_rgba_custom_background(tmp_path):
assert green_numpy[0][0][2] == 0 # B assert green_numpy[0][0][2] == 0 # B
def test_image_media_io_no_mode_conversion(tmp_path):
"""image_mode=None skips conversion and preserves the original mode."""
# RGBA image: opaque black pixel on a fully transparent background
rgba_image = Image.new("RGBA", (10, 10), (0, 0, 0, 0))
rgba_image.putpixel((5, 5), (0, 0, 0, 255))
test_image_path = tmp_path / "test_rgba.png"
rgba_image.save(test_image_path)
# Default behavior: RGBA is composited onto a white background
image_io_default = ImageMediaIO()
converted_default = image_io_default.load_file(test_image_path)
assert converted_default.media.mode == "RGB"
assert converted_default.media.getpixel((0, 0)) == (255, 255, 255)
assert converted_default.media.getpixel((5, 5)) == (0, 0, 0)
# image_mode=None: original mode and alpha channel are preserved
image_io_keep = ImageMediaIO(image_mode=None)
converted_keep = image_io_keep.load_file(test_image_path)
assert converted_keep.media.mode == "RGBA"
assert converted_keep.media.getpixel((0, 0)) == (0, 0, 0, 0)
assert converted_keep.media.getpixel((5, 5)) == (0, 0, 0, 255)
def test_image_media_io_rgba_background_color_validation(): def test_image_media_io_rgba_background_color_validation():
"""Test that invalid rgba_background_color values are properly rejected.""" """Test that invalid rgba_background_color values are properly rejected."""
+17 -11
View File
@@ -476,15 +476,17 @@ class MediaConnector:
self, self,
image_url: str, image_url: str,
*, *,
image_mode: str = "RGB", image_mode: str | None = "RGB",
) -> Image.Image: ) -> Image.Image:
""" """
Load a PIL image from an HTTP or base64 data URL. Load a PIL image from an HTTP or base64 data URL.
By default, the image is converted into RGB format. By default, the image is converted into RGB format. Set
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
original image mode (e.g. preserving the alpha channel).
""" """
image_io = ImageMediaIO( image_io = ImageMediaIO(
image_mode=image_mode, **self.media_io_kwargs.get("image", {}) **({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
) )
try: try:
@@ -501,15 +503,17 @@ class MediaConnector:
self, self,
image_url: str, image_url: str,
*, *,
image_mode: str = "RGB", image_mode: str | None = "RGB",
) -> Image.Image: ) -> Image.Image:
""" """
Asynchronously load a PIL image from an HTTP or base64 data URL. Asynchronously load a PIL image from an HTTP or base64 data URL.
By default, the image is converted into RGB format. By default, the image is converted into RGB format. Set
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
original image mode (e.g. preserving the alpha channel).
""" """
image_io = ImageMediaIO( image_io = ImageMediaIO(
image_mode=image_mode, **self.media_io_kwargs.get("image", {}) **({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
) )
try: try:
@@ -526,14 +530,14 @@ class MediaConnector:
self, self,
video_url: str, video_url: str,
*, *,
image_mode: str = "RGB", image_mode: str | None = "RGB",
video_processor: str | None = None, video_processor: str | None = None,
) -> tuple[npt.NDArray, dict[str, Any]]: ) -> tuple[npt.NDArray, dict[str, Any]]:
""" """
Load video from an HTTP or base64 data URL. Load video from an HTTP or base64 data URL.
""" """
image_io = ImageMediaIO( image_io = ImageMediaIO(
image_mode=image_mode, **self.media_io_kwargs.get("image", {}) **({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
) )
video_io_kwargs = dict(self.media_io_kwargs.get("video", {})) video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
if "video_backend" not in video_io_kwargs and ( if "video_backend" not in video_io_kwargs and (
@@ -552,16 +556,18 @@ class MediaConnector:
self, self,
video_url: str, video_url: str,
*, *,
image_mode: str = "RGB", image_mode: str | None = "RGB",
video_processor: str | None = None, video_processor: str | None = None,
) -> tuple[npt.NDArray, dict[str, Any]]: ) -> tuple[npt.NDArray, dict[str, Any]]:
""" """
Asynchronously load video from an HTTP or base64 data URL. Asynchronously load video from an HTTP or base64 data URL.
By default, the image is converted into RGB format. By default, the image is converted into RGB format. Set
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
original image mode (e.g. preserving the alpha channel).
""" """
image_io = ImageMediaIO( image_io = ImageMediaIO(
image_mode=image_mode, **self.media_io_kwargs.get("image", {}) **({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
) )
video_io_kwargs = dict(self.media_io_kwargs.get("video", {})) video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
if "video_backend" not in video_io_kwargs and ( if "video_backend" not in video_io_kwargs and (
+4 -2
View File
@@ -25,9 +25,11 @@ class ImageMediaIO(MediaIO[Image.Image]):
error handling. error handling.
""" """
def __init__(self, image_mode: str = "RGB", **kwargs) -> None: def __init__(self, image_mode: str | None = "RGB", **kwargs) -> None:
super().__init__() super().__init__()
# Target mode for loaded images; `None` keeps the original mode
# (i.e. no conversion, alpha channel is preserved as-is).
self.image_mode = image_mode self.image_mode = image_mode
# `kwargs` contains custom arguments from # `kwargs` contains custom arguments from
# --media-io-kwargs for this modality, merged with # --media-io-kwargs for this modality, merged with
@@ -62,7 +64,7 @@ class ImageMediaIO(MediaIO[Image.Image]):
"""Convert image mode with custom background color.""" """Convert image mode with custom background color."""
if isinstance(image, MediaWithBytes): if isinstance(image, MediaWithBytes):
image = image.media image = image.media
if image.mode == self.image_mode: if self.image_mode is None or image.mode == self.image_mode:
return image return image
elif image.mode == "RGBA" and self.image_mode == "RGB": elif image.mode == "RGBA" and self.image_mode == "RGB":
return rgba_to_rgb(image, self.rgba_background_color) return rgba_to_rgb(image, self.rgba_background_color)
+4 -2
View File
@@ -58,13 +58,14 @@ def encode_audio_url(
def encode_image_base64( def encode_image_base64(
image: Image.Image, image: Image.Image,
*, *,
image_mode: str = "RGB", image_mode: str | None = "RGB",
format: str = "PNG", format: str = "PNG",
) -> str: ) -> str:
""" """
Encode a pillow image to base64 format. Encode a pillow image to base64 format.
By default, the image is converted into RGB format before being encoded. By default, the image is converted into RGB format before being encoded.
Pass `image_mode=None` to keep the original image mode.
""" """
image_io = ImageMediaIO(image_mode=image_mode) image_io = ImageMediaIO(image_mode=image_mode)
return image_io.encode_base64(image, image_format=format) return image_io.encode_base64(image, image_format=format)
@@ -73,13 +74,14 @@ def encode_image_base64(
def encode_image_url( def encode_image_url(
image: Image.Image, image: Image.Image,
*, *,
image_mode: str = "RGB", image_mode: str | None = "RGB",
format: str = "PNG", format: str = "PNG",
) -> str: ) -> str:
""" """
Encode a pillow image as a data URL. Encode a pillow image as a data URL.
By default, the image is converted into RGB format before being encoded. By default, the image is converted into RGB format before being encoded.
Pass `image_mode=None` to keep the original image mode.
""" """
image_b64 = encode_image_base64(image, image_mode=image_mode, format=format) image_b64 = encode_image_base64(image, image_mode=image_mode, format=format)
mimetype = mimetypes.types_map.get("." + format.lower(), "image") mimetype = mimetypes.types_map.get("." + format.lower(), "image")