[Multimodal] Allow keeping original image mode for ImageIO (#49159)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -6,6 +6,7 @@ import mimetypes
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import time
|
import time
|
||||||
|
from io import BytesIO
|
||||||
from tempfile import NamedTemporaryFile, TemporaryDirectory
|
from tempfile import NamedTemporaryFile, TemporaryDirectory
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
@@ -111,6 +112,34 @@ async def test_fetch_image_base64(
|
|||||||
assert _image_equals(data_image_sync, data_image_async)
|
assert _image_equals(data_image_sync, data_image_async)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_image_keep_original_mode():
|
||||||
|
"""media_io_kwargs can disable the default RGB conversion."""
|
||||||
|
# RGBA image: opaque black pixel on a fully transparent background
|
||||||
|
rgba_image = Image.new("RGBA", (4, 4), (0, 0, 0, 0))
|
||||||
|
rgba_image.putpixel((2, 2), (0, 0, 0, 255))
|
||||||
|
buffer = BytesIO()
|
||||||
|
rgba_image.save(buffer, "PNG")
|
||||||
|
data_url = (
|
||||||
|
f"data:image/png;base64,{base64.b64encode(buffer.getvalue()).decode('utf-8')}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Default behavior: RGBA is composited onto a white background
|
||||||
|
default_image = MediaConnector().fetch_image(data_url)
|
||||||
|
assert default_image.mode == "RGB"
|
||||||
|
assert default_image.getpixel((0, 0)) == (255, 255, 255)
|
||||||
|
assert default_image.getpixel((2, 2)) == (0, 0, 0)
|
||||||
|
|
||||||
|
# image_mode=None via media_io_kwargs: original mode is preserved
|
||||||
|
connector = MediaConnector(media_io_kwargs={"image": {"image_mode": None}})
|
||||||
|
image_sync = connector.fetch_image(data_url)
|
||||||
|
image_async = await connector.fetch_image_async(data_url)
|
||||||
|
for image in (image_sync, image_async):
|
||||||
|
assert image.mode == "RGBA"
|
||||||
|
assert image.getpixel((0, 0)) == (0, 0, 0, 0)
|
||||||
|
assert image.getpixel((2, 2)) == (0, 0, 0, 255)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
|
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
|
||||||
async def test_fetch_image_local_files(image_url: str):
|
async def test_fetch_image_local_files(image_url: str):
|
||||||
|
|||||||
@@ -80,6 +80,29 @@ def test_image_media_io_rgba_custom_background(tmp_path):
|
|||||||
assert green_numpy[0][0][2] == 0 # B
|
assert green_numpy[0][0][2] == 0 # B
|
||||||
|
|
||||||
|
|
||||||
|
def test_image_media_io_no_mode_conversion(tmp_path):
|
||||||
|
"""image_mode=None skips conversion and preserves the original mode."""
|
||||||
|
# RGBA image: opaque black pixel on a fully transparent background
|
||||||
|
rgba_image = Image.new("RGBA", (10, 10), (0, 0, 0, 0))
|
||||||
|
rgba_image.putpixel((5, 5), (0, 0, 0, 255))
|
||||||
|
test_image_path = tmp_path / "test_rgba.png"
|
||||||
|
rgba_image.save(test_image_path)
|
||||||
|
|
||||||
|
# Default behavior: RGBA is composited onto a white background
|
||||||
|
image_io_default = ImageMediaIO()
|
||||||
|
converted_default = image_io_default.load_file(test_image_path)
|
||||||
|
assert converted_default.media.mode == "RGB"
|
||||||
|
assert converted_default.media.getpixel((0, 0)) == (255, 255, 255)
|
||||||
|
assert converted_default.media.getpixel((5, 5)) == (0, 0, 0)
|
||||||
|
|
||||||
|
# image_mode=None: original mode and alpha channel are preserved
|
||||||
|
image_io_keep = ImageMediaIO(image_mode=None)
|
||||||
|
converted_keep = image_io_keep.load_file(test_image_path)
|
||||||
|
assert converted_keep.media.mode == "RGBA"
|
||||||
|
assert converted_keep.media.getpixel((0, 0)) == (0, 0, 0, 0)
|
||||||
|
assert converted_keep.media.getpixel((5, 5)) == (0, 0, 0, 255)
|
||||||
|
|
||||||
|
|
||||||
def test_image_media_io_rgba_background_color_validation():
|
def test_image_media_io_rgba_background_color_validation():
|
||||||
"""Test that invalid rgba_background_color values are properly rejected."""
|
"""Test that invalid rgba_background_color values are properly rejected."""
|
||||||
|
|
||||||
|
|||||||
@@ -476,15 +476,17 @@ class MediaConnector:
|
|||||||
self,
|
self,
|
||||||
image_url: str,
|
image_url: str,
|
||||||
*,
|
*,
|
||||||
image_mode: str = "RGB",
|
image_mode: str | None = "RGB",
|
||||||
) -> Image.Image:
|
) -> Image.Image:
|
||||||
"""
|
"""
|
||||||
Load a PIL image from an HTTP or base64 data URL.
|
Load a PIL image from an HTTP or base64 data URL.
|
||||||
|
|
||||||
By default, the image is converted into RGB format.
|
By default, the image is converted into RGB format. Set
|
||||||
|
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
|
||||||
|
original image mode (e.g. preserving the alpha channel).
|
||||||
"""
|
"""
|
||||||
image_io = ImageMediaIO(
|
image_io = ImageMediaIO(
|
||||||
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
|
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -501,15 +503,17 @@ class MediaConnector:
|
|||||||
self,
|
self,
|
||||||
image_url: str,
|
image_url: str,
|
||||||
*,
|
*,
|
||||||
image_mode: str = "RGB",
|
image_mode: str | None = "RGB",
|
||||||
) -> Image.Image:
|
) -> Image.Image:
|
||||||
"""
|
"""
|
||||||
Asynchronously load a PIL image from an HTTP or base64 data URL.
|
Asynchronously load a PIL image from an HTTP or base64 data URL.
|
||||||
|
|
||||||
By default, the image is converted into RGB format.
|
By default, the image is converted into RGB format. Set
|
||||||
|
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
|
||||||
|
original image mode (e.g. preserving the alpha channel).
|
||||||
"""
|
"""
|
||||||
image_io = ImageMediaIO(
|
image_io = ImageMediaIO(
|
||||||
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
|
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -526,14 +530,14 @@ class MediaConnector:
|
|||||||
self,
|
self,
|
||||||
video_url: str,
|
video_url: str,
|
||||||
*,
|
*,
|
||||||
image_mode: str = "RGB",
|
image_mode: str | None = "RGB",
|
||||||
video_processor: str | None = None,
|
video_processor: str | None = None,
|
||||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||||
"""
|
"""
|
||||||
Load video from an HTTP or base64 data URL.
|
Load video from an HTTP or base64 data URL.
|
||||||
"""
|
"""
|
||||||
image_io = ImageMediaIO(
|
image_io = ImageMediaIO(
|
||||||
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
|
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
|
||||||
)
|
)
|
||||||
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
|
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
|
||||||
if "video_backend" not in video_io_kwargs and (
|
if "video_backend" not in video_io_kwargs and (
|
||||||
@@ -552,16 +556,18 @@ class MediaConnector:
|
|||||||
self,
|
self,
|
||||||
video_url: str,
|
video_url: str,
|
||||||
*,
|
*,
|
||||||
image_mode: str = "RGB",
|
image_mode: str | None = "RGB",
|
||||||
video_processor: str | None = None,
|
video_processor: str | None = None,
|
||||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||||
"""
|
"""
|
||||||
Asynchronously load video from an HTTP or base64 data URL.
|
Asynchronously load video from an HTTP or base64 data URL.
|
||||||
|
|
||||||
By default, the image is converted into RGB format.
|
By default, the image is converted into RGB format. Set
|
||||||
|
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
|
||||||
|
original image mode (e.g. preserving the alpha channel).
|
||||||
"""
|
"""
|
||||||
image_io = ImageMediaIO(
|
image_io = ImageMediaIO(
|
||||||
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
|
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
|
||||||
)
|
)
|
||||||
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
|
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
|
||||||
if "video_backend" not in video_io_kwargs and (
|
if "video_backend" not in video_io_kwargs and (
|
||||||
|
|||||||
@@ -25,9 +25,11 @@ class ImageMediaIO(MediaIO[Image.Image]):
|
|||||||
error handling.
|
error handling.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, image_mode: str = "RGB", **kwargs) -> None:
|
def __init__(self, image_mode: str | None = "RGB", **kwargs) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
|
# Target mode for loaded images; `None` keeps the original mode
|
||||||
|
# (i.e. no conversion, alpha channel is preserved as-is).
|
||||||
self.image_mode = image_mode
|
self.image_mode = image_mode
|
||||||
# `kwargs` contains custom arguments from
|
# `kwargs` contains custom arguments from
|
||||||
# --media-io-kwargs for this modality, merged with
|
# --media-io-kwargs for this modality, merged with
|
||||||
@@ -62,7 +64,7 @@ class ImageMediaIO(MediaIO[Image.Image]):
|
|||||||
"""Convert image mode with custom background color."""
|
"""Convert image mode with custom background color."""
|
||||||
if isinstance(image, MediaWithBytes):
|
if isinstance(image, MediaWithBytes):
|
||||||
image = image.media
|
image = image.media
|
||||||
if image.mode == self.image_mode:
|
if self.image_mode is None or image.mode == self.image_mode:
|
||||||
return image
|
return image
|
||||||
elif image.mode == "RGBA" and self.image_mode == "RGB":
|
elif image.mode == "RGBA" and self.image_mode == "RGB":
|
||||||
return rgba_to_rgb(image, self.rgba_background_color)
|
return rgba_to_rgb(image, self.rgba_background_color)
|
||||||
|
|||||||
@@ -58,13 +58,14 @@ def encode_audio_url(
|
|||||||
def encode_image_base64(
|
def encode_image_base64(
|
||||||
image: Image.Image,
|
image: Image.Image,
|
||||||
*,
|
*,
|
||||||
image_mode: str = "RGB",
|
image_mode: str | None = "RGB",
|
||||||
format: str = "PNG",
|
format: str = "PNG",
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Encode a pillow image to base64 format.
|
Encode a pillow image to base64 format.
|
||||||
|
|
||||||
By default, the image is converted into RGB format before being encoded.
|
By default, the image is converted into RGB format before being encoded.
|
||||||
|
Pass `image_mode=None` to keep the original image mode.
|
||||||
"""
|
"""
|
||||||
image_io = ImageMediaIO(image_mode=image_mode)
|
image_io = ImageMediaIO(image_mode=image_mode)
|
||||||
return image_io.encode_base64(image, image_format=format)
|
return image_io.encode_base64(image, image_format=format)
|
||||||
@@ -73,13 +74,14 @@ def encode_image_base64(
|
|||||||
def encode_image_url(
|
def encode_image_url(
|
||||||
image: Image.Image,
|
image: Image.Image,
|
||||||
*,
|
*,
|
||||||
image_mode: str = "RGB",
|
image_mode: str | None = "RGB",
|
||||||
format: str = "PNG",
|
format: str = "PNG",
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Encode a pillow image as a data URL.
|
Encode a pillow image as a data URL.
|
||||||
|
|
||||||
By default, the image is converted into RGB format before being encoded.
|
By default, the image is converted into RGB format before being encoded.
|
||||||
|
Pass `image_mode=None` to keep the original image mode.
|
||||||
"""
|
"""
|
||||||
image_b64 = encode_image_base64(image, image_mode=image_mode, format=format)
|
image_b64 = encode_image_base64(image, image_mode=image_mode, format=format)
|
||||||
mimetype = mimetypes.types_map.get("." + format.lower(), "image")
|
mimetype = mimetypes.types_map.get("." + format.lower(), "image")
|
||||||
|
|||||||
Reference in New Issue
Block a user