feat: expose media_io_kwargs at runtime (#34778)

Signed-off-by: Alexandre Milesi <milesial@users.noreply.github.com>
This commit is contained in:
milesial
2026-03-07 04:27:04 +00:00
committed by GitHub
parent 58928475e4
commit 755356b3d1
20 changed files with 298 additions and 16 deletions
+12 -1
View File
@@ -83,11 +83,17 @@ def extract_audio_from_video_bytes(
class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
"""Configuration values can be user-provided either by --media-io-kwargs or
by the runtime API field "media_io_kwargs". Ensure proper validation and
error handling.
"""
def __init__(self, **kwargs) -> None:
super().__init__()
# `kwargs` contains custom arguments from
# --media-io-kwargs for this modality.
# --media-io-kwargs for this modality, merged with
# per-request runtime media_io_kwargs via merge_kwargs().
# They can be passed to the underlying
# media loaders (e.g. custom implementations)
# for flexible control.
@@ -122,6 +128,11 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]):
"""Configuration values can be user-provided either by --media-io-kwargs or
by the runtime API field "media_io_kwargs". Ensure proper validation and
error handling.
"""
def __init__(self) -> None:
super().__init__()
+22
View File
@@ -44,6 +44,28 @@ class MediaWithBytes(Generic[_T]):
class MediaIO(ABC, Generic[_T]):
"""Configuration values can be user-provided either by --media-io-kwargs or
by the runtime API field "media_io_kwargs". Ensure proper validation and
error handling.
"""
@classmethod
def merge_kwargs(
cls,
default_kwargs: dict[str, Any] | None,
runtime_kwargs: dict[str, Any] | None,
) -> dict[str, Any]:
"""Merge config-level kwargs and request-level kwargs.
By default this performs a shallow merge where runtime kwargs override
keys in default kwargs. Subclasses may override to apply modality-
specific behavior.
"""
merged = dict(default_kwargs or {})
if runtime_kwargs:
merged.update(runtime_kwargs)
return merged
@abstractmethod
def load_bytes(self, data: bytes) -> _T:
raise NotImplementedError
+34
View File
@@ -32,9 +32,43 @@ atexit.register(global_thread_pool.shutdown)
MEDIA_CONNECTOR_REGISTRY = ExtensionManager()
MODALITY_IO_MAP: dict[str, type[MediaIO]] = {
"audio": AudioMediaIO,
"image": ImageMediaIO,
"video": VideoMediaIO,
}
def merge_media_io_kwargs(
defaults: dict[str, dict[str, Any]] | None,
overrides: dict[str, dict[str, Any]] | None,
) -> dict[str, dict[str, Any]] | None:
"""Merge config-level and per-request media_io_kwargs per modality.
Each modality key is merged using the corresponding MediaIO subclass's
``merge_kwargs``, which may apply modality-specific logic (e.g.
VideoMediaIO clears cross-dependent fps/num_frames fields).
"""
if not defaults and not overrides:
return None
all_keys = set(defaults or {}) | set(overrides or {})
merged = {}
for key in all_keys:
io_cls = MODALITY_IO_MAP.get(key, MediaIO)
merged[key] = io_cls.merge_kwargs(
(defaults or {}).get(key),
(overrides or {}).get(key),
)
return merged or None
@MEDIA_CONNECTOR_REGISTRY.register("http")
class MediaConnector:
"""Configuration values can be user-provided either by --media-io-kwargs or
by the runtime API field "media_io_kwargs". Ensure proper validation and
error handling.
"""
def __init__(
self,
media_io_kwargs: dict[str, dict[str, Any]] | None = None,
+14 -1
View File
@@ -15,12 +15,18 @@ from .base import MediaIO, MediaWithBytes
class ImageMediaIO(MediaIO[Image.Image]):
"""Configuration values can be user-provided either by --media-io-kwargs or
by the runtime API field "media_io_kwargs". Ensure proper validation and
error handling.
"""
def __init__(self, image_mode: str = "RGB", **kwargs) -> None:
super().__init__()
self.image_mode = image_mode
# `kwargs` contains custom arguments from
# --media-io-kwargs for this modality.
# --media-io-kwargs for this modality, merged with
# per-request runtime media_io_kwargs via merge_kwargs().
# They can be passed to the underlying
# media loaders (e.g. custom implementations)
# for flexible control.
@@ -88,6 +94,13 @@ class ImageMediaIO(MediaIO[Image.Image]):
class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
"""Image embedding MediaIO implementation.
Configuration values can be user-provided either by --media-io-kwargs or
by the runtime API field "media_io_kwargs". Ensure proper validation and
error handling.
"""
def __init__(self) -> None:
super().__init__()
+24 -1
View File
@@ -17,6 +17,28 @@ from .image import ImageMediaIO
class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
"""Configuration values can be user-provided either by --media-io-kwargs or
by the runtime API field "media_io_kwargs". Ensure proper validation and
error handling.
"""
@classmethod
def merge_kwargs(
cls,
default_kwargs: dict[str, Any] | None,
runtime_kwargs: dict[str, Any] | None,
) -> dict[str, Any]:
merged = super().merge_kwargs(default_kwargs, runtime_kwargs)
# fps and num_frames interact with each other, so if either is
# overridden at request time, wipe the other from defaults to
# avoid unintuitive cross-field interactions.
if runtime_kwargs:
if "num_frames" in runtime_kwargs and "fps" not in runtime_kwargs:
merged.pop("fps", None)
elif "fps" in runtime_kwargs and "num_frames" not in runtime_kwargs:
merged.pop("num_frames", None)
return merged
def __init__(
self,
image_io: ImageMediaIO,
@@ -28,7 +50,8 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
self.image_io = image_io
self.num_frames = num_frames
# `kwargs` contains custom arguments from
# --media-io-kwargs for this modality.
# --media-io-kwargs for this modality, merged with
# per-request runtime media_io_kwargs via merge_kwargs().
# They can be passed to the underlying
# media loaders (e.g. custom implementations)
# for flexible control.