feat: expose media_io_kwargs at runtime (#34778)
Signed-off-by: Alexandre Milesi <milesial@users.noreply.github.com>
This commit is contained in:
@@ -83,11 +83,17 @@ def extract_audio_from_video_bytes(
|
||||
|
||||
|
||||
class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
||||
"""Configuration values can be user-provided either by --media-io-kwargs or
|
||||
by the runtime API field "media_io_kwargs". Ensure proper validation and
|
||||
error handling.
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs) -> None:
|
||||
super().__init__()
|
||||
|
||||
# `kwargs` contains custom arguments from
|
||||
# --media-io-kwargs for this modality.
|
||||
# --media-io-kwargs for this modality, merged with
|
||||
# per-request runtime media_io_kwargs via merge_kwargs().
|
||||
# They can be passed to the underlying
|
||||
# media loaders (e.g. custom implementations)
|
||||
# for flexible control.
|
||||
@@ -122,6 +128,11 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
||||
|
||||
|
||||
class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]):
|
||||
"""Configuration values can be user-provided either by --media-io-kwargs or
|
||||
by the runtime API field "media_io_kwargs". Ensure proper validation and
|
||||
error handling.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
|
||||
@@ -44,6 +44,28 @@ class MediaWithBytes(Generic[_T]):
|
||||
|
||||
|
||||
class MediaIO(ABC, Generic[_T]):
|
||||
"""Configuration values can be user-provided either by --media-io-kwargs or
|
||||
by the runtime API field "media_io_kwargs". Ensure proper validation and
|
||||
error handling.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def merge_kwargs(
|
||||
cls,
|
||||
default_kwargs: dict[str, Any] | None,
|
||||
runtime_kwargs: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Merge config-level kwargs and request-level kwargs.
|
||||
|
||||
By default this performs a shallow merge where runtime kwargs override
|
||||
keys in default kwargs. Subclasses may override to apply modality-
|
||||
specific behavior.
|
||||
"""
|
||||
merged = dict(default_kwargs or {})
|
||||
if runtime_kwargs:
|
||||
merged.update(runtime_kwargs)
|
||||
return merged
|
||||
|
||||
@abstractmethod
|
||||
def load_bytes(self, data: bytes) -> _T:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -32,9 +32,43 @@ atexit.register(global_thread_pool.shutdown)
|
||||
|
||||
MEDIA_CONNECTOR_REGISTRY = ExtensionManager()
|
||||
|
||||
MODALITY_IO_MAP: dict[str, type[MediaIO]] = {
|
||||
"audio": AudioMediaIO,
|
||||
"image": ImageMediaIO,
|
||||
"video": VideoMediaIO,
|
||||
}
|
||||
|
||||
|
||||
def merge_media_io_kwargs(
|
||||
defaults: dict[str, dict[str, Any]] | None,
|
||||
overrides: dict[str, dict[str, Any]] | None,
|
||||
) -> dict[str, dict[str, Any]] | None:
|
||||
"""Merge config-level and per-request media_io_kwargs per modality.
|
||||
|
||||
Each modality key is merged using the corresponding MediaIO subclass's
|
||||
``merge_kwargs``, which may apply modality-specific logic (e.g.
|
||||
VideoMediaIO clears cross-dependent fps/num_frames fields).
|
||||
"""
|
||||
if not defaults and not overrides:
|
||||
return None
|
||||
all_keys = set(defaults or {}) | set(overrides or {})
|
||||
merged = {}
|
||||
for key in all_keys:
|
||||
io_cls = MODALITY_IO_MAP.get(key, MediaIO)
|
||||
merged[key] = io_cls.merge_kwargs(
|
||||
(defaults or {}).get(key),
|
||||
(overrides or {}).get(key),
|
||||
)
|
||||
return merged or None
|
||||
|
||||
|
||||
@MEDIA_CONNECTOR_REGISTRY.register("http")
|
||||
class MediaConnector:
|
||||
"""Configuration values can be user-provided either by --media-io-kwargs or
|
||||
by the runtime API field "media_io_kwargs". Ensure proper validation and
|
||||
error handling.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None = None,
|
||||
|
||||
@@ -15,12 +15,18 @@ from .base import MediaIO, MediaWithBytes
|
||||
|
||||
|
||||
class ImageMediaIO(MediaIO[Image.Image]):
|
||||
"""Configuration values can be user-provided either by --media-io-kwargs or
|
||||
by the runtime API field "media_io_kwargs". Ensure proper validation and
|
||||
error handling.
|
||||
"""
|
||||
|
||||
def __init__(self, image_mode: str = "RGB", **kwargs) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.image_mode = image_mode
|
||||
# `kwargs` contains custom arguments from
|
||||
# --media-io-kwargs for this modality.
|
||||
# --media-io-kwargs for this modality, merged with
|
||||
# per-request runtime media_io_kwargs via merge_kwargs().
|
||||
# They can be passed to the underlying
|
||||
# media loaders (e.g. custom implementations)
|
||||
# for flexible control.
|
||||
@@ -88,6 +94,13 @@ class ImageMediaIO(MediaIO[Image.Image]):
|
||||
|
||||
|
||||
class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
|
||||
"""Image embedding MediaIO implementation.
|
||||
|
||||
Configuration values can be user-provided either by --media-io-kwargs or
|
||||
by the runtime API field "media_io_kwargs". Ensure proper validation and
|
||||
error handling.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
|
||||
@@ -17,6 +17,28 @@ from .image import ImageMediaIO
|
||||
|
||||
|
||||
class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
|
||||
"""Configuration values can be user-provided either by --media-io-kwargs or
|
||||
by the runtime API field "media_io_kwargs". Ensure proper validation and
|
||||
error handling.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def merge_kwargs(
|
||||
cls,
|
||||
default_kwargs: dict[str, Any] | None,
|
||||
runtime_kwargs: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
merged = super().merge_kwargs(default_kwargs, runtime_kwargs)
|
||||
# fps and num_frames interact with each other, so if either is
|
||||
# overridden at request time, wipe the other from defaults to
|
||||
# avoid unintuitive cross-field interactions.
|
||||
if runtime_kwargs:
|
||||
if "num_frames" in runtime_kwargs and "fps" not in runtime_kwargs:
|
||||
merged.pop("fps", None)
|
||||
elif "fps" in runtime_kwargs and "num_frames" not in runtime_kwargs:
|
||||
merged.pop("num_frames", None)
|
||||
return merged
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_io: ImageMediaIO,
|
||||
@@ -28,7 +50,8 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
|
||||
self.image_io = image_io
|
||||
self.num_frames = num_frames
|
||||
# `kwargs` contains custom arguments from
|
||||
# --media-io-kwargs for this modality.
|
||||
# --media-io-kwargs for this modality, merged with
|
||||
# per-request runtime media_io_kwargs via merge_kwargs().
|
||||
# They can be passed to the underlying
|
||||
# media loaders (e.g. custom implementations)
|
||||
# for flexible control.
|
||||
|
||||
Reference in New Issue
Block a user