Signed-off-by: juanpas97 <juan.perezalgaba@gmail.com> Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
683 lines
23 KiB
Python
683 lines
23 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
import asyncio
|
|
import atexit
|
|
import contextlib
|
|
import hashlib
|
|
import os
|
|
import tempfile
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
from typing import Any, TypeVar
|
|
from urllib.request import url2pathname
|
|
|
|
import aiohttp
|
|
import numpy as np
|
|
import numpy.typing as npt
|
|
import requests
|
|
import torch
|
|
from PIL import Image, UnidentifiedImageError
|
|
from urllib3.util import Url, parse_url
|
|
|
|
import vllm.envs as envs
|
|
from vllm.connections import (
|
|
HTTPConnection,
|
|
MediaDownloadSizeExceededError,
|
|
MediaRequestDownloadBudget,
|
|
MediaRequestDownloadSizeExceededError,
|
|
global_http_connection,
|
|
)
|
|
from vllm.exceptions import VLLMUnprocessableEntityError, VLLMValidationError
|
|
from vllm.logger import init_logger
|
|
from vllm.multimodal.video import get_video_loader_backend_for_processor
|
|
from vllm.utils.registry import ExtensionManager
|
|
|
|
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
|
|
from .base import MediaDecodedBudget, MediaIO, MediaWithBytes
|
|
from .image import ImageEmbeddingMediaIO, ImageMediaIO
|
|
from .video import VideoEmbeddingMediaIO, VideoMediaIO
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
_M = TypeVar("_M")
|
|
|
|
global_thread_pool = ThreadPoolExecutor(
|
|
max_workers=envs.VLLM_MEDIA_LOADING_THREAD_COUNT
|
|
)
|
|
atexit.register(global_thread_pool.shutdown)
|
|
|
|
MEDIA_CONNECTOR_REGISTRY = ExtensionManager()
|
|
|
|
MODALITY_IO_MAP: dict[str, type[MediaIO]] = {
|
|
"audio": AudioMediaIO,
|
|
"image": ImageMediaIO,
|
|
"video": VideoMediaIO,
|
|
}
|
|
|
|
|
|
def _wrap_media_fetch_error(
|
|
url: str, exc: Exception
|
|
) -> VLLMUnprocessableEntityError | Exception:
|
|
"""Convert media fetch exceptions to VLLMUnprocessableEntityError.
|
|
|
|
This handles HTTP errors that indicate the media resource is invalid
|
|
(4xx responses except 408/429, malformed URLs) and converts them to a
|
|
422 Unprocessable Entity error instead of 500.
|
|
|
|
Transient errors (5xx, 408, 429, DNS failures, connection errors,
|
|
timeouts) are returned as-is to allow retry logic to handle them
|
|
appropriately.
|
|
|
|
Returns:
|
|
VLLMUnprocessableEntityError for permanent client errors (4xx except
|
|
408/429, invalid URL)
|
|
Original exception for transient errors (5xx, 408, 429, network blips)
|
|
or other exceptions
|
|
|
|
"""
|
|
if isinstance(exc, VLLMValidationError):
|
|
return exc
|
|
|
|
if isinstance(exc, aiohttp.ClientResponseError):
|
|
if exc.status in (408, 429):
|
|
return exc
|
|
if exc.status < 500:
|
|
return VLLMUnprocessableEntityError(
|
|
f"Failed to fetch media from URL: HTTP {exc.status} error",
|
|
parameter="image_url",
|
|
value=url,
|
|
)
|
|
return exc
|
|
|
|
if isinstance(exc, requests.exceptions.HTTPError):
|
|
if exc.response is not None:
|
|
status_code = exc.response.status_code
|
|
if status_code in (408, 429):
|
|
return exc
|
|
if status_code < 500:
|
|
return VLLMUnprocessableEntityError(
|
|
f"Failed to fetch media from URL: HTTP {status_code} error",
|
|
parameter="image_url",
|
|
value=url,
|
|
)
|
|
return exc
|
|
|
|
if isinstance(exc, requests.exceptions.InvalidURL):
|
|
return VLLMUnprocessableEntityError(
|
|
"Failed to fetch media from URL: Invalid URL format",
|
|
parameter="image_url",
|
|
value=url,
|
|
)
|
|
|
|
if isinstance(
|
|
exc, (MediaDownloadSizeExceededError, MediaRequestDownloadSizeExceededError)
|
|
):
|
|
return VLLMUnprocessableEntityError(
|
|
f"Failed to fetch media from URL: {exc}",
|
|
parameter="image_url",
|
|
value=url,
|
|
)
|
|
|
|
if isinstance(exc, ValueError):
|
|
return VLLMUnprocessableEntityError(
|
|
"Failed to fetch media from URL: Invalid URL",
|
|
parameter="image_url",
|
|
value=url,
|
|
)
|
|
return exc
|
|
|
|
|
|
def merge_media_io_kwargs(
|
|
defaults: dict[str, dict[str, Any]] | None,
|
|
overrides: dict[str, dict[str, Any]] | None,
|
|
) -> dict[str, dict[str, Any]] | None:
|
|
"""Merge config-level and per-request media_io_kwargs per modality.
|
|
|
|
Each modality key is merged using the corresponding MediaIO subclass's
|
|
``merge_kwargs``, which may apply modality-specific logic (e.g.
|
|
VideoMediaIO clears cross-dependent fps/num_frames fields).
|
|
"""
|
|
if not defaults and not overrides:
|
|
return None
|
|
all_keys = set(defaults or {}) | set(overrides or {})
|
|
merged = {}
|
|
for key in all_keys:
|
|
io_cls = MODALITY_IO_MAP.get(key, MediaIO)
|
|
merged[key] = io_cls.merge_kwargs(
|
|
(defaults or {}).get(key),
|
|
(overrides or {}).get(key),
|
|
)
|
|
return merged or None
|
|
|
|
|
|
@MEDIA_CONNECTOR_REGISTRY.register("http")
|
|
class MediaConnector:
|
|
"""Configuration values can be user-provided either by --media-io-kwargs or
|
|
by the runtime API field "media_io_kwargs". Ensure proper validation and
|
|
error handling.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
media_io_kwargs: dict[str, dict[str, Any]] | None = None,
|
|
connection: HTTPConnection = global_http_connection,
|
|
*,
|
|
allowed_local_media_path: str = "",
|
|
allowed_media_domains: list[str] | None = None,
|
|
decoded_budget: MediaDecodedBudget | None = None,
|
|
download_budget: MediaRequestDownloadBudget | None = None,
|
|
) -> None:
|
|
"""Args:
|
|
media_io_kwargs: Additional args passed to process media
|
|
inputs, keyed by modalities. For example,
|
|
to set num_frames for video, set
|
|
`--media-io-kwargs '{"video":{"num_frames":40}}'`
|
|
connection: HTTP connection client to download media contents.
|
|
allowed_local_media_path: A local directory to load media files from.
|
|
allowed_media_domains: If set, only media URLs that belong to this
|
|
domain can be used for multi-modal inputs.
|
|
|
|
"""
|
|
super().__init__()
|
|
|
|
self.media_io_kwargs: dict[str, dict[str, Any]] = (
|
|
media_io_kwargs if media_io_kwargs else {}
|
|
)
|
|
self.connection = connection
|
|
|
|
if allowed_local_media_path:
|
|
allowed_local_media_path_ = Path(allowed_local_media_path).resolve()
|
|
|
|
if not allowed_local_media_path_.exists():
|
|
raise ValueError(
|
|
"Invalid `--allowed-local-media-path`: The path "
|
|
f"{allowed_local_media_path_} does not exist."
|
|
)
|
|
if not allowed_local_media_path_.is_dir():
|
|
raise ValueError(
|
|
"Invalid `--allowed-local-media-path`: The path "
|
|
f"{allowed_local_media_path_} must be a directory."
|
|
)
|
|
else:
|
|
allowed_local_media_path_ = None
|
|
|
|
self.allowed_local_media_path = allowed_local_media_path_
|
|
if allowed_media_domains is None:
|
|
allowed_media_domains = []
|
|
self.allowed_media_domains = allowed_media_domains
|
|
self.decoded_budget = decoded_budget
|
|
self.download_budget = download_budget
|
|
|
|
# Media download cache (opt-in via VLLM_MEDIA_CACHE)
|
|
self._media_cache_dir: str | None = None
|
|
self._media_cache_max_bytes: int = 0
|
|
self._media_cache_ttl_secs: float = 0
|
|
media_cache = envs.VLLM_MEDIA_CACHE
|
|
if media_cache:
|
|
try:
|
|
os.makedirs(media_cache, exist_ok=True)
|
|
# Verify the directory is writable before enabling caching
|
|
with tempfile.NamedTemporaryFile(dir=media_cache, delete=True):
|
|
pass
|
|
self._media_cache_dir = media_cache
|
|
self._media_cache_max_bytes = (
|
|
envs.VLLM_MEDIA_CACHE_MAX_SIZE_MB * 1024 * 1024
|
|
)
|
|
self._media_cache_ttl_secs = envs.VLLM_MEDIA_CACHE_TTL_HOURS * 3600
|
|
logger.info(
|
|
"Media cache enabled at %s (max %d MB, TTL %s hours)",
|
|
media_cache,
|
|
envs.VLLM_MEDIA_CACHE_MAX_SIZE_MB,
|
|
envs.VLLM_MEDIA_CACHE_TTL_HOURS,
|
|
)
|
|
except OSError:
|
|
logger.warning(
|
|
"VLLM_MEDIA_CACHE path %s is not writable, media caching disabled",
|
|
media_cache,
|
|
)
|
|
|
|
def _image_io(self, image_mode: str | None = "RGB") -> ImageMediaIO:
|
|
return ImageMediaIO(
|
|
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {})),
|
|
decoded_budget=self.decoded_budget,
|
|
)
|
|
|
|
def _get_cached_bytes(self, url: str) -> bytes | None:
|
|
"""Return cached bytes for a URL, or None if not cached/expired."""
|
|
if not self._media_cache_dir:
|
|
return None
|
|
cache_path = self._media_cache_path(url)
|
|
# Check TTL
|
|
try:
|
|
age = time.time() - cache_path.stat().st_mtime
|
|
except OSError:
|
|
return None
|
|
if age > self._media_cache_ttl_secs:
|
|
cache_path.unlink(missing_ok=True)
|
|
return None
|
|
# Touch mtime for LRU ordering
|
|
try:
|
|
cache_path.touch()
|
|
return cache_path.read_bytes()
|
|
except OSError:
|
|
return None
|
|
|
|
def _put_cached_bytes(self, url: str, data: bytes) -> None:
|
|
"""Store downloaded bytes and evict if over budget."""
|
|
if not self._media_cache_dir:
|
|
return
|
|
cache_path = self._media_cache_path(url)
|
|
# Atomic write via temp file + rename
|
|
tmp_path = None
|
|
try:
|
|
with tempfile.NamedTemporaryFile(
|
|
mode="wb", dir=self._media_cache_dir, delete=False
|
|
) as tmp_file:
|
|
tmp_file.write(data)
|
|
tmp_path = tmp_file.name
|
|
os.rename(tmp_path, str(cache_path))
|
|
except OSError:
|
|
# Another process beat us or disk issue
|
|
if tmp_path is not None:
|
|
with contextlib.suppress(OSError):
|
|
os.remove(tmp_path)
|
|
return
|
|
self._maybe_evict(exclude=cache_path)
|
|
|
|
def _maybe_evict(self, exclude: Path | None = None) -> None:
|
|
"""Evict expired entries first, then LRU until under size limit."""
|
|
cache_dir = Path(self._media_cache_dir) # type: ignore[arg-type]
|
|
entries = []
|
|
expired = []
|
|
total_size = 0
|
|
now = time.time()
|
|
for f in cache_dir.iterdir():
|
|
if f.name.startswith("."):
|
|
continue
|
|
try:
|
|
stat = f.stat()
|
|
except OSError:
|
|
continue
|
|
age = now - stat.st_mtime
|
|
if age > self._media_cache_ttl_secs:
|
|
expired.append(f)
|
|
continue
|
|
total_size += stat.st_size
|
|
# Never evict the file we just wrote
|
|
if exclude is not None and f.name == exclude.name:
|
|
continue
|
|
entries.append((stat.st_mtime, stat.st_size, f))
|
|
|
|
# Evict items according to LRU policy
|
|
entries.sort(key=lambda e: e[0], reverse=True)
|
|
while total_size > self._media_cache_max_bytes and entries:
|
|
mtime, size, f = entries.pop()
|
|
expired.append(f)
|
|
total_size -= size
|
|
|
|
for f in expired:
|
|
f.unlink(missing_ok=True)
|
|
|
|
def _media_cache_path(self, url: str) -> Path:
|
|
url_hash = hashlib.sha256(url.encode()).hexdigest()[:20]
|
|
ext = Path(url.split("?", 1)[0]).suffix or ""
|
|
return Path(self._media_cache_dir) / f"{url_hash}{ext}" # type: ignore[arg-type]
|
|
|
|
def _load_data_url(
|
|
self,
|
|
url: str,
|
|
media_io: MediaIO[_M],
|
|
) -> _M: # type: ignore[type-var]
|
|
# Format per RFC 2397:
|
|
# data:[<mediatype>][;<param>=<value>]*[;base64],<data>
|
|
data_spec, sep, data = url[5:].partition(",")
|
|
if not sep:
|
|
msg = f"Invalid data URL {url[:32]!r}: missing ',' separator."
|
|
raise ValueError(msg)
|
|
|
|
media_type, sep, encoding = data_spec.rpartition(";")
|
|
if not sep or encoding != "base64":
|
|
msg = "Only base64 data URLs are supported for now."
|
|
raise NotImplementedError(msg)
|
|
|
|
media_type = media_type.partition(";")[0]
|
|
return media_io.load_base64(media_type, data)
|
|
|
|
def _load_file_url(
|
|
self,
|
|
url_spec: Url,
|
|
media_io: MediaIO[_M],
|
|
) -> _M: # type: ignore[type-var]
|
|
allowed_local_media_path = self.allowed_local_media_path
|
|
if allowed_local_media_path is None:
|
|
raise RuntimeError(
|
|
"Cannot load local files without `--allowed-local-media-path`."
|
|
)
|
|
|
|
url_spec_path = url_spec.path or ""
|
|
url_spec_netloc = url_spec.netloc or ""
|
|
filepath = Path(url2pathname(url_spec_netloc + url_spec_path))
|
|
if allowed_local_media_path not in filepath.resolve().parents:
|
|
raise ValueError(
|
|
f"The file path {filepath} must be a subpath "
|
|
f"of `--allowed-local-media-path {allowed_local_media_path}`."
|
|
)
|
|
|
|
return media_io.load_file(filepath)
|
|
|
|
def _assert_url_in_allowed_media_domains(self, url_spec: Url) -> None:
|
|
if (
|
|
self.allowed_media_domains
|
|
and url_spec.hostname not in self.allowed_media_domains
|
|
):
|
|
raise ValueError(
|
|
f"The URL must be from one of the allowed domains: "
|
|
f"{self.allowed_media_domains}. Input URL domain: "
|
|
f"{url_spec.hostname}"
|
|
)
|
|
|
|
def load_from_url(
|
|
self,
|
|
url: str,
|
|
media_io: MediaIO[_M],
|
|
*,
|
|
fetch_timeout: int | None = None,
|
|
) -> _M: # type: ignore[type-var]
|
|
if url[:5].lower() == "data:":
|
|
return self._load_data_url(url, media_io)
|
|
|
|
url_spec = parse_url(url)
|
|
|
|
if url_spec.scheme and url_spec.scheme.startswith("http"):
|
|
self._assert_url_in_allowed_media_domains(url_spec)
|
|
max_bytes = media_io.get_max_bytes()
|
|
|
|
cached = self._get_cached_bytes(url)
|
|
if cached is not None:
|
|
return media_io.load_bytes(cached)
|
|
|
|
connection = self.connection
|
|
budget_kwargs = (
|
|
{"request_budget": self.download_budget}
|
|
if self.download_budget is not None
|
|
else {}
|
|
)
|
|
try:
|
|
data = connection.get_bytes(
|
|
url_spec.url,
|
|
timeout=fetch_timeout,
|
|
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
|
|
max_bytes=max_bytes,
|
|
**budget_kwargs,
|
|
)
|
|
except Exception as e:
|
|
wrapped = _wrap_media_fetch_error(url, e)
|
|
if isinstance(wrapped, VLLMUnprocessableEntityError):
|
|
raise wrapped from e
|
|
raise
|
|
|
|
self._put_cached_bytes(url, data)
|
|
return media_io.load_bytes(data)
|
|
|
|
if url_spec.scheme == "file":
|
|
return self._load_file_url(url_spec, media_io)
|
|
|
|
msg = "The URL must be either a HTTP, data or file URL."
|
|
raise ValueError(msg)
|
|
|
|
async def load_from_url_async(
|
|
self,
|
|
url: str,
|
|
media_io: MediaIO[_M],
|
|
*,
|
|
fetch_timeout: int | None = None,
|
|
) -> _M:
|
|
loop = asyncio.get_running_loop()
|
|
|
|
if url[:5].lower() == "data:":
|
|
future = loop.run_in_executor(
|
|
global_thread_pool, self._load_data_url, url, media_io
|
|
)
|
|
return await future
|
|
|
|
url_spec = parse_url(url)
|
|
|
|
if url_spec.scheme and url_spec.scheme.startswith("http"):
|
|
self._assert_url_in_allowed_media_domains(url_spec)
|
|
max_bytes = media_io.get_max_bytes()
|
|
|
|
cached = await loop.run_in_executor(
|
|
global_thread_pool, self._get_cached_bytes, url
|
|
)
|
|
if cached is not None:
|
|
future = loop.run_in_executor(
|
|
global_thread_pool, media_io.load_bytes, cached
|
|
)
|
|
return await future
|
|
|
|
connection = self.connection
|
|
budget_kwargs = (
|
|
{"request_budget": self.download_budget}
|
|
if self.download_budget is not None
|
|
else {}
|
|
)
|
|
try:
|
|
data = await connection.async_get_bytes(
|
|
url_spec.url,
|
|
timeout=fetch_timeout,
|
|
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
|
|
max_bytes=max_bytes,
|
|
**budget_kwargs,
|
|
)
|
|
except Exception as e:
|
|
wrapped = _wrap_media_fetch_error(url, e)
|
|
if isinstance(wrapped, VLLMUnprocessableEntityError):
|
|
raise wrapped from e
|
|
raise
|
|
|
|
await loop.run_in_executor(
|
|
global_thread_pool, self._put_cached_bytes, url, data
|
|
)
|
|
future = loop.run_in_executor(global_thread_pool, media_io.load_bytes, data)
|
|
return await future
|
|
|
|
if url_spec.scheme == "file":
|
|
future = loop.run_in_executor(
|
|
global_thread_pool, self._load_file_url, url_spec, media_io
|
|
)
|
|
return await future
|
|
msg = "The URL must be either a HTTP, data or file URL."
|
|
raise ValueError(msg)
|
|
|
|
def fetch_audio(
|
|
self,
|
|
audio_url: str,
|
|
) -> tuple[np.ndarray, int | float]:
|
|
"""Load audio from a URL."""
|
|
audio_io = AudioMediaIO(
|
|
decoded_budget=self.decoded_budget, **self.media_io_kwargs.get("audio", {})
|
|
)
|
|
|
|
return self.load_from_url(
|
|
audio_url,
|
|
audio_io,
|
|
fetch_timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
|
|
)
|
|
|
|
async def fetch_audio_async(
|
|
self,
|
|
audio_url: str,
|
|
) -> tuple[np.ndarray, int | float]:
|
|
"""Asynchronously fetch audio from a URL."""
|
|
audio_io = AudioMediaIO(
|
|
decoded_budget=self.decoded_budget, **self.media_io_kwargs.get("audio", {})
|
|
)
|
|
|
|
return await self.load_from_url_async(
|
|
audio_url,
|
|
audio_io,
|
|
fetch_timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
|
|
)
|
|
|
|
def fetch_image(
|
|
self,
|
|
image_url: str,
|
|
*,
|
|
image_mode: str | None = "RGB",
|
|
) -> Image.Image:
|
|
"""Load a PIL image from an HTTP or base64 data URL.
|
|
|
|
By default, the image is converted into RGB format. Set
|
|
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
|
|
original image mode (e.g. preserving the alpha channel).
|
|
"""
|
|
image_io = self._image_io(image_mode)
|
|
|
|
try:
|
|
return self.load_from_url(
|
|
image_url,
|
|
image_io,
|
|
fetch_timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
|
|
)
|
|
except UnidentifiedImageError as e:
|
|
# convert to ValueError to be properly caught upstream
|
|
raise ValueError(str(e)) from e
|
|
|
|
async def fetch_image_async(
|
|
self,
|
|
image_url: str,
|
|
*,
|
|
image_mode: str | None = "RGB",
|
|
) -> Image.Image:
|
|
"""Asynchronously load a PIL image from an HTTP or base64 data URL.
|
|
|
|
By default, the image is converted into RGB format. Set
|
|
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
|
|
original image mode (e.g. preserving the alpha channel).
|
|
"""
|
|
image_io = self._image_io(image_mode)
|
|
|
|
try:
|
|
return await self.load_from_url_async(
|
|
image_url,
|
|
image_io,
|
|
fetch_timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
|
|
)
|
|
except UnidentifiedImageError as e:
|
|
# convert to ValueError to be properly caught upstream
|
|
raise ValueError(str(e)) from e
|
|
|
|
def fetch_video(
|
|
self,
|
|
video_url: str,
|
|
*,
|
|
image_mode: str | None = "RGB",
|
|
video_processor: str | None = None,
|
|
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
|
|
"""Load video from an HTTP or base64 data URL."""
|
|
image_io = self._image_io(image_mode)
|
|
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
|
|
if "video_backend" not in video_io_kwargs and (
|
|
video_backend := get_video_loader_backend_for_processor(video_processor)
|
|
):
|
|
video_io_kwargs["video_backend"] = video_backend
|
|
video_io = VideoMediaIO(image_io, **video_io_kwargs)
|
|
|
|
return self.load_from_url(
|
|
video_url,
|
|
video_io,
|
|
fetch_timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
|
|
)
|
|
|
|
async def fetch_video_async(
|
|
self,
|
|
video_url: str,
|
|
*,
|
|
image_mode: str | None = "RGB",
|
|
video_processor: str | None = None,
|
|
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
|
|
"""Asynchronously load video from an HTTP or base64 data URL.
|
|
|
|
By default, the image is converted into RGB format. Set
|
|
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
|
|
original image mode (e.g. preserving the alpha channel).
|
|
"""
|
|
image_io = self._image_io(image_mode)
|
|
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
|
|
if "video_backend" not in video_io_kwargs and (
|
|
video_backend := get_video_loader_backend_for_processor(video_processor)
|
|
):
|
|
video_io_kwargs["video_backend"] = video_backend
|
|
video_io = VideoMediaIO(image_io, **video_io_kwargs)
|
|
|
|
return await self.load_from_url_async(
|
|
video_url,
|
|
video_io,
|
|
fetch_timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
|
|
)
|
|
|
|
def fetch_image_embedding(
|
|
self,
|
|
data: str,
|
|
) -> torch.Tensor:
|
|
"""Load image embedding from a URL."""
|
|
image_embedding_io = ImageEmbeddingMediaIO()
|
|
|
|
return image_embedding_io.load_base64("", data)
|
|
|
|
async def fetch_image_embedding_async(
|
|
self,
|
|
data: str,
|
|
) -> torch.Tensor:
|
|
"""Asynchronously load image embedding from a URL."""
|
|
image_embedding_io = ImageEmbeddingMediaIO()
|
|
loop = asyncio.get_running_loop()
|
|
|
|
return await loop.run_in_executor(
|
|
global_thread_pool, image_embedding_io.load_base64, "", data
|
|
)
|
|
|
|
def fetch_audio_embedding(
|
|
self,
|
|
data: str,
|
|
) -> torch.Tensor:
|
|
"""Load audio embedding from a URL."""
|
|
audio_embedding_io = AudioEmbeddingMediaIO()
|
|
|
|
return audio_embedding_io.load_base64("", data)
|
|
|
|
async def fetch_audio_embedding_async(
|
|
self,
|
|
data: str,
|
|
) -> torch.Tensor:
|
|
"""Asynchronously load audio embedding from a URL."""
|
|
audio_embedding_io = AudioEmbeddingMediaIO()
|
|
loop = asyncio.get_running_loop()
|
|
|
|
return await loop.run_in_executor(
|
|
global_thread_pool, audio_embedding_io.load_base64, "", data
|
|
)
|
|
|
|
def fetch_video_embedding(
|
|
self,
|
|
data: str,
|
|
) -> torch.Tensor:
|
|
"""Load video embedding from a URL."""
|
|
video_embedding_io = VideoEmbeddingMediaIO()
|
|
|
|
return video_embedding_io.load_base64("", data)
|
|
|
|
async def fetch_video_embedding_async(
|
|
self,
|
|
data: str,
|
|
) -> torch.Tensor:
|
|
"""Asynchronously load video embedding from a URL."""
|
|
video_embedding_io = VideoEmbeddingMediaIO()
|
|
loop = asyncio.get_running_loop()
|
|
|
|
return await loop.run_in_executor(
|
|
global_thread_pool, video_embedding_io.load_base64, "", data
|
|
)
|