[Security] Bound decoded and downloaded media per request (#61226)

Signed-off-by: juanpas97 <juan.perezalgaba@gmail.com>
Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Juan Pérez de Algaba
2026-10-13 10:17:14 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 877f6df889
commit 7173c64e2e
16 changed files with 565 additions and 50 deletions
+57 -8
View File
@@ -3,6 +3,7 @@
import asyncio
import functools
import threading
import time
from collections.abc import Callable, Coroutine, Mapping, MutableMapping
from pathlib import Path
@@ -60,6 +61,33 @@ class MediaDownloadSizeExceededError(ValueError):
super().__init__(message)
class MediaRequestDownloadSizeExceededError(ValueError):
"""Raised when all HTTP media downloads in one request exceed the limit."""
class MediaRequestDownloadBudget:
"""Thread-safe total of HTTP media bytes received for one request."""
def __init__(self) -> None:
self.used = 0
self._lock = threading.Lock()
def charge(self, size: int) -> None:
limit_mb = envs.VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB
if limit_mb <= 0:
return
limit = limit_mb * MiB_bytes
with self._lock:
received = self.used + size
if received > limit:
raise MediaRequestDownloadSizeExceededError(
"Request media downloads exceed "
f"VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB={limit_mb:g} "
f"({received} bytes received)."
)
self.used = received
def _get_media_download_limit_bytes() -> int | None:
max_size_mb = envs.VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB
if max_size_mb <= 0:
@@ -105,19 +133,26 @@ def _append_chunk(
received_bytes: int,
max_bytes: int | None,
media_download_limit: bool,
request_budget: MediaRequestDownloadBudget | None = None,
) -> int:
if not chunk:
return received_bytes
received_bytes += len(chunk)
if max_bytes is not None and received_bytes > max_bytes:
_raise_size_exceeded(max_bytes, received_bytes, media_download_limit)
if request_budget is not None:
request_budget.charge(len(chunk))
chunks.append(chunk)
return received_bytes
def _read_response_bytes(response: requests.Response, max_bytes: int | None) -> bytes:
def _read_response_bytes(
response: requests.Response,
max_bytes: int | None,
request_budget: MediaRequestDownloadBudget | None = None,
) -> bytes:
max_bytes, media_download_limit = _get_response_size_limit(max_bytes)
if max_bytes is None:
if max_bytes is None and request_budget is None:
return response.content
_check_content_length(
@@ -129,16 +164,23 @@ def _read_response_bytes(response: requests.Response, max_bytes: int | None) ->
received_bytes = 0
for chunk in response.iter_content(_RESPONSE_READ_CHUNK_SIZE):
received_bytes = _append_chunk(
chunks, chunk, received_bytes, max_bytes, media_download_limit
chunks,
chunk,
received_bytes,
max_bytes,
media_download_limit,
request_budget,
)
return b"".join(chunks)
async def _async_read_response_bytes(
response: aiohttp.ClientResponse, max_bytes: int | None
response: aiohttp.ClientResponse,
max_bytes: int | None,
request_budget: MediaRequestDownloadBudget | None = None,
) -> bytes:
max_bytes, media_download_limit = _get_response_size_limit(max_bytes)
if max_bytes is None:
if max_bytes is None and request_budget is None:
return await response.read()
_check_content_length(response.content_length, max_bytes, media_download_limit)
@@ -146,7 +188,12 @@ async def _async_read_response_bytes(
received_bytes = 0
async for chunk in response.content.iter_chunked(_RESPONSE_READ_CHUNK_SIZE):
received_bytes = _append_chunk(
chunks, chunk, received_bytes, max_bytes, media_download_limit
chunks,
chunk,
received_bytes,
max_bytes,
media_download_limit,
request_budget,
)
return b"".join(chunks)
@@ -407,6 +454,7 @@ class HTTPConnection:
timeout: float | None = None,
allow_redirects: bool = True,
max_bytes: int | None = None,
request_budget: MediaRequestDownloadBudget | None = None,
) -> bytes:
with self.get_response(
url,
@@ -416,7 +464,7 @@ class HTTPConnection:
) as r:
r.raise_for_status()
return _read_response_bytes(r, max_bytes)
return _read_response_bytes(r, max_bytes, request_budget)
@_async_retry
async def async_get_bytes(
@@ -426,6 +474,7 @@ class HTTPConnection:
timeout: float | None = None,
allow_redirects: bool = True,
max_bytes: int | None = None,
request_budget: MediaRequestDownloadBudget | None = None,
) -> bytes:
async with await self.get_async_response(
url,
@@ -434,7 +483,7 @@ class HTTPConnection:
) as r:
r.raise_for_status()
return await _async_read_response_bytes(r, max_bytes)
return await _async_read_response_bytes(r, max_bytes, request_budget)
def get_text(self, url: str, *, timeout: float | None = None) -> str:
with self.get_response(url, timeout=timeout) as r:
+13
View File
@@ -78,6 +78,7 @@ if TYPE_CHECKING:
VLLM_MEDIA_CACHE_TTL_HOURS: float = 24
VLLM_MEDIA_FETCH_MAX_RETRIES: int = 3
VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB: int = 256
VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB: int = 256
VLLM_MEDIA_URL_ALLOW_REDIRECTS: bool = True
VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8
VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25
@@ -86,6 +87,7 @@ if TYPE_CHECKING:
VLLM_MAX_AUDIO_PREPROCESS_WORKERS: int = max(1, min(os.cpu_count() or 1, 2))
VLLM_MAX_EMBED_DECODE_BYTES: int = 2_147_483_648
VLLM_MAX_IMAGE_PIXELS: int = 178_956_970
VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES: int = 715_827_880
VLLM_VIDEO_LOADER_BACKEND: str = "opencv"
VLLM_MEDIA_CONNECTOR: str = "http"
VLLM_TARGET_DEVICE: str = "cuda"
@@ -1063,6 +1065,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_MAX_IMAGE_PIXELS": lambda: int(
os.getenv("VLLM_MAX_IMAGE_PIXELS", "178956970")
),
# Maximum aggregate decoded bytes across media in one request.
# The default allows one image at the per-image pixel limit at 4 bytes
# per pixel. Set to 0 to disable.
"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES": lambda: int(
os.getenv("VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES", "715827880")
),
"VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB": lambda: int(
os.getenv("VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB", "256")
),
# Backend for Video IO — selects the frame-sampling algorithm.
# - "opencv": uniform sampling.
# - "opencv_dynamic": duration-aware dynamic sampling.
@@ -2412,6 +2423,8 @@ def compile_factors() -> dict[str, object]:
"VLLM_MAX_AUDIO_PREPROCESS_WORKERS",
"VLLM_MAX_EMBED_DECODE_BYTES",
"VLLM_MAX_IMAGE_PIXELS",
"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES",
"VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB",
"VLLM_VIDEO_LOADER_BACKEND",
"VLLM_MEDIA_CONNECTOR",
"VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME",
+2 -1
View File
@@ -1,13 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
from .base import MediaIO, MediaWithBytes
from .base import MediaDecodedBudget, MediaIO, MediaWithBytes
from .connector import MEDIA_CONNECTOR_REGISTRY, MediaConnector
from .image import ImageEmbeddingMediaIO, ImageMediaIO
from .video import VIDEO_LOADER_REGISTRY, VideoMediaIO
__all__ = [
"MediaIO",
"MediaDecodedBudget",
"MediaWithBytes",
"AudioEmbeddingMediaIO",
"AudioMediaIO",
+44 -16
View File
@@ -23,7 +23,7 @@ from vllm.utils.sparse_utils import (
safe_to_dense,
)
from .base import MediaIO
from .base import MediaDecodedBudget, MediaIO
logger = init_logger(__name__)
@@ -527,8 +527,11 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
error handling.
"""
def __init__(self, **kwargs) -> None:
def __init__(
self, *, decoded_budget: MediaDecodedBudget | None = None, **kwargs
) -> None:
super().__init__()
self.decoded_budget = decoded_budget
# Allow per-server override of the audio decoding backend, e.g.:
# --media-io-kwargs '{"audio": {"audio_backend": "torchcodec"}}'
@@ -564,13 +567,44 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
self._validate_encoded_size(len(data))
return load_audio(
BytesIO(data),
sr=None,
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES,
backend=self.audio_backend,
)
return self._load_with_budget(BytesIO(data))
def _load_with_budget(self, source: BytesIO | Path) -> tuple[npt.NDArray, float]:
budget = self.decoded_budget
if budget is None:
return load_audio(
source,
sr=None,
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES,
backend=self.audio_backend,
)
max_decode_bytes = envs.VLLM_MAX_AUDIO_DECODE_BYTES
reserved = budget.reserve(max_decode_bytes)
decoded_bytes = 0
try:
try:
audio = load_audio(
source,
sr=None,
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
max_decode_bytes=reserved,
backend=self.audio_backend,
)
except ValueError as exc:
if reserved < max_decode_bytes and "VLLM_MAX_AUDIO_DECODE_BYTES" in str(
exc
):
raise ValueError(
"Request decoded media exceeds "
"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES."
) from exc
raise
decoded_bytes = audio[0].nbytes
return audio
finally:
budget.settle(reserved, decoded_bytes)
def load_base64(
self,
@@ -588,13 +622,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]:
self._validate_encoded_size(filepath.stat().st_size)
return load_audio(
filepath,
sr=None,
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES,
backend=self.audio_backend,
)
return self._load_with_budget(filepath)
def encode_base64(
self,
+71
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from abc import ABC, abstractmethod
from collections.abc import Iterable, Iterator
from dataclasses import dataclass, field
@@ -9,9 +10,79 @@ from typing import Any, Generic, TypeVar, cast
import numpy as np
import vllm.envs as envs
_T = TypeVar("_T")
class MediaDecodedBudget:
"""Thread-safe decoded-media byte accumulator for one request."""
def __init__(self) -> None:
self.used = 0
self._reserved = 0
self._condition = threading.Condition()
def reserve(self, max_bytes: int) -> int:
"""Reserve decoder capacity, waiting for in-flight decodes if needed."""
limit = envs.VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES
if limit <= 0:
return max_bytes
max_bytes = max(0, max_bytes)
with self._condition:
while True:
available = limit - self.used - self._reserved
if available >= max_bytes or self._reserved == 0:
if available <= 0:
raise ValueError(
"Request decoded media exceeds "
"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES."
)
reserved = min(max_bytes, available)
self._reserved += reserved
return reserved
self._condition.wait()
def settle(self, reserved: int, decoded_bytes: int) -> None:
"""Replace a decoder reservation with its actual decoded size."""
limit = envs.VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES
if limit <= 0:
return
with self._condition:
self._reserved -= reserved
try:
used = self.used + decoded_bytes
if decoded_bytes > reserved or used > limit:
raise ValueError(
"Request decoded media exceeds "
f"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES={limit} "
f"({used} bytes)."
)
self.used = used
finally:
self._condition.notify_all()
def charge(self, decoded_bytes: int) -> None:
limit = envs.VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES
if limit <= 0 or decoded_bytes <= 0:
return
with self._condition:
while (
self.used + self._reserved + decoded_bytes > limit
and self._reserved > 0
and self.used + decoded_bytes <= limit
):
self._condition.wait()
used = self.used + decoded_bytes
if used > limit:
raise ValueError(
"Request decoded media exceeds "
f"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES={limit} "
f"({used} bytes)."
)
self.used = used
@dataclass
class MediaWithBytes(Generic[_T]):
"""Wrapper that couples a media object with its original encoded bytes.
+38 -16
View File
@@ -25,6 +25,8 @@ import vllm.envs as envs
from vllm.connections import (
HTTPConnection,
MediaDownloadSizeExceededError,
MediaRequestDownloadBudget,
MediaRequestDownloadSizeExceededError,
global_http_connection,
)
from vllm.exceptions import VLLMUnprocessableEntityError, VLLMValidationError
@@ -33,7 +35,7 @@ from vllm.multimodal.video import get_video_loader_backend_for_processor
from vllm.utils.registry import ExtensionManager
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
from .base import MediaIO, MediaWithBytes
from .base import MediaDecodedBudget, MediaIO, MediaWithBytes
from .image import ImageEmbeddingMediaIO, ImageMediaIO
from .video import VideoEmbeddingMediaIO, VideoMediaIO
@@ -109,7 +111,9 @@ def _wrap_media_fetch_error(
value=url,
)
if isinstance(exc, MediaDownloadSizeExceededError):
if isinstance(
exc, (MediaDownloadSizeExceededError, MediaRequestDownloadSizeExceededError)
):
return VLLMUnprocessableEntityError(
f"Failed to fetch media from URL: {exc}",
parameter="image_url",
@@ -162,6 +166,8 @@ class MediaConnector:
*,
allowed_local_media_path: str = "",
allowed_media_domains: list[str] | None = None,
decoded_budget: MediaDecodedBudget | None = None,
download_budget: MediaRequestDownloadBudget | None = None,
) -> None:
"""Args:
media_io_kwargs: Additional args passed to process media
@@ -201,6 +207,8 @@ class MediaConnector:
if allowed_media_domains is None:
allowed_media_domains = []
self.allowed_media_domains = allowed_media_domains
self.decoded_budget = decoded_budget
self.download_budget = download_budget
# Media download cache (opt-in via VLLM_MEDIA_CACHE)
self._media_cache_dir: str | None = None
@@ -230,6 +238,12 @@ class MediaConnector:
media_cache,
)
def _image_io(self, image_mode: str | None = "RGB") -> ImageMediaIO:
return ImageMediaIO(
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {})),
decoded_budget=self.decoded_budget,
)
def _get_cached_bytes(self, url: str) -> bytes | None:
"""Return cached bytes for a URL, or None if not cached/expired."""
if not self._media_cache_dir:
@@ -385,12 +399,18 @@ class MediaConnector:
return media_io.load_bytes(cached)
connection = self.connection
budget_kwargs = (
{"request_budget": self.download_budget}
if self.download_budget is not None
else {}
)
try:
data = connection.get_bytes(
url_spec.url,
timeout=fetch_timeout,
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
max_bytes=max_bytes,
**budget_kwargs,
)
except Exception as e:
wrapped = _wrap_media_fetch_error(url, e)
@@ -438,12 +458,18 @@ class MediaConnector:
return await future
connection = self.connection
budget_kwargs = (
{"request_budget": self.download_budget}
if self.download_budget is not None
else {}
)
try:
data = await connection.async_get_bytes(
url_spec.url,
timeout=fetch_timeout,
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
max_bytes=max_bytes,
**budget_kwargs,
)
except Exception as e:
wrapped = _wrap_media_fetch_error(url, e)
@@ -470,7 +496,9 @@ class MediaConnector:
audio_url: str,
) -> tuple[np.ndarray, int | float]:
"""Load audio from a URL."""
audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))
audio_io = AudioMediaIO(
decoded_budget=self.decoded_budget, **self.media_io_kwargs.get("audio", {})
)
return self.load_from_url(
audio_url,
@@ -483,7 +511,9 @@ class MediaConnector:
audio_url: str,
) -> tuple[np.ndarray, int | float]:
"""Asynchronously fetch audio from a URL."""
audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))
audio_io = AudioMediaIO(
decoded_budget=self.decoded_budget, **self.media_io_kwargs.get("audio", {})
)
return await self.load_from_url_async(
audio_url,
@@ -503,9 +533,7 @@ class MediaConnector:
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
original image mode (e.g. preserving the alpha channel).
"""
image_io = ImageMediaIO(
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
)
image_io = self._image_io(image_mode)
try:
return self.load_from_url(
@@ -529,9 +557,7 @@ class MediaConnector:
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
original image mode (e.g. preserving the alpha channel).
"""
image_io = ImageMediaIO(
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
)
image_io = self._image_io(image_mode)
try:
return await self.load_from_url_async(
@@ -551,9 +577,7 @@ class MediaConnector:
video_processor: str | None = None,
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
"""Load video from an HTTP or base64 data URL."""
image_io = ImageMediaIO(
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
)
image_io = self._image_io(image_mode)
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
if "video_backend" not in video_io_kwargs and (
video_backend := get_video_loader_backend_for_processor(video_processor)
@@ -580,9 +604,7 @@ class MediaConnector:
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
original image mode (e.g. preserving the alpha channel).
"""
image_io = ImageMediaIO(
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
)
image_io = self._image_io(image_mode)
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
if "video_backend" not in video_io_kwargs and (
video_backend := get_video_loader_backend_for_processor(video_processor)
+11 -2
View File
@@ -17,7 +17,7 @@ from vllm.utils.sparse_utils import (
)
from ..image import convert_image_mode, normalize_image, open_image, rgba_to_rgb
from .base import MediaIO, MediaWithBytes
from .base import MediaDecodedBudget, MediaIO, MediaWithBytes
MAGIC_NUMPY_PREFIX = b"\x93NUMPY" # https://numpy.org/devdocs/reference/generated/numpy.lib.format.html#format-version-1-0
@@ -28,12 +28,19 @@ class ImageMediaIO(MediaIO[Image.Image]):
error handling.
"""
def __init__(self, image_mode: str | None = "RGB", **kwargs) -> None:
def __init__(
self,
image_mode: str | None = "RGB",
*,
decoded_budget: MediaDecodedBudget | None = None,
**kwargs,
) -> None:
super().__init__()
# Target mode for loaded images; `None` keeps the original mode
# (i.e. no conversion, alpha channel is preserved as-is).
self.image_mode = image_mode
self.decoded_budget = decoded_budget
# `kwargs` contains custom arguments from
# --media-io-kwargs for this modality, merged with
# per-request runtime media_io_kwargs via merge_kwargs().
@@ -87,6 +94,8 @@ class ImageMediaIO(MediaIO[Image.Image]):
f"the maximum of {max_pixels} pixels. Set "
f"VLLM_MAX_IMAGE_PIXELS to increase this limit."
)
if self.decoded_budget is not None:
self.decoded_budget.charge(w * h * 4)
image = normalize_image(image)
image.load()
converted = self._convert_image_mode(image)
+10
View File
@@ -122,6 +122,16 @@ class VideoMediaIO(MediaIO[MediaWithBytes[tuple[DecodedFrames, dict[str, Any]]]]
video = self.video_loader.load_bytes(
data, num_frames=self.num_frames, **self.kwargs
)
frames = video[0] if isinstance(video, tuple) else video
budget = getattr(self.image_io, "decoded_budget", None)
shape = getattr(frames, "shape", None)
if budget is not None and shape is not None and len(shape) >= 3:
decoded_bytes = (
frames.numel() * frames.element_size()
if isinstance(frames, torch.Tensor)
else frames.nbytes
)
budget.charge(decoded_bytes)
return MediaWithBytes(video, data)
def load_base64(
+16
View File
@@ -53,6 +53,7 @@ from typing_extensions import TypedDict, override
from vllm import envs
from vllm.config import ModelConfig
from vllm.connections import MediaRequestDownloadBudget
from vllm.exceptions import VLLMValidationError
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
from vllm.logger import init_logger
@@ -67,6 +68,7 @@ from vllm.multimodal.inputs import (
VisionChunkVideo,
)
from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY, MediaConnector
from vllm.multimodal.media.base import MediaDecodedBudget
from vllm.multimodal.processing import BaseMultiModalProcessor
from vllm.renderers.embed_utils import (
safe_load_prompt_embeds,
@@ -609,6 +611,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
self._items_by_modality = defaultdict[str, list[_T]](list)
# Track original modality for each vision_chunk item (image or video)
self._modality_order = defaultdict[str, list[str]](list)
self.decoded_budget = MediaDecodedBudget()
self.download_budget = MediaRequestDownloadBudget()
@cached_property
def use_unified_vision_chunk_modality(self) -> bool:
@@ -1052,6 +1056,8 @@ class MultiModalContentParser(BaseMultiModalContentParser):
media_io_kwargs=self._tracker.media_io_kwargs,
allowed_local_media_path=self._tracker.allowed_local_media_path,
allowed_media_domains=self._tracker.allowed_media_domains,
decoded_budget=self._tracker.decoded_budget,
download_budget=self._tracker.download_budget,
)
@property
@@ -1138,6 +1144,10 @@ class MultiModalContentParser(BaseMultiModalContentParser):
def parse_image_pil(
self, image_pil: Image.Image | None, uuid: str | None = None
) -> None:
if image_pil is not None:
self._tracker.decoded_budget.charge(
image_pil.size[0] * image_pil.size[1] * 4
)
placeholder = self._tracker.add("image", (image_pil, uuid))
self._add_placeholder("image", placeholder)
@@ -1233,6 +1243,8 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
media_io_kwargs=self._tracker.media_io_kwargs,
allowed_local_media_path=self._tracker.allowed_local_media_path,
allowed_media_domains=self._tracker.allowed_media_domains,
decoded_budget=self._tracker.decoded_budget,
download_budget=self._tracker.download_budget,
)
@property
@@ -1351,6 +1363,10 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
image_pil: Image.Image | None,
uuid: str | None = None,
) -> None:
if image_pil is not None:
self._tracker.decoded_budget.charge(
image_pil.size[0] * image_pil.size[1] * 4
)
placeholder = self._tracker.add(
"image", partial(self._item_with_uuid_async, image_pil, uuid)
)