[Security] Bound decoded and downloaded media per request (#61226)
Signed-off-by: juanpas97 <juan.perezalgaba@gmail.com> Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
877f6df889
commit
7173c64e2e
+57
-8
@@ -3,6 +3,7 @@
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Coroutine, Mapping, MutableMapping
|
||||
from pathlib import Path
|
||||
@@ -60,6 +61,33 @@ class MediaDownloadSizeExceededError(ValueError):
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class MediaRequestDownloadSizeExceededError(ValueError):
|
||||
"""Raised when all HTTP media downloads in one request exceed the limit."""
|
||||
|
||||
|
||||
class MediaRequestDownloadBudget:
|
||||
"""Thread-safe total of HTTP media bytes received for one request."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.used = 0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def charge(self, size: int) -> None:
|
||||
limit_mb = envs.VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB
|
||||
if limit_mb <= 0:
|
||||
return
|
||||
limit = limit_mb * MiB_bytes
|
||||
with self._lock:
|
||||
received = self.used + size
|
||||
if received > limit:
|
||||
raise MediaRequestDownloadSizeExceededError(
|
||||
"Request media downloads exceed "
|
||||
f"VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB={limit_mb:g} "
|
||||
f"({received} bytes received)."
|
||||
)
|
||||
self.used = received
|
||||
|
||||
|
||||
def _get_media_download_limit_bytes() -> int | None:
|
||||
max_size_mb = envs.VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB
|
||||
if max_size_mb <= 0:
|
||||
@@ -105,19 +133,26 @@ def _append_chunk(
|
||||
received_bytes: int,
|
||||
max_bytes: int | None,
|
||||
media_download_limit: bool,
|
||||
request_budget: MediaRequestDownloadBudget | None = None,
|
||||
) -> int:
|
||||
if not chunk:
|
||||
return received_bytes
|
||||
received_bytes += len(chunk)
|
||||
if max_bytes is not None and received_bytes > max_bytes:
|
||||
_raise_size_exceeded(max_bytes, received_bytes, media_download_limit)
|
||||
if request_budget is not None:
|
||||
request_budget.charge(len(chunk))
|
||||
chunks.append(chunk)
|
||||
return received_bytes
|
||||
|
||||
|
||||
def _read_response_bytes(response: requests.Response, max_bytes: int | None) -> bytes:
|
||||
def _read_response_bytes(
|
||||
response: requests.Response,
|
||||
max_bytes: int | None,
|
||||
request_budget: MediaRequestDownloadBudget | None = None,
|
||||
) -> bytes:
|
||||
max_bytes, media_download_limit = _get_response_size_limit(max_bytes)
|
||||
if max_bytes is None:
|
||||
if max_bytes is None and request_budget is None:
|
||||
return response.content
|
||||
|
||||
_check_content_length(
|
||||
@@ -129,16 +164,23 @@ def _read_response_bytes(response: requests.Response, max_bytes: int | None) ->
|
||||
received_bytes = 0
|
||||
for chunk in response.iter_content(_RESPONSE_READ_CHUNK_SIZE):
|
||||
received_bytes = _append_chunk(
|
||||
chunks, chunk, received_bytes, max_bytes, media_download_limit
|
||||
chunks,
|
||||
chunk,
|
||||
received_bytes,
|
||||
max_bytes,
|
||||
media_download_limit,
|
||||
request_budget,
|
||||
)
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
async def _async_read_response_bytes(
|
||||
response: aiohttp.ClientResponse, max_bytes: int | None
|
||||
response: aiohttp.ClientResponse,
|
||||
max_bytes: int | None,
|
||||
request_budget: MediaRequestDownloadBudget | None = None,
|
||||
) -> bytes:
|
||||
max_bytes, media_download_limit = _get_response_size_limit(max_bytes)
|
||||
if max_bytes is None:
|
||||
if max_bytes is None and request_budget is None:
|
||||
return await response.read()
|
||||
|
||||
_check_content_length(response.content_length, max_bytes, media_download_limit)
|
||||
@@ -146,7 +188,12 @@ async def _async_read_response_bytes(
|
||||
received_bytes = 0
|
||||
async for chunk in response.content.iter_chunked(_RESPONSE_READ_CHUNK_SIZE):
|
||||
received_bytes = _append_chunk(
|
||||
chunks, chunk, received_bytes, max_bytes, media_download_limit
|
||||
chunks,
|
||||
chunk,
|
||||
received_bytes,
|
||||
max_bytes,
|
||||
media_download_limit,
|
||||
request_budget,
|
||||
)
|
||||
return b"".join(chunks)
|
||||
|
||||
@@ -407,6 +454,7 @@ class HTTPConnection:
|
||||
timeout: float | None = None,
|
||||
allow_redirects: bool = True,
|
||||
max_bytes: int | None = None,
|
||||
request_budget: MediaRequestDownloadBudget | None = None,
|
||||
) -> bytes:
|
||||
with self.get_response(
|
||||
url,
|
||||
@@ -416,7 +464,7 @@ class HTTPConnection:
|
||||
) as r:
|
||||
r.raise_for_status()
|
||||
|
||||
return _read_response_bytes(r, max_bytes)
|
||||
return _read_response_bytes(r, max_bytes, request_budget)
|
||||
|
||||
@_async_retry
|
||||
async def async_get_bytes(
|
||||
@@ -426,6 +474,7 @@ class HTTPConnection:
|
||||
timeout: float | None = None,
|
||||
allow_redirects: bool = True,
|
||||
max_bytes: int | None = None,
|
||||
request_budget: MediaRequestDownloadBudget | None = None,
|
||||
) -> bytes:
|
||||
async with await self.get_async_response(
|
||||
url,
|
||||
@@ -434,7 +483,7 @@ class HTTPConnection:
|
||||
) as r:
|
||||
r.raise_for_status()
|
||||
|
||||
return await _async_read_response_bytes(r, max_bytes)
|
||||
return await _async_read_response_bytes(r, max_bytes, request_budget)
|
||||
|
||||
def get_text(self, url: str, *, timeout: float | None = None) -> str:
|
||||
with self.get_response(url, timeout=timeout) as r:
|
||||
|
||||
@@ -78,6 +78,7 @@ if TYPE_CHECKING:
|
||||
VLLM_MEDIA_CACHE_TTL_HOURS: float = 24
|
||||
VLLM_MEDIA_FETCH_MAX_RETRIES: int = 3
|
||||
VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB: int = 256
|
||||
VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB: int = 256
|
||||
VLLM_MEDIA_URL_ALLOW_REDIRECTS: bool = True
|
||||
VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8
|
||||
VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25
|
||||
@@ -86,6 +87,7 @@ if TYPE_CHECKING:
|
||||
VLLM_MAX_AUDIO_PREPROCESS_WORKERS: int = max(1, min(os.cpu_count() or 1, 2))
|
||||
VLLM_MAX_EMBED_DECODE_BYTES: int = 2_147_483_648
|
||||
VLLM_MAX_IMAGE_PIXELS: int = 178_956_970
|
||||
VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES: int = 715_827_880
|
||||
VLLM_VIDEO_LOADER_BACKEND: str = "opencv"
|
||||
VLLM_MEDIA_CONNECTOR: str = "http"
|
||||
VLLM_TARGET_DEVICE: str = "cuda"
|
||||
@@ -1063,6 +1065,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_MAX_IMAGE_PIXELS": lambda: int(
|
||||
os.getenv("VLLM_MAX_IMAGE_PIXELS", "178956970")
|
||||
),
|
||||
# Maximum aggregate decoded bytes across media in one request.
|
||||
# The default allows one image at the per-image pixel limit at 4 bytes
|
||||
# per pixel. Set to 0 to disable.
|
||||
"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES": lambda: int(
|
||||
os.getenv("VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES", "715827880")
|
||||
),
|
||||
"VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB": lambda: int(
|
||||
os.getenv("VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB", "256")
|
||||
),
|
||||
# Backend for Video IO — selects the frame-sampling algorithm.
|
||||
# - "opencv": uniform sampling.
|
||||
# - "opencv_dynamic": duration-aware dynamic sampling.
|
||||
@@ -2412,6 +2423,8 @@ def compile_factors() -> dict[str, object]:
|
||||
"VLLM_MAX_AUDIO_PREPROCESS_WORKERS",
|
||||
"VLLM_MAX_EMBED_DECODE_BYTES",
|
||||
"VLLM_MAX_IMAGE_PIXELS",
|
||||
"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES",
|
||||
"VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB",
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
"VLLM_MEDIA_CONNECTOR",
|
||||
"VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME",
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
|
||||
from .base import MediaIO, MediaWithBytes
|
||||
from .base import MediaDecodedBudget, MediaIO, MediaWithBytes
|
||||
from .connector import MEDIA_CONNECTOR_REGISTRY, MediaConnector
|
||||
from .image import ImageEmbeddingMediaIO, ImageMediaIO
|
||||
from .video import VIDEO_LOADER_REGISTRY, VideoMediaIO
|
||||
|
||||
__all__ = [
|
||||
"MediaIO",
|
||||
"MediaDecodedBudget",
|
||||
"MediaWithBytes",
|
||||
"AudioEmbeddingMediaIO",
|
||||
"AudioMediaIO",
|
||||
|
||||
@@ -23,7 +23,7 @@ from vllm.utils.sparse_utils import (
|
||||
safe_to_dense,
|
||||
)
|
||||
|
||||
from .base import MediaIO
|
||||
from .base import MediaDecodedBudget, MediaIO
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -527,8 +527,11 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
||||
error handling.
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs) -> None:
|
||||
def __init__(
|
||||
self, *, decoded_budget: MediaDecodedBudget | None = None, **kwargs
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.decoded_budget = decoded_budget
|
||||
|
||||
# Allow per-server override of the audio decoding backend, e.g.:
|
||||
# --media-io-kwargs '{"audio": {"audio_backend": "torchcodec"}}'
|
||||
@@ -564,13 +567,44 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
||||
|
||||
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
|
||||
self._validate_encoded_size(len(data))
|
||||
return load_audio(
|
||||
BytesIO(data),
|
||||
sr=None,
|
||||
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
|
||||
max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES,
|
||||
backend=self.audio_backend,
|
||||
)
|
||||
return self._load_with_budget(BytesIO(data))
|
||||
|
||||
def _load_with_budget(self, source: BytesIO | Path) -> tuple[npt.NDArray, float]:
|
||||
budget = self.decoded_budget
|
||||
if budget is None:
|
||||
return load_audio(
|
||||
source,
|
||||
sr=None,
|
||||
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
|
||||
max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES,
|
||||
backend=self.audio_backend,
|
||||
)
|
||||
|
||||
max_decode_bytes = envs.VLLM_MAX_AUDIO_DECODE_BYTES
|
||||
reserved = budget.reserve(max_decode_bytes)
|
||||
decoded_bytes = 0
|
||||
try:
|
||||
try:
|
||||
audio = load_audio(
|
||||
source,
|
||||
sr=None,
|
||||
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
|
||||
max_decode_bytes=reserved,
|
||||
backend=self.audio_backend,
|
||||
)
|
||||
except ValueError as exc:
|
||||
if reserved < max_decode_bytes and "VLLM_MAX_AUDIO_DECODE_BYTES" in str(
|
||||
exc
|
||||
):
|
||||
raise ValueError(
|
||||
"Request decoded media exceeds "
|
||||
"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES."
|
||||
) from exc
|
||||
raise
|
||||
decoded_bytes = audio[0].nbytes
|
||||
return audio
|
||||
finally:
|
||||
budget.settle(reserved, decoded_bytes)
|
||||
|
||||
def load_base64(
|
||||
self,
|
||||
@@ -588,13 +622,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
||||
|
||||
def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]:
|
||||
self._validate_encoded_size(filepath.stat().st_size)
|
||||
return load_audio(
|
||||
filepath,
|
||||
sr=None,
|
||||
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
|
||||
max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES,
|
||||
backend=self.audio_backend,
|
||||
)
|
||||
return self._load_with_budget(filepath)
|
||||
|
||||
def encode_base64(
|
||||
self,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import threading
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
@@ -9,9 +10,79 @@ from typing import Any, Generic, TypeVar, cast
|
||||
|
||||
import numpy as np
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
class MediaDecodedBudget:
|
||||
"""Thread-safe decoded-media byte accumulator for one request."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.used = 0
|
||||
self._reserved = 0
|
||||
self._condition = threading.Condition()
|
||||
|
||||
def reserve(self, max_bytes: int) -> int:
|
||||
"""Reserve decoder capacity, waiting for in-flight decodes if needed."""
|
||||
limit = envs.VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES
|
||||
if limit <= 0:
|
||||
return max_bytes
|
||||
max_bytes = max(0, max_bytes)
|
||||
with self._condition:
|
||||
while True:
|
||||
available = limit - self.used - self._reserved
|
||||
if available >= max_bytes or self._reserved == 0:
|
||||
if available <= 0:
|
||||
raise ValueError(
|
||||
"Request decoded media exceeds "
|
||||
"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES."
|
||||
)
|
||||
reserved = min(max_bytes, available)
|
||||
self._reserved += reserved
|
||||
return reserved
|
||||
self._condition.wait()
|
||||
|
||||
def settle(self, reserved: int, decoded_bytes: int) -> None:
|
||||
"""Replace a decoder reservation with its actual decoded size."""
|
||||
limit = envs.VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES
|
||||
if limit <= 0:
|
||||
return
|
||||
with self._condition:
|
||||
self._reserved -= reserved
|
||||
try:
|
||||
used = self.used + decoded_bytes
|
||||
if decoded_bytes > reserved or used > limit:
|
||||
raise ValueError(
|
||||
"Request decoded media exceeds "
|
||||
f"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES={limit} "
|
||||
f"({used} bytes)."
|
||||
)
|
||||
self.used = used
|
||||
finally:
|
||||
self._condition.notify_all()
|
||||
|
||||
def charge(self, decoded_bytes: int) -> None:
|
||||
limit = envs.VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES
|
||||
if limit <= 0 or decoded_bytes <= 0:
|
||||
return
|
||||
with self._condition:
|
||||
while (
|
||||
self.used + self._reserved + decoded_bytes > limit
|
||||
and self._reserved > 0
|
||||
and self.used + decoded_bytes <= limit
|
||||
):
|
||||
self._condition.wait()
|
||||
used = self.used + decoded_bytes
|
||||
if used > limit:
|
||||
raise ValueError(
|
||||
"Request decoded media exceeds "
|
||||
f"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES={limit} "
|
||||
f"({used} bytes)."
|
||||
)
|
||||
self.used = used
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaWithBytes(Generic[_T]):
|
||||
"""Wrapper that couples a media object with its original encoded bytes.
|
||||
|
||||
@@ -25,6 +25,8 @@ import vllm.envs as envs
|
||||
from vllm.connections import (
|
||||
HTTPConnection,
|
||||
MediaDownloadSizeExceededError,
|
||||
MediaRequestDownloadBudget,
|
||||
MediaRequestDownloadSizeExceededError,
|
||||
global_http_connection,
|
||||
)
|
||||
from vllm.exceptions import VLLMUnprocessableEntityError, VLLMValidationError
|
||||
@@ -33,7 +35,7 @@ from vllm.multimodal.video import get_video_loader_backend_for_processor
|
||||
from vllm.utils.registry import ExtensionManager
|
||||
|
||||
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
|
||||
from .base import MediaIO, MediaWithBytes
|
||||
from .base import MediaDecodedBudget, MediaIO, MediaWithBytes
|
||||
from .image import ImageEmbeddingMediaIO, ImageMediaIO
|
||||
from .video import VideoEmbeddingMediaIO, VideoMediaIO
|
||||
|
||||
@@ -109,7 +111,9 @@ def _wrap_media_fetch_error(
|
||||
value=url,
|
||||
)
|
||||
|
||||
if isinstance(exc, MediaDownloadSizeExceededError):
|
||||
if isinstance(
|
||||
exc, (MediaDownloadSizeExceededError, MediaRequestDownloadSizeExceededError)
|
||||
):
|
||||
return VLLMUnprocessableEntityError(
|
||||
f"Failed to fetch media from URL: {exc}",
|
||||
parameter="image_url",
|
||||
@@ -162,6 +166,8 @@ class MediaConnector:
|
||||
*,
|
||||
allowed_local_media_path: str = "",
|
||||
allowed_media_domains: list[str] | None = None,
|
||||
decoded_budget: MediaDecodedBudget | None = None,
|
||||
download_budget: MediaRequestDownloadBudget | None = None,
|
||||
) -> None:
|
||||
"""Args:
|
||||
media_io_kwargs: Additional args passed to process media
|
||||
@@ -201,6 +207,8 @@ class MediaConnector:
|
||||
if allowed_media_domains is None:
|
||||
allowed_media_domains = []
|
||||
self.allowed_media_domains = allowed_media_domains
|
||||
self.decoded_budget = decoded_budget
|
||||
self.download_budget = download_budget
|
||||
|
||||
# Media download cache (opt-in via VLLM_MEDIA_CACHE)
|
||||
self._media_cache_dir: str | None = None
|
||||
@@ -230,6 +238,12 @@ class MediaConnector:
|
||||
media_cache,
|
||||
)
|
||||
|
||||
def _image_io(self, image_mode: str | None = "RGB") -> ImageMediaIO:
|
||||
return ImageMediaIO(
|
||||
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {})),
|
||||
decoded_budget=self.decoded_budget,
|
||||
)
|
||||
|
||||
def _get_cached_bytes(self, url: str) -> bytes | None:
|
||||
"""Return cached bytes for a URL, or None if not cached/expired."""
|
||||
if not self._media_cache_dir:
|
||||
@@ -385,12 +399,18 @@ class MediaConnector:
|
||||
return media_io.load_bytes(cached)
|
||||
|
||||
connection = self.connection
|
||||
budget_kwargs = (
|
||||
{"request_budget": self.download_budget}
|
||||
if self.download_budget is not None
|
||||
else {}
|
||||
)
|
||||
try:
|
||||
data = connection.get_bytes(
|
||||
url_spec.url,
|
||||
timeout=fetch_timeout,
|
||||
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
|
||||
max_bytes=max_bytes,
|
||||
**budget_kwargs,
|
||||
)
|
||||
except Exception as e:
|
||||
wrapped = _wrap_media_fetch_error(url, e)
|
||||
@@ -438,12 +458,18 @@ class MediaConnector:
|
||||
return await future
|
||||
|
||||
connection = self.connection
|
||||
budget_kwargs = (
|
||||
{"request_budget": self.download_budget}
|
||||
if self.download_budget is not None
|
||||
else {}
|
||||
)
|
||||
try:
|
||||
data = await connection.async_get_bytes(
|
||||
url_spec.url,
|
||||
timeout=fetch_timeout,
|
||||
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
|
||||
max_bytes=max_bytes,
|
||||
**budget_kwargs,
|
||||
)
|
||||
except Exception as e:
|
||||
wrapped = _wrap_media_fetch_error(url, e)
|
||||
@@ -470,7 +496,9 @@ class MediaConnector:
|
||||
audio_url: str,
|
||||
) -> tuple[np.ndarray, int | float]:
|
||||
"""Load audio from a URL."""
|
||||
audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))
|
||||
audio_io = AudioMediaIO(
|
||||
decoded_budget=self.decoded_budget, **self.media_io_kwargs.get("audio", {})
|
||||
)
|
||||
|
||||
return self.load_from_url(
|
||||
audio_url,
|
||||
@@ -483,7 +511,9 @@ class MediaConnector:
|
||||
audio_url: str,
|
||||
) -> tuple[np.ndarray, int | float]:
|
||||
"""Asynchronously fetch audio from a URL."""
|
||||
audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))
|
||||
audio_io = AudioMediaIO(
|
||||
decoded_budget=self.decoded_budget, **self.media_io_kwargs.get("audio", {})
|
||||
)
|
||||
|
||||
return await self.load_from_url_async(
|
||||
audio_url,
|
||||
@@ -503,9 +533,7 @@ class MediaConnector:
|
||||
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
|
||||
original image mode (e.g. preserving the alpha channel).
|
||||
"""
|
||||
image_io = ImageMediaIO(
|
||||
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
|
||||
)
|
||||
image_io = self._image_io(image_mode)
|
||||
|
||||
try:
|
||||
return self.load_from_url(
|
||||
@@ -529,9 +557,7 @@ class MediaConnector:
|
||||
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
|
||||
original image mode (e.g. preserving the alpha channel).
|
||||
"""
|
||||
image_io = ImageMediaIO(
|
||||
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
|
||||
)
|
||||
image_io = self._image_io(image_mode)
|
||||
|
||||
try:
|
||||
return await self.load_from_url_async(
|
||||
@@ -551,9 +577,7 @@ class MediaConnector:
|
||||
video_processor: str | None = None,
|
||||
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
|
||||
"""Load video from an HTTP or base64 data URL."""
|
||||
image_io = ImageMediaIO(
|
||||
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
|
||||
)
|
||||
image_io = self._image_io(image_mode)
|
||||
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
|
||||
if "video_backend" not in video_io_kwargs and (
|
||||
video_backend := get_video_loader_backend_for_processor(video_processor)
|
||||
@@ -580,9 +604,7 @@ class MediaConnector:
|
||||
`media_io_kwargs={"image": {"image_mode": None}}` to keep the
|
||||
original image mode (e.g. preserving the alpha channel).
|
||||
"""
|
||||
image_io = ImageMediaIO(
|
||||
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
|
||||
)
|
||||
image_io = self._image_io(image_mode)
|
||||
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
|
||||
if "video_backend" not in video_io_kwargs and (
|
||||
video_backend := get_video_loader_backend_for_processor(video_processor)
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.utils.sparse_utils import (
|
||||
)
|
||||
|
||||
from ..image import convert_image_mode, normalize_image, open_image, rgba_to_rgb
|
||||
from .base import MediaIO, MediaWithBytes
|
||||
from .base import MediaDecodedBudget, MediaIO, MediaWithBytes
|
||||
|
||||
MAGIC_NUMPY_PREFIX = b"\x93NUMPY" # https://numpy.org/devdocs/reference/generated/numpy.lib.format.html#format-version-1-0
|
||||
|
||||
@@ -28,12 +28,19 @@ class ImageMediaIO(MediaIO[Image.Image]):
|
||||
error handling.
|
||||
"""
|
||||
|
||||
def __init__(self, image_mode: str | None = "RGB", **kwargs) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
image_mode: str | None = "RGB",
|
||||
*,
|
||||
decoded_budget: MediaDecodedBudget | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
# Target mode for loaded images; `None` keeps the original mode
|
||||
# (i.e. no conversion, alpha channel is preserved as-is).
|
||||
self.image_mode = image_mode
|
||||
self.decoded_budget = decoded_budget
|
||||
# `kwargs` contains custom arguments from
|
||||
# --media-io-kwargs for this modality, merged with
|
||||
# per-request runtime media_io_kwargs via merge_kwargs().
|
||||
@@ -87,6 +94,8 @@ class ImageMediaIO(MediaIO[Image.Image]):
|
||||
f"the maximum of {max_pixels} pixels. Set "
|
||||
f"VLLM_MAX_IMAGE_PIXELS to increase this limit."
|
||||
)
|
||||
if self.decoded_budget is not None:
|
||||
self.decoded_budget.charge(w * h * 4)
|
||||
image = normalize_image(image)
|
||||
image.load()
|
||||
converted = self._convert_image_mode(image)
|
||||
|
||||
@@ -122,6 +122,16 @@ class VideoMediaIO(MediaIO[MediaWithBytes[tuple[DecodedFrames, dict[str, Any]]]]
|
||||
video = self.video_loader.load_bytes(
|
||||
data, num_frames=self.num_frames, **self.kwargs
|
||||
)
|
||||
frames = video[0] if isinstance(video, tuple) else video
|
||||
budget = getattr(self.image_io, "decoded_budget", None)
|
||||
shape = getattr(frames, "shape", None)
|
||||
if budget is not None and shape is not None and len(shape) >= 3:
|
||||
decoded_bytes = (
|
||||
frames.numel() * frames.element_size()
|
||||
if isinstance(frames, torch.Tensor)
|
||||
else frames.nbytes
|
||||
)
|
||||
budget.charge(decoded_bytes)
|
||||
return MediaWithBytes(video, data)
|
||||
|
||||
def load_base64(
|
||||
|
||||
@@ -53,6 +53,7 @@ from typing_extensions import TypedDict, override
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.connections import MediaRequestDownloadBudget
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
|
||||
from vllm.logger import init_logger
|
||||
@@ -67,6 +68,7 @@ from vllm.multimodal.inputs import (
|
||||
VisionChunkVideo,
|
||||
)
|
||||
from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY, MediaConnector
|
||||
from vllm.multimodal.media.base import MediaDecodedBudget
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.renderers.embed_utils import (
|
||||
safe_load_prompt_embeds,
|
||||
@@ -609,6 +611,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
self._items_by_modality = defaultdict[str, list[_T]](list)
|
||||
# Track original modality for each vision_chunk item (image or video)
|
||||
self._modality_order = defaultdict[str, list[str]](list)
|
||||
self.decoded_budget = MediaDecodedBudget()
|
||||
self.download_budget = MediaRequestDownloadBudget()
|
||||
|
||||
@cached_property
|
||||
def use_unified_vision_chunk_modality(self) -> bool:
|
||||
@@ -1052,6 +1056,8 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
media_io_kwargs=self._tracker.media_io_kwargs,
|
||||
allowed_local_media_path=self._tracker.allowed_local_media_path,
|
||||
allowed_media_domains=self._tracker.allowed_media_domains,
|
||||
decoded_budget=self._tracker.decoded_budget,
|
||||
download_budget=self._tracker.download_budget,
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -1138,6 +1144,10 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
def parse_image_pil(
|
||||
self, image_pil: Image.Image | None, uuid: str | None = None
|
||||
) -> None:
|
||||
if image_pil is not None:
|
||||
self._tracker.decoded_budget.charge(
|
||||
image_pil.size[0] * image_pil.size[1] * 4
|
||||
)
|
||||
placeholder = self._tracker.add("image", (image_pil, uuid))
|
||||
self._add_placeholder("image", placeholder)
|
||||
|
||||
@@ -1233,6 +1243,8 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
media_io_kwargs=self._tracker.media_io_kwargs,
|
||||
allowed_local_media_path=self._tracker.allowed_local_media_path,
|
||||
allowed_media_domains=self._tracker.allowed_media_domains,
|
||||
decoded_budget=self._tracker.decoded_budget,
|
||||
download_budget=self._tracker.download_budget,
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -1351,6 +1363,10 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
image_pil: Image.Image | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
if image_pil is not None:
|
||||
self._tracker.decoded_budget.charge(
|
||||
image_pil.size[0] * image_pil.size[1] * 4
|
||||
)
|
||||
placeholder = self._tracker.add(
|
||||
"image", partial(self._item_with_uuid_async, image_pil, uuid)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user