[Security] Bound decoded and downloaded media per request (#61226)

Signed-off-by: juanpas97 <juan.perezalgaba@gmail.com>
Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Juan Pérez de Algaba
2026-10-13 10:17:14 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 877f6df889
commit 7173c64e2e
16 changed files with 565 additions and 50 deletions
+2
View File
@@ -94,7 +94,9 @@ denial of service:
| Environment Variable | Default | Description | | Environment Variable | Default | Description |
| --- | --- | --- | | --- | --- | --- |
| `VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB` | `256` | Maximum size in MB for a single remote media response. Oversized responses are rejected while streaming before the full body is materialized in memory. | | `VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB` | `256` | Maximum size in MB for a single remote media response. Oversized responses are rejected while streaming before the full body is materialized in memory. |
| `VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB` | `256` | Maximum total HTTP media bytes downloaded for one request, including concurrent downloads. Enforced while streaming. |
| `VLLM_MAX_IMAGE_PIXELS` | `178956970` (~179M pixels) | Maximum decoded image size in pixels. Images exceeding this are rejected before raster memory is allocated. Default matches PIL's built-in 2x decompression-bomb threshold (~680 MB for RGB). | | `VLLM_MAX_IMAGE_PIXELS` | `178956970` (~179M pixels) | Maximum decoded image size in pixels. Images exceeding this are rejected before raster memory is allocated. Default matches PIL's built-in 2x decompression-bomb threshold (~680 MB for RGB). |
| `VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES` | `715827880` (~683 MiB) | Maximum aggregate decoded image, video-frame, and audio bytes in one request. Images are charged at four bytes per pixel before decoding; audio is limited before decoding; video files are charged after frame extraction. |
| `VLLM_MAX_AUDIO_CLIP_FILESIZE_MB` | `25` | Maximum compressed filesize in MB for a single audio file. Enforced on all audio inputs (multimodal chat URLs, speech-to-text uploads, data: URLs, and local file paths) before decoding begins. | | `VLLM_MAX_AUDIO_CLIP_FILESIZE_MB` | `25` | Maximum compressed filesize in MB for a single audio file. Enforced on all audio inputs (multimodal chat URLs, speech-to-text uploads, data: URLs, and local file paths) before decoding begins. |
| `VLLM_MAX_AUDIO_DECODE_DURATION_S` | `600` | Maximum decoded audio duration in seconds. Prevents compressed audio from expanding into gigabytes of float32 PCM. | | `VLLM_MAX_AUDIO_DECODE_DURATION_S` | `600` | Maximum decoded audio duration in seconds. Prevents compressed audio from expanding into gigabytes of float32 PCM. |
| `VLLM_MAX_AUDIO_DECODE_BYTES` | `268435456` (256 MiB) | Maximum float32 PCM bytes that audio decoding may allocate. Guards against sample-rate forgery where an inflated header sample rate bypasses the duration guard while the actual frame count causes a multi-GiB allocation. | | `VLLM_MAX_AUDIO_DECODE_BYTES` | `268435456` (256 MiB) | Maximum float32 PCM bytes that audio decoding may allocate. Guards against sample-rate forgery where an inflated header sample rate bypasses the duration guard while the actual frame count causes a multi-GiB allocation. |
+100
View File
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from concurrent.futures import ThreadPoolExecutor
from io import BytesIO from io import BytesIO
from pathlib import Path from pathlib import Path
from unittest.mock import patch from unittest.mock import patch
@@ -9,6 +11,7 @@ import pybase64 as base64
import pytest import pytest
import soundfile as sf import soundfile as sf
import vllm.envs as envs
from vllm.multimodal.media import AudioMediaIO from vllm.multimodal.media import AudioMediaIO
from vllm.multimodal.media import audio as audio_module from vllm.multimodal.media import audio as audio_module
from vllm.multimodal.media.audio import ( from vllm.multimodal.media.audio import (
@@ -16,6 +19,7 @@ from vllm.multimodal.media.audio import (
load_audio_soundfile, load_audio_soundfile,
load_audio_torchcodec, load_audio_torchcodec,
) )
from vllm.multimodal.media.base import MediaDecodedBudget
from ...conftest import AudioTestAssets from ...conftest import AudioTestAssets
@@ -118,6 +122,102 @@ def _make_flac_bytes(frames: int, channels: int, samplerate: int) -> bytes:
return buf.getvalue() return buf.getvalue()
@pytest.mark.skip_global_cleanup
def test_request_budget_rejects_second_decoded_audio_clip(monkeypatch):
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES", 384)
clip = _make_flac_bytes(frames=64, channels=1, samplerate=16000)
audio_io = AudioMediaIO(decoded_budget=MediaDecodedBudget())
audio_io.load_bytes(clip)
with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES"):
audio_io.load_bytes(clip)
@pytest.mark.skip_global_cleanup
def test_request_budget_allows_parallel_audio_decodes(monkeypatch):
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES", 512)
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_DECODE_BYTES", 256)
budget = MediaDecodedBudget()
audio_io = AudioMediaIO(decoded_budget=budget)
decoding = threading.Barrier(2)
def decode(*args, **kwargs):
assert kwargs["max_decode_bytes"] == 256
decoding.wait(timeout=5)
return np.zeros(32, dtype=np.float32), 16000
with (
patch.object(audio_module, "load_audio", side_effect=decode),
ThreadPoolExecutor(max_workers=2) as pool,
):
results = list(pool.map(audio_io.load_bytes, [b"a", b"b"]))
assert len(results) == 2
assert budget.used == 256
@pytest.mark.skip_global_cleanup
def test_request_budget_waits_for_inflight_audio_decode(monkeypatch):
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES", 256)
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_DECODE_BYTES", 256)
budget = MediaDecodedBudget()
audio_io = AudioMediaIO(decoded_budget=budget)
decoding = threading.Event()
finish_decoding = threading.Event()
second_reservation = threading.Event()
reserve = budget.reserve
decode_calls = 0
def track_reservation(max_bytes):
if decoding.is_set():
second_reservation.set()
return reserve(max_bytes)
def decode(*args, **kwargs):
nonlocal decode_calls
decode_calls += 1
decoding.set()
assert finish_decoding.wait(timeout=5)
return np.zeros(64, dtype=np.float32), 16000
with (
patch.object(budget, "reserve", side_effect=track_reservation),
patch.object(audio_module, "load_audio", side_effect=decode),
ThreadPoolExecutor(max_workers=2) as pool,
):
first = pool.submit(audio_io.load_bytes, b"a")
assert decoding.wait(timeout=5)
second = pool.submit(audio_io.load_bytes, b"b")
try:
assert second_reservation.wait(timeout=5)
finally:
finish_decoding.set()
first.result(timeout=5)
with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES"):
second.result(timeout=5)
assert decode_calls == 1
assert budget.used == 256
@pytest.mark.skip_global_cleanup
def test_request_budget_releases_failed_audio_reservation(monkeypatch):
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES", 256)
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_DECODE_BYTES", 256)
budget = MediaDecodedBudget()
audio_io = AudioMediaIO(decoded_budget=budget)
audio = (np.zeros(64, dtype=np.float32), 16000)
with patch.object(
audio_module, "load_audio", side_effect=[ValueError("invalid audio"), audio]
):
with pytest.raises(ValueError, match="invalid audio"):
audio_io.load_bytes(b"a")
assert audio_io.load_bytes(b"b")[0].nbytes == 256
assert budget.used == 256
def test_small_file_passes_memory_guard(): def test_small_file_passes_memory_guard():
"""A small valid file should pass both duration and memory guards.""" """A small valid file should pass both duration and memory guards."""
payload = _make_flac_bytes(frames=16000, channels=1, samplerate=16000) payload = _make_flac_bytes(frames=16000, channels=1, samplerate=16000)
+64 -2
View File
@@ -11,9 +11,9 @@ from vllm.multimodal.image import (
ALLOWED_IMAGE_FORMATS, ALLOWED_IMAGE_FORMATS,
open_image, open_image,
) )
from vllm.multimodal.media import ImageMediaIO from vllm.multimodal.media import ImageMediaIO, MediaConnector, MediaDecodedBudget
pytestmark = pytest.mark.cpu_test pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
ASSETS_DIR = Path(__file__).parent.parent / "assets" ASSETS_DIR = Path(__file__).parent.parent / "assets"
assert ASSETS_DIR.exists() assert ASSETS_DIR.exists()
@@ -288,6 +288,68 @@ def test_image_pixel_limit_disabled(monkeypatch):
assert result.media.size == (1000, 1000) assert result.media.size == (1000, 1000)
def _png_bytes(width: int, height: int) -> bytes:
from io import BytesIO
buf = BytesIO()
Image.new("RGB", (width, height), (1, 2, 3)).save(buf, format="PNG")
return buf.getvalue()
def test_request_decoded_budget_rejects_aggregate(monkeypatch):
import vllm.envs as envs
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES", 600)
budget = MediaDecodedBudget()
image_io = ImageMediaIO(decoded_budget=budget)
data = _png_bytes(10, 10)
image_io.load_bytes(data)
with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES"):
image_io.load_bytes(data)
def test_request_decoded_budget_accepts_images_within_limit(monkeypatch):
import vllm.envs as envs
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES", 800)
budget = MediaDecodedBudget()
image_io = ImageMediaIO(decoded_budget=budget)
data = _png_bytes(10, 10)
image_io.load_bytes(data)
image_io.load_bytes(data)
assert budget.used == 800
def test_request_decoded_budget_disabled(monkeypatch):
import vllm.envs as envs
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES", 0)
budget = MediaDecodedBudget()
image_io = ImageMediaIO(decoded_budget=budget)
data = _png_bytes(100, 100)
image_io.load_bytes(data)
image_io.load_bytes(data)
assert budget.used == 0
def test_media_connector_enforces_request_decoded_budget(monkeypatch):
import pybase64
import vllm.envs as envs
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES", 600)
b64 = pybase64.b64encode(_png_bytes(10, 10)).decode()
url = f"data:image/png;base64,{b64}"
connector = MediaConnector(decoded_budget=MediaDecodedBudget())
connector.fetch_image(url)
with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES"):
connector.fetch_image(url)
def test_image_media_io_rejects_postscript_payload(): def test_image_media_io_rejects_postscript_payload():
"""EPS/PostScript bytes must not reach an external renderer.""" """EPS/PostScript bytes must not reach an external renderer."""
import pybase64 as base64 import pybase64 as base64
@@ -13,11 +13,16 @@ from unittest.mock import AsyncMock, MagicMock, patch
import aiohttp import aiohttp
import pytest import pytest
from vllm.connections import MediaDownloadSizeExceededError from vllm.connections import (
MediaDownloadSizeExceededError,
MediaRequestDownloadSizeExceededError,
)
from vllm.entrypoints.serve import create_error_response from vllm.entrypoints.serve import create_error_response
from vllm.exceptions import VLLMClientError, VLLMUnprocessableEntityError from vllm.exceptions import VLLMClientError, VLLMUnprocessableEntityError
from vllm.multimodal.media import MediaConnector from vllm.multimodal.media import MediaConnector
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
class TestVLLMUnprocessableEntityError: class TestVLLMUnprocessableEntityError:
"""Tests for VLLMUnprocessableEntityError exception.""" """Tests for VLLMUnprocessableEntityError exception."""
@@ -148,6 +153,21 @@ class TestMediaConnectorErrorHandling:
assert "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB=256" in str(exc_info.value) assert "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB=256" in str(exc_info.value)
def test_fetch_image_request_download_size_limit_error(self):
connector = MediaConnector()
with patch.object(
connector.connection, "get_bytes", new_callable=MagicMock
) as mock_get:
mock_get.side_effect = MediaRequestDownloadSizeExceededError(
"VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB=256"
)
with pytest.raises(VLLMUnprocessableEntityError) as exc_info:
connector.fetch_image("https://example.com/image.jpg")
assert "VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB=256" in str(exc_info.value)
class TestErrorResponse: class TestErrorResponse:
"""Tests for error response creation.""" """Tests for error response creation."""
+15
View File
@@ -9,8 +9,10 @@ import numpy as np
import numpy.typing as npt import numpy.typing as npt
import pybase64 import pybase64
import pytest import pytest
import torch
from PIL import Image from PIL import Image
import vllm.envs as envs
from vllm.assets.base import get_vllm_public_assets from vllm.assets.base import get_vllm_public_assets
from vllm.assets.video import ( from vllm.assets.video import (
video_get_metadata, video_get_metadata,
@@ -18,6 +20,7 @@ from vllm.assets.video import (
video_to_pil_images_list, video_to_pil_images_list,
) )
from vllm.multimodal.media import ImageMediaIO, VideoMediaIO from vllm.multimodal.media import ImageMediaIO, VideoMediaIO
from vllm.multimodal.media.base import MediaDecodedBudget
from vllm.multimodal.video import ( from vllm.multimodal.video import (
PYNVVIDEOCODEC_VIDEO_BACKEND, PYNVVIDEOCODEC_VIDEO_BACKEND,
VIDEO_LOADER_REGISTRY, VIDEO_LOADER_REGISTRY,
@@ -47,6 +50,18 @@ class Assert10Frames1FPSVideoLoader(VideoLoader):
return FAKE_OUTPUT_2 return FAKE_OUTPUT_2
@pytest.mark.skip_global_cleanup
def test_video_request_budget_counts_torch_frames(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES", 16)
budget = MediaDecodedBudget()
video_io = VideoMediaIO(ImageMediaIO(decoded_budget=budget))
frames = torch.zeros((2, 2, 2, 3), dtype=torch.uint8)
video_io.video_loader = SimpleNamespace(load_bytes=lambda *_args, **_kwargs: frames)
with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES"):
video_io.load_bytes(b"video")
def test_video_media_io_kwargs(monkeypatch: pytest.MonkeyPatch): def test_video_media_io_kwargs(monkeypatch: pytest.MonkeyPatch):
with monkeypatch.context() as m: with monkeypatch.context() as m:
m.setenv("VLLM_VIDEO_LOADER_BACKEND", "assert_10_frames_1_fps") m.setenv("VLLM_VIDEO_LOADER_BACKEND", "assert_10_frames_1_fps")
+51 -3
View File
@@ -2,15 +2,19 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import AsyncIterator, Iterator from collections.abc import AsyncIterator, Iterator
from io import BytesIO
from pathlib import Path from pathlib import Path
from unittest.mock import patch from unittest.mock import patch
import numpy as np
import pytest import pytest
from PIL import Image
import vllm.envs as envs import vllm.envs as envs
from vllm.connections import HTTPConnection from vllm.connections import HTTPConnection, MediaRequestDownloadBudget
from vllm.exceptions import VLLMValidationError from vllm.exceptions import VLLMValidationError
from vllm.multimodal.media import AudioMediaIO, MediaConnector from vllm.multimodal.media import AudioMediaIO, ImageMediaIO, MediaConnector
from vllm.multimodal.media.base import MediaDecodedBudget
from vllm.utils.mem_constants import KiB_bytes, MiB_bytes from vllm.utils.mem_constants import KiB_bytes, MiB_bytes
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup] pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
@@ -87,6 +91,46 @@ def test_audio_load_bytes_rejects_oversized(monkeypatch: pytest.MonkeyPatch):
AudioMediaIO().load_bytes(b"\x00" * (MiB_bytes + 1)) AudioMediaIO().load_bytes(b"\x00" * (MiB_bytes + 1))
def test_request_audio_budget_limits_multiple_clips_before_decode(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES", 24)
budget = MediaDecodedBudget()
clip = np.zeros(4, dtype=np.float32)
def decode(_source, **kwargs):
if kwargs["max_decode_bytes"] < clip.nbytes:
raise ValueError("VLLM_MAX_AUDIO_DECODE_BYTES")
return clip, 16000
with patch("vllm.multimodal.media.audio.load_audio", side_effect=decode) as load:
audio_io = AudioMediaIO(decoded_budget=budget)
audio_io.load_bytes(b"first")
with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES"):
audio_io.load_bytes(b"second")
assert budget.used == clip.nbytes
assert load.call_count == 2
def test_request_decoded_budget_is_shared_by_image_and_audio(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES", 24)
budget = MediaDecodedBudget()
image_bytes = BytesIO()
Image.new("RGB", (2, 3)).save(image_bytes, format="PNG")
ImageMediaIO(decoded_budget=budget).load_bytes(image_bytes.getvalue())
with (
patch("vllm.multimodal.media.audio.load_audio") as decode,
pytest.raises(ValueError, match="VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES"),
):
AudioMediaIO(decoded_budget=budget).load_bytes(b"audio")
decode.assert_not_called()
def test_audio_load_file_rejects_oversized( def test_audio_load_file_rejects_oversized(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path monkeypatch: pytest.MonkeyPatch, tmp_path: Path
): ):
@@ -138,12 +182,16 @@ async def test_audio_connector_threads_byte_limit_to_http_reader(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
): ):
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1) monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
budget = MediaRequestDownloadBudget()
class _Connection: class _Connection:
async def async_get_bytes(self, _url: str, **kwargs): async def async_get_bytes(self, _url: str, **kwargs):
assert kwargs["max_bytes"] == MiB_bytes assert kwargs["max_bytes"] == MiB_bytes
assert kwargs["request_budget"] is budget
raise VLLMValidationError("Maximum file size exceeded") raise VLLMValidationError("Maximum file size exceeded")
connector = MediaConnector(connection=_Connection()) connector = MediaConnector( # type: ignore[arg-type]
connection=_Connection(), download_budget=budget
)
with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"): with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"):
await connector.fetch_audio_async("https://example.test/audio") await connector.fetch_audio_async("https://example.test/audio")
+50 -1
View File
@@ -9,7 +9,13 @@ from unittest.mock import AsyncMock, patch
import pytest import pytest
import vllm.envs as envs import vllm.envs as envs
from vllm.connections import HTTPConnection, HTTPResponseSizeExceededError from vllm.connections import (
HTTPConnection,
HTTPResponseSizeExceededError,
MediaRequestDownloadBudget,
)
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
_ONE_MIB = 1024 * 1024 _ONE_MIB = 1024 * 1024
_SMALL_BODY = b"a" * (_ONE_MIB // 2) _SMALL_BODY = b"a" * (_ONE_MIB // 2)
@@ -150,6 +156,49 @@ async def test_async_get_bytes_with_limit_streams_instead_of_materializing() ->
assert response.content.iterated == 2 assert response.content.iterated == 2
def test_request_download_budget_counts_across_responses(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB", 1)
budget = MediaRequestDownloadBudget()
connection = HTTPConnection()
first = _SyncResponse([b"a" * (3 * _ONE_MIB // 4)])
second = _SyncResponse([b"b" * (_ONE_MIB // 2)])
with patch.object(connection, "get_response", side_effect=[first, second]):
connection.get_bytes("http://example.com/one", request_budget=budget)
with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB"):
connection.get_bytes("http://example.com/two", request_budget=budget)
assert budget.used == 3 * _ONE_MIB // 4
assert second.iterated == 1
@pytest.mark.asyncio
async def test_async_request_download_budget_counts_across_responses(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB", 1)
budget = MediaRequestDownloadBudget()
connection = HTTPConnection()
first = _AsyncResponse([b"a" * (3 * _ONE_MIB // 4)])
second = _AsyncResponse([b"b" * (_ONE_MIB // 2)])
with patch.object(
connection, "get_async_response", new=AsyncMock(side_effect=[first, second])
):
await connection.async_get_bytes(
"http://example.com/one", request_budget=budget
)
with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB"):
await connection.async_get_bytes(
"http://example.com/two", request_budget=budget
)
assert budget.used == 3 * _ONE_MIB // 4
assert second.content.iterated == 1
def test_get_bytes_rejects_content_length_over_limit( def test_get_bytes_rejects_content_length_over_limit(
monkeypatch: pytest.MonkeyPatch, local_media_server: str monkeypatch: pytest.MonkeyPatch, local_media_server: str
) -> None: ) -> None:
+57 -8
View File
@@ -3,6 +3,7 @@
import asyncio import asyncio
import functools import functools
import threading
import time import time
from collections.abc import Callable, Coroutine, Mapping, MutableMapping from collections.abc import Callable, Coroutine, Mapping, MutableMapping
from pathlib import Path from pathlib import Path
@@ -60,6 +61,33 @@ class MediaDownloadSizeExceededError(ValueError):
super().__init__(message) super().__init__(message)
class MediaRequestDownloadSizeExceededError(ValueError):
"""Raised when all HTTP media downloads in one request exceed the limit."""
class MediaRequestDownloadBudget:
"""Thread-safe total of HTTP media bytes received for one request."""
def __init__(self) -> None:
self.used = 0
self._lock = threading.Lock()
def charge(self, size: int) -> None:
limit_mb = envs.VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB
if limit_mb <= 0:
return
limit = limit_mb * MiB_bytes
with self._lock:
received = self.used + size
if received > limit:
raise MediaRequestDownloadSizeExceededError(
"Request media downloads exceed "
f"VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB={limit_mb:g} "
f"({received} bytes received)."
)
self.used = received
def _get_media_download_limit_bytes() -> int | None: def _get_media_download_limit_bytes() -> int | None:
max_size_mb = envs.VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB max_size_mb = envs.VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB
if max_size_mb <= 0: if max_size_mb <= 0:
@@ -105,19 +133,26 @@ def _append_chunk(
received_bytes: int, received_bytes: int,
max_bytes: int | None, max_bytes: int | None,
media_download_limit: bool, media_download_limit: bool,
request_budget: MediaRequestDownloadBudget | None = None,
) -> int: ) -> int:
if not chunk: if not chunk:
return received_bytes return received_bytes
received_bytes += len(chunk) received_bytes += len(chunk)
if max_bytes is not None and received_bytes > max_bytes: if max_bytes is not None and received_bytes > max_bytes:
_raise_size_exceeded(max_bytes, received_bytes, media_download_limit) _raise_size_exceeded(max_bytes, received_bytes, media_download_limit)
if request_budget is not None:
request_budget.charge(len(chunk))
chunks.append(chunk) chunks.append(chunk)
return received_bytes return received_bytes
def _read_response_bytes(response: requests.Response, max_bytes: int | None) -> bytes: def _read_response_bytes(
response: requests.Response,
max_bytes: int | None,
request_budget: MediaRequestDownloadBudget | None = None,
) -> bytes:
max_bytes, media_download_limit = _get_response_size_limit(max_bytes) max_bytes, media_download_limit = _get_response_size_limit(max_bytes)
if max_bytes is None: if max_bytes is None and request_budget is None:
return response.content return response.content
_check_content_length( _check_content_length(
@@ -129,16 +164,23 @@ def _read_response_bytes(response: requests.Response, max_bytes: int | None) ->
received_bytes = 0 received_bytes = 0
for chunk in response.iter_content(_RESPONSE_READ_CHUNK_SIZE): for chunk in response.iter_content(_RESPONSE_READ_CHUNK_SIZE):
received_bytes = _append_chunk( received_bytes = _append_chunk(
chunks, chunk, received_bytes, max_bytes, media_download_limit chunks,
chunk,
received_bytes,
max_bytes,
media_download_limit,
request_budget,
) )
return b"".join(chunks) return b"".join(chunks)
async def _async_read_response_bytes( async def _async_read_response_bytes(
response: aiohttp.ClientResponse, max_bytes: int | None response: aiohttp.ClientResponse,
max_bytes: int | None,
request_budget: MediaRequestDownloadBudget | None = None,
) -> bytes: ) -> bytes:
max_bytes, media_download_limit = _get_response_size_limit(max_bytes) max_bytes, media_download_limit = _get_response_size_limit(max_bytes)
if max_bytes is None: if max_bytes is None and request_budget is None:
return await response.read() return await response.read()
_check_content_length(response.content_length, max_bytes, media_download_limit) _check_content_length(response.content_length, max_bytes, media_download_limit)
@@ -146,7 +188,12 @@ async def _async_read_response_bytes(
received_bytes = 0 received_bytes = 0
async for chunk in response.content.iter_chunked(_RESPONSE_READ_CHUNK_SIZE): async for chunk in response.content.iter_chunked(_RESPONSE_READ_CHUNK_SIZE):
received_bytes = _append_chunk( received_bytes = _append_chunk(
chunks, chunk, received_bytes, max_bytes, media_download_limit chunks,
chunk,
received_bytes,
max_bytes,
media_download_limit,
request_budget,
) )
return b"".join(chunks) return b"".join(chunks)
@@ -407,6 +454,7 @@ class HTTPConnection:
timeout: float | None = None, timeout: float | None = None,
allow_redirects: bool = True, allow_redirects: bool = True,
max_bytes: int | None = None, max_bytes: int | None = None,
request_budget: MediaRequestDownloadBudget | None = None,
) -> bytes: ) -> bytes:
with self.get_response( with self.get_response(
url, url,
@@ -416,7 +464,7 @@ class HTTPConnection:
) as r: ) as r:
r.raise_for_status() r.raise_for_status()
return _read_response_bytes(r, max_bytes) return _read_response_bytes(r, max_bytes, request_budget)
@_async_retry @_async_retry
async def async_get_bytes( async def async_get_bytes(
@@ -426,6 +474,7 @@ class HTTPConnection:
timeout: float | None = None, timeout: float | None = None,
allow_redirects: bool = True, allow_redirects: bool = True,
max_bytes: int | None = None, max_bytes: int | None = None,
request_budget: MediaRequestDownloadBudget | None = None,
) -> bytes: ) -> bytes:
async with await self.get_async_response( async with await self.get_async_response(
url, url,
@@ -434,7 +483,7 @@ class HTTPConnection:
) as r: ) as r:
r.raise_for_status() r.raise_for_status()
return await _async_read_response_bytes(r, max_bytes) return await _async_read_response_bytes(r, max_bytes, request_budget)
def get_text(self, url: str, *, timeout: float | None = None) -> str: def get_text(self, url: str, *, timeout: float | None = None) -> str:
with self.get_response(url, timeout=timeout) as r: with self.get_response(url, timeout=timeout) as r:
+13
View File
@@ -78,6 +78,7 @@ if TYPE_CHECKING:
VLLM_MEDIA_CACHE_TTL_HOURS: float = 24 VLLM_MEDIA_CACHE_TTL_HOURS: float = 24
VLLM_MEDIA_FETCH_MAX_RETRIES: int = 3 VLLM_MEDIA_FETCH_MAX_RETRIES: int = 3
VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB: int = 256 VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB: int = 256
VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB: int = 256
VLLM_MEDIA_URL_ALLOW_REDIRECTS: bool = True VLLM_MEDIA_URL_ALLOW_REDIRECTS: bool = True
VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8 VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8
VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25 VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25
@@ -86,6 +87,7 @@ if TYPE_CHECKING:
VLLM_MAX_AUDIO_PREPROCESS_WORKERS: int = max(1, min(os.cpu_count() or 1, 2)) VLLM_MAX_AUDIO_PREPROCESS_WORKERS: int = max(1, min(os.cpu_count() or 1, 2))
VLLM_MAX_EMBED_DECODE_BYTES: int = 2_147_483_648 VLLM_MAX_EMBED_DECODE_BYTES: int = 2_147_483_648
VLLM_MAX_IMAGE_PIXELS: int = 178_956_970 VLLM_MAX_IMAGE_PIXELS: int = 178_956_970
VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES: int = 715_827_880
VLLM_VIDEO_LOADER_BACKEND: str = "opencv" VLLM_VIDEO_LOADER_BACKEND: str = "opencv"
VLLM_MEDIA_CONNECTOR: str = "http" VLLM_MEDIA_CONNECTOR: str = "http"
VLLM_TARGET_DEVICE: str = "cuda" VLLM_TARGET_DEVICE: str = "cuda"
@@ -1063,6 +1065,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_MAX_IMAGE_PIXELS": lambda: int( "VLLM_MAX_IMAGE_PIXELS": lambda: int(
os.getenv("VLLM_MAX_IMAGE_PIXELS", "178956970") os.getenv("VLLM_MAX_IMAGE_PIXELS", "178956970")
), ),
# Maximum aggregate decoded bytes across media in one request.
# The default allows one image at the per-image pixel limit at 4 bytes
# per pixel. Set to 0 to disable.
"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES": lambda: int(
os.getenv("VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES", "715827880")
),
"VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB": lambda: int(
os.getenv("VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB", "256")
),
# Backend for Video IO — selects the frame-sampling algorithm. # Backend for Video IO — selects the frame-sampling algorithm.
# - "opencv": uniform sampling. # - "opencv": uniform sampling.
# - "opencv_dynamic": duration-aware dynamic sampling. # - "opencv_dynamic": duration-aware dynamic sampling.
@@ -2412,6 +2423,8 @@ def compile_factors() -> dict[str, object]:
"VLLM_MAX_AUDIO_PREPROCESS_WORKERS", "VLLM_MAX_AUDIO_PREPROCESS_WORKERS",
"VLLM_MAX_EMBED_DECODE_BYTES", "VLLM_MAX_EMBED_DECODE_BYTES",
"VLLM_MAX_IMAGE_PIXELS", "VLLM_MAX_IMAGE_PIXELS",
"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES",
"VLLM_MAX_MEDIA_REQUEST_DOWNLOAD_SIZE_MB",
"VLLM_VIDEO_LOADER_BACKEND", "VLLM_VIDEO_LOADER_BACKEND",
"VLLM_MEDIA_CONNECTOR", "VLLM_MEDIA_CONNECTOR",
"VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME", "VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME",
+2 -1
View File
@@ -1,13 +1,14 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .audio import AudioEmbeddingMediaIO, AudioMediaIO from .audio import AudioEmbeddingMediaIO, AudioMediaIO
from .base import MediaIO, MediaWithBytes from .base import MediaDecodedBudget, MediaIO, MediaWithBytes
from .connector import MEDIA_CONNECTOR_REGISTRY, MediaConnector from .connector import MEDIA_CONNECTOR_REGISTRY, MediaConnector
from .image import ImageEmbeddingMediaIO, ImageMediaIO from .image import ImageEmbeddingMediaIO, ImageMediaIO
from .video import VIDEO_LOADER_REGISTRY, VideoMediaIO from .video import VIDEO_LOADER_REGISTRY, VideoMediaIO
__all__ = [ __all__ = [
"MediaIO", "MediaIO",
"MediaDecodedBudget",
"MediaWithBytes", "MediaWithBytes",
"AudioEmbeddingMediaIO", "AudioEmbeddingMediaIO",
"AudioMediaIO", "AudioMediaIO",
+44 -16
View File
@@ -23,7 +23,7 @@ from vllm.utils.sparse_utils import (
safe_to_dense, safe_to_dense,
) )
from .base import MediaIO from .base import MediaDecodedBudget, MediaIO
logger = init_logger(__name__) logger = init_logger(__name__)
@@ -527,8 +527,11 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
error handling. error handling.
""" """
def __init__(self, **kwargs) -> None: def __init__(
self, *, decoded_budget: MediaDecodedBudget | None = None, **kwargs
) -> None:
super().__init__() super().__init__()
self.decoded_budget = decoded_budget
# Allow per-server override of the audio decoding backend, e.g.: # Allow per-server override of the audio decoding backend, e.g.:
# --media-io-kwargs '{"audio": {"audio_backend": "torchcodec"}}' # --media-io-kwargs '{"audio": {"audio_backend": "torchcodec"}}'
@@ -564,13 +567,44 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]: def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
self._validate_encoded_size(len(data)) self._validate_encoded_size(len(data))
return load_audio( return self._load_with_budget(BytesIO(data))
BytesIO(data),
sr=None, def _load_with_budget(self, source: BytesIO | Path) -> tuple[npt.NDArray, float]:
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S, budget = self.decoded_budget
max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES, if budget is None:
backend=self.audio_backend, return load_audio(
) source,
sr=None,
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES,
backend=self.audio_backend,
)
max_decode_bytes = envs.VLLM_MAX_AUDIO_DECODE_BYTES
reserved = budget.reserve(max_decode_bytes)
decoded_bytes = 0
try:
try:
audio = load_audio(
source,
sr=None,
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
max_decode_bytes=reserved,
backend=self.audio_backend,
)
except ValueError as exc:
if reserved < max_decode_bytes and "VLLM_MAX_AUDIO_DECODE_BYTES" in str(
exc
):
raise ValueError(
"Request decoded media exceeds "
"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES."
) from exc
raise
decoded_bytes = audio[0].nbytes
return audio
finally:
budget.settle(reserved, decoded_bytes)
def load_base64( def load_base64(
self, self,
@@ -588,13 +622,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]: def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]:
self._validate_encoded_size(filepath.stat().st_size) self._validate_encoded_size(filepath.stat().st_size)
return load_audio( return self._load_with_budget(filepath)
filepath,
sr=None,
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES,
backend=self.audio_backend,
)
def encode_base64( def encode_base64(
self, self,
+71
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections.abc import Iterable, Iterator from collections.abc import Iterable, Iterator
from dataclasses import dataclass, field from dataclasses import dataclass, field
@@ -9,9 +10,79 @@ from typing import Any, Generic, TypeVar, cast
import numpy as np import numpy as np
import vllm.envs as envs
_T = TypeVar("_T") _T = TypeVar("_T")
class MediaDecodedBudget:
"""Thread-safe decoded-media byte accumulator for one request."""
def __init__(self) -> None:
self.used = 0
self._reserved = 0
self._condition = threading.Condition()
def reserve(self, max_bytes: int) -> int:
"""Reserve decoder capacity, waiting for in-flight decodes if needed."""
limit = envs.VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES
if limit <= 0:
return max_bytes
max_bytes = max(0, max_bytes)
with self._condition:
while True:
available = limit - self.used - self._reserved
if available >= max_bytes or self._reserved == 0:
if available <= 0:
raise ValueError(
"Request decoded media exceeds "
"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES."
)
reserved = min(max_bytes, available)
self._reserved += reserved
return reserved
self._condition.wait()
def settle(self, reserved: int, decoded_bytes: int) -> None:
"""Replace a decoder reservation with its actual decoded size."""
limit = envs.VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES
if limit <= 0:
return
with self._condition:
self._reserved -= reserved
try:
used = self.used + decoded_bytes
if decoded_bytes > reserved or used > limit:
raise ValueError(
"Request decoded media exceeds "
f"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES={limit} "
f"({used} bytes)."
)
self.used = used
finally:
self._condition.notify_all()
def charge(self, decoded_bytes: int) -> None:
limit = envs.VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES
if limit <= 0 or decoded_bytes <= 0:
return
with self._condition:
while (
self.used + self._reserved + decoded_bytes > limit
and self._reserved > 0
and self.used + decoded_bytes <= limit
):
self._condition.wait()
used = self.used + decoded_bytes
if used > limit:
raise ValueError(
"Request decoded media exceeds "
f"VLLM_MAX_MEDIA_REQUEST_DECODE_BYTES={limit} "
f"({used} bytes)."
)
self.used = used
@dataclass @dataclass
class MediaWithBytes(Generic[_T]): class MediaWithBytes(Generic[_T]):
"""Wrapper that couples a media object with its original encoded bytes. """Wrapper that couples a media object with its original encoded bytes.
+38 -16
View File
@@ -25,6 +25,8 @@ import vllm.envs as envs
from vllm.connections import ( from vllm.connections import (
HTTPConnection, HTTPConnection,
MediaDownloadSizeExceededError, MediaDownloadSizeExceededError,
MediaRequestDownloadBudget,
MediaRequestDownloadSizeExceededError,
global_http_connection, global_http_connection,
) )
from vllm.exceptions import VLLMUnprocessableEntityError, VLLMValidationError from vllm.exceptions import VLLMUnprocessableEntityError, VLLMValidationError
@@ -33,7 +35,7 @@ from vllm.multimodal.video import get_video_loader_backend_for_processor
from vllm.utils.registry import ExtensionManager from vllm.utils.registry import ExtensionManager
from .audio import AudioEmbeddingMediaIO, AudioMediaIO from .audio import AudioEmbeddingMediaIO, AudioMediaIO
from .base import MediaIO, MediaWithBytes from .base import MediaDecodedBudget, MediaIO, MediaWithBytes
from .image import ImageEmbeddingMediaIO, ImageMediaIO from .image import ImageEmbeddingMediaIO, ImageMediaIO
from .video import VideoEmbeddingMediaIO, VideoMediaIO from .video import VideoEmbeddingMediaIO, VideoMediaIO
@@ -109,7 +111,9 @@ def _wrap_media_fetch_error(
value=url, value=url,
) )
if isinstance(exc, MediaDownloadSizeExceededError): if isinstance(
exc, (MediaDownloadSizeExceededError, MediaRequestDownloadSizeExceededError)
):
return VLLMUnprocessableEntityError( return VLLMUnprocessableEntityError(
f"Failed to fetch media from URL: {exc}", f"Failed to fetch media from URL: {exc}",
parameter="image_url", parameter="image_url",
@@ -162,6 +166,8 @@ class MediaConnector:
*, *,
allowed_local_media_path: str = "", allowed_local_media_path: str = "",
allowed_media_domains: list[str] | None = None, allowed_media_domains: list[str] | None = None,
decoded_budget: MediaDecodedBudget | None = None,
download_budget: MediaRequestDownloadBudget | None = None,
) -> None: ) -> None:
"""Args: """Args:
media_io_kwargs: Additional args passed to process media media_io_kwargs: Additional args passed to process media
@@ -201,6 +207,8 @@ class MediaConnector:
if allowed_media_domains is None: if allowed_media_domains is None:
allowed_media_domains = [] allowed_media_domains = []
self.allowed_media_domains = allowed_media_domains self.allowed_media_domains = allowed_media_domains
self.decoded_budget = decoded_budget
self.download_budget = download_budget
# Media download cache (opt-in via VLLM_MEDIA_CACHE) # Media download cache (opt-in via VLLM_MEDIA_CACHE)
self._media_cache_dir: str | None = None self._media_cache_dir: str | None = None
@@ -230,6 +238,12 @@ class MediaConnector:
media_cache, media_cache,
) )
def _image_io(self, image_mode: str | None = "RGB") -> ImageMediaIO:
return ImageMediaIO(
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {})),
decoded_budget=self.decoded_budget,
)
def _get_cached_bytes(self, url: str) -> bytes | None: def _get_cached_bytes(self, url: str) -> bytes | None:
"""Return cached bytes for a URL, or None if not cached/expired.""" """Return cached bytes for a URL, or None if not cached/expired."""
if not self._media_cache_dir: if not self._media_cache_dir:
@@ -385,12 +399,18 @@ class MediaConnector:
return media_io.load_bytes(cached) return media_io.load_bytes(cached)
connection = self.connection connection = self.connection
budget_kwargs = (
{"request_budget": self.download_budget}
if self.download_budget is not None
else {}
)
try: try:
data = connection.get_bytes( data = connection.get_bytes(
url_spec.url, url_spec.url,
timeout=fetch_timeout, timeout=fetch_timeout,
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS, allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
max_bytes=max_bytes, max_bytes=max_bytes,
**budget_kwargs,
) )
except Exception as e: except Exception as e:
wrapped = _wrap_media_fetch_error(url, e) wrapped = _wrap_media_fetch_error(url, e)
@@ -438,12 +458,18 @@ class MediaConnector:
return await future return await future
connection = self.connection connection = self.connection
budget_kwargs = (
{"request_budget": self.download_budget}
if self.download_budget is not None
else {}
)
try: try:
data = await connection.async_get_bytes( data = await connection.async_get_bytes(
url_spec.url, url_spec.url,
timeout=fetch_timeout, timeout=fetch_timeout,
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS, allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
max_bytes=max_bytes, max_bytes=max_bytes,
**budget_kwargs,
) )
except Exception as e: except Exception as e:
wrapped = _wrap_media_fetch_error(url, e) wrapped = _wrap_media_fetch_error(url, e)
@@ -470,7 +496,9 @@ class MediaConnector:
audio_url: str, audio_url: str,
) -> tuple[np.ndarray, int | float]: ) -> tuple[np.ndarray, int | float]:
"""Load audio from a URL.""" """Load audio from a URL."""
audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {})) audio_io = AudioMediaIO(
decoded_budget=self.decoded_budget, **self.media_io_kwargs.get("audio", {})
)
return self.load_from_url( return self.load_from_url(
audio_url, audio_url,
@@ -483,7 +511,9 @@ class MediaConnector:
audio_url: str, audio_url: str,
) -> tuple[np.ndarray, int | float]: ) -> tuple[np.ndarray, int | float]:
"""Asynchronously fetch audio from a URL.""" """Asynchronously fetch audio from a URL."""
audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {})) audio_io = AudioMediaIO(
decoded_budget=self.decoded_budget, **self.media_io_kwargs.get("audio", {})
)
return await self.load_from_url_async( return await self.load_from_url_async(
audio_url, audio_url,
@@ -503,9 +533,7 @@ class MediaConnector:
`media_io_kwargs={"image": {"image_mode": None}}` to keep the `media_io_kwargs={"image": {"image_mode": None}}` to keep the
original image mode (e.g. preserving the alpha channel). original image mode (e.g. preserving the alpha channel).
""" """
image_io = ImageMediaIO( image_io = self._image_io(image_mode)
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
)
try: try:
return self.load_from_url( return self.load_from_url(
@@ -529,9 +557,7 @@ class MediaConnector:
`media_io_kwargs={"image": {"image_mode": None}}` to keep the `media_io_kwargs={"image": {"image_mode": None}}` to keep the
original image mode (e.g. preserving the alpha channel). original image mode (e.g. preserving the alpha channel).
""" """
image_io = ImageMediaIO( image_io = self._image_io(image_mode)
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
)
try: try:
return await self.load_from_url_async( return await self.load_from_url_async(
@@ -551,9 +577,7 @@ class MediaConnector:
video_processor: str | None = None, video_processor: str | None = None,
) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]: ) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
"""Load video from an HTTP or base64 data URL.""" """Load video from an HTTP or base64 data URL."""
image_io = ImageMediaIO( image_io = self._image_io(image_mode)
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
)
video_io_kwargs = dict(self.media_io_kwargs.get("video", {})) video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
if "video_backend" not in video_io_kwargs and ( if "video_backend" not in video_io_kwargs and (
video_backend := get_video_loader_backend_for_processor(video_processor) video_backend := get_video_loader_backend_for_processor(video_processor)
@@ -580,9 +604,7 @@ class MediaConnector:
`media_io_kwargs={"image": {"image_mode": None}}` to keep the `media_io_kwargs={"image": {"image_mode": None}}` to keep the
original image mode (e.g. preserving the alpha channel). original image mode (e.g. preserving the alpha channel).
""" """
image_io = ImageMediaIO( image_io = self._image_io(image_mode)
**({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
)
video_io_kwargs = dict(self.media_io_kwargs.get("video", {})) video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
if "video_backend" not in video_io_kwargs and ( if "video_backend" not in video_io_kwargs and (
video_backend := get_video_loader_backend_for_processor(video_processor) video_backend := get_video_loader_backend_for_processor(video_processor)
+11 -2
View File
@@ -17,7 +17,7 @@ from vllm.utils.sparse_utils import (
) )
from ..image import convert_image_mode, normalize_image, open_image, rgba_to_rgb from ..image import convert_image_mode, normalize_image, open_image, rgba_to_rgb
from .base import MediaIO, MediaWithBytes from .base import MediaDecodedBudget, MediaIO, MediaWithBytes
MAGIC_NUMPY_PREFIX = b"\x93NUMPY" # https://numpy.org/devdocs/reference/generated/numpy.lib.format.html#format-version-1-0 MAGIC_NUMPY_PREFIX = b"\x93NUMPY" # https://numpy.org/devdocs/reference/generated/numpy.lib.format.html#format-version-1-0
@@ -28,12 +28,19 @@ class ImageMediaIO(MediaIO[Image.Image]):
error handling. error handling.
""" """
def __init__(self, image_mode: str | None = "RGB", **kwargs) -> None: def __init__(
self,
image_mode: str | None = "RGB",
*,
decoded_budget: MediaDecodedBudget | None = None,
**kwargs,
) -> None:
super().__init__() super().__init__()
# Target mode for loaded images; `None` keeps the original mode # Target mode for loaded images; `None` keeps the original mode
# (i.e. no conversion, alpha channel is preserved as-is). # (i.e. no conversion, alpha channel is preserved as-is).
self.image_mode = image_mode self.image_mode = image_mode
self.decoded_budget = decoded_budget
# `kwargs` contains custom arguments from # `kwargs` contains custom arguments from
# --media-io-kwargs for this modality, merged with # --media-io-kwargs for this modality, merged with
# per-request runtime media_io_kwargs via merge_kwargs(). # per-request runtime media_io_kwargs via merge_kwargs().
@@ -87,6 +94,8 @@ class ImageMediaIO(MediaIO[Image.Image]):
f"the maximum of {max_pixels} pixels. Set " f"the maximum of {max_pixels} pixels. Set "
f"VLLM_MAX_IMAGE_PIXELS to increase this limit." f"VLLM_MAX_IMAGE_PIXELS to increase this limit."
) )
if self.decoded_budget is not None:
self.decoded_budget.charge(w * h * 4)
image = normalize_image(image) image = normalize_image(image)
image.load() image.load()
converted = self._convert_image_mode(image) converted = self._convert_image_mode(image)
+10
View File
@@ -122,6 +122,16 @@ class VideoMediaIO(MediaIO[MediaWithBytes[tuple[DecodedFrames, dict[str, Any]]]]
video = self.video_loader.load_bytes( video = self.video_loader.load_bytes(
data, num_frames=self.num_frames, **self.kwargs data, num_frames=self.num_frames, **self.kwargs
) )
frames = video[0] if isinstance(video, tuple) else video
budget = getattr(self.image_io, "decoded_budget", None)
shape = getattr(frames, "shape", None)
if budget is not None and shape is not None and len(shape) >= 3:
decoded_bytes = (
frames.numel() * frames.element_size()
if isinstance(frames, torch.Tensor)
else frames.nbytes
)
budget.charge(decoded_bytes)
return MediaWithBytes(video, data) return MediaWithBytes(video, data)
def load_base64( def load_base64(
+16
View File
@@ -53,6 +53,7 @@ from typing_extensions import TypedDict, override
from vllm import envs from vllm import envs
from vllm.config import ModelConfig from vllm.config import ModelConfig
from vllm.connections import MediaRequestDownloadBudget
from vllm.exceptions import VLLMValidationError from vllm.exceptions import VLLMValidationError
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
from vllm.logger import init_logger from vllm.logger import init_logger
@@ -67,6 +68,7 @@ from vllm.multimodal.inputs import (
VisionChunkVideo, VisionChunkVideo,
) )
from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY, MediaConnector from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY, MediaConnector
from vllm.multimodal.media.base import MediaDecodedBudget
from vllm.multimodal.processing import BaseMultiModalProcessor from vllm.multimodal.processing import BaseMultiModalProcessor
from vllm.renderers.embed_utils import ( from vllm.renderers.embed_utils import (
safe_load_prompt_embeds, safe_load_prompt_embeds,
@@ -609,6 +611,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
self._items_by_modality = defaultdict[str, list[_T]](list) self._items_by_modality = defaultdict[str, list[_T]](list)
# Track original modality for each vision_chunk item (image or video) # Track original modality for each vision_chunk item (image or video)
self._modality_order = defaultdict[str, list[str]](list) self._modality_order = defaultdict[str, list[str]](list)
self.decoded_budget = MediaDecodedBudget()
self.download_budget = MediaRequestDownloadBudget()
@cached_property @cached_property
def use_unified_vision_chunk_modality(self) -> bool: def use_unified_vision_chunk_modality(self) -> bool:
@@ -1052,6 +1056,8 @@ class MultiModalContentParser(BaseMultiModalContentParser):
media_io_kwargs=self._tracker.media_io_kwargs, media_io_kwargs=self._tracker.media_io_kwargs,
allowed_local_media_path=self._tracker.allowed_local_media_path, allowed_local_media_path=self._tracker.allowed_local_media_path,
allowed_media_domains=self._tracker.allowed_media_domains, allowed_media_domains=self._tracker.allowed_media_domains,
decoded_budget=self._tracker.decoded_budget,
download_budget=self._tracker.download_budget,
) )
@property @property
@@ -1138,6 +1144,10 @@ class MultiModalContentParser(BaseMultiModalContentParser):
def parse_image_pil( def parse_image_pil(
self, image_pil: Image.Image | None, uuid: str | None = None self, image_pil: Image.Image | None, uuid: str | None = None
) -> None: ) -> None:
if image_pil is not None:
self._tracker.decoded_budget.charge(
image_pil.size[0] * image_pil.size[1] * 4
)
placeholder = self._tracker.add("image", (image_pil, uuid)) placeholder = self._tracker.add("image", (image_pil, uuid))
self._add_placeholder("image", placeholder) self._add_placeholder("image", placeholder)
@@ -1233,6 +1243,8 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
media_io_kwargs=self._tracker.media_io_kwargs, media_io_kwargs=self._tracker.media_io_kwargs,
allowed_local_media_path=self._tracker.allowed_local_media_path, allowed_local_media_path=self._tracker.allowed_local_media_path,
allowed_media_domains=self._tracker.allowed_media_domains, allowed_media_domains=self._tracker.allowed_media_domains,
decoded_budget=self._tracker.decoded_budget,
download_budget=self._tracker.download_budget,
) )
@property @property
@@ -1351,6 +1363,10 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
image_pil: Image.Image | None, image_pil: Image.Image | None,
uuid: str | None = None, uuid: str | None = None,
) -> None: ) -> None:
if image_pil is not None:
self._tracker.decoded_budget.charge(
image_pil.size[0] * image_pil.size[1] * 4
)
placeholder = self._tracker.add( placeholder = self._tracker.add(
"image", partial(self._item_with_uuid_async, image_pil, uuid) "image", partial(self._item_with_uuid_async, image_pil, uuid)
) )