diff --git a/docs/usage/security.md b/docs/usage/security.md index 3a936bc9e6..b0abc710af 100644 --- a/docs/usage/security.md +++ b/docs/usage/security.md @@ -85,13 +85,15 @@ significantly reduce the attack surface for these types of abuse. Also, consider setting `VLLM_MEDIA_URL_ALLOW_REDIRECTS=0` to prevent HTTP redirects from being followed to bypass domain restrictions. -### 5. **Restrict Media Decode Sizes:** +### 5. **Restrict Media Download and Decode Sizes:** -Compressed media files can expand into gigabytes of memory during decoding. vLLM -enforces decode-size limits to prevent out-of-memory denial of service: +Remote media responses and compressed media files can expand into gigabytes of +memory. vLLM enforces download and decode-size limits to prevent out-of-memory +denial of service: | Environment Variable | Default | Description | | --- | --- | --- | +| `VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB` | `256` | Maximum size in MB for a single remote media response. Oversized responses are rejected while streaming before the full body is materialized in memory. | | `VLLM_MAX_IMAGE_PIXELS` | `178956970` (~179M pixels) | Maximum decoded image size in pixels. Images exceeding this are rejected before raster memory is allocated. Default matches PIL's built-in 2x decompression-bomb threshold (~680 MB for RGB). | | `VLLM_MAX_AUDIO_CLIP_FILESIZE_MB` | `25` | Maximum filesize in MB for a single audio file. | | `VLLM_MAX_AUDIO_DECODE_DURATION_S` | `600` | Maximum decoded audio duration in seconds. Prevents compressed audio from expanding into gigabytes of float32 PCM. | diff --git a/tests/entrypoints/openai/test_run_batch.py b/tests/entrypoints/openai/test_run_batch.py index b2f01cfd7e..58e19ab221 100644 --- a/tests/entrypoints/openai/test_run_batch.py +++ b/tests/entrypoints/openai/test_run_batch.py @@ -4,16 +4,25 @@ import json import subprocess import tempfile +import threading +from collections.abc import Generator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from unittest.mock import AsyncMock, MagicMock, patch import pytest +import vllm.envs as envs from vllm.assets.audio import AudioAsset +from vllm.connections import HTTPConnection +from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.entrypoints.openai.run_batch import ( BatchRequestOutput, + BatchTranscriptionRequest, download_bytes_from_url, + make_transcription_wrapper, ) from vllm.exceptions import VLLMValidationError +from vllm.utils.mem_constants import MiB_bytes CHAT_MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM" EMBEDDING_MODEL_NAME = "intfloat/multilingual-e5-small" @@ -281,6 +290,74 @@ INPUT_REASONING_BATCH = "\n".join( ) MINIMAL_WAV_BASE64 = "UklGRigAAABXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YQQAAAAAAP9/" +_EXACT_LIMIT_AUDIO = b"a" * MiB_bytes +_OVERSIZED_BASE64_AUDIO = "A" * (4 * ((MiB_bytes + 2) // 3)) +_OVERSIZED_HTTP_AUDIO = b"b" * (MiB_bytes + 1) + + +class _BatchHTTPServer(ThreadingHTTPServer): + request_paths: list[str] + + +class _BatchHTTPHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + server = self.server + assert isinstance(server, _BatchHTTPServer) + server.request_paths.append(self.path) + + if self.path == "/exact-limit": + self._send_body(_EXACT_LIMIT_AUDIO) + return + + if self.path == "/oversized-chunked": + self.send_response(200) + self.end_headers() + self._write_body(_OVERSIZED_HTTP_AUDIO) + return + + self.send_error(404, "Unknown batch HTTP test path") + + def _send_body(self, body: bytes) -> None: + self.send_response(200) + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self._write_body(body) + + def _write_body(self, body: bytes) -> None: + try: + for offset in range(0, len(body), 64 * 1024): + self.wfile.write(body[offset : offset + 64 * 1024]) + except (BrokenPipeError, ConnectionResetError): + self.close_connection = True + + def log_message(self, fmt: str, *args: object) -> None: + return + + +@pytest.fixture +def batch_http_server() -> Generator[_BatchHTTPServer, None, None]: + server = _BatchHTTPServer(("127.0.0.1", 0), _BatchHTTPHandler) + server.request_paths = [] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + thread.join() + server.server_close() + + +def _batch_http_url(server: _BatchHTTPServer, path: str) -> str: + return f"http://localhost:{server.server_port}{path}" + + +async def _close_async_connection(connection: HTTPConnection) -> None: + if connection._async_client is not None: + await connection._async_client.close() + + INPUT_TRANSCRIPTION_BATCH = ( json.dumps( { @@ -761,8 +838,15 @@ def test_tool_calling(): def _make_aiohttp_mocks(response_data: bytes = b"fake-data", status: int = 200): """Create mock objects that simulate aiohttp.ClientSession context managers.""" + + async def iter_chunked(chunk_size: int): + del chunk_size + yield response_data + mock_resp = MagicMock() mock_resp.status = status + mock_resp.content_length = len(response_data) + mock_resp.content.iter_chunked = iter_chunked mock_resp.read = AsyncMock(return_value=response_data) mock_resp.__aenter__ = AsyncMock(return_value=mock_resp) mock_resp.__aexit__ = AsyncMock(return_value=False) @@ -892,3 +976,79 @@ async def test_download_bytes_backslash_bypass(): await download_bytes_from_url( bypass_url, allowed_media_domains=["evil.internal"] ) + + +@pytest.mark.asyncio +async def test_transcription_wrapper_rejects_oversized_data_url_before_decode( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1) + handler = AsyncMock() + wrapped_handler = make_transcription_wrapper(is_translation=False)(handler) + request = BatchTranscriptionRequest.model_validate( + { + "model": SPEECH_LARGE_MODEL_NAME, + "file_url": f"data:audio/wav;base64,{_OVERSIZED_BASE64_AUDIO}", + "response_format": "json", + } + ) + + with patch("vllm.entrypoints.openai.run_batch.base64.b64decode") as decode: + response = await wrapped_handler(request) + + assert isinstance(response, ErrorResponse) + assert "Maximum file size exceeded" in response.error.message + decode.assert_not_called() + handler.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_download_bytes_allows_http_body_at_audio_limit( + monkeypatch: pytest.MonkeyPatch, + batch_http_server: _BatchHTTPServer, +): + monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1) + connection = HTTPConnection() + monkeypatch.setattr( + "vllm.entrypoints.openai.run_batch.global_http_connection", + connection, + ) + + try: + result = await download_bytes_from_url( + _batch_http_url(batch_http_server, "/exact-limit") + ) + assert result == _EXACT_LIMIT_AUDIO + finally: + await _close_async_connection(connection) + + +@pytest.mark.asyncio +async def test_transcription_wrapper_rejects_oversized_http_before_handler( + monkeypatch: pytest.MonkeyPatch, + batch_http_server: _BatchHTTPServer, +): + monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1) + connection = HTTPConnection() + monkeypatch.setattr( + "vllm.entrypoints.openai.run_batch.global_http_connection", + connection, + ) + handler = AsyncMock() + wrapped_handler = make_transcription_wrapper(is_translation=False)(handler) + request = BatchTranscriptionRequest.model_validate( + { + "model": SPEECH_LARGE_MODEL_NAME, + "file_url": _batch_http_url(batch_http_server, "/oversized-chunked"), + "response_format": "json", + } + ) + + try: + response = await wrapped_handler(request) + finally: + await _close_async_connection(connection) + + assert isinstance(response, ErrorResponse) + assert "Maximum file size exceeded" in response.error.message + handler.assert_not_awaited() diff --git a/tests/entrypoints/speech_to_text/transcription/test_chunk_timestamp_offset.py b/tests/entrypoints/speech_to_text/transcription/test_chunk_timestamp_offset.py index 22231e0d85..cd61507728 100644 --- a/tests/entrypoints/speech_to_text/transcription/test_chunk_timestamp_offset.py +++ b/tests/entrypoints/speech_to_text/transcription/test_chunk_timestamp_offset.py @@ -41,7 +41,6 @@ async def test_chunk_offsets_are_cumulative_not_nominal(): overlap_chunk_second=1, min_energy_split_window_size=1600, ) - serving.max_audio_filesize_mb = 100.0 serving.model_cls = MagicMock() serving.model_cls.validate_language.side_effect = lambda lang: lang serving.model_cls.supports_explicit_language_detection = False diff --git a/tests/multimodal/media/test_unprocessable_entity_error.py b/tests/multimodal/media/test_unprocessable_entity_error.py index bca2ac5f5e..6d3c2a0b0a 100644 --- a/tests/multimodal/media/test_unprocessable_entity_error.py +++ b/tests/multimodal/media/test_unprocessable_entity_error.py @@ -13,6 +13,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import aiohttp import pytest +from vllm.connections import MediaDownloadSizeExceededError from vllm.entrypoints.serve import create_error_response from vllm.exceptions import VLLMClientError, VLLMUnprocessableEntityError from vllm.multimodal.media import MediaConnector @@ -134,6 +135,19 @@ class TestMediaConnectorErrorHandling: assert isinstance(exc_info.value, aiohttp.ClientConnectionError) + def test_fetch_image_download_size_limit_error(self): + connector = MediaConnector() + + with patch.object( + connector.connection, "get_bytes", new_callable=MagicMock + ) as mock_get: + mock_get.side_effect = MediaDownloadSizeExceededError(256 * 1024 * 1024) + + with pytest.raises(VLLMUnprocessableEntityError) as exc_info: + connector.fetch_image("https://example.com/image.jpg") + + assert "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB=256" in str(exc_info.value) + class TestErrorResponse: """Tests for error response creation.""" diff --git a/tests/test_audio_media_size_precheck.py b/tests/test_audio_media_size_precheck.py new file mode 100644 index 0000000000..f9a4d772e8 --- /dev/null +++ b/tests/test_audio_media_size_precheck.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from collections.abc import AsyncIterator, Iterator +from unittest.mock import patch + +import pytest + +import vllm.envs as envs +from vllm.connections import HTTPConnection +from vllm.exceptions import VLLMValidationError +from vllm.multimodal.media import AudioMediaIO, MediaConnector +from vllm.utils.mem_constants import KiB_bytes, MiB_bytes + +pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup] + + +class _SyncResponse: + def __init__(self, chunks: list[bytes], content_length: int | None = None): + self.headers = ( + {} if content_length is None else {"content-length": str(content_length)} + ) + self._chunks = chunks + self.iterated = 0 + + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def raise_for_status(self) -> None: + return None + + def iter_content(self, chunk_size: int) -> Iterator[bytes]: + assert chunk_size == 64 * KiB_bytes + for chunk in self._chunks: + self.iterated += 1 + yield chunk + + +class _AsyncContent: + def __init__(self, chunks: list[bytes]): + self._chunks = chunks + self.iterated = 0 + + async def iter_chunked(self, chunk_size: int) -> AsyncIterator[bytes]: + assert chunk_size == 64 * KiB_bytes + for chunk in self._chunks: + self.iterated += 1 + yield chunk + + +class _AsyncResponse: + def __init__(self, chunks: list[bytes], content_length: int | None = None): + self.content_length = content_length + self.content = _AsyncContent(chunks) + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + def raise_for_status(self) -> None: + return None + + +def test_audio_base64_rejects_before_decode(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1) + max_encoded_chars = 4 * ((MiB_bytes + 2) // 3) + + with ( + patch("vllm.multimodal.media.audio.pybase64.b64decode") as decode, + pytest.raises(VLLMValidationError, match="Maximum file size exceeded"), + ): + AudioMediaIO().load_base64("audio/wav", "A" * (max_encoded_chars + 1)) + + decode.assert_not_called() + + +def test_sync_http_reader_rejects_from_content_length_before_body_read(): + response = _SyncResponse([b"A" * 10], content_length=MiB_bytes + 1) + connection = HTTPConnection() + + with ( + patch.object(connection, "get_response", return_value=response) as get_response, + pytest.raises(VLLMValidationError, match="Maximum file size exceeded"), + ): + connection.get_bytes("https://example.test/audio", max_bytes=MiB_bytes) + + assert get_response.call_args.kwargs["stream"] is True + assert response.iterated == 0 + + +@pytest.mark.asyncio +async def test_async_http_reader_stops_after_first_over_limit_chunk(): + response = _AsyncResponse([b"A" * (64 * KiB_bytes), b"B", b"C" * 10]) + connection = HTTPConnection() + + async def get_async_response(*_args, **_kwargs): + return response + + with ( + patch.object(connection, "get_async_response", side_effect=get_async_response), + pytest.raises(VLLMValidationError, match="Maximum file size exceeded"), + ): + await connection.async_get_bytes( + "https://example.test/audio", + max_bytes=64 * KiB_bytes, + ) + + assert response.content.iterated == 2 + + +@pytest.mark.asyncio +async def test_audio_connector_threads_byte_limit_to_http_reader( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1) + + class _Connection: + async def async_get_bytes(self, _url: str, **kwargs): + assert kwargs["max_bytes"] == MiB_bytes + raise VLLMValidationError("Maximum file size exceeded") + + connector = MediaConnector(connection=_Connection()) # type: ignore[arg-type] + with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"): + await connector.fetch_audio_async("https://example.test/audio") diff --git a/tests/test_connections.py b/tests/test_connections.py new file mode 100644 index 0000000000..983e20f6c1 --- /dev/null +++ b/tests/test_connections.py @@ -0,0 +1,192 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import threading +from collections.abc import Generator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest.mock import AsyncMock, patch + +import pytest + +import vllm.envs as envs +from vllm.connections import HTTPConnection, HTTPResponseSizeExceededError + +_ONE_MIB = 1024 * 1024 +_SMALL_BODY = b"a" * (_ONE_MIB // 2) +_LARGE_BODY = b"b" * (2 * _ONE_MIB) +_CHUNK_SIZE = 64 * 1024 + + +class _MediaHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + body = _LARGE_BODY if "large" in self.path else _SMALL_BODY + self.send_response(200) + self.send_header("Content-Type", "application/octet-stream") + if self.path != "/chunked-large": + self.send_header("Content-Length", str(len(body))) + self.end_headers() + + try: + for offset in range(0, len(body), _CHUNK_SIZE): + self.wfile.write(body[offset : offset + _CHUNK_SIZE]) + except (BrokenPipeError, ConnectionResetError): + self.close_connection = True + + def log_message(self, fmt: str, *args: object) -> None: + return + + +@pytest.fixture +def local_media_server() -> Generator[str, None, None]: + server = ThreadingHTTPServer(("127.0.0.1", 0), _MediaHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + thread.join() + server.server_close() + + +class _SyncResponse: + def __init__(self, chunks: list[bytes]) -> None: + self.headers: dict[str, str] = {} + self._chunks = chunks + self.content_accessed = False + self.iterated = 0 + + @property + def content(self) -> bytes: + self.content_accessed = True + raise AssertionError("bounded reads must not access response.content") + + def iter_content(self, chunk_size: int): + del chunk_size + for chunk in self._chunks: + self.iterated += 1 + yield chunk + + def raise_for_status(self) -> None: + return + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return False + + +class _AsyncContent: + def __init__(self, chunks: list[bytes]) -> None: + self._chunks = chunks + self.iterated = 0 + + async def iter_chunked(self, chunk_size: int): + del chunk_size + for chunk in self._chunks: + self.iterated += 1 + yield chunk + + +class _AsyncResponse: + def __init__(self, chunks: list[bytes]) -> None: + self.content = _AsyncContent(chunks) + self.content_length = None + self.read = AsyncMock( + side_effect=AssertionError("bounded reads must not call response.read()") + ) + + def raise_for_status(self) -> None: + return + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_value, traceback): + return False + + +def test_get_bytes_with_limit_streams_instead_of_materializing() -> None: + response = _SyncResponse([b"abcd", b"efgh", b"ijkl"]) + connection = HTTPConnection() + + with ( + patch.object( + connection, + "get_response", + return_value=response, + ) as get_response, + pytest.raises(HTTPResponseSizeExceededError, match="maximum size"), + ): + connection.get_bytes("http://example.com/media", max_bytes=4) + + get_response.assert_called_once_with( + "http://example.com/media", + stream=True, + timeout=None, + allow_redirects=True, + ) + assert response.content_accessed is False + assert response.iterated == 2 + + +@pytest.mark.asyncio +async def test_async_get_bytes_with_limit_streams_instead_of_materializing() -> None: + response = _AsyncResponse([b"abcd", b"efgh", b"ijkl"]) + connection = HTTPConnection() + + with ( + patch.object( + connection, + "get_async_response", + new=AsyncMock(return_value=response), + ), + pytest.raises(HTTPResponseSizeExceededError, match="maximum size"), + ): + await connection.async_get_bytes("http://example.com/media", max_bytes=4) + + response.read.assert_not_awaited() + assert response.content.iterated == 2 + + +def test_get_bytes_rejects_content_length_over_limit( + monkeypatch: pytest.MonkeyPatch, local_media_server: str +) -> None: + monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", 1, raising=False) + connection = HTTPConnection(reuse_client=False) + + with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB"): + connection.get_bytes(f"{local_media_server}/large") + + +@pytest.mark.asyncio +async def test_async_get_bytes_rejects_stream_over_limit_without_content_length( + monkeypatch: pytest.MonkeyPatch, local_media_server: str +) -> None: + monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", 1, raising=False) + connection = HTTPConnection(reuse_client=False) + + try: + with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB"): + await connection.async_get_bytes(f"{local_media_server}/chunked-large") + finally: + if connection._async_client is not None: + await connection._async_client.close() + + +@pytest.mark.asyncio +async def test_get_bytes_allows_body_within_limit( + monkeypatch: pytest.MonkeyPatch, local_media_server: str +) -> None: + monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", 1, raising=False) + connection = HTTPConnection(reuse_client=False) + + assert connection.get_bytes(f"{local_media_server}/small") == _SMALL_BODY + + try: + actual = await connection.async_get_bytes(f"{local_media_server}/small") + assert actual == _SMALL_BODY + finally: + if connection._async_client is not None: + await connection._async_client.close() diff --git a/vllm/connections.py b/vllm/connections.py index 8ef715f804..b06d2d563f 100644 --- a/vllm/connections.py +++ b/vllm/connections.py @@ -13,7 +13,9 @@ import requests from urllib3.util import parse_url import vllm.envs as envs +from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger +from vllm.utils.mem_constants import KiB_bytes, MiB_bytes from vllm.version import __version__ as VLLM_VERSION logger = init_logger(__name__) @@ -25,6 +27,128 @@ _T = TypeVar("_T") # Attempt N uses: base_timeout * (_RETRY_BACKOFF_FACTOR ** N) for the # per-attempt timeout and sleeps _RETRY_BACKOFF_FACTOR ** N seconds. _RETRY_BACKOFF_FACTOR = 4 +_RESPONSE_READ_CHUNK_SIZE = 64 * KiB_bytes + + +class HTTPResponseSizeExceededError(VLLMValidationError): + """Raised when an HTTP response exceeds a caller-supplied byte ceiling.""" + + def __init__(self, max_bytes: int, received_bytes: int) -> None: + self.max_bytes = max_bytes + self.received_bytes = received_bytes + super().__init__( + "Maximum file size exceeded: HTTP response exceeded maximum size " + f"of {max_bytes} bytes (received at least {received_bytes} bytes)", + parameter="audio_filesize_mb", + value=received_bytes / MiB_bytes, + ) + + +class MediaDownloadSizeExceededError(ValueError): + """Raised when a remote media download exceeds the configured byte limit.""" + + def __init__(self, max_bytes: int, received_bytes: int | None = None) -> None: + self.max_bytes = max_bytes + self.received_bytes = received_bytes + max_size_mb = max_bytes / MiB_bytes + message = ( + "Remote media download exceeds " + f"VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB={max_size_mb:g}" + ) + if received_bytes is not None: + message += f" (received at least {received_bytes} bytes)" + super().__init__(message) + + +def _get_media_download_limit_bytes() -> int | None: + max_size_mb = envs.VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB + if max_size_mb <= 0: + return None + return max_size_mb * MiB_bytes + + +def _get_response_size_limit(max_bytes: int | None) -> tuple[int | None, bool]: + media_max_bytes = _get_media_download_limit_bytes() + if media_max_bytes is not None and ( + max_bytes is None or media_max_bytes <= max_bytes + ): + return media_max_bytes, True + return max_bytes, False + + +def _raise_size_exceeded( + max_bytes: int, received_bytes: int, media_download_limit: bool +) -> None: + if media_download_limit: + raise MediaDownloadSizeExceededError(max_bytes, received_bytes) + raise HTTPResponseSizeExceededError(max_bytes, received_bytes) + + +def _check_content_length( + content_length: str | int | None, + max_bytes: int | None, + media_download_limit: bool, +) -> None: + if content_length is None or max_bytes is None: + return + try: + parsed_length = int(content_length) + except (TypeError, ValueError): + return + if parsed_length > max_bytes: + _raise_size_exceeded(max_bytes, parsed_length, media_download_limit) + + +def _append_chunk( + chunks: list[bytes], + chunk: bytes, + received_bytes: int, + max_bytes: int | None, + media_download_limit: bool, +) -> int: + if not chunk: + return received_bytes + received_bytes += len(chunk) + if max_bytes is not None and received_bytes > max_bytes: + _raise_size_exceeded(max_bytes, received_bytes, media_download_limit) + chunks.append(chunk) + return received_bytes + + +def _read_response_bytes(response: requests.Response, max_bytes: int | None) -> bytes: + max_bytes, media_download_limit = _get_response_size_limit(max_bytes) + if max_bytes is None: + return response.content + + _check_content_length( + response.headers.get("Content-Length"), + max_bytes, + media_download_limit, + ) + chunks: list[bytes] = [] + received_bytes = 0 + for chunk in response.iter_content(_RESPONSE_READ_CHUNK_SIZE): + received_bytes = _append_chunk( + chunks, chunk, received_bytes, max_bytes, media_download_limit + ) + return b"".join(chunks) + + +async def _async_read_response_bytes( + response: aiohttp.ClientResponse, max_bytes: int | None +) -> bytes: + max_bytes, media_download_limit = _get_response_size_limit(max_bytes) + if max_bytes is None: + return await response.read() + + _check_content_length(response.content_length, max_bytes, media_download_limit) + chunks: list[bytes] = [] + received_bytes = 0 + async for chunk in response.content.iter_chunked(_RESPONSE_READ_CHUNK_SIZE): + received_bytes = _append_chunk( + chunks, chunk, received_bytes, max_bytes, media_download_limit + ) + return b"".join(chunks) def _is_retryable(exc: Exception) -> bool: @@ -277,14 +401,22 @@ class HTTPConnection: @_sync_retry def get_bytes( - self, url: str, *, timeout: float | None = None, allow_redirects: bool = True + self, + url: str, + *, + timeout: float | None = None, + allow_redirects: bool = True, + max_bytes: int | None = None, ) -> bytes: with self.get_response( - url, timeout=timeout, allow_redirects=allow_redirects + url, + stream=True, + timeout=timeout, + allow_redirects=allow_redirects, ) as r: r.raise_for_status() - return r.content + return _read_response_bytes(r, max_bytes) @_async_retry async def async_get_bytes( @@ -293,13 +425,16 @@ class HTTPConnection: *, timeout: float | None = None, allow_redirects: bool = True, + max_bytes: int | None = None, ) -> bytes: async with await self.get_async_response( - url, timeout=timeout, allow_redirects=allow_redirects + url, + timeout=timeout, + allow_redirects=allow_redirects, ) as r: r.raise_for_status() - return await r.read() + return await _async_read_response_bytes(r, max_bytes) def get_text(self, url: str, *, timeout: float | None = None) -> str: with self.get_response(url, timeout=timeout) as r: diff --git a/vllm/entrypoints/openai/run_batch.py b/vllm/entrypoints/openai/run_batch.py index a367791764..d1e247086d 100644 --- a/vllm/entrypoints/openai/run_batch.py +++ b/vllm/entrypoints/openai/run_batch.py @@ -10,7 +10,7 @@ from argparse import Namespace from collections.abc import Awaitable, Callable from http import HTTPStatus from io import BytesIO, StringIO -from typing import Any, TypeAlias +from typing import Any, NoReturn, TypeAlias from urllib.parse import urlparse import aiohttp @@ -28,7 +28,7 @@ from urllib3.util import parse_url import vllm.envs as envs from vllm.config import config -from vllm.connections import global_http_connection +from vllm.connections import HTTPResponseSizeExceededError, global_http_connection from vllm.engine.arg_utils import AsyncEngineArgs from vllm.engine.protocol import EngineClient from vllm.entrypoints.launchers.api_server.entry import init_app_state @@ -68,11 +68,24 @@ from vllm.logger import init_logger from vllm.reasoning import ReasoningParserManager from vllm.utils import random_uuid from vllm.utils.argparse_utils import FlexibleArgumentParser +from vllm.utils.mem_constants import MiB_bytes from vllm.version import __version__ as VLLM_VERSION logger = init_logger(__name__) +def _get_audio_download_limit_bytes() -> int: + return int(envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB * MiB_bytes) + + +def _raise_audio_filesize_limit(size_bytes: int) -> NoReturn: + raise VLLMValidationError( + "Maximum file size exceeded", + parameter="audio_filesize_mb", + value=size_bytes / MiB_bytes, + ) + + class BatchTranscriptionRequest(TranscriptionRequest): """ Batch transcription request that uses file_url instead of file. @@ -474,6 +487,9 @@ async def download_bytes_from_url( if "," in url: header, data = url.split(",", 1) if "base64" in header: + decoded_size_bytes = len(data.rstrip("=")) * 3 // 4 + if decoded_size_bytes > _get_audio_download_limit_bytes(): + _raise_audio_filesize_limit(decoded_size_bytes) return base64.b64decode(data) else: raise VLLMValidationError( @@ -502,9 +518,14 @@ async def download_bytes_from_url( # between urllib3 and aiohttp (e.g. backslash-@ attacks). url = url_spec.url - return await global_http_connection.async_get_bytes( - url, allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS - ) + try: + return await global_http_connection.async_get_bytes( + url, + allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS, + max_bytes=_get_audio_download_limit_bytes(), + ) + except HTTPResponseSizeExceededError as exc: + _raise_audio_filesize_limit(exc.received_bytes) else: raise VLLMValidationError( diff --git a/vllm/entrypoints/speech_to_text/base/serving.py b/vllm/entrypoints/speech_to_text/base/serving.py index e7b864f3b7..b259ec3e5f 100644 --- a/vllm/entrypoints/speech_to_text/base/serving.py +++ b/vllm/entrypoints/speech_to_text/base/serving.py @@ -124,7 +124,6 @@ class SpeechToTextBaseServing(GenerateBaseServing): self.enable_force_include_usage = enable_force_include_usage - self.max_audio_filesize_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB self.max_audio_decode_duration_s: int = envs.VLLM_MAX_AUDIO_DECODE_DURATION_S self.max_audio_decode_bytes: int = envs.VLLM_MAX_AUDIO_DECODE_BYTES if self.model_cls.supports_segment_timestamp: @@ -274,13 +273,6 @@ class SpeechToTextBaseServing(GenerateBaseServing): else None ) - if len(audio_data) / 1024**2 > self.max_audio_filesize_mb: - raise VLLMValidationError( - "Maximum file size exceeded", - parameter="audio_filesize_mb", - value=len(audio_data) / 1024**2, - ) - # Run cpu intensive preprocess step in a separate thread pool executor. chunks, duration = await self._decode_and_chunk_speech_async(audio_data) diff --git a/vllm/envs.py b/vllm/envs.py index 7ef9c06592..fe9727aea7 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -75,6 +75,7 @@ if TYPE_CHECKING: VLLM_MEDIA_CACHE_MAX_SIZE_MB: int = 5120 VLLM_MEDIA_CACHE_TTL_HOURS: float = 24 VLLM_MEDIA_FETCH_MAX_RETRIES: int = 3 + VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB: int = 256 VLLM_MEDIA_URL_ALLOW_REDIRECTS: bool = True VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8 VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25 @@ -988,6 +989,12 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_MEDIA_FETCH_MAX_RETRIES": lambda: int( os.getenv("VLLM_MEDIA_FETCH_MAX_RETRIES", "3") ), + # Maximum size in MB for a single remote media download. The limit is + # enforced while streaming the response body so oversized or infinite + # responses cannot grow the API server heap without bound. Default is 256. + "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB": lambda: int( + os.getenv("VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", "256") + ), # Whether to allow HTTP redirects when fetching from media URLs. # Default to True "VLLM_MEDIA_URL_ALLOW_REDIRECTS": lambda: bool( diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py index 42dcff8e3e..1f5a41c9df 100644 --- a/vllm/multimodal/media/audio.py +++ b/vllm/multimodal/media/audio.py @@ -10,6 +10,7 @@ import pybase64 import torch import vllm.envs as envs +from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.multimodal.audio import resample_audio_pyav from vllm.utils.import_utils import PlaceholderModule @@ -275,7 +276,20 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]): # for flexible control. self.kwargs = kwargs + def get_max_bytes(self) -> int: + return int(envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB * MiB_bytes) + + def _validate_encoded_size(self, size: int) -> None: + max_bytes = self.get_max_bytes() + if size > max_bytes: + raise VLLMValidationError( + "Maximum file size exceeded", + parameter="audio_filesize_mb", + value=size / MiB_bytes, + ) + def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]: + self._validate_encoded_size(len(data)) return load_audio( BytesIO(data), sr=None, @@ -288,6 +302,13 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]): media_type: str, data: str, ) -> tuple[npt.NDArray, float]: + max_encoded_chars = 4 * ((self.get_max_bytes() + 2) // 3) + if len(data) > max_encoded_chars: + raise VLLMValidationError( + "Maximum file size exceeded", + parameter="audio_filesize_mb", + value=(len(data) * 3 / 4) / MiB_bytes, + ) return self.load_bytes(pybase64.b64decode(data)) def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]: diff --git a/vllm/multimodal/media/base.py b/vllm/multimodal/media/base.py index 2927914e64..5a58b9ff72 100644 --- a/vllm/multimodal/media/base.py +++ b/vllm/multimodal/media/base.py @@ -79,6 +79,10 @@ class MediaIO(ABC, Generic[_T]): merged.update(runtime_kwargs) return merged + def get_max_bytes(self) -> int | None: + """Return the maximum encoded payload size accepted by this media IO.""" + return None + @abstractmethod def load_bytes(self, data: bytes) -> _T: raise NotImplementedError diff --git a/vllm/multimodal/media/connector.py b/vllm/multimodal/media/connector.py index fed41657e9..bf3f9d85f1 100644 --- a/vllm/multimodal/media/connector.py +++ b/vllm/multimodal/media/connector.py @@ -22,8 +22,12 @@ from PIL import Image, UnidentifiedImageError from urllib3.util import Url, parse_url import vllm.envs as envs -from vllm.connections import HTTPConnection, global_http_connection -from vllm.exceptions import VLLMUnprocessableEntityError +from vllm.connections import ( + HTTPConnection, + MediaDownloadSizeExceededError, + global_http_connection, +) +from vllm.exceptions import VLLMUnprocessableEntityError, VLLMValidationError from vllm.logger import init_logger from vllm.multimodal.video import get_video_loader_backend_for_processor from vllm.utils.registry import ExtensionManager @@ -70,6 +74,9 @@ def _wrap_media_fetch_error( Original exception for transient errors (5xx, 408, 429, network blips) or other exceptions """ + if isinstance(exc, VLLMValidationError): + return exc + if isinstance(exc, aiohttp.ClientResponseError): if exc.status in (408, 429): return exc @@ -101,6 +108,13 @@ def _wrap_media_fetch_error( value=url, ) + if isinstance(exc, MediaDownloadSizeExceededError): + return VLLMUnprocessableEntityError( + f"Failed to fetch media from URL: {exc}", + parameter="image_url", + value=url, + ) + if isinstance(exc, ValueError): return VLLMUnprocessableEntityError( "Failed to fetch media from URL: Invalid URL", @@ -363,6 +377,7 @@ class MediaConnector: if url_spec.scheme and url_spec.scheme.startswith("http"): self._assert_url_in_allowed_media_domains(url_spec) + max_bytes = media_io.get_max_bytes() cached = self._get_cached_bytes(url) if cached is not None: @@ -374,6 +389,7 @@ class MediaConnector: url_spec.url, timeout=fetch_timeout, allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS, + max_bytes=max_bytes, ) except Exception as e: wrapped = _wrap_media_fetch_error(url, e) @@ -409,6 +425,7 @@ class MediaConnector: if url_spec.scheme and url_spec.scheme.startswith("http"): self._assert_url_in_allowed_media_domains(url_spec) + max_bytes = media_io.get_max_bytes() cached = await loop.run_in_executor( global_thread_pool, self._get_cached_bytes, url @@ -425,6 +442,7 @@ class MediaConnector: url_spec.url, timeout=fetch_timeout, allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS, + max_bytes=max_bytes, ) except Exception as e: wrapped = _wrap_media_fetch_error(url, e)