Reject oversized media before fully downloading it (#51896)
Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Bourtoule
parent
585bb07c70
commit
2ec6f0d71e
@@ -85,13 +85,15 @@ significantly reduce the attack surface for these types of abuse.
|
|||||||
Also, consider setting `VLLM_MEDIA_URL_ALLOW_REDIRECTS=0` to prevent HTTP
|
Also, consider setting `VLLM_MEDIA_URL_ALLOW_REDIRECTS=0` to prevent HTTP
|
||||||
redirects from being followed to bypass domain restrictions.
|
redirects from being followed to bypass domain restrictions.
|
||||||
|
|
||||||
### 5. **Restrict Media Decode Sizes:**
|
### 5. **Restrict Media Download and Decode Sizes:**
|
||||||
|
|
||||||
Compressed media files can expand into gigabytes of memory during decoding. vLLM
|
Remote media responses and compressed media files can expand into gigabytes of
|
||||||
enforces decode-size limits to prevent out-of-memory denial of service:
|
memory. vLLM enforces download and decode-size limits to prevent out-of-memory
|
||||||
|
denial of service:
|
||||||
|
|
||||||
| Environment Variable | Default | Description |
|
| Environment Variable | Default | Description |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
|
| `VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB` | `256` | Maximum size in MB for a single remote media response. Oversized responses are rejected while streaming before the full body is materialized in memory. |
|
||||||
| `VLLM_MAX_IMAGE_PIXELS` | `178956970` (~179M pixels) | Maximum decoded image size in pixels. Images exceeding this are rejected before raster memory is allocated. Default matches PIL's built-in 2x decompression-bomb threshold (~680 MB for RGB). |
|
| `VLLM_MAX_IMAGE_PIXELS` | `178956970` (~179M pixels) | Maximum decoded image size in pixels. Images exceeding this are rejected before raster memory is allocated. Default matches PIL's built-in 2x decompression-bomb threshold (~680 MB for RGB). |
|
||||||
| `VLLM_MAX_AUDIO_CLIP_FILESIZE_MB` | `25` | Maximum filesize in MB for a single audio file. |
|
| `VLLM_MAX_AUDIO_CLIP_FILESIZE_MB` | `25` | Maximum filesize in MB for a single audio file. |
|
||||||
| `VLLM_MAX_AUDIO_DECODE_DURATION_S` | `600` | Maximum decoded audio duration in seconds. Prevents compressed audio from expanding into gigabytes of float32 PCM. |
|
| `VLLM_MAX_AUDIO_DECODE_DURATION_S` | `600` | Maximum decoded audio duration in seconds. Prevents compressed audio from expanding into gigabytes of float32 PCM. |
|
||||||
|
|||||||
@@ -4,16 +4,25 @@
|
|||||||
import json
|
import json
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import threading
|
||||||
|
from collections.abc import Generator
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
from vllm.assets.audio import AudioAsset
|
from vllm.assets.audio import AudioAsset
|
||||||
|
from vllm.connections import HTTPConnection
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.openai.run_batch import (
|
from vllm.entrypoints.openai.run_batch import (
|
||||||
BatchRequestOutput,
|
BatchRequestOutput,
|
||||||
|
BatchTranscriptionRequest,
|
||||||
download_bytes_from_url,
|
download_bytes_from_url,
|
||||||
|
make_transcription_wrapper,
|
||||||
)
|
)
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
|
from vllm.utils.mem_constants import MiB_bytes
|
||||||
|
|
||||||
CHAT_MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
CHAT_MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||||
EMBEDDING_MODEL_NAME = "intfloat/multilingual-e5-small"
|
EMBEDDING_MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||||
@@ -281,6 +290,74 @@ INPUT_REASONING_BATCH = "\n".join(
|
|||||||
)
|
)
|
||||||
|
|
||||||
MINIMAL_WAV_BASE64 = "UklGRigAAABXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YQQAAAAAAP9/"
|
MINIMAL_WAV_BASE64 = "UklGRigAAABXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YQQAAAAAAP9/"
|
||||||
|
_EXACT_LIMIT_AUDIO = b"a" * MiB_bytes
|
||||||
|
_OVERSIZED_BASE64_AUDIO = "A" * (4 * ((MiB_bytes + 2) // 3))
|
||||||
|
_OVERSIZED_HTTP_AUDIO = b"b" * (MiB_bytes + 1)
|
||||||
|
|
||||||
|
|
||||||
|
class _BatchHTTPServer(ThreadingHTTPServer):
|
||||||
|
request_paths: list[str]
|
||||||
|
|
||||||
|
|
||||||
|
class _BatchHTTPHandler(BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
server = self.server
|
||||||
|
assert isinstance(server, _BatchHTTPServer)
|
||||||
|
server.request_paths.append(self.path)
|
||||||
|
|
||||||
|
if self.path == "/exact-limit":
|
||||||
|
self._send_body(_EXACT_LIMIT_AUDIO)
|
||||||
|
return
|
||||||
|
|
||||||
|
if self.path == "/oversized-chunked":
|
||||||
|
self.send_response(200)
|
||||||
|
self.end_headers()
|
||||||
|
self._write_body(_OVERSIZED_HTTP_AUDIO)
|
||||||
|
return
|
||||||
|
|
||||||
|
self.send_error(404, "Unknown batch HTTP test path")
|
||||||
|
|
||||||
|
def _send_body(self, body: bytes) -> None:
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "application/octet-stream")
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
self._write_body(body)
|
||||||
|
|
||||||
|
def _write_body(self, body: bytes) -> None:
|
||||||
|
try:
|
||||||
|
for offset in range(0, len(body), 64 * 1024):
|
||||||
|
self.wfile.write(body[offset : offset + 64 * 1024])
|
||||||
|
except (BrokenPipeError, ConnectionResetError):
|
||||||
|
self.close_connection = True
|
||||||
|
|
||||||
|
def log_message(self, fmt: str, *args: object) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def batch_http_server() -> Generator[_BatchHTTPServer, None, None]:
|
||||||
|
server = _BatchHTTPServer(("127.0.0.1", 0), _BatchHTTPHandler)
|
||||||
|
server.request_paths = []
|
||||||
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||||
|
thread.start()
|
||||||
|
try:
|
||||||
|
yield server
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
|
thread.join()
|
||||||
|
server.server_close()
|
||||||
|
|
||||||
|
|
||||||
|
def _batch_http_url(server: _BatchHTTPServer, path: str) -> str:
|
||||||
|
return f"http://localhost:{server.server_port}{path}"
|
||||||
|
|
||||||
|
|
||||||
|
async def _close_async_connection(connection: HTTPConnection) -> None:
|
||||||
|
if connection._async_client is not None:
|
||||||
|
await connection._async_client.close()
|
||||||
|
|
||||||
|
|
||||||
INPUT_TRANSCRIPTION_BATCH = (
|
INPUT_TRANSCRIPTION_BATCH = (
|
||||||
json.dumps(
|
json.dumps(
|
||||||
{
|
{
|
||||||
@@ -761,8 +838,15 @@ def test_tool_calling():
|
|||||||
|
|
||||||
def _make_aiohttp_mocks(response_data: bytes = b"fake-data", status: int = 200):
|
def _make_aiohttp_mocks(response_data: bytes = b"fake-data", status: int = 200):
|
||||||
"""Create mock objects that simulate aiohttp.ClientSession context managers."""
|
"""Create mock objects that simulate aiohttp.ClientSession context managers."""
|
||||||
|
|
||||||
|
async def iter_chunked(chunk_size: int):
|
||||||
|
del chunk_size
|
||||||
|
yield response_data
|
||||||
|
|
||||||
mock_resp = MagicMock()
|
mock_resp = MagicMock()
|
||||||
mock_resp.status = status
|
mock_resp.status = status
|
||||||
|
mock_resp.content_length = len(response_data)
|
||||||
|
mock_resp.content.iter_chunked = iter_chunked
|
||||||
mock_resp.read = AsyncMock(return_value=response_data)
|
mock_resp.read = AsyncMock(return_value=response_data)
|
||||||
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
||||||
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
||||||
@@ -892,3 +976,79 @@ async def test_download_bytes_backslash_bypass():
|
|||||||
await download_bytes_from_url(
|
await download_bytes_from_url(
|
||||||
bypass_url, allowed_media_domains=["evil.internal"]
|
bypass_url, allowed_media_domains=["evil.internal"]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_transcription_wrapper_rejects_oversized_data_url_before_decode(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
):
|
||||||
|
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
|
||||||
|
handler = AsyncMock()
|
||||||
|
wrapped_handler = make_transcription_wrapper(is_translation=False)(handler)
|
||||||
|
request = BatchTranscriptionRequest.model_validate(
|
||||||
|
{
|
||||||
|
"model": SPEECH_LARGE_MODEL_NAME,
|
||||||
|
"file_url": f"data:audio/wav;base64,{_OVERSIZED_BASE64_AUDIO}",
|
||||||
|
"response_format": "json",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("vllm.entrypoints.openai.run_batch.base64.b64decode") as decode:
|
||||||
|
response = await wrapped_handler(request)
|
||||||
|
|
||||||
|
assert isinstance(response, ErrorResponse)
|
||||||
|
assert "Maximum file size exceeded" in response.error.message
|
||||||
|
decode.assert_not_called()
|
||||||
|
handler.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_download_bytes_allows_http_body_at_audio_limit(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
batch_http_server: _BatchHTTPServer,
|
||||||
|
):
|
||||||
|
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
|
||||||
|
connection = HTTPConnection()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm.entrypoints.openai.run_batch.global_http_connection",
|
||||||
|
connection,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = await download_bytes_from_url(
|
||||||
|
_batch_http_url(batch_http_server, "/exact-limit")
|
||||||
|
)
|
||||||
|
assert result == _EXACT_LIMIT_AUDIO
|
||||||
|
finally:
|
||||||
|
await _close_async_connection(connection)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_transcription_wrapper_rejects_oversized_http_before_handler(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
batch_http_server: _BatchHTTPServer,
|
||||||
|
):
|
||||||
|
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
|
||||||
|
connection = HTTPConnection()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm.entrypoints.openai.run_batch.global_http_connection",
|
||||||
|
connection,
|
||||||
|
)
|
||||||
|
handler = AsyncMock()
|
||||||
|
wrapped_handler = make_transcription_wrapper(is_translation=False)(handler)
|
||||||
|
request = BatchTranscriptionRequest.model_validate(
|
||||||
|
{
|
||||||
|
"model": SPEECH_LARGE_MODEL_NAME,
|
||||||
|
"file_url": _batch_http_url(batch_http_server, "/oversized-chunked"),
|
||||||
|
"response_format": "json",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = await wrapped_handler(request)
|
||||||
|
finally:
|
||||||
|
await _close_async_connection(connection)
|
||||||
|
|
||||||
|
assert isinstance(response, ErrorResponse)
|
||||||
|
assert "Maximum file size exceeded" in response.error.message
|
||||||
|
handler.assert_not_awaited()
|
||||||
|
|||||||
@@ -41,7 +41,6 @@ async def test_chunk_offsets_are_cumulative_not_nominal():
|
|||||||
overlap_chunk_second=1,
|
overlap_chunk_second=1,
|
||||||
min_energy_split_window_size=1600,
|
min_energy_split_window_size=1600,
|
||||||
)
|
)
|
||||||
serving.max_audio_filesize_mb = 100.0
|
|
||||||
serving.model_cls = MagicMock()
|
serving.model_cls = MagicMock()
|
||||||
serving.model_cls.validate_language.side_effect = lambda lang: lang
|
serving.model_cls.validate_language.side_effect = lambda lang: lang
|
||||||
serving.model_cls.supports_explicit_language_detection = False
|
serving.model_cls.supports_explicit_language_detection = False
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||||||
import aiohttp
|
import aiohttp
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from vllm.connections import MediaDownloadSizeExceededError
|
||||||
from vllm.entrypoints.serve import create_error_response
|
from vllm.entrypoints.serve import create_error_response
|
||||||
from vllm.exceptions import VLLMClientError, VLLMUnprocessableEntityError
|
from vllm.exceptions import VLLMClientError, VLLMUnprocessableEntityError
|
||||||
from vllm.multimodal.media import MediaConnector
|
from vllm.multimodal.media import MediaConnector
|
||||||
@@ -134,6 +135,19 @@ class TestMediaConnectorErrorHandling:
|
|||||||
|
|
||||||
assert isinstance(exc_info.value, aiohttp.ClientConnectionError)
|
assert isinstance(exc_info.value, aiohttp.ClientConnectionError)
|
||||||
|
|
||||||
|
def test_fetch_image_download_size_limit_error(self):
|
||||||
|
connector = MediaConnector()
|
||||||
|
|
||||||
|
with patch.object(
|
||||||
|
connector.connection, "get_bytes", new_callable=MagicMock
|
||||||
|
) as mock_get:
|
||||||
|
mock_get.side_effect = MediaDownloadSizeExceededError(256 * 1024 * 1024)
|
||||||
|
|
||||||
|
with pytest.raises(VLLMUnprocessableEntityError) as exc_info:
|
||||||
|
connector.fetch_image("https://example.com/image.jpg")
|
||||||
|
|
||||||
|
assert "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB=256" in str(exc_info.value)
|
||||||
|
|
||||||
|
|
||||||
class TestErrorResponse:
|
class TestErrorResponse:
|
||||||
"""Tests for error response creation."""
|
"""Tests for error response creation."""
|
||||||
|
|||||||
@@ -0,0 +1,129 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from collections.abc import AsyncIterator, Iterator
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
|
from vllm.connections import HTTPConnection
|
||||||
|
from vllm.exceptions import VLLMValidationError
|
||||||
|
from vllm.multimodal.media import AudioMediaIO, MediaConnector
|
||||||
|
from vllm.utils.mem_constants import KiB_bytes, MiB_bytes
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||||
|
|
||||||
|
|
||||||
|
class _SyncResponse:
|
||||||
|
def __init__(self, chunks: list[bytes], content_length: int | None = None):
|
||||||
|
self.headers = (
|
||||||
|
{} if content_length is None else {"content-length": str(content_length)}
|
||||||
|
)
|
||||||
|
self._chunks = chunks
|
||||||
|
self.iterated = 0
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_args):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def raise_for_status(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def iter_content(self, chunk_size: int) -> Iterator[bytes]:
|
||||||
|
assert chunk_size == 64 * KiB_bytes
|
||||||
|
for chunk in self._chunks:
|
||||||
|
self.iterated += 1
|
||||||
|
yield chunk
|
||||||
|
|
||||||
|
|
||||||
|
class _AsyncContent:
|
||||||
|
def __init__(self, chunks: list[bytes]):
|
||||||
|
self._chunks = chunks
|
||||||
|
self.iterated = 0
|
||||||
|
|
||||||
|
async def iter_chunked(self, chunk_size: int) -> AsyncIterator[bytes]:
|
||||||
|
assert chunk_size == 64 * KiB_bytes
|
||||||
|
for chunk in self._chunks:
|
||||||
|
self.iterated += 1
|
||||||
|
yield chunk
|
||||||
|
|
||||||
|
|
||||||
|
class _AsyncResponse:
|
||||||
|
def __init__(self, chunks: list[bytes], content_length: int | None = None):
|
||||||
|
self.content_length = content_length
|
||||||
|
self.content = _AsyncContent(chunks)
|
||||||
|
|
||||||
|
async def __aenter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *_args):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def raise_for_status(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def test_audio_base64_rejects_before_decode(monkeypatch: pytest.MonkeyPatch):
|
||||||
|
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
|
||||||
|
max_encoded_chars = 4 * ((MiB_bytes + 2) // 3)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("vllm.multimodal.media.audio.pybase64.b64decode") as decode,
|
||||||
|
pytest.raises(VLLMValidationError, match="Maximum file size exceeded"),
|
||||||
|
):
|
||||||
|
AudioMediaIO().load_base64("audio/wav", "A" * (max_encoded_chars + 1))
|
||||||
|
|
||||||
|
decode.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_sync_http_reader_rejects_from_content_length_before_body_read():
|
||||||
|
response = _SyncResponse([b"A" * 10], content_length=MiB_bytes + 1)
|
||||||
|
connection = HTTPConnection()
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(connection, "get_response", return_value=response) as get_response,
|
||||||
|
pytest.raises(VLLMValidationError, match="Maximum file size exceeded"),
|
||||||
|
):
|
||||||
|
connection.get_bytes("https://example.test/audio", max_bytes=MiB_bytes)
|
||||||
|
|
||||||
|
assert get_response.call_args.kwargs["stream"] is True
|
||||||
|
assert response.iterated == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_async_http_reader_stops_after_first_over_limit_chunk():
|
||||||
|
response = _AsyncResponse([b"A" * (64 * KiB_bytes), b"B", b"C" * 10])
|
||||||
|
connection = HTTPConnection()
|
||||||
|
|
||||||
|
async def get_async_response(*_args, **_kwargs):
|
||||||
|
return response
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(connection, "get_async_response", side_effect=get_async_response),
|
||||||
|
pytest.raises(VLLMValidationError, match="Maximum file size exceeded"),
|
||||||
|
):
|
||||||
|
await connection.async_get_bytes(
|
||||||
|
"https://example.test/audio",
|
||||||
|
max_bytes=64 * KiB_bytes,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.content.iterated == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_audio_connector_threads_byte_limit_to_http_reader(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
):
|
||||||
|
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
|
||||||
|
|
||||||
|
class _Connection:
|
||||||
|
async def async_get_bytes(self, _url: str, **kwargs):
|
||||||
|
assert kwargs["max_bytes"] == MiB_bytes
|
||||||
|
raise VLLMValidationError("Maximum file size exceeded")
|
||||||
|
|
||||||
|
connector = MediaConnector(connection=_Connection()) # type: ignore[arg-type]
|
||||||
|
with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"):
|
||||||
|
await connector.fetch_audio_async("https://example.test/audio")
|
||||||
@@ -0,0 +1,192 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import threading
|
||||||
|
from collections.abc import Generator
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
|
from vllm.connections import HTTPConnection, HTTPResponseSizeExceededError
|
||||||
|
|
||||||
|
_ONE_MIB = 1024 * 1024
|
||||||
|
_SMALL_BODY = b"a" * (_ONE_MIB // 2)
|
||||||
|
_LARGE_BODY = b"b" * (2 * _ONE_MIB)
|
||||||
|
_CHUNK_SIZE = 64 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
class _MediaHandler(BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
body = _LARGE_BODY if "large" in self.path else _SMALL_BODY
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "application/octet-stream")
|
||||||
|
if self.path != "/chunked-large":
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
|
||||||
|
try:
|
||||||
|
for offset in range(0, len(body), _CHUNK_SIZE):
|
||||||
|
self.wfile.write(body[offset : offset + _CHUNK_SIZE])
|
||||||
|
except (BrokenPipeError, ConnectionResetError):
|
||||||
|
self.close_connection = True
|
||||||
|
|
||||||
|
def log_message(self, fmt: str, *args: object) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def local_media_server() -> Generator[str, None, None]:
|
||||||
|
server = ThreadingHTTPServer(("127.0.0.1", 0), _MediaHandler)
|
||||||
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||||
|
thread.start()
|
||||||
|
try:
|
||||||
|
yield f"http://127.0.0.1:{server.server_port}"
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
|
thread.join()
|
||||||
|
server.server_close()
|
||||||
|
|
||||||
|
|
||||||
|
class _SyncResponse:
|
||||||
|
def __init__(self, chunks: list[bytes]) -> None:
|
||||||
|
self.headers: dict[str, str] = {}
|
||||||
|
self._chunks = chunks
|
||||||
|
self.content_accessed = False
|
||||||
|
self.iterated = 0
|
||||||
|
|
||||||
|
@property
|
||||||
|
def content(self) -> bytes:
|
||||||
|
self.content_accessed = True
|
||||||
|
raise AssertionError("bounded reads must not access response.content")
|
||||||
|
|
||||||
|
def iter_content(self, chunk_size: int):
|
||||||
|
del chunk_size
|
||||||
|
for chunk in self._chunks:
|
||||||
|
self.iterated += 1
|
||||||
|
yield chunk
|
||||||
|
|
||||||
|
def raise_for_status(self) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, exc_type, exc_value, traceback):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
class _AsyncContent:
|
||||||
|
def __init__(self, chunks: list[bytes]) -> None:
|
||||||
|
self._chunks = chunks
|
||||||
|
self.iterated = 0
|
||||||
|
|
||||||
|
async def iter_chunked(self, chunk_size: int):
|
||||||
|
del chunk_size
|
||||||
|
for chunk in self._chunks:
|
||||||
|
self.iterated += 1
|
||||||
|
yield chunk
|
||||||
|
|
||||||
|
|
||||||
|
class _AsyncResponse:
|
||||||
|
def __init__(self, chunks: list[bytes]) -> None:
|
||||||
|
self.content = _AsyncContent(chunks)
|
||||||
|
self.content_length = None
|
||||||
|
self.read = AsyncMock(
|
||||||
|
side_effect=AssertionError("bounded reads must not call response.read()")
|
||||||
|
)
|
||||||
|
|
||||||
|
def raise_for_status(self) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
async def __aenter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, exc_type, exc_value, traceback):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_bytes_with_limit_streams_instead_of_materializing() -> None:
|
||||||
|
response = _SyncResponse([b"abcd", b"efgh", b"ijkl"])
|
||||||
|
connection = HTTPConnection()
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(
|
||||||
|
connection,
|
||||||
|
"get_response",
|
||||||
|
return_value=response,
|
||||||
|
) as get_response,
|
||||||
|
pytest.raises(HTTPResponseSizeExceededError, match="maximum size"),
|
||||||
|
):
|
||||||
|
connection.get_bytes("http://example.com/media", max_bytes=4)
|
||||||
|
|
||||||
|
get_response.assert_called_once_with(
|
||||||
|
"http://example.com/media",
|
||||||
|
stream=True,
|
||||||
|
timeout=None,
|
||||||
|
allow_redirects=True,
|
||||||
|
)
|
||||||
|
assert response.content_accessed is False
|
||||||
|
assert response.iterated == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_async_get_bytes_with_limit_streams_instead_of_materializing() -> None:
|
||||||
|
response = _AsyncResponse([b"abcd", b"efgh", b"ijkl"])
|
||||||
|
connection = HTTPConnection()
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(
|
||||||
|
connection,
|
||||||
|
"get_async_response",
|
||||||
|
new=AsyncMock(return_value=response),
|
||||||
|
),
|
||||||
|
pytest.raises(HTTPResponseSizeExceededError, match="maximum size"),
|
||||||
|
):
|
||||||
|
await connection.async_get_bytes("http://example.com/media", max_bytes=4)
|
||||||
|
|
||||||
|
response.read.assert_not_awaited()
|
||||||
|
assert response.content.iterated == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_bytes_rejects_content_length_over_limit(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, local_media_server: str
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", 1, raising=False)
|
||||||
|
connection = HTTPConnection(reuse_client=False)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB"):
|
||||||
|
connection.get_bytes(f"{local_media_server}/large")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_async_get_bytes_rejects_stream_over_limit_without_content_length(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, local_media_server: str
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", 1, raising=False)
|
||||||
|
connection = HTTPConnection(reuse_client=False)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB"):
|
||||||
|
await connection.async_get_bytes(f"{local_media_server}/chunked-large")
|
||||||
|
finally:
|
||||||
|
if connection._async_client is not None:
|
||||||
|
await connection._async_client.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_get_bytes_allows_body_within_limit(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, local_media_server: str
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", 1, raising=False)
|
||||||
|
connection = HTTPConnection(reuse_client=False)
|
||||||
|
|
||||||
|
assert connection.get_bytes(f"{local_media_server}/small") == _SMALL_BODY
|
||||||
|
|
||||||
|
try:
|
||||||
|
actual = await connection.async_get_bytes(f"{local_media_server}/small")
|
||||||
|
assert actual == _SMALL_BODY
|
||||||
|
finally:
|
||||||
|
if connection._async_client is not None:
|
||||||
|
await connection._async_client.close()
|
||||||
+140
-5
@@ -13,7 +13,9 @@ import requests
|
|||||||
from urllib3.util import parse_url
|
from urllib3.util import parse_url
|
||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
from vllm.utils.mem_constants import KiB_bytes, MiB_bytes
|
||||||
from vllm.version import __version__ as VLLM_VERSION
|
from vllm.version import __version__ as VLLM_VERSION
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
@@ -25,6 +27,128 @@ _T = TypeVar("_T")
|
|||||||
# Attempt N uses: base_timeout * (_RETRY_BACKOFF_FACTOR ** N) for the
|
# Attempt N uses: base_timeout * (_RETRY_BACKOFF_FACTOR ** N) for the
|
||||||
# per-attempt timeout and sleeps _RETRY_BACKOFF_FACTOR ** N seconds.
|
# per-attempt timeout and sleeps _RETRY_BACKOFF_FACTOR ** N seconds.
|
||||||
_RETRY_BACKOFF_FACTOR = 4
|
_RETRY_BACKOFF_FACTOR = 4
|
||||||
|
_RESPONSE_READ_CHUNK_SIZE = 64 * KiB_bytes
|
||||||
|
|
||||||
|
|
||||||
|
class HTTPResponseSizeExceededError(VLLMValidationError):
|
||||||
|
"""Raised when an HTTP response exceeds a caller-supplied byte ceiling."""
|
||||||
|
|
||||||
|
def __init__(self, max_bytes: int, received_bytes: int) -> None:
|
||||||
|
self.max_bytes = max_bytes
|
||||||
|
self.received_bytes = received_bytes
|
||||||
|
super().__init__(
|
||||||
|
"Maximum file size exceeded: HTTP response exceeded maximum size "
|
||||||
|
f"of {max_bytes} bytes (received at least {received_bytes} bytes)",
|
||||||
|
parameter="audio_filesize_mb",
|
||||||
|
value=received_bytes / MiB_bytes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class MediaDownloadSizeExceededError(ValueError):
|
||||||
|
"""Raised when a remote media download exceeds the configured byte limit."""
|
||||||
|
|
||||||
|
def __init__(self, max_bytes: int, received_bytes: int | None = None) -> None:
|
||||||
|
self.max_bytes = max_bytes
|
||||||
|
self.received_bytes = received_bytes
|
||||||
|
max_size_mb = max_bytes / MiB_bytes
|
||||||
|
message = (
|
||||||
|
"Remote media download exceeds "
|
||||||
|
f"VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB={max_size_mb:g}"
|
||||||
|
)
|
||||||
|
if received_bytes is not None:
|
||||||
|
message += f" (received at least {received_bytes} bytes)"
|
||||||
|
super().__init__(message)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_media_download_limit_bytes() -> int | None:
|
||||||
|
max_size_mb = envs.VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB
|
||||||
|
if max_size_mb <= 0:
|
||||||
|
return None
|
||||||
|
return max_size_mb * MiB_bytes
|
||||||
|
|
||||||
|
|
||||||
|
def _get_response_size_limit(max_bytes: int | None) -> tuple[int | None, bool]:
|
||||||
|
media_max_bytes = _get_media_download_limit_bytes()
|
||||||
|
if media_max_bytes is not None and (
|
||||||
|
max_bytes is None or media_max_bytes <= max_bytes
|
||||||
|
):
|
||||||
|
return media_max_bytes, True
|
||||||
|
return max_bytes, False
|
||||||
|
|
||||||
|
|
||||||
|
def _raise_size_exceeded(
|
||||||
|
max_bytes: int, received_bytes: int, media_download_limit: bool
|
||||||
|
) -> None:
|
||||||
|
if media_download_limit:
|
||||||
|
raise MediaDownloadSizeExceededError(max_bytes, received_bytes)
|
||||||
|
raise HTTPResponseSizeExceededError(max_bytes, received_bytes)
|
||||||
|
|
||||||
|
|
||||||
|
def _check_content_length(
|
||||||
|
content_length: str | int | None,
|
||||||
|
max_bytes: int | None,
|
||||||
|
media_download_limit: bool,
|
||||||
|
) -> None:
|
||||||
|
if content_length is None or max_bytes is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
parsed_length = int(content_length)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return
|
||||||
|
if parsed_length > max_bytes:
|
||||||
|
_raise_size_exceeded(max_bytes, parsed_length, media_download_limit)
|
||||||
|
|
||||||
|
|
||||||
|
def _append_chunk(
|
||||||
|
chunks: list[bytes],
|
||||||
|
chunk: bytes,
|
||||||
|
received_bytes: int,
|
||||||
|
max_bytes: int | None,
|
||||||
|
media_download_limit: bool,
|
||||||
|
) -> int:
|
||||||
|
if not chunk:
|
||||||
|
return received_bytes
|
||||||
|
received_bytes += len(chunk)
|
||||||
|
if max_bytes is not None and received_bytes > max_bytes:
|
||||||
|
_raise_size_exceeded(max_bytes, received_bytes, media_download_limit)
|
||||||
|
chunks.append(chunk)
|
||||||
|
return received_bytes
|
||||||
|
|
||||||
|
|
||||||
|
def _read_response_bytes(response: requests.Response, max_bytes: int | None) -> bytes:
|
||||||
|
max_bytes, media_download_limit = _get_response_size_limit(max_bytes)
|
||||||
|
if max_bytes is None:
|
||||||
|
return response.content
|
||||||
|
|
||||||
|
_check_content_length(
|
||||||
|
response.headers.get("Content-Length"),
|
||||||
|
max_bytes,
|
||||||
|
media_download_limit,
|
||||||
|
)
|
||||||
|
chunks: list[bytes] = []
|
||||||
|
received_bytes = 0
|
||||||
|
for chunk in response.iter_content(_RESPONSE_READ_CHUNK_SIZE):
|
||||||
|
received_bytes = _append_chunk(
|
||||||
|
chunks, chunk, received_bytes, max_bytes, media_download_limit
|
||||||
|
)
|
||||||
|
return b"".join(chunks)
|
||||||
|
|
||||||
|
|
||||||
|
async def _async_read_response_bytes(
|
||||||
|
response: aiohttp.ClientResponse, max_bytes: int | None
|
||||||
|
) -> bytes:
|
||||||
|
max_bytes, media_download_limit = _get_response_size_limit(max_bytes)
|
||||||
|
if max_bytes is None:
|
||||||
|
return await response.read()
|
||||||
|
|
||||||
|
_check_content_length(response.content_length, max_bytes, media_download_limit)
|
||||||
|
chunks: list[bytes] = []
|
||||||
|
received_bytes = 0
|
||||||
|
async for chunk in response.content.iter_chunked(_RESPONSE_READ_CHUNK_SIZE):
|
||||||
|
received_bytes = _append_chunk(
|
||||||
|
chunks, chunk, received_bytes, max_bytes, media_download_limit
|
||||||
|
)
|
||||||
|
return b"".join(chunks)
|
||||||
|
|
||||||
|
|
||||||
def _is_retryable(exc: Exception) -> bool:
|
def _is_retryable(exc: Exception) -> bool:
|
||||||
@@ -277,14 +401,22 @@ class HTTPConnection:
|
|||||||
|
|
||||||
@_sync_retry
|
@_sync_retry
|
||||||
def get_bytes(
|
def get_bytes(
|
||||||
self, url: str, *, timeout: float | None = None, allow_redirects: bool = True
|
self,
|
||||||
|
url: str,
|
||||||
|
*,
|
||||||
|
timeout: float | None = None,
|
||||||
|
allow_redirects: bool = True,
|
||||||
|
max_bytes: int | None = None,
|
||||||
) -> bytes:
|
) -> bytes:
|
||||||
with self.get_response(
|
with self.get_response(
|
||||||
url, timeout=timeout, allow_redirects=allow_redirects
|
url,
|
||||||
|
stream=True,
|
||||||
|
timeout=timeout,
|
||||||
|
allow_redirects=allow_redirects,
|
||||||
) as r:
|
) as r:
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
|
|
||||||
return r.content
|
return _read_response_bytes(r, max_bytes)
|
||||||
|
|
||||||
@_async_retry
|
@_async_retry
|
||||||
async def async_get_bytes(
|
async def async_get_bytes(
|
||||||
@@ -293,13 +425,16 @@ class HTTPConnection:
|
|||||||
*,
|
*,
|
||||||
timeout: float | None = None,
|
timeout: float | None = None,
|
||||||
allow_redirects: bool = True,
|
allow_redirects: bool = True,
|
||||||
|
max_bytes: int | None = None,
|
||||||
) -> bytes:
|
) -> bytes:
|
||||||
async with await self.get_async_response(
|
async with await self.get_async_response(
|
||||||
url, timeout=timeout, allow_redirects=allow_redirects
|
url,
|
||||||
|
timeout=timeout,
|
||||||
|
allow_redirects=allow_redirects,
|
||||||
) as r:
|
) as r:
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
|
|
||||||
return await r.read()
|
return await _async_read_response_bytes(r, max_bytes)
|
||||||
|
|
||||||
def get_text(self, url: str, *, timeout: float | None = None) -> str:
|
def get_text(self, url: str, *, timeout: float | None = None) -> str:
|
||||||
with self.get_response(url, timeout=timeout) as r:
|
with self.get_response(url, timeout=timeout) as r:
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from argparse import Namespace
|
|||||||
from collections.abc import Awaitable, Callable
|
from collections.abc import Awaitable, Callable
|
||||||
from http import HTTPStatus
|
from http import HTTPStatus
|
||||||
from io import BytesIO, StringIO
|
from io import BytesIO, StringIO
|
||||||
from typing import Any, TypeAlias
|
from typing import Any, NoReturn, TypeAlias
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
@@ -28,7 +28,7 @@ from urllib3.util import parse_url
|
|||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm.config import config
|
from vllm.config import config
|
||||||
from vllm.connections import global_http_connection
|
from vllm.connections import HTTPResponseSizeExceededError, global_http_connection
|
||||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.launchers.api_server.entry import init_app_state
|
from vllm.entrypoints.launchers.api_server.entry import init_app_state
|
||||||
@@ -68,11 +68,24 @@ from vllm.logger import init_logger
|
|||||||
from vllm.reasoning import ReasoningParserManager
|
from vllm.reasoning import ReasoningParserManager
|
||||||
from vllm.utils import random_uuid
|
from vllm.utils import random_uuid
|
||||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
from vllm.utils.mem_constants import MiB_bytes
|
||||||
from vllm.version import __version__ as VLLM_VERSION
|
from vllm.version import __version__ as VLLM_VERSION
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_audio_download_limit_bytes() -> int:
|
||||||
|
return int(envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB * MiB_bytes)
|
||||||
|
|
||||||
|
|
||||||
|
def _raise_audio_filesize_limit(size_bytes: int) -> NoReturn:
|
||||||
|
raise VLLMValidationError(
|
||||||
|
"Maximum file size exceeded",
|
||||||
|
parameter="audio_filesize_mb",
|
||||||
|
value=size_bytes / MiB_bytes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class BatchTranscriptionRequest(TranscriptionRequest):
|
class BatchTranscriptionRequest(TranscriptionRequest):
|
||||||
"""
|
"""
|
||||||
Batch transcription request that uses file_url instead of file.
|
Batch transcription request that uses file_url instead of file.
|
||||||
@@ -474,6 +487,9 @@ async def download_bytes_from_url(
|
|||||||
if "," in url:
|
if "," in url:
|
||||||
header, data = url.split(",", 1)
|
header, data = url.split(",", 1)
|
||||||
if "base64" in header:
|
if "base64" in header:
|
||||||
|
decoded_size_bytes = len(data.rstrip("=")) * 3 // 4
|
||||||
|
if decoded_size_bytes > _get_audio_download_limit_bytes():
|
||||||
|
_raise_audio_filesize_limit(decoded_size_bytes)
|
||||||
return base64.b64decode(data)
|
return base64.b64decode(data)
|
||||||
else:
|
else:
|
||||||
raise VLLMValidationError(
|
raise VLLMValidationError(
|
||||||
@@ -502,9 +518,14 @@ async def download_bytes_from_url(
|
|||||||
# between urllib3 and aiohttp (e.g. backslash-@ attacks).
|
# between urllib3 and aiohttp (e.g. backslash-@ attacks).
|
||||||
url = url_spec.url
|
url = url_spec.url
|
||||||
|
|
||||||
return await global_http_connection.async_get_bytes(
|
try:
|
||||||
url, allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS
|
return await global_http_connection.async_get_bytes(
|
||||||
)
|
url,
|
||||||
|
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
|
||||||
|
max_bytes=_get_audio_download_limit_bytes(),
|
||||||
|
)
|
||||||
|
except HTTPResponseSizeExceededError as exc:
|
||||||
|
_raise_audio_filesize_limit(exc.received_bytes)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
raise VLLMValidationError(
|
raise VLLMValidationError(
|
||||||
|
|||||||
@@ -124,7 +124,6 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
|||||||
|
|
||||||
self.enable_force_include_usage = enable_force_include_usage
|
self.enable_force_include_usage = enable_force_include_usage
|
||||||
|
|
||||||
self.max_audio_filesize_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB
|
|
||||||
self.max_audio_decode_duration_s: int = envs.VLLM_MAX_AUDIO_DECODE_DURATION_S
|
self.max_audio_decode_duration_s: int = envs.VLLM_MAX_AUDIO_DECODE_DURATION_S
|
||||||
self.max_audio_decode_bytes: int = envs.VLLM_MAX_AUDIO_DECODE_BYTES
|
self.max_audio_decode_bytes: int = envs.VLLM_MAX_AUDIO_DECODE_BYTES
|
||||||
if self.model_cls.supports_segment_timestamp:
|
if self.model_cls.supports_segment_timestamp:
|
||||||
@@ -274,13 +273,6 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
|||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
|
||||||
if len(audio_data) / 1024**2 > self.max_audio_filesize_mb:
|
|
||||||
raise VLLMValidationError(
|
|
||||||
"Maximum file size exceeded",
|
|
||||||
parameter="audio_filesize_mb",
|
|
||||||
value=len(audio_data) / 1024**2,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Run cpu intensive preprocess step in a separate thread pool executor.
|
# Run cpu intensive preprocess step in a separate thread pool executor.
|
||||||
chunks, duration = await self._decode_and_chunk_speech_async(audio_data)
|
chunks, duration = await self._decode_and_chunk_speech_async(audio_data)
|
||||||
|
|
||||||
|
|||||||
@@ -75,6 +75,7 @@ if TYPE_CHECKING:
|
|||||||
VLLM_MEDIA_CACHE_MAX_SIZE_MB: int = 5120
|
VLLM_MEDIA_CACHE_MAX_SIZE_MB: int = 5120
|
||||||
VLLM_MEDIA_CACHE_TTL_HOURS: float = 24
|
VLLM_MEDIA_CACHE_TTL_HOURS: float = 24
|
||||||
VLLM_MEDIA_FETCH_MAX_RETRIES: int = 3
|
VLLM_MEDIA_FETCH_MAX_RETRIES: int = 3
|
||||||
|
VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB: int = 256
|
||||||
VLLM_MEDIA_URL_ALLOW_REDIRECTS: bool = True
|
VLLM_MEDIA_URL_ALLOW_REDIRECTS: bool = True
|
||||||
VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8
|
VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8
|
||||||
VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25
|
VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25
|
||||||
@@ -988,6 +989,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
"VLLM_MEDIA_FETCH_MAX_RETRIES": lambda: int(
|
"VLLM_MEDIA_FETCH_MAX_RETRIES": lambda: int(
|
||||||
os.getenv("VLLM_MEDIA_FETCH_MAX_RETRIES", "3")
|
os.getenv("VLLM_MEDIA_FETCH_MAX_RETRIES", "3")
|
||||||
),
|
),
|
||||||
|
# Maximum size in MB for a single remote media download. The limit is
|
||||||
|
# enforced while streaming the response body so oversized or infinite
|
||||||
|
# responses cannot grow the API server heap without bound. Default is 256.
|
||||||
|
"VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB": lambda: int(
|
||||||
|
os.getenv("VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", "256")
|
||||||
|
),
|
||||||
# Whether to allow HTTP redirects when fetching from media URLs.
|
# Whether to allow HTTP redirects when fetching from media URLs.
|
||||||
# Default to True
|
# Default to True
|
||||||
"VLLM_MEDIA_URL_ALLOW_REDIRECTS": lambda: bool(
|
"VLLM_MEDIA_URL_ALLOW_REDIRECTS": lambda: bool(
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import pybase64
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.multimodal.audio import resample_audio_pyav
|
from vllm.multimodal.audio import resample_audio_pyav
|
||||||
from vllm.utils.import_utils import PlaceholderModule
|
from vllm.utils.import_utils import PlaceholderModule
|
||||||
@@ -275,7 +276,20 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
|||||||
# for flexible control.
|
# for flexible control.
|
||||||
self.kwargs = kwargs
|
self.kwargs = kwargs
|
||||||
|
|
||||||
|
def get_max_bytes(self) -> int:
|
||||||
|
return int(envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB * MiB_bytes)
|
||||||
|
|
||||||
|
def _validate_encoded_size(self, size: int) -> None:
|
||||||
|
max_bytes = self.get_max_bytes()
|
||||||
|
if size > max_bytes:
|
||||||
|
raise VLLMValidationError(
|
||||||
|
"Maximum file size exceeded",
|
||||||
|
parameter="audio_filesize_mb",
|
||||||
|
value=size / MiB_bytes,
|
||||||
|
)
|
||||||
|
|
||||||
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
|
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
|
||||||
|
self._validate_encoded_size(len(data))
|
||||||
return load_audio(
|
return load_audio(
|
||||||
BytesIO(data),
|
BytesIO(data),
|
||||||
sr=None,
|
sr=None,
|
||||||
@@ -288,6 +302,13 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
|
|||||||
media_type: str,
|
media_type: str,
|
||||||
data: str,
|
data: str,
|
||||||
) -> tuple[npt.NDArray, float]:
|
) -> tuple[npt.NDArray, float]:
|
||||||
|
max_encoded_chars = 4 * ((self.get_max_bytes() + 2) // 3)
|
||||||
|
if len(data) > max_encoded_chars:
|
||||||
|
raise VLLMValidationError(
|
||||||
|
"Maximum file size exceeded",
|
||||||
|
parameter="audio_filesize_mb",
|
||||||
|
value=(len(data) * 3 / 4) / MiB_bytes,
|
||||||
|
)
|
||||||
return self.load_bytes(pybase64.b64decode(data))
|
return self.load_bytes(pybase64.b64decode(data))
|
||||||
|
|
||||||
def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]:
|
def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]:
|
||||||
|
|||||||
@@ -79,6 +79,10 @@ class MediaIO(ABC, Generic[_T]):
|
|||||||
merged.update(runtime_kwargs)
|
merged.update(runtime_kwargs)
|
||||||
return merged
|
return merged
|
||||||
|
|
||||||
|
def get_max_bytes(self) -> int | None:
|
||||||
|
"""Return the maximum encoded payload size accepted by this media IO."""
|
||||||
|
return None
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def load_bytes(self, data: bytes) -> _T:
|
def load_bytes(self, data: bytes) -> _T:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|||||||
@@ -22,8 +22,12 @@ from PIL import Image, UnidentifiedImageError
|
|||||||
from urllib3.util import Url, parse_url
|
from urllib3.util import Url, parse_url
|
||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm.connections import HTTPConnection, global_http_connection
|
from vllm.connections import (
|
||||||
from vllm.exceptions import VLLMUnprocessableEntityError
|
HTTPConnection,
|
||||||
|
MediaDownloadSizeExceededError,
|
||||||
|
global_http_connection,
|
||||||
|
)
|
||||||
|
from vllm.exceptions import VLLMUnprocessableEntityError, VLLMValidationError
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.multimodal.video import get_video_loader_backend_for_processor
|
from vllm.multimodal.video import get_video_loader_backend_for_processor
|
||||||
from vllm.utils.registry import ExtensionManager
|
from vllm.utils.registry import ExtensionManager
|
||||||
@@ -70,6 +74,9 @@ def _wrap_media_fetch_error(
|
|||||||
Original exception for transient errors (5xx, 408, 429, network blips)
|
Original exception for transient errors (5xx, 408, 429, network blips)
|
||||||
or other exceptions
|
or other exceptions
|
||||||
"""
|
"""
|
||||||
|
if isinstance(exc, VLLMValidationError):
|
||||||
|
return exc
|
||||||
|
|
||||||
if isinstance(exc, aiohttp.ClientResponseError):
|
if isinstance(exc, aiohttp.ClientResponseError):
|
||||||
if exc.status in (408, 429):
|
if exc.status in (408, 429):
|
||||||
return exc
|
return exc
|
||||||
@@ -101,6 +108,13 @@ def _wrap_media_fetch_error(
|
|||||||
value=url,
|
value=url,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if isinstance(exc, MediaDownloadSizeExceededError):
|
||||||
|
return VLLMUnprocessableEntityError(
|
||||||
|
f"Failed to fetch media from URL: {exc}",
|
||||||
|
parameter="image_url",
|
||||||
|
value=url,
|
||||||
|
)
|
||||||
|
|
||||||
if isinstance(exc, ValueError):
|
if isinstance(exc, ValueError):
|
||||||
return VLLMUnprocessableEntityError(
|
return VLLMUnprocessableEntityError(
|
||||||
"Failed to fetch media from URL: Invalid URL",
|
"Failed to fetch media from URL: Invalid URL",
|
||||||
@@ -363,6 +377,7 @@ class MediaConnector:
|
|||||||
|
|
||||||
if url_spec.scheme and url_spec.scheme.startswith("http"):
|
if url_spec.scheme and url_spec.scheme.startswith("http"):
|
||||||
self._assert_url_in_allowed_media_domains(url_spec)
|
self._assert_url_in_allowed_media_domains(url_spec)
|
||||||
|
max_bytes = media_io.get_max_bytes()
|
||||||
|
|
||||||
cached = self._get_cached_bytes(url)
|
cached = self._get_cached_bytes(url)
|
||||||
if cached is not None:
|
if cached is not None:
|
||||||
@@ -374,6 +389,7 @@ class MediaConnector:
|
|||||||
url_spec.url,
|
url_spec.url,
|
||||||
timeout=fetch_timeout,
|
timeout=fetch_timeout,
|
||||||
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
|
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
|
||||||
|
max_bytes=max_bytes,
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
wrapped = _wrap_media_fetch_error(url, e)
|
wrapped = _wrap_media_fetch_error(url, e)
|
||||||
@@ -409,6 +425,7 @@ class MediaConnector:
|
|||||||
|
|
||||||
if url_spec.scheme and url_spec.scheme.startswith("http"):
|
if url_spec.scheme and url_spec.scheme.startswith("http"):
|
||||||
self._assert_url_in_allowed_media_domains(url_spec)
|
self._assert_url_in_allowed_media_domains(url_spec)
|
||||||
|
max_bytes = media_io.get_max_bytes()
|
||||||
|
|
||||||
cached = await loop.run_in_executor(
|
cached = await loop.run_in_executor(
|
||||||
global_thread_pool, self._get_cached_bytes, url
|
global_thread_pool, self._get_cached_bytes, url
|
||||||
@@ -425,6 +442,7 @@ class MediaConnector:
|
|||||||
url_spec.url,
|
url_spec.url,
|
||||||
timeout=fetch_timeout,
|
timeout=fetch_timeout,
|
||||||
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
|
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
|
||||||
|
max_bytes=max_bytes,
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
wrapped = _wrap_media_fetch_error(url, e)
|
wrapped = _wrap_media_fetch_error(url, e)
|
||||||
|
|||||||
Reference in New Issue
Block a user