diff --git a/docs/usage/security.md b/docs/usage/security.md index 4631d7ce52..6e4822efc7 100644 --- a/docs/usage/security.md +++ b/docs/usage/security.md @@ -98,6 +98,7 @@ denial of service: | `VLLM_MAX_AUDIO_CLIP_FILESIZE_MB` | `25` | Maximum compressed filesize in MB for a single audio file. Enforced on all audio inputs (multimodal chat URLs, speech-to-text uploads, data: URLs, and local file paths) before decoding begins. | | `VLLM_MAX_AUDIO_DECODE_DURATION_S` | `600` | Maximum decoded audio duration in seconds. Prevents compressed audio from expanding into gigabytes of float32 PCM. | | `VLLM_MAX_AUDIO_DECODE_BYTES` | `268435456` (256 MiB) | Maximum float32 PCM bytes that audio decoding may allocate. Guards against sample-rate forgery where an inflated header sample rate bypasses the duration guard while the actual frame count causes a multi-GiB allocation. | +| `VLLM_MAX_EMBED_DECODE_BYTES` | `2147483648` (2 GiB) | Maximum bytes a client-supplied embedding payload (`prompt_embeds`, `image_embeds`, `audio_embeds`, `video_embeds`) may allocate once densified. A sparse tensor carries its own declared shape, so a payload of a few hundred bytes can expand into hundreds of GiB. Checked before `to_dense()`, so the memory is never allocated. Set to `0` to disable. | Setting any of these to `0` disables the corresponding limit. This is **not recommended** for deployments exposed to untrusted users, as it removes the diff --git a/tests/renderers/test_sparse_tensor_validation.py b/tests/renderers/test_sparse_tensor_validation.py index 642867086f..c119b090c0 100644 --- a/tests/renderers/test_sparse_tensor_validation.py +++ b/tests/renderers/test_sparse_tensor_validation.py @@ -2,7 +2,13 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ Tests verify that malicious sparse tensors are rejected before they can trigger -out-of-bounds memory writes during to_dense() operations. +out-of-bounds memory writes, or an unbounded allocation, during to_dense() +operations. + +A sparse tensor carries its own declared shape, so the two failure modes are +independent: invalid *indices* are what the invariant checks catch, while a +valid tensor with an enormous declared *shape* passes those checks and is +bounded by `VLLM_MAX_EMBED_DECODE_BYTES` instead. """ import io @@ -12,9 +18,12 @@ import pybase64 as base64 import pytest import torch +import vllm.envs as envs from vllm.exceptions import VLLMValidationError from vllm.multimodal.media import AudioEmbeddingMediaIO, ImageEmbeddingMediaIO +from vllm.multimodal.media.video import VideoEmbeddingMediaIO from vllm.renderers.embed_utils import safe_load_prompt_embeds +from vllm.utils.sparse_utils import safe_to_dense @pytest.fixture @@ -397,3 +406,136 @@ class TestSparseTensorValidationIntegration: with pytest.raises((RuntimeError, ValueError)): io_handler.load_base64("", attack_payload.decode("utf-8")) + + +# The real attack payload: 4 TiB of float32 once densified, from a single +# stored element. Only ever measured, never decoded. +BOMB_SHAPE = (2**20, 2**20) + +# What the size-limit tests below actually build. Each of them also lowers +# `VLLM_MAX_EMBED_DECODE_BYTES` to `TEST_LIMIT_BYTES`, so the guard still has +# something to reject while nothing here can allocate more than 4 MiB even if +# the guard regressed. Keeping the 4 TiB shape in these tests would make the +# outcome depend on the runner: with `vm.overcommit_memory=0` the allocator +# refuses it outright, but with `vm.overcommit_memory=1` the malloc succeeds +# and the zero-fill is what fails, which would OOM the test process. +OVERSIZED_SHAPE = (1024, 1024) # 4 MiB dense +TEST_LIMIT_BYTES = 4096 + +EMBEDDING_MEDIA_IO = [ + (ImageEmbeddingMediaIO, "image_embeds"), + (AudioEmbeddingMediaIO, "audio_embeds"), + (VideoEmbeddingMediaIO, "video_embeds"), +] + + +def _create_oversized_sparse_tensor(shape: tuple[int, int]) -> torch.Tensor: + """A *valid* sparse tensor with one stored element and a declared `shape`. + + Indices are in bounds, so `check_sparse_tensor_invariants()` accepts it. + That is the point: the invariant checks are not what stops this one. + """ + indices = torch.tensor([[0], [0]]) + values = torch.tensor([1.0]) + return torch.sparse_coo_tensor(indices, values, shape, dtype=torch.float32) + + +@pytest.fixture +def small_decode_limit(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("VLLM_MAX_EMBED_DECODE_BYTES", str(TEST_LIMIT_BYTES)) + + +class TestEmbeddingDecodeSizeLimit: + """Bound the dense allocation a payload can declare. + + `weights_only=True` stops code execution and the invariant checks stop + out-of-bounds indices, but neither bounds the declared shape. + """ + + def test_oversized_sparse_tensor_rejected(self, small_decode_limit): + with pytest.raises(VLLMValidationError) as exc_info: + safe_to_dense( + _create_oversized_sparse_tensor(OVERSIZED_SHAPE), + parameter="prompt_embeds", + ) + + assert "VLLM_MAX_EMBED_DECODE_BYTES" in str(exc_info.value) + assert "prompt_embeds" in str(exc_info.value) + + def test_default_limit_would_reject_the_real_payload(self): + """The shipped default has to bound the actual attack, not just the + shrunken shape used above. Checked arithmetically so no test ever asks + the allocator for 4 TiB.""" + dense_bytes = BOMB_SHAPE[0] * BOMB_SHAPE[1] * 4 + assert dense_bytes > envs.VLLM_MAX_EMBED_DECODE_BYTES > 0 + + def test_attack_payload_is_tiny(self): + """The payload really is small - this is amplification, not an upload.""" + assert len(_encode_tensor(_create_oversized_sparse_tensor(BOMB_SHAPE))) < 4096 + + def test_valid_tensors_still_densify(self): + assert safe_to_dense( + _create_valid_dense_tensor(), parameter="prompt_embeds" + ).shape == (10, 768) + + dense = safe_to_dense(_create_valid_sparse_tensor(), parameter="image_embeds") + assert dense.shape == (3, 3) + assert not dense.is_sparse + + def test_limit_is_configurable(self, monkeypatch: pytest.MonkeyPatch): + # 32 x 32 float32 == 4096 bytes, exactly at the limit. + monkeypatch.setenv("VLLM_MAX_EMBED_DECODE_BYTES", str(TEST_LIMIT_BYTES)) + + at_limit = _create_oversized_sparse_tensor((32, 32)) + assert safe_to_dense(at_limit, parameter="image_embeds").shape == (32, 32) + + over_limit = _create_oversized_sparse_tensor((33, 32)) + with pytest.raises(VLLMValidationError): + safe_to_dense(over_limit, parameter="image_embeds") + + def test_limit_can_be_disabled(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("VLLM_MAX_EMBED_DECODE_BYTES", "0") + tensor = _create_oversized_sparse_tensor(OVERSIZED_SHAPE) + + assert safe_to_dense(tensor, parameter="image_embeds").shape == OVERSIZED_SHAPE + + @pytest.mark.parametrize("io_cls,parameter", EMBEDDING_MEDIA_IO) + def test_media_io_rejects_oversized_base64( + self, io_cls, parameter, small_decode_limit + ): + encoded = _encode_tensor(_create_oversized_sparse_tensor(OVERSIZED_SHAPE)) + + with pytest.raises(VLLMValidationError) as exc_info: + io_cls().load_base64("", encoded.decode("utf-8")) + + assert parameter in str(exc_info.value) + + @pytest.mark.parametrize("io_cls,parameter", EMBEDDING_MEDIA_IO) + def test_media_io_rejects_oversized_file( + self, io_cls, parameter, tmp_path, small_decode_limit + ): + path = tmp_path / "embeds.pt" + torch.save(_create_oversized_sparse_tensor(OVERSIZED_SHAPE), path) + + with pytest.raises(VLLMValidationError): + io_cls().load_file(path) + + @pytest.mark.parametrize("io_cls,parameter", EMBEDDING_MEDIA_IO) + def test_media_io_rejects_non_tensor_payload(self, io_cls, parameter): + """These three had no `isinstance` check, so a non-tensor payload + surfaced as an `AttributeError` (500) rather than a 4xx.""" + buffer = io.BytesIO() + torch.save({"not": "a tensor"}, buffer) + encoded = base64.b64encode(buffer.getvalue()).decode("utf-8") + + with pytest.raises(VLLMValidationError) as exc_info: + io_cls().load_base64("", encoded) + + assert "torch.Tensor" in str(exc_info.value) + + @pytest.mark.parametrize("io_cls,parameter", EMBEDDING_MEDIA_IO) + def test_media_io_still_accepts_valid_embeddings(self, io_cls, parameter): + tensor = torch.randn(4, 16, dtype=torch.float32) + encoded = _encode_tensor(tensor) + + assert torch.equal(io_cls().load_base64("", encoded.decode("utf-8")), tensor) diff --git a/vllm/envs.py b/vllm/envs.py index f2e24c576e..e33758d808 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -82,6 +82,7 @@ if TYPE_CHECKING: VLLM_MAX_AUDIO_DECODE_DURATION_S: int = 600 VLLM_MAX_AUDIO_DECODE_BYTES: int = 268_435_456 VLLM_MAX_AUDIO_PREPROCESS_WORKERS: int = max(1, min(os.cpu_count() or 1, 2)) + VLLM_MAX_EMBED_DECODE_BYTES: int = 2_147_483_648 VLLM_MAX_IMAGE_PIXELS: int = 178_956_970 VLLM_VIDEO_LOADER_BACKEND: str = "opencv" VLLM_MEDIA_CONNECTOR: str = "http" @@ -1011,6 +1012,16 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_MAX_AUDIO_DECODE_BYTES": lambda: int( os.getenv("VLLM_MAX_AUDIO_DECODE_BYTES", "268435456") ), + # Maximum bytes a client-supplied embedding payload (`prompt_embeds`, + # `image_embeds`, `audio_embeds`, `video_embeds`) may allocate once it is + # densified. A sparse tensor carries its own declared shape, so a payload + # of a few hundred bytes can expand into hundreds of GiB. The limit is + # checked before `to_dense()` so the memory is never allocated. Set to 0 + # to disable. Default is 2 GiB, which covers a 128K-token float32 + # embedding at hidden_size 4096. + "VLLM_MAX_EMBED_DECODE_BYTES": lambda: int( + os.getenv("VLLM_MAX_EMBED_DECODE_BYTES", "2147483648") + ), # Maximum number of worker threads used for STT preprocessing. The default # intentionally caps at 2 because that performed best in profiling. # https://github.com/vllm-project/vllm/pull/44612#issuecomment-4662757781 @@ -2325,6 +2336,7 @@ def compile_factors() -> dict[str, object]: "VLLM_MAX_AUDIO_DECODE_DURATION_S", "VLLM_MAX_AUDIO_DECODE_BYTES", "VLLM_MAX_AUDIO_PREPROCESS_WORKERS", + "VLLM_MAX_EMBED_DECODE_BYTES", "VLLM_MAX_IMAGE_PIXELS", "VLLM_VIDEO_LOADER_BACKEND", "VLLM_MEDIA_CONNECTOR", diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py index 3408ab635d..e1e27d469e 100644 --- a/vllm/multimodal/media/audio.py +++ b/vllm/multimodal/media/audio.py @@ -16,7 +16,10 @@ from vllm.multimodal.audio import resample_audio_pyav from vllm.utils.import_utils import PlaceholderModule from vllm.utils.mem_constants import MiB_bytes from vllm.utils.serial_utils import tensor2base64 -from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe +from vllm.utils.sparse_utils import ( + check_sparse_tensor_invariants_threadsafe, + safe_to_dense, +) from .base import MediaIO @@ -348,7 +351,7 @@ class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]): buffer = BytesIO(data) with check_sparse_tensor_invariants_threadsafe(): tensor = torch.load(buffer, weights_only=True) - return tensor.to_dense() + return safe_to_dense(tensor, parameter="audio_embeds") def load_base64(self, media_type: str, data: str) -> torch.Tensor: return self.load_bytes(pybase64.b64decode(data, validate=True)) @@ -356,7 +359,7 @@ class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]): def load_file(self, filepath: Path) -> torch.Tensor: with check_sparse_tensor_invariants_threadsafe(): tensor = torch.load(filepath, weights_only=True) - return tensor.to_dense() + return safe_to_dense(tensor, parameter="audio_embeds") def encode_base64(self, media: torch.Tensor) -> str: return tensor2base64(media) diff --git a/vllm/multimodal/media/image.py b/vllm/multimodal/media/image.py index d9fc2132ed..b63a12f824 100644 --- a/vllm/multimodal/media/image.py +++ b/vllm/multimodal/media/image.py @@ -11,7 +11,10 @@ from PIL import Image import vllm.envs as envs from vllm.utils.serial_utils import tensor2base64 -from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe +from vllm.utils.sparse_utils import ( + check_sparse_tensor_invariants_threadsafe, + safe_to_dense, +) from ..image import convert_image_mode, normalize_image, rgba_to_rgb from .base import MediaIO, MediaWithBytes @@ -135,7 +138,7 @@ class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]): buffer = BytesIO(data) with check_sparse_tensor_invariants_threadsafe(): tensor = torch.load(buffer, weights_only=True) - return tensor.to_dense() + return safe_to_dense(tensor, parameter="image_embeds") def _load_numpy(self, data: bytes) -> torch.Tensor: with BytesIO(data) as buffer: @@ -156,7 +159,7 @@ class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]): with check_sparse_tensor_invariants_threadsafe(): tensor = torch.load(filepath, weights_only=True) - return tensor.to_dense() + return safe_to_dense(tensor, parameter="image_embeds") def encode_base64(self, media: torch.Tensor) -> str: return tensor2base64(media) diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py index b53f93785a..e5c073f5f3 100644 --- a/vllm/multimodal/media/video.py +++ b/vllm/multimodal/media/video.py @@ -14,7 +14,10 @@ from PIL import Image from vllm import envs from vllm.logger import init_logger from vllm.utils.serial_utils import tensor2base64 -from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe +from vllm.utils.sparse_utils import ( + check_sparse_tensor_invariants_threadsafe, + safe_to_dense, +) from ..video import VIDEO_LOADER_REGISTRY from .base import MediaIO, MediaWithBytes @@ -215,7 +218,7 @@ class VideoEmbeddingMediaIO(MediaIO[torch.Tensor]): buffer = BytesIO(data) with check_sparse_tensor_invariants_threadsafe(): tensor = torch.load(buffer, weights_only=True) - return tensor.to_dense() + return safe_to_dense(tensor, parameter="video_embeds") def _load_numpy(self, data: bytes) -> torch.Tensor: with BytesIO(data) as buffer: @@ -236,7 +239,7 @@ class VideoEmbeddingMediaIO(MediaIO[torch.Tensor]): with check_sparse_tensor_invariants_threadsafe(): tensor = torch.load(filepath, weights_only=True) - return tensor.to_dense() + return safe_to_dense(tensor, parameter="video_embeds") def encode_base64(self, media: torch.Tensor) -> str: return tensor2base64(media) diff --git a/vllm/renderers/embed_utils.py b/vllm/renderers/embed_utils.py index 90c6cc1b27..c4079005a5 100644 --- a/vllm/renderers/embed_utils.py +++ b/vllm/renderers/embed_utils.py @@ -8,7 +8,10 @@ import torch from vllm.exceptions import VLLMValidationError from vllm.utils.async_utils import make_async -from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe +from vllm.utils.sparse_utils import ( + check_sparse_tensor_invariants_threadsafe, + safe_to_dense, +) if TYPE_CHECKING: from vllm.config import ModelConfig @@ -30,12 +33,7 @@ def safe_load_prompt_embeds( weights_only=True, map_location=torch.device("cpu"), ) - if not isinstance(tensor, torch.Tensor): - raise VLLMValidationError( - "`prompt_embeds` payload did not deserialize to a torch.Tensor.", - parameter="prompt_embeds", - ) - tensor = tensor.to_dense() + tensor = safe_to_dense(tensor, parameter="prompt_embeds") if tensor.dim() > 2: tensor = tensor.squeeze(0) diff --git a/vllm/utils/sparse_utils.py b/vllm/utils/sparse_utils.py index b80e002130..79330271dd 100644 --- a/vllm/utils/sparse_utils.py +++ b/vllm/utils/sparse_utils.py @@ -1,7 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -Thread-safe wrapper for sparse tensor invariant validation. +Safe deserialization helpers for client-supplied embedding tensors. + +`check_sparse_tensor_invariants_threadsafe()` is a thread-safe wrapper for +sparse tensor invariant validation. `safe_to_dense()` bounds the memory a +deserialized tensor may allocate when densified. PyTorch's `torch.sparse.check_sparse_tensor_invariants()` context manager manipulates a **process-global** flag (save/enable/restore). When multiple @@ -18,6 +22,10 @@ import threading import torch +import vllm.envs as envs +from vllm.exceptions import VLLMValidationError +from vllm.utils.mem_constants import MiB_bytes + _SPARSE_LOAD_LOCK = threading.Lock() @@ -25,3 +33,41 @@ _SPARSE_LOAD_LOCK = threading.Lock() def check_sparse_tensor_invariants_threadsafe(): with _SPARSE_LOAD_LOCK, torch.sparse.check_sparse_tensor_invariants(): yield + + +def safe_to_dense(tensor: object, *, parameter: str) -> torch.Tensor: + """Densify a client-supplied embedding tensor within a memory budget. + + `torch.load(..., weights_only=True)` prevents arbitrary code execution but + not memory amplification: a sparse COO tensor carries its own declared + shape, so the dense allocation is bounded by that shape rather than by the + request. A 2.6 KB payload declaring `(30000, 30000)` materializes 3.4 GiB, + and the shape can be picked to fit whatever host is being targeted. The + invariant checks only reject *invalid* tensors (indices outside the + declared shape); a single non-zero element inside a `(2**20, 2**20)` shape + is perfectly valid and passes them. + + So reject oversized payloads based on the dense size they declare, before + `to_dense()` allocates anything. + """ + if not isinstance(tensor, torch.Tensor): + raise VLLMValidationError( + f"`{parameter}` payload did not deserialize to a torch.Tensor.", + parameter=parameter, + ) + + max_bytes = envs.VLLM_MAX_EMBED_DECODE_BYTES + if max_bytes > 0: + dense_bytes = tensor.numel() * tensor.element_size() + if dense_bytes > max_bytes: + raise VLLMValidationError( + f"`{parameter}` declares shape {tuple(tensor.shape)}, which " + f"would allocate {dense_bytes} bytes " + f"({dense_bytes / MiB_bytes:.0f} MiB) once densified, " + f"exceeding the {max_bytes} byte limit. Set " + f"VLLM_MAX_EMBED_DECODE_BYTES to increase this limit.", + parameter=parameter, + value=dense_bytes, + ) + + return tensor.to_dense()