[Bugfix][Security] Bound embedding densification before to_dense() (#54632)
Signed-off-by: lzhan011 <35493221+lzhan011@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -98,6 +98,7 @@ denial of service:
|
||||
| `VLLM_MAX_AUDIO_CLIP_FILESIZE_MB` | `25` | Maximum compressed filesize in MB for a single audio file. Enforced on all audio inputs (multimodal chat URLs, speech-to-text uploads, data: URLs, and local file paths) before decoding begins. |
|
||||
| `VLLM_MAX_AUDIO_DECODE_DURATION_S` | `600` | Maximum decoded audio duration in seconds. Prevents compressed audio from expanding into gigabytes of float32 PCM. |
|
||||
| `VLLM_MAX_AUDIO_DECODE_BYTES` | `268435456` (256 MiB) | Maximum float32 PCM bytes that audio decoding may allocate. Guards against sample-rate forgery where an inflated header sample rate bypasses the duration guard while the actual frame count causes a multi-GiB allocation. |
|
||||
| `VLLM_MAX_EMBED_DECODE_BYTES` | `2147483648` (2 GiB) | Maximum bytes a client-supplied embedding payload (`prompt_embeds`, `image_embeds`, `audio_embeds`, `video_embeds`) may allocate once densified. A sparse tensor carries its own declared shape, so a payload of a few hundred bytes can expand into hundreds of GiB. Checked before `to_dense()`, so the memory is never allocated. Set to `0` to disable. |
|
||||
|
||||
Setting any of these to `0` disables the corresponding limit. This is **not
|
||||
recommended** for deployments exposed to untrusted users, as it removes the
|
||||
|
||||
@@ -2,7 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests verify that malicious sparse tensors are rejected before they can trigger
|
||||
out-of-bounds memory writes during to_dense() operations.
|
||||
out-of-bounds memory writes, or an unbounded allocation, during to_dense()
|
||||
operations.
|
||||
|
||||
A sparse tensor carries its own declared shape, so the two failure modes are
|
||||
independent: invalid *indices* are what the invariant checks catch, while a
|
||||
valid tensor with an enormous declared *shape* passes those checks and is
|
||||
bounded by `VLLM_MAX_EMBED_DECODE_BYTES` instead.
|
||||
"""
|
||||
|
||||
import io
|
||||
@@ -12,9 +18,12 @@ import pybase64 as base64
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.multimodal.media import AudioEmbeddingMediaIO, ImageEmbeddingMediaIO
|
||||
from vllm.multimodal.media.video import VideoEmbeddingMediaIO
|
||||
from vllm.renderers.embed_utils import safe_load_prompt_embeds
|
||||
from vllm.utils.sparse_utils import safe_to_dense
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -397,3 +406,136 @@ class TestSparseTensorValidationIntegration:
|
||||
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
io_handler.load_base64("", attack_payload.decode("utf-8"))
|
||||
|
||||
|
||||
# The real attack payload: 4 TiB of float32 once densified, from a single
|
||||
# stored element. Only ever measured, never decoded.
|
||||
BOMB_SHAPE = (2**20, 2**20)
|
||||
|
||||
# What the size-limit tests below actually build. Each of them also lowers
|
||||
# `VLLM_MAX_EMBED_DECODE_BYTES` to `TEST_LIMIT_BYTES`, so the guard still has
|
||||
# something to reject while nothing here can allocate more than 4 MiB even if
|
||||
# the guard regressed. Keeping the 4 TiB shape in these tests would make the
|
||||
# outcome depend on the runner: with `vm.overcommit_memory=0` the allocator
|
||||
# refuses it outright, but with `vm.overcommit_memory=1` the malloc succeeds
|
||||
# and the zero-fill is what fails, which would OOM the test process.
|
||||
OVERSIZED_SHAPE = (1024, 1024) # 4 MiB dense
|
||||
TEST_LIMIT_BYTES = 4096
|
||||
|
||||
EMBEDDING_MEDIA_IO = [
|
||||
(ImageEmbeddingMediaIO, "image_embeds"),
|
||||
(AudioEmbeddingMediaIO, "audio_embeds"),
|
||||
(VideoEmbeddingMediaIO, "video_embeds"),
|
||||
]
|
||||
|
||||
|
||||
def _create_oversized_sparse_tensor(shape: tuple[int, int]) -> torch.Tensor:
|
||||
"""A *valid* sparse tensor with one stored element and a declared `shape`.
|
||||
|
||||
Indices are in bounds, so `check_sparse_tensor_invariants()` accepts it.
|
||||
That is the point: the invariant checks are not what stops this one.
|
||||
"""
|
||||
indices = torch.tensor([[0], [0]])
|
||||
values = torch.tensor([1.0])
|
||||
return torch.sparse_coo_tensor(indices, values, shape, dtype=torch.float32)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def small_decode_limit(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("VLLM_MAX_EMBED_DECODE_BYTES", str(TEST_LIMIT_BYTES))
|
||||
|
||||
|
||||
class TestEmbeddingDecodeSizeLimit:
|
||||
"""Bound the dense allocation a payload can declare.
|
||||
|
||||
`weights_only=True` stops code execution and the invariant checks stop
|
||||
out-of-bounds indices, but neither bounds the declared shape.
|
||||
"""
|
||||
|
||||
def test_oversized_sparse_tensor_rejected(self, small_decode_limit):
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
safe_to_dense(
|
||||
_create_oversized_sparse_tensor(OVERSIZED_SHAPE),
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
|
||||
assert "VLLM_MAX_EMBED_DECODE_BYTES" in str(exc_info.value)
|
||||
assert "prompt_embeds" in str(exc_info.value)
|
||||
|
||||
def test_default_limit_would_reject_the_real_payload(self):
|
||||
"""The shipped default has to bound the actual attack, not just the
|
||||
shrunken shape used above. Checked arithmetically so no test ever asks
|
||||
the allocator for 4 TiB."""
|
||||
dense_bytes = BOMB_SHAPE[0] * BOMB_SHAPE[1] * 4
|
||||
assert dense_bytes > envs.VLLM_MAX_EMBED_DECODE_BYTES > 0
|
||||
|
||||
def test_attack_payload_is_tiny(self):
|
||||
"""The payload really is small - this is amplification, not an upload."""
|
||||
assert len(_encode_tensor(_create_oversized_sparse_tensor(BOMB_SHAPE))) < 4096
|
||||
|
||||
def test_valid_tensors_still_densify(self):
|
||||
assert safe_to_dense(
|
||||
_create_valid_dense_tensor(), parameter="prompt_embeds"
|
||||
).shape == (10, 768)
|
||||
|
||||
dense = safe_to_dense(_create_valid_sparse_tensor(), parameter="image_embeds")
|
||||
assert dense.shape == (3, 3)
|
||||
assert not dense.is_sparse
|
||||
|
||||
def test_limit_is_configurable(self, monkeypatch: pytest.MonkeyPatch):
|
||||
# 32 x 32 float32 == 4096 bytes, exactly at the limit.
|
||||
monkeypatch.setenv("VLLM_MAX_EMBED_DECODE_BYTES", str(TEST_LIMIT_BYTES))
|
||||
|
||||
at_limit = _create_oversized_sparse_tensor((32, 32))
|
||||
assert safe_to_dense(at_limit, parameter="image_embeds").shape == (32, 32)
|
||||
|
||||
over_limit = _create_oversized_sparse_tensor((33, 32))
|
||||
with pytest.raises(VLLMValidationError):
|
||||
safe_to_dense(over_limit, parameter="image_embeds")
|
||||
|
||||
def test_limit_can_be_disabled(self, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("VLLM_MAX_EMBED_DECODE_BYTES", "0")
|
||||
tensor = _create_oversized_sparse_tensor(OVERSIZED_SHAPE)
|
||||
|
||||
assert safe_to_dense(tensor, parameter="image_embeds").shape == OVERSIZED_SHAPE
|
||||
|
||||
@pytest.mark.parametrize("io_cls,parameter", EMBEDDING_MEDIA_IO)
|
||||
def test_media_io_rejects_oversized_base64(
|
||||
self, io_cls, parameter, small_decode_limit
|
||||
):
|
||||
encoded = _encode_tensor(_create_oversized_sparse_tensor(OVERSIZED_SHAPE))
|
||||
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
io_cls().load_base64("", encoded.decode("utf-8"))
|
||||
|
||||
assert parameter in str(exc_info.value)
|
||||
|
||||
@pytest.mark.parametrize("io_cls,parameter", EMBEDDING_MEDIA_IO)
|
||||
def test_media_io_rejects_oversized_file(
|
||||
self, io_cls, parameter, tmp_path, small_decode_limit
|
||||
):
|
||||
path = tmp_path / "embeds.pt"
|
||||
torch.save(_create_oversized_sparse_tensor(OVERSIZED_SHAPE), path)
|
||||
|
||||
with pytest.raises(VLLMValidationError):
|
||||
io_cls().load_file(path)
|
||||
|
||||
@pytest.mark.parametrize("io_cls,parameter", EMBEDDING_MEDIA_IO)
|
||||
def test_media_io_rejects_non_tensor_payload(self, io_cls, parameter):
|
||||
"""These three had no `isinstance` check, so a non-tensor payload
|
||||
surfaced as an `AttributeError` (500) rather than a 4xx."""
|
||||
buffer = io.BytesIO()
|
||||
torch.save({"not": "a tensor"}, buffer)
|
||||
encoded = base64.b64encode(buffer.getvalue()).decode("utf-8")
|
||||
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
io_cls().load_base64("", encoded)
|
||||
|
||||
assert "torch.Tensor" in str(exc_info.value)
|
||||
|
||||
@pytest.mark.parametrize("io_cls,parameter", EMBEDDING_MEDIA_IO)
|
||||
def test_media_io_still_accepts_valid_embeddings(self, io_cls, parameter):
|
||||
tensor = torch.randn(4, 16, dtype=torch.float32)
|
||||
encoded = _encode_tensor(tensor)
|
||||
|
||||
assert torch.equal(io_cls().load_base64("", encoded.decode("utf-8")), tensor)
|
||||
|
||||
@@ -82,6 +82,7 @@ if TYPE_CHECKING:
|
||||
VLLM_MAX_AUDIO_DECODE_DURATION_S: int = 600
|
||||
VLLM_MAX_AUDIO_DECODE_BYTES: int = 268_435_456
|
||||
VLLM_MAX_AUDIO_PREPROCESS_WORKERS: int = max(1, min(os.cpu_count() or 1, 2))
|
||||
VLLM_MAX_EMBED_DECODE_BYTES: int = 2_147_483_648
|
||||
VLLM_MAX_IMAGE_PIXELS: int = 178_956_970
|
||||
VLLM_VIDEO_LOADER_BACKEND: str = "opencv"
|
||||
VLLM_MEDIA_CONNECTOR: str = "http"
|
||||
@@ -1011,6 +1012,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_MAX_AUDIO_DECODE_BYTES": lambda: int(
|
||||
os.getenv("VLLM_MAX_AUDIO_DECODE_BYTES", "268435456")
|
||||
),
|
||||
# Maximum bytes a client-supplied embedding payload (`prompt_embeds`,
|
||||
# `image_embeds`, `audio_embeds`, `video_embeds`) may allocate once it is
|
||||
# densified. A sparse tensor carries its own declared shape, so a payload
|
||||
# of a few hundred bytes can expand into hundreds of GiB. The limit is
|
||||
# checked before `to_dense()` so the memory is never allocated. Set to 0
|
||||
# to disable. Default is 2 GiB, which covers a 128K-token float32
|
||||
# embedding at hidden_size 4096.
|
||||
"VLLM_MAX_EMBED_DECODE_BYTES": lambda: int(
|
||||
os.getenv("VLLM_MAX_EMBED_DECODE_BYTES", "2147483648")
|
||||
),
|
||||
# Maximum number of worker threads used for STT preprocessing. The default
|
||||
# intentionally caps at 2 because that performed best in profiling.
|
||||
# https://github.com/vllm-project/vllm/pull/44612#issuecomment-4662757781
|
||||
@@ -2325,6 +2336,7 @@ def compile_factors() -> dict[str, object]:
|
||||
"VLLM_MAX_AUDIO_DECODE_DURATION_S",
|
||||
"VLLM_MAX_AUDIO_DECODE_BYTES",
|
||||
"VLLM_MAX_AUDIO_PREPROCESS_WORKERS",
|
||||
"VLLM_MAX_EMBED_DECODE_BYTES",
|
||||
"VLLM_MAX_IMAGE_PIXELS",
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
"VLLM_MEDIA_CONNECTOR",
|
||||
|
||||
@@ -16,7 +16,10 @@ from vllm.multimodal.audio import resample_audio_pyav
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
from vllm.utils.mem_constants import MiB_bytes
|
||||
from vllm.utils.serial_utils import tensor2base64
|
||||
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
|
||||
from vllm.utils.sparse_utils import (
|
||||
check_sparse_tensor_invariants_threadsafe,
|
||||
safe_to_dense,
|
||||
)
|
||||
|
||||
from .base import MediaIO
|
||||
|
||||
@@ -348,7 +351,7 @@ class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]):
|
||||
buffer = BytesIO(data)
|
||||
with check_sparse_tensor_invariants_threadsafe():
|
||||
tensor = torch.load(buffer, weights_only=True)
|
||||
return tensor.to_dense()
|
||||
return safe_to_dense(tensor, parameter="audio_embeds")
|
||||
|
||||
def load_base64(self, media_type: str, data: str) -> torch.Tensor:
|
||||
return self.load_bytes(pybase64.b64decode(data, validate=True))
|
||||
@@ -356,7 +359,7 @@ class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]):
|
||||
def load_file(self, filepath: Path) -> torch.Tensor:
|
||||
with check_sparse_tensor_invariants_threadsafe():
|
||||
tensor = torch.load(filepath, weights_only=True)
|
||||
return tensor.to_dense()
|
||||
return safe_to_dense(tensor, parameter="audio_embeds")
|
||||
|
||||
def encode_base64(self, media: torch.Tensor) -> str:
|
||||
return tensor2base64(media)
|
||||
|
||||
@@ -11,7 +11,10 @@ from PIL import Image
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.utils.serial_utils import tensor2base64
|
||||
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
|
||||
from vllm.utils.sparse_utils import (
|
||||
check_sparse_tensor_invariants_threadsafe,
|
||||
safe_to_dense,
|
||||
)
|
||||
|
||||
from ..image import convert_image_mode, normalize_image, rgba_to_rgb
|
||||
from .base import MediaIO, MediaWithBytes
|
||||
@@ -135,7 +138,7 @@ class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
|
||||
buffer = BytesIO(data)
|
||||
with check_sparse_tensor_invariants_threadsafe():
|
||||
tensor = torch.load(buffer, weights_only=True)
|
||||
return tensor.to_dense()
|
||||
return safe_to_dense(tensor, parameter="image_embeds")
|
||||
|
||||
def _load_numpy(self, data: bytes) -> torch.Tensor:
|
||||
with BytesIO(data) as buffer:
|
||||
@@ -156,7 +159,7 @@ class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
|
||||
|
||||
with check_sparse_tensor_invariants_threadsafe():
|
||||
tensor = torch.load(filepath, weights_only=True)
|
||||
return tensor.to_dense()
|
||||
return safe_to_dense(tensor, parameter="image_embeds")
|
||||
|
||||
def encode_base64(self, media: torch.Tensor) -> str:
|
||||
return tensor2base64(media)
|
||||
|
||||
@@ -14,7 +14,10 @@ from PIL import Image
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.serial_utils import tensor2base64
|
||||
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
|
||||
from vllm.utils.sparse_utils import (
|
||||
check_sparse_tensor_invariants_threadsafe,
|
||||
safe_to_dense,
|
||||
)
|
||||
|
||||
from ..video import VIDEO_LOADER_REGISTRY
|
||||
from .base import MediaIO, MediaWithBytes
|
||||
@@ -215,7 +218,7 @@ class VideoEmbeddingMediaIO(MediaIO[torch.Tensor]):
|
||||
buffer = BytesIO(data)
|
||||
with check_sparse_tensor_invariants_threadsafe():
|
||||
tensor = torch.load(buffer, weights_only=True)
|
||||
return tensor.to_dense()
|
||||
return safe_to_dense(tensor, parameter="video_embeds")
|
||||
|
||||
def _load_numpy(self, data: bytes) -> torch.Tensor:
|
||||
with BytesIO(data) as buffer:
|
||||
@@ -236,7 +239,7 @@ class VideoEmbeddingMediaIO(MediaIO[torch.Tensor]):
|
||||
|
||||
with check_sparse_tensor_invariants_threadsafe():
|
||||
tensor = torch.load(filepath, weights_only=True)
|
||||
return tensor.to_dense()
|
||||
return safe_to_dense(tensor, parameter="video_embeds")
|
||||
|
||||
def encode_base64(self, media: torch.Tensor) -> str:
|
||||
return tensor2base64(media)
|
||||
|
||||
@@ -8,7 +8,10 @@ import torch
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
|
||||
from vllm.utils.sparse_utils import (
|
||||
check_sparse_tensor_invariants_threadsafe,
|
||||
safe_to_dense,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
@@ -30,12 +33,7 @@ def safe_load_prompt_embeds(
|
||||
weights_only=True,
|
||||
map_location=torch.device("cpu"),
|
||||
)
|
||||
if not isinstance(tensor, torch.Tensor):
|
||||
raise VLLMValidationError(
|
||||
"`prompt_embeds` payload did not deserialize to a torch.Tensor.",
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
tensor = tensor.to_dense()
|
||||
tensor = safe_to_dense(tensor, parameter="prompt_embeds")
|
||||
|
||||
if tensor.dim() > 2:
|
||||
tensor = tensor.squeeze(0)
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Thread-safe wrapper for sparse tensor invariant validation.
|
||||
Safe deserialization helpers for client-supplied embedding tensors.
|
||||
|
||||
`check_sparse_tensor_invariants_threadsafe()` is a thread-safe wrapper for
|
||||
sparse tensor invariant validation. `safe_to_dense()` bounds the memory a
|
||||
deserialized tensor may allocate when densified.
|
||||
|
||||
PyTorch's `torch.sparse.check_sparse_tensor_invariants()` context manager
|
||||
manipulates a **process-global** flag (save/enable/restore). When multiple
|
||||
@@ -18,6 +22,10 @@ import threading
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.utils.mem_constants import MiB_bytes
|
||||
|
||||
_SPARSE_LOAD_LOCK = threading.Lock()
|
||||
|
||||
|
||||
@@ -25,3 +33,41 @@ _SPARSE_LOAD_LOCK = threading.Lock()
|
||||
def check_sparse_tensor_invariants_threadsafe():
|
||||
with _SPARSE_LOAD_LOCK, torch.sparse.check_sparse_tensor_invariants():
|
||||
yield
|
||||
|
||||
|
||||
def safe_to_dense(tensor: object, *, parameter: str) -> torch.Tensor:
|
||||
"""Densify a client-supplied embedding tensor within a memory budget.
|
||||
|
||||
`torch.load(..., weights_only=True)` prevents arbitrary code execution but
|
||||
not memory amplification: a sparse COO tensor carries its own declared
|
||||
shape, so the dense allocation is bounded by that shape rather than by the
|
||||
request. A 2.6 KB payload declaring `(30000, 30000)` materializes 3.4 GiB,
|
||||
and the shape can be picked to fit whatever host is being targeted. The
|
||||
invariant checks only reject *invalid* tensors (indices outside the
|
||||
declared shape); a single non-zero element inside a `(2**20, 2**20)` shape
|
||||
is perfectly valid and passes them.
|
||||
|
||||
So reject oversized payloads based on the dense size they declare, before
|
||||
`to_dense()` allocates anything.
|
||||
"""
|
||||
if not isinstance(tensor, torch.Tensor):
|
||||
raise VLLMValidationError(
|
||||
f"`{parameter}` payload did not deserialize to a torch.Tensor.",
|
||||
parameter=parameter,
|
||||
)
|
||||
|
||||
max_bytes = envs.VLLM_MAX_EMBED_DECODE_BYTES
|
||||
if max_bytes > 0:
|
||||
dense_bytes = tensor.numel() * tensor.element_size()
|
||||
if dense_bytes > max_bytes:
|
||||
raise VLLMValidationError(
|
||||
f"`{parameter}` declares shape {tuple(tensor.shape)}, which "
|
||||
f"would allocate {dense_bytes} bytes "
|
||||
f"({dense_bytes / MiB_bytes:.0f} MiB) once densified, "
|
||||
f"exceeding the {max_bytes} byte limit. Set "
|
||||
f"VLLM_MAX_EMBED_DECODE_BYTES to increase this limit.",
|
||||
parameter=parameter,
|
||||
value=dense_bytes,
|
||||
)
|
||||
|
||||
return tensor.to_dense()
|
||||
|
||||
Reference in New Issue
Block a user