[Bugfix] Unreadable prompt_embeds payload should be a 400, not a 500 (#55451)

Signed-off-by: lzhan011 <35493221+lzhan011@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
lzhan011
2026-09-16 05:53:36 +00:00
committed by GitHub
co-authored by Claude Opus 5 Cyrus Leung
parent 4fe9e6f6e5
commit 323a29d8f2
4 changed files with 230 additions and 17 deletions
@@ -0,0 +1,163 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""`prompt_embeds` payloads that are not tensor files must be client errors.
`prompt_embeds` is a base64 blob the caller writes, and every other way of
getting it wrong -- wrong rank, wrong hidden_size, wrong dtype, not a tensor at
all -- already answers 400. A payload that `torch.load` cannot read at all was
the exception: torch has no single error type for that (the zip reader raises
`RuntimeError`, the legacy pickle path raises an unpickling error, `KeyError` or
`IndexError`, an empty payload raises `EOFError`), none of them is a
`ValueError`, and the entrypoints' fallback therefore mapped them to 500.
The assertions go through `create_error_response`, which is what actually turns
an exception into a status code, so they pin the HTTP answer rather than the
exception class alone.
"""
import io
import zipfile
from unittest.mock import patch
import pybase64 as base64
import pytest
import torch
from vllm.entrypoints.serve.exception_handling.error_response import (
create_error_response,
)
from vllm.exceptions import VLLMValidationError
from vllm.renderers.embed_utils import (
_MAX_EMBED_ERROR_REASON_CHARS,
safe_load_prompt_embeds,
)
@pytest.fixture
def model_config():
from vllm.config import ModelConfig
return ModelConfig(
model="facebook/opt-125m",
tokenizer="facebook/opt-125m",
tokenizer_mode="auto",
trust_remote_code=False,
dtype="float32",
seed=0,
enable_prompt_embeds=True,
)
def _encode(raw: bytes) -> bytes:
return base64.b64encode(raw)
def _zip_without_a_tensor() -> bytes:
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, "w") as archive:
archive.writestr("junk.txt", "hello")
return buffer.getvalue()
def _saved_tensor(tensor: torch.Tensor) -> bytes:
buffer = io.BytesIO()
torch.save(tensor, buffer)
return buffer.getvalue()
UNPARSABLE_PAYLOADS = [
pytest.param(b"", id="empty"),
pytest.param(b"\x80", id="single-byte"),
pytest.param(b"\x00\x01\x02\x03not-a-tensor", id="random-bytes"),
pytest.param(b"hello world this is not a torch file", id="ascii-text"),
pytest.param(b"PK\x03\x04" + b"\x00" * 20, id="truncated-zip"),
pytest.param(_zip_without_a_tensor(), id="zip-without-a-tensor"),
# pickle.dumps(42): a valid pickle stream that is not a torch checkpoint.
# Spelled out rather than built, so this file does not import pickle.
pytest.param(b"\x80\x04K*.", id="legacy-pickle-of-an-int"),
]
@pytest.mark.parametrize("payload", UNPARSABLE_PAYLOADS)
def test_unreadable_prompt_embeds_payload_is_a_client_error(model_config, payload):
with pytest.raises(VLLMValidationError) as excinfo:
safe_load_prompt_embeds(model_config, _encode(payload))
assert excinfo.value.parameter == "prompt_embeds"
assert int(create_error_response(excinfo.value).error.code) == 400
@pytest.mark.parametrize("payload", UNPARSABLE_PAYLOADS)
def test_the_error_body_stays_bounded(model_config, payload):
"""torch's reason is built from the caller's bytes, so it must be truncated.
An unpickler that reports the offending global, for instance, quotes a name
the caller chose; without a bound a large one would be reflected whole.
"""
padding = b"A" * 100_000
with pytest.raises(VLLMValidationError) as excinfo:
safe_load_prompt_embeds(model_config, _encode(payload + padding))
assert len(str(excinfo.value)) < 500
def test_a_long_reason_is_truncated_and_says_how_much_was_left_out(model_config):
"""The cap is pinned against a reason of known length, not a real payload.
Which of torch's messages happen to exceed the cap is torch's business and
changes between releases, so the reason is injected here: that keeps the
assertion about *this* code's truncation rule rather than about whichever
error text the installed torch produces.
"""
reason = "E" * (_MAX_EMBED_ERROR_REASON_CHARS + 37)
with (
patch(
"vllm.renderers.embed_utils.torch.load", side_effect=RuntimeError(reason)
),
pytest.raises(VLLMValidationError) as excinfo,
):
safe_load_prompt_embeds(model_config, _encode(b"ignored"))
message = str(excinfo.value)
assert "E" * _MAX_EMBED_ERROR_REASON_CHARS in message
assert "E" * (_MAX_EMBED_ERROR_REASON_CHARS + 1) not in message
assert "... (37 more characters truncated)" in message
def test_a_short_reason_is_passed_through_whole(model_config):
"""A reason that fits is not annotated -- the count only appears when it is
telling the reader something."""
reason = "invalid magic number; corrupt file?"
assert len(reason) <= _MAX_EMBED_ERROR_REASON_CHARS
with (
patch(
"vllm.renderers.embed_utils.torch.load", side_effect=RuntimeError(reason)
),
pytest.raises(VLLMValidationError) as excinfo,
):
safe_load_prompt_embeds(model_config, _encode(b"ignored"))
message = str(excinfo.value)
assert reason in message
assert "truncated" not in message
def test_a_well_formed_payload_still_loads(model_config):
"""Positive control: the healthy path is untouched."""
hidden_size = model_config.get_hidden_size()
tensor = torch.zeros(3, hidden_size, dtype=torch.float32)
loaded = safe_load_prompt_embeds(model_config, _encode(_saved_tensor(tensor)))
assert loaded.shape == (3, hidden_size)
assert loaded.dtype == model_config.dtype
def test_a_payload_that_is_not_base64_is_still_a_client_error(model_config):
"""Positive control: the base64 stage already answered 400 and still does."""
with pytest.raises(ValueError) as excinfo:
safe_load_prompt_embeds(model_config, b"!!!!not base64!!!!")
assert int(create_error_response(excinfo.value).error.code) == 400
@@ -11,6 +11,7 @@ import pytest
import torch
from vllm.config import ModelConfig
from vllm.exceptions import VLLMValidationError
from vllm.multimodal.media import AudioEmbeddingMediaIO, ImageEmbeddingMediaIO
from vllm.renderers.embed_utils import safe_load_prompt_embeds
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
@@ -54,7 +55,7 @@ class TestNegativeControlWithoutRace:
def test_malicious_sparse_rejected_by_prompt_loader(self, model_config):
encoded = _encode_tensor(_create_malicious_sparse_tensor())
with pytest.raises((RuntimeError, ValueError)):
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
safe_load_prompt_embeds(model_config, encoded)
def test_malicious_sparse_rejected_by_image_loader(self):
@@ -147,7 +148,7 @@ class TestConcurrentRaceProtection:
try:
safe_load_prompt_embeds(model_config, malicious_encoded)
bypassed.append(True)
except (RuntimeError, ValueError):
except (RuntimeError, ValueError, VLLMValidationError):
rejected.append(True)
with ThreadPoolExecutor(max_workers=num_workers) as pool:
@@ -192,7 +193,7 @@ class TestGlobalFlagRestoration:
initial = torch.sparse.check_sparse_tensor_invariants.is_enabled()
malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor())
with pytest.raises((RuntimeError, ValueError)):
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
safe_load_prompt_embeds(model_config, malicious_encoded)
assert torch.sparse.check_sparse_tensor_invariants.is_enabled() == initial
@@ -203,7 +204,7 @@ class TestGlobalFlagRestoration:
malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor())
def attempt_load(_):
with contextlib.suppress(RuntimeError, ValueError):
with contextlib.suppress(RuntimeError, ValueError, VLLMValidationError):
safe_load_prompt_embeds(model_config, malicious_encoded)
with ThreadPoolExecutor(max_workers=4) as pool:
@@ -244,7 +245,7 @@ class TestCrossLoaderLockSharing:
# serialization — one finishes before the other starts), AND invalid
# tensors are still rejected.
malicious = _encode_tensor(_create_malicious_sparse_tensor())
with pytest.raises((RuntimeError, ValueError)):
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
safe_load_prompt_embeds(model_config, malicious)
with pytest.raises((RuntimeError, ValueError)):
io_handler.load_base64("", malicious.decode("utf-8"))
@@ -117,8 +117,10 @@ class TestPromptEmbedsValidation:
malicious_tensor = _create_malicious_sparse_tensor()
encoded = _encode_tensor(malicious_tensor)
# Should raise RuntimeError due to invalid sparse tensor
with pytest.raises((RuntimeError, ValueError)) as exc_info:
# The invariant check raises RuntimeError from inside torch.load;
# safe_load_prompt_embeds now reports that as a client error so the
# caller gets a 400 rather than the 500 a bare RuntimeError produced.
with pytest.raises(VLLMValidationError) as exc_info:
safe_load_prompt_embeds(model_config, encoded)
# Error should indicate sparse tensor validation failure
@@ -137,7 +139,7 @@ class TestPromptEmbedsValidation:
)
encoded = _encode_tensor(malicious_tensor)
with pytest.raises((RuntimeError, ValueError)):
with pytest.raises(VLLMValidationError):
safe_load_prompt_embeds(model_config, encoded)
def test_negative_indices_rejected(self, model_config):
@@ -152,7 +154,7 @@ class TestPromptEmbedsValidation:
)
encoded = _encode_tensor(malicious_tensor)
with pytest.raises((RuntimeError, ValueError)):
with pytest.raises(VLLMValidationError):
safe_load_prompt_embeds(model_config, encoded)
def test_hidden_size_mismatch_rejected(self, model_config):
@@ -379,8 +381,8 @@ class TestSparseTensorValidationIntegration:
# Step 1-2: Attacker creates malicious payload
attack_payload = _encode_tensor(_create_malicious_sparse_tensor())
# Step 3-4: Server processes and should reject
with pytest.raises((RuntimeError, ValueError)):
# Step 3-4: Server processes and should reject, as a client error
with pytest.raises(VLLMValidationError):
safe_load_prompt_embeds(model_config, attack_payload)
def test_attack_scenario_chat_api_image(self):
+53 -6
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from io import BytesIO
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Final
import pybase64
import torch
@@ -17,6 +17,27 @@ if TYPE_CHECKING:
from vllm.config import ModelConfig
# How much of torch's own reason is repeated back to the caller. The reason is
# built from bytes the caller sent, so it is bounded rather than echoed whole.
_MAX_EMBED_ERROR_REASON_CHARS: Final = 200
def _truncated_reason(exc: Exception) -> str:
"""`exc`'s message, capped, saying how much was left out when it is capped.
Without the count a capped reason reads like the whole one, so a caller
debugging a large payload cannot tell that torch said more than this.
"""
reason: Final = str(exc).strip() or type(exc).__name__
omitted: Final = len(reason) - _MAX_EMBED_ERROR_REASON_CHARS
if omitted <= 0:
return reason
return (
f"{reason[:_MAX_EMBED_ERROR_REASON_CHARS]}"
f"... ({omitted} more characters truncated)"
)
def safe_load_prompt_embeds(
model_config: "ModelConfig",
embed: bytes,
@@ -27,12 +48,38 @@ def safe_load_prompt_embeds(
parameter="prompt_embeds",
)
# Decoded outside the try below so that a body which is not base64 keeps
# raising binascii.Error, which is a ValueError and already answered 400.
payload: Final = pybase64.b64decode(embed, validate=True)
with check_sparse_tensor_invariants_threadsafe():
tensor = torch.load(
BytesIO(pybase64.b64decode(embed, validate=True)),
weights_only=True,
map_location=torch.device("cpu"),
)
try:
tensor = torch.load(
BytesIO(payload),
weights_only=True,
map_location=torch.device("cpu"),
)
except (torch.OutOfMemoryError, MemoryError):
# Running out of memory is the server's condition, not the caller's.
raise
except Exception as exc:
# Everything else here is about the bytes the caller sent, and torch
# has no single error type for "this is not a tensor file": the zip
# reader raises `RuntimeError`, the legacy pickle path raises
# `UnpicklingError`, `KeyError` or `IndexError`, an empty payload
# raises `EOFError`. None of them is a `ValueError`, so the
# entrypoints' fallback mapped them all to 500. Catching by
# behaviour rather than by type also survives torch changing which
# error it raises for a given malformed payload.
#
# torch's reason is worth keeping -- for a malformed sparse tensor
# it names the offending index -- but it is built from the caller's
# own bytes, so it is truncated rather than echoed whole.
raise VLLMValidationError(
"`prompt_embeds` could not be deserialized as a torch tensor: "
f"{_truncated_reason(exc)}",
parameter="prompt_embeds",
) from exc
tensor = safe_to_dense(tensor, parameter="prompt_embeds")
if tensor.dim() > 2: