[Bugfix] Unreadable prompt_embeds payload should be a 400, not a 500 (#55451)
Signed-off-by: lzhan011 <35493221+lzhan011@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Claude Opus 5
Cyrus Leung
parent
4fe9e6f6e5
commit
323a29d8f2
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""`prompt_embeds` payloads that are not tensor files must be client errors.
|
||||
|
||||
`prompt_embeds` is a base64 blob the caller writes, and every other way of
|
||||
getting it wrong -- wrong rank, wrong hidden_size, wrong dtype, not a tensor at
|
||||
all -- already answers 400. A payload that `torch.load` cannot read at all was
|
||||
the exception: torch has no single error type for that (the zip reader raises
|
||||
`RuntimeError`, the legacy pickle path raises an unpickling error, `KeyError` or
|
||||
`IndexError`, an empty payload raises `EOFError`), none of them is a
|
||||
`ValueError`, and the entrypoints' fallback therefore mapped them to 500.
|
||||
|
||||
The assertions go through `create_error_response`, which is what actually turns
|
||||
an exception into a status code, so they pin the HTTP answer rather than the
|
||||
exception class alone.
|
||||
"""
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
from unittest.mock import patch
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers.embed_utils import (
|
||||
_MAX_EMBED_ERROR_REASON_CHARS,
|
||||
safe_load_prompt_embeds,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def model_config():
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
return ModelConfig(
|
||||
model="facebook/opt-125m",
|
||||
tokenizer="facebook/opt-125m",
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=False,
|
||||
dtype="float32",
|
||||
seed=0,
|
||||
enable_prompt_embeds=True,
|
||||
)
|
||||
|
||||
|
||||
def _encode(raw: bytes) -> bytes:
|
||||
return base64.b64encode(raw)
|
||||
|
||||
|
||||
def _zip_without_a_tensor() -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(buffer, "w") as archive:
|
||||
archive.writestr("junk.txt", "hello")
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def _saved_tensor(tensor: torch.Tensor) -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
torch.save(tensor, buffer)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
UNPARSABLE_PAYLOADS = [
|
||||
pytest.param(b"", id="empty"),
|
||||
pytest.param(b"\x80", id="single-byte"),
|
||||
pytest.param(b"\x00\x01\x02\x03not-a-tensor", id="random-bytes"),
|
||||
pytest.param(b"hello world this is not a torch file", id="ascii-text"),
|
||||
pytest.param(b"PK\x03\x04" + b"\x00" * 20, id="truncated-zip"),
|
||||
pytest.param(_zip_without_a_tensor(), id="zip-without-a-tensor"),
|
||||
# pickle.dumps(42): a valid pickle stream that is not a torch checkpoint.
|
||||
# Spelled out rather than built, so this file does not import pickle.
|
||||
pytest.param(b"\x80\x04K*.", id="legacy-pickle-of-an-int"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", UNPARSABLE_PAYLOADS)
|
||||
def test_unreadable_prompt_embeds_payload_is_a_client_error(model_config, payload):
|
||||
with pytest.raises(VLLMValidationError) as excinfo:
|
||||
safe_load_prompt_embeds(model_config, _encode(payload))
|
||||
|
||||
assert excinfo.value.parameter == "prompt_embeds"
|
||||
assert int(create_error_response(excinfo.value).error.code) == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", UNPARSABLE_PAYLOADS)
|
||||
def test_the_error_body_stays_bounded(model_config, payload):
|
||||
"""torch's reason is built from the caller's bytes, so it must be truncated.
|
||||
|
||||
An unpickler that reports the offending global, for instance, quotes a name
|
||||
the caller chose; without a bound a large one would be reflected whole.
|
||||
"""
|
||||
padding = b"A" * 100_000
|
||||
with pytest.raises(VLLMValidationError) as excinfo:
|
||||
safe_load_prompt_embeds(model_config, _encode(payload + padding))
|
||||
|
||||
assert len(str(excinfo.value)) < 500
|
||||
|
||||
|
||||
def test_a_long_reason_is_truncated_and_says_how_much_was_left_out(model_config):
|
||||
"""The cap is pinned against a reason of known length, not a real payload.
|
||||
|
||||
Which of torch's messages happen to exceed the cap is torch's business and
|
||||
changes between releases, so the reason is injected here: that keeps the
|
||||
assertion about *this* code's truncation rule rather than about whichever
|
||||
error text the installed torch produces.
|
||||
"""
|
||||
reason = "E" * (_MAX_EMBED_ERROR_REASON_CHARS + 37)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.renderers.embed_utils.torch.load", side_effect=RuntimeError(reason)
|
||||
),
|
||||
pytest.raises(VLLMValidationError) as excinfo,
|
||||
):
|
||||
safe_load_prompt_embeds(model_config, _encode(b"ignored"))
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "E" * _MAX_EMBED_ERROR_REASON_CHARS in message
|
||||
assert "E" * (_MAX_EMBED_ERROR_REASON_CHARS + 1) not in message
|
||||
assert "... (37 more characters truncated)" in message
|
||||
|
||||
|
||||
def test_a_short_reason_is_passed_through_whole(model_config):
|
||||
"""A reason that fits is not annotated -- the count only appears when it is
|
||||
telling the reader something."""
|
||||
reason = "invalid magic number; corrupt file?"
|
||||
assert len(reason) <= _MAX_EMBED_ERROR_REASON_CHARS
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.renderers.embed_utils.torch.load", side_effect=RuntimeError(reason)
|
||||
),
|
||||
pytest.raises(VLLMValidationError) as excinfo,
|
||||
):
|
||||
safe_load_prompt_embeds(model_config, _encode(b"ignored"))
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert reason in message
|
||||
assert "truncated" not in message
|
||||
|
||||
|
||||
def test_a_well_formed_payload_still_loads(model_config):
|
||||
"""Positive control: the healthy path is untouched."""
|
||||
hidden_size = model_config.get_hidden_size()
|
||||
tensor = torch.zeros(3, hidden_size, dtype=torch.float32)
|
||||
|
||||
loaded = safe_load_prompt_embeds(model_config, _encode(_saved_tensor(tensor)))
|
||||
|
||||
assert loaded.shape == (3, hidden_size)
|
||||
assert loaded.dtype == model_config.dtype
|
||||
|
||||
|
||||
def test_a_payload_that_is_not_base64_is_still_a_client_error(model_config):
|
||||
"""Positive control: the base64 stage already answered 400 and still does."""
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
safe_load_prompt_embeds(model_config, b"!!!!not base64!!!!")
|
||||
|
||||
assert int(create_error_response(excinfo.value).error.code) == 400
|
||||
@@ -11,6 +11,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.multimodal.media import AudioEmbeddingMediaIO, ImageEmbeddingMediaIO
|
||||
from vllm.renderers.embed_utils import safe_load_prompt_embeds
|
||||
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
|
||||
@@ -54,7 +55,7 @@ class TestNegativeControlWithoutRace:
|
||||
|
||||
def test_malicious_sparse_rejected_by_prompt_loader(self, model_config):
|
||||
encoded = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
def test_malicious_sparse_rejected_by_image_loader(self):
|
||||
@@ -147,7 +148,7 @@ class TestConcurrentRaceProtection:
|
||||
try:
|
||||
safe_load_prompt_embeds(model_config, malicious_encoded)
|
||||
bypassed.append(True)
|
||||
except (RuntimeError, ValueError):
|
||||
except (RuntimeError, ValueError, VLLMValidationError):
|
||||
rejected.append(True)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=num_workers) as pool:
|
||||
@@ -192,7 +193,7 @@ class TestGlobalFlagRestoration:
|
||||
initial = torch.sparse.check_sparse_tensor_invariants.is_enabled()
|
||||
malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
|
||||
safe_load_prompt_embeds(model_config, malicious_encoded)
|
||||
|
||||
assert torch.sparse.check_sparse_tensor_invariants.is_enabled() == initial
|
||||
@@ -203,7 +204,7 @@ class TestGlobalFlagRestoration:
|
||||
malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
|
||||
def attempt_load(_):
|
||||
with contextlib.suppress(RuntimeError, ValueError):
|
||||
with contextlib.suppress(RuntimeError, ValueError, VLLMValidationError):
|
||||
safe_load_prompt_embeds(model_config, malicious_encoded)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
@@ -244,7 +245,7 @@ class TestCrossLoaderLockSharing:
|
||||
# serialization — one finishes before the other starts), AND invalid
|
||||
# tensors are still rejected.
|
||||
malicious = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
|
||||
safe_load_prompt_embeds(model_config, malicious)
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
io_handler.load_base64("", malicious.decode("utf-8"))
|
||||
|
||||
@@ -117,8 +117,10 @@ class TestPromptEmbedsValidation:
|
||||
malicious_tensor = _create_malicious_sparse_tensor()
|
||||
encoded = _encode_tensor(malicious_tensor)
|
||||
|
||||
# Should raise RuntimeError due to invalid sparse tensor
|
||||
with pytest.raises((RuntimeError, ValueError)) as exc_info:
|
||||
# The invariant check raises RuntimeError from inside torch.load;
|
||||
# safe_load_prompt_embeds now reports that as a client error so the
|
||||
# caller gets a 400 rather than the 500 a bare RuntimeError produced.
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
# Error should indicate sparse tensor validation failure
|
||||
@@ -137,7 +139,7 @@ class TestPromptEmbedsValidation:
|
||||
)
|
||||
encoded = _encode_tensor(malicious_tensor)
|
||||
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises(VLLMValidationError):
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
def test_negative_indices_rejected(self, model_config):
|
||||
@@ -152,7 +154,7 @@ class TestPromptEmbedsValidation:
|
||||
)
|
||||
encoded = _encode_tensor(malicious_tensor)
|
||||
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises(VLLMValidationError):
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
def test_hidden_size_mismatch_rejected(self, model_config):
|
||||
@@ -379,8 +381,8 @@ class TestSparseTensorValidationIntegration:
|
||||
# Step 1-2: Attacker creates malicious payload
|
||||
attack_payload = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
|
||||
# Step 3-4: Server processes and should reject
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
# Step 3-4: Server processes and should reject, as a client error
|
||||
with pytest.raises(VLLMValidationError):
|
||||
safe_load_prompt_embeds(model_config, attack_payload)
|
||||
|
||||
def test_attack_scenario_chat_api_image(self):
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from io import BytesIO
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import pybase64
|
||||
import torch
|
||||
@@ -17,6 +17,27 @@ if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
|
||||
# How much of torch's own reason is repeated back to the caller. The reason is
|
||||
# built from bytes the caller sent, so it is bounded rather than echoed whole.
|
||||
_MAX_EMBED_ERROR_REASON_CHARS: Final = 200
|
||||
|
||||
|
||||
def _truncated_reason(exc: Exception) -> str:
|
||||
"""`exc`'s message, capped, saying how much was left out when it is capped.
|
||||
|
||||
Without the count a capped reason reads like the whole one, so a caller
|
||||
debugging a large payload cannot tell that torch said more than this.
|
||||
"""
|
||||
reason: Final = str(exc).strip() or type(exc).__name__
|
||||
omitted: Final = len(reason) - _MAX_EMBED_ERROR_REASON_CHARS
|
||||
if omitted <= 0:
|
||||
return reason
|
||||
return (
|
||||
f"{reason[:_MAX_EMBED_ERROR_REASON_CHARS]}"
|
||||
f"... ({omitted} more characters truncated)"
|
||||
)
|
||||
|
||||
|
||||
def safe_load_prompt_embeds(
|
||||
model_config: "ModelConfig",
|
||||
embed: bytes,
|
||||
@@ -27,12 +48,38 @@ def safe_load_prompt_embeds(
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
|
||||
# Decoded outside the try below so that a body which is not base64 keeps
|
||||
# raising binascii.Error, which is a ValueError and already answered 400.
|
||||
payload: Final = pybase64.b64decode(embed, validate=True)
|
||||
|
||||
with check_sparse_tensor_invariants_threadsafe():
|
||||
tensor = torch.load(
|
||||
BytesIO(pybase64.b64decode(embed, validate=True)),
|
||||
weights_only=True,
|
||||
map_location=torch.device("cpu"),
|
||||
)
|
||||
try:
|
||||
tensor = torch.load(
|
||||
BytesIO(payload),
|
||||
weights_only=True,
|
||||
map_location=torch.device("cpu"),
|
||||
)
|
||||
except (torch.OutOfMemoryError, MemoryError):
|
||||
# Running out of memory is the server's condition, not the caller's.
|
||||
raise
|
||||
except Exception as exc:
|
||||
# Everything else here is about the bytes the caller sent, and torch
|
||||
# has no single error type for "this is not a tensor file": the zip
|
||||
# reader raises `RuntimeError`, the legacy pickle path raises
|
||||
# `UnpicklingError`, `KeyError` or `IndexError`, an empty payload
|
||||
# raises `EOFError`. None of them is a `ValueError`, so the
|
||||
# entrypoints' fallback mapped them all to 500. Catching by
|
||||
# behaviour rather than by type also survives torch changing which
|
||||
# error it raises for a given malformed payload.
|
||||
#
|
||||
# torch's reason is worth keeping -- for a malformed sparse tensor
|
||||
# it names the offending index -- but it is built from the caller's
|
||||
# own bytes, so it is truncated rather than echoed whole.
|
||||
raise VLLMValidationError(
|
||||
"`prompt_embeds` could not be deserialized as a torch tensor: "
|
||||
f"{_truncated_reason(exc)}",
|
||||
parameter="prompt_embeds",
|
||||
) from exc
|
||||
tensor = safe_to_dense(tensor, parameter="prompt_embeds")
|
||||
|
||||
if tensor.dim() > 2:
|
||||
|
||||
Reference in New Issue
Block a user