[Bugfix] Unreadable prompt_embeds payload should be a 400, not a 500 (#55451)
Signed-off-by: lzhan011 <35493221+lzhan011@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Claude Opus 5
Cyrus Leung
parent
4fe9e6f6e5
commit
323a29d8f2
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""`prompt_embeds` payloads that are not tensor files must be client errors.
|
||||
|
||||
`prompt_embeds` is a base64 blob the caller writes, and every other way of
|
||||
getting it wrong -- wrong rank, wrong hidden_size, wrong dtype, not a tensor at
|
||||
all -- already answers 400. A payload that `torch.load` cannot read at all was
|
||||
the exception: torch has no single error type for that (the zip reader raises
|
||||
`RuntimeError`, the legacy pickle path raises an unpickling error, `KeyError` or
|
||||
`IndexError`, an empty payload raises `EOFError`), none of them is a
|
||||
`ValueError`, and the entrypoints' fallback therefore mapped them to 500.
|
||||
|
||||
The assertions go through `create_error_response`, which is what actually turns
|
||||
an exception into a status code, so they pin the HTTP answer rather than the
|
||||
exception class alone.
|
||||
"""
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
from unittest.mock import patch
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers.embed_utils import (
|
||||
_MAX_EMBED_ERROR_REASON_CHARS,
|
||||
safe_load_prompt_embeds,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def model_config():
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
return ModelConfig(
|
||||
model="facebook/opt-125m",
|
||||
tokenizer="facebook/opt-125m",
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=False,
|
||||
dtype="float32",
|
||||
seed=0,
|
||||
enable_prompt_embeds=True,
|
||||
)
|
||||
|
||||
|
||||
def _encode(raw: bytes) -> bytes:
|
||||
return base64.b64encode(raw)
|
||||
|
||||
|
||||
def _zip_without_a_tensor() -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(buffer, "w") as archive:
|
||||
archive.writestr("junk.txt", "hello")
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def _saved_tensor(tensor: torch.Tensor) -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
torch.save(tensor, buffer)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
UNPARSABLE_PAYLOADS = [
|
||||
pytest.param(b"", id="empty"),
|
||||
pytest.param(b"\x80", id="single-byte"),
|
||||
pytest.param(b"\x00\x01\x02\x03not-a-tensor", id="random-bytes"),
|
||||
pytest.param(b"hello world this is not a torch file", id="ascii-text"),
|
||||
pytest.param(b"PK\x03\x04" + b"\x00" * 20, id="truncated-zip"),
|
||||
pytest.param(_zip_without_a_tensor(), id="zip-without-a-tensor"),
|
||||
# pickle.dumps(42): a valid pickle stream that is not a torch checkpoint.
|
||||
# Spelled out rather than built, so this file does not import pickle.
|
||||
pytest.param(b"\x80\x04K*.", id="legacy-pickle-of-an-int"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", UNPARSABLE_PAYLOADS)
|
||||
def test_unreadable_prompt_embeds_payload_is_a_client_error(model_config, payload):
|
||||
with pytest.raises(VLLMValidationError) as excinfo:
|
||||
safe_load_prompt_embeds(model_config, _encode(payload))
|
||||
|
||||
assert excinfo.value.parameter == "prompt_embeds"
|
||||
assert int(create_error_response(excinfo.value).error.code) == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", UNPARSABLE_PAYLOADS)
|
||||
def test_the_error_body_stays_bounded(model_config, payload):
|
||||
"""torch's reason is built from the caller's bytes, so it must be truncated.
|
||||
|
||||
An unpickler that reports the offending global, for instance, quotes a name
|
||||
the caller chose; without a bound a large one would be reflected whole.
|
||||
"""
|
||||
padding = b"A" * 100_000
|
||||
with pytest.raises(VLLMValidationError) as excinfo:
|
||||
safe_load_prompt_embeds(model_config, _encode(payload + padding))
|
||||
|
||||
assert len(str(excinfo.value)) < 500
|
||||
|
||||
|
||||
def test_a_long_reason_is_truncated_and_says_how_much_was_left_out(model_config):
|
||||
"""The cap is pinned against a reason of known length, not a real payload.
|
||||
|
||||
Which of torch's messages happen to exceed the cap is torch's business and
|
||||
changes between releases, so the reason is injected here: that keeps the
|
||||
assertion about *this* code's truncation rule rather than about whichever
|
||||
error text the installed torch produces.
|
||||
"""
|
||||
reason = "E" * (_MAX_EMBED_ERROR_REASON_CHARS + 37)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.renderers.embed_utils.torch.load", side_effect=RuntimeError(reason)
|
||||
),
|
||||
pytest.raises(VLLMValidationError) as excinfo,
|
||||
):
|
||||
safe_load_prompt_embeds(model_config, _encode(b"ignored"))
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "E" * _MAX_EMBED_ERROR_REASON_CHARS in message
|
||||
assert "E" * (_MAX_EMBED_ERROR_REASON_CHARS + 1) not in message
|
||||
assert "... (37 more characters truncated)" in message
|
||||
|
||||
|
||||
def test_a_short_reason_is_passed_through_whole(model_config):
|
||||
"""A reason that fits is not annotated -- the count only appears when it is
|
||||
telling the reader something."""
|
||||
reason = "invalid magic number; corrupt file?"
|
||||
assert len(reason) <= _MAX_EMBED_ERROR_REASON_CHARS
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.renderers.embed_utils.torch.load", side_effect=RuntimeError(reason)
|
||||
),
|
||||
pytest.raises(VLLMValidationError) as excinfo,
|
||||
):
|
||||
safe_load_prompt_embeds(model_config, _encode(b"ignored"))
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert reason in message
|
||||
assert "truncated" not in message
|
||||
|
||||
|
||||
def test_a_well_formed_payload_still_loads(model_config):
|
||||
"""Positive control: the healthy path is untouched."""
|
||||
hidden_size = model_config.get_hidden_size()
|
||||
tensor = torch.zeros(3, hidden_size, dtype=torch.float32)
|
||||
|
||||
loaded = safe_load_prompt_embeds(model_config, _encode(_saved_tensor(tensor)))
|
||||
|
||||
assert loaded.shape == (3, hidden_size)
|
||||
assert loaded.dtype == model_config.dtype
|
||||
|
||||
|
||||
def test_a_payload_that_is_not_base64_is_still_a_client_error(model_config):
|
||||
"""Positive control: the base64 stage already answered 400 and still does."""
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
safe_load_prompt_embeds(model_config, b"!!!!not base64!!!!")
|
||||
|
||||
assert int(create_error_response(excinfo.value).error.code) == 400
|
||||
@@ -11,6 +11,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.multimodal.media import AudioEmbeddingMediaIO, ImageEmbeddingMediaIO
|
||||
from vllm.renderers.embed_utils import safe_load_prompt_embeds
|
||||
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
|
||||
@@ -54,7 +55,7 @@ class TestNegativeControlWithoutRace:
|
||||
|
||||
def test_malicious_sparse_rejected_by_prompt_loader(self, model_config):
|
||||
encoded = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
def test_malicious_sparse_rejected_by_image_loader(self):
|
||||
@@ -147,7 +148,7 @@ class TestConcurrentRaceProtection:
|
||||
try:
|
||||
safe_load_prompt_embeds(model_config, malicious_encoded)
|
||||
bypassed.append(True)
|
||||
except (RuntimeError, ValueError):
|
||||
except (RuntimeError, ValueError, VLLMValidationError):
|
||||
rejected.append(True)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=num_workers) as pool:
|
||||
@@ -192,7 +193,7 @@ class TestGlobalFlagRestoration:
|
||||
initial = torch.sparse.check_sparse_tensor_invariants.is_enabled()
|
||||
malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
|
||||
safe_load_prompt_embeds(model_config, malicious_encoded)
|
||||
|
||||
assert torch.sparse.check_sparse_tensor_invariants.is_enabled() == initial
|
||||
@@ -203,7 +204,7 @@ class TestGlobalFlagRestoration:
|
||||
malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
|
||||
def attempt_load(_):
|
||||
with contextlib.suppress(RuntimeError, ValueError):
|
||||
with contextlib.suppress(RuntimeError, ValueError, VLLMValidationError):
|
||||
safe_load_prompt_embeds(model_config, malicious_encoded)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
@@ -244,7 +245,7 @@ class TestCrossLoaderLockSharing:
|
||||
# serialization — one finishes before the other starts), AND invalid
|
||||
# tensors are still rejected.
|
||||
malicious = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
|
||||
safe_load_prompt_embeds(model_config, malicious)
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
io_handler.load_base64("", malicious.decode("utf-8"))
|
||||
|
||||
@@ -117,8 +117,10 @@ class TestPromptEmbedsValidation:
|
||||
malicious_tensor = _create_malicious_sparse_tensor()
|
||||
encoded = _encode_tensor(malicious_tensor)
|
||||
|
||||
# Should raise RuntimeError due to invalid sparse tensor
|
||||
with pytest.raises((RuntimeError, ValueError)) as exc_info:
|
||||
# The invariant check raises RuntimeError from inside torch.load;
|
||||
# safe_load_prompt_embeds now reports that as a client error so the
|
||||
# caller gets a 400 rather than the 500 a bare RuntimeError produced.
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
# Error should indicate sparse tensor validation failure
|
||||
@@ -137,7 +139,7 @@ class TestPromptEmbedsValidation:
|
||||
)
|
||||
encoded = _encode_tensor(malicious_tensor)
|
||||
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises(VLLMValidationError):
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
def test_negative_indices_rejected(self, model_config):
|
||||
@@ -152,7 +154,7 @@ class TestPromptEmbedsValidation:
|
||||
)
|
||||
encoded = _encode_tensor(malicious_tensor)
|
||||
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises(VLLMValidationError):
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
def test_hidden_size_mismatch_rejected(self, model_config):
|
||||
@@ -379,8 +381,8 @@ class TestSparseTensorValidationIntegration:
|
||||
# Step 1-2: Attacker creates malicious payload
|
||||
attack_payload = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
|
||||
# Step 3-4: Server processes and should reject
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
# Step 3-4: Server processes and should reject, as a client error
|
||||
with pytest.raises(VLLMValidationError):
|
||||
safe_load_prompt_embeds(model_config, attack_payload)
|
||||
|
||||
def test_attack_scenario_chat_api_image(self):
|
||||
|
||||
Reference in New Issue
Block a user