From 323a29d8f2b2a50ed77a6173f120822376a8b14a Mon Sep 17 00:00:00 2001 From: lzhan011 <35493221+lzhan011@users.noreply.github.com> Date: Wed, 16 Sep 2026 00:53:36 -0500 Subject: [PATCH] [Bugfix] Unreadable `prompt_embeds` payload should be a 400, not a 500 (#55451) Signed-off-by: lzhan011 <35493221+lzhan011@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Cyrus Leung --- .../test_prompt_embeds_payload_validation.py | 163 ++++++++++++++++++ .../test_sparse_tensor_concurrent_race.py | 11 +- .../test_sparse_tensor_validation.py | 14 +- vllm/renderers/embed_utils.py | 59 ++++++- 4 files changed, 230 insertions(+), 17 deletions(-) create mode 100644 tests/renderers/test_prompt_embeds_payload_validation.py diff --git a/tests/renderers/test_prompt_embeds_payload_validation.py b/tests/renderers/test_prompt_embeds_payload_validation.py new file mode 100644 index 0000000000..1e425805e1 --- /dev/null +++ b/tests/renderers/test_prompt_embeds_payload_validation.py @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""`prompt_embeds` payloads that are not tensor files must be client errors. + +`prompt_embeds` is a base64 blob the caller writes, and every other way of +getting it wrong -- wrong rank, wrong hidden_size, wrong dtype, not a tensor at +all -- already answers 400. A payload that `torch.load` cannot read at all was +the exception: torch has no single error type for that (the zip reader raises +`RuntimeError`, the legacy pickle path raises an unpickling error, `KeyError` or +`IndexError`, an empty payload raises `EOFError`), none of them is a +`ValueError`, and the entrypoints' fallback therefore mapped them to 500. + +The assertions go through `create_error_response`, which is what actually turns +an exception into a status code, so they pin the HTTP answer rather than the +exception class alone. +""" + +import io +import zipfile +from unittest.mock import patch + +import pybase64 as base64 +import pytest +import torch + +from vllm.entrypoints.serve.exception_handling.error_response import ( + create_error_response, +) +from vllm.exceptions import VLLMValidationError +from vllm.renderers.embed_utils import ( + _MAX_EMBED_ERROR_REASON_CHARS, + safe_load_prompt_embeds, +) + + +@pytest.fixture +def model_config(): + from vllm.config import ModelConfig + + return ModelConfig( + model="facebook/opt-125m", + tokenizer="facebook/opt-125m", + tokenizer_mode="auto", + trust_remote_code=False, + dtype="float32", + seed=0, + enable_prompt_embeds=True, + ) + + +def _encode(raw: bytes) -> bytes: + return base64.b64encode(raw) + + +def _zip_without_a_tensor() -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("junk.txt", "hello") + return buffer.getvalue() + + +def _saved_tensor(tensor: torch.Tensor) -> bytes: + buffer = io.BytesIO() + torch.save(tensor, buffer) + return buffer.getvalue() + + +UNPARSABLE_PAYLOADS = [ + pytest.param(b"", id="empty"), + pytest.param(b"\x80", id="single-byte"), + pytest.param(b"\x00\x01\x02\x03not-a-tensor", id="random-bytes"), + pytest.param(b"hello world this is not a torch file", id="ascii-text"), + pytest.param(b"PK\x03\x04" + b"\x00" * 20, id="truncated-zip"), + pytest.param(_zip_without_a_tensor(), id="zip-without-a-tensor"), + # pickle.dumps(42): a valid pickle stream that is not a torch checkpoint. + # Spelled out rather than built, so this file does not import pickle. + pytest.param(b"\x80\x04K*.", id="legacy-pickle-of-an-int"), +] + + +@pytest.mark.parametrize("payload", UNPARSABLE_PAYLOADS) +def test_unreadable_prompt_embeds_payload_is_a_client_error(model_config, payload): + with pytest.raises(VLLMValidationError) as excinfo: + safe_load_prompt_embeds(model_config, _encode(payload)) + + assert excinfo.value.parameter == "prompt_embeds" + assert int(create_error_response(excinfo.value).error.code) == 400 + + +@pytest.mark.parametrize("payload", UNPARSABLE_PAYLOADS) +def test_the_error_body_stays_bounded(model_config, payload): + """torch's reason is built from the caller's bytes, so it must be truncated. + + An unpickler that reports the offending global, for instance, quotes a name + the caller chose; without a bound a large one would be reflected whole. + """ + padding = b"A" * 100_000 + with pytest.raises(VLLMValidationError) as excinfo: + safe_load_prompt_embeds(model_config, _encode(payload + padding)) + + assert len(str(excinfo.value)) < 500 + + +def test_a_long_reason_is_truncated_and_says_how_much_was_left_out(model_config): + """The cap is pinned against a reason of known length, not a real payload. + + Which of torch's messages happen to exceed the cap is torch's business and + changes between releases, so the reason is injected here: that keeps the + assertion about *this* code's truncation rule rather than about whichever + error text the installed torch produces. + """ + reason = "E" * (_MAX_EMBED_ERROR_REASON_CHARS + 37) + + with ( + patch( + "vllm.renderers.embed_utils.torch.load", side_effect=RuntimeError(reason) + ), + pytest.raises(VLLMValidationError) as excinfo, + ): + safe_load_prompt_embeds(model_config, _encode(b"ignored")) + + message = str(excinfo.value) + assert "E" * _MAX_EMBED_ERROR_REASON_CHARS in message + assert "E" * (_MAX_EMBED_ERROR_REASON_CHARS + 1) not in message + assert "... (37 more characters truncated)" in message + + +def test_a_short_reason_is_passed_through_whole(model_config): + """A reason that fits is not annotated -- the count only appears when it is + telling the reader something.""" + reason = "invalid magic number; corrupt file?" + assert len(reason) <= _MAX_EMBED_ERROR_REASON_CHARS + + with ( + patch( + "vllm.renderers.embed_utils.torch.load", side_effect=RuntimeError(reason) + ), + pytest.raises(VLLMValidationError) as excinfo, + ): + safe_load_prompt_embeds(model_config, _encode(b"ignored")) + + message = str(excinfo.value) + assert reason in message + assert "truncated" not in message + + +def test_a_well_formed_payload_still_loads(model_config): + """Positive control: the healthy path is untouched.""" + hidden_size = model_config.get_hidden_size() + tensor = torch.zeros(3, hidden_size, dtype=torch.float32) + + loaded = safe_load_prompt_embeds(model_config, _encode(_saved_tensor(tensor))) + + assert loaded.shape == (3, hidden_size) + assert loaded.dtype == model_config.dtype + + +def test_a_payload_that_is_not_base64_is_still_a_client_error(model_config): + """Positive control: the base64 stage already answered 400 and still does.""" + with pytest.raises(ValueError) as excinfo: + safe_load_prompt_embeds(model_config, b"!!!!not base64!!!!") + + assert int(create_error_response(excinfo.value).error.code) == 400 diff --git a/tests/renderers/test_sparse_tensor_concurrent_race.py b/tests/renderers/test_sparse_tensor_concurrent_race.py index 22925a9f19..a1683eee0b 100644 --- a/tests/renderers/test_sparse_tensor_concurrent_race.py +++ b/tests/renderers/test_sparse_tensor_concurrent_race.py @@ -11,6 +11,7 @@ import pytest import torch from vllm.config import ModelConfig +from vllm.exceptions import VLLMValidationError from vllm.multimodal.media import AudioEmbeddingMediaIO, ImageEmbeddingMediaIO from vllm.renderers.embed_utils import safe_load_prompt_embeds from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe @@ -54,7 +55,7 @@ class TestNegativeControlWithoutRace: def test_malicious_sparse_rejected_by_prompt_loader(self, model_config): encoded = _encode_tensor(_create_malicious_sparse_tensor()) - with pytest.raises((RuntimeError, ValueError)): + with pytest.raises((RuntimeError, ValueError, VLLMValidationError)): safe_load_prompt_embeds(model_config, encoded) def test_malicious_sparse_rejected_by_image_loader(self): @@ -147,7 +148,7 @@ class TestConcurrentRaceProtection: try: safe_load_prompt_embeds(model_config, malicious_encoded) bypassed.append(True) - except (RuntimeError, ValueError): + except (RuntimeError, ValueError, VLLMValidationError): rejected.append(True) with ThreadPoolExecutor(max_workers=num_workers) as pool: @@ -192,7 +193,7 @@ class TestGlobalFlagRestoration: initial = torch.sparse.check_sparse_tensor_invariants.is_enabled() malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor()) - with pytest.raises((RuntimeError, ValueError)): + with pytest.raises((RuntimeError, ValueError, VLLMValidationError)): safe_load_prompt_embeds(model_config, malicious_encoded) assert torch.sparse.check_sparse_tensor_invariants.is_enabled() == initial @@ -203,7 +204,7 @@ class TestGlobalFlagRestoration: malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor()) def attempt_load(_): - with contextlib.suppress(RuntimeError, ValueError): + with contextlib.suppress(RuntimeError, ValueError, VLLMValidationError): safe_load_prompt_embeds(model_config, malicious_encoded) with ThreadPoolExecutor(max_workers=4) as pool: @@ -244,7 +245,7 @@ class TestCrossLoaderLockSharing: # serialization — one finishes before the other starts), AND invalid # tensors are still rejected. malicious = _encode_tensor(_create_malicious_sparse_tensor()) - with pytest.raises((RuntimeError, ValueError)): + with pytest.raises((RuntimeError, ValueError, VLLMValidationError)): safe_load_prompt_embeds(model_config, malicious) with pytest.raises((RuntimeError, ValueError)): io_handler.load_base64("", malicious.decode("utf-8")) diff --git a/tests/renderers/test_sparse_tensor_validation.py b/tests/renderers/test_sparse_tensor_validation.py index c119b090c0..e49ac560dc 100644 --- a/tests/renderers/test_sparse_tensor_validation.py +++ b/tests/renderers/test_sparse_tensor_validation.py @@ -117,8 +117,10 @@ class TestPromptEmbedsValidation: malicious_tensor = _create_malicious_sparse_tensor() encoded = _encode_tensor(malicious_tensor) - # Should raise RuntimeError due to invalid sparse tensor - with pytest.raises((RuntimeError, ValueError)) as exc_info: + # The invariant check raises RuntimeError from inside torch.load; + # safe_load_prompt_embeds now reports that as a client error so the + # caller gets a 400 rather than the 500 a bare RuntimeError produced. + with pytest.raises(VLLMValidationError) as exc_info: safe_load_prompt_embeds(model_config, encoded) # Error should indicate sparse tensor validation failure @@ -137,7 +139,7 @@ class TestPromptEmbedsValidation: ) encoded = _encode_tensor(malicious_tensor) - with pytest.raises((RuntimeError, ValueError)): + with pytest.raises(VLLMValidationError): safe_load_prompt_embeds(model_config, encoded) def test_negative_indices_rejected(self, model_config): @@ -152,7 +154,7 @@ class TestPromptEmbedsValidation: ) encoded = _encode_tensor(malicious_tensor) - with pytest.raises((RuntimeError, ValueError)): + with pytest.raises(VLLMValidationError): safe_load_prompt_embeds(model_config, encoded) def test_hidden_size_mismatch_rejected(self, model_config): @@ -379,8 +381,8 @@ class TestSparseTensorValidationIntegration: # Step 1-2: Attacker creates malicious payload attack_payload = _encode_tensor(_create_malicious_sparse_tensor()) - # Step 3-4: Server processes and should reject - with pytest.raises((RuntimeError, ValueError)): + # Step 3-4: Server processes and should reject, as a client error + with pytest.raises(VLLMValidationError): safe_load_prompt_embeds(model_config, attack_payload) def test_attack_scenario_chat_api_image(self): diff --git a/vllm/renderers/embed_utils.py b/vllm/renderers/embed_utils.py index c4079005a5..981aeee3fb 100644 --- a/vllm/renderers/embed_utils.py +++ b/vllm/renderers/embed_utils.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from io import BytesIO -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Final import pybase64 import torch @@ -17,6 +17,27 @@ if TYPE_CHECKING: from vllm.config import ModelConfig +# How much of torch's own reason is repeated back to the caller. The reason is +# built from bytes the caller sent, so it is bounded rather than echoed whole. +_MAX_EMBED_ERROR_REASON_CHARS: Final = 200 + + +def _truncated_reason(exc: Exception) -> str: + """`exc`'s message, capped, saying how much was left out when it is capped. + + Without the count a capped reason reads like the whole one, so a caller + debugging a large payload cannot tell that torch said more than this. + """ + reason: Final = str(exc).strip() or type(exc).__name__ + omitted: Final = len(reason) - _MAX_EMBED_ERROR_REASON_CHARS + if omitted <= 0: + return reason + return ( + f"{reason[:_MAX_EMBED_ERROR_REASON_CHARS]}" + f"... ({omitted} more characters truncated)" + ) + + def safe_load_prompt_embeds( model_config: "ModelConfig", embed: bytes, @@ -27,12 +48,38 @@ def safe_load_prompt_embeds( parameter="prompt_embeds", ) + # Decoded outside the try below so that a body which is not base64 keeps + # raising binascii.Error, which is a ValueError and already answered 400. + payload: Final = pybase64.b64decode(embed, validate=True) + with check_sparse_tensor_invariants_threadsafe(): - tensor = torch.load( - BytesIO(pybase64.b64decode(embed, validate=True)), - weights_only=True, - map_location=torch.device("cpu"), - ) + try: + tensor = torch.load( + BytesIO(payload), + weights_only=True, + map_location=torch.device("cpu"), + ) + except (torch.OutOfMemoryError, MemoryError): + # Running out of memory is the server's condition, not the caller's. + raise + except Exception as exc: + # Everything else here is about the bytes the caller sent, and torch + # has no single error type for "this is not a tensor file": the zip + # reader raises `RuntimeError`, the legacy pickle path raises + # `UnpicklingError`, `KeyError` or `IndexError`, an empty payload + # raises `EOFError`. None of them is a `ValueError`, so the + # entrypoints' fallback mapped them all to 500. Catching by + # behaviour rather than by type also survives torch changing which + # error it raises for a given malformed payload. + # + # torch's reason is worth keeping -- for a malformed sparse tensor + # it names the offending index -- but it is built from the caller's + # own bytes, so it is truncated rather than echoed whole. + raise VLLMValidationError( + "`prompt_embeds` could not be deserialized as a torch tensor: " + f"{_truncated_reason(exc)}", + parameter="prompt_embeds", + ) from exc tensor = safe_to_dense(tensor, parameter="prompt_embeds") if tensor.dim() > 2: