[Bugfix] Unreadable prompt_embeds payload should be a 400, not a 500 (#55451)

Signed-off-by: lzhan011 <35493221+lzhan011@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
lzhan011
2026-09-16 05:53:36 +00:00
committed by GitHub
co-authored by Claude Opus 5 Cyrus Leung
parent 4fe9e6f6e5
commit 323a29d8f2
4 changed files with 230 additions and 17 deletions
+53 -6
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from io import BytesIO
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Final
import pybase64
import torch
@@ -17,6 +17,27 @@ if TYPE_CHECKING:
from vllm.config import ModelConfig
# How much of torch's own reason is repeated back to the caller. The reason is
# built from bytes the caller sent, so it is bounded rather than echoed whole.
_MAX_EMBED_ERROR_REASON_CHARS: Final = 200
def _truncated_reason(exc: Exception) -> str:
"""`exc`'s message, capped, saying how much was left out when it is capped.
Without the count a capped reason reads like the whole one, so a caller
debugging a large payload cannot tell that torch said more than this.
"""
reason: Final = str(exc).strip() or type(exc).__name__
omitted: Final = len(reason) - _MAX_EMBED_ERROR_REASON_CHARS
if omitted <= 0:
return reason
return (
f"{reason[:_MAX_EMBED_ERROR_REASON_CHARS]}"
f"... ({omitted} more characters truncated)"
)
def safe_load_prompt_embeds(
model_config: "ModelConfig",
embed: bytes,
@@ -27,12 +48,38 @@ def safe_load_prompt_embeds(
parameter="prompt_embeds",
)
# Decoded outside the try below so that a body which is not base64 keeps
# raising binascii.Error, which is a ValueError and already answered 400.
payload: Final = pybase64.b64decode(embed, validate=True)
with check_sparse_tensor_invariants_threadsafe():
tensor = torch.load(
BytesIO(pybase64.b64decode(embed, validate=True)),
weights_only=True,
map_location=torch.device("cpu"),
)
try:
tensor = torch.load(
BytesIO(payload),
weights_only=True,
map_location=torch.device("cpu"),
)
except (torch.OutOfMemoryError, MemoryError):
# Running out of memory is the server's condition, not the caller's.
raise
except Exception as exc:
# Everything else here is about the bytes the caller sent, and torch
# has no single error type for "this is not a tensor file": the zip
# reader raises `RuntimeError`, the legacy pickle path raises
# `UnpicklingError`, `KeyError` or `IndexError`, an empty payload
# raises `EOFError`. None of them is a `ValueError`, so the
# entrypoints' fallback mapped them all to 500. Catching by
# behaviour rather than by type also survives torch changing which
# error it raises for a given malformed payload.
#
# torch's reason is worth keeping -- for a malformed sparse tensor
# it names the offending index -- but it is built from the caller's
# own bytes, so it is truncated rather than echoed whole.
raise VLLMValidationError(
"`prompt_embeds` could not be deserialized as a torch tensor: "
f"{_truncated_reason(exc)}",
parameter="prompt_embeds",
) from exc
tensor = safe_to_dense(tensor, parameter="prompt_embeds")
if tensor.dim() > 2: