[Bugfix] Unreadable prompt_embeds payload should be a 400, not a 500 (#55451)
Signed-off-by: lzhan011 <35493221+lzhan011@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Claude Opus 5
Cyrus Leung
parent
4fe9e6f6e5
commit
323a29d8f2
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from io import BytesIO
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import pybase64
|
||||
import torch
|
||||
@@ -17,6 +17,27 @@ if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
|
||||
# How much of torch's own reason is repeated back to the caller. The reason is
|
||||
# built from bytes the caller sent, so it is bounded rather than echoed whole.
|
||||
_MAX_EMBED_ERROR_REASON_CHARS: Final = 200
|
||||
|
||||
|
||||
def _truncated_reason(exc: Exception) -> str:
|
||||
"""`exc`'s message, capped, saying how much was left out when it is capped.
|
||||
|
||||
Without the count a capped reason reads like the whole one, so a caller
|
||||
debugging a large payload cannot tell that torch said more than this.
|
||||
"""
|
||||
reason: Final = str(exc).strip() or type(exc).__name__
|
||||
omitted: Final = len(reason) - _MAX_EMBED_ERROR_REASON_CHARS
|
||||
if omitted <= 0:
|
||||
return reason
|
||||
return (
|
||||
f"{reason[:_MAX_EMBED_ERROR_REASON_CHARS]}"
|
||||
f"... ({omitted} more characters truncated)"
|
||||
)
|
||||
|
||||
|
||||
def safe_load_prompt_embeds(
|
||||
model_config: "ModelConfig",
|
||||
embed: bytes,
|
||||
@@ -27,12 +48,38 @@ def safe_load_prompt_embeds(
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
|
||||
# Decoded outside the try below so that a body which is not base64 keeps
|
||||
# raising binascii.Error, which is a ValueError and already answered 400.
|
||||
payload: Final = pybase64.b64decode(embed, validate=True)
|
||||
|
||||
with check_sparse_tensor_invariants_threadsafe():
|
||||
tensor = torch.load(
|
||||
BytesIO(pybase64.b64decode(embed, validate=True)),
|
||||
weights_only=True,
|
||||
map_location=torch.device("cpu"),
|
||||
)
|
||||
try:
|
||||
tensor = torch.load(
|
||||
BytesIO(payload),
|
||||
weights_only=True,
|
||||
map_location=torch.device("cpu"),
|
||||
)
|
||||
except (torch.OutOfMemoryError, MemoryError):
|
||||
# Running out of memory is the server's condition, not the caller's.
|
||||
raise
|
||||
except Exception as exc:
|
||||
# Everything else here is about the bytes the caller sent, and torch
|
||||
# has no single error type for "this is not a tensor file": the zip
|
||||
# reader raises `RuntimeError`, the legacy pickle path raises
|
||||
# `UnpicklingError`, `KeyError` or `IndexError`, an empty payload
|
||||
# raises `EOFError`. None of them is a `ValueError`, so the
|
||||
# entrypoints' fallback mapped them all to 500. Catching by
|
||||
# behaviour rather than by type also survives torch changing which
|
||||
# error it raises for a given malformed payload.
|
||||
#
|
||||
# torch's reason is worth keeping -- for a malformed sparse tensor
|
||||
# it names the offending index -- but it is built from the caller's
|
||||
# own bytes, so it is truncated rather than echoed whole.
|
||||
raise VLLMValidationError(
|
||||
"`prompt_embeds` could not be deserialized as a torch tensor: "
|
||||
f"{_truncated_reason(exc)}",
|
||||
parameter="prompt_embeds",
|
||||
) from exc
|
||||
tensor = safe_to_dense(tensor, parameter="prompt_embeds")
|
||||
|
||||
if tensor.dim() > 2:
|
||||
|
||||
Reference in New Issue
Block a user