[Frontend] Consolidate entrypoint exception handler (#52261)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -8,12 +8,9 @@ import uuid
|
||||
from argparse import Namespace
|
||||
from collections.abc import Awaitable
|
||||
from contextlib import asynccontextmanager
|
||||
from http import HTTPStatus
|
||||
|
||||
import pydantic
|
||||
import regex as re
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.concurrency import iterate_in_threadpool
|
||||
from starlette.datastructures import Headers, MutableHeaders
|
||||
@@ -21,20 +18,8 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from vllm import envs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.launcher import terminate_if_errored
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorInfo,
|
||||
ErrorResponse,
|
||||
GenerationError,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.error_response import (
|
||||
create_error_response,
|
||||
sanitize_message,
|
||||
)
|
||||
from vllm.exceptions import VLLMError, VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gc_utils import freeze_gc_heap
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
|
||||
logger = init_logger("vllm.entrypoints.openai.server_utils")
|
||||
|
||||
@@ -325,222 +310,6 @@ async def log_response(request: Request, call_next):
|
||||
return response
|
||||
|
||||
|
||||
async def vllm_error_handler(req: Request, exc: VLLMError):
|
||||
"""Dispatch a vLLM-specific error to the appropriate handler."""
|
||||
if isinstance(exc, (EngineGenerateError, EngineDeadError)):
|
||||
return await engine_error_handler(req, exc)
|
||||
elif isinstance(exc, GenerationError):
|
||||
return await generation_error_handler(req, exc)
|
||||
else:
|
||||
return await exception_handler(req, exc)
|
||||
|
||||
|
||||
async def engine_error_handler(
|
||||
req: Request, exc: EngineDeadError | EngineGenerateError
|
||||
):
|
||||
"""
|
||||
VLLM V1 AsyncLLM catches exceptions and returns
|
||||
only two types: EngineGenerateError and EngineDeadError.
|
||||
|
||||
EngineGenerateError is raised by the per request generate()
|
||||
method. This error could be request specific (and therefore
|
||||
recoverable - e.g. if there is an error in input processing).
|
||||
|
||||
EngineDeadError is raised by the background output_handler
|
||||
method. This error is global and therefore not recoverable.
|
||||
|
||||
We register these @app.exception_handlers to return nice
|
||||
responses to the end user if they occur and shut down if needed.
|
||||
See https://fastapi.tiangolo.com/tutorial/handling-errors/
|
||||
for more details on how exception handlers work.
|
||||
|
||||
If an exception is encountered in a StreamingResponse
|
||||
generator, the exception is not raised, since we already sent
|
||||
a 200 status. Rather, we send an error message as the next chunk.
|
||||
Since the exception is not raised, this means that the server
|
||||
will not automatically shut down. Instead, we use the watchdog
|
||||
background task for check for errored state.
|
||||
"""
|
||||
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"Engine Exception caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
terminate_if_errored(
|
||||
server=req.app.state.server,
|
||||
engine=req.app.state.engine_client,
|
||||
)
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
|
||||
async def generation_error_handler(req: Request, exc: GenerationError):
|
||||
"""Handle GenerationError without logging stack traces.
|
||||
|
||||
GenerationError is a known, expected error (e.g. KV cache load failure)
|
||||
that should be returned to the client as a 500 response without polluting
|
||||
server logs with stack traces.
|
||||
"""
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
|
||||
async def exception_handler(req: Request, exc: Exception):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.error(
|
||||
"Exception caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
|
||||
async def http_exception_handler(req: Request, exc: HTTPException):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"HTTPException caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(
|
||||
message=sanitize_message(exc.detail),
|
||||
type=HTTPStatus(exc.status_code).phrase,
|
||||
code=exc.status_code,
|
||||
)
|
||||
)
|
||||
return JSONResponse(err.model_dump(), status_code=exc.status_code)
|
||||
|
||||
|
||||
_BRACKETED_INTERNAL_RE = re.compile(r"[\[\]{}()]")
|
||||
|
||||
# NOTE: this list is pydantic-core's internal schema-kind vocabulary,
|
||||
# not a stable public API -- it can grow when pydantic-core adds new
|
||||
# wrapper/validator kinds. To refresh it after a pydantic upgrade:
|
||||
# 1. Fuzz the validation-error-prone endpoints (e.g. /tokenize,
|
||||
# /v1/completions, /v1/chat/completions) with deliberately
|
||||
# malformed values for union-typed and wrapped fields (e.g. `stop`,
|
||||
# `prompt`), and inspect the raw `loc` tuples in the response.
|
||||
# 2. Any *unbracketed* segment that isn't a real field name or list
|
||||
# index is a new internal marker -- add it here. Bracketed/
|
||||
# parenthesized markers (e.g. "list[...]", "function-wrap[...]")
|
||||
# are already caught structurally by _BRACKETED_INTERNAL_RE and
|
||||
# don't need a list entry.
|
||||
# 3. pydantic-core's source (the `error.rs`/schema-kind definitions
|
||||
# in the pydantic-core Rust crate) is the canonical reference if
|
||||
# you want to check before it shows up in a live fuzz run.
|
||||
_INTERNAL_LOC_MARKERS = frozenset(
|
||||
{
|
||||
"function-wrap",
|
||||
"function-after",
|
||||
"function-before",
|
||||
"function-plain",
|
||||
"json-or-python",
|
||||
"lax-or-strict",
|
||||
"chain",
|
||||
"default",
|
||||
"nullable",
|
||||
"tagged-union",
|
||||
"union",
|
||||
"call",
|
||||
"arguments",
|
||||
"is-instance",
|
||||
"is-subclass",
|
||||
"callable",
|
||||
"str",
|
||||
"int",
|
||||
"float",
|
||||
"bool",
|
||||
"bytes",
|
||||
"bytearray",
|
||||
"list",
|
||||
"tuple",
|
||||
"dict",
|
||||
"set",
|
||||
"frozenset",
|
||||
"complex",
|
||||
"none",
|
||||
"nonetype",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _is_internal_loc_segment(segment: str) -> bool:
|
||||
"""True if `segment` is a Pydantic-internal wrapper/union-branch
|
||||
marker rather than a user-meaningful field name or list index."""
|
||||
if _BRACKETED_INTERNAL_RE.search(segment):
|
||||
return True
|
||||
return segment.lower() in _INTERNAL_LOC_MARKERS
|
||||
|
||||
|
||||
def clean_loc_for_param(loc: tuple) -> str:
|
||||
"""Join a Pydantic error `loc` tuple into a clean dotted `param`
|
||||
path, dropping internal wrapper/union-branch markers that don't
|
||||
correspond to a real field name an API consumer would recognize.
|
||||
|
||||
E.g. ('body', 'function-wrap[__log_extra_fields__()]', 'prompt')
|
||||
-> "body.prompt", not "body.function-wrap[__log_extra_fields__()].prompt".
|
||||
"""
|
||||
parts = [str(p) for p in loc if not _is_internal_loc_segment(str(p))]
|
||||
if not parts:
|
||||
return ".".join(str(p) for p in loc)
|
||||
return ".".join(parts)
|
||||
|
||||
|
||||
async def validation_exception_handler(req: Request, exc: RequestValidationError):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"RequestValidationError caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
param = None
|
||||
errors = exc.errors()
|
||||
for error in errors:
|
||||
if "ctx" in error and "error" in error["ctx"]:
|
||||
ctx_error = error["ctx"]["error"]
|
||||
if isinstance(ctx_error, VLLMValidationError):
|
||||
param = ctx_error.parameter
|
||||
break
|
||||
|
||||
if param is None and errors:
|
||||
first_error = errors[0]
|
||||
loc = first_error.get("loc") if isinstance(first_error, dict) else None
|
||||
if loc:
|
||||
param = clean_loc_for_param(loc)
|
||||
|
||||
# Build the message from exc.errors() instead of str(exc) - str(exc)
|
||||
# leaks the server's file path via FastAPI's endpoint context.
|
||||
if errors:
|
||||
count = len(errors)
|
||||
label = "error" if count == 1 else "errors"
|
||||
message = f"{count} validation {label}:\n"
|
||||
message += "".join(f" {err}\n" for err in errors)
|
||||
message = message.rstrip()
|
||||
else:
|
||||
message = "Validation error"
|
||||
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(
|
||||
message=sanitize_message(message),
|
||||
type=HTTPStatus.BAD_REQUEST.phrase,
|
||||
code=HTTPStatus.BAD_REQUEST,
|
||||
param=param,
|
||||
)
|
||||
)
|
||||
return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
|
||||
|
||||
|
||||
_running_tasks: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user