[Frontend] Consolidate entrypoint exception handler (#52261)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-08-14 09:27:24 +00:00
committed by GitHub
parent 103c419f04
commit b8165e5e58
31 changed files with 418 additions and 343 deletions
+1 -232
View File
@@ -8,12 +8,9 @@ import uuid
from argparse import Namespace
from collections.abc import Awaitable
from contextlib import asynccontextmanager
from http import HTTPStatus
import pydantic
import regex as re
from fastapi import FastAPI, HTTPException, Request
from fastapi.exceptions import RequestValidationError
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
from starlette.concurrency import iterate_in_threadpool
from starlette.datastructures import Headers, MutableHeaders
@@ -21,20 +18,8 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send
from vllm import envs
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.launcher import terminate_if_errored
from vllm.entrypoints.openai.engine.protocol import (
ErrorInfo,
ErrorResponse,
GenerationError,
)
from vllm.entrypoints.serve.utils.error_response import (
create_error_response,
sanitize_message,
)
from vllm.exceptions import VLLMError, VLLMValidationError
from vllm.logger import init_logger
from vllm.utils.gc_utils import freeze_gc_heap
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
logger = init_logger("vllm.entrypoints.openai.server_utils")
@@ -325,222 +310,6 @@ async def log_response(request: Request, call_next):
return response
async def vllm_error_handler(req: Request, exc: VLLMError):
"""Dispatch a vLLM-specific error to the appropriate handler."""
if isinstance(exc, (EngineGenerateError, EngineDeadError)):
return await engine_error_handler(req, exc)
elif isinstance(exc, GenerationError):
return await generation_error_handler(req, exc)
else:
return await exception_handler(req, exc)
async def engine_error_handler(
req: Request, exc: EngineDeadError | EngineGenerateError
):
"""
VLLM V1 AsyncLLM catches exceptions and returns
only two types: EngineGenerateError and EngineDeadError.
EngineGenerateError is raised by the per request generate()
method. This error could be request specific (and therefore
recoverable - e.g. if there is an error in input processing).
EngineDeadError is raised by the background output_handler
method. This error is global and therefore not recoverable.
We register these @app.exception_handlers to return nice
responses to the end user if they occur and shut down if needed.
See https://fastapi.tiangolo.com/tutorial/handling-errors/
for more details on how exception handlers work.
If an exception is encountered in a StreamingResponse
generator, the exception is not raised, since we already sent
a 200 status. Rather, we send an error message as the next chunk.
Since the exception is not raised, this means that the server
will not automatically shut down. Instead, we use the watchdog
background task for check for errored state.
"""
if req.app.state.args.log_error_stack:
logger.exception(
"Engine Exception caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
terminate_if_errored(
server=req.app.state.server,
engine=req.app.state.engine_client,
)
err = create_error_response(exc)
return JSONResponse(err.model_dump(), status_code=err.error.code)
async def generation_error_handler(req: Request, exc: GenerationError):
"""Handle GenerationError without logging stack traces.
GenerationError is a known, expected error (e.g. KV cache load failure)
that should be returned to the client as a 500 response without polluting
server logs with stack traces.
"""
err = create_error_response(exc)
return JSONResponse(err.model_dump(), status_code=err.error.code)
async def exception_handler(req: Request, exc: Exception):
if req.app.state.args.log_error_stack:
logger.error(
"Exception caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
err = create_error_response(exc)
return JSONResponse(err.model_dump(), status_code=err.error.code)
async def http_exception_handler(req: Request, exc: HTTPException):
if req.app.state.args.log_error_stack:
logger.exception(
"HTTPException caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
err = ErrorResponse(
error=ErrorInfo(
message=sanitize_message(exc.detail),
type=HTTPStatus(exc.status_code).phrase,
code=exc.status_code,
)
)
return JSONResponse(err.model_dump(), status_code=exc.status_code)
_BRACKETED_INTERNAL_RE = re.compile(r"[\[\]{}()]")
# NOTE: this list is pydantic-core's internal schema-kind vocabulary,
# not a stable public API -- it can grow when pydantic-core adds new
# wrapper/validator kinds. To refresh it after a pydantic upgrade:
# 1. Fuzz the validation-error-prone endpoints (e.g. /tokenize,
# /v1/completions, /v1/chat/completions) with deliberately
# malformed values for union-typed and wrapped fields (e.g. `stop`,
# `prompt`), and inspect the raw `loc` tuples in the response.
# 2. Any *unbracketed* segment that isn't a real field name or list
# index is a new internal marker -- add it here. Bracketed/
# parenthesized markers (e.g. "list[...]", "function-wrap[...]")
# are already caught structurally by _BRACKETED_INTERNAL_RE and
# don't need a list entry.
# 3. pydantic-core's source (the `error.rs`/schema-kind definitions
# in the pydantic-core Rust crate) is the canonical reference if
# you want to check before it shows up in a live fuzz run.
_INTERNAL_LOC_MARKERS = frozenset(
{
"function-wrap",
"function-after",
"function-before",
"function-plain",
"json-or-python",
"lax-or-strict",
"chain",
"default",
"nullable",
"tagged-union",
"union",
"call",
"arguments",
"is-instance",
"is-subclass",
"callable",
"str",
"int",
"float",
"bool",
"bytes",
"bytearray",
"list",
"tuple",
"dict",
"set",
"frozenset",
"complex",
"none",
"nonetype",
}
)
def _is_internal_loc_segment(segment: str) -> bool:
"""True if `segment` is a Pydantic-internal wrapper/union-branch
marker rather than a user-meaningful field name or list index."""
if _BRACKETED_INTERNAL_RE.search(segment):
return True
return segment.lower() in _INTERNAL_LOC_MARKERS
def clean_loc_for_param(loc: tuple) -> str:
"""Join a Pydantic error `loc` tuple into a clean dotted `param`
path, dropping internal wrapper/union-branch markers that don't
correspond to a real field name an API consumer would recognize.
E.g. ('body', 'function-wrap[__log_extra_fields__()]', 'prompt')
-> "body.prompt", not "body.function-wrap[__log_extra_fields__()].prompt".
"""
parts = [str(p) for p in loc if not _is_internal_loc_segment(str(p))]
if not parts:
return ".".join(str(p) for p in loc)
return ".".join(parts)
async def validation_exception_handler(req: Request, exc: RequestValidationError):
if req.app.state.args.log_error_stack:
logger.exception(
"RequestValidationError caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
param = None
errors = exc.errors()
for error in errors:
if "ctx" in error and "error" in error["ctx"]:
ctx_error = error["ctx"]["error"]
if isinstance(ctx_error, VLLMValidationError):
param = ctx_error.parameter
break
if param is None and errors:
first_error = errors[0]
loc = first_error.get("loc") if isinstance(first_error, dict) else None
if loc:
param = clean_loc_for_param(loc)
# Build the message from exc.errors() instead of str(exc) - str(exc)
# leaks the server's file path via FastAPI's endpoint context.
if errors:
count = len(errors)
label = "error" if count == 1 else "errors"
message = f"{count} validation {label}:\n"
message += "".join(f" {err}\n" for err in errors)
message = message.rstrip()
else:
message = "Validation error"
err = ErrorResponse(
error=ErrorInfo(
message=sanitize_message(message),
type=HTTPStatus.BAD_REQUEST.phrase,
code=HTTPStatus.BAD_REQUEST,
param=param,
)
)
return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
_running_tasks: set[asyncio.Task] = set()