[Frontend] Consolidate entrypoint exception handler (#52261)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -44,7 +44,9 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
PromptTokenUsageInfo,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.server_utils import validation_exception_handler
|
||||
from vllm.entrypoints.serve.exception_handling.handlers.validation import (
|
||||
validation_exception_handler,
|
||||
)
|
||||
|
||||
_convert = AnthropicServingMessages._convert_anthropic_to_openai_request
|
||||
_img_url = AnthropicServingMessages._convert_image_source_to_url
|
||||
|
||||
@@ -30,8 +30,10 @@ from vllm.entrypoints.cohere.protocol import (
|
||||
CohereChatV2Response,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.server_utils import (
|
||||
from vllm.entrypoints.serve.exception_handling.handlers.http import (
|
||||
http_exception_handler,
|
||||
)
|
||||
from vllm.entrypoints.serve.exception_handling.handlers.validation import (
|
||||
validation_exception_handler,
|
||||
)
|
||||
|
||||
|
||||
+54
-1
@@ -10,7 +10,60 @@ speech-to-text WebSocket paths.
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
|
||||
|
||||
def test_sanitize_message():
|
||||
assert (
|
||||
sanitize_message("<_io.BytesIO object at 0x7a95e299e750>")
|
||||
== "<_io.BytesIO object>"
|
||||
)
|
||||
|
||||
|
||||
class TestSanitizeMessageFilePaths:
|
||||
"""sanitize_message should also strip file paths and traceback
|
||||
frames, not just memory addresses - see #31683."""
|
||||
|
||||
def test_strips_traceback_style_frame(self):
|
||||
msg = (
|
||||
"1 validation error:\n"
|
||||
" {'type': 'list_type', 'loc': ('body', 'messages')}\n"
|
||||
'\n File "/usr/local/lib/python3.12/dist-packages/vllm/'
|
||||
'entrypoints/serve/utils/api_utils.py", line 40, '
|
||||
"in create_chat_completion\n"
|
||||
" POST /v1/chat/completions"
|
||||
)
|
||||
result = sanitize_message(msg)
|
||||
assert "/usr/local/" not in result
|
||||
assert "api_utils.py" not in result
|
||||
assert "list_type" in result
|
||||
|
||||
def test_strips_arbitrary_absolute_path(self):
|
||||
result = sanitize_message("Error in /home/user/project/vllm/server.py")
|
||||
assert "/home/user" not in result
|
||||
|
||||
def test_strips_single_parent_container_path(self):
|
||||
"""Regression: /app/server.py and /workspace/server.py (common in
|
||||
container deployments) were missed by the original {2,} quantifier."""
|
||||
assert "/app/" not in sanitize_message("Error in /app/server.py")
|
||||
assert "/workspace/" not in sanitize_message("Error in /workspace/server.py")
|
||||
|
||||
def test_preserves_api_endpoint_paths(self):
|
||||
msg = "POST /v1/chat/completions failed"
|
||||
assert "/v1/chat/completions" in sanitize_message(msg)
|
||||
|
||||
def test_preserves_short_field_references(self):
|
||||
msg = "Invalid value for field 'body.messages'"
|
||||
assert sanitize_message(msg) == msg
|
||||
|
||||
def test_strips_both_address_and_path(self):
|
||||
msg = (
|
||||
"<Request at 0x7f123> failed at "
|
||||
"/usr/local/lib/python3.12/dist-packages/vllm/server.py"
|
||||
)
|
||||
result = sanitize_message(msg)
|
||||
assert "0x" not in result
|
||||
assert "/usr/local/" not in result
|
||||
|
||||
|
||||
class TestSanitizeMessageCoversLeakPatterns:
|
||||
+1
-1
@@ -16,7 +16,7 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
|
||||
from vllm.entrypoints.serve.utils.server_utils import (
|
||||
from vllm.entrypoints.serve.exception_handling.handlers.validation import (
|
||||
clean_loc_for_param,
|
||||
validation_exception_handler,
|
||||
)
|
||||
@@ -6,18 +6,10 @@ import pytest
|
||||
from vllm.entrypoints.openai.engine.protocol import StreamOptions
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
get_max_tokens,
|
||||
sanitize_message,
|
||||
should_include_usage,
|
||||
)
|
||||
|
||||
|
||||
def test_sanitize_message():
|
||||
assert (
|
||||
sanitize_message("<_io.BytesIO object at 0x7a95e299e750>")
|
||||
== "<_io.BytesIO object>"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("stream_options", "expected"),
|
||||
[
|
||||
@@ -118,49 +110,3 @@ class TestGetMaxTokens:
|
||||
input_length=150,
|
||||
default_sampling_params={"max_tokens": 2048},
|
||||
)
|
||||
|
||||
|
||||
class TestSanitizeMessageFilePaths:
|
||||
"""sanitize_message should also strip file paths and traceback
|
||||
frames, not just memory addresses - see #31683."""
|
||||
|
||||
def test_strips_traceback_style_frame(self):
|
||||
msg = (
|
||||
"1 validation error:\n"
|
||||
" {'type': 'list_type', 'loc': ('body', 'messages')}\n"
|
||||
'\n File "/usr/local/lib/python3.12/dist-packages/vllm/'
|
||||
'entrypoints/serve/utils/api_utils.py", line 40, '
|
||||
"in create_chat_completion\n"
|
||||
" POST /v1/chat/completions"
|
||||
)
|
||||
result = sanitize_message(msg)
|
||||
assert "/usr/local/" not in result
|
||||
assert "api_utils.py" not in result
|
||||
assert "list_type" in result
|
||||
|
||||
def test_strips_arbitrary_absolute_path(self):
|
||||
result = sanitize_message("Error in /home/user/project/vllm/server.py")
|
||||
assert "/home/user" not in result
|
||||
|
||||
def test_strips_single_parent_container_path(self):
|
||||
"""Regression: /app/server.py and /workspace/server.py (common in
|
||||
container deployments) were missed by the original {2,} quantifier."""
|
||||
assert "/app/" not in sanitize_message("Error in /app/server.py")
|
||||
assert "/workspace/" not in sanitize_message("Error in /workspace/server.py")
|
||||
|
||||
def test_preserves_api_endpoint_paths(self):
|
||||
msg = "POST /v1/chat/completions failed"
|
||||
assert "/v1/chat/completions" in sanitize_message(msg)
|
||||
|
||||
def test_preserves_short_field_references(self):
|
||||
msg = "Invalid value for field 'body.messages'"
|
||||
assert sanitize_message(msg) == msg
|
||||
|
||||
def test_strips_both_address_and_path(self):
|
||||
msg = (
|
||||
"<Request at 0x7f123> failed at "
|
||||
"/usr/local/lib/python3.12/dist-packages/vllm/server.py"
|
||||
)
|
||||
result = sanitize_message(msg)
|
||||
assert "0x" not in result
|
||||
assert "/usr/local/" not in result
|
||||
|
||||
@@ -13,7 +13,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import aiohttp
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.serve.utils.error_response import create_error_response
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.exceptions import VLLMClientError, VLLMUnprocessableEntityError
|
||||
from vllm.multimodal.media import MediaConnector
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import pytest
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.entrypoints.serve.utils.error_response import create_error_response
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.multimodal.parse import parse_mm_uuids
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
|
||||
@@ -17,9 +17,9 @@ from vllm.entrypoints.anthropic.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
sanitize_message,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
|
||||
@@ -46,7 +46,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
|
||||
@@ -33,9 +33,9 @@ from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
sanitize_message,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
|
||||
@@ -86,7 +86,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
StreamOptions,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
from vllm.renderers.cohere import MESSAGES_CITATIONS_KEY, POSITION_TO_SOURCE_KEY
|
||||
|
||||
@@ -16,8 +16,7 @@ from contextlib import asynccontextmanager
|
||||
from typing import Any, cast
|
||||
|
||||
import uvloop
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from starlette.datastructures import State
|
||||
|
||||
@@ -31,6 +30,7 @@ from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_se
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
@@ -41,15 +41,10 @@ from vllm.entrypoints.serve.utils.api_utils import (
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.entrypoints.serve.utils.server_utils import (
|
||||
exception_handler,
|
||||
get_uvicorn_log_config,
|
||||
http_exception_handler,
|
||||
lifespan,
|
||||
log_response,
|
||||
validation_exception_handler,
|
||||
vllm_error_handler,
|
||||
)
|
||||
from vllm.exceptions import VLLMError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
@@ -275,6 +270,8 @@ def build_app(
|
||||
# `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).
|
||||
_attach_endpoint_plugins(app, supported_tasks)
|
||||
|
||||
init_exception_handler(app)
|
||||
|
||||
app.root_path = args.root_path
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
@@ -284,28 +281,6 @@ def build_app(
|
||||
allow_headers=args.allowed_headers,
|
||||
)
|
||||
|
||||
# Exception handlers are registered in four layers:
|
||||
# 1. framework errors raised by FastAPI/Starlette
|
||||
# 2. vLLM-specific errors dispatched via a single ``VLLMError`` handler
|
||||
# 3. fallback handlers for raw exceptions not yet migrated to ``VLLMError``
|
||||
# 4. the raw ``Exception`` handler as a safety net
|
||||
# Registering specific exception types (rather than only ``Exception``)
|
||||
# ensures they are handled by ``ExceptionMiddleware`` (inside the Prometheus
|
||||
# middleware) rather than ``ServerErrorMiddleware`` (outside it), so their
|
||||
# status codes are recorded correctly.
|
||||
app.exception_handler(HTTPException)(http_exception_handler)
|
||||
app.exception_handler(RequestValidationError)(validation_exception_handler)
|
||||
|
||||
app.exception_handler(VLLMError)(vllm_error_handler)
|
||||
|
||||
# TODO(zqzten): remove these fallback handlers after migration to VLLMError
|
||||
app.exception_handler(ValueError)(exception_handler)
|
||||
app.exception_handler(TypeError)(exception_handler)
|
||||
app.exception_handler(OverflowError)(exception_handler)
|
||||
app.exception_handler(NotImplementedError)(exception_handler)
|
||||
|
||||
app.exception_handler(Exception)(exception_handler)
|
||||
|
||||
# Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
|
||||
if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]:
|
||||
from vllm.entrypoints.serve.utils.server_utils import AuthenticationMiddleware
|
||||
|
||||
@@ -14,11 +14,11 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
ModelPermission,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.lora.protocol import (
|
||||
LoadLoRAAdapterRequest,
|
||||
UnloadLoRAAdapterRequest,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.error_response import create_error_response
|
||||
from vllm.exceptions import LoRAAdapterNotFoundError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
|
||||
@@ -52,7 +52,7 @@ from vllm.entrypoints.pooling.scoring.protocol import (
|
||||
ScoreRequest,
|
||||
ScoreResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.error_response import create_error_response
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.speech_to_text.transcription.protocol import (
|
||||
TranscriptionRequest,
|
||||
TranscriptionResponse,
|
||||
|
||||
@@ -5,8 +5,16 @@ from fastapi import FastAPI
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .exception_handling.error_response import create_error_response
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"create_error_response",
|
||||
"register_vllm_serve_api_routers",
|
||||
"register_vllm_dev_api_routers",
|
||||
]
|
||||
|
||||
|
||||
def register_vllm_serve_api_routers(app: FastAPI):
|
||||
from .instrumentator import register_instrumentator_api_routers
|
||||
|
||||
@@ -12,8 +12,8 @@ from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.pooling.typing import AnyPoolingRequest
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
from vllm.entrypoints.serve.utils.error_response import create_error_response
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import VLLMNotFoundError
|
||||
from vllm.inputs import EngineInput
|
||||
|
||||
+2
-1
@@ -7,9 +7,10 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
GenerationError,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .utils import sanitize_message
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from fastapi import Request
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..error_response import create_error_response
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
async def exception_handler(req: Request, exc: Exception):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.error(
|
||||
"Exception caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
@@ -0,0 +1,31 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..utils import sanitize_message
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
async def http_exception_handler(req: Request, exc: HTTPException):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"HTTPException caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(
|
||||
message=sanitize_message(exc.detail),
|
||||
type=HTTPStatus(exc.status_code).phrase,
|
||||
code=exc.status_code,
|
||||
)
|
||||
)
|
||||
return JSONResponse(err.model_dump(), status_code=exc.status_code)
|
||||
@@ -0,0 +1,138 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from http import HTTPStatus
|
||||
|
||||
import regex as re
|
||||
from fastapi import Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..utils import sanitize_message
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
_BRACKETED_INTERNAL_RE = re.compile(r"[\[\]{}()]")
|
||||
|
||||
# NOTE: this list is pydantic-core's internal schema-kind vocabulary,
|
||||
# not a stable public API -- it can grow when pydantic-core adds new
|
||||
# wrapper/validator kinds. To refresh it after a pydantic upgrade:
|
||||
# 1. Fuzz the validation-error-prone endpoints (e.g. /tokenize,
|
||||
# /v1/completions, /v1/chat/completions) with deliberately
|
||||
# malformed values for union-typed and wrapped fields (e.g. `stop`,
|
||||
# `prompt`), and inspect the raw `loc` tuples in the response.
|
||||
# 2. Any *unbracketed* segment that isn't a real field name or list
|
||||
# index is a new internal marker -- add it here. Bracketed/
|
||||
# parenthesized markers (e.g. "list[...]", "function-wrap[...]")
|
||||
# are already caught structurally by _BRACKETED_INTERNAL_RE and
|
||||
# don't need a list entry.
|
||||
# 3. pydantic-core's source (the `error.rs`/schema-kind definitions
|
||||
# in the pydantic-core Rust crate) is the canonical reference if
|
||||
# you want to check before it shows up in a live fuzz run.
|
||||
_INTERNAL_LOC_MARKERS = frozenset(
|
||||
{
|
||||
"function-wrap",
|
||||
"function-after",
|
||||
"function-before",
|
||||
"function-plain",
|
||||
"json-or-python",
|
||||
"lax-or-strict",
|
||||
"chain",
|
||||
"default",
|
||||
"nullable",
|
||||
"tagged-union",
|
||||
"union",
|
||||
"call",
|
||||
"arguments",
|
||||
"is-instance",
|
||||
"is-subclass",
|
||||
"callable",
|
||||
"str",
|
||||
"int",
|
||||
"float",
|
||||
"bool",
|
||||
"bytes",
|
||||
"bytearray",
|
||||
"list",
|
||||
"tuple",
|
||||
"dict",
|
||||
"set",
|
||||
"frozenset",
|
||||
"complex",
|
||||
"none",
|
||||
"nonetype",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _is_internal_loc_segment(segment: str) -> bool:
|
||||
"""True if `segment` is a Pydantic-internal wrapper/union-branch
|
||||
marker rather than a user-meaningful field name or list index."""
|
||||
if _BRACKETED_INTERNAL_RE.search(segment):
|
||||
return True
|
||||
return segment.lower() in _INTERNAL_LOC_MARKERS
|
||||
|
||||
|
||||
def clean_loc_for_param(loc: tuple) -> str:
|
||||
"""Join a Pydantic error `loc` tuple into a clean dotted `param`
|
||||
path, dropping internal wrapper/union-branch markers that don't
|
||||
correspond to a real field name an API consumer would recognize.
|
||||
|
||||
E.g. ('body', 'function-wrap[__log_extra_fields__()]', 'prompt')
|
||||
-> "body.prompt", not "body.function-wrap[__log_extra_fields__()].prompt".
|
||||
"""
|
||||
parts = [str(p) for p in loc if not _is_internal_loc_segment(str(p))]
|
||||
if not parts:
|
||||
return ".".join(str(p) for p in loc)
|
||||
return ".".join(parts)
|
||||
|
||||
|
||||
async def validation_exception_handler(req: Request, exc: RequestValidationError):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"RequestValidationError caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
param = None
|
||||
errors = exc.errors()
|
||||
for error in errors:
|
||||
if "ctx" in error and "error" in error["ctx"]:
|
||||
ctx_error = error["ctx"]["error"]
|
||||
if isinstance(ctx_error, VLLMValidationError):
|
||||
param = ctx_error.parameter
|
||||
break
|
||||
|
||||
if param is None and errors:
|
||||
first_error = errors[0]
|
||||
loc = first_error.get("loc") if isinstance(first_error, dict) else None
|
||||
if loc:
|
||||
param = clean_loc_for_param(loc)
|
||||
|
||||
# Build the message from exc.errors() instead of str(exc) - str(exc)
|
||||
# leaks the server's file path via FastAPI's endpoint context.
|
||||
if errors:
|
||||
count = len(errors)
|
||||
label = "error" if count == 1 else "errors"
|
||||
message = f"{count} validation {label}:\n"
|
||||
message += "".join(f" {err}\n" for err in errors)
|
||||
message = message.rstrip()
|
||||
else:
|
||||
message = "Validation error"
|
||||
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(
|
||||
message=sanitize_message(message),
|
||||
type=HTTPStatus.BAD_REQUEST.phrase,
|
||||
code=HTTPStatus.BAD_REQUEST,
|
||||
param=param,
|
||||
)
|
||||
)
|
||||
return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
|
||||
@@ -0,0 +1,80 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from fastapi import Request
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.launcher import terminate_if_errored
|
||||
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
||||
from vllm.exceptions import VLLMError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
|
||||
from ..error_response import create_error_response
|
||||
from .exception import exception_handler
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
async def vllm_error_handler(req: Request, exc: VLLMError):
|
||||
"""Dispatch a vLLM-specific error to the appropriate handler."""
|
||||
if isinstance(exc, (EngineGenerateError, EngineDeadError)):
|
||||
return await engine_error_handler(req, exc)
|
||||
elif isinstance(exc, GenerationError):
|
||||
return await generation_error_handler(req, exc)
|
||||
else:
|
||||
return await exception_handler(req, exc)
|
||||
|
||||
|
||||
async def engine_error_handler(
|
||||
req: Request, exc: EngineDeadError | EngineGenerateError
|
||||
):
|
||||
"""
|
||||
VLLM V1 AsyncLLM catches exceptions and returns
|
||||
only two types: EngineGenerateError and EngineDeadError.
|
||||
|
||||
EngineGenerateError is raised by the per request generate()
|
||||
method. This error could be request specific (and therefore
|
||||
recoverable - e.g. if there is an error in input processing).
|
||||
|
||||
EngineDeadError is raised by the background output_handler
|
||||
method. This error is global and therefore not recoverable.
|
||||
|
||||
We register these @app.exception_handlers to return nice
|
||||
responses to the end user if they occur and shut down if needed.
|
||||
See https://fastapi.tiangolo.com/tutorial/handling-errors/
|
||||
for more details on how exception handlers work.
|
||||
|
||||
If an exception is encountered in a StreamingResponse
|
||||
generator, the exception is not raised, since we already sent
|
||||
a 200 status. Rather, we send an error message as the next chunk.
|
||||
Since the exception is not raised, this means that the server
|
||||
will not automatically shut down. Instead, we use the watchdog
|
||||
background task for check for errored state.
|
||||
"""
|
||||
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"Engine Exception caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
terminate_if_errored(
|
||||
server=req.app.state.server,
|
||||
engine=req.app.state.engine_client,
|
||||
)
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
|
||||
async def generation_error_handler(req: Request, exc: GenerationError):
|
||||
"""Handle GenerationError without logging stack traces.
|
||||
|
||||
GenerationError is a known, expected error (e.g. KV cache load failure)
|
||||
that should be returned to the client as a 500 response without polluting
|
||||
server logs with stack traces.
|
||||
"""
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
@@ -0,0 +1,43 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
vLLM Exception handlers are registered in four layers:
|
||||
1. framework errors raised by FastAPI/Starlette
|
||||
2. vLLM-specific errors dispatched via a single ``VLLMError`` handler
|
||||
3. fallback handlers for raw exceptions not yet migrated to ``VLLMError``
|
||||
4. the raw ``Exception`` handler as a safety net
|
||||
Registering specific exception types (rather than only ``Exception``)
|
||||
ensures they are handled by ``ExceptionMiddleware`` (inside the Prometheus
|
||||
middleware) rather than ``ServerErrorMiddleware`` (outside it), so their
|
||||
status codes are recorded correctly.
|
||||
"""
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
|
||||
from vllm.exceptions import VLLMError
|
||||
|
||||
from .handlers.exception import exception_handler
|
||||
from .handlers.http import http_exception_handler
|
||||
from .handlers.validation import validation_exception_handler
|
||||
from .handlers.vllm_error import vllm_error_handler
|
||||
|
||||
|
||||
def init_exception_handler(app: FastAPI):
|
||||
# 1. framework errors raised by FastAPI/Starlette
|
||||
app.exception_handler(HTTPException)(http_exception_handler)
|
||||
app.exception_handler(RequestValidationError)(validation_exception_handler)
|
||||
|
||||
# 2. vLLM-specific errors dispatched via a single ``VLLMError`` handler
|
||||
app.exception_handler(VLLMError)(vllm_error_handler)
|
||||
|
||||
# 3. fallback handlers for raw exceptions not yet migrated to ``VLLMError``
|
||||
# TODO(zqzten): remove these fallback handlers after migration to VLLMError
|
||||
app.exception_handler(ValueError)(exception_handler)
|
||||
app.exception_handler(TypeError)(exception_handler)
|
||||
app.exception_handler(OverflowError)(exception_handler)
|
||||
app.exception_handler(NotImplementedError)(exception_handler)
|
||||
|
||||
# 4. the raw ``Exception`` handler as a safety net
|
||||
app.exception_handler(Exception)(exception_handler)
|
||||
@@ -0,0 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import regex as re
|
||||
|
||||
|
||||
def sanitize_message(message: str) -> str:
|
||||
"""Strip memory addresses, tracebacks, and file paths from error messages."""
|
||||
message = re.sub(r" at 0x[0-9a-f]+>", ">", message)
|
||||
message = re.sub(r'\n?\s*File "[^"]+", line \d+, in \S+(\n\s+.*)?', "", message)
|
||||
message = re.sub(
|
||||
r"/(?:home|usr|opt|var|tmp|root|lib|mnt|srv)(?:/[\w.\-]+)+", "<path>", message
|
||||
)
|
||||
message = re.sub(r"(?:/[\w\-]+)+/[\w\-]+\.\w+", "<path>", message)
|
||||
return message.strip()
|
||||
@@ -10,7 +10,6 @@ from logging import Logger
|
||||
from string import Template
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
from fastapi import Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
@@ -309,17 +308,6 @@ def process_lora_modules(
|
||||
return lora_modules
|
||||
|
||||
|
||||
def sanitize_message(message: str) -> str:
|
||||
"""Strip memory addresses, tracebacks, and file paths from error messages."""
|
||||
message = re.sub(r" at 0x[0-9a-f]+>", ">", message)
|
||||
message = re.sub(r'\n?\s*File "[^"]+", line \d+, in \S+(\n\s+.*)?', "", message)
|
||||
message = re.sub(
|
||||
r"/(?:home|usr|opt|var|tmp|root|lib|mnt|srv)(?:/[\w.\-]+)+", "<path>", message
|
||||
)
|
||||
message = re.sub(r"(?:/[\w\-]+)+/[\w\-]+\.\w+", "<path>", message)
|
||||
return message.strip()
|
||||
|
||||
|
||||
def log_version_and_model(lgr: Logger, version: str, model_name: str) -> None:
|
||||
if envs.VLLM_DISABLE_LOG_LOGO or (formatter := current_formatter_type(lgr)) is None:
|
||||
message = "vLLM server version %s, serving model %s"
|
||||
|
||||
@@ -8,12 +8,9 @@ import uuid
|
||||
from argparse import Namespace
|
||||
from collections.abc import Awaitable
|
||||
from contextlib import asynccontextmanager
|
||||
from http import HTTPStatus
|
||||
|
||||
import pydantic
|
||||
import regex as re
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.concurrency import iterate_in_threadpool
|
||||
from starlette.datastructures import Headers, MutableHeaders
|
||||
@@ -21,20 +18,8 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from vllm import envs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.launcher import terminate_if_errored
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorInfo,
|
||||
ErrorResponse,
|
||||
GenerationError,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.error_response import (
|
||||
create_error_response,
|
||||
sanitize_message,
|
||||
)
|
||||
from vllm.exceptions import VLLMError, VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gc_utils import freeze_gc_heap
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
|
||||
logger = init_logger("vllm.entrypoints.openai.server_utils")
|
||||
|
||||
@@ -325,222 +310,6 @@ async def log_response(request: Request, call_next):
|
||||
return response
|
||||
|
||||
|
||||
async def vllm_error_handler(req: Request, exc: VLLMError):
|
||||
"""Dispatch a vLLM-specific error to the appropriate handler."""
|
||||
if isinstance(exc, (EngineGenerateError, EngineDeadError)):
|
||||
return await engine_error_handler(req, exc)
|
||||
elif isinstance(exc, GenerationError):
|
||||
return await generation_error_handler(req, exc)
|
||||
else:
|
||||
return await exception_handler(req, exc)
|
||||
|
||||
|
||||
async def engine_error_handler(
|
||||
req: Request, exc: EngineDeadError | EngineGenerateError
|
||||
):
|
||||
"""
|
||||
VLLM V1 AsyncLLM catches exceptions and returns
|
||||
only two types: EngineGenerateError and EngineDeadError.
|
||||
|
||||
EngineGenerateError is raised by the per request generate()
|
||||
method. This error could be request specific (and therefore
|
||||
recoverable - e.g. if there is an error in input processing).
|
||||
|
||||
EngineDeadError is raised by the background output_handler
|
||||
method. This error is global and therefore not recoverable.
|
||||
|
||||
We register these @app.exception_handlers to return nice
|
||||
responses to the end user if they occur and shut down if needed.
|
||||
See https://fastapi.tiangolo.com/tutorial/handling-errors/
|
||||
for more details on how exception handlers work.
|
||||
|
||||
If an exception is encountered in a StreamingResponse
|
||||
generator, the exception is not raised, since we already sent
|
||||
a 200 status. Rather, we send an error message as the next chunk.
|
||||
Since the exception is not raised, this means that the server
|
||||
will not automatically shut down. Instead, we use the watchdog
|
||||
background task for check for errored state.
|
||||
"""
|
||||
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"Engine Exception caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
terminate_if_errored(
|
||||
server=req.app.state.server,
|
||||
engine=req.app.state.engine_client,
|
||||
)
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
|
||||
async def generation_error_handler(req: Request, exc: GenerationError):
|
||||
"""Handle GenerationError without logging stack traces.
|
||||
|
||||
GenerationError is a known, expected error (e.g. KV cache load failure)
|
||||
that should be returned to the client as a 500 response without polluting
|
||||
server logs with stack traces.
|
||||
"""
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
|
||||
async def exception_handler(req: Request, exc: Exception):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.error(
|
||||
"Exception caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
|
||||
async def http_exception_handler(req: Request, exc: HTTPException):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"HTTPException caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(
|
||||
message=sanitize_message(exc.detail),
|
||||
type=HTTPStatus(exc.status_code).phrase,
|
||||
code=exc.status_code,
|
||||
)
|
||||
)
|
||||
return JSONResponse(err.model_dump(), status_code=exc.status_code)
|
||||
|
||||
|
||||
_BRACKETED_INTERNAL_RE = re.compile(r"[\[\]{}()]")
|
||||
|
||||
# NOTE: this list is pydantic-core's internal schema-kind vocabulary,
|
||||
# not a stable public API -- it can grow when pydantic-core adds new
|
||||
# wrapper/validator kinds. To refresh it after a pydantic upgrade:
|
||||
# 1. Fuzz the validation-error-prone endpoints (e.g. /tokenize,
|
||||
# /v1/completions, /v1/chat/completions) with deliberately
|
||||
# malformed values for union-typed and wrapped fields (e.g. `stop`,
|
||||
# `prompt`), and inspect the raw `loc` tuples in the response.
|
||||
# 2. Any *unbracketed* segment that isn't a real field name or list
|
||||
# index is a new internal marker -- add it here. Bracketed/
|
||||
# parenthesized markers (e.g. "list[...]", "function-wrap[...]")
|
||||
# are already caught structurally by _BRACKETED_INTERNAL_RE and
|
||||
# don't need a list entry.
|
||||
# 3. pydantic-core's source (the `error.rs`/schema-kind definitions
|
||||
# in the pydantic-core Rust crate) is the canonical reference if
|
||||
# you want to check before it shows up in a live fuzz run.
|
||||
_INTERNAL_LOC_MARKERS = frozenset(
|
||||
{
|
||||
"function-wrap",
|
||||
"function-after",
|
||||
"function-before",
|
||||
"function-plain",
|
||||
"json-or-python",
|
||||
"lax-or-strict",
|
||||
"chain",
|
||||
"default",
|
||||
"nullable",
|
||||
"tagged-union",
|
||||
"union",
|
||||
"call",
|
||||
"arguments",
|
||||
"is-instance",
|
||||
"is-subclass",
|
||||
"callable",
|
||||
"str",
|
||||
"int",
|
||||
"float",
|
||||
"bool",
|
||||
"bytes",
|
||||
"bytearray",
|
||||
"list",
|
||||
"tuple",
|
||||
"dict",
|
||||
"set",
|
||||
"frozenset",
|
||||
"complex",
|
||||
"none",
|
||||
"nonetype",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _is_internal_loc_segment(segment: str) -> bool:
|
||||
"""True if `segment` is a Pydantic-internal wrapper/union-branch
|
||||
marker rather than a user-meaningful field name or list index."""
|
||||
if _BRACKETED_INTERNAL_RE.search(segment):
|
||||
return True
|
||||
return segment.lower() in _INTERNAL_LOC_MARKERS
|
||||
|
||||
|
||||
def clean_loc_for_param(loc: tuple) -> str:
|
||||
"""Join a Pydantic error `loc` tuple into a clean dotted `param`
|
||||
path, dropping internal wrapper/union-branch markers that don't
|
||||
correspond to a real field name an API consumer would recognize.
|
||||
|
||||
E.g. ('body', 'function-wrap[__log_extra_fields__()]', 'prompt')
|
||||
-> "body.prompt", not "body.function-wrap[__log_extra_fields__()].prompt".
|
||||
"""
|
||||
parts = [str(p) for p in loc if not _is_internal_loc_segment(str(p))]
|
||||
if not parts:
|
||||
return ".".join(str(p) for p in loc)
|
||||
return ".".join(parts)
|
||||
|
||||
|
||||
async def validation_exception_handler(req: Request, exc: RequestValidationError):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"RequestValidationError caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
param = None
|
||||
errors = exc.errors()
|
||||
for error in errors:
|
||||
if "ctx" in error and "error" in error["ctx"]:
|
||||
ctx_error = error["ctx"]["error"]
|
||||
if isinstance(ctx_error, VLLMValidationError):
|
||||
param = ctx_error.parameter
|
||||
break
|
||||
|
||||
if param is None and errors:
|
||||
first_error = errors[0]
|
||||
loc = first_error.get("loc") if isinstance(first_error, dict) else None
|
||||
if loc:
|
||||
param = clean_loc_for_param(loc)
|
||||
|
||||
# Build the message from exc.errors() instead of str(exc) - str(exc)
|
||||
# leaks the server's file path via FastAPI's endpoint context.
|
||||
if errors:
|
||||
count = len(errors)
|
||||
label = "error" if count == 1 else "errors"
|
||||
message = f"{count} validation {label}:\n"
|
||||
message += "".join(f" {err}\n" for err in errors)
|
||||
message = message.rstrip()
|
||||
else:
|
||||
message = "Validation error"
|
||||
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(
|
||||
message=sanitize_message(message),
|
||||
type=HTTPStatus.BAD_REQUEST.phrase,
|
||||
code=HTTPStatus.BAD_REQUEST,
|
||||
param=param,
|
||||
)
|
||||
)
|
||||
return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
|
||||
|
||||
|
||||
_running_tasks: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from starlette.websockets import WebSocketDisconnect
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
render_for_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.serve.utils.error_response import create_error_response
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import (
|
||||
EngineInput,
|
||||
|
||||
Reference in New Issue
Block a user