[Frontend] Consolidate entrypoint exception handler (#52261)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-08-14 09:27:24 +00:00
committed by GitHub
parent 103c419f04
commit b8165e5e58
31 changed files with 418 additions and 343 deletions
@@ -44,7 +44,9 @@ from vllm.entrypoints.openai.engine.protocol import (
PromptTokenUsageInfo,
UsageInfo,
)
from vllm.entrypoints.serve.utils.server_utils import validation_exception_handler
from vllm.entrypoints.serve.exception_handling.handlers.validation import (
validation_exception_handler,
)
_convert = AnthropicServingMessages._convert_anthropic_to_openai_request
_img_url = AnthropicServingMessages._convert_image_source_to_url
+3 -1
View File
@@ -30,8 +30,10 @@ from vllm.entrypoints.cohere.protocol import (
CohereChatV2Response,
)
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
from vllm.entrypoints.serve.utils.server_utils import (
from vllm.entrypoints.serve.exception_handling.handlers.http import (
http_exception_handler,
)
from vllm.entrypoints.serve.exception_handling.handlers.validation import (
validation_exception_handler,
)
@@ -10,7 +10,60 @@ speech-to-text WebSocket paths.
import pytest
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
def test_sanitize_message():
assert (
sanitize_message("<_io.BytesIO object at 0x7a95e299e750>")
== "<_io.BytesIO object>"
)
class TestSanitizeMessageFilePaths:
"""sanitize_message should also strip file paths and traceback
frames, not just memory addresses - see #31683."""
def test_strips_traceback_style_frame(self):
msg = (
"1 validation error:\n"
" {'type': 'list_type', 'loc': ('body', 'messages')}\n"
'\n File "/usr/local/lib/python3.12/dist-packages/vllm/'
'entrypoints/serve/utils/api_utils.py", line 40, '
"in create_chat_completion\n"
" POST /v1/chat/completions"
)
result = sanitize_message(msg)
assert "/usr/local/" not in result
assert "api_utils.py" not in result
assert "list_type" in result
def test_strips_arbitrary_absolute_path(self):
result = sanitize_message("Error in /home/user/project/vllm/server.py")
assert "/home/user" not in result
def test_strips_single_parent_container_path(self):
"""Regression: /app/server.py and /workspace/server.py (common in
container deployments) were missed by the original {2,} quantifier."""
assert "/app/" not in sanitize_message("Error in /app/server.py")
assert "/workspace/" not in sanitize_message("Error in /workspace/server.py")
def test_preserves_api_endpoint_paths(self):
msg = "POST /v1/chat/completions failed"
assert "/v1/chat/completions" in sanitize_message(msg)
def test_preserves_short_field_references(self):
msg = "Invalid value for field 'body.messages'"
assert sanitize_message(msg) == msg
def test_strips_both_address_and_path(self):
msg = (
"<Request at 0x7f123> failed at "
"/usr/local/lib/python3.12/dist-packages/vllm/server.py"
)
result = sanitize_message(msg)
assert "0x" not in result
assert "/usr/local/" not in result
class TestSanitizeMessageCoversLeakPatterns:
@@ -16,7 +16,7 @@ from types import SimpleNamespace
import pytest
from fastapi.exceptions import RequestValidationError
from vllm.entrypoints.serve.utils.server_utils import (
from vllm.entrypoints.serve.exception_handling.handlers.validation import (
clean_loc_for_param,
validation_exception_handler,
)
@@ -6,18 +6,10 @@ import pytest
from vllm.entrypoints.openai.engine.protocol import StreamOptions
from vllm.entrypoints.serve.utils.api_utils import (
get_max_tokens,
sanitize_message,
should_include_usage,
)
def test_sanitize_message():
assert (
sanitize_message("<_io.BytesIO object at 0x7a95e299e750>")
== "<_io.BytesIO object>"
)
@pytest.mark.parametrize(
("stream_options", "expected"),
[
@@ -118,49 +110,3 @@ class TestGetMaxTokens:
input_length=150,
default_sampling_params={"max_tokens": 2048},
)
class TestSanitizeMessageFilePaths:
"""sanitize_message should also strip file paths and traceback
frames, not just memory addresses - see #31683."""
def test_strips_traceback_style_frame(self):
msg = (
"1 validation error:\n"
" {'type': 'list_type', 'loc': ('body', 'messages')}\n"
'\n File "/usr/local/lib/python3.12/dist-packages/vllm/'
'entrypoints/serve/utils/api_utils.py", line 40, '
"in create_chat_completion\n"
" POST /v1/chat/completions"
)
result = sanitize_message(msg)
assert "/usr/local/" not in result
assert "api_utils.py" not in result
assert "list_type" in result
def test_strips_arbitrary_absolute_path(self):
result = sanitize_message("Error in /home/user/project/vllm/server.py")
assert "/home/user" not in result
def test_strips_single_parent_container_path(self):
"""Regression: /app/server.py and /workspace/server.py (common in
container deployments) were missed by the original {2,} quantifier."""
assert "/app/" not in sanitize_message("Error in /app/server.py")
assert "/workspace/" not in sanitize_message("Error in /workspace/server.py")
def test_preserves_api_endpoint_paths(self):
msg = "POST /v1/chat/completions failed"
assert "/v1/chat/completions" in sanitize_message(msg)
def test_preserves_short_field_references(self):
msg = "Invalid value for field 'body.messages'"
assert sanitize_message(msg) == msg
def test_strips_both_address_and_path(self):
msg = (
"<Request at 0x7f123> failed at "
"/usr/local/lib/python3.12/dist-packages/vllm/server.py"
)
result = sanitize_message(msg)
assert "0x" not in result
assert "/usr/local/" not in result
@@ -13,7 +13,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import aiohttp
import pytest
from vllm.entrypoints.serve.utils.error_response import create_error_response
from vllm.entrypoints.serve import create_error_response
from vllm.exceptions import VLLMClientError, VLLMUnprocessableEntityError
from vllm.multimodal.media import MediaConnector
@@ -8,7 +8,7 @@ import pytest
from vllm.assets.image import ImageAsset
from vllm.assets.video import VideoAsset
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.entrypoints.serve.utils.error_response import create_error_response
from vllm.entrypoints.serve import create_error_response
from vllm.multimodal.parse import parse_mm_uuids
from vllm.renderers.hf import HfRenderer
from vllm.tokenizers.registry import cached_tokenizer_from_config
+1 -1
View File
@@ -17,9 +17,9 @@ from vllm.entrypoints.anthropic.protocol import (
)
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
sanitize_message,
validate_json_request,
with_cancellation,
)
+1 -1
View File
@@ -46,7 +46,7 @@ from vllm.entrypoints.openai.engine.protocol import (
UsageInfo,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.renderers.online_renderer import OnlineRenderer
+1 -1
View File
@@ -33,9 +33,9 @@ from starlette.middleware.base import BaseHTTPMiddleware
import vllm.envs as envs
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
sanitize_message,
validate_json_request,
with_cancellation,
)
+1 -1
View File
@@ -86,7 +86,7 @@ from vllm.entrypoints.openai.engine.protocol import (
StreamOptions,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.parser.abstract_parser import Parser
from vllm.renderers.cohere import MESSAGES_CITATIONS_KEY, POSITION_TO_SOURCE_KEY
+4 -29
View File
@@ -16,8 +16,7 @@ from contextlib import asynccontextmanager
from typing import Any, cast
import uvloop
from fastapi import FastAPI, HTTPException
from fastapi.exceptions import RequestValidationError
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from starlette.datastructures import State
@@ -31,6 +30,7 @@ from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_se
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.entrypoints.serve.utils.api_utils import (
@@ -41,15 +41,10 @@ from vllm.entrypoints.serve.utils.api_utils import (
)
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.entrypoints.serve.utils.server_utils import (
exception_handler,
get_uvicorn_log_config,
http_exception_handler,
lifespan,
log_response,
validation_exception_handler,
vllm_error_handler,
)
from vllm.exceptions import VLLMError
from vllm.logger import init_logger
from vllm.reasoning import ReasoningParserManager
from vllm.renderers.online_derenderer import OnlineDerenderer
@@ -275,6 +270,8 @@ def build_app(
# `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).
_attach_endpoint_plugins(app, supported_tasks)
init_exception_handler(app)
app.root_path = args.root_path
app.add_middleware(
CORSMiddleware,
@@ -284,28 +281,6 @@ def build_app(
allow_headers=args.allowed_headers,
)
# Exception handlers are registered in four layers:
# 1. framework errors raised by FastAPI/Starlette
# 2. vLLM-specific errors dispatched via a single ``VLLMError`` handler
# 3. fallback handlers for raw exceptions not yet migrated to ``VLLMError``
# 4. the raw ``Exception`` handler as a safety net
# Registering specific exception types (rather than only ``Exception``)
# ensures they are handled by ``ExceptionMiddleware`` (inside the Prometheus
# middleware) rather than ``ServerErrorMiddleware`` (outside it), so their
# status codes are recorded correctly.
app.exception_handler(HTTPException)(http_exception_handler)
app.exception_handler(RequestValidationError)(validation_exception_handler)
app.exception_handler(VLLMError)(vllm_error_handler)
# TODO(zqzten): remove these fallback handlers after migration to VLLMError
app.exception_handler(ValueError)(exception_handler)
app.exception_handler(TypeError)(exception_handler)
app.exception_handler(OverflowError)(exception_handler)
app.exception_handler(NotImplementedError)(exception_handler)
app.exception_handler(Exception)(exception_handler)
# Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]:
from vllm.entrypoints.serve.utils.server_utils import AuthenticationMiddleware
+1 -1
View File
@@ -14,11 +14,11 @@ from vllm.entrypoints.openai.engine.protocol import (
ModelPermission,
)
from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath
from vllm.entrypoints.serve import create_error_response
from vllm.entrypoints.serve.lora.protocol import (
LoadLoRAAdapterRequest,
UnloadLoRAAdapterRequest,
)
from vllm.entrypoints.serve.utils.error_response import create_error_response
from vllm.exceptions import LoRAAdapterNotFoundError
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
+1 -1
View File
@@ -52,7 +52,7 @@ from vllm.entrypoints.pooling.scoring.protocol import (
ScoreRequest,
ScoreResponse,
)
from vllm.entrypoints.serve.utils.error_response import create_error_response
from vllm.entrypoints.serve import create_error_response
from vllm.entrypoints.speech_to_text.transcription.protocol import (
TranscriptionRequest,
TranscriptionResponse,
+8
View File
@@ -5,8 +5,16 @@ from fastapi import FastAPI
from vllm.logger import init_logger
from .exception_handling.error_response import create_error_response
logger = init_logger(__name__)
__all__ = [
"create_error_response",
"register_vllm_serve_api_routers",
"register_vllm_dev_api_routers",
]
def register_vllm_serve_api_routers(app: FastAPI):
from .instrumentator import register_instrumentator_api_routers
+1 -1
View File
@@ -12,8 +12,8 @@ from vllm.entrypoints.openai.models.serving import (
OpenAIServingModels,
)
from vllm.entrypoints.pooling.typing import AnyPoolingRequest
from vllm.entrypoints.serve import create_error_response
from vllm.entrypoints.serve.engine.typing import AnyRequest
from vllm.entrypoints.serve.utils.error_response import create_error_response
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.exceptions import VLLMNotFoundError
from vllm.inputs import EngineInput
@@ -7,9 +7,10 @@ from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
GenerationError,
)
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
from vllm.logger import init_logger
from .utils import sanitize_message
logger = init_logger(__name__)
@@ -0,0 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fastapi import Request
from starlette.responses import JSONResponse
from vllm.logger import init_logger
from ..error_response import create_error_response
logger = init_logger(__name__)
async def exception_handler(req: Request, exc: Exception):
if req.app.state.args.log_error_stack:
logger.error(
"Exception caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
err = create_error_response(exc)
return JSONResponse(err.model_dump(), status_code=err.error.code)
@@ -0,0 +1,31 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import HTTPException, Request
from starlette.responses import JSONResponse
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
from vllm.logger import init_logger
from ..utils import sanitize_message
logger = init_logger(__name__)
async def http_exception_handler(req: Request, exc: HTTPException):
if req.app.state.args.log_error_stack:
logger.exception(
"HTTPException caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
err = ErrorResponse(
error=ErrorInfo(
message=sanitize_message(exc.detail),
type=HTTPStatus(exc.status_code).phrase,
code=exc.status_code,
)
)
return JSONResponse(err.model_dump(), status_code=exc.status_code)
@@ -0,0 +1,138 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
import regex as re
from fastapi import Request
from fastapi.exceptions import RequestValidationError
from starlette.responses import JSONResponse
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from ..utils import sanitize_message
logger = init_logger(__name__)
_BRACKETED_INTERNAL_RE = re.compile(r"[\[\]{}()]")
# NOTE: this list is pydantic-core's internal schema-kind vocabulary,
# not a stable public API -- it can grow when pydantic-core adds new
# wrapper/validator kinds. To refresh it after a pydantic upgrade:
# 1. Fuzz the validation-error-prone endpoints (e.g. /tokenize,
# /v1/completions, /v1/chat/completions) with deliberately
# malformed values for union-typed and wrapped fields (e.g. `stop`,
# `prompt`), and inspect the raw `loc` tuples in the response.
# 2. Any *unbracketed* segment that isn't a real field name or list
# index is a new internal marker -- add it here. Bracketed/
# parenthesized markers (e.g. "list[...]", "function-wrap[...]")
# are already caught structurally by _BRACKETED_INTERNAL_RE and
# don't need a list entry.
# 3. pydantic-core's source (the `error.rs`/schema-kind definitions
# in the pydantic-core Rust crate) is the canonical reference if
# you want to check before it shows up in a live fuzz run.
_INTERNAL_LOC_MARKERS = frozenset(
{
"function-wrap",
"function-after",
"function-before",
"function-plain",
"json-or-python",
"lax-or-strict",
"chain",
"default",
"nullable",
"tagged-union",
"union",
"call",
"arguments",
"is-instance",
"is-subclass",
"callable",
"str",
"int",
"float",
"bool",
"bytes",
"bytearray",
"list",
"tuple",
"dict",
"set",
"frozenset",
"complex",
"none",
"nonetype",
}
)
def _is_internal_loc_segment(segment: str) -> bool:
"""True if `segment` is a Pydantic-internal wrapper/union-branch
marker rather than a user-meaningful field name or list index."""
if _BRACKETED_INTERNAL_RE.search(segment):
return True
return segment.lower() in _INTERNAL_LOC_MARKERS
def clean_loc_for_param(loc: tuple) -> str:
"""Join a Pydantic error `loc` tuple into a clean dotted `param`
path, dropping internal wrapper/union-branch markers that don't
correspond to a real field name an API consumer would recognize.
E.g. ('body', 'function-wrap[__log_extra_fields__()]', 'prompt')
-> "body.prompt", not "body.function-wrap[__log_extra_fields__()].prompt".
"""
parts = [str(p) for p in loc if not _is_internal_loc_segment(str(p))]
if not parts:
return ".".join(str(p) for p in loc)
return ".".join(parts)
async def validation_exception_handler(req: Request, exc: RequestValidationError):
if req.app.state.args.log_error_stack:
logger.exception(
"RequestValidationError caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
param = None
errors = exc.errors()
for error in errors:
if "ctx" in error and "error" in error["ctx"]:
ctx_error = error["ctx"]["error"]
if isinstance(ctx_error, VLLMValidationError):
param = ctx_error.parameter
break
if param is None and errors:
first_error = errors[0]
loc = first_error.get("loc") if isinstance(first_error, dict) else None
if loc:
param = clean_loc_for_param(loc)
# Build the message from exc.errors() instead of str(exc) - str(exc)
# leaks the server's file path via FastAPI's endpoint context.
if errors:
count = len(errors)
label = "error" if count == 1 else "errors"
message = f"{count} validation {label}:\n"
message += "".join(f" {err}\n" for err in errors)
message = message.rstrip()
else:
message = "Validation error"
err = ErrorResponse(
error=ErrorInfo(
message=sanitize_message(message),
type=HTTPStatus.BAD_REQUEST.phrase,
code=HTTPStatus.BAD_REQUEST,
param=param,
)
)
return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
@@ -0,0 +1,80 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fastapi import Request
from starlette.responses import JSONResponse
from vllm.entrypoints.launcher import terminate_if_errored
from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.exceptions import VLLMError
from vllm.logger import init_logger
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
from ..error_response import create_error_response
from .exception import exception_handler
logger = init_logger(__name__)
async def vllm_error_handler(req: Request, exc: VLLMError):
"""Dispatch a vLLM-specific error to the appropriate handler."""
if isinstance(exc, (EngineGenerateError, EngineDeadError)):
return await engine_error_handler(req, exc)
elif isinstance(exc, GenerationError):
return await generation_error_handler(req, exc)
else:
return await exception_handler(req, exc)
async def engine_error_handler(
req: Request, exc: EngineDeadError | EngineGenerateError
):
"""
VLLM V1 AsyncLLM catches exceptions and returns
only two types: EngineGenerateError and EngineDeadError.
EngineGenerateError is raised by the per request generate()
method. This error could be request specific (and therefore
recoverable - e.g. if there is an error in input processing).
EngineDeadError is raised by the background output_handler
method. This error is global and therefore not recoverable.
We register these @app.exception_handlers to return nice
responses to the end user if they occur and shut down if needed.
See https://fastapi.tiangolo.com/tutorial/handling-errors/
for more details on how exception handlers work.
If an exception is encountered in a StreamingResponse
generator, the exception is not raised, since we already sent
a 200 status. Rather, we send an error message as the next chunk.
Since the exception is not raised, this means that the server
will not automatically shut down. Instead, we use the watchdog
background task for check for errored state.
"""
if req.app.state.args.log_error_stack:
logger.exception(
"Engine Exception caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
terminate_if_errored(
server=req.app.state.server,
engine=req.app.state.engine_client,
)
err = create_error_response(exc)
return JSONResponse(err.model_dump(), status_code=err.error.code)
async def generation_error_handler(req: Request, exc: GenerationError):
"""Handle GenerationError without logging stack traces.
GenerationError is a known, expected error (e.g. KV cache load failure)
that should be returned to the client as a 500 response without polluting
server logs with stack traces.
"""
err = create_error_response(exc)
return JSONResponse(err.model_dump(), status_code=err.error.code)
@@ -0,0 +1,43 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
vLLM Exception handlers are registered in four layers:
1. framework errors raised by FastAPI/Starlette
2. vLLM-specific errors dispatched via a single ``VLLMError`` handler
3. fallback handlers for raw exceptions not yet migrated to ``VLLMError``
4. the raw ``Exception`` handler as a safety net
Registering specific exception types (rather than only ``Exception``)
ensures they are handled by ``ExceptionMiddleware`` (inside the Prometheus
middleware) rather than ``ServerErrorMiddleware`` (outside it), so their
status codes are recorded correctly.
"""
from fastapi import FastAPI, HTTPException
from fastapi.exceptions import RequestValidationError
from vllm.exceptions import VLLMError
from .handlers.exception import exception_handler
from .handlers.http import http_exception_handler
from .handlers.validation import validation_exception_handler
from .handlers.vllm_error import vllm_error_handler
def init_exception_handler(app: FastAPI):
# 1. framework errors raised by FastAPI/Starlette
app.exception_handler(HTTPException)(http_exception_handler)
app.exception_handler(RequestValidationError)(validation_exception_handler)
# 2. vLLM-specific errors dispatched via a single ``VLLMError`` handler
app.exception_handler(VLLMError)(vllm_error_handler)
# 3. fallback handlers for raw exceptions not yet migrated to ``VLLMError``
# TODO(zqzten): remove these fallback handlers after migration to VLLMError
app.exception_handler(ValueError)(exception_handler)
app.exception_handler(TypeError)(exception_handler)
app.exception_handler(OverflowError)(exception_handler)
app.exception_handler(NotImplementedError)(exception_handler)
# 4. the raw ``Exception`` handler as a safety net
app.exception_handler(Exception)(exception_handler)
@@ -0,0 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import regex as re
def sanitize_message(message: str) -> str:
"""Strip memory addresses, tracebacks, and file paths from error messages."""
message = re.sub(r" at 0x[0-9a-f]+>", ">", message)
message = re.sub(r'\n?\s*File "[^"]+", line \d+, in \S+(\n\s+.*)?', "", message)
message = re.sub(
r"/(?:home|usr|opt|var|tmp|root|lib|mnt|srv)(?:/[\w.\-]+)+", "<path>", message
)
message = re.sub(r"(?:/[\w\-]+)+/[\w\-]+\.\w+", "<path>", message)
return message.strip()
-12
View File
@@ -10,7 +10,6 @@ from logging import Logger
from string import Template
from typing import Any
import regex as re
from fastapi import Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, StreamingResponse
@@ -309,17 +308,6 @@ def process_lora_modules(
return lora_modules
def sanitize_message(message: str) -> str:
"""Strip memory addresses, tracebacks, and file paths from error messages."""
message = re.sub(r" at 0x[0-9a-f]+>", ">", message)
message = re.sub(r'\n?\s*File "[^"]+", line \d+, in \S+(\n\s+.*)?', "", message)
message = re.sub(
r"/(?:home|usr|opt|var|tmp|root|lib|mnt|srv)(?:/[\w.\-]+)+", "<path>", message
)
message = re.sub(r"(?:/[\w\-]+)+/[\w\-]+\.\w+", "<path>", message)
return message.strip()
def log_version_and_model(lgr: Logger, version: str, model_name: str) -> None:
if envs.VLLM_DISABLE_LOG_LOGO or (formatter := current_formatter_type(lgr)) is None:
message = "vLLM server version %s, serving model %s"
+1 -232
View File
@@ -8,12 +8,9 @@ import uuid
from argparse import Namespace
from collections.abc import Awaitable
from contextlib import asynccontextmanager
from http import HTTPStatus
import pydantic
import regex as re
from fastapi import FastAPI, HTTPException, Request
from fastapi.exceptions import RequestValidationError
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
from starlette.concurrency import iterate_in_threadpool
from starlette.datastructures import Headers, MutableHeaders
@@ -21,20 +18,8 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send
from vllm import envs
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.launcher import terminate_if_errored
from vllm.entrypoints.openai.engine.protocol import (
ErrorInfo,
ErrorResponse,
GenerationError,
)
from vllm.entrypoints.serve.utils.error_response import (
create_error_response,
sanitize_message,
)
from vllm.exceptions import VLLMError, VLLMValidationError
from vllm.logger import init_logger
from vllm.utils.gc_utils import freeze_gc_heap
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
logger = init_logger("vllm.entrypoints.openai.server_utils")
@@ -325,222 +310,6 @@ async def log_response(request: Request, call_next):
return response
async def vllm_error_handler(req: Request, exc: VLLMError):
"""Dispatch a vLLM-specific error to the appropriate handler."""
if isinstance(exc, (EngineGenerateError, EngineDeadError)):
return await engine_error_handler(req, exc)
elif isinstance(exc, GenerationError):
return await generation_error_handler(req, exc)
else:
return await exception_handler(req, exc)
async def engine_error_handler(
req: Request, exc: EngineDeadError | EngineGenerateError
):
"""
VLLM V1 AsyncLLM catches exceptions and returns
only two types: EngineGenerateError and EngineDeadError.
EngineGenerateError is raised by the per request generate()
method. This error could be request specific (and therefore
recoverable - e.g. if there is an error in input processing).
EngineDeadError is raised by the background output_handler
method. This error is global and therefore not recoverable.
We register these @app.exception_handlers to return nice
responses to the end user if they occur and shut down if needed.
See https://fastapi.tiangolo.com/tutorial/handling-errors/
for more details on how exception handlers work.
If an exception is encountered in a StreamingResponse
generator, the exception is not raised, since we already sent
a 200 status. Rather, we send an error message as the next chunk.
Since the exception is not raised, this means that the server
will not automatically shut down. Instead, we use the watchdog
background task for check for errored state.
"""
if req.app.state.args.log_error_stack:
logger.exception(
"Engine Exception caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
terminate_if_errored(
server=req.app.state.server,
engine=req.app.state.engine_client,
)
err = create_error_response(exc)
return JSONResponse(err.model_dump(), status_code=err.error.code)
async def generation_error_handler(req: Request, exc: GenerationError):
"""Handle GenerationError without logging stack traces.
GenerationError is a known, expected error (e.g. KV cache load failure)
that should be returned to the client as a 500 response without polluting
server logs with stack traces.
"""
err = create_error_response(exc)
return JSONResponse(err.model_dump(), status_code=err.error.code)
async def exception_handler(req: Request, exc: Exception):
if req.app.state.args.log_error_stack:
logger.error(
"Exception caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
err = create_error_response(exc)
return JSONResponse(err.model_dump(), status_code=err.error.code)
async def http_exception_handler(req: Request, exc: HTTPException):
if req.app.state.args.log_error_stack:
logger.exception(
"HTTPException caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
err = ErrorResponse(
error=ErrorInfo(
message=sanitize_message(exc.detail),
type=HTTPStatus(exc.status_code).phrase,
code=exc.status_code,
)
)
return JSONResponse(err.model_dump(), status_code=exc.status_code)
_BRACKETED_INTERNAL_RE = re.compile(r"[\[\]{}()]")
# NOTE: this list is pydantic-core's internal schema-kind vocabulary,
# not a stable public API -- it can grow when pydantic-core adds new
# wrapper/validator kinds. To refresh it after a pydantic upgrade:
# 1. Fuzz the validation-error-prone endpoints (e.g. /tokenize,
# /v1/completions, /v1/chat/completions) with deliberately
# malformed values for union-typed and wrapped fields (e.g. `stop`,
# `prompt`), and inspect the raw `loc` tuples in the response.
# 2. Any *unbracketed* segment that isn't a real field name or list
# index is a new internal marker -- add it here. Bracketed/
# parenthesized markers (e.g. "list[...]", "function-wrap[...]")
# are already caught structurally by _BRACKETED_INTERNAL_RE and
# don't need a list entry.
# 3. pydantic-core's source (the `error.rs`/schema-kind definitions
# in the pydantic-core Rust crate) is the canonical reference if
# you want to check before it shows up in a live fuzz run.
_INTERNAL_LOC_MARKERS = frozenset(
{
"function-wrap",
"function-after",
"function-before",
"function-plain",
"json-or-python",
"lax-or-strict",
"chain",
"default",
"nullable",
"tagged-union",
"union",
"call",
"arguments",
"is-instance",
"is-subclass",
"callable",
"str",
"int",
"float",
"bool",
"bytes",
"bytearray",
"list",
"tuple",
"dict",
"set",
"frozenset",
"complex",
"none",
"nonetype",
}
)
def _is_internal_loc_segment(segment: str) -> bool:
"""True if `segment` is a Pydantic-internal wrapper/union-branch
marker rather than a user-meaningful field name or list index."""
if _BRACKETED_INTERNAL_RE.search(segment):
return True
return segment.lower() in _INTERNAL_LOC_MARKERS
def clean_loc_for_param(loc: tuple) -> str:
"""Join a Pydantic error `loc` tuple into a clean dotted `param`
path, dropping internal wrapper/union-branch markers that don't
correspond to a real field name an API consumer would recognize.
E.g. ('body', 'function-wrap[__log_extra_fields__()]', 'prompt')
-> "body.prompt", not "body.function-wrap[__log_extra_fields__()].prompt".
"""
parts = [str(p) for p in loc if not _is_internal_loc_segment(str(p))]
if not parts:
return ".".join(str(p) for p in loc)
return ".".join(parts)
async def validation_exception_handler(req: Request, exc: RequestValidationError):
if req.app.state.args.log_error_stack:
logger.exception(
"RequestValidationError caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
param = None
errors = exc.errors()
for error in errors:
if "ctx" in error and "error" in error["ctx"]:
ctx_error = error["ctx"]["error"]
if isinstance(ctx_error, VLLMValidationError):
param = ctx_error.parameter
break
if param is None and errors:
first_error = errors[0]
loc = first_error.get("loc") if isinstance(first_error, dict) else None
if loc:
param = clean_loc_for_param(loc)
# Build the message from exc.errors() instead of str(exc) - str(exc)
# leaks the server's file path via FastAPI's endpoint context.
if errors:
count = len(errors)
label = "error" if count == 1 else "errors"
message = f"{count} validation {label}:\n"
message += "".join(f" {err}\n" for err in errors)
message = message.rstrip()
else:
message = "Validation error"
err = ErrorResponse(
error=ErrorInfo(
message=sanitize_message(message),
type=HTTPStatus.BAD_REQUEST.phrase,
code=HTTPStatus.BAD_REQUEST,
param=param,
)
)
return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
_running_tasks: set[asyncio.Task] = set()
@@ -14,7 +14,7 @@ from starlette.websockets import WebSocketDisconnect
from vllm import envs
from vllm.entrypoints.openai.engine.protocol import ErrorResponse, UsageInfo
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
+1 -1
View File
@@ -26,7 +26,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
render_for_completion,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.serve.utils.error_response import create_error_response
from vllm.entrypoints.serve import create_error_response
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import (
EngineInput,