diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index db792f8c73..a54e13666f 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -44,7 +44,9 @@ from vllm.entrypoints.openai.engine.protocol import ( PromptTokenUsageInfo, UsageInfo, ) -from vllm.entrypoints.serve.utils.server_utils import validation_exception_handler +from vllm.entrypoints.serve.exception_handling.handlers.validation import ( + validation_exception_handler, +) _convert = AnthropicServingMessages._convert_anthropic_to_openai_request _img_url = AnthropicServingMessages._convert_image_source_to_url diff --git a/tests/entrypoints/cohere/test_api_router.py b/tests/entrypoints/cohere/test_api_router.py index 8380faa29e..45e2ec5cb0 100644 --- a/tests/entrypoints/cohere/test_api_router.py +++ b/tests/entrypoints/cohere/test_api_router.py @@ -30,8 +30,10 @@ from vllm.entrypoints.cohere.protocol import ( CohereChatV2Response, ) from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse -from vllm.entrypoints.serve.utils.server_utils import ( +from vllm.entrypoints.serve.exception_handling.handlers.http import ( http_exception_handler, +) +from vllm.entrypoints.serve.exception_handling.handlers.validation import ( validation_exception_handler, ) diff --git a/tests/entrypoints/serve/exception_handling/__init__.py b/tests/entrypoints/serve/exception_handling/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/entrypoints/serve/utils/test_error_sanitization.py b/tests/entrypoints/serve/exception_handling/test_error_sanitization.py similarity index 57% rename from tests/entrypoints/serve/utils/test_error_sanitization.py rename to tests/entrypoints/serve/exception_handling/test_error_sanitization.py index c871dffb40..f98d870318 100644 --- a/tests/entrypoints/serve/utils/test_error_sanitization.py +++ b/tests/entrypoints/serve/exception_handling/test_error_sanitization.py @@ -10,7 +10,60 @@ speech-to-text WebSocket paths. import pytest -from vllm.entrypoints.serve.utils.api_utils import sanitize_message +from vllm.entrypoints.serve.exception_handling.utils import sanitize_message + + +def test_sanitize_message(): + assert ( + sanitize_message("<_io.BytesIO object at 0x7a95e299e750>") + == "<_io.BytesIO object>" + ) + + +class TestSanitizeMessageFilePaths: + """sanitize_message should also strip file paths and traceback + frames, not just memory addresses - see #31683.""" + + def test_strips_traceback_style_frame(self): + msg = ( + "1 validation error:\n" + " {'type': 'list_type', 'loc': ('body', 'messages')}\n" + '\n File "/usr/local/lib/python3.12/dist-packages/vllm/' + 'entrypoints/serve/utils/api_utils.py", line 40, ' + "in create_chat_completion\n" + " POST /v1/chat/completions" + ) + result = sanitize_message(msg) + assert "/usr/local/" not in result + assert "api_utils.py" not in result + assert "list_type" in result + + def test_strips_arbitrary_absolute_path(self): + result = sanitize_message("Error in /home/user/project/vllm/server.py") + assert "/home/user" not in result + + def test_strips_single_parent_container_path(self): + """Regression: /app/server.py and /workspace/server.py (common in + container deployments) were missed by the original {2,} quantifier.""" + assert "/app/" not in sanitize_message("Error in /app/server.py") + assert "/workspace/" not in sanitize_message("Error in /workspace/server.py") + + def test_preserves_api_endpoint_paths(self): + msg = "POST /v1/chat/completions failed" + assert "/v1/chat/completions" in sanitize_message(msg) + + def test_preserves_short_field_references(self): + msg = "Invalid value for field 'body.messages'" + assert sanitize_message(msg) == msg + + def test_strips_both_address_and_path(self): + msg = ( + " failed at " + "/usr/local/lib/python3.12/dist-packages/vllm/server.py" + ) + result = sanitize_message(msg) + assert "0x" not in result + assert "/usr/local/" not in result class TestSanitizeMessageCoversLeakPatterns: diff --git a/tests/entrypoints/serve/instrumentator/test_http_status_metrics.py b/tests/entrypoints/serve/exception_handling/test_http_status_metrics.py similarity index 100% rename from tests/entrypoints/serve/instrumentator/test_http_status_metrics.py rename to tests/entrypoints/serve/exception_handling/test_http_status_metrics.py diff --git a/tests/entrypoints/serve/utils/test_server_utils.py b/tests/entrypoints/serve/exception_handling/test_validation_exception_handler.py similarity index 98% rename from tests/entrypoints/serve/utils/test_server_utils.py rename to tests/entrypoints/serve/exception_handling/test_validation_exception_handler.py index b135d1c97c..9d393b414f 100644 --- a/tests/entrypoints/serve/utils/test_server_utils.py +++ b/tests/entrypoints/serve/exception_handling/test_validation_exception_handler.py @@ -16,7 +16,7 @@ from types import SimpleNamespace import pytest from fastapi.exceptions import RequestValidationError -from vllm.entrypoints.serve.utils.server_utils import ( +from vllm.entrypoints.serve.exception_handling.handlers.validation import ( clean_loc_for_param, validation_exception_handler, ) diff --git a/tests/entrypoints/serve/utils/test_api_utils.py b/tests/entrypoints/serve/utils/test_api_utils.py index 2429da27e1..c23d0e9e68 100644 --- a/tests/entrypoints/serve/utils/test_api_utils.py +++ b/tests/entrypoints/serve/utils/test_api_utils.py @@ -6,18 +6,10 @@ import pytest from vllm.entrypoints.openai.engine.protocol import StreamOptions from vllm.entrypoints.serve.utils.api_utils import ( get_max_tokens, - sanitize_message, should_include_usage, ) -def test_sanitize_message(): - assert ( - sanitize_message("<_io.BytesIO object at 0x7a95e299e750>") - == "<_io.BytesIO object>" - ) - - @pytest.mark.parametrize( ("stream_options", "expected"), [ @@ -118,49 +110,3 @@ class TestGetMaxTokens: input_length=150, default_sampling_params={"max_tokens": 2048}, ) - - -class TestSanitizeMessageFilePaths: - """sanitize_message should also strip file paths and traceback - frames, not just memory addresses - see #31683.""" - - def test_strips_traceback_style_frame(self): - msg = ( - "1 validation error:\n" - " {'type': 'list_type', 'loc': ('body', 'messages')}\n" - '\n File "/usr/local/lib/python3.12/dist-packages/vllm/' - 'entrypoints/serve/utils/api_utils.py", line 40, ' - "in create_chat_completion\n" - " POST /v1/chat/completions" - ) - result = sanitize_message(msg) - assert "/usr/local/" not in result - assert "api_utils.py" not in result - assert "list_type" in result - - def test_strips_arbitrary_absolute_path(self): - result = sanitize_message("Error in /home/user/project/vllm/server.py") - assert "/home/user" not in result - - def test_strips_single_parent_container_path(self): - """Regression: /app/server.py and /workspace/server.py (common in - container deployments) were missed by the original {2,} quantifier.""" - assert "/app/" not in sanitize_message("Error in /app/server.py") - assert "/workspace/" not in sanitize_message("Error in /workspace/server.py") - - def test_preserves_api_endpoint_paths(self): - msg = "POST /v1/chat/completions failed" - assert "/v1/chat/completions" in sanitize_message(msg) - - def test_preserves_short_field_references(self): - msg = "Invalid value for field 'body.messages'" - assert sanitize_message(msg) == msg - - def test_strips_both_address_and_path(self): - msg = ( - " failed at " - "/usr/local/lib/python3.12/dist-packages/vllm/server.py" - ) - result = sanitize_message(msg) - assert "0x" not in result - assert "/usr/local/" not in result diff --git a/tests/multimodal/media/test_unprocessable_entity_error.py b/tests/multimodal/media/test_unprocessable_entity_error.py index 7cad429557..bca2ac5f5e 100644 --- a/tests/multimodal/media/test_unprocessable_entity_error.py +++ b/tests/multimodal/media/test_unprocessable_entity_error.py @@ -13,7 +13,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import aiohttp import pytest -from vllm.entrypoints.serve.utils.error_response import create_error_response +from vllm.entrypoints.serve import create_error_response from vllm.exceptions import VLLMClientError, VLLMUnprocessableEntityError from vllm.multimodal.media import MediaConnector diff --git a/tests/renderers/test_process_multi_modal_uuids.py b/tests/renderers/test_process_multi_modal_uuids.py index 9f3440fc2e..c811630c85 100644 --- a/tests/renderers/test_process_multi_modal_uuids.py +++ b/tests/renderers/test_process_multi_modal_uuids.py @@ -8,7 +8,7 @@ import pytest from vllm.assets.image import ImageAsset from vllm.assets.video import VideoAsset from vllm.config import CacheConfig, ModelConfig, VllmConfig -from vllm.entrypoints.serve.utils.error_response import create_error_response +from vllm.entrypoints.serve import create_error_response from vllm.multimodal.parse import parse_mm_uuids from vllm.renderers.hf import HfRenderer from vllm.tokenizers.registry import cached_tokenizer_from_config diff --git a/vllm/entrypoints/anthropic/api_router.py b/vllm/entrypoints/anthropic/api_router.py index 414f87f308..7c6d23183d 100644 --- a/vllm/entrypoints/anthropic/api_router.py +++ b/vllm/entrypoints/anthropic/api_router.py @@ -17,9 +17,9 @@ from vllm.entrypoints.anthropic.protocol import ( ) from vllm.entrypoints.anthropic.serving import AnthropicServingMessages from vllm.entrypoints.openai.engine.protocol import ErrorResponse +from vllm.entrypoints.serve.exception_handling.utils import sanitize_message from vllm.entrypoints.serve.utils.api_utils import ( load_aware_call, - sanitize_message, validate_json_request, with_cancellation, ) diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 2f5532b578..4ec5c14e01 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -46,7 +46,7 @@ from vllm.entrypoints.openai.engine.protocol import ( UsageInfo, ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.entrypoints.serve.utils.api_utils import sanitize_message +from vllm.entrypoints.serve.exception_handling.utils import sanitize_message from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.renderers.online_renderer import OnlineRenderer diff --git a/vllm/entrypoints/cohere/api_router.py b/vllm/entrypoints/cohere/api_router.py index b532e60920..85eee0fd55 100644 --- a/vllm/entrypoints/cohere/api_router.py +++ b/vllm/entrypoints/cohere/api_router.py @@ -33,9 +33,9 @@ from starlette.middleware.base import BaseHTTPMiddleware import vllm.envs as envs from vllm.entrypoints.openai.engine.protocol import ErrorResponse +from vllm.entrypoints.serve.exception_handling.utils import sanitize_message from vllm.entrypoints.serve.utils.api_utils import ( load_aware_call, - sanitize_message, validate_json_request, with_cancellation, ) diff --git a/vllm/entrypoints/cohere/serving.py b/vllm/entrypoints/cohere/serving.py index 3873f70433..b285319e7e 100644 --- a/vllm/entrypoints/cohere/serving.py +++ b/vllm/entrypoints/cohere/serving.py @@ -86,7 +86,7 @@ from vllm.entrypoints.openai.engine.protocol import ( StreamOptions, ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.entrypoints.serve.utils.api_utils import sanitize_message +from vllm.entrypoints.serve.exception_handling.utils import sanitize_message from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.parser.abstract_parser import Parser from vllm.renderers.cohere import MESSAGES_CITATIONS_KEY, POSITION_TO_SOURCE_KEY diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 5a6dd8c83d..808420d191 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -16,8 +16,7 @@ from contextlib import asynccontextmanager from typing import Any, cast import uvloop -from fastapi import FastAPI, HTTPException -from fastapi.exceptions import RequestValidationError +from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from starlette.datastructures import State @@ -31,6 +30,7 @@ from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_se from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware +from vllm.entrypoints.serve.exception_handling.register import init_exception_handler from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.entrypoints.serve.utils.api_utils import ( @@ -41,15 +41,10 @@ from vllm.entrypoints.serve.utils.api_utils import ( ) from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.entrypoints.serve.utils.server_utils import ( - exception_handler, get_uvicorn_log_config, - http_exception_handler, lifespan, log_response, - validation_exception_handler, - vllm_error_handler, ) -from vllm.exceptions import VLLMError from vllm.logger import init_logger from vllm.reasoning import ReasoningParserManager from vllm.renderers.online_derenderer import OnlineDerenderer @@ -275,6 +270,8 @@ def build_app( # `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`). _attach_endpoint_plugins(app, supported_tasks) + init_exception_handler(app) + app.root_path = args.root_path app.add_middleware( CORSMiddleware, @@ -284,28 +281,6 @@ def build_app( allow_headers=args.allowed_headers, ) - # Exception handlers are registered in four layers: - # 1. framework errors raised by FastAPI/Starlette - # 2. vLLM-specific errors dispatched via a single ``VLLMError`` handler - # 3. fallback handlers for raw exceptions not yet migrated to ``VLLMError`` - # 4. the raw ``Exception`` handler as a safety net - # Registering specific exception types (rather than only ``Exception``) - # ensures they are handled by ``ExceptionMiddleware`` (inside the Prometheus - # middleware) rather than ``ServerErrorMiddleware`` (outside it), so their - # status codes are recorded correctly. - app.exception_handler(HTTPException)(http_exception_handler) - app.exception_handler(RequestValidationError)(validation_exception_handler) - - app.exception_handler(VLLMError)(vllm_error_handler) - - # TODO(zqzten): remove these fallback handlers after migration to VLLMError - app.exception_handler(ValueError)(exception_handler) - app.exception_handler(TypeError)(exception_handler) - app.exception_handler(OverflowError)(exception_handler) - app.exception_handler(NotImplementedError)(exception_handler) - - app.exception_handler(Exception)(exception_handler) - # Ensure --api-key option from CLI takes precedence over VLLM_API_KEY if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]: from vllm.entrypoints.serve.utils.server_utils import AuthenticationMiddleware diff --git a/vllm/entrypoints/openai/models/serving.py b/vllm/entrypoints/openai/models/serving.py index b886d92641..9ceb8c7dfb 100644 --- a/vllm/entrypoints/openai/models/serving.py +++ b/vllm/entrypoints/openai/models/serving.py @@ -14,11 +14,11 @@ from vllm.entrypoints.openai.engine.protocol import ( ModelPermission, ) from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath +from vllm.entrypoints.serve import create_error_response from vllm.entrypoints.serve.lora.protocol import ( LoadLoRAAdapterRequest, UnloadLoRAAdapterRequest, ) -from vllm.entrypoints.serve.utils.error_response import create_error_response from vllm.exceptions import LoRAAdapterNotFoundError from vllm.logger import init_logger from vllm.lora.request import LoRARequest diff --git a/vllm/entrypoints/openai/run_batch.py b/vllm/entrypoints/openai/run_batch.py index 6ae608da0a..e9b7c582d6 100644 --- a/vllm/entrypoints/openai/run_batch.py +++ b/vllm/entrypoints/openai/run_batch.py @@ -52,7 +52,7 @@ from vllm.entrypoints.pooling.scoring.protocol import ( ScoreRequest, ScoreResponse, ) -from vllm.entrypoints.serve.utils.error_response import create_error_response +from vllm.entrypoints.serve import create_error_response from vllm.entrypoints.speech_to_text.transcription.protocol import ( TranscriptionRequest, TranscriptionResponse, diff --git a/vllm/entrypoints/serve/__init__.py b/vllm/entrypoints/serve/__init__.py index 57491d45f6..c358b9a6db 100644 --- a/vllm/entrypoints/serve/__init__.py +++ b/vllm/entrypoints/serve/__init__.py @@ -5,8 +5,16 @@ from fastapi import FastAPI from vllm.logger import init_logger +from .exception_handling.error_response import create_error_response + logger = init_logger(__name__) +__all__ = [ + "create_error_response", + "register_vllm_serve_api_routers", + "register_vllm_dev_api_routers", +] + def register_vllm_serve_api_routers(app: FastAPI): from .instrumentator import register_instrumentator_api_routers diff --git a/vllm/entrypoints/serve/engine/serving.py b/vllm/entrypoints/serve/engine/serving.py index 5908e20f71..a31e47d9c5 100644 --- a/vllm/entrypoints/serve/engine/serving.py +++ b/vllm/entrypoints/serve/engine/serving.py @@ -12,8 +12,8 @@ from vllm.entrypoints.openai.models.serving import ( OpenAIServingModels, ) from vllm.entrypoints.pooling.typing import AnyPoolingRequest +from vllm.entrypoints.serve import create_error_response from vllm.entrypoints.serve.engine.typing import AnyRequest -from vllm.entrypoints.serve.utils.error_response import create_error_response from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.exceptions import VLLMNotFoundError from vllm.inputs import EngineInput diff --git a/vllm/entrypoints/serve/exception_handling/__init__.py b/vllm/entrypoints/serve/exception_handling/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/entrypoints/serve/utils/error_response.py b/vllm/entrypoints/serve/exception_handling/error_response.py similarity index 97% rename from vllm/entrypoints/serve/utils/error_response.py rename to vllm/entrypoints/serve/exception_handling/error_response.py index 2aa785c53b..bb31c1d335 100644 --- a/vllm/entrypoints/serve/utils/error_response.py +++ b/vllm/entrypoints/serve/exception_handling/error_response.py @@ -7,9 +7,10 @@ from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, GenerationError, ) -from vllm.entrypoints.serve.utils.api_utils import sanitize_message from vllm.logger import init_logger +from .utils import sanitize_message + logger = init_logger(__name__) diff --git a/vllm/entrypoints/serve/exception_handling/handlers/__init__.py b/vllm/entrypoints/serve/exception_handling/handlers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/entrypoints/serve/exception_handling/handlers/exception.py b/vllm/entrypoints/serve/exception_handling/handlers/exception.py new file mode 100644 index 0000000000..5afef771ab --- /dev/null +++ b/vllm/entrypoints/serve/exception_handling/handlers/exception.py @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from fastapi import Request +from starlette.responses import JSONResponse + +from vllm.logger import init_logger + +from ..error_response import create_error_response + +logger = init_logger(__name__) + + +async def exception_handler(req: Request, exc: Exception): + if req.app.state.args.log_error_stack: + logger.error( + "Exception caught. Request id: %s", + req.state.request_metadata.request_id + if hasattr(req.state, "request_metadata") + else None, + ) + + err = create_error_response(exc) + return JSONResponse(err.model_dump(), status_code=err.error.code) diff --git a/vllm/entrypoints/serve/exception_handling/handlers/http.py b/vllm/entrypoints/serve/exception_handling/handlers/http.py new file mode 100644 index 0000000000..d838c0ae10 --- /dev/null +++ b/vllm/entrypoints/serve/exception_handling/handlers/http.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from http import HTTPStatus + +from fastapi import HTTPException, Request +from starlette.responses import JSONResponse + +from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse +from vllm.logger import init_logger + +from ..utils import sanitize_message + +logger = init_logger(__name__) + + +async def http_exception_handler(req: Request, exc: HTTPException): + if req.app.state.args.log_error_stack: + logger.exception( + "HTTPException caught. Request id: %s", + req.state.request_metadata.request_id + if hasattr(req.state, "request_metadata") + else None, + ) + err = ErrorResponse( + error=ErrorInfo( + message=sanitize_message(exc.detail), + type=HTTPStatus(exc.status_code).phrase, + code=exc.status_code, + ) + ) + return JSONResponse(err.model_dump(), status_code=exc.status_code) diff --git a/vllm/entrypoints/serve/exception_handling/handlers/validation.py b/vllm/entrypoints/serve/exception_handling/handlers/validation.py new file mode 100644 index 0000000000..fc02389ef6 --- /dev/null +++ b/vllm/entrypoints/serve/exception_handling/handlers/validation.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from http import HTTPStatus + +import regex as re +from fastapi import Request +from fastapi.exceptions import RequestValidationError +from starlette.responses import JSONResponse + +from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse +from vllm.exceptions import VLLMValidationError +from vllm.logger import init_logger + +from ..utils import sanitize_message + +logger = init_logger(__name__) + + +_BRACKETED_INTERNAL_RE = re.compile(r"[\[\]{}()]") + +# NOTE: this list is pydantic-core's internal schema-kind vocabulary, +# not a stable public API -- it can grow when pydantic-core adds new +# wrapper/validator kinds. To refresh it after a pydantic upgrade: +# 1. Fuzz the validation-error-prone endpoints (e.g. /tokenize, +# /v1/completions, /v1/chat/completions) with deliberately +# malformed values for union-typed and wrapped fields (e.g. `stop`, +# `prompt`), and inspect the raw `loc` tuples in the response. +# 2. Any *unbracketed* segment that isn't a real field name or list +# index is a new internal marker -- add it here. Bracketed/ +# parenthesized markers (e.g. "list[...]", "function-wrap[...]") +# are already caught structurally by _BRACKETED_INTERNAL_RE and +# don't need a list entry. +# 3. pydantic-core's source (the `error.rs`/schema-kind definitions +# in the pydantic-core Rust crate) is the canonical reference if +# you want to check before it shows up in a live fuzz run. +_INTERNAL_LOC_MARKERS = frozenset( + { + "function-wrap", + "function-after", + "function-before", + "function-plain", + "json-or-python", + "lax-or-strict", + "chain", + "default", + "nullable", + "tagged-union", + "union", + "call", + "arguments", + "is-instance", + "is-subclass", + "callable", + "str", + "int", + "float", + "bool", + "bytes", + "bytearray", + "list", + "tuple", + "dict", + "set", + "frozenset", + "complex", + "none", + "nonetype", + } +) + + +def _is_internal_loc_segment(segment: str) -> bool: + """True if `segment` is a Pydantic-internal wrapper/union-branch + marker rather than a user-meaningful field name or list index.""" + if _BRACKETED_INTERNAL_RE.search(segment): + return True + return segment.lower() in _INTERNAL_LOC_MARKERS + + +def clean_loc_for_param(loc: tuple) -> str: + """Join a Pydantic error `loc` tuple into a clean dotted `param` + path, dropping internal wrapper/union-branch markers that don't + correspond to a real field name an API consumer would recognize. + + E.g. ('body', 'function-wrap[__log_extra_fields__()]', 'prompt') + -> "body.prompt", not "body.function-wrap[__log_extra_fields__()].prompt". + """ + parts = [str(p) for p in loc if not _is_internal_loc_segment(str(p))] + if not parts: + return ".".join(str(p) for p in loc) + return ".".join(parts) + + +async def validation_exception_handler(req: Request, exc: RequestValidationError): + if req.app.state.args.log_error_stack: + logger.exception( + "RequestValidationError caught. Request id: %s", + req.state.request_metadata.request_id + if hasattr(req.state, "request_metadata") + else None, + ) + + param = None + errors = exc.errors() + for error in errors: + if "ctx" in error and "error" in error["ctx"]: + ctx_error = error["ctx"]["error"] + if isinstance(ctx_error, VLLMValidationError): + param = ctx_error.parameter + break + + if param is None and errors: + first_error = errors[0] + loc = first_error.get("loc") if isinstance(first_error, dict) else None + if loc: + param = clean_loc_for_param(loc) + + # Build the message from exc.errors() instead of str(exc) - str(exc) + # leaks the server's file path via FastAPI's endpoint context. + if errors: + count = len(errors) + label = "error" if count == 1 else "errors" + message = f"{count} validation {label}:\n" + message += "".join(f" {err}\n" for err in errors) + message = message.rstrip() + else: + message = "Validation error" + + err = ErrorResponse( + error=ErrorInfo( + message=sanitize_message(message), + type=HTTPStatus.BAD_REQUEST.phrase, + code=HTTPStatus.BAD_REQUEST, + param=param, + ) + ) + return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST) diff --git a/vllm/entrypoints/serve/exception_handling/handlers/vllm_error.py b/vllm/entrypoints/serve/exception_handling/handlers/vllm_error.py new file mode 100644 index 0000000000..8ac9756f3d --- /dev/null +++ b/vllm/entrypoints/serve/exception_handling/handlers/vllm_error.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from fastapi import Request +from starlette.responses import JSONResponse + +from vllm.entrypoints.launcher import terminate_if_errored +from vllm.entrypoints.openai.engine.protocol import GenerationError +from vllm.exceptions import VLLMError +from vllm.logger import init_logger +from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError + +from ..error_response import create_error_response +from .exception import exception_handler + +logger = init_logger(__name__) + + +async def vllm_error_handler(req: Request, exc: VLLMError): + """Dispatch a vLLM-specific error to the appropriate handler.""" + if isinstance(exc, (EngineGenerateError, EngineDeadError)): + return await engine_error_handler(req, exc) + elif isinstance(exc, GenerationError): + return await generation_error_handler(req, exc) + else: + return await exception_handler(req, exc) + + +async def engine_error_handler( + req: Request, exc: EngineDeadError | EngineGenerateError +): + """ + VLLM V1 AsyncLLM catches exceptions and returns + only two types: EngineGenerateError and EngineDeadError. + + EngineGenerateError is raised by the per request generate() + method. This error could be request specific (and therefore + recoverable - e.g. if there is an error in input processing). + + EngineDeadError is raised by the background output_handler + method. This error is global and therefore not recoverable. + + We register these @app.exception_handlers to return nice + responses to the end user if they occur and shut down if needed. + See https://fastapi.tiangolo.com/tutorial/handling-errors/ + for more details on how exception handlers work. + + If an exception is encountered in a StreamingResponse + generator, the exception is not raised, since we already sent + a 200 status. Rather, we send an error message as the next chunk. + Since the exception is not raised, this means that the server + will not automatically shut down. Instead, we use the watchdog + background task for check for errored state. + """ + + if req.app.state.args.log_error_stack: + logger.exception( + "Engine Exception caught. Request id: %s", + req.state.request_metadata.request_id + if hasattr(req.state, "request_metadata") + else None, + ) + + terminate_if_errored( + server=req.app.state.server, + engine=req.app.state.engine_client, + ) + err = create_error_response(exc) + return JSONResponse(err.model_dump(), status_code=err.error.code) + + +async def generation_error_handler(req: Request, exc: GenerationError): + """Handle GenerationError without logging stack traces. + + GenerationError is a known, expected error (e.g. KV cache load failure) + that should be returned to the client as a 500 response without polluting + server logs with stack traces. + """ + err = create_error_response(exc) + return JSONResponse(err.model_dump(), status_code=err.error.code) diff --git a/vllm/entrypoints/serve/exception_handling/register.py b/vllm/entrypoints/serve/exception_handling/register.py new file mode 100644 index 0000000000..9f94873939 --- /dev/null +++ b/vllm/entrypoints/serve/exception_handling/register.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +vLLM Exception handlers are registered in four layers: +1. framework errors raised by FastAPI/Starlette +2. vLLM-specific errors dispatched via a single ``VLLMError`` handler +3. fallback handlers for raw exceptions not yet migrated to ``VLLMError`` +4. the raw ``Exception`` handler as a safety net +Registering specific exception types (rather than only ``Exception``) +ensures they are handled by ``ExceptionMiddleware`` (inside the Prometheus +middleware) rather than ``ServerErrorMiddleware`` (outside it), so their +status codes are recorded correctly. +""" + +from fastapi import FastAPI, HTTPException +from fastapi.exceptions import RequestValidationError + +from vllm.exceptions import VLLMError + +from .handlers.exception import exception_handler +from .handlers.http import http_exception_handler +from .handlers.validation import validation_exception_handler +from .handlers.vllm_error import vllm_error_handler + + +def init_exception_handler(app: FastAPI): + # 1. framework errors raised by FastAPI/Starlette + app.exception_handler(HTTPException)(http_exception_handler) + app.exception_handler(RequestValidationError)(validation_exception_handler) + + # 2. vLLM-specific errors dispatched via a single ``VLLMError`` handler + app.exception_handler(VLLMError)(vllm_error_handler) + + # 3. fallback handlers for raw exceptions not yet migrated to ``VLLMError`` + # TODO(zqzten): remove these fallback handlers after migration to VLLMError + app.exception_handler(ValueError)(exception_handler) + app.exception_handler(TypeError)(exception_handler) + app.exception_handler(OverflowError)(exception_handler) + app.exception_handler(NotImplementedError)(exception_handler) + + # 4. the raw ``Exception`` handler as a safety net + app.exception_handler(Exception)(exception_handler) diff --git a/vllm/entrypoints/serve/exception_handling/utils.py b/vllm/entrypoints/serve/exception_handling/utils.py new file mode 100644 index 0000000000..73d9992237 --- /dev/null +++ b/vllm/entrypoints/serve/exception_handling/utils.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import regex as re + + +def sanitize_message(message: str) -> str: + """Strip memory addresses, tracebacks, and file paths from error messages.""" + message = re.sub(r" at 0x[0-9a-f]+>", ">", message) + message = re.sub(r'\n?\s*File "[^"]+", line \d+, in \S+(\n\s+.*)?', "", message) + message = re.sub( + r"/(?:home|usr|opt|var|tmp|root|lib|mnt|srv)(?:/[\w.\-]+)+", "", message + ) + message = re.sub(r"(?:/[\w\-]+)+/[\w\-]+\.\w+", "", message) + return message.strip() diff --git a/vllm/entrypoints/serve/utils/api_utils.py b/vllm/entrypoints/serve/utils/api_utils.py index 5fc855d9b5..c75cd16e54 100644 --- a/vllm/entrypoints/serve/utils/api_utils.py +++ b/vllm/entrypoints/serve/utils/api_utils.py @@ -10,7 +10,6 @@ from logging import Logger from string import Template from typing import Any -import regex as re from fastapi import Request from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse, StreamingResponse @@ -309,17 +308,6 @@ def process_lora_modules( return lora_modules -def sanitize_message(message: str) -> str: - """Strip memory addresses, tracebacks, and file paths from error messages.""" - message = re.sub(r" at 0x[0-9a-f]+>", ">", message) - message = re.sub(r'\n?\s*File "[^"]+", line \d+, in \S+(\n\s+.*)?', "", message) - message = re.sub( - r"/(?:home|usr|opt|var|tmp|root|lib|mnt|srv)(?:/[\w.\-]+)+", "", message - ) - message = re.sub(r"(?:/[\w\-]+)+/[\w\-]+\.\w+", "", message) - return message.strip() - - def log_version_and_model(lgr: Logger, version: str, model_name: str) -> None: if envs.VLLM_DISABLE_LOG_LOGO or (formatter := current_formatter_type(lgr)) is None: message = "vLLM server version %s, serving model %s" diff --git a/vllm/entrypoints/serve/utils/server_utils.py b/vllm/entrypoints/serve/utils/server_utils.py index 03e02e34f7..9eba5d3a67 100644 --- a/vllm/entrypoints/serve/utils/server_utils.py +++ b/vllm/entrypoints/serve/utils/server_utils.py @@ -8,12 +8,9 @@ import uuid from argparse import Namespace from collections.abc import Awaitable from contextlib import asynccontextmanager -from http import HTTPStatus import pydantic -import regex as re -from fastapi import FastAPI, HTTPException, Request -from fastapi.exceptions import RequestValidationError +from fastapi import FastAPI, Request from fastapi.responses import JSONResponse from starlette.concurrency import iterate_in_threadpool from starlette.datastructures import Headers, MutableHeaders @@ -21,20 +18,8 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send from vllm import envs from vllm.engine.protocol import EngineClient -from vllm.entrypoints.launcher import terminate_if_errored -from vllm.entrypoints.openai.engine.protocol import ( - ErrorInfo, - ErrorResponse, - GenerationError, -) -from vllm.entrypoints.serve.utils.error_response import ( - create_error_response, - sanitize_message, -) -from vllm.exceptions import VLLMError, VLLMValidationError from vllm.logger import init_logger from vllm.utils.gc_utils import freeze_gc_heap -from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError logger = init_logger("vllm.entrypoints.openai.server_utils") @@ -325,222 +310,6 @@ async def log_response(request: Request, call_next): return response -async def vllm_error_handler(req: Request, exc: VLLMError): - """Dispatch a vLLM-specific error to the appropriate handler.""" - if isinstance(exc, (EngineGenerateError, EngineDeadError)): - return await engine_error_handler(req, exc) - elif isinstance(exc, GenerationError): - return await generation_error_handler(req, exc) - else: - return await exception_handler(req, exc) - - -async def engine_error_handler( - req: Request, exc: EngineDeadError | EngineGenerateError -): - """ - VLLM V1 AsyncLLM catches exceptions and returns - only two types: EngineGenerateError and EngineDeadError. - - EngineGenerateError is raised by the per request generate() - method. This error could be request specific (and therefore - recoverable - e.g. if there is an error in input processing). - - EngineDeadError is raised by the background output_handler - method. This error is global and therefore not recoverable. - - We register these @app.exception_handlers to return nice - responses to the end user if they occur and shut down if needed. - See https://fastapi.tiangolo.com/tutorial/handling-errors/ - for more details on how exception handlers work. - - If an exception is encountered in a StreamingResponse - generator, the exception is not raised, since we already sent - a 200 status. Rather, we send an error message as the next chunk. - Since the exception is not raised, this means that the server - will not automatically shut down. Instead, we use the watchdog - background task for check for errored state. - """ - - if req.app.state.args.log_error_stack: - logger.exception( - "Engine Exception caught. Request id: %s", - req.state.request_metadata.request_id - if hasattr(req.state, "request_metadata") - else None, - ) - - terminate_if_errored( - server=req.app.state.server, - engine=req.app.state.engine_client, - ) - err = create_error_response(exc) - return JSONResponse(err.model_dump(), status_code=err.error.code) - - -async def generation_error_handler(req: Request, exc: GenerationError): - """Handle GenerationError without logging stack traces. - - GenerationError is a known, expected error (e.g. KV cache load failure) - that should be returned to the client as a 500 response without polluting - server logs with stack traces. - """ - err = create_error_response(exc) - return JSONResponse(err.model_dump(), status_code=err.error.code) - - -async def exception_handler(req: Request, exc: Exception): - if req.app.state.args.log_error_stack: - logger.error( - "Exception caught. Request id: %s", - req.state.request_metadata.request_id - if hasattr(req.state, "request_metadata") - else None, - ) - - err = create_error_response(exc) - return JSONResponse(err.model_dump(), status_code=err.error.code) - - -async def http_exception_handler(req: Request, exc: HTTPException): - if req.app.state.args.log_error_stack: - logger.exception( - "HTTPException caught. Request id: %s", - req.state.request_metadata.request_id - if hasattr(req.state, "request_metadata") - else None, - ) - err = ErrorResponse( - error=ErrorInfo( - message=sanitize_message(exc.detail), - type=HTTPStatus(exc.status_code).phrase, - code=exc.status_code, - ) - ) - return JSONResponse(err.model_dump(), status_code=exc.status_code) - - -_BRACKETED_INTERNAL_RE = re.compile(r"[\[\]{}()]") - -# NOTE: this list is pydantic-core's internal schema-kind vocabulary, -# not a stable public API -- it can grow when pydantic-core adds new -# wrapper/validator kinds. To refresh it after a pydantic upgrade: -# 1. Fuzz the validation-error-prone endpoints (e.g. /tokenize, -# /v1/completions, /v1/chat/completions) with deliberately -# malformed values for union-typed and wrapped fields (e.g. `stop`, -# `prompt`), and inspect the raw `loc` tuples in the response. -# 2. Any *unbracketed* segment that isn't a real field name or list -# index is a new internal marker -- add it here. Bracketed/ -# parenthesized markers (e.g. "list[...]", "function-wrap[...]") -# are already caught structurally by _BRACKETED_INTERNAL_RE and -# don't need a list entry. -# 3. pydantic-core's source (the `error.rs`/schema-kind definitions -# in the pydantic-core Rust crate) is the canonical reference if -# you want to check before it shows up in a live fuzz run. -_INTERNAL_LOC_MARKERS = frozenset( - { - "function-wrap", - "function-after", - "function-before", - "function-plain", - "json-or-python", - "lax-or-strict", - "chain", - "default", - "nullable", - "tagged-union", - "union", - "call", - "arguments", - "is-instance", - "is-subclass", - "callable", - "str", - "int", - "float", - "bool", - "bytes", - "bytearray", - "list", - "tuple", - "dict", - "set", - "frozenset", - "complex", - "none", - "nonetype", - } -) - - -def _is_internal_loc_segment(segment: str) -> bool: - """True if `segment` is a Pydantic-internal wrapper/union-branch - marker rather than a user-meaningful field name or list index.""" - if _BRACKETED_INTERNAL_RE.search(segment): - return True - return segment.lower() in _INTERNAL_LOC_MARKERS - - -def clean_loc_for_param(loc: tuple) -> str: - """Join a Pydantic error `loc` tuple into a clean dotted `param` - path, dropping internal wrapper/union-branch markers that don't - correspond to a real field name an API consumer would recognize. - - E.g. ('body', 'function-wrap[__log_extra_fields__()]', 'prompt') - -> "body.prompt", not "body.function-wrap[__log_extra_fields__()].prompt". - """ - parts = [str(p) for p in loc if not _is_internal_loc_segment(str(p))] - if not parts: - return ".".join(str(p) for p in loc) - return ".".join(parts) - - -async def validation_exception_handler(req: Request, exc: RequestValidationError): - if req.app.state.args.log_error_stack: - logger.exception( - "RequestValidationError caught. Request id: %s", - req.state.request_metadata.request_id - if hasattr(req.state, "request_metadata") - else None, - ) - - param = None - errors = exc.errors() - for error in errors: - if "ctx" in error and "error" in error["ctx"]: - ctx_error = error["ctx"]["error"] - if isinstance(ctx_error, VLLMValidationError): - param = ctx_error.parameter - break - - if param is None and errors: - first_error = errors[0] - loc = first_error.get("loc") if isinstance(first_error, dict) else None - if loc: - param = clean_loc_for_param(loc) - - # Build the message from exc.errors() instead of str(exc) - str(exc) - # leaks the server's file path via FastAPI's endpoint context. - if errors: - count = len(errors) - label = "error" if count == 1 else "errors" - message = f"{count} validation {label}:\n" - message += "".join(f" {err}\n" for err in errors) - message = message.rstrip() - else: - message = "Validation error" - - err = ErrorResponse( - error=ErrorInfo( - message=sanitize_message(message), - type=HTTPStatus.BAD_REQUEST.phrase, - code=HTTPStatus.BAD_REQUEST, - param=param, - ) - ) - return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST) - - _running_tasks: set[asyncio.Task] = set() diff --git a/vllm/entrypoints/speech_to_text/realtime/connection.py b/vllm/entrypoints/speech_to_text/realtime/connection.py index 32f501f104..15d38509c5 100644 --- a/vllm/entrypoints/speech_to_text/realtime/connection.py +++ b/vllm/entrypoints/speech_to_text/realtime/connection.py @@ -14,7 +14,7 @@ from starlette.websockets import WebSocketDisconnect from vllm import envs from vllm.entrypoints.openai.engine.protocol import ErrorResponse, UsageInfo -from vllm.entrypoints.serve.utils.api_utils import sanitize_message +from vllm.entrypoints.serve.exception_handling.utils import sanitize_message from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index 4d3b991135..712869c2bf 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -26,7 +26,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import ( render_for_completion, ) from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.entrypoints.serve.utils.error_response import create_error_response +from vllm.entrypoints.serve import create_error_response from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.inputs import ( EngineInput,