diff --git a/tests/entrypoints/serve/middleware/__init__.py b/tests/entrypoints/serve/middleware/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/entrypoints/serve/middleware/test_authentication_middleware.py b/tests/entrypoints/serve/middleware/test_authentication_middleware.py new file mode 100644 index 0000000000..528261e644 --- /dev/null +++ b/tests/entrypoints/serve/middleware/test_authentication_middleware.py @@ -0,0 +1,150 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from argparse import Namespace +from typing import get_args + +import pytest +import regex as re +from fastapi import FastAPI +from starlette.responses import JSONResponse +from starlette.routing import Route +from starlette.testclient import TestClient + +from vllm.entrypoints.launchers.api_server.routers import register_api_routers +from vllm.entrypoints.serve.middleware.authenticate import ( + GUARDED_PREFIX, + AuthenticationMiddleware, +) +from vllm.tasks import POOLING_TASKS, SupportedTask + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def get_all_http_routes(app: FastAPI) -> list[tuple[str, list[str]]]: + """Extract all HTTP routes (path, methods) from the FastAPI app.""" + routes = [] + for route in app.routes: + if not isinstance(route, Route): + continue + path = route.path + methods = list(route.methods or {"GET"}) + routes.append((path, methods)) + return routes + + +def generate_test_path(path_template: str) -> str: + """Replace path parameters (e.g. {response_id}) with 'test'.""" + return re.sub(r"\{[^}]+\}", "test", path_template) + + +def _create_app_with_mock_routes(routes: list[tuple[str, list[str]]]) -> FastAPI: + """Create a FastAPI app with AuthenticationMiddleware and mock endpoints.""" + app = FastAPI() + app.add_middleware(AuthenticationMiddleware, tokens=["valid-token"]) + + async def mock_endpoint(): + return JSONResponse({"status": "ok"}) + + for path_template, methods in routes: + allowed_methods = list(set(methods + ["OPTIONS"])) + app.add_api_route( + path_template, + mock_endpoint, + methods=allowed_methods, + include_in_schema=False, + ) + return app + + +class MockModelConfig: + def __init__(self): + self.hf_config = Namespace() + self.hf_config.num_labels = 1 + + def get_pooling_task(self, supported_tasks: tuple["SupportedTask", ...]): + pooling_tasks = [s for s in supported_tasks if s in POOLING_TASKS] + return pooling_tasks[0] if len(pooling_tasks) > 0 else None + + +@pytest.fixture(params=get_args(SupportedTask)) +def task_routes(request, monkeypatch) -> tuple[str, list[tuple[str, list[str]]]]: + """For each supported task, build an app with only that task's routers, + extract all routes, and return the task name and routes.""" + task = request.param + # Enable development mode to register all routes (including dev-only routes). + monkeypatch.setenv("VLLM_SERVER_DEV_MODE", "1") + + app = FastAPI() + args = Namespace() + app.state = Namespace() + app.state.args = args + + # Register routers for this specific task (development mode already enabled). + register_api_routers( + args, app, supported_tasks=(task,), model_config=MockModelConfig() + ) + + routes = get_all_http_routes(app) + return task, routes + + +# --------------------------------------------------------------------------- +# Tests for auto-discovered routes +# --------------------------------------------------------------------------- + + +def test_auto_discovered_protected_routes_require_auth(task_routes): + """For every auto-discovered route that starts with a guarded prefix, + verify that authentication is enforced.""" + task, routes = task_routes + app = _create_app_with_mock_routes(routes) + client = TestClient(app) + + for path_template, methods in routes: + if not path_template.startswith(GUARDED_PREFIX): + continue + + test_path = generate_test_path(path_template) + test_method = methods[0] if methods else "GET" + + resp = client.request(test_method, test_path) + assert resp.status_code == 401, ( + f"[{task}] {test_method} {test_path} should reject missing token" + ) + + resp = client.request( + test_method, test_path, headers={"Authorization": "Bearer wrong"} + ) + assert resp.status_code == 401, ( + f"[{task}] {test_method} {test_path} should reject invalid token" + ) + + resp = client.request( + test_method, test_path, headers={"Authorization": "Bearer valid-token"} + ) + assert resp.status_code == 200, ( + f"[{task}] {test_method} {test_path} should accept valid token" + ) + + +def test_auto_discovered_unprotected_routes_no_auth(task_routes): + """For every auto-discovered route that does NOT start with a guarded + prefix, verify that no authentication is required.""" + task, routes = task_routes + app = _create_app_with_mock_routes(routes) + client = TestClient(app) + + for path_template, methods in routes: + if path_template.startswith(GUARDED_PREFIX): + continue + + test_path = generate_test_path(path_template) + test_method = methods[0] if methods else "GET" + + resp = client.request(test_method, test_path) + assert resp.status_code == 200, ( + f"[{task}] {test_method} {test_path} should be accessible without token" + ) diff --git a/tests/entrypoints/serve/instrumentator/test_optional_middleware.py b/tests/entrypoints/serve/middleware/test_optional_middleware.py similarity index 100% rename from tests/entrypoints/serve/instrumentator/test_optional_middleware.py rename to tests/entrypoints/serve/middleware/test_optional_middleware.py diff --git a/vllm/entrypoints/launchers/__init__.py b/vllm/entrypoints/launchers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/entrypoints/launchers/api_server/__init__.py b/vllm/entrypoints/launchers/api_server/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/entrypoints/launchers/api_server/routers.py b/vllm/entrypoints/launchers/api_server/routers.py new file mode 100644 index 0000000000..39d670c019 --- /dev/null +++ b/vllm/entrypoints/launchers/api_server/routers.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from argparse import Namespace + +from fastapi import FastAPI + +from vllm import envs +from vllm.config import ModelConfig +from vllm.tasks import POOLING_TASKS, SupportedTask + + +def register_api_routers( + args: Namespace, + app: FastAPI, + supported_tasks: tuple["SupportedTask", ...], + model_config: ModelConfig | None = None, +): + from vllm.entrypoints.serve import register_vllm_serve_api_routers + + register_vllm_serve_api_routers(app) + + from vllm.entrypoints.openai.models.api_router import ( + attach_router as register_models_api_router, + ) + + register_models_api_router(app) + + from vllm.entrypoints.serve.sagemaker.api_router import ( + attach_router as register_sagemaker_api_router, + ) + + register_sagemaker_api_router(app, supported_tasks, model_config) + + if envs.VLLM_SERVER_DEV_MODE: + from vllm.entrypoints.serve import register_vllm_dev_api_routers + + register_vllm_dev_api_routers(app) + + if "generate" in supported_tasks: + from vllm.entrypoints.generate.api_router import ( + register_generate_api_routers, + ) + + register_generate_api_routers(app) + + from vllm.entrypoints.serve.elastic_ep.api_router import ( + attach_router as elastic_ep_attach_router, + ) + + elastic_ep_attach_router(app) + + if "generate" in supported_tasks or "render" in supported_tasks: + from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers + + register_scale_out_api_routers(app, supported_tasks) + + if "transcription" in supported_tasks or "realtime" in supported_tasks: + from vllm.entrypoints.speech_to_text.factories import ( + register_speech_to_text_api_routers, + ) + + register_speech_to_text_api_routers(app, supported_tasks) + + if any(task in POOLING_TASKS for task in supported_tasks): + from vllm.entrypoints.pooling.factories import register_pooling_api_routers + + register_pooling_api_routers(app, supported_tasks, model_config) + + if getattr(args, "enable_fault_tolerance", False): + from vllm.entrypoints.serve.fault_tolerance.api_router import ( + register_fault_tolerance_api_router, + ) + + register_fault_tolerance_api_router(app) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 808420d191..c541f03f44 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio -import importlib -import inspect import multiprocessing import multiprocessing.forkserver as forkserver import os @@ -17,7 +15,6 @@ from typing import Any, cast import uvloop from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware from starlette.datastructures import State import vllm.envs as envs @@ -26,11 +23,12 @@ from vllm.engine.arg_utils import AsyncEngineArgs from vllm.engine.protocol import EngineClient from vllm.entrypoints.chat_utils import load_chat_template from vllm.entrypoints.launcher import serve_http +from vllm.entrypoints.launchers.api_server.routers import register_api_routers from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware from vllm.entrypoints.serve.exception_handling.register import init_exception_handler +from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.entrypoints.serve.utils.api_utils import ( @@ -43,7 +41,6 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.entrypoints.serve.utils.server_utils import ( get_uvicorn_log_config, lifespan, - log_response, ) from vllm.logger import init_logger from vllm.reasoning import ReasoningParserManager @@ -205,64 +202,9 @@ def build_app( else: app = FastAPI(lifespan=lifespan) app.state.args = args + app.root_path = args.root_path - from vllm.entrypoints.serve import register_vllm_serve_api_routers - - register_vllm_serve_api_routers(app) - - from vllm.entrypoints.openai.models.api_router import ( - attach_router as register_models_api_router, - ) - - register_models_api_router(app) - - from vllm.entrypoints.serve.sagemaker.api_router import ( - attach_router as register_sagemaker_api_router, - ) - - register_sagemaker_api_router(app, supported_tasks, model_config) - - if envs.VLLM_SERVER_DEV_MODE: - from vllm.entrypoints.serve import register_vllm_dev_api_routers - - register_vllm_dev_api_routers(app) - - if "generate" in supported_tasks: - from vllm.entrypoints.generate.api_router import ( - register_generate_api_routers, - ) - - register_generate_api_routers(app) - - from vllm.entrypoints.serve.elastic_ep.api_router import ( - attach_router as elastic_ep_attach_router, - ) - - elastic_ep_attach_router(app) - - if "generate" in supported_tasks or "render" in supported_tasks: - from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers - - register_scale_out_api_routers(app, supported_tasks) - - if "transcription" in supported_tasks or "realtime" in supported_tasks: - from vllm.entrypoints.speech_to_text.factories import ( - register_speech_to_text_api_routers, - ) - - register_speech_to_text_api_routers(app, supported_tasks) - - if any(task in POOLING_TASKS for task in supported_tasks): - from vllm.entrypoints.pooling.factories import register_pooling_api_routers - - register_pooling_api_routers(app, supported_tasks, model_config) - - if args.enable_fault_tolerance: - from vllm.entrypoints.serve.fault_tolerance.api_router import ( - register_fault_tolerance_api_router, - ) - - register_fault_tolerance_api_router(app) + register_api_routers(args, app, supported_tasks, model_config) # Endpoint plugins are attached last so their routes are registered after all core # routers. This runs even for the CPU only render server. A plugin eligible for @@ -271,58 +213,7 @@ def build_app( _attach_endpoint_plugins(app, supported_tasks) init_exception_handler(app) - - app.root_path = args.root_path - app.add_middleware( - CORSMiddleware, - allow_origins=args.allowed_origins, - allow_credentials=args.allow_credentials, - allow_methods=args.allowed_methods, - allow_headers=args.allowed_headers, - ) - - # Ensure --api-key option from CLI takes precedence over VLLM_API_KEY - if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]: - from vllm.entrypoints.serve.utils.server_utils import AuthenticationMiddleware - - app.add_middleware(AuthenticationMiddleware, tokens=tokens) - - if args.enable_request_id_headers: - from vllm.entrypoints.serve.utils.server_utils import XRequestIdMiddleware - - app.add_middleware(XRequestIdMiddleware) - - # Add scaling middleware to check for scaling state - app.add_middleware(ScalingMiddleware) - - if "realtime" in supported_tasks: - # Add WebSocket metrics middleware - from vllm.entrypoints.speech_to_text.factories import ( - add_websocket_metrics_middleware, - ) - - add_websocket_metrics_middleware(app) - - if envs.VLLM_DEBUG_LOG_API_SERVER_RESPONSE: - logger.warning( - "CAUTION: Enabling log response in the API Server. " - "This can include sensitive information and should be " - "avoided in production." - ) - app.middleware("http")(log_response) - - for middleware in args.middleware: - module_path, object_name = middleware.rsplit(".", 1) - imported = getattr(importlib.import_module(module_path), object_name) - if inspect.isclass(imported): - app.add_middleware(imported) # type: ignore[arg-type] - elif inspect.iscoroutinefunction(imported): - app.middleware("http")(imported) - else: - raise ValueError( - f"Invalid middleware {middleware}. Must be a function or a class." - ) - + init_entrypoints_middleware(args, app, supported_tasks) app = sagemaker_standards_bootstrap(app) return app diff --git a/vllm/entrypoints/serve/middleware/__init__.py b/vllm/entrypoints/serve/middleware/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/entrypoints/serve/middleware/authenticate.py b/vllm/entrypoints/serve/middleware/authenticate.py new file mode 100644 index 0000000000..e8a61b21ee --- /dev/null +++ b/vllm/entrypoints/serve/middleware/authenticate.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import hashlib +import secrets +from collections.abc import Awaitable + +from starlette.datastructures import Headers +from starlette.responses import JSONResponse +from starlette.types import ASGIApp, Receive, Scope, Send + +GUARDED_PREFIX = ("/v1", "/v2", "/inference", "/cohere") + + +class AuthenticationMiddleware: + """ + Pure ASGI middleware that authenticates each request by checking + if the Authorization Bearer token exists and equals anyof "{api_key}". + + Notes + ----- + There are two cases in which authentication is skipped: + 1. The HTTP method is OPTIONS. + 2. The request path doesn't start with GUARDED_PREFIX (e.g. /health). + """ + + def __init__(self, app: ASGIApp, tokens: list[str]) -> None: + self.app = app + self.api_tokens = [hashlib.sha256(t.encode("utf-8")).digest() for t in tokens] + + def verify_token(self, headers: Headers) -> bool: + authorization_header_value = headers.get("Authorization") + if not authorization_header_value: + return False + + scheme, _, param = authorization_header_value.partition(" ") + if scheme.lower() != "bearer": + return False + + param_hash = hashlib.sha256(param.encode("utf-8")).digest() + + token_match = False + for token_hash in self.api_tokens: + token_match |= secrets.compare_digest(param_hash, token_hash) + + return token_match + + def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]: + if ( + scope["type"] not in ("http", "websocket") + or scope.get("method") == "OPTIONS" + ): + # scope["type"] can be "lifespan" or "startup" for example, + # in which case we don't need to do anything + return self.app(scope, receive, send) + root_path = scope.get("root_path", "") + url_path = scope["path"].removeprefix(root_path) + headers = Headers(scope=scope) + # Type narrow to satisfy mypy. + if url_path.startswith(GUARDED_PREFIX) and not self.verify_token(headers): + response = JSONResponse(content={"error": "Unauthorized"}, status_code=401) + return response(scope, receive, send) + return self.app(scope, receive, send) diff --git a/vllm/entrypoints/serve/middleware/log_response.py b/vllm/entrypoints/serve/middleware/log_response.py new file mode 100644 index 0000000000..e2f5e5bde5 --- /dev/null +++ b/vllm/entrypoints/serve/middleware/log_response.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pydantic +from fastapi import Request +from starlette.concurrency import iterate_in_threadpool + +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +class SSEDecoder: + """Robust Server-Sent Events decoder for streaming responses.""" + + def __init__(self): + self.buffer = "" + self.content_buffer = [] + + def decode_chunk(self, chunk: bytes) -> list[dict]: + """Decode a chunk of SSE data and return parsed events.""" + import json + + try: + chunk_str = chunk.decode("utf-8") + except UnicodeDecodeError: + # Skip malformed chunks + return [] + + self.buffer += chunk_str + events = [] + + # Process complete lines + while "\n" in self.buffer: + line, self.buffer = self.buffer.split("\n", 1) + line = line.rstrip("\r") # Handle CRLF + + if line.startswith("data: "): + data_str = line[6:].strip() + if data_str == "[DONE]": + events.append({"type": "done"}) + elif data_str: + try: + event_data = json.loads(data_str) + events.append({"type": "data", "data": event_data}) + except json.JSONDecodeError: + # Skip malformed JSON + continue + + return events + + def extract_content(self, event_data: dict) -> str: + """Extract content from event data.""" + return _extract_content_from_chunk(event_data) + + def add_content(self, content: str) -> None: + """Add content to the buffer.""" + if content: + self.content_buffer.append(content) + + def get_complete_content(self) -> str: + """Get the complete buffered content.""" + return "".join(self.content_buffer) + + +def _extract_content_from_chunk(chunk_data: dict) -> str: + """Extract content from a streaming response chunk.""" + try: + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionStreamResponse, + ) + from vllm.entrypoints.openai.completion.protocol import ( + CompletionStreamResponse, + ) + + # Try using Completion types for type-safe parsing + if chunk_data.get("object") == "chat.completion.chunk": + chat_response = ChatCompletionStreamResponse.model_validate(chunk_data) + if chat_response.choices and chat_response.choices[0].delta.content: + return chat_response.choices[0].delta.content + elif chunk_data.get("object") == "text_completion": + completion_response = CompletionStreamResponse.model_validate(chunk_data) + if completion_response.choices and completion_response.choices[0].text: + return completion_response.choices[0].text + except pydantic.ValidationError: + # Fallback to manual parsing + if "choices" in chunk_data and chunk_data["choices"]: + choice = chunk_data["choices"][0] + if "delta" in choice and choice["delta"].get("content"): + return choice["delta"]["content"] + elif choice.get("text"): + return choice["text"] + return "" + + +def _log_streaming_response(response, response_body: list) -> None: + """Log streaming response with robust SSE parsing.""" + + sse_decoder = SSEDecoder() + chunk_count = 0 + + def buffered_iterator(): + nonlocal chunk_count + + for chunk in response_body: + chunk_count += 1 + yield chunk + + # Parse SSE events from chunk + events = sse_decoder.decode_chunk(chunk) + + for event in events: + if event["type"] == "data": + content = sse_decoder.extract_content(event["data"]) + sse_decoder.add_content(content) + elif event["type"] == "done": + # Log complete content when done + full_content = sse_decoder.get_complete_content() + if full_content: + # Truncate if too long + if len(full_content) > 2048: + full_content = full_content[:2048] + "...[truncated]" + logger.info( + "response_body={streaming_complete: content=%r, chunks=%d}", + full_content, + chunk_count, + ) + else: + logger.info( + "response_body={streaming_complete: no_content, chunks=%d}", + chunk_count, + ) + return + + response.body_iterator = iterate_in_threadpool(buffered_iterator()) + logger.info("response_body={streaming_started: chunks=%d}", len(response_body)) + + +def _log_non_streaming_response(response_body: list) -> None: + """Log non-streaming response.""" + try: + decoded_body = response_body[0].decode() + logger.info("response_body={%s}", decoded_body) + except UnicodeDecodeError: + logger.info("response_body={}") + + +async def log_response(request: Request, call_next): + response = await call_next(request) + response_body = [section async for section in response.body_iterator] + response.body_iterator = iterate_in_threadpool(iter(response_body)) + # Check if this is a streaming response by looking at content-type + content_type = response.headers.get("content-type", "") + is_streaming = content_type == "text/event-stream; charset=utf-8" + + # Log response body based on type + if not response_body: + logger.info("response_body={}") + elif is_streaming: + _log_streaming_response(response, response_body) + else: + _log_non_streaming_response(response_body) + return response diff --git a/vllm/entrypoints/serve/middleware/register.py b/vllm/entrypoints/serve/middleware/register.py new file mode 100644 index 0000000000..3b132ce088 --- /dev/null +++ b/vllm/entrypoints/serve/middleware/register.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import importlib +import inspect +from argparse import Namespace + +from fastapi import FastAPI +from starlette.middleware.cors import CORSMiddleware + +from vllm import envs +from vllm.logger import init_logger +from vllm.tasks import SupportedTask + +from .log_response import log_response + +logger = init_logger(__name__) + + +def init_entrypoints_middleware( + args: Namespace, + app: FastAPI, + supported_tasks: tuple["SupportedTask", ...], +): + app.add_middleware( + CORSMiddleware, + allow_origins=args.allowed_origins, + allow_credentials=args.allow_credentials, + allow_methods=args.allowed_methods, + allow_headers=args.allowed_headers, + ) + + # Ensure --api-key option from CLI takes precedence over VLLM_API_KEY + if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]: + from .authenticate import AuthenticationMiddleware + + app.add_middleware(AuthenticationMiddleware, tokens=tokens) + + if args.enable_request_id_headers: + from .x_request_id import XRequestIdMiddleware + + app.add_middleware(XRequestIdMiddleware) + + if "generate" in supported_tasks: + # Add scaling middleware to check for scaling state + from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware + + app.add_middleware(ScalingMiddleware) + + if "realtime" in supported_tasks: + # Add WebSocket metrics middleware + from vllm.entrypoints.speech_to_text.realtime.metrics import ( + WebSocketMetricsMiddleware, + ) + + app.add_middleware(WebSocketMetricsMiddleware) + + if envs.VLLM_DEBUG_LOG_API_SERVER_RESPONSE: + logger.warning( + "CAUTION: Enabling log response in the API Server. " + "This can include sensitive information and should be " + "avoided in production." + ) + app.middleware("http")(log_response) + + for middleware in args.middleware: + module_path, object_name = middleware.rsplit(".", 1) + imported = getattr(importlib.import_module(module_path), object_name) + if inspect.isclass(imported): + app.add_middleware(imported) # type: ignore[arg-type] + elif inspect.iscoroutinefunction(imported): + app.middleware("http")(imported) + else: + raise ValueError( + f"Invalid middleware {middleware}. Must be a function or a class." + ) diff --git a/vllm/entrypoints/serve/middleware/x_request_id.py b/vllm/entrypoints/serve/middleware/x_request_id.py new file mode 100644 index 0000000000..02e85306cd --- /dev/null +++ b/vllm/entrypoints/serve/middleware/x_request_id.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import uuid +from collections.abc import Awaitable + +from starlette.datastructures import Headers, MutableHeaders +from starlette.types import ASGIApp, Message, Receive, Scope, Send + + +class XRequestIdMiddleware: + """ + Middleware the set's the X-Request-Id header for each response + to a random uuid4 (hex) value if the header isn't already + present in the request, otherwise use the provided request id. + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]: + if scope["type"] not in ("http", "websocket"): + return self.app(scope, receive, send) + + # Extract the request headers. + request_headers = Headers(scope=scope) + + async def send_with_request_id(message: Message) -> None: + """ + Custom send function to mutate the response headers + and append X-Request-Id to it. + """ + if message["type"] == "http.response.start": + response_headers = MutableHeaders(raw=message["headers"]) + request_id = request_headers.get("X-Request-Id", uuid.uuid4().hex) + response_headers.append("X-Request-Id", request_id) + await send(message) + + return self.app(scope, receive, send_with_request_id) diff --git a/vllm/entrypoints/serve/utils/server_utils.py b/vllm/entrypoints/serve/utils/server_utils.py index 9eba5d3a67..97910d019e 100644 --- a/vllm/entrypoints/serve/utils/server_utils.py +++ b/vllm/entrypoints/serve/utils/server_utils.py @@ -1,20 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio -import hashlib import json -import secrets -import uuid from argparse import Namespace -from collections.abc import Awaitable from contextlib import asynccontextmanager -import pydantic -from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse -from starlette.concurrency import iterate_in_threadpool -from starlette.datastructures import Headers, MutableHeaders -from starlette.types import ASGIApp, Message, Receive, Scope, Send +from fastapi import FastAPI from vllm import envs from vllm.engine.protocol import EngineClient @@ -24,91 +15,6 @@ from vllm.utils.gc_utils import freeze_gc_heap logger = init_logger("vllm.entrypoints.openai.server_utils") -GUARDED_PREFIX = ("/v1", "/v2", "/inference", "/cohere") - - -class AuthenticationMiddleware: - """ - Pure ASGI middleware that authenticates each request by checking - if the Authorization Bearer token exists and equals anyof "{api_key}". - - Notes - ----- - There are two cases in which authentication is skipped: - 1. The HTTP method is OPTIONS. - 2. The request path doesn't start with GUARDED_PREFIX (e.g. /health). - """ - - def __init__(self, app: ASGIApp, tokens: list[str]) -> None: - self.app = app - self.api_tokens = [hashlib.sha256(t.encode("utf-8")).digest() for t in tokens] - - def verify_token(self, headers: Headers) -> bool: - authorization_header_value = headers.get("Authorization") - if not authorization_header_value: - return False - - scheme, _, param = authorization_header_value.partition(" ") - if scheme.lower() != "bearer": - return False - - param_hash = hashlib.sha256(param.encode("utf-8")).digest() - - token_match = False - for token_hash in self.api_tokens: - token_match |= secrets.compare_digest(param_hash, token_hash) - - return token_match - - def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]: - if ( - scope["type"] not in ("http", "websocket") - or scope.get("method") == "OPTIONS" - ): - # scope["type"] can be "lifespan" or "startup" for example, - # in which case we don't need to do anything - return self.app(scope, receive, send) - root_path = scope.get("root_path", "") - url_path = scope["path"].removeprefix(root_path) - headers = Headers(scope=scope) - # Type narrow to satisfy mypy. - if url_path.startswith(GUARDED_PREFIX) and not self.verify_token(headers): - response = JSONResponse(content={"error": "Unauthorized"}, status_code=401) - return response(scope, receive, send) - return self.app(scope, receive, send) - - -class XRequestIdMiddleware: - """ - Middleware the set's the X-Request-Id header for each response - to a random uuid4 (hex) value if the header isn't already - present in the request, otherwise use the provided request id. - """ - - def __init__(self, app: ASGIApp) -> None: - self.app = app - - def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]: - if scope["type"] not in ("http", "websocket"): - return self.app(scope, receive, send) - - # Extract the request headers. - request_headers = Headers(scope=scope) - - async def send_with_request_id(message: Message) -> None: - """ - Custom send function to mutate the response headers - and append X-Request-Id to it. - """ - if message["type"] == "http.response.start": - response_headers = MutableHeaders(raw=message["headers"]) - request_id = request_headers.get("X-Request-Id", uuid.uuid4().hex) - response_headers.append("X-Request-Id", request_id) - await send(message) - - return self.app(scope, receive, send_with_request_id) - - def load_log_config(log_config_file: str | None) -> dict | None: if not log_config_file: return None @@ -155,161 +61,6 @@ def get_uvicorn_log_config(args: Namespace) -> dict | None: return None -def _extract_content_from_chunk(chunk_data: dict) -> str: - """Extract content from a streaming response chunk.""" - try: - from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionStreamResponse, - ) - from vllm.entrypoints.openai.completion.protocol import ( - CompletionStreamResponse, - ) - - # Try using Completion types for type-safe parsing - if chunk_data.get("object") == "chat.completion.chunk": - chat_response = ChatCompletionStreamResponse.model_validate(chunk_data) - if chat_response.choices and chat_response.choices[0].delta.content: - return chat_response.choices[0].delta.content - elif chunk_data.get("object") == "text_completion": - completion_response = CompletionStreamResponse.model_validate(chunk_data) - if completion_response.choices and completion_response.choices[0].text: - return completion_response.choices[0].text - except pydantic.ValidationError: - # Fallback to manual parsing - if "choices" in chunk_data and chunk_data["choices"]: - choice = chunk_data["choices"][0] - if "delta" in choice and choice["delta"].get("content"): - return choice["delta"]["content"] - elif choice.get("text"): - return choice["text"] - return "" - - -class SSEDecoder: - """Robust Server-Sent Events decoder for streaming responses.""" - - def __init__(self): - self.buffer = "" - self.content_buffer = [] - - def decode_chunk(self, chunk: bytes) -> list[dict]: - """Decode a chunk of SSE data and return parsed events.""" - import json - - try: - chunk_str = chunk.decode("utf-8") - except UnicodeDecodeError: - # Skip malformed chunks - return [] - - self.buffer += chunk_str - events = [] - - # Process complete lines - while "\n" in self.buffer: - line, self.buffer = self.buffer.split("\n", 1) - line = line.rstrip("\r") # Handle CRLF - - if line.startswith("data: "): - data_str = line[6:].strip() - if data_str == "[DONE]": - events.append({"type": "done"}) - elif data_str: - try: - event_data = json.loads(data_str) - events.append({"type": "data", "data": event_data}) - except json.JSONDecodeError: - # Skip malformed JSON - continue - - return events - - def extract_content(self, event_data: dict) -> str: - """Extract content from event data.""" - return _extract_content_from_chunk(event_data) - - def add_content(self, content: str) -> None: - """Add content to the buffer.""" - if content: - self.content_buffer.append(content) - - def get_complete_content(self) -> str: - """Get the complete buffered content.""" - return "".join(self.content_buffer) - - -def _log_streaming_response(response, response_body: list) -> None: - """Log streaming response with robust SSE parsing.""" - from starlette.concurrency import iterate_in_threadpool - - sse_decoder = SSEDecoder() - chunk_count = 0 - - def buffered_iterator(): - nonlocal chunk_count - - for chunk in response_body: - chunk_count += 1 - yield chunk - - # Parse SSE events from chunk - events = sse_decoder.decode_chunk(chunk) - - for event in events: - if event["type"] == "data": - content = sse_decoder.extract_content(event["data"]) - sse_decoder.add_content(content) - elif event["type"] == "done": - # Log complete content when done - full_content = sse_decoder.get_complete_content() - if full_content: - # Truncate if too long - if len(full_content) > 2048: - full_content = full_content[:2048] + "" - "...[truncated]" - logger.info( - "response_body={streaming_complete: content=%r, chunks=%d}", - full_content, - chunk_count, - ) - else: - logger.info( - "response_body={streaming_complete: no_content, chunks=%d}", - chunk_count, - ) - return - - response.body_iterator = iterate_in_threadpool(buffered_iterator()) - logger.info("response_body={streaming_started: chunks=%d}", len(response_body)) - - -def _log_non_streaming_response(response_body: list) -> None: - """Log non-streaming response.""" - try: - decoded_body = response_body[0].decode() - logger.info("response_body={%s}", decoded_body) - except UnicodeDecodeError: - logger.info("response_body={}") - - -async def log_response(request: Request, call_next): - response = await call_next(request) - response_body = [section async for section in response.body_iterator] - response.body_iterator = iterate_in_threadpool(iter(response_body)) - # Check if this is a streaming response by looking at content-type - content_type = response.headers.get("content-type", "") - is_streaming = content_type == "text/event-stream; charset=utf-8" - - # Log response body based on type - if not response_body: - logger.info("response_body={}") - elif is_streaming: - _log_streaming_response(response, response_body) - else: - _log_non_streaming_response(response_body) - return response - - _running_tasks: set[asyncio.Task] = set() diff --git a/vllm/entrypoints/speech_to_text/factories.py b/vllm/entrypoints/speech_to_text/factories.py index 1971e32b98..21633b22f3 100644 --- a/vllm/entrypoints/speech_to_text/factories.py +++ b/vllm/entrypoints/speech_to_text/factories.py @@ -37,12 +37,6 @@ def register_speech_to_text_api_routers( app.include_router(translation_router) -def add_websocket_metrics_middleware(app: FastAPI): - from .realtime.metrics import WebSocketMetricsMiddleware - - app.add_middleware(WebSocketMetricsMiddleware) - - def init_speech_to_text_state( engine_client: "EngineClient", state: "State",