diff --git a/docs/usage/security.md b/docs/usage/security.md index ab5aa96bec..50c4149be8 100644 --- a/docs/usage/security.md +++ b/docs/usage/security.md @@ -178,7 +178,7 @@ When `--api-key` is configured, the following endpoints require Bearer token aut - `/v1/systemone` - Structured decisions API - `/v1/load_lora_adapter` - Load a LoRA adapter (can alter model behavior; only available when `--enable-lora` is set and `VLLM_ALLOW_RUNTIME_LORA_UPDATING=True`) - `/v1/unload_lora_adapter` - Unload a LoRA adapter (can alter model behavior; only available when `--enable-lora` is set and `VLLM_ALLOW_RUNTIME_LORA_UPDATING=True`) -- `/inference/v1/generate` - Generate completions (available when `--enable-scale-out` is set, or with `--tokens-only`) +- `/inference/v1/generate` - Generate completions (available when `--enable-scale-out` is set, or with `--tokens-only`; otherwise returns HTTP 403 with instructions to enable it) - `/cohere/v2/chat/render` - Render Cohere Chat v2 requests (requires `VLLM_ENABLE_COHERE_API=1` and either `--enable-scale-out` or `--tokens-only`) - `/v2/embed` - Cohere Embed API - `/v2/rerank` - Cohere Rerank API diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index 28f3557324..251398611b 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -24,17 +24,36 @@ mod world_size; use std::sync::Arc; -use axum::Router; use axum::extract::DefaultBodyLimit; +use axum::http::StatusCode; use axum::middleware::{from_fn, from_fn_with_state}; use axum::routing::{get, post}; +use axum::{Json, Router}; use tower_http::trace::TraceLayer; use tracing::{info, warn}; use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES; use crate::middleware; +use crate::routes::openai::utils::types::{ErrorDetail, ErrorResponse}; use crate::state::AppState; +const SCALE_OUT_DISABLED_MESSAGE: &str = + "Scale-out endpoints are disabled. Restart vllm serve with --enable-scale-out to enable them."; + +async fn disabled_scale_out() -> (StatusCode, Json) { + ( + StatusCode::FORBIDDEN, + Json(ErrorResponse { + error: ErrorDetail { + message: SCALE_OUT_DISABLED_MESSAGE.to_string(), + error_type: "PermissionDeniedError".to_string(), + param: None, + code: Some("403".to_string()), + }, + }), + ) +} + fn server_dev_mode_enabled() -> bool { std::env::var("VLLM_SERVER_DEV_MODE") .ok() @@ -105,7 +124,8 @@ fn build_router_with_options( if scale_out_endpoints_enabled { router = router.route("/inference/v1/generate", post(inference::generate)); } else { - info!("scale-out endpoints are disabled; pass --enable-scale-out to enable them"); + info!("{SCALE_OUT_DISABLED_MESSAGE}"); + router = router.route("/inference/v1/generate", post(disabled_scale_out)); } if runtime_lora_updating_enabled { diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 7ba4bbf0f4..d574e02a92 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -758,7 +758,7 @@ async fn scale_out_generate_route_follows_api_server_options() { .await .expect("call app"); - assert_ne!(response.status(), StatusCode::NOT_FOUND); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -770,10 +770,7 @@ async fn scale_out_generate_route_is_disabled_by_default() { Arc::new(FakeChatBackend::new()), ) .await; - let mut app = build_router_with_scale_out_endpoints( - Arc::new(AppState::new(vec!["model".to_string()], chat)), - false, - ); + let mut app = build_router(Arc::new(AppState::new(vec!["model".to_string()], chat))); let response = app .call( @@ -781,13 +778,22 @@ async fn scale_out_generate_route_is_disabled_by_default() { .method("POST") .uri("/inference/v1/generate") .header("content-type", "application/json") - .body(Body::from("{}")) + .body(Body::from("not JSON")) .expect("build request"), ) .await .expect("call app"); - assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read response"); + let error: serde_json::Value = serde_json::from_slice(&body).expect("decode error"); + assert_eq!(error["error"]["type"], "PermissionDeniedError"); + assert!( + error["error"]["message"] + .as_str() + .expect("error message") + .contains("--enable-scale-out") + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -816,7 +822,7 @@ async fn scale_out_generate_route_can_be_enabled() { .await .expect("call app"); - assert_ne!(response.status(), StatusCode::NOT_FOUND); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); } /// Build a dev-mode router backed by a mock engine using a custom ready diff --git a/tests/entrypoints/scale_out/test_factories.py b/tests/entrypoints/scale_out/test_factories.py index ea96fff876..7937c0d45b 100644 --- a/tests/entrypoints/scale_out/test_factories.py +++ b/tests/entrypoints/scale_out/test_factories.py @@ -1,13 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import logging from argparse import Namespace +import pytest from fastapi import FastAPI +from fastapi.testclient import TestClient from vllm.entrypoints.launchers.api_server.routers import register_api_routers +from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers from vllm.entrypoints.scale_out.token_in_token_out.api_router import attach_router -from vllm.entrypoints.serve.middleware.authenticate import GUARDED_PREFIX +from vllm.entrypoints.serve.middleware.authenticate import ( + GUARDED_PREFIX, + AuthenticationMiddleware, +) from vllm.tasks import SupportedTask RENDER_PATHS = { @@ -41,17 +48,50 @@ def registered_paths( return {route.path for route in app.routes} -def test_scale_out_routes_are_disabled_by_default(): - paths = registered_paths(("generate",)) +@pytest.mark.parametrize("path", sorted(SCALE_OUT_PATHS)) +@pytest.mark.parametrize("authorized", [False, True]) +def test_disabled_scale_out_routes_explain_opt_in_without_parsing(path, authorized): + app = FastAPI() + register_scale_out_api_routers(app, ("generate",)) + app.add_middleware(AuthenticationMiddleware, tokens=["secret"]) + headers = {"Authorization": "Bearer secret"} if authorized else {} + client = TestClient(app) + response = client.post(path, content="not JSON", headers=headers) - assert SCALE_OUT_PATHS.isdisjoint(paths) + assert response.status_code == (403 if authorized else 401) + if authorized: + assert "--enable-scale-out" in response.json()["error"]["message"] + assert response.json()["error"]["type"] == "PermissionDeniedError" + assert path not in app.openapi()["paths"] + assert client.post("/unknown", headers=headers).status_code == 404 -def test_disabled_scale_out_routes_are_logged(caplog): +def test_disabled_scale_out_routes_are_logged_at_info(caplog): with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"): registered_paths(("generate",)) - assert any("--enable-scale-out" in record.message for record in caplog.records) + assert [ + record.levelno + for record in caplog.records + if "--enable-scale-out" in record.message + ] == [logging.INFO] + + +@pytest.mark.parametrize("tokens_only", [False, True]) +def test_enabled_generate_routes_use_native_request_validation(tokens_only): + app = FastAPI() + app.state.args = Namespace( + tokens_only=tokens_only, enable_scale_out=not tokens_only + ) + register_scale_out_api_routers(app, ("generate",)) + response = TestClient(app).post( + GENERATE_PATH, + content="not JSON", + headers={"Content-Type": "application/json"}, + ) + + assert response.status_code == 422 + assert response.json()["detail"][0]["type"] == "json_invalid" def test_scale_out_routes_can_be_enabled_for_generate_server(): diff --git a/vllm/entrypoints/scale_out/factories.py b/vllm/entrypoints/scale_out/factories.py index bd48c709dd..45bb9dbcde 100644 --- a/vllm/entrypoints/scale_out/factories.py +++ b/vllm/entrypoints/scale_out/factories.py @@ -1,11 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from argparse import Namespace +from http import HTTPStatus from typing import TYPE_CHECKING from fastapi import FastAPI +from fastapi.responses import JSONResponse +from fastapi.routing import APIRoute from vllm.engine.protocol import EngineClient +from vllm.entrypoints.serve.exception_handling.error_response import ( + create_error_response, +) from vllm.logger import init_logger from vllm.tasks import SupportedTask @@ -66,6 +72,9 @@ def register_scale_out_api_routers( app: FastAPI, supported_tasks: tuple["SupportedTask", ...], ): + from .derender.api_router import router as derender_render + from .render.api_router import router as render_render + args = getattr(app.state, "args", None) # `vllm launch render` and `vllm serve --tokens-only` are dedicated to # serving these endpoints, so they always register them regardless of @@ -73,17 +82,35 @@ def register_scale_out_api_routers( dedicated_mode = "render" in supported_tasks or getattr(args, "tokens_only", False) enabled = dedicated_mode or getattr(args, "enable_scale_out", False) if not enabled: - logger.info( - "Scale-out endpoints are disabled. Set --enable-scale-out to enable them." + message = ( + "Scale-out endpoints are disabled. Restart vllm serve with " + "--enable-scale-out to enable them." ) + logger.info(message) + + async def disabled_scale_out() -> JSONResponse: + error = create_error_response( + message, "PermissionDeniedError", HTTPStatus.FORBIDDEN + ) + return JSONResponse(error.model_dump(), status_code=HTTPStatus.FORBIDDEN) + + from .token_in_token_out.api_router import router as token_router + + for route in ( + *render_render.routes, + *derender_render.routes, + *token_router.routes, + ): + if isinstance(route, APIRoute): + app.add_api_route( + route.path, + disabled_scale_out, + methods=sorted(route.methods), + include_in_schema=False, + ) return - from .render.api_router import router as render_render - app.include_router(render_render) - - from .derender.api_router import router as derender_render - app.include_router(derender_render) if "generate" in supported_tasks: