[Frontend] Explain disabled scale-out endpoints with opt-in hints (#61042)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
@@ -178,7 +178,7 @@ When `--api-key` is configured, the following endpoints require Bearer token aut
|
||||
- `/v1/systemone` - Structured decisions API
|
||||
- `/v1/load_lora_adapter` - Load a LoRA adapter (can alter model behavior; only available when `--enable-lora` is set and `VLLM_ALLOW_RUNTIME_LORA_UPDATING=True`)
|
||||
- `/v1/unload_lora_adapter` - Unload a LoRA adapter (can alter model behavior; only available when `--enable-lora` is set and `VLLM_ALLOW_RUNTIME_LORA_UPDATING=True`)
|
||||
- `/inference/v1/generate` - Generate completions (available when `--enable-scale-out` is set, or with `--tokens-only`)
|
||||
- `/inference/v1/generate` - Generate completions (available when `--enable-scale-out` is set, or with `--tokens-only`; otherwise returns HTTP 403 with instructions to enable it)
|
||||
- `/cohere/v2/chat/render` - Render Cohere Chat v2 requests (requires `VLLM_ENABLE_COHERE_API=1` and either `--enable-scale-out` or `--tokens-only`)
|
||||
- `/v2/embed` - Cohere Embed API
|
||||
- `/v2/rerank` - Cohere Rerank API
|
||||
|
||||
@@ -24,17 +24,36 @@ mod world_size;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Router;
|
||||
use axum::extract::DefaultBodyLimit;
|
||||
use axum::http::StatusCode;
|
||||
use axum::middleware::{from_fn, from_fn_with_state};
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Json, Router};
|
||||
use tower_http::trace::TraceLayer;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
|
||||
use crate::middleware;
|
||||
use crate::routes::openai::utils::types::{ErrorDetail, ErrorResponse};
|
||||
use crate::state::AppState;
|
||||
|
||||
const SCALE_OUT_DISABLED_MESSAGE: &str =
|
||||
"Scale-out endpoints are disabled. Restart vllm serve with --enable-scale-out to enable them.";
|
||||
|
||||
async fn disabled_scale_out() -> (StatusCode, Json<ErrorResponse>) {
|
||||
(
|
||||
StatusCode::FORBIDDEN,
|
||||
Json(ErrorResponse {
|
||||
error: ErrorDetail {
|
||||
message: SCALE_OUT_DISABLED_MESSAGE.to_string(),
|
||||
error_type: "PermissionDeniedError".to_string(),
|
||||
param: None,
|
||||
code: Some("403".to_string()),
|
||||
},
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
fn server_dev_mode_enabled() -> bool {
|
||||
std::env::var("VLLM_SERVER_DEV_MODE")
|
||||
.ok()
|
||||
@@ -105,7 +124,8 @@ fn build_router_with_options(
|
||||
if scale_out_endpoints_enabled {
|
||||
router = router.route("/inference/v1/generate", post(inference::generate));
|
||||
} else {
|
||||
info!("scale-out endpoints are disabled; pass --enable-scale-out to enable them");
|
||||
info!("{SCALE_OUT_DISABLED_MESSAGE}");
|
||||
router = router.route("/inference/v1/generate", post(disabled_scale_out));
|
||||
}
|
||||
|
||||
if runtime_lora_updating_enabled {
|
||||
|
||||
@@ -758,7 +758,7 @@ async fn scale_out_generate_route_follows_api_server_options() {
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_ne!(response.status(), StatusCode::NOT_FOUND);
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
@@ -770,10 +770,7 @@ async fn scale_out_generate_route_is_disabled_by_default() {
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
)
|
||||
.await;
|
||||
let mut app = build_router_with_scale_out_endpoints(
|
||||
Arc::new(AppState::new(vec!["model".to_string()], chat)),
|
||||
false,
|
||||
);
|
||||
let mut app = build_router(Arc::new(AppState::new(vec!["model".to_string()], chat)));
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
@@ -781,13 +778,22 @@ async fn scale_out_generate_route_is_disabled_by_default() {
|
||||
.method("POST")
|
||||
.uri("/inference/v1/generate")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from("{}"))
|
||||
.body(Body::from("not JSON"))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
assert_eq!(response.status(), StatusCode::FORBIDDEN);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read response");
|
||||
let error: serde_json::Value = serde_json::from_slice(&body).expect("decode error");
|
||||
assert_eq!(error["error"]["type"], "PermissionDeniedError");
|
||||
assert!(
|
||||
error["error"]["message"]
|
||||
.as_str()
|
||||
.expect("error message")
|
||||
.contains("--enable-scale-out")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
@@ -816,7 +822,7 @@ async fn scale_out_generate_route_can_be_enabled() {
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_ne!(response.status(), StatusCode::NOT_FOUND);
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
/// Build a dev-mode router backed by a mock engine using a custom ready
|
||||
|
||||
@@ -1,13 +1,20 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
from argparse import Namespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
|
||||
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.api_router import attach_router
|
||||
from vllm.entrypoints.serve.middleware.authenticate import GUARDED_PREFIX
|
||||
from vllm.entrypoints.serve.middleware.authenticate import (
|
||||
GUARDED_PREFIX,
|
||||
AuthenticationMiddleware,
|
||||
)
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
RENDER_PATHS = {
|
||||
@@ -41,17 +48,50 @@ def registered_paths(
|
||||
return {route.path for route in app.routes}
|
||||
|
||||
|
||||
def test_scale_out_routes_are_disabled_by_default():
|
||||
paths = registered_paths(("generate",))
|
||||
@pytest.mark.parametrize("path", sorted(SCALE_OUT_PATHS))
|
||||
@pytest.mark.parametrize("authorized", [False, True])
|
||||
def test_disabled_scale_out_routes_explain_opt_in_without_parsing(path, authorized):
|
||||
app = FastAPI()
|
||||
register_scale_out_api_routers(app, ("generate",))
|
||||
app.add_middleware(AuthenticationMiddleware, tokens=["secret"])
|
||||
headers = {"Authorization": "Bearer secret"} if authorized else {}
|
||||
client = TestClient(app)
|
||||
response = client.post(path, content="not JSON", headers=headers)
|
||||
|
||||
assert SCALE_OUT_PATHS.isdisjoint(paths)
|
||||
assert response.status_code == (403 if authorized else 401)
|
||||
if authorized:
|
||||
assert "--enable-scale-out" in response.json()["error"]["message"]
|
||||
assert response.json()["error"]["type"] == "PermissionDeniedError"
|
||||
assert path not in app.openapi()["paths"]
|
||||
assert client.post("/unknown", headers=headers).status_code == 404
|
||||
|
||||
|
||||
def test_disabled_scale_out_routes_are_logged(caplog):
|
||||
def test_disabled_scale_out_routes_are_logged_at_info(caplog):
|
||||
with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"):
|
||||
registered_paths(("generate",))
|
||||
|
||||
assert any("--enable-scale-out" in record.message for record in caplog.records)
|
||||
assert [
|
||||
record.levelno
|
||||
for record in caplog.records
|
||||
if "--enable-scale-out" in record.message
|
||||
] == [logging.INFO]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokens_only", [False, True])
|
||||
def test_enabled_generate_routes_use_native_request_validation(tokens_only):
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(
|
||||
tokens_only=tokens_only, enable_scale_out=not tokens_only
|
||||
)
|
||||
register_scale_out_api_routers(app, ("generate",))
|
||||
response = TestClient(app).post(
|
||||
GENERATE_PATH,
|
||||
content="not JSON",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
assert response.status_code == 422
|
||||
assert response.json()["detail"][0]["type"] == "json_invalid"
|
||||
|
||||
|
||||
def test_scale_out_routes_can_be_enabled_for_generate_server():
|
||||
|
||||
@@ -1,11 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.routing import APIRoute
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
@@ -66,6 +72,9 @@ def register_scale_out_api_routers(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
):
|
||||
from .derender.api_router import router as derender_render
|
||||
from .render.api_router import router as render_render
|
||||
|
||||
args = getattr(app.state, "args", None)
|
||||
# `vllm launch render` and `vllm serve --tokens-only` are dedicated to
|
||||
# serving these endpoints, so they always register them regardless of
|
||||
@@ -73,17 +82,35 @@ def register_scale_out_api_routers(
|
||||
dedicated_mode = "render" in supported_tasks or getattr(args, "tokens_only", False)
|
||||
enabled = dedicated_mode or getattr(args, "enable_scale_out", False)
|
||||
if not enabled:
|
||||
logger.info(
|
||||
"Scale-out endpoints are disabled. Set --enable-scale-out to enable them."
|
||||
message = (
|
||||
"Scale-out endpoints are disabled. Restart vllm serve with "
|
||||
"--enable-scale-out to enable them."
|
||||
)
|
||||
logger.info(message)
|
||||
|
||||
async def disabled_scale_out() -> JSONResponse:
|
||||
error = create_error_response(
|
||||
message, "PermissionDeniedError", HTTPStatus.FORBIDDEN
|
||||
)
|
||||
return JSONResponse(error.model_dump(), status_code=HTTPStatus.FORBIDDEN)
|
||||
|
||||
from .token_in_token_out.api_router import router as token_router
|
||||
|
||||
for route in (
|
||||
*render_render.routes,
|
||||
*derender_render.routes,
|
||||
*token_router.routes,
|
||||
):
|
||||
if isinstance(route, APIRoute):
|
||||
app.add_api_route(
|
||||
route.path,
|
||||
disabled_scale_out,
|
||||
methods=sorted(route.methods),
|
||||
include_in_schema=False,
|
||||
)
|
||||
return
|
||||
|
||||
from .render.api_router import router as render_render
|
||||
|
||||
app.include_router(render_render)
|
||||
|
||||
from .derender.api_router import router as derender_render
|
||||
|
||||
app.include_router(derender_render)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
|
||||
Reference in New Issue
Block a user