[Frontend] Explain disabled scale-out endpoints with opt-in hints (#61042)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
aoshen02
2026-10-11 08:58:07 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent fcae5c9887
commit 1789f09588
5 changed files with 117 additions and 24 deletions
+1 -1
View File
@@ -178,7 +178,7 @@ When `--api-key` is configured, the following endpoints require Bearer token aut
- `/v1/systemone` - Structured decisions API
- `/v1/load_lora_adapter` - Load a LoRA adapter (can alter model behavior; only available when `--enable-lora` is set and `VLLM_ALLOW_RUNTIME_LORA_UPDATING=True`)
- `/v1/unload_lora_adapter` - Unload a LoRA adapter (can alter model behavior; only available when `--enable-lora` is set and `VLLM_ALLOW_RUNTIME_LORA_UPDATING=True`)
- `/inference/v1/generate` - Generate completions (available when `--enable-scale-out` is set, or with `--tokens-only`)
- `/inference/v1/generate` - Generate completions (available when `--enable-scale-out` is set, or with `--tokens-only`; otherwise returns HTTP 403 with instructions to enable it)
- `/cohere/v2/chat/render` - Render Cohere Chat v2 requests (requires `VLLM_ENABLE_COHERE_API=1` and either `--enable-scale-out` or `--tokens-only`)
- `/v2/embed` - Cohere Embed API
- `/v2/rerank` - Cohere Rerank API
+22 -2
View File
@@ -24,17 +24,36 @@ mod world_size;
use std::sync::Arc;
use axum::Router;
use axum::extract::DefaultBodyLimit;
use axum::http::StatusCode;
use axum::middleware::{from_fn, from_fn_with_state};
use axum::routing::{get, post};
use axum::{Json, Router};
use tower_http::trace::TraceLayer;
use tracing::{info, warn};
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
use crate::middleware;
use crate::routes::openai::utils::types::{ErrorDetail, ErrorResponse};
use crate::state::AppState;
const SCALE_OUT_DISABLED_MESSAGE: &str =
"Scale-out endpoints are disabled. Restart vllm serve with --enable-scale-out to enable them.";
async fn disabled_scale_out() -> (StatusCode, Json<ErrorResponse>) {
(
StatusCode::FORBIDDEN,
Json(ErrorResponse {
error: ErrorDetail {
message: SCALE_OUT_DISABLED_MESSAGE.to_string(),
error_type: "PermissionDeniedError".to_string(),
param: None,
code: Some("403".to_string()),
},
}),
)
}
fn server_dev_mode_enabled() -> bool {
std::env::var("VLLM_SERVER_DEV_MODE")
.ok()
@@ -105,7 +124,8 @@ fn build_router_with_options(
if scale_out_endpoints_enabled {
router = router.route("/inference/v1/generate", post(inference::generate));
} else {
info!("scale-out endpoints are disabled; pass --enable-scale-out to enable them");
info!("{SCALE_OUT_DISABLED_MESSAGE}");
router = router.route("/inference/v1/generate", post(disabled_scale_out));
}
if runtime_lora_updating_enabled {
+14 -8
View File
@@ -758,7 +758,7 @@ async fn scale_out_generate_route_follows_api_server_options() {
.await
.expect("call app");
assert_ne!(response.status(), StatusCode::NOT_FOUND);
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -770,10 +770,7 @@ async fn scale_out_generate_route_is_disabled_by_default() {
Arc::new(FakeChatBackend::new()),
)
.await;
let mut app = build_router_with_scale_out_endpoints(
Arc::new(AppState::new(vec!["model".to_string()], chat)),
false,
);
let mut app = build_router(Arc::new(AppState::new(vec!["model".to_string()], chat)));
let response = app
.call(
@@ -781,13 +778,22 @@ async fn scale_out_generate_route_is_disabled_by_default() {
.method("POST")
.uri("/inference/v1/generate")
.header("content-type", "application/json")
.body(Body::from("{}"))
.body(Body::from("not JSON"))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::NOT_FOUND);
assert_eq!(response.status(), StatusCode::FORBIDDEN);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read response");
let error: serde_json::Value = serde_json::from_slice(&body).expect("decode error");
assert_eq!(error["error"]["type"], "PermissionDeniedError");
assert!(
error["error"]["message"]
.as_str()
.expect("error message")
.contains("--enable-scale-out")
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -816,7 +822,7 @@ async fn scale_out_generate_route_can_be_enabled() {
.await
.expect("call app");
assert_ne!(response.status(), StatusCode::NOT_FOUND);
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}
/// Build a dev-mode router backed by a mock engine using a custom ready
+46 -6
View File
@@ -1,13 +1,20 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
from argparse import Namespace
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
from vllm.entrypoints.scale_out.token_in_token_out.api_router import attach_router
from vllm.entrypoints.serve.middleware.authenticate import GUARDED_PREFIX
from vllm.entrypoints.serve.middleware.authenticate import (
GUARDED_PREFIX,
AuthenticationMiddleware,
)
from vllm.tasks import SupportedTask
RENDER_PATHS = {
@@ -41,17 +48,50 @@ def registered_paths(
return {route.path for route in app.routes}
def test_scale_out_routes_are_disabled_by_default():
paths = registered_paths(("generate",))
@pytest.mark.parametrize("path", sorted(SCALE_OUT_PATHS))
@pytest.mark.parametrize("authorized", [False, True])
def test_disabled_scale_out_routes_explain_opt_in_without_parsing(path, authorized):
app = FastAPI()
register_scale_out_api_routers(app, ("generate",))
app.add_middleware(AuthenticationMiddleware, tokens=["secret"])
headers = {"Authorization": "Bearer secret"} if authorized else {}
client = TestClient(app)
response = client.post(path, content="not JSON", headers=headers)
assert SCALE_OUT_PATHS.isdisjoint(paths)
assert response.status_code == (403 if authorized else 401)
if authorized:
assert "--enable-scale-out" in response.json()["error"]["message"]
assert response.json()["error"]["type"] == "PermissionDeniedError"
assert path not in app.openapi()["paths"]
assert client.post("/unknown", headers=headers).status_code == 404
def test_disabled_scale_out_routes_are_logged(caplog):
def test_disabled_scale_out_routes_are_logged_at_info(caplog):
with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"):
registered_paths(("generate",))
assert any("--enable-scale-out" in record.message for record in caplog.records)
assert [
record.levelno
for record in caplog.records
if "--enable-scale-out" in record.message
] == [logging.INFO]
@pytest.mark.parametrize("tokens_only", [False, True])
def test_enabled_generate_routes_use_native_request_validation(tokens_only):
app = FastAPI()
app.state.args = Namespace(
tokens_only=tokens_only, enable_scale_out=not tokens_only
)
register_scale_out_api_routers(app, ("generate",))
response = TestClient(app).post(
GENERATE_PATH,
content="not JSON",
headers={"Content-Type": "application/json"},
)
assert response.status_code == 422
assert response.json()["detail"][0]["type"] == "json_invalid"
def test_scale_out_routes_can_be_enabled_for_generate_server():
+34 -7
View File
@@ -1,11 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
from http import HTTPStatus
from typing import TYPE_CHECKING
from fastapi import FastAPI
from fastapi.responses import JSONResponse
from fastapi.routing import APIRoute
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.exception_handling.error_response import (
create_error_response,
)
from vllm.logger import init_logger
from vllm.tasks import SupportedTask
@@ -66,6 +72,9 @@ def register_scale_out_api_routers(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
):
from .derender.api_router import router as derender_render
from .render.api_router import router as render_render
args = getattr(app.state, "args", None)
# `vllm launch render` and `vllm serve --tokens-only` are dedicated to
# serving these endpoints, so they always register them regardless of
@@ -73,17 +82,35 @@ def register_scale_out_api_routers(
dedicated_mode = "render" in supported_tasks or getattr(args, "tokens_only", False)
enabled = dedicated_mode or getattr(args, "enable_scale_out", False)
if not enabled:
logger.info(
"Scale-out endpoints are disabled. Set --enable-scale-out to enable them."
message = (
"Scale-out endpoints are disabled. Restart vllm serve with "
"--enable-scale-out to enable them."
)
logger.info(message)
async def disabled_scale_out() -> JSONResponse:
error = create_error_response(
message, "PermissionDeniedError", HTTPStatus.FORBIDDEN
)
return JSONResponse(error.model_dump(), status_code=HTTPStatus.FORBIDDEN)
from .token_in_token_out.api_router import router as token_router
for route in (
*render_render.routes,
*derender_render.routes,
*token_router.routes,
):
if isinstance(route, APIRoute):
app.add_api_route(
route.path,
disabled_scale_out,
methods=sorted(route.methods),
include_in_schema=False,
)
return
from .render.api_router import router as render_render
app.include_router(render_render)
from .derender.api_router import router as derender_render
app.include_router(derender_render)
if "generate" in supported_tasks: