[Frontend] Explain disabled scale-out endpoints with opt-in hints (#61042)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
aoshen02
2026-10-11 08:58:07 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent fcae5c9887
commit 1789f09588
5 changed files with 117 additions and 24 deletions
+22 -2
View File
@@ -24,17 +24,36 @@ mod world_size;
use std::sync::Arc;
use axum::Router;
use axum::extract::DefaultBodyLimit;
use axum::http::StatusCode;
use axum::middleware::{from_fn, from_fn_with_state};
use axum::routing::{get, post};
use axum::{Json, Router};
use tower_http::trace::TraceLayer;
use tracing::{info, warn};
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
use crate::middleware;
use crate::routes::openai::utils::types::{ErrorDetail, ErrorResponse};
use crate::state::AppState;
const SCALE_OUT_DISABLED_MESSAGE: &str =
"Scale-out endpoints are disabled. Restart vllm serve with --enable-scale-out to enable them.";
async fn disabled_scale_out() -> (StatusCode, Json<ErrorResponse>) {
(
StatusCode::FORBIDDEN,
Json(ErrorResponse {
error: ErrorDetail {
message: SCALE_OUT_DISABLED_MESSAGE.to_string(),
error_type: "PermissionDeniedError".to_string(),
param: None,
code: Some("403".to_string()),
},
}),
)
}
fn server_dev_mode_enabled() -> bool {
std::env::var("VLLM_SERVER_DEV_MODE")
.ok()
@@ -105,7 +124,8 @@ fn build_router_with_options(
if scale_out_endpoints_enabled {
router = router.route("/inference/v1/generate", post(inference::generate));
} else {
info!("scale-out endpoints are disabled; pass --enable-scale-out to enable them");
info!("{SCALE_OUT_DISABLED_MESSAGE}");
router = router.route("/inference/v1/generate", post(disabled_scale_out));
}
if runtime_lora_updating_enabled {
+14 -8
View File
@@ -758,7 +758,7 @@ async fn scale_out_generate_route_follows_api_server_options() {
.await
.expect("call app");
assert_ne!(response.status(), StatusCode::NOT_FOUND);
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -770,10 +770,7 @@ async fn scale_out_generate_route_is_disabled_by_default() {
Arc::new(FakeChatBackend::new()),
)
.await;
let mut app = build_router_with_scale_out_endpoints(
Arc::new(AppState::new(vec!["model".to_string()], chat)),
false,
);
let mut app = build_router(Arc::new(AppState::new(vec!["model".to_string()], chat)));
let response = app
.call(
@@ -781,13 +778,22 @@ async fn scale_out_generate_route_is_disabled_by_default() {
.method("POST")
.uri("/inference/v1/generate")
.header("content-type", "application/json")
.body(Body::from("{}"))
.body(Body::from("not JSON"))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::NOT_FOUND);
assert_eq!(response.status(), StatusCode::FORBIDDEN);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read response");
let error: serde_json::Value = serde_json::from_slice(&body).expect("decode error");
assert_eq!(error["error"]["type"], "PermissionDeniedError");
assert!(
error["error"]["message"]
.as_str()
.expect("error message")
.contains("--enable-scale-out")
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -816,7 +822,7 @@ async fn scale_out_generate_route_can_be_enabled() {
.await
.expect("call app");
assert_ne!(response.status(), StatusCode::NOT_FOUND);
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}
/// Build a dev-mode router backed by a mock engine using a custom ready