[Frontend] Explain disabled scale-out endpoints with opt-in hints (#61042)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
@@ -24,17 +24,36 @@ mod world_size;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Router;
|
||||
use axum::extract::DefaultBodyLimit;
|
||||
use axum::http::StatusCode;
|
||||
use axum::middleware::{from_fn, from_fn_with_state};
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Json, Router};
|
||||
use tower_http::trace::TraceLayer;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
|
||||
use crate::middleware;
|
||||
use crate::routes::openai::utils::types::{ErrorDetail, ErrorResponse};
|
||||
use crate::state::AppState;
|
||||
|
||||
const SCALE_OUT_DISABLED_MESSAGE: &str =
|
||||
"Scale-out endpoints are disabled. Restart vllm serve with --enable-scale-out to enable them.";
|
||||
|
||||
async fn disabled_scale_out() -> (StatusCode, Json<ErrorResponse>) {
|
||||
(
|
||||
StatusCode::FORBIDDEN,
|
||||
Json(ErrorResponse {
|
||||
error: ErrorDetail {
|
||||
message: SCALE_OUT_DISABLED_MESSAGE.to_string(),
|
||||
error_type: "PermissionDeniedError".to_string(),
|
||||
param: None,
|
||||
code: Some("403".to_string()),
|
||||
},
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
fn server_dev_mode_enabled() -> bool {
|
||||
std::env::var("VLLM_SERVER_DEV_MODE")
|
||||
.ok()
|
||||
@@ -105,7 +124,8 @@ fn build_router_with_options(
|
||||
if scale_out_endpoints_enabled {
|
||||
router = router.route("/inference/v1/generate", post(inference::generate));
|
||||
} else {
|
||||
info!("scale-out endpoints are disabled; pass --enable-scale-out to enable them");
|
||||
info!("{SCALE_OUT_DISABLED_MESSAGE}");
|
||||
router = router.route("/inference/v1/generate", post(disabled_scale_out));
|
||||
}
|
||||
|
||||
if runtime_lora_updating_enabled {
|
||||
|
||||
@@ -758,7 +758,7 @@ async fn scale_out_generate_route_follows_api_server_options() {
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_ne!(response.status(), StatusCode::NOT_FOUND);
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
@@ -770,10 +770,7 @@ async fn scale_out_generate_route_is_disabled_by_default() {
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
)
|
||||
.await;
|
||||
let mut app = build_router_with_scale_out_endpoints(
|
||||
Arc::new(AppState::new(vec!["model".to_string()], chat)),
|
||||
false,
|
||||
);
|
||||
let mut app = build_router(Arc::new(AppState::new(vec!["model".to_string()], chat)));
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
@@ -781,13 +778,22 @@ async fn scale_out_generate_route_is_disabled_by_default() {
|
||||
.method("POST")
|
||||
.uri("/inference/v1/generate")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from("{}"))
|
||||
.body(Body::from("not JSON"))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
assert_eq!(response.status(), StatusCode::FORBIDDEN);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read response");
|
||||
let error: serde_json::Value = serde_json::from_slice(&body).expect("decode error");
|
||||
assert_eq!(error["error"]["type"], "PermissionDeniedError");
|
||||
assert!(
|
||||
error["error"]["message"]
|
||||
.as_str()
|
||||
.expect("error message")
|
||||
.contains("--enable-scale-out")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
@@ -816,7 +822,7 @@ async fn scale_out_generate_route_can_be_enabled() {
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_ne!(response.status(), StatusCode::NOT_FOUND);
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
/// Build a dev-mode router backed by a mock engine using a custom ready
|
||||
|
||||
Reference in New Issue
Block a user