[Frontend] Gate scale-out endpoints behind opt-in flag (#54579)

Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
This commit is contained in:
Francisco Javier Arceo
2026-09-02 14:31:14 +08:00
committed by GitHub
parent 300f688321
commit c00091e026
20 changed files with 386 additions and 39 deletions
+42 -4
View File
@@ -26,7 +26,7 @@ use axum::extract::DefaultBodyLimit;
use axum::middleware::{from_fn, from_fn_with_state};
use axum::routing::{get, post};
use tower_http::trace::TraceLayer;
use tracing::warn;
use tracing::{info, warn};
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
use crate::middleware;
@@ -45,12 +45,33 @@ fn runtime_lora_updating_enabled() -> bool {
.is_some_and(|value| matches!(value.trim().to_lowercase().as_str(), "1" | "true"))
}
fn parse_scale_out_endpoints_flag(value: Option<&str>) -> Result<bool, String> {
let Some(value) = value else {
return Ok(false);
};
if value.is_empty() {
return Ok(false);
}
match value.trim() {
"0" => Ok(false),
"1" => Ok(true),
_ => Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string()),
}
}
fn scale_out_endpoints_enabled() -> bool {
let value = std::env::var("VLLM_ENABLE_SCALE_OUT_ENDPOINTS").ok();
parse_scale_out_endpoints_flag(value.as_deref()).unwrap_or_else(|message| panic!("{message}"))
}
/// Build the minimal OpenAI-compatible router for one configured model.
pub fn build_router(state: Arc<AppState>) -> Router {
build_router_with_options(
state,
server_dev_mode_enabled(),
runtime_lora_updating_enabled(),
scale_out_endpoints_enabled(),
)
}
@@ -65,13 +86,22 @@ fn build_router_with_dev_mode_and_lora(
dev_mode_enabled: bool,
runtime_lora_updating_enabled: bool,
) -> Router {
build_router_with_options(state, dev_mode_enabled, runtime_lora_updating_enabled)
build_router_with_options(state, dev_mode_enabled, runtime_lora_updating_enabled, true)
}
#[cfg(test)]
fn build_router_with_scale_out_endpoints(
state: Arc<AppState>,
scale_out_endpoints_enabled: bool,
) -> Router {
build_router_with_options(state, false, false, scale_out_endpoints_enabled)
}
fn build_router_with_options(
state: Arc<AppState>,
dev_mode_enabled: bool,
runtime_lora_updating_enabled: bool,
scale_out_endpoints_enabled: bool,
) -> Router {
let mut router = Router::new()
// Health & monitoring
@@ -85,8 +115,16 @@ fn build_router_with_options(
.route("/v1/chat/completions", post(openai::chat_completions))
// vLLM specific endpoints
.route("/tokenize", post(tokenize::tokenize))
.route("/detokenize", post(tokenize::detokenize))
.route("/inference/v1/generate", post(inference::generate));
.route("/detokenize", post(tokenize::detokenize));
if scale_out_endpoints_enabled {
router = router.route("/inference/v1/generate", post(inference::generate));
} else {
info!(
"scale-out endpoints are disabled; set \
VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 to enable them"
);
}
if runtime_lora_updating_enabled {
router = router
+103 -16
View File
@@ -54,6 +54,7 @@ use zeromq::{DealerSocket, PushSocket, ZmqMessage};
use super::{
build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora,
build_router_with_scale_out_endpoints, parse_scale_out_endpoints_flag,
render::build_router as build_render_router,
};
use crate::config::{ApiServerOptions, CorsConfig};
@@ -716,6 +717,83 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
)
}
#[test]
fn scale_out_flag_accepts_unset_empty_zero_one_and_whitespace() {
assert_eq!(parse_scale_out_endpoints_flag(None), Ok(false));
assert_eq!(parse_scale_out_endpoints_flag(Some("")), Ok(false));
assert_eq!(parse_scale_out_endpoints_flag(Some("0")), Ok(false));
assert_eq!(parse_scale_out_endpoints_flag(Some("1")), Ok(true));
assert_eq!(parse_scale_out_endpoints_flag(Some(" 1 ")), Ok(true));
}
#[test]
fn scale_out_flag_rejects_values_other_than_zero_or_one() {
for value in ["-1", "2", "01", "+1", "invalid", " "] {
assert_eq!(
parse_scale_out_endpoints_flag(Some(value)),
Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string())
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn scale_out_generate_route_is_disabled_by_default() {
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
b"engine-scale-out-disabled",
default_stream_output_specs(),
Arc::new(FakeChatBackend::new()),
)
.await;
let mut app = build_router_with_scale_out_endpoints(
Arc::new(AppState::new(vec!["model".to_string()], chat)),
false,
);
let response = app
.call(
Request::builder()
.method("POST")
.uri("/inference/v1/generate")
.header("content-type", "application/json")
.body(Body::from("{}"))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn scale_out_generate_route_can_be_enabled() {
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
b"engine-scale-out-enabled",
default_stream_output_specs(),
Arc::new(FakeChatBackend::new()),
)
.await;
let mut app = build_router_with_scale_out_endpoints(
Arc::new(AppState::new(vec!["model".to_string()], chat)),
true,
);
let response = app
.call(
Request::builder()
.method("POST")
.uri("/inference/v1/generate")
.header("content-type", "application/json")
.body(Body::from("{}"))
.expect("build request"),
)
.await
.expect("call app");
assert_ne!(response.status(), StatusCode::NOT_FOUND);
}
/// Build a dev-mode router backed by a mock engine using a custom ready
/// response, returning the router and the engine task handle so the engine
/// stays alive for the duration of the test.
@@ -916,10 +994,13 @@ async fn test_app_with_stream_output_specs(
)
.await;
(
build_router(Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
))),
build_router_with_scale_out_endpoints(
Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)),
true,
),
engine_task,
)
}
@@ -1236,10 +1317,10 @@ async fn render_chat_response_can_be_submitted_to_generate_unchanged() {
Arc::new(FakeChatBackend::new()),
)
.await;
let mut generate_app = build_router(Arc::new(AppState::new(
vec!["render-model".to_string()],
chat,
)));
let mut generate_app = build_router_with_scale_out_endpoints(
Arc::new(AppState::new(vec!["render-model".to_string()], chat)),
true,
);
let generate_response = generate_app
.call(
Request::builder()
@@ -3986,10 +4067,13 @@ async fn non_stream_raw_generate_returns_token_output_envelope() {
.await
.expect("connect client");
let chat = ChatLlm::from_shared_backend(Llm::new(client), Arc::new(FakeChatBackend::new()));
let mut app = build_router(Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)));
let mut app = build_router_with_scale_out_endpoints(
Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)),
true,
);
let response = app
.call(
@@ -4100,10 +4184,13 @@ async fn stream_raw_generate_returns_sse_chunks_and_usage() {
.await
.expect("connect client");
let chat = ChatLlm::from_shared_backend(Llm::new(client), Arc::new(FakeChatBackend::new()));
let mut app = build_router(Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)));
let mut app = build_router_with_scale_out_endpoints(
Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)),
true,
);
let response = app
.call(