[Frontend] Replace VLLM_ENABLE_SCALE_OUT_ENDPOINTS with --enable-scale-out (#55176)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-09-14 10:02:14 +00:00
committed by GitHub
parent 2c7ee87223
commit 934b1fcbcb
24 changed files with 130 additions and 215 deletions
+10
View File
@@ -329,6 +329,15 @@ pub struct SharedRuntimeArgs {
#[serde(default)]
pub enable_request_id_headers: bool,
/// Register the scale-out `/inference/v1/generate` endpoint.
#[arg(
long,
default_missing_value = "true",
num_args = 0..=1
)]
#[serde(default)]
pub enable_scale_out: bool,
/// If provided, the server will require one of these keys to be presented
/// in the Authorization header.
#[educe(Debug(ignore))]
@@ -577,6 +586,7 @@ impl SharedRuntimeArgs {
enable_log_requests: self.enable_log_requests,
enable_prompt_tokens_details: self.enable_prompt_tokens_details,
enable_request_id_headers: self.enable_request_id_headers,
enable_scale_out: self.enable_scale_out,
}
}
+6
View File
@@ -195,6 +195,7 @@ fn serve_args_forward_python_flags_with_separator() {
enable_log_requests: false,
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
disable_log_stats: false,
served_model_name: [],
allowed_origins: JsonStringList(
@@ -1030,6 +1031,7 @@ fn frontend_args_accept_json() {
enable_log_requests: false,
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
disable_log_stats: false,
served_model_name: [],
allowed_origins: JsonStringList(
@@ -1629,6 +1631,7 @@ fn serve_args_accept_handshake_aliases() {
enable_log_requests: false,
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
disable_log_stats: false,
served_model_name: [],
allowed_origins: JsonStringList(
@@ -1780,6 +1783,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
enable_log_requests: false,
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
},
cors: CorsConfig {
allow_origins: [
@@ -1868,6 +1872,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
enable_log_requests: false,
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
},
cors: CorsConfig {
allow_origins: [
@@ -1977,6 +1982,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
enable_log_requests: false,
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
},
cors: CorsConfig {
allow_origins: [
+2
View File
@@ -55,6 +55,8 @@ pub struct ApiServerOptions {
pub enable_prompt_tokens_details: bool,
/// When `true`, set `X-Request-Id` on every HTTP response.
pub enable_request_id_headers: bool,
/// When `true`, register the scale-out `/inference/v1/generate` route.
pub enable_scale_out: bool,
}
/// CORS settings mirroring Python's `CORSMiddleware`; the default is permissive.
+3 -25
View File
@@ -46,33 +46,14 @@ fn runtime_lora_updating_enabled() -> bool {
.is_some_and(|value| matches!(value.trim().to_lowercase().as_str(), "1" | "true"))
}
fn parse_scale_out_endpoints_flag(value: Option<&str>) -> Result<bool, String> {
let Some(value) = value else {
return Ok(false);
};
if value.is_empty() {
return Ok(false);
}
match value.trim() {
"0" => Ok(false),
"1" => Ok(true),
_ => Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string()),
}
}
fn scale_out_endpoints_enabled() -> bool {
let value = std::env::var("VLLM_ENABLE_SCALE_OUT_ENDPOINTS").ok();
parse_scale_out_endpoints_flag(value.as_deref()).unwrap_or_else(|message| panic!("{message}"))
}
/// Build the minimal OpenAI-compatible router for one configured model.
pub fn build_router(state: Arc<AppState>) -> Router {
let scale_out_endpoints_enabled = state.api_server_options.enable_scale_out;
build_router_with_options(
state,
server_dev_mode_enabled(),
runtime_lora_updating_enabled(),
scale_out_endpoints_enabled(),
scale_out_endpoints_enabled,
)
}
@@ -121,10 +102,7 @@ fn build_router_with_options(
if scale_out_endpoints_enabled {
router = router.route("/inference/v1/generate", post(inference::generate));
} else {
info!(
"scale-out endpoints are disabled; set \
VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 to enable them"
);
info!("scale-out endpoints are disabled; pass --enable-scale-out to enable them");
}
if runtime_lora_updating_enabled {
+30 -18
View File
@@ -54,8 +54,7 @@ use zeromq::{DealerSocket, PushSocket, ZmqMessage};
use super::{
build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora,
build_router_with_scale_out_endpoints, parse_scale_out_endpoints_flag,
render::build_router as build_render_router,
build_router_with_scale_out_endpoints, render::build_router as build_render_router,
};
use crate::config::{ApiServerOptions, CorsConfig, LoraModulePath};
use crate::lora::LoadLoraError;
@@ -731,23 +730,36 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
)
}
#[test]
fn scale_out_flag_accepts_unset_empty_zero_one_and_whitespace() {
assert_eq!(parse_scale_out_endpoints_flag(None), Ok(false));
assert_eq!(parse_scale_out_endpoints_flag(Some("")), Ok(false));
assert_eq!(parse_scale_out_endpoints_flag(Some("0")), Ok(false));
assert_eq!(parse_scale_out_endpoints_flag(Some("1")), Ok(true));
assert_eq!(parse_scale_out_endpoints_flag(Some(" 1 ")), Ok(true));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn scale_out_generate_route_follows_api_server_options() {
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
b"engine-scale-out-api-server-options",
default_stream_output_specs(),
Arc::new(FakeChatBackend::new()),
)
.await;
let state = Arc::new(
AppState::new(vec!["model".to_string()], chat).with_api_server_options(ApiServerOptions {
enable_scale_out: true,
..Default::default()
}),
);
let mut app = build_router(state);
#[test]
fn scale_out_flag_rejects_values_other_than_zero_or_one() {
for value in ["-1", "2", "01", "+1", "invalid", " "] {
assert_eq!(
parse_scale_out_endpoints_flag(Some(value)),
Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string())
);
}
let response = app
.call(
Request::builder()
.method("POST")
.uri("/inference/v1/generate")
.header("content-type", "application/json")
.body(Body::from("{}"))
.expect("build request"),
)
.await
.expect("call app");
assert_ne!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]