[Frontend] Replace VLLM_ENABLE_SCALE_OUT_ENDPOINTS with --enable-scale-out (#55176)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -329,6 +329,15 @@ pub struct SharedRuntimeArgs {
|
||||
#[serde(default)]
|
||||
pub enable_request_id_headers: bool,
|
||||
|
||||
/// Register the scale-out `/inference/v1/generate` endpoint.
|
||||
#[arg(
|
||||
long,
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
#[serde(default)]
|
||||
pub enable_scale_out: bool,
|
||||
|
||||
/// If provided, the server will require one of these keys to be presented
|
||||
/// in the Authorization header.
|
||||
#[educe(Debug(ignore))]
|
||||
@@ -577,6 +586,7 @@ impl SharedRuntimeArgs {
|
||||
enable_log_requests: self.enable_log_requests,
|
||||
enable_prompt_tokens_details: self.enable_prompt_tokens_details,
|
||||
enable_request_id_headers: self.enable_request_id_headers,
|
||||
enable_scale_out: self.enable_scale_out,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -195,6 +195,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
allowed_origins: JsonStringList(
|
||||
@@ -1030,6 +1031,7 @@ fn frontend_args_accept_json() {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
allowed_origins: JsonStringList(
|
||||
@@ -1629,6 +1631,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
allowed_origins: JsonStringList(
|
||||
@@ -1780,6 +1783,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
},
|
||||
cors: CorsConfig {
|
||||
allow_origins: [
|
||||
@@ -1868,6 +1872,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
},
|
||||
cors: CorsConfig {
|
||||
allow_origins: [
|
||||
@@ -1977,6 +1982,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
},
|
||||
cors: CorsConfig {
|
||||
allow_origins: [
|
||||
|
||||
@@ -55,6 +55,8 @@ pub struct ApiServerOptions {
|
||||
pub enable_prompt_tokens_details: bool,
|
||||
/// When `true`, set `X-Request-Id` on every HTTP response.
|
||||
pub enable_request_id_headers: bool,
|
||||
/// When `true`, register the scale-out `/inference/v1/generate` route.
|
||||
pub enable_scale_out: bool,
|
||||
}
|
||||
|
||||
/// CORS settings mirroring Python's `CORSMiddleware`; the default is permissive.
|
||||
|
||||
@@ -46,33 +46,14 @@ fn runtime_lora_updating_enabled() -> bool {
|
||||
.is_some_and(|value| matches!(value.trim().to_lowercase().as_str(), "1" | "true"))
|
||||
}
|
||||
|
||||
fn parse_scale_out_endpoints_flag(value: Option<&str>) -> Result<bool, String> {
|
||||
let Some(value) = value else {
|
||||
return Ok(false);
|
||||
};
|
||||
if value.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
match value.trim() {
|
||||
"0" => Ok(false),
|
||||
"1" => Ok(true),
|
||||
_ => Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn scale_out_endpoints_enabled() -> bool {
|
||||
let value = std::env::var("VLLM_ENABLE_SCALE_OUT_ENDPOINTS").ok();
|
||||
parse_scale_out_endpoints_flag(value.as_deref()).unwrap_or_else(|message| panic!("{message}"))
|
||||
}
|
||||
|
||||
/// Build the minimal OpenAI-compatible router for one configured model.
|
||||
pub fn build_router(state: Arc<AppState>) -> Router {
|
||||
let scale_out_endpoints_enabled = state.api_server_options.enable_scale_out;
|
||||
build_router_with_options(
|
||||
state,
|
||||
server_dev_mode_enabled(),
|
||||
runtime_lora_updating_enabled(),
|
||||
scale_out_endpoints_enabled(),
|
||||
scale_out_endpoints_enabled,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -121,10 +102,7 @@ fn build_router_with_options(
|
||||
if scale_out_endpoints_enabled {
|
||||
router = router.route("/inference/v1/generate", post(inference::generate));
|
||||
} else {
|
||||
info!(
|
||||
"scale-out endpoints are disabled; set \
|
||||
VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 to enable them"
|
||||
);
|
||||
info!("scale-out endpoints are disabled; pass --enable-scale-out to enable them");
|
||||
}
|
||||
|
||||
if runtime_lora_updating_enabled {
|
||||
|
||||
@@ -54,8 +54,7 @@ use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
use super::{
|
||||
build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora,
|
||||
build_router_with_scale_out_endpoints, parse_scale_out_endpoints_flag,
|
||||
render::build_router as build_render_router,
|
||||
build_router_with_scale_out_endpoints, render::build_router as build_render_router,
|
||||
};
|
||||
use crate::config::{ApiServerOptions, CorsConfig, LoraModulePath};
|
||||
use crate::lora::LoadLoraError;
|
||||
@@ -731,23 +730,36 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scale_out_flag_accepts_unset_empty_zero_one_and_whitespace() {
|
||||
assert_eq!(parse_scale_out_endpoints_flag(None), Ok(false));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some("")), Ok(false));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some("0")), Ok(false));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some("1")), Ok(true));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some(" 1 ")), Ok(true));
|
||||
}
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn scale_out_generate_route_follows_api_server_options() {
|
||||
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
|
||||
b"engine-scale-out-api-server-options",
|
||||
default_stream_output_specs(),
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
)
|
||||
.await;
|
||||
let state = Arc::new(
|
||||
AppState::new(vec!["model".to_string()], chat).with_api_server_options(ApiServerOptions {
|
||||
enable_scale_out: true,
|
||||
..Default::default()
|
||||
}),
|
||||
);
|
||||
let mut app = build_router(state);
|
||||
|
||||
#[test]
|
||||
fn scale_out_flag_rejects_values_other_than_zero_or_one() {
|
||||
for value in ["-1", "2", "01", "+1", "invalid", " "] {
|
||||
assert_eq!(
|
||||
parse_scale_out_endpoints_flag(Some(value)),
|
||||
Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string())
|
||||
);
|
||||
}
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/inference/v1/generate")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from("{}"))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_ne!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
|
||||
Reference in New Issue
Block a user