[Frontend] Replace VLLM_ENABLE_SCALE_OUT_ENDPOINTS with --enable-scale-out (#55176)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -55,6 +55,8 @@ pub struct ApiServerOptions {
|
||||
pub enable_prompt_tokens_details: bool,
|
||||
/// When `true`, set `X-Request-Id` on every HTTP response.
|
||||
pub enable_request_id_headers: bool,
|
||||
/// When `true`, register the scale-out `/inference/v1/generate` route.
|
||||
pub enable_scale_out: bool,
|
||||
}
|
||||
|
||||
/// CORS settings mirroring Python's `CORSMiddleware`; the default is permissive.
|
||||
|
||||
@@ -46,33 +46,14 @@ fn runtime_lora_updating_enabled() -> bool {
|
||||
.is_some_and(|value| matches!(value.trim().to_lowercase().as_str(), "1" | "true"))
|
||||
}
|
||||
|
||||
fn parse_scale_out_endpoints_flag(value: Option<&str>) -> Result<bool, String> {
|
||||
let Some(value) = value else {
|
||||
return Ok(false);
|
||||
};
|
||||
if value.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
match value.trim() {
|
||||
"0" => Ok(false),
|
||||
"1" => Ok(true),
|
||||
_ => Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn scale_out_endpoints_enabled() -> bool {
|
||||
let value = std::env::var("VLLM_ENABLE_SCALE_OUT_ENDPOINTS").ok();
|
||||
parse_scale_out_endpoints_flag(value.as_deref()).unwrap_or_else(|message| panic!("{message}"))
|
||||
}
|
||||
|
||||
/// Build the minimal OpenAI-compatible router for one configured model.
|
||||
pub fn build_router(state: Arc<AppState>) -> Router {
|
||||
let scale_out_endpoints_enabled = state.api_server_options.enable_scale_out;
|
||||
build_router_with_options(
|
||||
state,
|
||||
server_dev_mode_enabled(),
|
||||
runtime_lora_updating_enabled(),
|
||||
scale_out_endpoints_enabled(),
|
||||
scale_out_endpoints_enabled,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -121,10 +102,7 @@ fn build_router_with_options(
|
||||
if scale_out_endpoints_enabled {
|
||||
router = router.route("/inference/v1/generate", post(inference::generate));
|
||||
} else {
|
||||
info!(
|
||||
"scale-out endpoints are disabled; set \
|
||||
VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 to enable them"
|
||||
);
|
||||
info!("scale-out endpoints are disabled; pass --enable-scale-out to enable them");
|
||||
}
|
||||
|
||||
if runtime_lora_updating_enabled {
|
||||
|
||||
@@ -54,8 +54,7 @@ use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
use super::{
|
||||
build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora,
|
||||
build_router_with_scale_out_endpoints, parse_scale_out_endpoints_flag,
|
||||
render::build_router as build_render_router,
|
||||
build_router_with_scale_out_endpoints, render::build_router as build_render_router,
|
||||
};
|
||||
use crate::config::{ApiServerOptions, CorsConfig, LoraModulePath};
|
||||
use crate::lora::LoadLoraError;
|
||||
@@ -731,23 +730,36 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scale_out_flag_accepts_unset_empty_zero_one_and_whitespace() {
|
||||
assert_eq!(parse_scale_out_endpoints_flag(None), Ok(false));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some("")), Ok(false));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some("0")), Ok(false));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some("1")), Ok(true));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some(" 1 ")), Ok(true));
|
||||
}
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn scale_out_generate_route_follows_api_server_options() {
|
||||
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
|
||||
b"engine-scale-out-api-server-options",
|
||||
default_stream_output_specs(),
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
)
|
||||
.await;
|
||||
let state = Arc::new(
|
||||
AppState::new(vec!["model".to_string()], chat).with_api_server_options(ApiServerOptions {
|
||||
enable_scale_out: true,
|
||||
..Default::default()
|
||||
}),
|
||||
);
|
||||
let mut app = build_router(state);
|
||||
|
||||
#[test]
|
||||
fn scale_out_flag_rejects_values_other_than_zero_or_one() {
|
||||
for value in ["-1", "2", "01", "+1", "invalid", " "] {
|
||||
assert_eq!(
|
||||
parse_scale_out_endpoints_flag(Some(value)),
|
||||
Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string())
|
||||
);
|
||||
}
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/inference/v1/generate")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from("{}"))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_ne!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
|
||||
Reference in New Issue
Block a user