[Frontend] Gate scale-out endpoints behind opt-in flag (#54579)
Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
This commit is contained in:
@@ -26,7 +26,7 @@ use axum::extract::DefaultBodyLimit;
|
||||
use axum::middleware::{from_fn, from_fn_with_state};
|
||||
use axum::routing::{get, post};
|
||||
use tower_http::trace::TraceLayer;
|
||||
use tracing::warn;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
|
||||
use crate::middleware;
|
||||
@@ -45,12 +45,33 @@ fn runtime_lora_updating_enabled() -> bool {
|
||||
.is_some_and(|value| matches!(value.trim().to_lowercase().as_str(), "1" | "true"))
|
||||
}
|
||||
|
||||
fn parse_scale_out_endpoints_flag(value: Option<&str>) -> Result<bool, String> {
|
||||
let Some(value) = value else {
|
||||
return Ok(false);
|
||||
};
|
||||
if value.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
match value.trim() {
|
||||
"0" => Ok(false),
|
||||
"1" => Ok(true),
|
||||
_ => Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn scale_out_endpoints_enabled() -> bool {
|
||||
let value = std::env::var("VLLM_ENABLE_SCALE_OUT_ENDPOINTS").ok();
|
||||
parse_scale_out_endpoints_flag(value.as_deref()).unwrap_or_else(|message| panic!("{message}"))
|
||||
}
|
||||
|
||||
/// Build the minimal OpenAI-compatible router for one configured model.
|
||||
pub fn build_router(state: Arc<AppState>) -> Router {
|
||||
build_router_with_options(
|
||||
state,
|
||||
server_dev_mode_enabled(),
|
||||
runtime_lora_updating_enabled(),
|
||||
scale_out_endpoints_enabled(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -65,13 +86,22 @@ fn build_router_with_dev_mode_and_lora(
|
||||
dev_mode_enabled: bool,
|
||||
runtime_lora_updating_enabled: bool,
|
||||
) -> Router {
|
||||
build_router_with_options(state, dev_mode_enabled, runtime_lora_updating_enabled)
|
||||
build_router_with_options(state, dev_mode_enabled, runtime_lora_updating_enabled, true)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn build_router_with_scale_out_endpoints(
|
||||
state: Arc<AppState>,
|
||||
scale_out_endpoints_enabled: bool,
|
||||
) -> Router {
|
||||
build_router_with_options(state, false, false, scale_out_endpoints_enabled)
|
||||
}
|
||||
|
||||
fn build_router_with_options(
|
||||
state: Arc<AppState>,
|
||||
dev_mode_enabled: bool,
|
||||
runtime_lora_updating_enabled: bool,
|
||||
scale_out_endpoints_enabled: bool,
|
||||
) -> Router {
|
||||
let mut router = Router::new()
|
||||
// Health & monitoring
|
||||
@@ -85,8 +115,16 @@ fn build_router_with_options(
|
||||
.route("/v1/chat/completions", post(openai::chat_completions))
|
||||
// vLLM specific endpoints
|
||||
.route("/tokenize", post(tokenize::tokenize))
|
||||
.route("/detokenize", post(tokenize::detokenize))
|
||||
.route("/inference/v1/generate", post(inference::generate));
|
||||
.route("/detokenize", post(tokenize::detokenize));
|
||||
|
||||
if scale_out_endpoints_enabled {
|
||||
router = router.route("/inference/v1/generate", post(inference::generate));
|
||||
} else {
|
||||
info!(
|
||||
"scale-out endpoints are disabled; set \
|
||||
VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 to enable them"
|
||||
);
|
||||
}
|
||||
|
||||
if runtime_lora_updating_enabled {
|
||||
router = router
|
||||
|
||||
@@ -54,6 +54,7 @@ use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
use super::{
|
||||
build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora,
|
||||
build_router_with_scale_out_endpoints, parse_scale_out_endpoints_flag,
|
||||
render::build_router as build_render_router,
|
||||
};
|
||||
use crate::config::{ApiServerOptions, CorsConfig};
|
||||
@@ -716,6 +717,83 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scale_out_flag_accepts_unset_empty_zero_one_and_whitespace() {
|
||||
assert_eq!(parse_scale_out_endpoints_flag(None), Ok(false));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some("")), Ok(false));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some("0")), Ok(false));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some("1")), Ok(true));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some(" 1 ")), Ok(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scale_out_flag_rejects_values_other_than_zero_or_one() {
|
||||
for value in ["-1", "2", "01", "+1", "invalid", " "] {
|
||||
assert_eq!(
|
||||
parse_scale_out_endpoints_flag(Some(value)),
|
||||
Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn scale_out_generate_route_is_disabled_by_default() {
|
||||
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
|
||||
b"engine-scale-out-disabled",
|
||||
default_stream_output_specs(),
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
)
|
||||
.await;
|
||||
let mut app = build_router_with_scale_out_endpoints(
|
||||
Arc::new(AppState::new(vec!["model".to_string()], chat)),
|
||||
false,
|
||||
);
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/inference/v1/generate")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from("{}"))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn scale_out_generate_route_can_be_enabled() {
|
||||
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
|
||||
b"engine-scale-out-enabled",
|
||||
default_stream_output_specs(),
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
)
|
||||
.await;
|
||||
let mut app = build_router_with_scale_out_endpoints(
|
||||
Arc::new(AppState::new(vec!["model".to_string()], chat)),
|
||||
true,
|
||||
);
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/inference/v1/generate")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from("{}"))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_ne!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
/// Build a dev-mode router backed by a mock engine using a custom ready
|
||||
/// response, returning the router and the engine task handle so the engine
|
||||
/// stays alive for the duration of the test.
|
||||
@@ -916,10 +994,13 @@ async fn test_app_with_stream_output_specs(
|
||||
)
|
||||
.await;
|
||||
(
|
||||
build_router(Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
))),
|
||||
build_router_with_scale_out_endpoints(
|
||||
Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)),
|
||||
true,
|
||||
),
|
||||
engine_task,
|
||||
)
|
||||
}
|
||||
@@ -1236,10 +1317,10 @@ async fn render_chat_response_can_be_submitted_to_generate_unchanged() {
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
)
|
||||
.await;
|
||||
let mut generate_app = build_router(Arc::new(AppState::new(
|
||||
vec!["render-model".to_string()],
|
||||
chat,
|
||||
)));
|
||||
let mut generate_app = build_router_with_scale_out_endpoints(
|
||||
Arc::new(AppState::new(vec!["render-model".to_string()], chat)),
|
||||
true,
|
||||
);
|
||||
let generate_response = generate_app
|
||||
.call(
|
||||
Request::builder()
|
||||
@@ -3986,10 +4067,13 @@ async fn non_stream_raw_generate_returns_token_output_envelope() {
|
||||
.await
|
||||
.expect("connect client");
|
||||
let chat = ChatLlm::from_shared_backend(Llm::new(client), Arc::new(FakeChatBackend::new()));
|
||||
let mut app = build_router(Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)));
|
||||
let mut app = build_router_with_scale_out_endpoints(
|
||||
Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)),
|
||||
true,
|
||||
);
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
@@ -4100,10 +4184,13 @@ async fn stream_raw_generate_returns_sse_chunks_and_usage() {
|
||||
.await
|
||||
.expect("connect client");
|
||||
let chat = ChatLlm::from_shared_backend(Llm::new(client), Arc::new(FakeChatBackend::new()));
|
||||
let mut app = build_router(Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)));
|
||||
let mut app = build_router_with_scale_out_endpoints(
|
||||
Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)),
|
||||
true,
|
||||
);
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
|
||||
Reference in New Issue
Block a user