feat(rust): expose profiler control routes in Rust frontend (#46306)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
b0b8a286dd
commit
08a8a4af3f
@@ -77,6 +77,7 @@ async fn main() -> Result<()> {
|
||||
grpc_port: None,
|
||||
shutdown_timeout: Duration::ZERO,
|
||||
keep_alive_timeout: Duration::from_secs(5),
|
||||
profiler: None,
|
||||
};
|
||||
|
||||
let bind_address = format!("127.0.0.1:{port}");
|
||||
|
||||
@@ -208,6 +208,9 @@ pub struct Config {
|
||||
/// Maximum idle time on a keep-alive HTTP connection before the server
|
||||
/// closes it (`VLLM_HTTP_TIMEOUT_KEEP_ALIVE`, default 5s).
|
||||
pub keep_alive_timeout: Duration,
|
||||
/// Profiler mode that registers `/start_profile` and `/stop_profile`
|
||||
/// routes when present.
|
||||
pub profiler: Option<String>,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
|
||||
@@ -134,7 +134,8 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
.with_api_server_options(config.api_server_options)
|
||||
.with_server_info(ServerInfoSnapshot::from_config(config))
|
||||
.with_api_keys(config.api_keys.clone())
|
||||
.with_cors(config.cors.clone()),
|
||||
.with_cors(config.cors.clone())
|
||||
.with_profiler(config.profiler.clone()),
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ mod lora;
|
||||
mod metrics;
|
||||
pub(crate) mod openai;
|
||||
mod pause;
|
||||
mod profile;
|
||||
mod server_info;
|
||||
mod sleep;
|
||||
mod tokenize;
|
||||
@@ -21,6 +22,7 @@ use axum::extract::DefaultBodyLimit;
|
||||
use axum::middleware::{from_fn, from_fn_with_state};
|
||||
use axum::routing::{get, post};
|
||||
use tower_http::trace::TraceLayer;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::middleware;
|
||||
use crate::state::AppState;
|
||||
@@ -107,6 +109,17 @@ fn build_router_with_options(
|
||||
.route("/get_world_size", get(world_size::get_world_size))
|
||||
}
|
||||
|
||||
if let Some(profiler) = &state.profiler {
|
||||
warn!(
|
||||
mode = profiler,
|
||||
"profiler is enabled in the API server; \
|
||||
this should only be used for local development",
|
||||
);
|
||||
router = router
|
||||
.route("/start_profile", post(profile::start_profile))
|
||||
.route("/stop_profile", post(profile::stop_profile));
|
||||
}
|
||||
|
||||
let enable_request_id_headers = state.api_server_options.enable_request_id_headers;
|
||||
let enable_api_key_auth = state.has_api_keys();
|
||||
let mut router = router
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::http::StatusCode;
|
||||
use tracing::info;
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::utility_call_error;
|
||||
|
||||
/// Start profiling the engine.
|
||||
pub async fn start_profile(State(state): State<Arc<AppState>>) -> Result<StatusCode, ApiError> {
|
||||
info!("starting profiler");
|
||||
state
|
||||
.engine_core_client()
|
||||
.start_profile(None)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("start_profile", error))?;
|
||||
info!("profiler started");
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
|
||||
/// Stop profiling the engine.
|
||||
pub async fn stop_profile(State(state): State<Arc<AppState>>) -> Result<StatusCode, ApiError> {
|
||||
info!("stopping profiler");
|
||||
state
|
||||
.engine_core_client()
|
||||
.stop_profile(None)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("stop_profile", error))?;
|
||||
info!("profiler stopped");
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
@@ -6183,3 +6183,146 @@ async fn world_size_excludes_data_parallelism_when_include_dp_false() {
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
assert_eq!(json, json!({"world_size": 2}));
|
||||
}
|
||||
|
||||
// ========================= Profiler route tests =========================
|
||||
|
||||
async fn test_profiling_app_with_engine_script<F>(script: F) -> (axum::Router, MockEngineTask)
|
||||
where
|
||||
F: for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static,
|
||||
{
|
||||
let ipc = IpcNamespace::new().expect("create ipc namespace");
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-openai-profiler".to_vec();
|
||||
|
||||
let engine_task = MockEngineTask::new(spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
move |dealer, push| script(dealer, push),
|
||||
));
|
||||
|
||||
let client = EngineCoreClient::connect(
|
||||
EngineCoreClientConfig::new_single(handshake_address)
|
||||
.with_model_name("test-model")
|
||||
.with_local_input_output_addresses(
|
||||
Some(ipc.input_endpoint()),
|
||||
Some(ipc.output_endpoint()),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("connect client");
|
||||
|
||||
let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
|
||||
(
|
||||
build_router(Arc::new(
|
||||
AppState::new(vec!["test-model".to_string()], chat)
|
||||
.with_profiler(Some("torch".to_string())),
|
||||
)),
|
||||
engine_task,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn start_profile_route_sends_expected_utility_call() {
|
||||
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
|
||||
assert_eq!(array[2], Value::from("profile"));
|
||||
assert_eq!(array[3], Value::Array(vec![Value::from(true), Value::Nil]));
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/start_profile")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn stop_profile_route_sends_expected_utility_call() {
|
||||
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
|
||||
assert_eq!(array[2], Value::from("profile"));
|
||||
assert_eq!(array[3], Value::Array(vec![Value::from(false), Value::Nil]));
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/stop_profile")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn profile_routes_are_hidden_when_profiling_is_disabled() {
|
||||
let (chat, engine_task) = test_chat_with_engine_handle().await;
|
||||
let app = build_router(Arc::new(AppState::new(
|
||||
vec!["test-model".to_string()],
|
||||
chat,
|
||||
)));
|
||||
|
||||
for (method, uri) in [("POST", "/start_profile"), ("POST", "/stop_profile")] {
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method(method)
|
||||
.uri(uri)
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND, "{method} {uri}");
|
||||
}
|
||||
|
||||
engine_task.abort_and_join().await;
|
||||
}
|
||||
|
||||
@@ -47,6 +47,9 @@ pub struct AppState {
|
||||
model_path: Option<String>,
|
||||
/// Lazily initialized runtime for heavyweight request paths.
|
||||
request_runtime: OnceLock<BackgroundShutdownRuntime>,
|
||||
/// Profiler mode that registers `/start_profile` and `/stop_profile`
|
||||
/// routes when present.
|
||||
pub profiler: Option<String>,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
@@ -74,6 +77,7 @@ impl AppState {
|
||||
lora_manager: LoraManager::new(),
|
||||
model_path: None,
|
||||
request_runtime: OnceLock::new(),
|
||||
profiler: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,6 +99,12 @@ impl AppState {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the profiler mode that enables `/start_profile` and `/stop_profile`.
|
||||
pub fn with_profiler(mut self, profiler: Option<String>) -> Self {
|
||||
self.profiler = profiler;
|
||||
self
|
||||
}
|
||||
|
||||
/// Attach the runtime server information snapshot used by `/server_info`.
|
||||
pub(crate) fn with_server_info(mut self, server_info: ServerInfoSnapshot) -> Self {
|
||||
self.server_info = Some(server_info);
|
||||
|
||||
Reference in New Issue
Block a user