feat(rust): expose profiler control routes in Rust frontend (#46306)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Pranav Thakur
2026-07-02 08:46:07 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent b0b8a286dd
commit 08a8a4af3f
11 changed files with 381 additions and 2 deletions
@@ -77,6 +77,7 @@ async fn main() -> Result<()> {
grpc_port: None,
shutdown_timeout: Duration::ZERO,
keep_alive_timeout: Duration::from_secs(5),
profiler: None,
};
let bind_address = format!("127.0.0.1:{port}");
+3
View File
@@ -208,6 +208,9 @@ pub struct Config {
/// Maximum idle time on a keep-alive HTTP connection before the server
/// closes it (`VLLM_HTTP_TIMEOUT_KEEP_ALIVE`, default 5s).
pub keep_alive_timeout: Duration,
/// Profiler mode that registers `/start_profile` and `/stop_profile`
/// routes when present.
pub profiler: Option<String>,
}
impl Config {
+2 -1
View File
@@ -134,7 +134,8 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
.with_api_server_options(config.api_server_options)
.with_server_info(ServerInfoSnapshot::from_config(config))
.with_api_keys(config.api_keys.clone())
.with_cors(config.cors.clone()),
.with_cors(config.cors.clone())
.with_profiler(config.profiler.clone()),
))
}
+13
View File
@@ -8,6 +8,7 @@ mod lora;
mod metrics;
pub(crate) mod openai;
mod pause;
mod profile;
mod server_info;
mod sleep;
mod tokenize;
@@ -21,6 +22,7 @@ use axum::extract::DefaultBodyLimit;
use axum::middleware::{from_fn, from_fn_with_state};
use axum::routing::{get, post};
use tower_http::trace::TraceLayer;
use tracing::warn;
use crate::middleware;
use crate::state::AppState;
@@ -107,6 +109,17 @@ fn build_router_with_options(
.route("/get_world_size", get(world_size::get_world_size))
}
if let Some(profiler) = &state.profiler {
warn!(
mode = profiler,
"profiler is enabled in the API server; \
this should only be used for local development",
);
router = router
.route("/start_profile", post(profile::start_profile))
.route("/stop_profile", post(profile::stop_profile));
}
let enable_request_id_headers = state.api_server_options.enable_request_id_headers;
let enable_api_key_auth = state.has_api_keys();
let mut router = router
+33
View File
@@ -0,0 +1,33 @@
use std::sync::Arc;
use axum::extract::State;
use axum::http::StatusCode;
use tracing::info;
use crate::error::ApiError;
use crate::state::AppState;
use crate::utils::utility_call_error;
/// Start profiling the engine.
pub async fn start_profile(State(state): State<Arc<AppState>>) -> Result<StatusCode, ApiError> {
info!("starting profiler");
state
.engine_core_client()
.start_profile(None)
.await
.map_err(|error| utility_call_error("start_profile", error))?;
info!("profiler started");
Ok(StatusCode::OK)
}
/// Stop profiling the engine.
pub async fn stop_profile(State(state): State<Arc<AppState>>) -> Result<StatusCode, ApiError> {
info!("stopping profiler");
state
.engine_core_client()
.stop_profile(None)
.await
.map_err(|error| utility_call_error("stop_profile", error))?;
info!("profiler stopped");
Ok(StatusCode::OK)
}
+143
View File
@@ -6183,3 +6183,146 @@ async fn world_size_excludes_data_parallelism_when_include_dp_false() {
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json, json!({"world_size": 2}));
}
// ========================= Profiler route tests =========================
async fn test_profiling_app_with_engine_script<F>(script: F) -> (axum::Router, MockEngineTask)
where
F: for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static,
{
let ipc = IpcNamespace::new().expect("create ipc namespace");
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-openai-profiler".to_vec();
let engine_task = MockEngineTask::new(spawn_mock_engine_task(
handshake_address.clone(),
engine_id.clone(),
move |dealer, push| script(dealer, push),
));
let client = EngineCoreClient::connect(
EngineCoreClientConfig::new_single(handshake_address)
.with_model_name("test-model")
.with_local_input_output_addresses(
Some(ipc.input_endpoint()),
Some(ipc.output_endpoint()),
),
)
.await
.expect("connect client");
let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
(
build_router(Arc::new(
AppState::new(vec!["test-model".to_string()], chat)
.with_profiler(Some("torch".to_string())),
)),
engine_task,
)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn start_profile_route_sends_expected_utility_call() {
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("profile"));
assert_eq!(array[3], Value::Array(vec![Value::from(true), Value::Nil]));
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
})
})
.await;
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/start_profile")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn stop_profile_route_sends_expected_utility_call() {
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("profile"));
assert_eq!(array[3], Value::Array(vec![Value::from(false), Value::Nil]));
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
})
})
.await;
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/stop_profile")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn profile_routes_are_hidden_when_profiling_is_disabled() {
let (chat, engine_task) = test_chat_with_engine_handle().await;
let app = build_router(Arc::new(AppState::new(
vec!["test-model".to_string()],
chat,
)));
for (method, uri) in [("POST", "/start_profile"), ("POST", "/stop_profile")] {
let response = app
.clone()
.call(
Request::builder()
.method(method)
.uri(uri)
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::NOT_FOUND, "{method} {uri}");
}
engine_task.abort_and_join().await;
}
+10
View File
@@ -47,6 +47,9 @@ pub struct AppState {
model_path: Option<String>,
/// Lazily initialized runtime for heavyweight request paths.
request_runtime: OnceLock<BackgroundShutdownRuntime>,
/// Profiler mode that registers `/start_profile` and `/stop_profile`
/// routes when present.
pub profiler: Option<String>,
}
impl AppState {
@@ -74,6 +77,7 @@ impl AppState {
lora_manager: LoraManager::new(),
model_path: None,
request_runtime: OnceLock::new(),
profiler: None,
}
}
@@ -95,6 +99,12 @@ impl AppState {
self
}
/// Set the profiler mode that enables `/start_profile` and `/stop_profile`.
pub fn with_profiler(mut self, profiler: Option<String>) -> Self {
self.profiler = profiler;
self
}
/// Attach the runtime server information snapshot used by `/server_info`.
pub(crate) fn with_server_info(mut self, server_info: ServerInfoSnapshot) -> Self {
self.server_info = Some(server_info);