[Profiler] Add per-session profiling controls to Python and Rust frontends (#57875)

Signed-off-by: czhu <conway.zhu@cohere.com>
Signed-off-by: Conway Zhu <conway.zhu@cohere.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
czhu-cohere
2026-10-06 05:21:19 +00:00
committed by GitHub
co-authored by aoshen02
parent 9a1f6fd4cb
commit 6fddc9b21e
25 changed files with 1497 additions and 164 deletions
+9
View File
@@ -22,6 +22,8 @@ pub enum ApiError {
ModelNotFound { model: String },
/// The request body could not be parsed as valid JSON.
JsonParseError { message: String },
/// The requested operation conflicts with current server state.
Conflict { message: String },
/// An unexpected internal failure happened before streaming started.
ServerError { message: String },
}
@@ -32,6 +34,7 @@ impl ApiError {
match self {
Self::InvalidRequest { .. } => StatusCode::BAD_REQUEST,
Self::ModelNotFound { .. } => StatusCode::NOT_FOUND,
Self::Conflict { .. } => StatusCode::CONFLICT,
Self::ServerError { .. } => StatusCode::INTERNAL_SERVER_ERROR,
Self::JsonParseError { .. } => StatusCode::BAD_REQUEST,
}
@@ -65,6 +68,12 @@ impl ApiError {
param: None,
code: Some("json_parse_error".to_string()),
},
Self::Conflict { message } => ErrorDetail {
message: message.clone(),
error_type: "conflict_error".to_string(),
param: None,
code: Some("conflict_error".to_string()),
},
};
ErrorResponse { error }
+73 -4
View File
@@ -3,22 +3,75 @@
use std::sync::Arc;
use axum::Json;
use axum::body::Bytes;
use axum::extract::State;
use axum::http::StatusCode;
use serde::Deserialize;
use tracing::info;
use vllm_engine_core_client::Error as EngineCoreClientError;
use crate::error::ApiError;
use crate::state::AppState;
use crate::utils::utility_call_error;
#[derive(Debug, Default, Deserialize)]
pub(crate) struct StartProfileRequest {
profile_prefix: Option<String>,
delay_iterations: Option<u64>,
max_iterations: Option<u64>,
}
fn valid_profile_prefix(prefix: &str) -> bool {
let bytes = prefix.as_bytes();
(1..=128).contains(&bytes.len())
&& bytes[0].is_ascii_alphanumeric()
&& bytes[1..]
.iter()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
}
/// Start profiling the engine.
pub async fn start_profile(State(state): State<Arc<AppState>>) -> Result<StatusCode, ApiError> {
pub async fn start_profile(
State(state): State<Arc<AppState>>,
body: Bytes,
) -> Result<StatusCode, ApiError> {
let body = if body.is_empty() {
StartProfileRequest::default()
} else {
Json::<StartProfileRequest>::from_bytes(&body)?.0
};
if body
.profile_prefix
.as_deref()
.is_some_and(|prefix| !valid_profile_prefix(prefix))
{
return Err(ApiError::invalid_request(
"profile_prefix must be 1-128 ASCII characters, start with a letter or \
number, and contain only letters, numbers, '.', '_', or '-'"
.to_string(),
Some("profile_prefix"),
));
}
info!("starting profiler");
state
if let Err(error) = state
.engine_core_client()
.start_profile(None)
.start_profile(
body.profile_prefix.as_deref(),
body.delay_iterations,
body.max_iterations,
)
.await
.map_err(|error| utility_call_error("start_profile", error))?;
{
if matches!(error, EngineCoreClientError::ProfileAlreadyActive) {
return Err(ApiError::conflict(
"A profiling session is already active. Call /stop_profile before \
starting another session."
.to_string(),
));
}
return Err(utility_call_error("start_profile", error));
}
info!("profiler started");
Ok(StatusCode::OK)
}
@@ -34,3 +87,19 @@ pub async fn stop_profile(State(state): State<Arc<AppState>>) -> Result<StatusCo
info!("profiler stopped");
Ok(StatusCode::OK)
}
#[cfg(test)]
mod tests {
use super::valid_profile_prefix;
#[test]
fn validates_profile_prefixes() {
for prefix in ["benchmark", "sharegpt_run-1", "profile.2"] {
assert!(valid_profile_prefix(prefix), "{prefix}");
}
for prefix in ["", "../trace", "/tmp/trace", "trace name", "_trace"] {
assert!(!valid_profile_prefix(prefix), "{prefix}");
}
assert!(!valid_profile_prefix(&"a".repeat(129)));
}
}
+276 -24
View File
@@ -7802,27 +7802,146 @@ where
)
}
/// An empty body starts profiling without overrides, with or without a JSON
/// content type.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn start_profile_route_sends_expected_utility_call() {
for content_type in [None, Some("application/json")] {
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let start = vec![Value::from(true), Value::Nil, Value::Nil, Value::Nil];
let call_id = recv_profile_call(dealer, start).await;
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
})
})
.await;
let mut request = Request::builder().method("POST").uri("/start_profile");
if let Some(content_type) = content_type {
request = request.header("content-type", content_type);
}
let response = app
.clone()
.call(request.body(Body::empty()).expect("build request"))
.await
.expect("call app");
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(
status,
StatusCode::OK,
"{content_type:?}: {}",
String::from_utf8_lossy(&body)
);
assert!(body.is_empty());
engine_task.await.expect("mock engine task");
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn start_profile_route_forwards_session_overrides() {
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("profile"));
assert_eq!(array[3], Value::Array(vec![Value::from(true), Value::Nil]));
let start = vec![
Value::from(true),
Value::from("sharegpt_run-1"),
Value::from(5000),
Value::from(20),
];
let call_id = recv_profile_call(dealer, start).await;
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
})
})
.await;
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/start_profile")
.header("content-type", "application/json")
.body(Body::from(
json!({
"profile_prefix": "sharegpt_run-1",
"delay_iterations": 5000,
"max_iterations": 20,
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn start_profile_route_ignores_completion_payload() {
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let start = vec![Value::from(true), Value::Nil, Value::Nil, Value::Nil];
let call_id = recv_profile_call(dealer, start).await;
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
})
})
.await;
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/start_profile")
.header("content-type", "application/json")
.body(Body::from(
json!({
"model": "Qwen/Qwen3-0.6B",
"prompt": "profile this request",
"max_tokens": 16,
"stream": false,
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn duplicate_start_profile_returns_conflict() {
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let start = vec![Value::from(true), Value::Nil, Value::Nil, Value::Nil];
let call_id = recv_profile_call(dealer, start).await;
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
})
})
.await;
let first = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/start_profile")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
let second = app
.clone()
.call(
Request::builder()
@@ -7834,10 +7953,151 @@ async fn start_profile_route_sends_expected_utility_call() {
.await
.expect("call app");
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
assert_eq!(first.status(), StatusCode::OK);
assert_eq!(second.status(), StatusCode::CONFLICT);
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn start_profile_succeeds_again_after_stop() {
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let expected_args = [
vec![Value::from(true), Value::Nil, Value::Nil, Value::Nil],
vec![Value::from(false), Value::Nil],
vec![Value::from(true), Value::Nil, Value::Nil, Value::Nil],
vec![Value::from(false), Value::Nil],
];
for expected in expected_args {
let call_id = recv_profile_call(dealer, expected).await;
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
}
})
})
.await;
for uri in [
"/start_profile",
"/stop_profile",
"/start_profile",
"/stop_profile",
] {
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri(uri)
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK, "{uri}");
}
engine_task.await.expect("mock engine task");
}
/// Receive a `profile` utility call with the expected args and return its call id.
async fn recv_profile_call(dealer: &mut DealerSocket, expected_args: Vec<Value>) -> u64 {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
assert_eq!(array[2], Value::from("profile"));
assert_eq!(array[3], Value::Array(expected_args));
array[1].as_u64().expect("call id")
}
/// A start issued while a stop is in flight waits for that stop instead of
/// racing it, so a stale stop reply cannot clear the new session.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn start_profile_waits_for_inflight_stop() {
let (stop_received_tx, stop_received_rx) = tokio::sync::oneshot::channel();
let (release_stop_tx, release_stop_rx) = tokio::sync::oneshot::channel::<()>();
let (app, engine_task) = test_profiling_app_with_engine_script(move |dealer, push| {
boxed_test_future(async move {
let start = || vec![Value::from(true), Value::Nil, Value::Nil, Value::Nil];
let call_id = recv_profile_call(dealer, start()).await;
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
let call_id = recv_profile_call(dealer, vec![Value::from(false), Value::Nil]).await;
stop_received_tx.send(()).expect("notify stop received");
release_stop_rx.await.expect("release stop");
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
let call_id = recv_profile_call(dealer, start()).await;
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
})
})
.await;
let post = |uri: &'static str| {
let mut app = app.clone();
async move {
app.call(
Request::builder()
.method("POST")
.uri(uri)
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app")
.status()
}
};
let timeout = Duration::from_secs(2);
assert_eq!(post("/start_profile").await, StatusCode::OK);
let stop = tokio::spawn(post("/stop_profile"));
tokio::time::timeout(timeout, stop_received_rx)
.await
.expect("stop dispatch timeout")
.expect("stop received");
// Nothing before the profile lock awaits, so one poll reaches it; an
// unserialized client would instead reject against the stale session.
let mut restart = std::pin::pin!(post("/start_profile"));
assert!(futures::poll!(restart.as_mut()).is_pending());
release_stop_tx.send(()).expect("release stop");
let stop = tokio::time::timeout(timeout, stop).await.expect("stop timeout");
assert_eq!(stop.expect("stop task"), StatusCode::OK);
let restart = tokio::time::timeout(timeout, restart).await.expect("restart timeout");
assert_eq!(restart, StatusCode::OK);
tokio::time::timeout(timeout, engine_task)
.await
.expect("mock engine timeout")
.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn start_profile_route_rejects_invalid_prefix() {
let (app, engine_task) =
test_profiling_app_with_engine_script(|_, _| boxed_test_future(async move {})).await;
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/start_profile")
.header("content-type", "application/json")
.body(Body::from(
json!({"profile_prefix": "../trace"}).to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
engine_task.await.expect("mock engine task");
}
@@ -7846,16 +8106,8 @@ async fn start_profile_route_sends_expected_utility_call() {
async fn stop_profile_route_sends_expected_utility_call() {
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("profile"));
assert_eq!(array[3], Value::Array(vec![Value::from(false), Value::Nil]));
let stop = vec![Value::from(false), Value::Nil];
let call_id = recv_profile_call(dealer, stop).await;
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
})
})