[Profiler] Add per-session profiling controls to Python and Rust frontends (#57875)
Signed-off-by: czhu <conway.zhu@cohere.com> Signed-off-by: Conway Zhu <conway.zhu@cohere.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -22,6 +22,8 @@ pub enum ApiError {
|
||||
ModelNotFound { model: String },
|
||||
/// The request body could not be parsed as valid JSON.
|
||||
JsonParseError { message: String },
|
||||
/// The requested operation conflicts with current server state.
|
||||
Conflict { message: String },
|
||||
/// An unexpected internal failure happened before streaming started.
|
||||
ServerError { message: String },
|
||||
}
|
||||
@@ -32,6 +34,7 @@ impl ApiError {
|
||||
match self {
|
||||
Self::InvalidRequest { .. } => StatusCode::BAD_REQUEST,
|
||||
Self::ModelNotFound { .. } => StatusCode::NOT_FOUND,
|
||||
Self::Conflict { .. } => StatusCode::CONFLICT,
|
||||
Self::ServerError { .. } => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Self::JsonParseError { .. } => StatusCode::BAD_REQUEST,
|
||||
}
|
||||
@@ -65,6 +68,12 @@ impl ApiError {
|
||||
param: None,
|
||||
code: Some("json_parse_error".to_string()),
|
||||
},
|
||||
Self::Conflict { message } => ErrorDetail {
|
||||
message: message.clone(),
|
||||
error_type: "conflict_error".to_string(),
|
||||
param: None,
|
||||
code: Some("conflict_error".to_string()),
|
||||
},
|
||||
};
|
||||
|
||||
ErrorResponse { error }
|
||||
|
||||
@@ -3,22 +3,75 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::body::Bytes;
|
||||
use axum::extract::State;
|
||||
use axum::http::StatusCode;
|
||||
use serde::Deserialize;
|
||||
use tracing::info;
|
||||
use vllm_engine_core_client::Error as EngineCoreClientError;
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::utility_call_error;
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
pub(crate) struct StartProfileRequest {
|
||||
profile_prefix: Option<String>,
|
||||
delay_iterations: Option<u64>,
|
||||
max_iterations: Option<u64>,
|
||||
}
|
||||
|
||||
fn valid_profile_prefix(prefix: &str) -> bool {
|
||||
let bytes = prefix.as_bytes();
|
||||
(1..=128).contains(&bytes.len())
|
||||
&& bytes[0].is_ascii_alphanumeric()
|
||||
&& bytes[1..]
|
||||
.iter()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
|
||||
}
|
||||
|
||||
/// Start profiling the engine.
|
||||
pub async fn start_profile(State(state): State<Arc<AppState>>) -> Result<StatusCode, ApiError> {
|
||||
pub async fn start_profile(
|
||||
State(state): State<Arc<AppState>>,
|
||||
body: Bytes,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
let body = if body.is_empty() {
|
||||
StartProfileRequest::default()
|
||||
} else {
|
||||
Json::<StartProfileRequest>::from_bytes(&body)?.0
|
||||
};
|
||||
if body
|
||||
.profile_prefix
|
||||
.as_deref()
|
||||
.is_some_and(|prefix| !valid_profile_prefix(prefix))
|
||||
{
|
||||
return Err(ApiError::invalid_request(
|
||||
"profile_prefix must be 1-128 ASCII characters, start with a letter or \
|
||||
number, and contain only letters, numbers, '.', '_', or '-'"
|
||||
.to_string(),
|
||||
Some("profile_prefix"),
|
||||
));
|
||||
}
|
||||
info!("starting profiler");
|
||||
state
|
||||
if let Err(error) = state
|
||||
.engine_core_client()
|
||||
.start_profile(None)
|
||||
.start_profile(
|
||||
body.profile_prefix.as_deref(),
|
||||
body.delay_iterations,
|
||||
body.max_iterations,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("start_profile", error))?;
|
||||
{
|
||||
if matches!(error, EngineCoreClientError::ProfileAlreadyActive) {
|
||||
return Err(ApiError::conflict(
|
||||
"A profiling session is already active. Call /stop_profile before \
|
||||
starting another session."
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
return Err(utility_call_error("start_profile", error));
|
||||
}
|
||||
info!("profiler started");
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
@@ -34,3 +87,19 @@ pub async fn stop_profile(State(state): State<Arc<AppState>>) -> Result<StatusCo
|
||||
info!("profiler stopped");
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::valid_profile_prefix;
|
||||
|
||||
#[test]
|
||||
fn validates_profile_prefixes() {
|
||||
for prefix in ["benchmark", "sharegpt_run-1", "profile.2"] {
|
||||
assert!(valid_profile_prefix(prefix), "{prefix}");
|
||||
}
|
||||
for prefix in ["", "../trace", "/tmp/trace", "trace name", "_trace"] {
|
||||
assert!(!valid_profile_prefix(prefix), "{prefix}");
|
||||
}
|
||||
assert!(!valid_profile_prefix(&"a".repeat(129)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7802,27 +7802,146 @@ where
|
||||
)
|
||||
}
|
||||
|
||||
/// An empty body starts profiling without overrides, with or without a JSON
|
||||
/// content type.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn start_profile_route_sends_expected_utility_call() {
|
||||
for content_type in [None, Some("application/json")] {
|
||||
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let start = vec![Value::from(true), Value::Nil, Value::Nil, Value::Nil];
|
||||
let call_id = recv_profile_call(dealer, start).await;
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let mut request = Request::builder().method("POST").uri("/start_profile");
|
||||
if let Some(content_type) = content_type {
|
||||
request = request.header("content-type", content_type);
|
||||
}
|
||||
let response = app
|
||||
.clone()
|
||||
.call(request.body(Body::empty()).expect("build request"))
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::OK,
|
||||
"{content_type:?}: {}",
|
||||
String::from_utf8_lossy(&body)
|
||||
);
|
||||
assert!(body.is_empty());
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn start_profile_route_forwards_session_overrides() {
|
||||
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
|
||||
assert_eq!(array[2], Value::from("profile"));
|
||||
assert_eq!(array[3], Value::Array(vec![Value::from(true), Value::Nil]));
|
||||
|
||||
let start = vec![
|
||||
Value::from(true),
|
||||
Value::from("sharegpt_run-1"),
|
||||
Value::from(5000),
|
||||
Value::from(20),
|
||||
];
|
||||
let call_id = recv_profile_call(dealer, start).await;
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/start_profile")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"profile_prefix": "sharegpt_run-1",
|
||||
"delay_iterations": 5000,
|
||||
"max_iterations": 20,
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn start_profile_route_ignores_completion_payload() {
|
||||
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let start = vec![Value::from(true), Value::Nil, Value::Nil, Value::Nil];
|
||||
let call_id = recv_profile_call(dealer, start).await;
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/start_profile")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"prompt": "profile this request",
|
||||
"max_tokens": 16,
|
||||
"stream": false,
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn duplicate_start_profile_returns_conflict() {
|
||||
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let start = vec![Value::from(true), Value::Nil, Value::Nil, Value::Nil];
|
||||
let call_id = recv_profile_call(dealer, start).await;
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let first = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/start_profile")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
let second = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
@@ -7834,10 +7953,151 @@ async fn start_profile_route_sends_expected_utility_call() {
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
assert_eq!(first.status(), StatusCode::OK);
|
||||
assert_eq!(second.status(), StatusCode::CONFLICT);
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn start_profile_succeeds_again_after_stop() {
|
||||
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let expected_args = [
|
||||
vec![Value::from(true), Value::Nil, Value::Nil, Value::Nil],
|
||||
vec![Value::from(false), Value::Nil],
|
||||
vec![Value::from(true), Value::Nil, Value::Nil, Value::Nil],
|
||||
vec![Value::from(false), Value::Nil],
|
||||
];
|
||||
for expected in expected_args {
|
||||
let call_id = recv_profile_call(dealer, expected).await;
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
}
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
for uri in [
|
||||
"/start_profile",
|
||||
"/stop_profile",
|
||||
"/start_profile",
|
||||
"/stop_profile",
|
||||
] {
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(uri)
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK, "{uri}");
|
||||
}
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
/// Receive a `profile` utility call with the expected args and return its call id.
|
||||
async fn recv_profile_call(dealer: &mut DealerSocket, expected_args: Vec<Value>) -> u64 {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
assert_eq!(array[2], Value::from("profile"));
|
||||
assert_eq!(array[3], Value::Array(expected_args));
|
||||
array[1].as_u64().expect("call id")
|
||||
}
|
||||
|
||||
/// A start issued while a stop is in flight waits for that stop instead of
|
||||
/// racing it, so a stale stop reply cannot clear the new session.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn start_profile_waits_for_inflight_stop() {
|
||||
let (stop_received_tx, stop_received_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_stop_tx, release_stop_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let (app, engine_task) = test_profiling_app_with_engine_script(move |dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let start = || vec![Value::from(true), Value::Nil, Value::Nil, Value::Nil];
|
||||
|
||||
let call_id = recv_profile_call(dealer, start()).await;
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
|
||||
let call_id = recv_profile_call(dealer, vec![Value::from(false), Value::Nil]).await;
|
||||
stop_received_tx.send(()).expect("notify stop received");
|
||||
release_stop_rx.await.expect("release stop");
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
|
||||
let call_id = recv_profile_call(dealer, start()).await;
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let post = |uri: &'static str| {
|
||||
let mut app = app.clone();
|
||||
async move {
|
||||
app.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(uri)
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app")
|
||||
.status()
|
||||
}
|
||||
};
|
||||
|
||||
let timeout = Duration::from_secs(2);
|
||||
assert_eq!(post("/start_profile").await, StatusCode::OK);
|
||||
let stop = tokio::spawn(post("/stop_profile"));
|
||||
tokio::time::timeout(timeout, stop_received_rx)
|
||||
.await
|
||||
.expect("stop dispatch timeout")
|
||||
.expect("stop received");
|
||||
|
||||
// Nothing before the profile lock awaits, so one poll reaches it; an
|
||||
// unserialized client would instead reject against the stale session.
|
||||
let mut restart = std::pin::pin!(post("/start_profile"));
|
||||
assert!(futures::poll!(restart.as_mut()).is_pending());
|
||||
|
||||
release_stop_tx.send(()).expect("release stop");
|
||||
let stop = tokio::time::timeout(timeout, stop).await.expect("stop timeout");
|
||||
assert_eq!(stop.expect("stop task"), StatusCode::OK);
|
||||
let restart = tokio::time::timeout(timeout, restart).await.expect("restart timeout");
|
||||
assert_eq!(restart, StatusCode::OK);
|
||||
tokio::time::timeout(timeout, engine_task)
|
||||
.await
|
||||
.expect("mock engine timeout")
|
||||
.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn start_profile_route_rejects_invalid_prefix() {
|
||||
let (app, engine_task) =
|
||||
test_profiling_app_with_engine_script(|_, _| boxed_test_future(async move {})).await;
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/start_profile")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({"profile_prefix": "../trace"}).to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
@@ -7846,16 +8106,8 @@ async fn start_profile_route_sends_expected_utility_call() {
|
||||
async fn stop_profile_route_sends_expected_utility_call() {
|
||||
let (app, engine_task) = test_profiling_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
|
||||
assert_eq!(array[2], Value::from("profile"));
|
||||
assert_eq!(array[3], Value::Array(vec![Value::from(false), Value::Nil]));
|
||||
|
||||
let stop = vec![Value::from(false), Value::Nil];
|
||||
let call_id = recv_profile_call(dealer, stop).await;
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user