[Frontend][RL] Align sleep-mode API responses and operation metrics (#52864)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5.5
parent
29773c722b
commit
e5f3d08d72
@@ -25,10 +25,6 @@ const EXCLUDED_HANDLERS: &[&str] = &[
|
||||
"/reset_mm_cache",
|
||||
"/reset_encoder_cache",
|
||||
"/collective_rpc",
|
||||
"/sleep",
|
||||
"/release_kv_cache_memory",
|
||||
"/wake_up",
|
||||
"/is_sleeping",
|
||||
];
|
||||
|
||||
/// Map the request method to a bounded Prometheus label.
|
||||
|
||||
@@ -6,9 +6,9 @@ use std::sync::Arc;
|
||||
use axum::Json;
|
||||
use axum::extract::rejection::QueryRejection;
|
||||
use axum::extract::{Query, RawQuery, State};
|
||||
use axum::http::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use vllm_engine_core_client::protocol::utility::PauseMode;
|
||||
use vllm_metrics::METRICS;
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::state::AppState;
|
||||
@@ -19,6 +19,24 @@ pub(crate) struct IsSleepingResponse {
|
||||
is_sleeping: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct SleepResponse {
|
||||
status: &'static str,
|
||||
level: u32,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ReleaseKvCacheMemoryResponse {
|
||||
status: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct WakeUpResponse {
|
||||
status: &'static str,
|
||||
/// Tags requested by the caller; null means wake all resources.
|
||||
tags: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct SleepParams {
|
||||
#[serde(default = "default_sleep_level")]
|
||||
@@ -32,43 +50,56 @@ const fn default_sleep_level() -> u32 {
|
||||
}
|
||||
|
||||
fn invalid_query(error: QueryRejection) -> ApiError {
|
||||
ApiError::invalid_request(error.body_text(), Some("mode"))
|
||||
ApiError::invalid_request(error.body_text(), None)
|
||||
}
|
||||
|
||||
/// Put the engine to sleep.
|
||||
pub async fn sleep(
|
||||
State(state): State<Arc<AppState>>,
|
||||
params: Result<Query<SleepParams>, QueryRejection>,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
let Query(params) = params.map_err(invalid_query)?;
|
||||
) -> Result<Json<SleepResponse>, ApiError> {
|
||||
let Query(SleepParams { level, mode }) = params.map_err(invalid_query)?;
|
||||
if level > 2 {
|
||||
return Err(ApiError::invalid_request(
|
||||
"level must be between 0 and 2",
|
||||
Some("level"),
|
||||
));
|
||||
}
|
||||
let _recorder = METRICS.api_server.record_sleep_mode_operation("sleep");
|
||||
|
||||
state
|
||||
.engine_core_client()
|
||||
.sleep(params.level, params.mode)
|
||||
.sleep(level, mode)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("sleep", error))?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
Ok(Json(SleepResponse {
|
||||
status: "sleeping",
|
||||
level,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Release KV cache memory while keeping model weights resident.
|
||||
pub async fn release_kv_cache_memory(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
) -> Result<Json<ReleaseKvCacheMemoryResponse>, ApiError> {
|
||||
let _recorder = METRICS.api_server.record_sleep_mode_operation("release_kv_cache_memory");
|
||||
state
|
||||
.engine_core_client()
|
||||
.release_kv_cache_memory()
|
||||
.await
|
||||
.map_err(|error| utility_call_error("release_kv_cache_memory", error))?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
Ok(Json(ReleaseKvCacheMemoryResponse {
|
||||
status: "kv_cache_released",
|
||||
}))
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep mode.
|
||||
pub async fn wake_up(
|
||||
State(state): State<Arc<AppState>>,
|
||||
RawQuery(query): RawQuery,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
) -> Result<Json<WakeUpResponse>, ApiError> {
|
||||
let tags = query.and_then(|query| {
|
||||
let tags = url::form_urlencoded::parse(query.as_bytes())
|
||||
.filter(|(key, _)| key == "tags")
|
||||
@@ -77,13 +108,17 @@ pub async fn wake_up(
|
||||
(!tags.is_empty()).then_some(tags)
|
||||
});
|
||||
|
||||
state
|
||||
let _recorder = METRICS.api_server.record_sleep_mode_operation("wake");
|
||||
let fully_awake = state
|
||||
.engine_core_client()
|
||||
.wake_up(tags)
|
||||
.wake_up(tags.clone())
|
||||
.await
|
||||
.map_err(|error| utility_call_error("wake_up", error))?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
Ok(Json(WakeUpResponse {
|
||||
status: if fully_awake { "awake" } else { "sleeping" },
|
||||
tags,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Return whether the engine is currently sleeping at any level.
|
||||
|
||||
@@ -5892,6 +5892,7 @@ async fn weight_checker_route_rejects_invalid_requests_before_engine_calls() {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn sleep_route_uses_python_compatible_default_query_values() {
|
||||
let before = METRICS.render().expect("render metrics");
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
@@ -5927,13 +5928,95 @@ async fn sleep_route_uses_python_compatible_default_query_values() {
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
|
||||
json!({ "status": "sleeping", "level": 1 })
|
||||
);
|
||||
engine_task.await.expect("mock engine task");
|
||||
let after = METRICS.render().expect("render metrics");
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"http_requests_total",
|
||||
Some("method=\"POST\",status=\"2xx\",handler=\"/sleep\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"vllm:sleep_mode_operation_duration_seconds_count",
|
||||
Some("operation=\"sleep\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
assert_eq!(
|
||||
metric_value(
|
||||
&after,
|
||||
"vllm:sleep_mode_operations_in_flight",
|
||||
Some("operation=\"sleep\""),
|
||||
),
|
||||
Some(0.0)
|
||||
);
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"http_request_duration_seconds_count",
|
||||
Some("method=\"POST\",handler=\"/sleep\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn sleep_route_rejects_invalid_query_before_engine_dispatch() {
|
||||
let before = METRICS.render().expect("render metrics");
|
||||
let (app, engine_task) =
|
||||
test_admin_app_with_engine_script(|_dealer, _push| boxed_test_future(async move {})).await;
|
||||
for query in ["level=invalid", "level=-1", "level=3", "mode=invalid"] {
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(format!("/sleep?{query}"))
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{query}");
|
||||
}
|
||||
engine_task.abort_and_join().await;
|
||||
let after = METRICS.render().expect("render metrics");
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"http_requests_total",
|
||||
Some("method=\"POST\",status=\"4xx\",handler=\"/sleep\""),
|
||||
),
|
||||
4.0
|
||||
);
|
||||
for metric in [
|
||||
"vllm:sleep_mode_operation_duration_seconds_count",
|
||||
"vllm:sleep_mode_operations_in_flight",
|
||||
] {
|
||||
assert_eq!(
|
||||
metric_delta(&before, &after, metric, Some("operation=\"sleep\"")),
|
||||
0.0
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn release_kv_cache_memory_route_sends_expected_utility_call() {
|
||||
let before = METRICS.render().expect("render metrics");
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
@@ -5966,13 +6049,177 @@ async fn release_kv_cache_memory_route_sends_expected_utility_call() {
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
|
||||
json!({ "status": "kv_cache_released" })
|
||||
);
|
||||
engine_task.await.expect("mock engine task");
|
||||
let after = METRICS.render().expect("render metrics");
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"http_requests_total",
|
||||
Some("method=\"POST\",status=\"2xx\",handler=\"/release_kv_cache_memory\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn release_kv_cache_memory_error_records_http_status_and_operation_duration() {
|
||||
let before = METRICS.render().expect("render metrics");
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let call_id =
|
||||
payload.as_array().expect("utility payload array")[1].as_u64().expect("call id");
|
||||
send_outputs(
|
||||
push,
|
||||
UtilityCallOutput {
|
||||
output: UtilityOutput {
|
||||
call_id: call_id.into(),
|
||||
failure_message: Some("requires a completed pause first".to_owned()),
|
||||
result: None,
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/release_kv_cache_memory")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
engine_task.await.expect("mock engine task");
|
||||
let after = METRICS.render().expect("render metrics");
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"http_requests_total",
|
||||
Some("method=\"POST\",status=\"5xx\",handler=\"/release_kv_cache_memory\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"vllm:sleep_mode_operation_duration_seconds_count",
|
||||
Some("operation=\"release_kv_cache_memory\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
assert_eq!(
|
||||
metric_value(
|
||||
&after,
|
||||
"vllm:sleep_mode_operations_in_flight",
|
||||
Some("operation=\"release_kv_cache_memory\""),
|
||||
),
|
||||
Some(0.0)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn cancelled_wake_operation_clears_in_flight_and_is_scraped() {
|
||||
let before = METRICS.render().expect("render metrics");
|
||||
let (dispatched_tx, dispatched_rx) = tokio::sync::oneshot::channel();
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(move |dealer, _push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
assert_eq!(
|
||||
payload.as_array().expect("utility payload array")[2],
|
||||
Value::from("wake_up")
|
||||
);
|
||||
dispatched_tx.send(()).expect("notify dispatch");
|
||||
std::future::pending::<()>().await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
let mut request_app = app.clone();
|
||||
let request_task = tokio::spawn(async move {
|
||||
request_app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/wake_up")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(2), dispatched_rx)
|
||||
.await
|
||||
.expect("wake dispatch timeout")
|
||||
.expect("wake dispatched");
|
||||
let active = METRICS.render().expect("render metrics");
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&active,
|
||||
"vllm:sleep_mode_operations_in_flight",
|
||||
Some("operation=\"wake\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
request_task.abort();
|
||||
assert!(request_task.await.expect_err("request cancelled").is_cancelled());
|
||||
engine_task.abort_and_join().await;
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.uri("/metrics")
|
||||
.body(Body::empty())
|
||||
.expect("build scrape request"),
|
||||
)
|
||||
.await
|
||||
.expect("scrape metrics");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read metrics");
|
||||
let after = String::from_utf8(body.to_vec()).expect("utf8 metrics");
|
||||
assert_eq!(
|
||||
metric_value(
|
||||
&after,
|
||||
"vllm:sleep_mode_operations_in_flight",
|
||||
Some("operation=\"wake\""),
|
||||
),
|
||||
Some(0.0)
|
||||
);
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"vllm:sleep_mode_operation_duration_seconds_count",
|
||||
Some("operation=\"wake\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
assert!(!after.contains("rl_sleep_mode"));
|
||||
assert!(!after.contains("sleep_mode_operations_total"));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn wake_up_route_without_tags_accepts_bool_result() {
|
||||
let before = METRICS.render().expect("render metrics");
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
@@ -6005,8 +6252,21 @@ async fn wake_up_route_without_tags_accepts_bool_result() {
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
|
||||
json!({ "status": "awake", "tags": null })
|
||||
);
|
||||
engine_task.await.expect("mock engine task");
|
||||
let after = METRICS.render().expect("render metrics");
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"http_requests_total",
|
||||
Some("method=\"POST\",status=\"2xx\",handler=\"/wake_up\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
@@ -6030,7 +6290,7 @@ async fn wake_up_route_accepts_repeated_tags() {
|
||||
])])
|
||||
);
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
|
||||
send_outputs(push, utility_outputs(call_id, utility_result_value(false))).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
@@ -6050,13 +6310,17 @@ async fn wake_up_route_accepts_repeated_tags() {
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
|
||||
json!({ "status": "sleeping", "tags": ["weights", "kv_cache"] })
|
||||
);
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn is_sleeping_route_returns_json_payload() {
|
||||
let before = METRICS.render().expect("render metrics");
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
@@ -6094,6 +6358,16 @@ async fn is_sleeping_route_returns_json_payload() {
|
||||
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
|
||||
json!({ "is_sleeping": true })
|
||||
);
|
||||
let after = METRICS.render().expect("render metrics");
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"http_requests_total",
|
||||
Some("method=\"GET\",status=\"2xx\",handler=\"/is_sleeping\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
|
||||
Reference in New Issue
Block a user