[Frontend][RL] Align sleep-mode API responses and operation metrics (#52864)

Signed-off-by: Ronald1995 <ronaldautomobile@163.com>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ronald
2026-09-29 23:16:38 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent 29773c722b
commit e5f3d08d72
14 changed files with 834 additions and 48 deletions
@@ -2290,6 +2290,89 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() {
client.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn wake_up_requires_every_engine_to_be_fully_awake() {
init_tracing();
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let mut engines = Vec::new();
for engine_index in 0..2 {
let engine = spawn_mock_engine_task(
handshake_address.clone(),
EngineId::from_engine_index(engine_index).into_frame().to_vec(),
move |dealer, push| {
Box::pin(async move {
let results = if engine_index == 0 {
[true, false, true]
} else {
[false, true, true]
};
for (attempt, fully_awake) in results.into_iter().enumerate() {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]);
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call_id");
assert_eq!(array[2], Value::from("wake_up"));
let tags = if attempt == 1 {
Value::Array(vec![Value::from("weights")])
} else {
Value::Nil
};
assert_eq!(array[3], Value::Array(vec![tags]));
send_outputs(
push,
UtilityCallOutput {
engine_index: u32::from(engine_index),
timestamp: 0.0,
output: UtilityOutput {
call_id: call_id.into(),
failure_message: None,
result: Some(utility_result_value(fully_awake)),
},
}
.into(),
)
.await;
}
})
},
);
engines.push(engine);
}
let client = connect_client_with_ipc(
handshake_test_config(
handshake_address,
2,
"test-model",
Duration::from_secs(2),
5,
None,
),
&ipc,
)
.await;
for (tags, expected) in [
(None, false),
(Some(vec!["weights".to_owned()]), false),
(None, true),
] {
assert_eq!(
timeout(Duration::from_secs(2), client.wake_up(tags))
.await
.expect("wake timeout")
.expect("wake result"),
expected
);
}
for (shutdown_tx, engine_task) in engines {
let _ = shutdown_tx.send(());
engine_task.await.unwrap();
}
client.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn collective_rpc_flattens_results_from_all_engines() {
init_tracing();
+118 -1
View File
@@ -1,11 +1,13 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::time::Instant;
use prometheus_client::encoding::EncodeLabelSet;
use prometheus_client::metrics::family::Family;
use prometheus_client::registry::Registry;
use crate::{Histogram, U64Counter};
use crate::{Histogram, U64Counter, U64Gauge};
const HTTP_REQUEST_DURATION_BUCKETS: [f64; 3] = [0.1, 0.5, 1.0];
const HTTP_REQUEST_DURATION_HIGHR_BUCKETS: [f64; 21] = [
@@ -21,6 +23,10 @@ fn http_request_duration_highr_histogram() -> Histogram {
Histogram::new(HTTP_REQUEST_DURATION_HIGHR_BUCKETS.iter().copied())
}
fn sleep_mode_operation_duration_histogram() -> Histogram {
Histogram::new([0.01, 0.1, 1.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0])
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct HttpRequestLabels {
pub method: String,
@@ -34,15 +40,25 @@ pub struct HttpHandlerLabels {
pub handler: String,
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct SleepModeOperationLabels {
pub operation: &'static str,
}
pub(crate) type HttpRequestCounterFamily = Family<HttpRequestLabels, U64Counter>;
pub(crate) type HttpHandlerHistogramFamily =
Family<HttpHandlerLabels, Histogram, fn() -> Histogram>;
pub(crate) type SleepModeOperationHistogramFamily =
Family<SleepModeOperationLabels, Histogram, fn() -> Histogram>;
pub(crate) type SleepModeOperationGaugeFamily = Family<SleepModeOperationLabels, U64Gauge>;
/// API-server Prometheus families exported from the HTTP middleware layer.
pub struct ApiServerMetrics {
pub http_requests: HttpRequestCounterFamily,
pub http_request_duration_seconds: HttpHandlerHistogramFamily,
pub http_request_duration_highr_seconds: Histogram,
pub sleep_mode_operation_duration_seconds: SleepModeOperationHistogramFamily,
pub sleep_mode_operations_in_flight: SleepModeOperationGaugeFamily,
}
impl ApiServerMetrics {
@@ -71,10 +87,111 @@ impl ApiServerMetrics {
http_request_duration_highr_seconds.clone(),
);
let sleep_mode_operation_duration_seconds = Family::new_with_constructor(
sleep_mode_operation_duration_histogram as fn() -> Histogram,
);
registry.register(
"vllm:sleep_mode_operation_duration_seconds",
"Duration of one sleep-mode operation.",
sleep_mode_operation_duration_seconds.clone(),
);
let sleep_mode_operations_in_flight = SleepModeOperationGaugeFamily::default();
registry.register(
"vllm:sleep_mode_operations_in_flight",
"Sleep-mode operations currently awaited.",
sleep_mode_operations_in_flight.clone(),
);
Self {
http_requests,
http_request_duration_seconds,
http_request_duration_highr_seconds,
sleep_mode_operation_duration_seconds,
sleep_mode_operations_in_flight,
}
}
/// Record one dispatched sleep-mode engine operation.
pub fn record_sleep_mode_operation(
&self,
operation: &'static str,
) -> SleepModeOperationRecorder {
let labels = SleepModeOperationLabels { operation };
let in_flight = self.sleep_mode_operations_in_flight.get_or_create_owned(&labels);
in_flight.inc();
SleepModeOperationRecorder {
started_at: Instant::now(),
duration: self.sleep_mode_operation_duration_seconds.get_or_create_owned(&labels),
in_flight,
}
}
}
/// Records duration and clears in-flight tracking on completion or cancellation.
pub struct SleepModeOperationRecorder {
started_at: Instant,
duration: Histogram,
in_flight: U64Gauge,
}
impl Drop for SleepModeOperationRecorder {
fn drop(&mut self) {
self.in_flight.dec();
self.duration.observe(self.started_at.elapsed().as_secs_f64());
}
}
#[cfg(test)]
mod tests {
use prometheus_client::encoding::text::encode;
use prometheus_client::registry::Registry;
use super::ApiServerMetrics;
fn rendered_metrics(registry: &Registry) -> String {
let mut rendered = String::new();
encode(&mut rendered, registry).expect("encode metrics");
rendered
}
#[test]
fn sleep_mode_recorder_tracks_duration_and_concurrency() {
let mut registry = Registry::default();
let metrics = ApiServerMetrics::register(&mut registry);
for operation in ["sleep", "release_kv_cache_memory", "wake"] {
let recorder = metrics.record_sleep_mode_operation(operation);
let active = rendered_metrics(&registry);
assert!(active.contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 1"
)));
drop(recorder);
let succeeded = rendered_metrics(&registry);
assert!(succeeded.contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 0"
)));
assert!(succeeded.contains(&format!(
"vllm:sleep_mode_operation_duration_seconds_count{{operation=\"{operation}\"}} 1"
)));
let first = metrics.record_sleep_mode_operation(operation);
let second = metrics.record_sleep_mode_operation(operation);
assert!(rendered_metrics(&registry).contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 2"
)));
drop(first);
assert!(rendered_metrics(&registry).contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 1"
)));
drop(second);
let rendered = rendered_metrics(&registry);
assert!(rendered.contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 0"
)));
assert!(rendered.contains(&format!(
"vllm:sleep_mode_operation_duration_seconds_count{{operation=\"{operation}\"}} 3"
)));
}
}
}
@@ -25,10 +25,6 @@ const EXCLUDED_HANDLERS: &[&str] = &[
"/reset_mm_cache",
"/reset_encoder_cache",
"/collective_rpc",
"/sleep",
"/release_kv_cache_memory",
"/wake_up",
"/is_sleeping",
];
/// Map the request method to a bounded Prometheus label.
+47 -12
View File
@@ -6,9 +6,9 @@ use std::sync::Arc;
use axum::Json;
use axum::extract::rejection::QueryRejection;
use axum::extract::{Query, RawQuery, State};
use axum::http::StatusCode;
use serde::{Deserialize, Serialize};
use vllm_engine_core_client::protocol::utility::PauseMode;
use vllm_metrics::METRICS;
use crate::error::ApiError;
use crate::state::AppState;
@@ -19,6 +19,24 @@ pub(crate) struct IsSleepingResponse {
is_sleeping: bool,
}
#[derive(Serialize)]
pub struct SleepResponse {
status: &'static str,
level: u32,
}
#[derive(Serialize)]
pub struct ReleaseKvCacheMemoryResponse {
status: &'static str,
}
#[derive(Serialize)]
pub struct WakeUpResponse {
status: &'static str,
/// Tags requested by the caller; null means wake all resources.
tags: Option<Vec<String>>,
}
#[derive(Debug, Deserialize)]
pub(crate) struct SleepParams {
#[serde(default = "default_sleep_level")]
@@ -32,43 +50,56 @@ const fn default_sleep_level() -> u32 {
}
fn invalid_query(error: QueryRejection) -> ApiError {
ApiError::invalid_request(error.body_text(), Some("mode"))
ApiError::invalid_request(error.body_text(), None)
}
/// Put the engine to sleep.
pub async fn sleep(
State(state): State<Arc<AppState>>,
params: Result<Query<SleepParams>, QueryRejection>,
) -> Result<StatusCode, ApiError> {
let Query(params) = params.map_err(invalid_query)?;
) -> Result<Json<SleepResponse>, ApiError> {
let Query(SleepParams { level, mode }) = params.map_err(invalid_query)?;
if level > 2 {
return Err(ApiError::invalid_request(
"level must be between 0 and 2",
Some("level"),
));
}
let _recorder = METRICS.api_server.record_sleep_mode_operation("sleep");
state
.engine_core_client()
.sleep(params.level, params.mode)
.sleep(level, mode)
.await
.map_err(|error| utility_call_error("sleep", error))?;
Ok(StatusCode::OK)
Ok(Json(SleepResponse {
status: "sleeping",
level,
}))
}
/// Release KV cache memory while keeping model weights resident.
pub async fn release_kv_cache_memory(
State(state): State<Arc<AppState>>,
) -> Result<StatusCode, ApiError> {
) -> Result<Json<ReleaseKvCacheMemoryResponse>, ApiError> {
let _recorder = METRICS.api_server.record_sleep_mode_operation("release_kv_cache_memory");
state
.engine_core_client()
.release_kv_cache_memory()
.await
.map_err(|error| utility_call_error("release_kv_cache_memory", error))?;
Ok(StatusCode::OK)
Ok(Json(ReleaseKvCacheMemoryResponse {
status: "kv_cache_released",
}))
}
/// Wake the engine from sleep mode.
pub async fn wake_up(
State(state): State<Arc<AppState>>,
RawQuery(query): RawQuery,
) -> Result<StatusCode, ApiError> {
) -> Result<Json<WakeUpResponse>, ApiError> {
let tags = query.and_then(|query| {
let tags = url::form_urlencoded::parse(query.as_bytes())
.filter(|(key, _)| key == "tags")
@@ -77,13 +108,17 @@ pub async fn wake_up(
(!tags.is_empty()).then_some(tags)
});
state
let _recorder = METRICS.api_server.record_sleep_mode_operation("wake");
let fully_awake = state
.engine_core_client()
.wake_up(tags)
.wake_up(tags.clone())
.await
.map_err(|error| utility_call_error("wake_up", error))?;
Ok(StatusCode::OK)
Ok(Json(WakeUpResponse {
status: if fully_awake { "awake" } else { "sleeping" },
tags,
}))
}
/// Return whether the engine is currently sleeping at any level.
+279 -5
View File
@@ -5892,6 +5892,7 @@ async fn weight_checker_route_rejects_invalid_requests_before_engine_calls() {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn sleep_route_uses_python_compatible_default_query_values() {
let before = METRICS.render().expect("render metrics");
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
@@ -5927,13 +5928,95 @@ async fn sleep_route_uses_python_compatible_default_query_values() {
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
json!({ "status": "sleeping", "level": 1 })
);
engine_task.await.expect("mock engine task");
let after = METRICS.render().expect("render metrics");
assert_eq!(
metric_delta(
&before,
&after,
"http_requests_total",
Some("method=\"POST\",status=\"2xx\",handler=\"/sleep\""),
),
1.0
);
assert_eq!(
metric_delta(
&before,
&after,
"vllm:sleep_mode_operation_duration_seconds_count",
Some("operation=\"sleep\""),
),
1.0
);
assert_eq!(
metric_value(
&after,
"vllm:sleep_mode_operations_in_flight",
Some("operation=\"sleep\""),
),
Some(0.0)
);
assert_eq!(
metric_delta(
&before,
&after,
"http_request_duration_seconds_count",
Some("method=\"POST\",handler=\"/sleep\""),
),
1.0
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn sleep_route_rejects_invalid_query_before_engine_dispatch() {
let before = METRICS.render().expect("render metrics");
let (app, engine_task) =
test_admin_app_with_engine_script(|_dealer, _push| boxed_test_future(async move {})).await;
for query in ["level=invalid", "level=-1", "level=3", "mode=invalid"] {
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri(format!("/sleep?{query}"))
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{query}");
}
engine_task.abort_and_join().await;
let after = METRICS.render().expect("render metrics");
assert_eq!(
metric_delta(
&before,
&after,
"http_requests_total",
Some("method=\"POST\",status=\"4xx\",handler=\"/sleep\""),
),
4.0
);
for metric in [
"vllm:sleep_mode_operation_duration_seconds_count",
"vllm:sleep_mode_operations_in_flight",
] {
assert_eq!(
metric_delta(&before, &after, metric, Some("operation=\"sleep\"")),
0.0
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn release_kv_cache_memory_route_sends_expected_utility_call() {
let before = METRICS.render().expect("render metrics");
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
@@ -5966,13 +6049,177 @@ async fn release_kv_cache_memory_route_sends_expected_utility_call() {
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
json!({ "status": "kv_cache_released" })
);
engine_task.await.expect("mock engine task");
let after = METRICS.render().expect("render metrics");
assert_eq!(
metric_delta(
&before,
&after,
"http_requests_total",
Some("method=\"POST\",status=\"2xx\",handler=\"/release_kv_cache_memory\""),
),
1.0
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn release_kv_cache_memory_error_records_http_status_and_operation_duration() {
let before = METRICS.render().expect("render metrics");
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
let payload = decode_value(&utility[1]).expect("decode utility payload");
let call_id =
payload.as_array().expect("utility payload array")[1].as_u64().expect("call id");
send_outputs(
push,
UtilityCallOutput {
output: UtilityOutput {
call_id: call_id.into(),
failure_message: Some("requires a completed pause first".to_owned()),
result: None,
},
..Default::default()
}
.into(),
)
.await;
})
})
.await;
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/release_kv_cache_memory")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
engine_task.await.expect("mock engine task");
let after = METRICS.render().expect("render metrics");
assert_eq!(
metric_delta(
&before,
&after,
"http_requests_total",
Some("method=\"POST\",status=\"5xx\",handler=\"/release_kv_cache_memory\""),
),
1.0
);
assert_eq!(
metric_delta(
&before,
&after,
"vllm:sleep_mode_operation_duration_seconds_count",
Some("operation=\"release_kv_cache_memory\""),
),
1.0
);
assert_eq!(
metric_value(
&after,
"vllm:sleep_mode_operations_in_flight",
Some("operation=\"release_kv_cache_memory\""),
),
Some(0.0)
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn cancelled_wake_operation_clears_in_flight_and_is_scraped() {
let before = METRICS.render().expect("render metrics");
let (dispatched_tx, dispatched_rx) = tokio::sync::oneshot::channel();
let (app, engine_task) = test_admin_app_with_engine_script(move |dealer, _push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
let payload = decode_value(&utility[1]).expect("decode utility payload");
assert_eq!(
payload.as_array().expect("utility payload array")[2],
Value::from("wake_up")
);
dispatched_tx.send(()).expect("notify dispatch");
std::future::pending::<()>().await;
})
})
.await;
let mut request_app = app.clone();
let request_task = tokio::spawn(async move {
request_app
.call(
Request::builder()
.method("POST")
.uri("/wake_up")
.body(Body::empty())
.expect("build request"),
)
.await
});
tokio::time::timeout(Duration::from_secs(2), dispatched_rx)
.await
.expect("wake dispatch timeout")
.expect("wake dispatched");
let active = METRICS.render().expect("render metrics");
assert_eq!(
metric_delta(
&before,
&active,
"vllm:sleep_mode_operations_in_flight",
Some("operation=\"wake\""),
),
1.0
);
request_task.abort();
assert!(request_task.await.expect_err("request cancelled").is_cancelled());
engine_task.abort_and_join().await;
let response = app
.clone()
.call(
Request::builder()
.uri("/metrics")
.body(Body::empty())
.expect("build scrape request"),
)
.await
.expect("scrape metrics");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read metrics");
let after = String::from_utf8(body.to_vec()).expect("utf8 metrics");
assert_eq!(
metric_value(
&after,
"vllm:sleep_mode_operations_in_flight",
Some("operation=\"wake\""),
),
Some(0.0)
);
assert_eq!(
metric_delta(
&before,
&after,
"vllm:sleep_mode_operation_duration_seconds_count",
Some("operation=\"wake\""),
),
1.0
);
assert!(!after.contains("rl_sleep_mode"));
assert!(!after.contains("sleep_mode_operations_total"));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn wake_up_route_without_tags_accepts_bool_result() {
let before = METRICS.render().expect("render metrics");
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
@@ -6005,8 +6252,21 @@ async fn wake_up_route_without_tags_accepts_bool_result() {
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
json!({ "status": "awake", "tags": null })
);
engine_task.await.expect("mock engine task");
let after = METRICS.render().expect("render metrics");
assert_eq!(
metric_delta(
&before,
&after,
"http_requests_total",
Some("method=\"POST\",status=\"2xx\",handler=\"/wake_up\""),
),
1.0
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -6030,7 +6290,7 @@ async fn wake_up_route_accepts_repeated_tags() {
])])
);
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
send_outputs(push, utility_outputs(call_id, utility_result_value(false))).await;
})
})
.await;
@@ -6050,13 +6310,17 @@ async fn wake_up_route_accepts_repeated_tags() {
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
json!({ "status": "sleeping", "tags": ["weights", "kv_cache"] })
);
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn is_sleeping_route_returns_json_payload() {
let before = METRICS.render().expect("render metrics");
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
@@ -6094,6 +6358,16 @@ async fn is_sleeping_route_returns_json_payload() {
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
json!({ "is_sleeping": true })
);
let after = METRICS.render().expect("render metrics");
assert_eq!(
metric_delta(
&before,
&after,
"http_requests_total",
Some("method=\"GET\",status=\"2xx\",handler=\"/is_sleeping\""),
),
1.0
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]