From e5f3d08d722870a5301560fad0cfa65b7e333900 Mon Sep 17 00:00:00 2001 From: Ronald Date: Wed, 30 Sep 2026 14:16:38 +0800 Subject: [PATCH] [Frontend][RL] Align sleep-mode API responses and operation metrics (#52864) Signed-off-by: Ronald1995 Signed-off-by: aoshen02 Co-authored-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- .../engine-core-client/src/tests/client.rs | 83 +++++ rust/src/metrics/src/api_server.rs | 119 +++++++- rust/src/server/src/middleware/metrics.rs | 4 - rust/src/server/src/routes/sleep.rs | 59 +++- rust/src/server/src/routes/tests.rs | 284 +++++++++++++++++- tests/entrypoints/serve/dev/test_sleep.py | 184 +++++++++++- tests/v1/engine/test_wake_up_contract.py | 32 ++ vllm/engine/protocol.py | 4 +- vllm/entrypoints/llm.py | 7 +- .../entrypoints/serve/dev/sleep/api_router.py | 39 ++- vllm/entrypoints/serve/dev/sleep/metrics.py | 57 ++++ vllm/v1/engine/async_llm.py | 4 +- vllm/v1/engine/core_client.py | 2 +- vllm/v1/engine/llm_engine.py | 4 +- 14 files changed, 834 insertions(+), 48 deletions(-) create mode 100644 tests/v1/engine/test_wake_up_contract.py create mode 100644 vllm/entrypoints/serve/dev/sleep/metrics.py diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index 5f2dc9c81e..0ac81764a9 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -2290,6 +2290,89 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() { client.shutdown().await.unwrap(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn wake_up_requires_every_engine_to_be_fully_awake() { + init_tracing(); + let ipc = IpcNamespace::new().unwrap(); + let handshake_address = ipc.handshake_endpoint(); + let mut engines = Vec::new(); + for engine_index in 0..2 { + let engine = spawn_mock_engine_task( + handshake_address.clone(), + EngineId::from_engine_index(engine_index).into_frame().to_vec(), + move |dealer, push| { + Box::pin(async move { + let results = if engine_index == 0 { + [true, false, true] + } else { + [false, true, true] + }; + for (attempt, fully_awake) in results.into_iter().enumerate() { + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + let payload = decode_value(&utility[1]); + let array = payload.as_array().expect("utility payload array"); + let call_id = array[1].as_u64().expect("call_id"); + assert_eq!(array[2], Value::from("wake_up")); + let tags = if attempt == 1 { + Value::Array(vec![Value::from("weights")]) + } else { + Value::Nil + }; + assert_eq!(array[3], Value::Array(vec![tags])); + send_outputs( + push, + UtilityCallOutput { + engine_index: u32::from(engine_index), + timestamp: 0.0, + output: UtilityOutput { + call_id: call_id.into(), + failure_message: None, + result: Some(utility_result_value(fully_awake)), + }, + } + .into(), + ) + .await; + } + }) + }, + ); + engines.push(engine); + } + let client = connect_client_with_ipc( + handshake_test_config( + handshake_address, + 2, + "test-model", + Duration::from_secs(2), + 5, + None, + ), + &ipc, + ) + .await; + + for (tags, expected) in [ + (None, false), + (Some(vec!["weights".to_owned()]), false), + (None, true), + ] { + assert_eq!( + timeout(Duration::from_secs(2), client.wake_up(tags)) + .await + .expect("wake timeout") + .expect("wake result"), + expected + ); + } + for (shutdown_tx, engine_task) in engines { + let _ = shutdown_tx.send(()); + engine_task.await.unwrap(); + } + client.shutdown().await.unwrap(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn collective_rpc_flattens_results_from_all_engines() { init_tracing(); diff --git a/rust/src/metrics/src/api_server.rs b/rust/src/metrics/src/api_server.rs index 855e9fbbb9..1dc7154998 100644 --- a/rust/src/metrics/src/api_server.rs +++ b/rust/src/metrics/src/api_server.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project +use std::time::Instant; + use prometheus_client::encoding::EncodeLabelSet; use prometheus_client::metrics::family::Family; use prometheus_client::registry::Registry; -use crate::{Histogram, U64Counter}; +use crate::{Histogram, U64Counter, U64Gauge}; const HTTP_REQUEST_DURATION_BUCKETS: [f64; 3] = [0.1, 0.5, 1.0]; const HTTP_REQUEST_DURATION_HIGHR_BUCKETS: [f64; 21] = [ @@ -21,6 +23,10 @@ fn http_request_duration_highr_histogram() -> Histogram { Histogram::new(HTTP_REQUEST_DURATION_HIGHR_BUCKETS.iter().copied()) } +fn sleep_mode_operation_duration_histogram() -> Histogram { + Histogram::new([0.01, 0.1, 1.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0]) +} + #[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] pub struct HttpRequestLabels { pub method: String, @@ -34,15 +40,25 @@ pub struct HttpHandlerLabels { pub handler: String, } +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +pub struct SleepModeOperationLabels { + pub operation: &'static str, +} + pub(crate) type HttpRequestCounterFamily = Family; pub(crate) type HttpHandlerHistogramFamily = Family Histogram>; +pub(crate) type SleepModeOperationHistogramFamily = + Family Histogram>; +pub(crate) type SleepModeOperationGaugeFamily = Family; /// API-server Prometheus families exported from the HTTP middleware layer. pub struct ApiServerMetrics { pub http_requests: HttpRequestCounterFamily, pub http_request_duration_seconds: HttpHandlerHistogramFamily, pub http_request_duration_highr_seconds: Histogram, + pub sleep_mode_operation_duration_seconds: SleepModeOperationHistogramFamily, + pub sleep_mode_operations_in_flight: SleepModeOperationGaugeFamily, } impl ApiServerMetrics { @@ -71,10 +87,111 @@ impl ApiServerMetrics { http_request_duration_highr_seconds.clone(), ); + let sleep_mode_operation_duration_seconds = Family::new_with_constructor( + sleep_mode_operation_duration_histogram as fn() -> Histogram, + ); + registry.register( + "vllm:sleep_mode_operation_duration_seconds", + "Duration of one sleep-mode operation.", + sleep_mode_operation_duration_seconds.clone(), + ); + + let sleep_mode_operations_in_flight = SleepModeOperationGaugeFamily::default(); + registry.register( + "vllm:sleep_mode_operations_in_flight", + "Sleep-mode operations currently awaited.", + sleep_mode_operations_in_flight.clone(), + ); + Self { http_requests, http_request_duration_seconds, http_request_duration_highr_seconds, + sleep_mode_operation_duration_seconds, + sleep_mode_operations_in_flight, + } + } + + /// Record one dispatched sleep-mode engine operation. + pub fn record_sleep_mode_operation( + &self, + operation: &'static str, + ) -> SleepModeOperationRecorder { + let labels = SleepModeOperationLabels { operation }; + let in_flight = self.sleep_mode_operations_in_flight.get_or_create_owned(&labels); + in_flight.inc(); + SleepModeOperationRecorder { + started_at: Instant::now(), + duration: self.sleep_mode_operation_duration_seconds.get_or_create_owned(&labels), + in_flight, + } + } +} + +/// Records duration and clears in-flight tracking on completion or cancellation. +pub struct SleepModeOperationRecorder { + started_at: Instant, + duration: Histogram, + in_flight: U64Gauge, +} + +impl Drop for SleepModeOperationRecorder { + fn drop(&mut self) { + self.in_flight.dec(); + self.duration.observe(self.started_at.elapsed().as_secs_f64()); + } +} + +#[cfg(test)] +mod tests { + use prometheus_client::encoding::text::encode; + use prometheus_client::registry::Registry; + + use super::ApiServerMetrics; + + fn rendered_metrics(registry: &Registry) -> String { + let mut rendered = String::new(); + encode(&mut rendered, registry).expect("encode metrics"); + rendered + } + + #[test] + fn sleep_mode_recorder_tracks_duration_and_concurrency() { + let mut registry = Registry::default(); + let metrics = ApiServerMetrics::register(&mut registry); + for operation in ["sleep", "release_kv_cache_memory", "wake"] { + let recorder = metrics.record_sleep_mode_operation(operation); + let active = rendered_metrics(®istry); + assert!(active.contains(&format!( + "vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 1" + ))); + + drop(recorder); + let succeeded = rendered_metrics(®istry); + assert!(succeeded.contains(&format!( + "vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 0" + ))); + assert!(succeeded.contains(&format!( + "vllm:sleep_mode_operation_duration_seconds_count{{operation=\"{operation}\"}} 1" + ))); + + let first = metrics.record_sleep_mode_operation(operation); + let second = metrics.record_sleep_mode_operation(operation); + assert!(rendered_metrics(®istry).contains(&format!( + "vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 2" + ))); + drop(first); + assert!(rendered_metrics(®istry).contains(&format!( + "vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 1" + ))); + drop(second); + let rendered = rendered_metrics(®istry); + assert!(rendered.contains(&format!( + "vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 0" + ))); + assert!(rendered.contains(&format!( + "vllm:sleep_mode_operation_duration_seconds_count{{operation=\"{operation}\"}} 3" + ))); } } } diff --git a/rust/src/server/src/middleware/metrics.rs b/rust/src/server/src/middleware/metrics.rs index de07620cf7..46f10934ec 100644 --- a/rust/src/server/src/middleware/metrics.rs +++ b/rust/src/server/src/middleware/metrics.rs @@ -25,10 +25,6 @@ const EXCLUDED_HANDLERS: &[&str] = &[ "/reset_mm_cache", "/reset_encoder_cache", "/collective_rpc", - "/sleep", - "/release_kv_cache_memory", - "/wake_up", - "/is_sleeping", ]; /// Map the request method to a bounded Prometheus label. diff --git a/rust/src/server/src/routes/sleep.rs b/rust/src/server/src/routes/sleep.rs index d64a0850b6..b6239dc2d4 100644 --- a/rust/src/server/src/routes/sleep.rs +++ b/rust/src/server/src/routes/sleep.rs @@ -6,9 +6,9 @@ use std::sync::Arc; use axum::Json; use axum::extract::rejection::QueryRejection; use axum::extract::{Query, RawQuery, State}; -use axum::http::StatusCode; use serde::{Deserialize, Serialize}; use vllm_engine_core_client::protocol::utility::PauseMode; +use vllm_metrics::METRICS; use crate::error::ApiError; use crate::state::AppState; @@ -19,6 +19,24 @@ pub(crate) struct IsSleepingResponse { is_sleeping: bool, } +#[derive(Serialize)] +pub struct SleepResponse { + status: &'static str, + level: u32, +} + +#[derive(Serialize)] +pub struct ReleaseKvCacheMemoryResponse { + status: &'static str, +} + +#[derive(Serialize)] +pub struct WakeUpResponse { + status: &'static str, + /// Tags requested by the caller; null means wake all resources. + tags: Option>, +} + #[derive(Debug, Deserialize)] pub(crate) struct SleepParams { #[serde(default = "default_sleep_level")] @@ -32,43 +50,56 @@ const fn default_sleep_level() -> u32 { } fn invalid_query(error: QueryRejection) -> ApiError { - ApiError::invalid_request(error.body_text(), Some("mode")) + ApiError::invalid_request(error.body_text(), None) } /// Put the engine to sleep. pub async fn sleep( State(state): State>, params: Result, QueryRejection>, -) -> Result { - let Query(params) = params.map_err(invalid_query)?; +) -> Result, ApiError> { + let Query(SleepParams { level, mode }) = params.map_err(invalid_query)?; + if level > 2 { + return Err(ApiError::invalid_request( + "level must be between 0 and 2", + Some("level"), + )); + } + let _recorder = METRICS.api_server.record_sleep_mode_operation("sleep"); state .engine_core_client() - .sleep(params.level, params.mode) + .sleep(level, mode) .await .map_err(|error| utility_call_error("sleep", error))?; - Ok(StatusCode::OK) + Ok(Json(SleepResponse { + status: "sleeping", + level, + })) } /// Release KV cache memory while keeping model weights resident. pub async fn release_kv_cache_memory( State(state): State>, -) -> Result { +) -> Result, ApiError> { + let _recorder = METRICS.api_server.record_sleep_mode_operation("release_kv_cache_memory"); state .engine_core_client() .release_kv_cache_memory() .await .map_err(|error| utility_call_error("release_kv_cache_memory", error))?; - Ok(StatusCode::OK) + Ok(Json(ReleaseKvCacheMemoryResponse { + status: "kv_cache_released", + })) } /// Wake the engine from sleep mode. pub async fn wake_up( State(state): State>, RawQuery(query): RawQuery, -) -> Result { +) -> Result, ApiError> { let tags = query.and_then(|query| { let tags = url::form_urlencoded::parse(query.as_bytes()) .filter(|(key, _)| key == "tags") @@ -77,13 +108,17 @@ pub async fn wake_up( (!tags.is_empty()).then_some(tags) }); - state + let _recorder = METRICS.api_server.record_sleep_mode_operation("wake"); + let fully_awake = state .engine_core_client() - .wake_up(tags) + .wake_up(tags.clone()) .await .map_err(|error| utility_call_error("wake_up", error))?; - Ok(StatusCode::OK) + Ok(Json(WakeUpResponse { + status: if fully_awake { "awake" } else { "sleeping" }, + tags, + })) } /// Return whether the engine is currently sleeping at any level. diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index b016810b59..d229be25fa 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -5892,6 +5892,7 @@ async fn weight_checker_route_rejects_invalid_requests_before_engine_calls() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn sleep_route_uses_python_compatible_default_query_values() { + let before = METRICS.render().expect("render metrics"); let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { boxed_test_future(async move { let utility = recv_engine_message(dealer).await; @@ -5927,13 +5928,95 @@ async fn sleep_route_uses_python_compatible_default_query_values() { let status = response.status(); let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); - assert!(body.is_empty()); + assert_eq!( + serde_json::from_slice::(&body).expect("decode json"), + json!({ "status": "sleeping", "level": 1 }) + ); engine_task.await.expect("mock engine task"); + let after = METRICS.render().expect("render metrics"); + assert_eq!( + metric_delta( + &before, + &after, + "http_requests_total", + Some("method=\"POST\",status=\"2xx\",handler=\"/sleep\""), + ), + 1.0 + ); + assert_eq!( + metric_delta( + &before, + &after, + "vllm:sleep_mode_operation_duration_seconds_count", + Some("operation=\"sleep\""), + ), + 1.0 + ); + assert_eq!( + metric_value( + &after, + "vllm:sleep_mode_operations_in_flight", + Some("operation=\"sleep\""), + ), + Some(0.0) + ); + assert_eq!( + metric_delta( + &before, + &after, + "http_request_duration_seconds_count", + Some("method=\"POST\",handler=\"/sleep\""), + ), + 1.0 + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn sleep_route_rejects_invalid_query_before_engine_dispatch() { + let before = METRICS.render().expect("render metrics"); + let (app, engine_task) = + test_admin_app_with_engine_script(|_dealer, _push| boxed_test_future(async move {})).await; + for query in ["level=invalid", "level=-1", "level=3", "mode=invalid"] { + let response = app + .clone() + .call( + Request::builder() + .method("POST") + .uri(format!("/sleep?{query}")) + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{query}"); + } + engine_task.abort_and_join().await; + let after = METRICS.render().expect("render metrics"); + assert_eq!( + metric_delta( + &before, + &after, + "http_requests_total", + Some("method=\"POST\",status=\"4xx\",handler=\"/sleep\""), + ), + 4.0 + ); + for metric in [ + "vllm:sleep_mode_operation_duration_seconds_count", + "vllm:sleep_mode_operations_in_flight", + ] { + assert_eq!( + metric_delta(&before, &after, metric, Some("operation=\"sleep\"")), + 0.0 + ); + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn release_kv_cache_memory_route_sends_expected_utility_call() { + let before = METRICS.render().expect("render metrics"); let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { boxed_test_future(async move { let utility = recv_engine_message(dealer).await; @@ -5966,13 +6049,177 @@ async fn release_kv_cache_memory_route_sends_expected_utility_call() { let status = response.status(); let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); - assert!(body.is_empty()); + assert_eq!( + serde_json::from_slice::(&body).expect("decode json"), + json!({ "status": "kv_cache_released" }) + ); engine_task.await.expect("mock engine task"); + let after = METRICS.render().expect("render metrics"); + assert_eq!( + metric_delta( + &before, + &after, + "http_requests_total", + Some("method=\"POST\",status=\"2xx\",handler=\"/release_kv_cache_memory\""), + ), + 1.0 + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn release_kv_cache_memory_error_records_http_status_and_operation_duration() { + let before = METRICS.render().expect("render metrics"); + let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let call_id = + payload.as_array().expect("utility payload array")[1].as_u64().expect("call id"); + send_outputs( + push, + UtilityCallOutput { + output: UtilityOutput { + call_id: call_id.into(), + failure_message: Some("requires a completed pause first".to_owned()), + result: None, + }, + ..Default::default() + } + .into(), + ) + .await; + }) + }) + .await; + let response = app + .clone() + .call( + Request::builder() + .method("POST") + .uri("/release_kv_cache_memory") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + engine_task.await.expect("mock engine task"); + let after = METRICS.render().expect("render metrics"); + assert_eq!( + metric_delta( + &before, + &after, + "http_requests_total", + Some("method=\"POST\",status=\"5xx\",handler=\"/release_kv_cache_memory\""), + ), + 1.0 + ); + assert_eq!( + metric_delta( + &before, + &after, + "vllm:sleep_mode_operation_duration_seconds_count", + Some("operation=\"release_kv_cache_memory\""), + ), + 1.0 + ); + assert_eq!( + metric_value( + &after, + "vllm:sleep_mode_operations_in_flight", + Some("operation=\"release_kv_cache_memory\""), + ), + Some(0.0) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn cancelled_wake_operation_clears_in_flight_and_is_scraped() { + let before = METRICS.render().expect("render metrics"); + let (dispatched_tx, dispatched_rx) = tokio::sync::oneshot::channel(); + let (app, engine_task) = test_admin_app_with_engine_script(move |dealer, _push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + let payload = decode_value(&utility[1]).expect("decode utility payload"); + assert_eq!( + payload.as_array().expect("utility payload array")[2], + Value::from("wake_up") + ); + dispatched_tx.send(()).expect("notify dispatch"); + std::future::pending::<()>().await; + }) + }) + .await; + let mut request_app = app.clone(); + let request_task = tokio::spawn(async move { + request_app + .call( + Request::builder() + .method("POST") + .uri("/wake_up") + .body(Body::empty()) + .expect("build request"), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(2), dispatched_rx) + .await + .expect("wake dispatch timeout") + .expect("wake dispatched"); + let active = METRICS.render().expect("render metrics"); + assert_eq!( + metric_delta( + &before, + &active, + "vllm:sleep_mode_operations_in_flight", + Some("operation=\"wake\""), + ), + 1.0 + ); + request_task.abort(); + assert!(request_task.await.expect_err("request cancelled").is_cancelled()); + engine_task.abort_and_join().await; + + let response = app + .clone() + .call( + Request::builder() + .uri("/metrics") + .body(Body::empty()) + .expect("build scrape request"), + ) + .await + .expect("scrape metrics"); + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read metrics"); + let after = String::from_utf8(body.to_vec()).expect("utf8 metrics"); + assert_eq!( + metric_value( + &after, + "vllm:sleep_mode_operations_in_flight", + Some("operation=\"wake\""), + ), + Some(0.0) + ); + assert_eq!( + metric_delta( + &before, + &after, + "vllm:sleep_mode_operation_duration_seconds_count", + Some("operation=\"wake\""), + ), + 1.0 + ); + assert!(!after.contains("rl_sleep_mode")); + assert!(!after.contains("sleep_mode_operations_total")); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn wake_up_route_without_tags_accepts_bool_result() { + let before = METRICS.render().expect("render metrics"); let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { boxed_test_future(async move { let utility = recv_engine_message(dealer).await; @@ -6005,8 +6252,21 @@ async fn wake_up_route_without_tags_accepts_bool_result() { let status = response.status(); let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); - assert!(body.is_empty()); + assert_eq!( + serde_json::from_slice::(&body).expect("decode json"), + json!({ "status": "awake", "tags": null }) + ); engine_task.await.expect("mock engine task"); + let after = METRICS.render().expect("render metrics"); + assert_eq!( + metric_delta( + &before, + &after, + "http_requests_total", + Some("method=\"POST\",status=\"2xx\",handler=\"/wake_up\""), + ), + 1.0 + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -6030,7 +6290,7 @@ async fn wake_up_route_accepts_repeated_tags() { ])]) ); - send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await; + send_outputs(push, utility_outputs(call_id, utility_result_value(false))).await; }) }) .await; @@ -6050,13 +6310,17 @@ async fn wake_up_route_accepts_repeated_tags() { let status = response.status(); let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); - assert!(body.is_empty()); + assert_eq!( + serde_json::from_slice::(&body).expect("decode json"), + json!({ "status": "sleeping", "tags": ["weights", "kv_cache"] }) + ); engine_task.await.expect("mock engine task"); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn is_sleeping_route_returns_json_payload() { + let before = METRICS.render().expect("render metrics"); let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { boxed_test_future(async move { let utility = recv_engine_message(dealer).await; @@ -6094,6 +6358,16 @@ async fn is_sleeping_route_returns_json_payload() { serde_json::from_slice::(&body).expect("decode json"), json!({ "is_sleeping": true }) ); + let after = METRICS.render().expect("render metrics"); + assert_eq!( + metric_delta( + &before, + &after, + "http_requests_total", + Some("method=\"GET\",status=\"2xx\",handler=\"/is_sleeping\""), + ), + 1.0 + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/tests/entrypoints/serve/dev/test_sleep.py b/tests/entrypoints/serve/dev/test_sleep.py index 34e69c9ece..1f91945327 100644 --- a/tests/entrypoints/serve/dev/test_sleep.py +++ b/tests/entrypoints/serve/dev/test_sleep.py @@ -1,35 +1,209 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import asyncio +from types import SimpleNamespace from unittest.mock import AsyncMock import pytest import requests from fastapi import FastAPI from fastapi.testclient import TestClient +from prometheus_client import CollectorRegistry from prometheus_client.parser import text_string_to_metric_families from tests.utils import RemoteOpenAIServer +from vllm.entrypoints.serve.dev.sleep import metrics as sleep_metrics from vllm.entrypoints.serve.dev.sleep.api_router import attach_router +from vllm.entrypoints.serve.dev.sleep.metrics import SleepModeOperationMetrics +from vllm.entrypoints.serve.exception_handling.register import init_exception_handler +from vllm.entrypoints.serve.instrumentator.metrics import ( + attach_router as attach_metrics_router, +) +from vllm.v1.metrics import prometheus as prometheus_metrics MODEL_NAME = "meta-llama/Llama-3.2-1B" +@pytest.fixture +def sleep_route_app(monkeypatch): + app = FastAPI() + app.state.args = SimpleNamespace(log_error_stack=False) + app.state.engine_client = AsyncMock() + metrics = SleepModeOperationMetrics(CollectorRegistry()) + monkeypatch.setattr( + "vllm.entrypoints.serve.dev.sleep.api_router.sleep_mode_operation_metrics", + lambda: metrics, + ) + + attach_router(app) + init_exception_handler(app) + return app, metrics + + +@pytest.mark.cpu_test +@pytest.mark.parametrize("level", [0, 1, 2]) +def test_sleep_route_response_and_engine_arguments(sleep_route_app, level): + app, _ = sleep_route_app + with TestClient(app) as client: + response = client.post("/sleep", params={"level": level}) + assert response.status_code == 200 + assert response.json() == {"status": "sleeping", "level": level} + app.state.engine_client.sleep.assert_awaited_once_with(level, "abort") + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + ("query", "expected_param"), + [ + ("level=invalid", "query.level"), + ("level=-1", "query.level"), + ("level=3", "query.level"), + ("mode=invalid", "query.mode"), + ], +) +def test_sleep_route_rejects_invalid_query_before_dispatch( + sleep_route_app, query, expected_param +): + app, metrics = sleep_route_app + with TestClient(app) as client: + response = client.post(f"/sleep?{query}") + assert response.status_code == 400 + assert response.json()["error"]["param"] == expected_param + app.state.engine_client.sleep.assert_not_awaited() + assert list(metrics.duration.collect()[0].samples) == [] + assert list(metrics.in_flight.collect()[0].samples) == [] + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + ("fully_awake", "tags", "expected"), + [ + (True, "", {"status": "awake", "tags": None}), + (False, "?tags=weights", {"status": "sleeping", "tags": ["weights"]}), + ], +) +def test_wake_route_maps_engine_result_without_state_query( + sleep_route_app, fully_awake, tags, expected +): + app, _ = sleep_route_app + app.state.engine_client.wake_up.return_value = fully_awake + with TestClient(app) as client: + response = client.post(f"/wake_up{tags}") + assert response.status_code == 200 + assert response.json() == expected + app.state.engine_client.wake_up.assert_awaited_once_with(expected["tags"]) + app.state.engine_client.is_sleeping.assert_not_awaited() + + @pytest.mark.cpu_test @pytest.mark.parametrize("fails", [False, True]) -def test_release_kv_cache_memory_route(fails): - app = FastAPI() - app.state.engine_client = AsyncMock() +def test_release_kv_cache_memory_route(sleep_route_app, fails): + app, metrics = sleep_route_app release = app.state.engine_client.release_kv_cache_memory if fails: release.side_effect = RuntimeError("requires a completed pause first") - attach_router(app) - with TestClient(app, raise_server_exceptions=False) as client: response = client.post("/release_kv_cache_memory") assert response.status_code == (500 if fails else 200) + if not fails: + assert response.json() == {"status": "kv_cache_released"} release.assert_awaited_once_with() + assert metrics.in_flight.labels("release_kv_cache_memory")._value.get() == 0 + assert _duration_count(metrics, "release_kv_cache_memory") == 1 + + +@pytest.mark.cpu_test +@pytest.mark.parametrize("operation", ["sleep", "release_kv_cache_memory", "wake"]) +@pytest.mark.parametrize("exception", [RuntimeError, asyncio.CancelledError]) +def test_sleep_mode_recorder_tracks_duration_and_cleans_up(operation, exception): + metrics = SleepModeOperationMetrics(CollectorRegistry()) + with metrics.record(operation): + assert metrics.in_flight.labels(operation)._value.get() == 1 + with pytest.raises(exception), metrics.record(operation): + raise exception() + assert metrics.in_flight.labels(operation)._value.get() == 0 + assert metrics.duration.labels(operation)._sum.get() >= 0 + assert _duration_count(metrics, operation) == 2 + + +def _duration_count(metrics, operation): + return next( + sample.value + for sample in metrics.duration.collect()[0].samples + if sample.name.endswith("_count") and sample.labels == {"operation": operation} + ) + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + ("path", "method", "operation", "status", "fails"), + [ + ("/sleep", "POST", "sleep", 200, False), + ("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 200, False), + ("/wake_up", "POST", "wake", 200, False), + ("/is_sleeping", "GET", None, 200, False), + ("/sleep?level=3", "POST", None, 400, False), + ("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 500, True), + ], +) +def test_sleep_routes_visible_on_production_metrics_endpoint( + monkeypatch, path, method, operation, status, fails +): + registry = CollectorRegistry() + monkeypatch.delenv("PROMETHEUS_MULTIPROC_DIR", raising=False) + monkeypatch.setattr(sleep_metrics, "REGISTRY", registry) + monkeypatch.setattr(sleep_metrics, "_metrics", None) + monkeypatch.setattr(prometheus_metrics, "REGISTRY", registry) + + app = FastAPI() + app.state.args = SimpleNamespace(log_error_stack=False) + app.state.engine_client = AsyncMock() + app.state.engine_client.wake_up.return_value = True + app.state.engine_client.is_sleeping.return_value = False + if fails: + app.state.engine_client.release_kv_cache_memory.side_effect = RuntimeError( + "engine failed" + ) + attach_router(app) + init_exception_handler(app) + attach_metrics_router(app) + + with TestClient(app, raise_server_exceptions=False) as client: + assert client.request(method, path).status_code == status + response = client.get("/metrics") + + assert response.status_code == 200 + samples = [ + sample + for family in text_string_to_metric_families(response.text) + for sample in family.samples + ] + assert any( + sample.name == "http_requests_total" + and sample.labels + == { + "handler": path.split("?")[0], + "method": method, + "status": f"{status // 100}xx", + } + and sample.value == 1 + for sample in samples + ) + assert not any("sleep_mode_operations_total" in sample.name for sample in samples) + assert any( + sample.name == "vllm:sleep_mode_operations_in_flight" + and sample.labels == {"operation": operation} + and sample.value == 0 + for sample in samples + ) == (operation is not None) + assert any( + sample.name == "vllm:sleep_mode_operation_duration_seconds_count" + and sample.labels == {"operation": operation} + and sample.value == 1 + for sample in samples + ) == (operation is not None) def test_sleep_mode(): diff --git a/tests/v1/engine/test_wake_up_contract.py b/tests/v1/engine/test_wake_up_contract.py new file mode 100644 index 0000000000..1369cece95 --- /dev/null +++ b/tests/v1/engine/test_wake_up_contract.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from unittest.mock import AsyncMock, call + +import pytest + +from vllm.v1.engine.core_client import DPLBAsyncMPClient + +pytestmark = pytest.mark.cpu_test + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "results, expected", + [ + ([True, True], True), + ([True, False], False), + ([False, True], False), + ([False, False], False), + ], +) +async def test_dp_wake_up_requires_all_engines_awake(results, expected): + client = object.__new__(DPLBAsyncMPClient) + client.core_engines = [b"engine-0", b"engine-1"] + client._call_utility_async = AsyncMock(side_effect=results) + + assert await client.wake_up_async(["weights"]) is expected + assert client._call_utility_async.await_args_list == [ + call("wake_up", ["weights"], engine=b"engine-0"), + call("wake_up", ["weights"], engine=b"engine-1"), + ] diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index 71a61b1648..a34981d651 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -199,8 +199,8 @@ class EngineClient(ABC): ... @abstractmethod - async def wake_up(self, tags: list[str] | None = None) -> None: - """Wake up the engine.""" + async def wake_up(self, tags: list[str] | None = None) -> bool: + """Wake the tagged resources; return whether every engine is fully awake.""" ... @abstractmethod diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 37348533b8..3de7bab133 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -839,7 +839,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): """ self.llm_engine.release_kv_cache_memory() - def wake_up(self, tags: list[str] | None = None): + def wake_up(self, tags: list[str] | None = None) -> bool: """Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep] method for more details. @@ -851,8 +851,11 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): (or None) before the engine is used again. Use tags=["scheduling"] to resume from level 0 sleep. + Returns: + Whether the engine is fully awake. + """ - self.llm_engine.wake_up(tags) + return self.llm_engine.wake_up(tags) def get_metrics(self) -> list["Metric"]: """Return a snapshot of aggregated metrics from Prometheus. diff --git a/vllm/entrypoints/serve/dev/sleep/api_router.py b/vllm/entrypoints/serve/dev/sleep/api_router.py index 846a487706..eb2ad9d7d4 100644 --- a/vllm/entrypoints/serve/dev/sleep/api_router.py +++ b/vllm/entrypoints/serve/dev/sleep/api_router.py @@ -2,11 +2,16 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from fastapi import APIRouter, FastAPI, Request -from fastapi.responses import JSONResponse, Response +from typing import Annotated + +from fastapi import APIRouter, FastAPI, Query, Request +from fastapi.responses import JSONResponse from vllm.engine.protocol import EngineClient from vllm.logger import init_logger +from vllm.v1.engine import PauseMode + +from .metrics import sleep_mode_operation_metrics logger = init_logger(__name__) @@ -19,29 +24,35 @@ router = APIRouter() @router.post("/sleep") -async def sleep(raw_request: Request): - # get POST params - level = raw_request.query_params.get("level", "1") - mode = raw_request.query_params.get("mode", "abort") - await engine_client(raw_request).sleep(int(level), mode) - return Response(status_code=200) +async def sleep( + raw_request: Request, + level: Annotated[int, Query(ge=0, le=2)] = 1, + mode: Annotated[PauseMode, Query()] = "abort", +) -> JSONResponse: + with sleep_mode_operation_metrics().record("sleep"): + await engine_client(raw_request).sleep(level, mode) + return JSONResponse(content={"status": "sleeping", "level": level}) @router.post("/release_kv_cache_memory") -async def release_kv_cache_memory(raw_request: Request): - await engine_client(raw_request).release_kv_cache_memory() - return Response(status_code=200) +async def release_kv_cache_memory(raw_request: Request) -> JSONResponse: + with sleep_mode_operation_metrics().record("release_kv_cache_memory"): + await engine_client(raw_request).release_kv_cache_memory() + return JSONResponse(content={"status": "kv_cache_released"}) @router.post("/wake_up") -async def wake_up(raw_request: Request): +async def wake_up(raw_request: Request) -> JSONResponse: tags = raw_request.query_params.getlist("tags") if tags == []: # set to None to wake up all tags if no tags are provided tags = None logger.info("wake up the engine with tags: %s", tags) - await engine_client(raw_request).wake_up(tags) - return Response(status_code=200) + with sleep_mode_operation_metrics().record("wake"): + fully_awake = await engine_client(raw_request).wake_up(tags) + return JSONResponse( + content={"status": "awake" if fully_awake else "sleeping", "tags": tags} + ) @router.get("/is_sleeping") diff --git a/vllm/entrypoints/serve/dev/sleep/metrics.py b/vllm/entrypoints/serve/dev/sleep/metrics.py new file mode 100644 index 0000000000..e1dbeaf175 --- /dev/null +++ b/vllm/entrypoints/serve/dev/sleep/metrics.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Metrics for dispatched sleep-mode frontend operations.""" + +from collections.abc import Iterator +from contextlib import contextmanager +from threading import Lock +from time import perf_counter +from typing import Literal + +from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram + +Operation = Literal["sleep", "release_kv_cache_memory", "wake"] +BUCKETS = (0.01, 0.1, 1, 10, 30, 60, 120, 300, 600) + + +class SleepModeOperationMetrics: + def __init__(self, registry: CollectorRegistry | None = None): + registry = REGISTRY if registry is None else registry + self.duration = Histogram( + "vllm:sleep_mode_operation_duration_seconds", + "Duration of one sleep-mode operation.", + ["operation"], + buckets=BUCKETS, + registry=registry, + ) + self.in_flight = Gauge( + "vllm:sleep_mode_operations_in_flight", + "Sleep-mode operations currently awaited.", + ["operation"], + multiprocess_mode="livesum", + registry=registry, + ) + + @contextmanager + def record(self, operation: Operation) -> Iterator[None]: + active = self.in_flight.labels(operation) + started = perf_counter() + active.inc() + try: + yield + finally: + active.dec() + self.duration.labels(operation).observe(perf_counter() - started) + + +_metrics: SleepModeOperationMetrics | None = None +_metrics_lock = Lock() + + +def sleep_mode_operation_metrics() -> SleepModeOperationMetrics: + global _metrics + if _metrics is None: + with _metrics_lock: + if _metrics is None: + _metrics = SleepModeOperationMetrics() + return _metrics diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index ad9420247f..b1cc2edbc4 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -1110,12 +1110,14 @@ class AsyncLLM(EngineClient): if self.logger_manager is not None: self.logger_manager.record_sleep_state(1, 0) - async def wake_up(self, tags: list[str] | None = None) -> None: + async def wake_up(self, tags: list[str] | None = None) -> bool: fully_awake = await self.engine_core.wake_up_async(tags) if self.logger_manager is not None and fully_awake: self.logger_manager.record_sleep_state(0, 0) + return fully_awake + async def checkpoint_prepare(self) -> None: await self.collective_rpc("checkpoint_prepare") diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index 9372c8a172..cb6705966a 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -1320,7 +1320,7 @@ class AsyncMPClient(MPClient): await self.call_utility_async("release_kv_cache_memory") async def wake_up_async(self, tags: list[str] | None = None) -> bool: - return await self.call_utility_async("wake_up", tags) + return all(await self.call_utility_all_async("wake_up", tags)) async def is_sleeping_async(self) -> bool: return await self.call_utility_consensus_async("is_sleeping") diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 73f6c18b34..d14b3b9755 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -386,12 +386,14 @@ class LLMEngine: if self.logger_manager is not None: self.logger_manager.record_sleep_state(1, 0) - def wake_up(self, tags: list[str] | None = None): + def wake_up(self, tags: list[str] | None = None) -> bool: fully_awake = self.engine_core.wake_up(tags) if self.logger_manager is not None and fully_awake: self.logger_manager.record_sleep_state(0, 0) + return fully_awake + def is_sleeping(self) -> bool: return self.engine_core.is_sleeping()