[Frontend][RL] Align sleep-mode API responses and operation metrics (#52864)

Signed-off-by: Ronald1995 <ronaldautomobile@163.com>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ronald
2026-09-29 23:16:38 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent 29773c722b
commit e5f3d08d72
14 changed files with 834 additions and 48 deletions
@@ -2290,6 +2290,89 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() {
client.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn wake_up_requires_every_engine_to_be_fully_awake() {
init_tracing();
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let mut engines = Vec::new();
for engine_index in 0..2 {
let engine = spawn_mock_engine_task(
handshake_address.clone(),
EngineId::from_engine_index(engine_index).into_frame().to_vec(),
move |dealer, push| {
Box::pin(async move {
let results = if engine_index == 0 {
[true, false, true]
} else {
[false, true, true]
};
for (attempt, fully_awake) in results.into_iter().enumerate() {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]);
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call_id");
assert_eq!(array[2], Value::from("wake_up"));
let tags = if attempt == 1 {
Value::Array(vec![Value::from("weights")])
} else {
Value::Nil
};
assert_eq!(array[3], Value::Array(vec![tags]));
send_outputs(
push,
UtilityCallOutput {
engine_index: u32::from(engine_index),
timestamp: 0.0,
output: UtilityOutput {
call_id: call_id.into(),
failure_message: None,
result: Some(utility_result_value(fully_awake)),
},
}
.into(),
)
.await;
}
})
},
);
engines.push(engine);
}
let client = connect_client_with_ipc(
handshake_test_config(
handshake_address,
2,
"test-model",
Duration::from_secs(2),
5,
None,
),
&ipc,
)
.await;
for (tags, expected) in [
(None, false),
(Some(vec!["weights".to_owned()]), false),
(None, true),
] {
assert_eq!(
timeout(Duration::from_secs(2), client.wake_up(tags))
.await
.expect("wake timeout")
.expect("wake result"),
expected
);
}
for (shutdown_tx, engine_task) in engines {
let _ = shutdown_tx.send(());
engine_task.await.unwrap();
}
client.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn collective_rpc_flattens_results_from_all_engines() {
init_tracing();
+118 -1
View File
@@ -1,11 +1,13 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::time::Instant;
use prometheus_client::encoding::EncodeLabelSet;
use prometheus_client::metrics::family::Family;
use prometheus_client::registry::Registry;
use crate::{Histogram, U64Counter};
use crate::{Histogram, U64Counter, U64Gauge};
const HTTP_REQUEST_DURATION_BUCKETS: [f64; 3] = [0.1, 0.5, 1.0];
const HTTP_REQUEST_DURATION_HIGHR_BUCKETS: [f64; 21] = [
@@ -21,6 +23,10 @@ fn http_request_duration_highr_histogram() -> Histogram {
Histogram::new(HTTP_REQUEST_DURATION_HIGHR_BUCKETS.iter().copied())
}
fn sleep_mode_operation_duration_histogram() -> Histogram {
Histogram::new([0.01, 0.1, 1.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0])
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct HttpRequestLabels {
pub method: String,
@@ -34,15 +40,25 @@ pub struct HttpHandlerLabels {
pub handler: String,
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct SleepModeOperationLabels {
pub operation: &'static str,
}
pub(crate) type HttpRequestCounterFamily = Family<HttpRequestLabels, U64Counter>;
pub(crate) type HttpHandlerHistogramFamily =
Family<HttpHandlerLabels, Histogram, fn() -> Histogram>;
pub(crate) type SleepModeOperationHistogramFamily =
Family<SleepModeOperationLabels, Histogram, fn() -> Histogram>;
pub(crate) type SleepModeOperationGaugeFamily = Family<SleepModeOperationLabels, U64Gauge>;
/// API-server Prometheus families exported from the HTTP middleware layer.
pub struct ApiServerMetrics {
pub http_requests: HttpRequestCounterFamily,
pub http_request_duration_seconds: HttpHandlerHistogramFamily,
pub http_request_duration_highr_seconds: Histogram,
pub sleep_mode_operation_duration_seconds: SleepModeOperationHistogramFamily,
pub sleep_mode_operations_in_flight: SleepModeOperationGaugeFamily,
}
impl ApiServerMetrics {
@@ -71,10 +87,111 @@ impl ApiServerMetrics {
http_request_duration_highr_seconds.clone(),
);
let sleep_mode_operation_duration_seconds = Family::new_with_constructor(
sleep_mode_operation_duration_histogram as fn() -> Histogram,
);
registry.register(
"vllm:sleep_mode_operation_duration_seconds",
"Duration of one sleep-mode operation.",
sleep_mode_operation_duration_seconds.clone(),
);
let sleep_mode_operations_in_flight = SleepModeOperationGaugeFamily::default();
registry.register(
"vllm:sleep_mode_operations_in_flight",
"Sleep-mode operations currently awaited.",
sleep_mode_operations_in_flight.clone(),
);
Self {
http_requests,
http_request_duration_seconds,
http_request_duration_highr_seconds,
sleep_mode_operation_duration_seconds,
sleep_mode_operations_in_flight,
}
}
/// Record one dispatched sleep-mode engine operation.
pub fn record_sleep_mode_operation(
&self,
operation: &'static str,
) -> SleepModeOperationRecorder {
let labels = SleepModeOperationLabels { operation };
let in_flight = self.sleep_mode_operations_in_flight.get_or_create_owned(&labels);
in_flight.inc();
SleepModeOperationRecorder {
started_at: Instant::now(),
duration: self.sleep_mode_operation_duration_seconds.get_or_create_owned(&labels),
in_flight,
}
}
}
/// Records duration and clears in-flight tracking on completion or cancellation.
pub struct SleepModeOperationRecorder {
started_at: Instant,
duration: Histogram,
in_flight: U64Gauge,
}
impl Drop for SleepModeOperationRecorder {
fn drop(&mut self) {
self.in_flight.dec();
self.duration.observe(self.started_at.elapsed().as_secs_f64());
}
}
#[cfg(test)]
mod tests {
use prometheus_client::encoding::text::encode;
use prometheus_client::registry::Registry;
use super::ApiServerMetrics;
fn rendered_metrics(registry: &Registry) -> String {
let mut rendered = String::new();
encode(&mut rendered, registry).expect("encode metrics");
rendered
}
#[test]
fn sleep_mode_recorder_tracks_duration_and_concurrency() {
let mut registry = Registry::default();
let metrics = ApiServerMetrics::register(&mut registry);
for operation in ["sleep", "release_kv_cache_memory", "wake"] {
let recorder = metrics.record_sleep_mode_operation(operation);
let active = rendered_metrics(&registry);
assert!(active.contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 1"
)));
drop(recorder);
let succeeded = rendered_metrics(&registry);
assert!(succeeded.contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 0"
)));
assert!(succeeded.contains(&format!(
"vllm:sleep_mode_operation_duration_seconds_count{{operation=\"{operation}\"}} 1"
)));
let first = metrics.record_sleep_mode_operation(operation);
let second = metrics.record_sleep_mode_operation(operation);
assert!(rendered_metrics(&registry).contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 2"
)));
drop(first);
assert!(rendered_metrics(&registry).contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 1"
)));
drop(second);
let rendered = rendered_metrics(&registry);
assert!(rendered.contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 0"
)));
assert!(rendered.contains(&format!(
"vllm:sleep_mode_operation_duration_seconds_count{{operation=\"{operation}\"}} 3"
)));
}
}
}
@@ -25,10 +25,6 @@ const EXCLUDED_HANDLERS: &[&str] = &[
"/reset_mm_cache",
"/reset_encoder_cache",
"/collective_rpc",
"/sleep",
"/release_kv_cache_memory",
"/wake_up",
"/is_sleeping",
];
/// Map the request method to a bounded Prometheus label.
+47 -12
View File
@@ -6,9 +6,9 @@ use std::sync::Arc;
use axum::Json;
use axum::extract::rejection::QueryRejection;
use axum::extract::{Query, RawQuery, State};
use axum::http::StatusCode;
use serde::{Deserialize, Serialize};
use vllm_engine_core_client::protocol::utility::PauseMode;
use vllm_metrics::METRICS;
use crate::error::ApiError;
use crate::state::AppState;
@@ -19,6 +19,24 @@ pub(crate) struct IsSleepingResponse {
is_sleeping: bool,
}
#[derive(Serialize)]
pub struct SleepResponse {
status: &'static str,
level: u32,
}
#[derive(Serialize)]
pub struct ReleaseKvCacheMemoryResponse {
status: &'static str,
}
#[derive(Serialize)]
pub struct WakeUpResponse {
status: &'static str,
/// Tags requested by the caller; null means wake all resources.
tags: Option<Vec<String>>,
}
#[derive(Debug, Deserialize)]
pub(crate) struct SleepParams {
#[serde(default = "default_sleep_level")]
@@ -32,43 +50,56 @@ const fn default_sleep_level() -> u32 {
}
fn invalid_query(error: QueryRejection) -> ApiError {
ApiError::invalid_request(error.body_text(), Some("mode"))
ApiError::invalid_request(error.body_text(), None)
}
/// Put the engine to sleep.
pub async fn sleep(
State(state): State<Arc<AppState>>,
params: Result<Query<SleepParams>, QueryRejection>,
) -> Result<StatusCode, ApiError> {
let Query(params) = params.map_err(invalid_query)?;
) -> Result<Json<SleepResponse>, ApiError> {
let Query(SleepParams { level, mode }) = params.map_err(invalid_query)?;
if level > 2 {
return Err(ApiError::invalid_request(
"level must be between 0 and 2",
Some("level"),
));
}
let _recorder = METRICS.api_server.record_sleep_mode_operation("sleep");
state
.engine_core_client()
.sleep(params.level, params.mode)
.sleep(level, mode)
.await
.map_err(|error| utility_call_error("sleep", error))?;
Ok(StatusCode::OK)
Ok(Json(SleepResponse {
status: "sleeping",
level,
}))
}
/// Release KV cache memory while keeping model weights resident.
pub async fn release_kv_cache_memory(
State(state): State<Arc<AppState>>,
) -> Result<StatusCode, ApiError> {
) -> Result<Json<ReleaseKvCacheMemoryResponse>, ApiError> {
let _recorder = METRICS.api_server.record_sleep_mode_operation("release_kv_cache_memory");
state
.engine_core_client()
.release_kv_cache_memory()
.await
.map_err(|error| utility_call_error("release_kv_cache_memory", error))?;
Ok(StatusCode::OK)
Ok(Json(ReleaseKvCacheMemoryResponse {
status: "kv_cache_released",
}))
}
/// Wake the engine from sleep mode.
pub async fn wake_up(
State(state): State<Arc<AppState>>,
RawQuery(query): RawQuery,
) -> Result<StatusCode, ApiError> {
) -> Result<Json<WakeUpResponse>, ApiError> {
let tags = query.and_then(|query| {
let tags = url::form_urlencoded::parse(query.as_bytes())
.filter(|(key, _)| key == "tags")
@@ -77,13 +108,17 @@ pub async fn wake_up(
(!tags.is_empty()).then_some(tags)
});
state
let _recorder = METRICS.api_server.record_sleep_mode_operation("wake");
let fully_awake = state
.engine_core_client()
.wake_up(tags)
.wake_up(tags.clone())
.await
.map_err(|error| utility_call_error("wake_up", error))?;
Ok(StatusCode::OK)
Ok(Json(WakeUpResponse {
status: if fully_awake { "awake" } else { "sleeping" },
tags,
}))
}
/// Return whether the engine is currently sleeping at any level.
+279 -5
View File
@@ -5892,6 +5892,7 @@ async fn weight_checker_route_rejects_invalid_requests_before_engine_calls() {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn sleep_route_uses_python_compatible_default_query_values() {
let before = METRICS.render().expect("render metrics");
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
@@ -5927,13 +5928,95 @@ async fn sleep_route_uses_python_compatible_default_query_values() {
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
json!({ "status": "sleeping", "level": 1 })
);
engine_task.await.expect("mock engine task");
let after = METRICS.render().expect("render metrics");
assert_eq!(
metric_delta(
&before,
&after,
"http_requests_total",
Some("method=\"POST\",status=\"2xx\",handler=\"/sleep\""),
),
1.0
);
assert_eq!(
metric_delta(
&before,
&after,
"vllm:sleep_mode_operation_duration_seconds_count",
Some("operation=\"sleep\""),
),
1.0
);
assert_eq!(
metric_value(
&after,
"vllm:sleep_mode_operations_in_flight",
Some("operation=\"sleep\""),
),
Some(0.0)
);
assert_eq!(
metric_delta(
&before,
&after,
"http_request_duration_seconds_count",
Some("method=\"POST\",handler=\"/sleep\""),
),
1.0
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn sleep_route_rejects_invalid_query_before_engine_dispatch() {
let before = METRICS.render().expect("render metrics");
let (app, engine_task) =
test_admin_app_with_engine_script(|_dealer, _push| boxed_test_future(async move {})).await;
for query in ["level=invalid", "level=-1", "level=3", "mode=invalid"] {
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri(format!("/sleep?{query}"))
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{query}");
}
engine_task.abort_and_join().await;
let after = METRICS.render().expect("render metrics");
assert_eq!(
metric_delta(
&before,
&after,
"http_requests_total",
Some("method=\"POST\",status=\"4xx\",handler=\"/sleep\""),
),
4.0
);
for metric in [
"vllm:sleep_mode_operation_duration_seconds_count",
"vllm:sleep_mode_operations_in_flight",
] {
assert_eq!(
metric_delta(&before, &after, metric, Some("operation=\"sleep\"")),
0.0
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn release_kv_cache_memory_route_sends_expected_utility_call() {
let before = METRICS.render().expect("render metrics");
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
@@ -5966,13 +6049,177 @@ async fn release_kv_cache_memory_route_sends_expected_utility_call() {
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
json!({ "status": "kv_cache_released" })
);
engine_task.await.expect("mock engine task");
let after = METRICS.render().expect("render metrics");
assert_eq!(
metric_delta(
&before,
&after,
"http_requests_total",
Some("method=\"POST\",status=\"2xx\",handler=\"/release_kv_cache_memory\""),
),
1.0
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn release_kv_cache_memory_error_records_http_status_and_operation_duration() {
let before = METRICS.render().expect("render metrics");
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
let payload = decode_value(&utility[1]).expect("decode utility payload");
let call_id =
payload.as_array().expect("utility payload array")[1].as_u64().expect("call id");
send_outputs(
push,
UtilityCallOutput {
output: UtilityOutput {
call_id: call_id.into(),
failure_message: Some("requires a completed pause first".to_owned()),
result: None,
},
..Default::default()
}
.into(),
)
.await;
})
})
.await;
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/release_kv_cache_memory")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
engine_task.await.expect("mock engine task");
let after = METRICS.render().expect("render metrics");
assert_eq!(
metric_delta(
&before,
&after,
"http_requests_total",
Some("method=\"POST\",status=\"5xx\",handler=\"/release_kv_cache_memory\""),
),
1.0
);
assert_eq!(
metric_delta(
&before,
&after,
"vllm:sleep_mode_operation_duration_seconds_count",
Some("operation=\"release_kv_cache_memory\""),
),
1.0
);
assert_eq!(
metric_value(
&after,
"vllm:sleep_mode_operations_in_flight",
Some("operation=\"release_kv_cache_memory\""),
),
Some(0.0)
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn cancelled_wake_operation_clears_in_flight_and_is_scraped() {
let before = METRICS.render().expect("render metrics");
let (dispatched_tx, dispatched_rx) = tokio::sync::oneshot::channel();
let (app, engine_task) = test_admin_app_with_engine_script(move |dealer, _push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
let payload = decode_value(&utility[1]).expect("decode utility payload");
assert_eq!(
payload.as_array().expect("utility payload array")[2],
Value::from("wake_up")
);
dispatched_tx.send(()).expect("notify dispatch");
std::future::pending::<()>().await;
})
})
.await;
let mut request_app = app.clone();
let request_task = tokio::spawn(async move {
request_app
.call(
Request::builder()
.method("POST")
.uri("/wake_up")
.body(Body::empty())
.expect("build request"),
)
.await
});
tokio::time::timeout(Duration::from_secs(2), dispatched_rx)
.await
.expect("wake dispatch timeout")
.expect("wake dispatched");
let active = METRICS.render().expect("render metrics");
assert_eq!(
metric_delta(
&before,
&active,
"vllm:sleep_mode_operations_in_flight",
Some("operation=\"wake\""),
),
1.0
);
request_task.abort();
assert!(request_task.await.expect_err("request cancelled").is_cancelled());
engine_task.abort_and_join().await;
let response = app
.clone()
.call(
Request::builder()
.uri("/metrics")
.body(Body::empty())
.expect("build scrape request"),
)
.await
.expect("scrape metrics");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read metrics");
let after = String::from_utf8(body.to_vec()).expect("utf8 metrics");
assert_eq!(
metric_value(
&after,
"vllm:sleep_mode_operations_in_flight",
Some("operation=\"wake\""),
),
Some(0.0)
);
assert_eq!(
metric_delta(
&before,
&after,
"vllm:sleep_mode_operation_duration_seconds_count",
Some("operation=\"wake\""),
),
1.0
);
assert!(!after.contains("rl_sleep_mode"));
assert!(!after.contains("sleep_mode_operations_total"));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn wake_up_route_without_tags_accepts_bool_result() {
let before = METRICS.render().expect("render metrics");
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
@@ -6005,8 +6252,21 @@ async fn wake_up_route_without_tags_accepts_bool_result() {
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
json!({ "status": "awake", "tags": null })
);
engine_task.await.expect("mock engine task");
let after = METRICS.render().expect("render metrics");
assert_eq!(
metric_delta(
&before,
&after,
"http_requests_total",
Some("method=\"POST\",status=\"2xx\",handler=\"/wake_up\""),
),
1.0
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -6030,7 +6290,7 @@ async fn wake_up_route_accepts_repeated_tags() {
])])
);
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
send_outputs(push, utility_outputs(call_id, utility_result_value(false))).await;
})
})
.await;
@@ -6050,13 +6310,17 @@ async fn wake_up_route_accepts_repeated_tags() {
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
json!({ "status": "sleeping", "tags": ["weights", "kv_cache"] })
);
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn is_sleeping_route_returns_json_payload() {
let before = METRICS.render().expect("render metrics");
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
@@ -6094,6 +6358,16 @@ async fn is_sleeping_route_returns_json_payload() {
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
json!({ "is_sleeping": true })
);
let after = METRICS.render().expect("render metrics");
assert_eq!(
metric_delta(
&before,
&after,
"http_requests_total",
Some("method=\"GET\",status=\"2xx\",handler=\"/is_sleeping\""),
),
1.0
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+179 -5
View File
@@ -1,35 +1,209 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
import requests
from fastapi import FastAPI
from fastapi.testclient import TestClient
from prometheus_client import CollectorRegistry
from prometheus_client.parser import text_string_to_metric_families
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.serve.dev.sleep import metrics as sleep_metrics
from vllm.entrypoints.serve.dev.sleep.api_router import attach_router
from vllm.entrypoints.serve.dev.sleep.metrics import SleepModeOperationMetrics
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
from vllm.entrypoints.serve.instrumentator.metrics import (
attach_router as attach_metrics_router,
)
from vllm.v1.metrics import prometheus as prometheus_metrics
MODEL_NAME = "meta-llama/Llama-3.2-1B"
@pytest.fixture
def sleep_route_app(monkeypatch):
app = FastAPI()
app.state.args = SimpleNamespace(log_error_stack=False)
app.state.engine_client = AsyncMock()
metrics = SleepModeOperationMetrics(CollectorRegistry())
monkeypatch.setattr(
"vllm.entrypoints.serve.dev.sleep.api_router.sleep_mode_operation_metrics",
lambda: metrics,
)
attach_router(app)
init_exception_handler(app)
return app, metrics
@pytest.mark.cpu_test
@pytest.mark.parametrize("level", [0, 1, 2])
def test_sleep_route_response_and_engine_arguments(sleep_route_app, level):
app, _ = sleep_route_app
with TestClient(app) as client:
response = client.post("/sleep", params={"level": level})
assert response.status_code == 200
assert response.json() == {"status": "sleeping", "level": level}
app.state.engine_client.sleep.assert_awaited_once_with(level, "abort")
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("query", "expected_param"),
[
("level=invalid", "query.level"),
("level=-1", "query.level"),
("level=3", "query.level"),
("mode=invalid", "query.mode"),
],
)
def test_sleep_route_rejects_invalid_query_before_dispatch(
sleep_route_app, query, expected_param
):
app, metrics = sleep_route_app
with TestClient(app) as client:
response = client.post(f"/sleep?{query}")
assert response.status_code == 400
assert response.json()["error"]["param"] == expected_param
app.state.engine_client.sleep.assert_not_awaited()
assert list(metrics.duration.collect()[0].samples) == []
assert list(metrics.in_flight.collect()[0].samples) == []
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("fully_awake", "tags", "expected"),
[
(True, "", {"status": "awake", "tags": None}),
(False, "?tags=weights", {"status": "sleeping", "tags": ["weights"]}),
],
)
def test_wake_route_maps_engine_result_without_state_query(
sleep_route_app, fully_awake, tags, expected
):
app, _ = sleep_route_app
app.state.engine_client.wake_up.return_value = fully_awake
with TestClient(app) as client:
response = client.post(f"/wake_up{tags}")
assert response.status_code == 200
assert response.json() == expected
app.state.engine_client.wake_up.assert_awaited_once_with(expected["tags"])
app.state.engine_client.is_sleeping.assert_not_awaited()
@pytest.mark.cpu_test
@pytest.mark.parametrize("fails", [False, True])
def test_release_kv_cache_memory_route(fails):
app = FastAPI()
app.state.engine_client = AsyncMock()
def test_release_kv_cache_memory_route(sleep_route_app, fails):
app, metrics = sleep_route_app
release = app.state.engine_client.release_kv_cache_memory
if fails:
release.side_effect = RuntimeError("requires a completed pause first")
attach_router(app)
with TestClient(app, raise_server_exceptions=False) as client:
response = client.post("/release_kv_cache_memory")
assert response.status_code == (500 if fails else 200)
if not fails:
assert response.json() == {"status": "kv_cache_released"}
release.assert_awaited_once_with()
assert metrics.in_flight.labels("release_kv_cache_memory")._value.get() == 0
assert _duration_count(metrics, "release_kv_cache_memory") == 1
@pytest.mark.cpu_test
@pytest.mark.parametrize("operation", ["sleep", "release_kv_cache_memory", "wake"])
@pytest.mark.parametrize("exception", [RuntimeError, asyncio.CancelledError])
def test_sleep_mode_recorder_tracks_duration_and_cleans_up(operation, exception):
metrics = SleepModeOperationMetrics(CollectorRegistry())
with metrics.record(operation):
assert metrics.in_flight.labels(operation)._value.get() == 1
with pytest.raises(exception), metrics.record(operation):
raise exception()
assert metrics.in_flight.labels(operation)._value.get() == 0
assert metrics.duration.labels(operation)._sum.get() >= 0
assert _duration_count(metrics, operation) == 2
def _duration_count(metrics, operation):
return next(
sample.value
for sample in metrics.duration.collect()[0].samples
if sample.name.endswith("_count") and sample.labels == {"operation": operation}
)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("path", "method", "operation", "status", "fails"),
[
("/sleep", "POST", "sleep", 200, False),
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 200, False),
("/wake_up", "POST", "wake", 200, False),
("/is_sleeping", "GET", None, 200, False),
("/sleep?level=3", "POST", None, 400, False),
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 500, True),
],
)
def test_sleep_routes_visible_on_production_metrics_endpoint(
monkeypatch, path, method, operation, status, fails
):
registry = CollectorRegistry()
monkeypatch.delenv("PROMETHEUS_MULTIPROC_DIR", raising=False)
monkeypatch.setattr(sleep_metrics, "REGISTRY", registry)
monkeypatch.setattr(sleep_metrics, "_metrics", None)
monkeypatch.setattr(prometheus_metrics, "REGISTRY", registry)
app = FastAPI()
app.state.args = SimpleNamespace(log_error_stack=False)
app.state.engine_client = AsyncMock()
app.state.engine_client.wake_up.return_value = True
app.state.engine_client.is_sleeping.return_value = False
if fails:
app.state.engine_client.release_kv_cache_memory.side_effect = RuntimeError(
"engine failed"
)
attach_router(app)
init_exception_handler(app)
attach_metrics_router(app)
with TestClient(app, raise_server_exceptions=False) as client:
assert client.request(method, path).status_code == status
response = client.get("/metrics")
assert response.status_code == 200
samples = [
sample
for family in text_string_to_metric_families(response.text)
for sample in family.samples
]
assert any(
sample.name == "http_requests_total"
and sample.labels
== {
"handler": path.split("?")[0],
"method": method,
"status": f"{status // 100}xx",
}
and sample.value == 1
for sample in samples
)
assert not any("sleep_mode_operations_total" in sample.name for sample in samples)
assert any(
sample.name == "vllm:sleep_mode_operations_in_flight"
and sample.labels == {"operation": operation}
and sample.value == 0
for sample in samples
) == (operation is not None)
assert any(
sample.name == "vllm:sleep_mode_operation_duration_seconds_count"
and sample.labels == {"operation": operation}
and sample.value == 1
for sample in samples
) == (operation is not None)
def test_sleep_mode():
+32
View File
@@ -0,0 +1,32 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import AsyncMock, call
import pytest
from vllm.v1.engine.core_client import DPLBAsyncMPClient
pytestmark = pytest.mark.cpu_test
@pytest.mark.asyncio
@pytest.mark.parametrize(
"results, expected",
[
([True, True], True),
([True, False], False),
([False, True], False),
([False, False], False),
],
)
async def test_dp_wake_up_requires_all_engines_awake(results, expected):
client = object.__new__(DPLBAsyncMPClient)
client.core_engines = [b"engine-0", b"engine-1"]
client._call_utility_async = AsyncMock(side_effect=results)
assert await client.wake_up_async(["weights"]) is expected
assert client._call_utility_async.await_args_list == [
call("wake_up", ["weights"], engine=b"engine-0"),
call("wake_up", ["weights"], engine=b"engine-1"),
]
+2 -2
View File
@@ -199,8 +199,8 @@ class EngineClient(ABC):
...
@abstractmethod
async def wake_up(self, tags: list[str] | None = None) -> None:
"""Wake up the engine."""
async def wake_up(self, tags: list[str] | None = None) -> bool:
"""Wake the tagged resources; return whether every engine is fully awake."""
...
@abstractmethod
+5 -2
View File
@@ -839,7 +839,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
"""
self.llm_engine.release_kv_cache_memory()
def wake_up(self, tags: list[str] | None = None):
def wake_up(self, tags: list[str] | None = None) -> bool:
"""Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep]
method for more details.
@@ -851,8 +851,11 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
(or None) before the engine is used again.
Use tags=["scheduling"] to resume from level 0 sleep.
Returns:
Whether the engine is fully awake.
"""
self.llm_engine.wake_up(tags)
return self.llm_engine.wake_up(tags)
def get_metrics(self) -> list["Metric"]:
"""Return a snapshot of aggregated metrics from Prometheus.
+25 -14
View File
@@ -2,11 +2,16 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fastapi import APIRouter, FastAPI, Request
from fastapi.responses import JSONResponse, Response
from typing import Annotated
from fastapi import APIRouter, FastAPI, Query, Request
from fastapi.responses import JSONResponse
from vllm.engine.protocol import EngineClient
from vllm.logger import init_logger
from vllm.v1.engine import PauseMode
from .metrics import sleep_mode_operation_metrics
logger = init_logger(__name__)
@@ -19,29 +24,35 @@ router = APIRouter()
@router.post("/sleep")
async def sleep(raw_request: Request):
# get POST params
level = raw_request.query_params.get("level", "1")
mode = raw_request.query_params.get("mode", "abort")
await engine_client(raw_request).sleep(int(level), mode)
return Response(status_code=200)
async def sleep(
raw_request: Request,
level: Annotated[int, Query(ge=0, le=2)] = 1,
mode: Annotated[PauseMode, Query()] = "abort",
) -> JSONResponse:
with sleep_mode_operation_metrics().record("sleep"):
await engine_client(raw_request).sleep(level, mode)
return JSONResponse(content={"status": "sleeping", "level": level})
@router.post("/release_kv_cache_memory")
async def release_kv_cache_memory(raw_request: Request):
await engine_client(raw_request).release_kv_cache_memory()
return Response(status_code=200)
async def release_kv_cache_memory(raw_request: Request) -> JSONResponse:
with sleep_mode_operation_metrics().record("release_kv_cache_memory"):
await engine_client(raw_request).release_kv_cache_memory()
return JSONResponse(content={"status": "kv_cache_released"})
@router.post("/wake_up")
async def wake_up(raw_request: Request):
async def wake_up(raw_request: Request) -> JSONResponse:
tags = raw_request.query_params.getlist("tags")
if tags == []:
# set to None to wake up all tags if no tags are provided
tags = None
logger.info("wake up the engine with tags: %s", tags)
await engine_client(raw_request).wake_up(tags)
return Response(status_code=200)
with sleep_mode_operation_metrics().record("wake"):
fully_awake = await engine_client(raw_request).wake_up(tags)
return JSONResponse(
content={"status": "awake" if fully_awake else "sleeping", "tags": tags}
)
@router.get("/is_sleeping")
@@ -0,0 +1,57 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Metrics for dispatched sleep-mode frontend operations."""
from collections.abc import Iterator
from contextlib import contextmanager
from threading import Lock
from time import perf_counter
from typing import Literal
from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram
Operation = Literal["sleep", "release_kv_cache_memory", "wake"]
BUCKETS = (0.01, 0.1, 1, 10, 30, 60, 120, 300, 600)
class SleepModeOperationMetrics:
def __init__(self, registry: CollectorRegistry | None = None):
registry = REGISTRY if registry is None else registry
self.duration = Histogram(
"vllm:sleep_mode_operation_duration_seconds",
"Duration of one sleep-mode operation.",
["operation"],
buckets=BUCKETS,
registry=registry,
)
self.in_flight = Gauge(
"vllm:sleep_mode_operations_in_flight",
"Sleep-mode operations currently awaited.",
["operation"],
multiprocess_mode="livesum",
registry=registry,
)
@contextmanager
def record(self, operation: Operation) -> Iterator[None]:
active = self.in_flight.labels(operation)
started = perf_counter()
active.inc()
try:
yield
finally:
active.dec()
self.duration.labels(operation).observe(perf_counter() - started)
_metrics: SleepModeOperationMetrics | None = None
_metrics_lock = Lock()
def sleep_mode_operation_metrics() -> SleepModeOperationMetrics:
global _metrics
if _metrics is None:
with _metrics_lock:
if _metrics is None:
_metrics = SleepModeOperationMetrics()
return _metrics
+3 -1
View File
@@ -1110,12 +1110,14 @@ class AsyncLLM(EngineClient):
if self.logger_manager is not None:
self.logger_manager.record_sleep_state(1, 0)
async def wake_up(self, tags: list[str] | None = None) -> None:
async def wake_up(self, tags: list[str] | None = None) -> bool:
fully_awake = await self.engine_core.wake_up_async(tags)
if self.logger_manager is not None and fully_awake:
self.logger_manager.record_sleep_state(0, 0)
return fully_awake
async def checkpoint_prepare(self) -> None:
await self.collective_rpc("checkpoint_prepare")
+1 -1
View File
@@ -1320,7 +1320,7 @@ class AsyncMPClient(MPClient):
await self.call_utility_async("release_kv_cache_memory")
async def wake_up_async(self, tags: list[str] | None = None) -> bool:
return await self.call_utility_async("wake_up", tags)
return all(await self.call_utility_all_async("wake_up", tags))
async def is_sleeping_async(self) -> bool:
return await self.call_utility_consensus_async("is_sleeping")
+3 -1
View File
@@ -386,12 +386,14 @@ class LLMEngine:
if self.logger_manager is not None:
self.logger_manager.record_sleep_state(1, 0)
def wake_up(self, tags: list[str] | None = None):
def wake_up(self, tags: list[str] | None = None) -> bool:
fully_awake = self.engine_core.wake_up(tags)
if self.logger_manager is not None and fully_awake:
self.logger_manager.record_sleep_state(0, 0)
return fully_awake
def is_sleeping(self) -> bool:
return self.engine_core.is_sleeping()