[Frontend][RL] Align sleep-mode API responses and operation metrics (#52864)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5.5
parent
29773c722b
commit
e5f3d08d72
@@ -2290,6 +2290,89 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() {
|
||||
client.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn wake_up_requires_every_engine_to_be_fully_awake() {
|
||||
init_tracing();
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let mut engines = Vec::new();
|
||||
for engine_index in 0..2 {
|
||||
let engine = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
EngineId::from_engine_index(engine_index).into_frame().to_vec(),
|
||||
move |dealer, push| {
|
||||
Box::pin(async move {
|
||||
let results = if engine_index == 0 {
|
||||
[true, false, true]
|
||||
} else {
|
||||
[false, true, true]
|
||||
};
|
||||
for (attempt, fully_awake) in results.into_iter().enumerate() {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
let payload = decode_value(&utility[1]);
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call_id");
|
||||
assert_eq!(array[2], Value::from("wake_up"));
|
||||
let tags = if attempt == 1 {
|
||||
Value::Array(vec![Value::from("weights")])
|
||||
} else {
|
||||
Value::Nil
|
||||
};
|
||||
assert_eq!(array[3], Value::Array(vec![tags]));
|
||||
send_outputs(
|
||||
push,
|
||||
UtilityCallOutput {
|
||||
engine_index: u32::from(engine_index),
|
||||
timestamp: 0.0,
|
||||
output: UtilityOutput {
|
||||
call_id: call_id.into(),
|
||||
failure_message: None,
|
||||
result: Some(utility_result_value(fully_awake)),
|
||||
},
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
})
|
||||
},
|
||||
);
|
||||
engines.push(engine);
|
||||
}
|
||||
let client = connect_client_with_ipc(
|
||||
handshake_test_config(
|
||||
handshake_address,
|
||||
2,
|
||||
"test-model",
|
||||
Duration::from_secs(2),
|
||||
5,
|
||||
None,
|
||||
),
|
||||
&ipc,
|
||||
)
|
||||
.await;
|
||||
|
||||
for (tags, expected) in [
|
||||
(None, false),
|
||||
(Some(vec!["weights".to_owned()]), false),
|
||||
(None, true),
|
||||
] {
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(2), client.wake_up(tags))
|
||||
.await
|
||||
.expect("wake timeout")
|
||||
.expect("wake result"),
|
||||
expected
|
||||
);
|
||||
}
|
||||
for (shutdown_tx, engine_task) in engines {
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
}
|
||||
client.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn collective_rpc_flattens_results_from_all_engines() {
|
||||
init_tracing();
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::time::Instant;
|
||||
|
||||
use prometheus_client::encoding::EncodeLabelSet;
|
||||
use prometheus_client::metrics::family::Family;
|
||||
use prometheus_client::registry::Registry;
|
||||
|
||||
use crate::{Histogram, U64Counter};
|
||||
use crate::{Histogram, U64Counter, U64Gauge};
|
||||
|
||||
const HTTP_REQUEST_DURATION_BUCKETS: [f64; 3] = [0.1, 0.5, 1.0];
|
||||
const HTTP_REQUEST_DURATION_HIGHR_BUCKETS: [f64; 21] = [
|
||||
@@ -21,6 +23,10 @@ fn http_request_duration_highr_histogram() -> Histogram {
|
||||
Histogram::new(HTTP_REQUEST_DURATION_HIGHR_BUCKETS.iter().copied())
|
||||
}
|
||||
|
||||
fn sleep_mode_operation_duration_histogram() -> Histogram {
|
||||
Histogram::new([0.01, 0.1, 1.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0])
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
|
||||
pub struct HttpRequestLabels {
|
||||
pub method: String,
|
||||
@@ -34,15 +40,25 @@ pub struct HttpHandlerLabels {
|
||||
pub handler: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
|
||||
pub struct SleepModeOperationLabels {
|
||||
pub operation: &'static str,
|
||||
}
|
||||
|
||||
pub(crate) type HttpRequestCounterFamily = Family<HttpRequestLabels, U64Counter>;
|
||||
pub(crate) type HttpHandlerHistogramFamily =
|
||||
Family<HttpHandlerLabels, Histogram, fn() -> Histogram>;
|
||||
pub(crate) type SleepModeOperationHistogramFamily =
|
||||
Family<SleepModeOperationLabels, Histogram, fn() -> Histogram>;
|
||||
pub(crate) type SleepModeOperationGaugeFamily = Family<SleepModeOperationLabels, U64Gauge>;
|
||||
|
||||
/// API-server Prometheus families exported from the HTTP middleware layer.
|
||||
pub struct ApiServerMetrics {
|
||||
pub http_requests: HttpRequestCounterFamily,
|
||||
pub http_request_duration_seconds: HttpHandlerHistogramFamily,
|
||||
pub http_request_duration_highr_seconds: Histogram,
|
||||
pub sleep_mode_operation_duration_seconds: SleepModeOperationHistogramFamily,
|
||||
pub sleep_mode_operations_in_flight: SleepModeOperationGaugeFamily,
|
||||
}
|
||||
|
||||
impl ApiServerMetrics {
|
||||
@@ -71,10 +87,111 @@ impl ApiServerMetrics {
|
||||
http_request_duration_highr_seconds.clone(),
|
||||
);
|
||||
|
||||
let sleep_mode_operation_duration_seconds = Family::new_with_constructor(
|
||||
sleep_mode_operation_duration_histogram as fn() -> Histogram,
|
||||
);
|
||||
registry.register(
|
||||
"vllm:sleep_mode_operation_duration_seconds",
|
||||
"Duration of one sleep-mode operation.",
|
||||
sleep_mode_operation_duration_seconds.clone(),
|
||||
);
|
||||
|
||||
let sleep_mode_operations_in_flight = SleepModeOperationGaugeFamily::default();
|
||||
registry.register(
|
||||
"vllm:sleep_mode_operations_in_flight",
|
||||
"Sleep-mode operations currently awaited.",
|
||||
sleep_mode_operations_in_flight.clone(),
|
||||
);
|
||||
|
||||
Self {
|
||||
http_requests,
|
||||
http_request_duration_seconds,
|
||||
http_request_duration_highr_seconds,
|
||||
sleep_mode_operation_duration_seconds,
|
||||
sleep_mode_operations_in_flight,
|
||||
}
|
||||
}
|
||||
|
||||
/// Record one dispatched sleep-mode engine operation.
|
||||
pub fn record_sleep_mode_operation(
|
||||
&self,
|
||||
operation: &'static str,
|
||||
) -> SleepModeOperationRecorder {
|
||||
let labels = SleepModeOperationLabels { operation };
|
||||
let in_flight = self.sleep_mode_operations_in_flight.get_or_create_owned(&labels);
|
||||
in_flight.inc();
|
||||
SleepModeOperationRecorder {
|
||||
started_at: Instant::now(),
|
||||
duration: self.sleep_mode_operation_duration_seconds.get_or_create_owned(&labels),
|
||||
in_flight,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Records duration and clears in-flight tracking on completion or cancellation.
|
||||
pub struct SleepModeOperationRecorder {
|
||||
started_at: Instant,
|
||||
duration: Histogram,
|
||||
in_flight: U64Gauge,
|
||||
}
|
||||
|
||||
impl Drop for SleepModeOperationRecorder {
|
||||
fn drop(&mut self) {
|
||||
self.in_flight.dec();
|
||||
self.duration.observe(self.started_at.elapsed().as_secs_f64());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use prometheus_client::encoding::text::encode;
|
||||
use prometheus_client::registry::Registry;
|
||||
|
||||
use super::ApiServerMetrics;
|
||||
|
||||
fn rendered_metrics(registry: &Registry) -> String {
|
||||
let mut rendered = String::new();
|
||||
encode(&mut rendered, registry).expect("encode metrics");
|
||||
rendered
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sleep_mode_recorder_tracks_duration_and_concurrency() {
|
||||
let mut registry = Registry::default();
|
||||
let metrics = ApiServerMetrics::register(&mut registry);
|
||||
for operation in ["sleep", "release_kv_cache_memory", "wake"] {
|
||||
let recorder = metrics.record_sleep_mode_operation(operation);
|
||||
let active = rendered_metrics(®istry);
|
||||
assert!(active.contains(&format!(
|
||||
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 1"
|
||||
)));
|
||||
|
||||
drop(recorder);
|
||||
let succeeded = rendered_metrics(®istry);
|
||||
assert!(succeeded.contains(&format!(
|
||||
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 0"
|
||||
)));
|
||||
assert!(succeeded.contains(&format!(
|
||||
"vllm:sleep_mode_operation_duration_seconds_count{{operation=\"{operation}\"}} 1"
|
||||
)));
|
||||
|
||||
let first = metrics.record_sleep_mode_operation(operation);
|
||||
let second = metrics.record_sleep_mode_operation(operation);
|
||||
assert!(rendered_metrics(®istry).contains(&format!(
|
||||
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 2"
|
||||
)));
|
||||
drop(first);
|
||||
assert!(rendered_metrics(®istry).contains(&format!(
|
||||
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 1"
|
||||
)));
|
||||
drop(second);
|
||||
let rendered = rendered_metrics(®istry);
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 0"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:sleep_mode_operation_duration_seconds_count{{operation=\"{operation}\"}} 3"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,10 +25,6 @@ const EXCLUDED_HANDLERS: &[&str] = &[
|
||||
"/reset_mm_cache",
|
||||
"/reset_encoder_cache",
|
||||
"/collective_rpc",
|
||||
"/sleep",
|
||||
"/release_kv_cache_memory",
|
||||
"/wake_up",
|
||||
"/is_sleeping",
|
||||
];
|
||||
|
||||
/// Map the request method to a bounded Prometheus label.
|
||||
|
||||
@@ -6,9 +6,9 @@ use std::sync::Arc;
|
||||
use axum::Json;
|
||||
use axum::extract::rejection::QueryRejection;
|
||||
use axum::extract::{Query, RawQuery, State};
|
||||
use axum::http::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use vllm_engine_core_client::protocol::utility::PauseMode;
|
||||
use vllm_metrics::METRICS;
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::state::AppState;
|
||||
@@ -19,6 +19,24 @@ pub(crate) struct IsSleepingResponse {
|
||||
is_sleeping: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct SleepResponse {
|
||||
status: &'static str,
|
||||
level: u32,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ReleaseKvCacheMemoryResponse {
|
||||
status: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct WakeUpResponse {
|
||||
status: &'static str,
|
||||
/// Tags requested by the caller; null means wake all resources.
|
||||
tags: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct SleepParams {
|
||||
#[serde(default = "default_sleep_level")]
|
||||
@@ -32,43 +50,56 @@ const fn default_sleep_level() -> u32 {
|
||||
}
|
||||
|
||||
fn invalid_query(error: QueryRejection) -> ApiError {
|
||||
ApiError::invalid_request(error.body_text(), Some("mode"))
|
||||
ApiError::invalid_request(error.body_text(), None)
|
||||
}
|
||||
|
||||
/// Put the engine to sleep.
|
||||
pub async fn sleep(
|
||||
State(state): State<Arc<AppState>>,
|
||||
params: Result<Query<SleepParams>, QueryRejection>,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
let Query(params) = params.map_err(invalid_query)?;
|
||||
) -> Result<Json<SleepResponse>, ApiError> {
|
||||
let Query(SleepParams { level, mode }) = params.map_err(invalid_query)?;
|
||||
if level > 2 {
|
||||
return Err(ApiError::invalid_request(
|
||||
"level must be between 0 and 2",
|
||||
Some("level"),
|
||||
));
|
||||
}
|
||||
let _recorder = METRICS.api_server.record_sleep_mode_operation("sleep");
|
||||
|
||||
state
|
||||
.engine_core_client()
|
||||
.sleep(params.level, params.mode)
|
||||
.sleep(level, mode)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("sleep", error))?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
Ok(Json(SleepResponse {
|
||||
status: "sleeping",
|
||||
level,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Release KV cache memory while keeping model weights resident.
|
||||
pub async fn release_kv_cache_memory(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
) -> Result<Json<ReleaseKvCacheMemoryResponse>, ApiError> {
|
||||
let _recorder = METRICS.api_server.record_sleep_mode_operation("release_kv_cache_memory");
|
||||
state
|
||||
.engine_core_client()
|
||||
.release_kv_cache_memory()
|
||||
.await
|
||||
.map_err(|error| utility_call_error("release_kv_cache_memory", error))?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
Ok(Json(ReleaseKvCacheMemoryResponse {
|
||||
status: "kv_cache_released",
|
||||
}))
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep mode.
|
||||
pub async fn wake_up(
|
||||
State(state): State<Arc<AppState>>,
|
||||
RawQuery(query): RawQuery,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
) -> Result<Json<WakeUpResponse>, ApiError> {
|
||||
let tags = query.and_then(|query| {
|
||||
let tags = url::form_urlencoded::parse(query.as_bytes())
|
||||
.filter(|(key, _)| key == "tags")
|
||||
@@ -77,13 +108,17 @@ pub async fn wake_up(
|
||||
(!tags.is_empty()).then_some(tags)
|
||||
});
|
||||
|
||||
state
|
||||
let _recorder = METRICS.api_server.record_sleep_mode_operation("wake");
|
||||
let fully_awake = state
|
||||
.engine_core_client()
|
||||
.wake_up(tags)
|
||||
.wake_up(tags.clone())
|
||||
.await
|
||||
.map_err(|error| utility_call_error("wake_up", error))?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
Ok(Json(WakeUpResponse {
|
||||
status: if fully_awake { "awake" } else { "sleeping" },
|
||||
tags,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Return whether the engine is currently sleeping at any level.
|
||||
|
||||
@@ -5892,6 +5892,7 @@ async fn weight_checker_route_rejects_invalid_requests_before_engine_calls() {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn sleep_route_uses_python_compatible_default_query_values() {
|
||||
let before = METRICS.render().expect("render metrics");
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
@@ -5927,13 +5928,95 @@ async fn sleep_route_uses_python_compatible_default_query_values() {
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
|
||||
json!({ "status": "sleeping", "level": 1 })
|
||||
);
|
||||
engine_task.await.expect("mock engine task");
|
||||
let after = METRICS.render().expect("render metrics");
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"http_requests_total",
|
||||
Some("method=\"POST\",status=\"2xx\",handler=\"/sleep\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"vllm:sleep_mode_operation_duration_seconds_count",
|
||||
Some("operation=\"sleep\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
assert_eq!(
|
||||
metric_value(
|
||||
&after,
|
||||
"vllm:sleep_mode_operations_in_flight",
|
||||
Some("operation=\"sleep\""),
|
||||
),
|
||||
Some(0.0)
|
||||
);
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"http_request_duration_seconds_count",
|
||||
Some("method=\"POST\",handler=\"/sleep\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn sleep_route_rejects_invalid_query_before_engine_dispatch() {
|
||||
let before = METRICS.render().expect("render metrics");
|
||||
let (app, engine_task) =
|
||||
test_admin_app_with_engine_script(|_dealer, _push| boxed_test_future(async move {})).await;
|
||||
for query in ["level=invalid", "level=-1", "level=3", "mode=invalid"] {
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(format!("/sleep?{query}"))
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{query}");
|
||||
}
|
||||
engine_task.abort_and_join().await;
|
||||
let after = METRICS.render().expect("render metrics");
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"http_requests_total",
|
||||
Some("method=\"POST\",status=\"4xx\",handler=\"/sleep\""),
|
||||
),
|
||||
4.0
|
||||
);
|
||||
for metric in [
|
||||
"vllm:sleep_mode_operation_duration_seconds_count",
|
||||
"vllm:sleep_mode_operations_in_flight",
|
||||
] {
|
||||
assert_eq!(
|
||||
metric_delta(&before, &after, metric, Some("operation=\"sleep\"")),
|
||||
0.0
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn release_kv_cache_memory_route_sends_expected_utility_call() {
|
||||
let before = METRICS.render().expect("render metrics");
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
@@ -5966,13 +6049,177 @@ async fn release_kv_cache_memory_route_sends_expected_utility_call() {
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
|
||||
json!({ "status": "kv_cache_released" })
|
||||
);
|
||||
engine_task.await.expect("mock engine task");
|
||||
let after = METRICS.render().expect("render metrics");
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"http_requests_total",
|
||||
Some("method=\"POST\",status=\"2xx\",handler=\"/release_kv_cache_memory\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn release_kv_cache_memory_error_records_http_status_and_operation_duration() {
|
||||
let before = METRICS.render().expect("render metrics");
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let call_id =
|
||||
payload.as_array().expect("utility payload array")[1].as_u64().expect("call id");
|
||||
send_outputs(
|
||||
push,
|
||||
UtilityCallOutput {
|
||||
output: UtilityOutput {
|
||||
call_id: call_id.into(),
|
||||
failure_message: Some("requires a completed pause first".to_owned()),
|
||||
result: None,
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/release_kv_cache_memory")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
engine_task.await.expect("mock engine task");
|
||||
let after = METRICS.render().expect("render metrics");
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"http_requests_total",
|
||||
Some("method=\"POST\",status=\"5xx\",handler=\"/release_kv_cache_memory\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"vllm:sleep_mode_operation_duration_seconds_count",
|
||||
Some("operation=\"release_kv_cache_memory\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
assert_eq!(
|
||||
metric_value(
|
||||
&after,
|
||||
"vllm:sleep_mode_operations_in_flight",
|
||||
Some("operation=\"release_kv_cache_memory\""),
|
||||
),
|
||||
Some(0.0)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn cancelled_wake_operation_clears_in_flight_and_is_scraped() {
|
||||
let before = METRICS.render().expect("render metrics");
|
||||
let (dispatched_tx, dispatched_rx) = tokio::sync::oneshot::channel();
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(move |dealer, _push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
assert_eq!(
|
||||
payload.as_array().expect("utility payload array")[2],
|
||||
Value::from("wake_up")
|
||||
);
|
||||
dispatched_tx.send(()).expect("notify dispatch");
|
||||
std::future::pending::<()>().await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
let mut request_app = app.clone();
|
||||
let request_task = tokio::spawn(async move {
|
||||
request_app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/wake_up")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(2), dispatched_rx)
|
||||
.await
|
||||
.expect("wake dispatch timeout")
|
||||
.expect("wake dispatched");
|
||||
let active = METRICS.render().expect("render metrics");
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&active,
|
||||
"vllm:sleep_mode_operations_in_flight",
|
||||
Some("operation=\"wake\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
request_task.abort();
|
||||
assert!(request_task.await.expect_err("request cancelled").is_cancelled());
|
||||
engine_task.abort_and_join().await;
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.uri("/metrics")
|
||||
.body(Body::empty())
|
||||
.expect("build scrape request"),
|
||||
)
|
||||
.await
|
||||
.expect("scrape metrics");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read metrics");
|
||||
let after = String::from_utf8(body.to_vec()).expect("utf8 metrics");
|
||||
assert_eq!(
|
||||
metric_value(
|
||||
&after,
|
||||
"vllm:sleep_mode_operations_in_flight",
|
||||
Some("operation=\"wake\""),
|
||||
),
|
||||
Some(0.0)
|
||||
);
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"vllm:sleep_mode_operation_duration_seconds_count",
|
||||
Some("operation=\"wake\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
assert!(!after.contains("rl_sleep_mode"));
|
||||
assert!(!after.contains("sleep_mode_operations_total"));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn wake_up_route_without_tags_accepts_bool_result() {
|
||||
let before = METRICS.render().expect("render metrics");
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
@@ -6005,8 +6252,21 @@ async fn wake_up_route_without_tags_accepts_bool_result() {
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
|
||||
json!({ "status": "awake", "tags": null })
|
||||
);
|
||||
engine_task.await.expect("mock engine task");
|
||||
let after = METRICS.render().expect("render metrics");
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"http_requests_total",
|
||||
Some("method=\"POST\",status=\"2xx\",handler=\"/wake_up\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
@@ -6030,7 +6290,7 @@ async fn wake_up_route_accepts_repeated_tags() {
|
||||
])])
|
||||
);
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
|
||||
send_outputs(push, utility_outputs(call_id, utility_result_value(false))).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
@@ -6050,13 +6310,17 @@ async fn wake_up_route_accepts_repeated_tags() {
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
|
||||
json!({ "status": "sleeping", "tags": ["weights", "kv_cache"] })
|
||||
);
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn is_sleeping_route_returns_json_payload() {
|
||||
let before = METRICS.render().expect("render metrics");
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
@@ -6094,6 +6358,16 @@ async fn is_sleeping_route_returns_json_payload() {
|
||||
serde_json::from_slice::<serde_json::Value>(&body).expect("decode json"),
|
||||
json!({ "is_sleeping": true })
|
||||
);
|
||||
let after = METRICS.render().expect("render metrics");
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&before,
|
||||
&after,
|
||||
"http_requests_total",
|
||||
Some("method=\"GET\",status=\"2xx\",handler=\"/is_sleeping\""),
|
||||
),
|
||||
1.0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
|
||||
Reference in New Issue
Block a user