[Frontend][RL] Align sleep-mode API responses and operation metrics (#52864)

Signed-off-by: Ronald1995 <ronaldautomobile@163.com>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ronald
2026-09-29 23:16:38 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent 29773c722b
commit e5f3d08d72
14 changed files with 834 additions and 48 deletions
+118 -1
View File
@@ -1,11 +1,13 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::time::Instant;
use prometheus_client::encoding::EncodeLabelSet;
use prometheus_client::metrics::family::Family;
use prometheus_client::registry::Registry;
use crate::{Histogram, U64Counter};
use crate::{Histogram, U64Counter, U64Gauge};
const HTTP_REQUEST_DURATION_BUCKETS: [f64; 3] = [0.1, 0.5, 1.0];
const HTTP_REQUEST_DURATION_HIGHR_BUCKETS: [f64; 21] = [
@@ -21,6 +23,10 @@ fn http_request_duration_highr_histogram() -> Histogram {
Histogram::new(HTTP_REQUEST_DURATION_HIGHR_BUCKETS.iter().copied())
}
fn sleep_mode_operation_duration_histogram() -> Histogram {
Histogram::new([0.01, 0.1, 1.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0])
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct HttpRequestLabels {
pub method: String,
@@ -34,15 +40,25 @@ pub struct HttpHandlerLabels {
pub handler: String,
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct SleepModeOperationLabels {
pub operation: &'static str,
}
pub(crate) type HttpRequestCounterFamily = Family<HttpRequestLabels, U64Counter>;
pub(crate) type HttpHandlerHistogramFamily =
Family<HttpHandlerLabels, Histogram, fn() -> Histogram>;
pub(crate) type SleepModeOperationHistogramFamily =
Family<SleepModeOperationLabels, Histogram, fn() -> Histogram>;
pub(crate) type SleepModeOperationGaugeFamily = Family<SleepModeOperationLabels, U64Gauge>;
/// API-server Prometheus families exported from the HTTP middleware layer.
pub struct ApiServerMetrics {
pub http_requests: HttpRequestCounterFamily,
pub http_request_duration_seconds: HttpHandlerHistogramFamily,
pub http_request_duration_highr_seconds: Histogram,
pub sleep_mode_operation_duration_seconds: SleepModeOperationHistogramFamily,
pub sleep_mode_operations_in_flight: SleepModeOperationGaugeFamily,
}
impl ApiServerMetrics {
@@ -71,10 +87,111 @@ impl ApiServerMetrics {
http_request_duration_highr_seconds.clone(),
);
let sleep_mode_operation_duration_seconds = Family::new_with_constructor(
sleep_mode_operation_duration_histogram as fn() -> Histogram,
);
registry.register(
"vllm:sleep_mode_operation_duration_seconds",
"Duration of one sleep-mode operation.",
sleep_mode_operation_duration_seconds.clone(),
);
let sleep_mode_operations_in_flight = SleepModeOperationGaugeFamily::default();
registry.register(
"vllm:sleep_mode_operations_in_flight",
"Sleep-mode operations currently awaited.",
sleep_mode_operations_in_flight.clone(),
);
Self {
http_requests,
http_request_duration_seconds,
http_request_duration_highr_seconds,
sleep_mode_operation_duration_seconds,
sleep_mode_operations_in_flight,
}
}
/// Record one dispatched sleep-mode engine operation.
pub fn record_sleep_mode_operation(
&self,
operation: &'static str,
) -> SleepModeOperationRecorder {
let labels = SleepModeOperationLabels { operation };
let in_flight = self.sleep_mode_operations_in_flight.get_or_create_owned(&labels);
in_flight.inc();
SleepModeOperationRecorder {
started_at: Instant::now(),
duration: self.sleep_mode_operation_duration_seconds.get_or_create_owned(&labels),
in_flight,
}
}
}
/// Records duration and clears in-flight tracking on completion or cancellation.
pub struct SleepModeOperationRecorder {
started_at: Instant,
duration: Histogram,
in_flight: U64Gauge,
}
impl Drop for SleepModeOperationRecorder {
fn drop(&mut self) {
self.in_flight.dec();
self.duration.observe(self.started_at.elapsed().as_secs_f64());
}
}
#[cfg(test)]
mod tests {
use prometheus_client::encoding::text::encode;
use prometheus_client::registry::Registry;
use super::ApiServerMetrics;
fn rendered_metrics(registry: &Registry) -> String {
let mut rendered = String::new();
encode(&mut rendered, registry).expect("encode metrics");
rendered
}
#[test]
fn sleep_mode_recorder_tracks_duration_and_concurrency() {
let mut registry = Registry::default();
let metrics = ApiServerMetrics::register(&mut registry);
for operation in ["sleep", "release_kv_cache_memory", "wake"] {
let recorder = metrics.record_sleep_mode_operation(operation);
let active = rendered_metrics(&registry);
assert!(active.contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 1"
)));
drop(recorder);
let succeeded = rendered_metrics(&registry);
assert!(succeeded.contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 0"
)));
assert!(succeeded.contains(&format!(
"vllm:sleep_mode_operation_duration_seconds_count{{operation=\"{operation}\"}} 1"
)));
let first = metrics.record_sleep_mode_operation(operation);
let second = metrics.record_sleep_mode_operation(operation);
assert!(rendered_metrics(&registry).contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 2"
)));
drop(first);
assert!(rendered_metrics(&registry).contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 1"
)));
drop(second);
let rendered = rendered_metrics(&registry);
assert!(rendered.contains(&format!(
"vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 0"
)));
assert!(rendered.contains(&format!(
"vllm:sleep_mode_operation_duration_seconds_count{{operation=\"{operation}\"}} 3"
)));
}
}
}