[Frontend][RL] Track HTTP weight operation outcomes and concurrency (#55781)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5.5
aoshen02
parent
ae66f9a97c
commit
713ec07655
@@ -23,6 +23,10 @@ fn http_request_duration_highr_histogram() -> Histogram {
|
||||
Histogram::new(HTTP_REQUEST_DURATION_HIGHR_BUCKETS.iter().copied())
|
||||
}
|
||||
|
||||
fn weight_operation_duration_histogram() -> Histogram {
|
||||
Histogram::new([0.01, 0.1, 1.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0])
|
||||
}
|
||||
|
||||
fn sleep_mode_operation_duration_histogram() -> Histogram {
|
||||
Histogram::new([0.01, 0.1, 1.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0])
|
||||
}
|
||||
@@ -40,6 +44,11 @@ pub struct HttpHandlerLabels {
|
||||
pub handler: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
|
||||
pub struct WeightOperationLabels {
|
||||
pub operation: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
|
||||
pub struct SleepModeOperationLabels {
|
||||
pub operation: &'static str,
|
||||
@@ -48,6 +57,9 @@ pub struct SleepModeOperationLabels {
|
||||
pub(crate) type HttpRequestCounterFamily = Family<HttpRequestLabels, U64Counter>;
|
||||
pub(crate) type HttpHandlerHistogramFamily =
|
||||
Family<HttpHandlerLabels, Histogram, fn() -> Histogram>;
|
||||
pub(crate) type WeightOperationHistogramFamily =
|
||||
Family<WeightOperationLabels, Histogram, fn() -> Histogram>;
|
||||
pub(crate) type WeightOperationGaugeFamily = Family<WeightOperationLabels, U64Gauge>;
|
||||
pub(crate) type SleepModeOperationHistogramFamily =
|
||||
Family<SleepModeOperationLabels, Histogram, fn() -> Histogram>;
|
||||
pub(crate) type SleepModeOperationGaugeFamily = Family<SleepModeOperationLabels, U64Gauge>;
|
||||
@@ -57,6 +69,8 @@ pub struct ApiServerMetrics {
|
||||
pub http_requests: HttpRequestCounterFamily,
|
||||
pub http_request_duration_seconds: HttpHandlerHistogramFamily,
|
||||
pub http_request_duration_highr_seconds: Histogram,
|
||||
pub weight_operation_duration_seconds: WeightOperationHistogramFamily,
|
||||
pub weight_operations_in_flight: WeightOperationGaugeFamily,
|
||||
pub sleep_mode_operation_duration_seconds: SleepModeOperationHistogramFamily,
|
||||
pub sleep_mode_operations_in_flight: SleepModeOperationGaugeFamily,
|
||||
}
|
||||
@@ -87,6 +101,21 @@ impl ApiServerMetrics {
|
||||
http_request_duration_highr_seconds.clone(),
|
||||
);
|
||||
|
||||
let weight_operation_duration_seconds =
|
||||
Family::new_with_constructor(weight_operation_duration_histogram as fn() -> Histogram);
|
||||
registry.register(
|
||||
"vllm:rl_weight_update_operation_duration_seconds",
|
||||
"Duration of one weight-transfer operation.",
|
||||
weight_operation_duration_seconds.clone(),
|
||||
);
|
||||
|
||||
let weight_operations_in_flight = WeightOperationGaugeFamily::default();
|
||||
registry.register(
|
||||
"vllm:rl_weight_update_operations_in_flight",
|
||||
"Weight-transfer operations currently awaited.",
|
||||
weight_operations_in_flight.clone(),
|
||||
);
|
||||
|
||||
let sleep_mode_operation_duration_seconds = Family::new_with_constructor(
|
||||
sleep_mode_operation_duration_histogram as fn() -> Histogram,
|
||||
);
|
||||
@@ -107,11 +136,25 @@ impl ApiServerMetrics {
|
||||
http_requests,
|
||||
http_request_duration_seconds,
|
||||
http_request_duration_highr_seconds,
|
||||
weight_operation_duration_seconds,
|
||||
weight_operations_in_flight,
|
||||
sleep_mode_operation_duration_seconds,
|
||||
sleep_mode_operations_in_flight,
|
||||
}
|
||||
}
|
||||
|
||||
/// Track one weight-transfer operation until the returned guard is dropped.
|
||||
pub fn record_weight_operation(&self, operation: &'static str) -> WeightOperationRecorder {
|
||||
let labels = WeightOperationLabels { operation };
|
||||
let in_flight = self.weight_operations_in_flight.get_or_create_owned(&labels);
|
||||
in_flight.inc();
|
||||
WeightOperationRecorder {
|
||||
started_at: Instant::now(),
|
||||
duration: self.weight_operation_duration_seconds.get_or_create_owned(&labels),
|
||||
in_flight,
|
||||
}
|
||||
}
|
||||
|
||||
/// Record one dispatched sleep-mode engine operation.
|
||||
pub fn record_sleep_mode_operation(
|
||||
&self,
|
||||
@@ -128,6 +171,19 @@ impl ApiServerMetrics {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct WeightOperationRecorder {
|
||||
started_at: Instant,
|
||||
duration: Histogram,
|
||||
in_flight: U64Gauge,
|
||||
}
|
||||
|
||||
impl Drop for WeightOperationRecorder {
|
||||
fn drop(&mut self) {
|
||||
self.in_flight.dec();
|
||||
self.duration.observe(self.started_at.elapsed().as_secs_f64());
|
||||
}
|
||||
}
|
||||
|
||||
/// Records duration and clears in-flight tracking on completion or cancellation.
|
||||
pub struct SleepModeOperationRecorder {
|
||||
started_at: Instant,
|
||||
@@ -194,4 +250,25 @@ mod tests {
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn weight_operation_recorder_tracks_in_flight_and_duration() {
|
||||
let mut registry = Registry::default();
|
||||
let metrics = ApiServerMetrics::register(&mut registry);
|
||||
|
||||
let recorder = metrics.record_weight_operation("update");
|
||||
assert!(
|
||||
rendered_metrics(®istry)
|
||||
.contains("vllm:rl_weight_update_operations_in_flight{operation=\"update\"} 1")
|
||||
);
|
||||
drop(recorder);
|
||||
|
||||
let after = rendered_metrics(®istry);
|
||||
assert!(
|
||||
after.contains("vllm:rl_weight_update_operations_in_flight{operation=\"update\"} 0")
|
||||
);
|
||||
assert!(after.contains(
|
||||
"vllm:rl_weight_update_operation_duration_seconds_count{operation=\"update\"} 1"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6723,6 +6723,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
|
||||
})
|
||||
})
|
||||
.await;
|
||||
let metrics_before = METRICS.render().unwrap();
|
||||
|
||||
let mut responses = Vec::new();
|
||||
for (method, path, body) in [
|
||||
@@ -6785,6 +6786,26 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() {
|
||||
);
|
||||
}
|
||||
engine_task.await.expect("mock engine task");
|
||||
let metrics_after = METRICS.render().unwrap();
|
||||
for operation in ["update", "finish"] {
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&metrics_before,
|
||||
&metrics_after,
|
||||
"vllm:rl_weight_update_operation_duration_seconds_count",
|
||||
Some(&format!("operation=\"{operation}\"")),
|
||||
),
|
||||
2.0
|
||||
);
|
||||
assert_eq!(
|
||||
metric_value(
|
||||
&metrics_after,
|
||||
"vllm:rl_weight_update_operations_in_flight",
|
||||
Some(&format!("operation=\"{operation}\"")),
|
||||
),
|
||||
Some(0.0)
|
||||
);
|
||||
}
|
||||
expect_test::expect![[r#"
|
||||
[
|
||||
"{\"message\":\"Weight transfer initialized\"}",
|
||||
@@ -6811,6 +6832,7 @@ async fn weight_transfer_routes_reject_invalid_payloads_before_engine_calls() {
|
||||
})
|
||||
})
|
||||
.await;
|
||||
let metrics_before = METRICS.render().unwrap();
|
||||
|
||||
for (path, body) in [
|
||||
("/init_weight_transfer_engine", "{"),
|
||||
@@ -6840,6 +6862,18 @@ async fn weight_transfer_routes_reject_invalid_payloads_before_engine_calls() {
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{path}: {body}");
|
||||
}
|
||||
let metrics_after = METRICS.render().unwrap();
|
||||
for operation in ["init", "update", "finish"] {
|
||||
assert_eq!(
|
||||
metric_delta(
|
||||
&metrics_before,
|
||||
&metrics_after,
|
||||
"vllm:rl_weight_update_operation_duration_seconds_count",
|
||||
Some(&format!("operation=\"{operation}\"")),
|
||||
),
|
||||
0.0
|
||||
);
|
||||
}
|
||||
engine_task.abort_and_join().await;
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ use axum::extract::State;
|
||||
use axum::extract::rejection::JsonRejection;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value as JsonValue;
|
||||
use vllm_metrics::METRICS;
|
||||
|
||||
use crate::error::{ApiError, invalid_request};
|
||||
use crate::state::AppState;
|
||||
@@ -62,6 +63,7 @@ pub async fn init_weight_transfer_engine(
|
||||
)
|
||||
})?;
|
||||
|
||||
let _recorder = METRICS.api_server.record_weight_operation("init");
|
||||
state
|
||||
.engine_core_client()
|
||||
.init_weight_transfer_engine(init_info)
|
||||
@@ -77,6 +79,7 @@ pub async fn init_weight_transfer_engine(
|
||||
pub async fn start_weight_update(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<Json<MessageResponse>, ApiError> {
|
||||
let _recorder = METRICS.api_server.record_weight_operation("start");
|
||||
state
|
||||
.engine_core_client()
|
||||
.start_weight_update()
|
||||
@@ -92,6 +95,7 @@ pub async fn start_weight_update(
|
||||
pub async fn start_draft_weight_update(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<Json<MessageResponse>, ApiError> {
|
||||
let _recorder = METRICS.api_server.record_weight_operation("start_draft");
|
||||
state
|
||||
.engine_core_client()
|
||||
.start_draft_weight_update()
|
||||
@@ -122,6 +126,7 @@ pub async fn update_weights(
|
||||
)
|
||||
})?;
|
||||
|
||||
let _recorder = METRICS.api_server.record_weight_operation("update");
|
||||
state
|
||||
.engine_core_client()
|
||||
.update_weights(update_info)
|
||||
@@ -142,6 +147,7 @@ pub async fn finish_weight_update(
|
||||
let request = body?.map(|Json(request)| request).unwrap_or_default();
|
||||
|
||||
let client = state.engine_core_client();
|
||||
let _recorder = METRICS.api_server.record_weight_operation("finish");
|
||||
client
|
||||
.finish_weight_update()
|
||||
.await
|
||||
|
||||
Reference in New Issue
Block a user