[Rust Frontend][Metrics] Add per-request preemption histogram (#57033)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: reidliu41 <reid201711@gmail.com>
This commit is contained in:
@@ -45,6 +45,7 @@ pub(crate) struct RequestMetricsTracker {
|
||||
last_token_ts: f64,
|
||||
first_token_latency: f64,
|
||||
num_generation_tokens: u32,
|
||||
num_preemptions: u64,
|
||||
latest_num_cached_tokens: u32,
|
||||
}
|
||||
|
||||
@@ -69,6 +70,7 @@ struct RequestMetricHandles {
|
||||
request_max_num_generation_tokens: HistogramMetric,
|
||||
request_params_max_tokens: HistogramMetric,
|
||||
request_params_n: HistogramMetric,
|
||||
request_num_preemptions: HistogramMetric,
|
||||
request_prefill_kv_computed_tokens: HistogramMetric,
|
||||
time_to_first_token_seconds: HistogramMetric,
|
||||
inter_token_latency_seconds: HistogramMetric,
|
||||
@@ -104,6 +106,7 @@ impl RequestMetricsTracker {
|
||||
last_token_ts: 0.0,
|
||||
first_token_latency: 0.0,
|
||||
num_generation_tokens: 0,
|
||||
num_preemptions: 0,
|
||||
latest_num_cached_tokens: 0,
|
||||
}
|
||||
}
|
||||
@@ -184,6 +187,7 @@ impl RequestMetricsTracker {
|
||||
self.handles.request_params_max_tokens.observe(max_tokens_param as f64);
|
||||
}
|
||||
self.handles.request_params_n.observe(self.n_param as f64);
|
||||
self.handles.request_num_preemptions.observe(self.num_preemptions as f64);
|
||||
self.handles
|
||||
.request_prefill_kv_computed_tokens
|
||||
.observe(prefill_kv_computed_tokens as f64);
|
||||
@@ -223,6 +227,7 @@ impl RequestMetricsTracker {
|
||||
}
|
||||
}
|
||||
EngineCoreEventType::Preempted => {
|
||||
self.num_preemptions += 1;
|
||||
self.handles.num_preemptions.inc();
|
||||
}
|
||||
}
|
||||
@@ -276,6 +281,7 @@ fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetri
|
||||
.get_or_create_owned(&labels),
|
||||
request_params_max_tokens: metrics.request_params_max_tokens.get_or_create_owned(&labels),
|
||||
request_params_n: metrics.request_params_n.get_or_create_owned(&labels),
|
||||
request_num_preemptions: metrics.request_num_preemptions.get_or_create_owned(&labels),
|
||||
request_prefill_kv_computed_tokens: metrics
|
||||
.request_prefill_kv_computed_tokens
|
||||
.get_or_create_owned(&labels),
|
||||
|
||||
@@ -790,6 +790,12 @@ async fn generate_records_request_metrics_in_prometheus_output() {
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:num_preemptions_total{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_num_preemptions_sum{{model_name=\"{model_name}\",engine=\"4\"}} 1.0"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_num_preemptions_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:time_to_first_token_seconds_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
@@ -883,6 +889,12 @@ async fn dropping_stream_records_abort_terminal_request_metrics() {
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:e2e_request_latency_seconds_count{{model_name=\"{model_name}\",engine=\"5\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_num_preemptions_sum{{model_name=\"{model_name}\",engine=\"5\"}} 0.0"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_num_preemptions_count{{model_name=\"{model_name}\",engine=\"5\"}} 1"
|
||||
)));
|
||||
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user