[Rust Frontend][Metrics] Add per-request preemption histogram (#57033)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: reidliu41 <reid201711@gmail.com>
This commit is contained in:
Reid
2026-09-16 02:51:17 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent 7d4bba5498
commit 0673de9800
3 changed files with 33 additions and 0 deletions
+15
View File
@@ -21,6 +21,7 @@ const REQUEST_LATENCY_BUCKETS: [f64; 21] = [
480.0, 960.0, 1920.0, 7680.0,
];
const REQUEST_PARAMS_N_BUCKETS: [f64; 5] = [1.0, 2.0, 5.0, 10.0, 20.0];
const REQUEST_NUM_PREEMPTIONS_BUCKETS: [f64; 7] = [1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0];
const ITERATION_TOKENS_BUCKETS: [f64; 13] = [
1.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0, 2048.0, 4096.0, 8192.0, 16384.0,
];
@@ -69,6 +70,10 @@ fn request_params_n_histogram() -> Histogram {
Histogram::new(REQUEST_PARAMS_N_BUCKETS.iter().copied())
}
fn request_num_preemptions_histogram() -> Histogram {
Histogram::new(REQUEST_NUM_PREEMPTIONS_BUCKETS.iter().copied())
}
fn iteration_tokens_histogram() -> Histogram {
Histogram::new(ITERATION_TOKENS_BUCKETS.iter().copied())
}
@@ -110,6 +115,7 @@ pub struct RequestMetrics {
pub request_max_num_generation_tokens: HistogramFamily,
pub request_params_max_tokens: HistogramFamily,
pub request_params_n: HistogramFamily,
pub request_num_preemptions: HistogramFamily,
pub request_prefill_kv_computed_tokens: HistogramFamily,
pub time_to_first_token_seconds: HistogramFamily,
pub inter_token_latency_seconds: HistogramFamily,
@@ -216,6 +222,14 @@ impl RequestMetrics {
request_params_n.clone(),
);
let request_num_preemptions =
Family::new_with_constructor(request_num_preemptions_histogram as fn() -> Histogram);
registry.register(
"vllm:request_num_preemptions",
"Histogram of the number of times a request was preempted.",
request_num_preemptions.clone(),
);
let request_prefill_kv_computed_tokens =
Family::new_with_constructor(request_token_count_histogram as fn() -> Histogram);
registry.register(
@@ -302,6 +316,7 @@ impl RequestMetrics {
request_max_num_generation_tokens,
request_params_max_tokens,
request_params_n,
request_num_preemptions,
request_prefill_kv_computed_tokens,
time_to_first_token_seconds,
inter_token_latency_seconds,