[Rust Frontend][Metrics] Add per-request preemption histogram (#57033)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: reidliu41 <reid201711@gmail.com>
This commit is contained in:
@@ -21,6 +21,7 @@ const REQUEST_LATENCY_BUCKETS: [f64; 21] = [
|
||||
480.0, 960.0, 1920.0, 7680.0,
|
||||
];
|
||||
const REQUEST_PARAMS_N_BUCKETS: [f64; 5] = [1.0, 2.0, 5.0, 10.0, 20.0];
|
||||
const REQUEST_NUM_PREEMPTIONS_BUCKETS: [f64; 7] = [1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0];
|
||||
const ITERATION_TOKENS_BUCKETS: [f64; 13] = [
|
||||
1.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0, 2048.0, 4096.0, 8192.0, 16384.0,
|
||||
];
|
||||
@@ -69,6 +70,10 @@ fn request_params_n_histogram() -> Histogram {
|
||||
Histogram::new(REQUEST_PARAMS_N_BUCKETS.iter().copied())
|
||||
}
|
||||
|
||||
fn request_num_preemptions_histogram() -> Histogram {
|
||||
Histogram::new(REQUEST_NUM_PREEMPTIONS_BUCKETS.iter().copied())
|
||||
}
|
||||
|
||||
fn iteration_tokens_histogram() -> Histogram {
|
||||
Histogram::new(ITERATION_TOKENS_BUCKETS.iter().copied())
|
||||
}
|
||||
@@ -110,6 +115,7 @@ pub struct RequestMetrics {
|
||||
pub request_max_num_generation_tokens: HistogramFamily,
|
||||
pub request_params_max_tokens: HistogramFamily,
|
||||
pub request_params_n: HistogramFamily,
|
||||
pub request_num_preemptions: HistogramFamily,
|
||||
pub request_prefill_kv_computed_tokens: HistogramFamily,
|
||||
pub time_to_first_token_seconds: HistogramFamily,
|
||||
pub inter_token_latency_seconds: HistogramFamily,
|
||||
@@ -216,6 +222,14 @@ impl RequestMetrics {
|
||||
request_params_n.clone(),
|
||||
);
|
||||
|
||||
let request_num_preemptions =
|
||||
Family::new_with_constructor(request_num_preemptions_histogram as fn() -> Histogram);
|
||||
registry.register(
|
||||
"vllm:request_num_preemptions",
|
||||
"Histogram of the number of times a request was preempted.",
|
||||
request_num_preemptions.clone(),
|
||||
);
|
||||
|
||||
let request_prefill_kv_computed_tokens =
|
||||
Family::new_with_constructor(request_token_count_histogram as fn() -> Histogram);
|
||||
registry.register(
|
||||
@@ -302,6 +316,7 @@ impl RequestMetrics {
|
||||
request_max_num_generation_tokens,
|
||||
request_params_max_tokens,
|
||||
request_params_n,
|
||||
request_num_preemptions,
|
||||
request_prefill_kv_computed_tokens,
|
||||
time_to_first_token_seconds,
|
||||
inter_token_latency_seconds,
|
||||
|
||||
Reference in New Issue
Block a user