[Rust Frontend] Move code from vllm-frontend-rs (#43283)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Eric Curtin <eric.curtin@docker.com>
Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Signed-off-by: Will.hou <1205157517@qq.com>
Signed-off-by: Will.hou <willamhou@ceresman.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Eric Curtin <eric.curtin@docker.com>
Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Co-authored-by: Will.hou <1205157517@qq.com>
Co-authored-by: Will.hou <willamhou@ceresman.com>

Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
Bugen Zhao
2026-05-21 17:21:48 -07:00
committed by GitHub
co-authored by Nick Hill Eric Curtin Dev-X25874 Will.hou Will.hou
parent 39d5fa96a7
commit 39910f2b25
284 changed files with 72569 additions and 90 deletions
+11
View File
@@ -0,0 +1,11 @@
[package]
name = "vllm-metrics"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
prometheus-client.workspace = true
[lints]
workspace = true
+78
View File
@@ -0,0 +1,78 @@
use prometheus_client::encoding::EncodeLabelSet;
use prometheus_client::metrics::family::Family;
use prometheus_client::metrics::histogram::Histogram;
use prometheus_client::registry::Registry;
use crate::U64Counter;
const HTTP_REQUEST_DURATION_BUCKETS: [f64; 3] = [0.1, 0.5, 1.0];
const HTTP_REQUEST_DURATION_HIGHR_BUCKETS: [f64; 21] = [
0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0,
7.5, 10.0, 30.0, 60.0,
];
fn http_request_duration_histogram() -> Histogram {
Histogram::new(HTTP_REQUEST_DURATION_BUCKETS.iter().copied())
}
fn http_request_duration_highr_histogram() -> Histogram {
Histogram::new(HTTP_REQUEST_DURATION_HIGHR_BUCKETS.iter().copied())
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct HttpRequestLabels {
pub method: String,
pub status: &'static str,
pub handler: String,
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct HttpHandlerLabels {
pub method: String,
pub handler: String,
}
pub(crate) type HttpRequestCounterFamily = Family<HttpRequestLabels, U64Counter>;
pub(crate) type HttpHandlerHistogramFamily =
Family<HttpHandlerLabels, Histogram, fn() -> Histogram>;
/// API-server Prometheus families exported from the HTTP middleware layer.
pub struct ApiServerMetrics {
pub http_requests: HttpRequestCounterFamily,
pub http_request_duration_seconds: HttpHandlerHistogramFamily,
pub http_request_duration_highr_seconds: Histogram,
}
impl ApiServerMetrics {
/// Register the API-server metric families into the shared registry.
pub(crate) fn register(registry: &mut Registry) -> Self {
let http_requests = HttpRequestCounterFamily::default();
registry.register(
"http_requests",
"Total number of HTTP requests by method, status, and handler.",
http_requests.clone(),
);
let http_request_duration_seconds = HttpHandlerHistogramFamily::new_with_constructor(
http_request_duration_histogram as fn() -> Histogram,
);
registry.register(
"http_request_duration_seconds",
"Duration of HTTP requests in seconds grouped by method and handler.",
http_request_duration_seconds.clone(),
);
let http_request_duration_highr_seconds = http_request_duration_highr_histogram();
registry.register(
"http_request_duration_highr_seconds",
"High-resolution duration of HTTP requests in seconds.",
http_request_duration_highr_seconds.clone(),
);
Self {
http_requests,
http_request_duration_seconds,
http_request_duration_highr_seconds,
}
}
}
+76
View File
@@ -0,0 +1,76 @@
use std::fmt;
use std::sync::LazyLock;
use std::sync::atomic::AtomicU64;
use prometheus_client::encoding::text::encode;
use prometheus_client::metrics::counter::Counter;
use prometheus_client::metrics::family::Family;
use prometheus_client::metrics::gauge::Gauge;
use prometheus_client::metrics::histogram::Histogram;
use prometheus_client::registry::Registry;
mod api_server;
mod request;
mod scheduler;
pub use api_server::*;
pub use request::*;
pub use scheduler::*;
// Note: `prometheus-client` appends the `_total` suffix automatically when
// encoding counters, so all counter family registration names in this crate
// must use the base metric name without a trailing `_total`.
pub type U64Counter = Counter<u64, AtomicU64>;
pub type U64Gauge = Gauge<u64, AtomicU64>;
pub type F64Gauge = Gauge<f64, AtomicU64>;
pub(crate) type HistogramFamily = Family<EngineLabels, Histogram, fn() -> Histogram>;
/// Shared Prometheus registry for frontend metrics.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/loggers.py#L389-L1004>
pub struct Metrics {
registry: Registry,
pub scheduler: SchedulerMetrics,
pub request: RequestMetrics,
pub api_server: ApiServerMetrics,
}
impl Metrics {
/// Construct a new metrics registry.
pub fn new() -> Self {
let mut registry = Registry::default();
let scheduler = SchedulerMetrics::register(&mut registry);
let request = RequestMetrics::register(&mut registry);
let api_server = ApiServerMetrics::register(&mut registry);
Self {
registry,
scheduler,
request,
api_server,
}
}
/// Render the current metrics registry into Prometheus/OpenMetrics text
/// format.
pub fn render(&self) -> Result<String, fmt::Error> {
let mut output = String::new();
encode(&mut output, &self.registry)?;
Ok(output)
}
/// Return the registry owned by this metrics object.
pub fn registry(&self) -> &Registry {
&self.registry
}
}
impl Default for Metrics {
fn default() -> Self {
Self::new()
}
}
/// Process-global metrics registry shared by the frontend crates.
pub static METRICS: LazyLock<Metrics> = LazyLock::new(Metrics::new);
+298
View File
@@ -0,0 +1,298 @@
use prometheus_client::encoding::EncodeLabelSet;
use prometheus_client::metrics::family::Family;
use prometheus_client::metrics::histogram::Histogram;
use prometheus_client::registry::Registry;
use crate::{EngineLabels, HistogramFamily, U64Counter};
const TTFT_BUCKETS: [f64; 22] = [
0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0,
20.0, 40.0, 80.0, 160.0, 640.0, 2560.0,
];
const ITL_BUCKETS: [f64; 19] = [
0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0, 20.0,
40.0, 80.0,
];
const REQUEST_LATENCY_BUCKETS: [f64; 21] = [
0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0, 120.0, 240.0,
480.0, 960.0, 1920.0, 7680.0,
];
const REQUEST_PARAMS_N_BUCKETS: [f64; 5] = [1.0, 2.0, 5.0, 10.0, 20.0];
fn build_1_2_5_buckets(max_value: u32) -> Vec<f64> {
let mut buckets = Vec::new();
let mut exponent = 0;
loop {
for mantissa in [1_u32, 2, 5] {
let value = mantissa * 10_u32.pow(exponent);
if value <= max_value {
buckets.push(value as f64);
} else {
if buckets.last().copied() != Some(max_value as f64) {
buckets.push(max_value as f64);
}
return buckets;
}
}
exponent += 1;
}
}
fn time_to_first_token_histogram() -> Histogram {
Histogram::new(TTFT_BUCKETS.iter().copied())
}
fn inter_token_latency_histogram() -> Histogram {
Histogram::new(ITL_BUCKETS.iter().copied())
}
fn request_time_per_output_token_histogram() -> Histogram {
Histogram::new(ITL_BUCKETS.iter().copied())
}
fn request_latency_histogram() -> Histogram {
Histogram::new(REQUEST_LATENCY_BUCKETS.iter().copied())
}
fn request_token_count_histogram() -> Histogram {
// TODO: determine max value based on `max_model_len`.
Histogram::new(build_1_2_5_buckets(131_072))
}
fn request_params_n_histogram() -> Histogram {
Histogram::new(REQUEST_PARAMS_N_BUCKETS.iter().copied())
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct FinishedReasonLabels {
pub model_name: String,
pub engine: u32,
pub finished_reason: &'static str,
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct PromptTokenSourceLabels {
pub model_name: String,
pub engine: u32,
pub source: &'static str,
}
pub(crate) type FinishedReasonCounterFamily = Family<FinishedReasonLabels, U64Counter>;
pub(crate) type PromptTokenSourceCounterFamily = Family<PromptTokenSourceLabels, U64Counter>;
/// Request-lifecycle Prometheus families exported from the `llm` layer.
pub struct RequestMetrics {
// Request-derived counters.
pub num_preemptions: Family<EngineLabels, U64Counter>,
pub prompt_tokens: Family<EngineLabels, U64Counter>,
pub prompt_tokens_by_source: PromptTokenSourceCounterFamily,
pub prompt_tokens_cached: Family<EngineLabels, U64Counter>,
pub generation_tokens: Family<EngineLabels, U64Counter>,
// We intentionally don't support iteration-level histograms for now, since it seems to make
// more sense if the engine maintains these metrics and frontend simply forwards.
//
// pub iteration_tokens_total: HistogramFamily,
// Request lifecycle counters and histograms.
pub request_success: FinishedReasonCounterFamily,
pub request_prompt_tokens: HistogramFamily,
pub request_generation_tokens: HistogramFamily,
pub request_max_num_generation_tokens: HistogramFamily,
pub request_params_max_tokens: HistogramFamily,
pub request_params_n: HistogramFamily,
pub request_prefill_kv_computed_tokens: HistogramFamily,
pub time_to_first_token_seconds: HistogramFamily,
pub inter_token_latency_seconds: HistogramFamily,
pub e2e_request_latency_seconds: HistogramFamily,
pub request_queue_time_seconds: HistogramFamily,
pub request_prefill_time_seconds: HistogramFamily,
pub request_decode_time_seconds: HistogramFamily,
pub request_inference_time_seconds: HistogramFamily,
pub request_time_per_output_token_seconds: HistogramFamily,
}
impl RequestMetrics {
/// Register the request-oriented metric families into the shared registry.
pub(crate) fn register(registry: &mut Registry) -> Self {
// Request-derived counters.
let num_preemptions = Family::default();
registry.register(
"vllm:num_preemptions",
"Cumulative number of preemption events.",
num_preemptions.clone(),
);
let prompt_tokens = Family::default();
registry.register(
"vllm:prompt_tokens",
"Number of prefill tokens processed.",
prompt_tokens.clone(),
);
let prompt_tokens_by_source = Family::default();
registry.register(
"vllm:prompt_tokens_by_source",
"Number of prompt tokens by source.",
prompt_tokens_by_source.clone(),
);
let prompt_tokens_cached = Family::default();
registry.register(
"vllm:prompt_tokens_cached",
"Number of prompt tokens with prefix cache hits.",
prompt_tokens_cached.clone(),
);
let generation_tokens = Family::default();
registry.register(
"vllm:generation_tokens",
"Number of generation tokens processed.",
generation_tokens.clone(),
);
// Request lifecycle counters and histograms.
let request_success = Family::default();
registry.register(
"vllm:request_success",
"Count of successfully processed requests.",
request_success.clone(),
);
let request_prompt_tokens =
Family::new_with_constructor(request_token_count_histogram as fn() -> Histogram);
registry.register(
"vllm:request_prompt_tokens",
"Number of prefill tokens processed.",
request_prompt_tokens.clone(),
);
let request_generation_tokens =
Family::new_with_constructor(request_token_count_histogram as fn() -> Histogram);
registry.register(
"vllm:request_generation_tokens",
"Number of generation tokens processed.",
request_generation_tokens.clone(),
);
let request_max_num_generation_tokens =
Family::new_with_constructor(request_token_count_histogram as fn() -> Histogram);
registry.register(
"vllm:request_max_num_generation_tokens",
"Histogram of maximum number of requested generation tokens.",
request_max_num_generation_tokens.clone(),
);
let request_params_max_tokens =
Family::new_with_constructor(request_token_count_histogram as fn() -> Histogram);
registry.register(
"vllm:request_params_max_tokens",
"Histogram of the max_tokens request parameter.",
request_params_max_tokens.clone(),
);
let request_params_n =
Family::new_with_constructor(request_params_n_histogram as fn() -> Histogram);
registry.register(
"vllm:request_params_n",
"Histogram of the n request parameter.",
request_params_n.clone(),
);
let request_prefill_kv_computed_tokens =
Family::new_with_constructor(request_token_count_histogram as fn() -> Histogram);
registry.register(
"vllm:request_prefill_kv_computed_tokens",
"Histogram of new KV tokens computed during prefill (excluding cached tokens).",
request_prefill_kv_computed_tokens.clone(),
);
let time_to_first_token_seconds =
Family::new_with_constructor(time_to_first_token_histogram as fn() -> Histogram);
registry.register(
"vllm:time_to_first_token_seconds",
"Histogram of time to first token in seconds.",
time_to_first_token_seconds.clone(),
);
let inter_token_latency_seconds =
Family::new_with_constructor(inter_token_latency_histogram as fn() -> Histogram);
registry.register(
"vllm:inter_token_latency_seconds",
"Histogram of inter-token latency in seconds.",
inter_token_latency_seconds.clone(),
);
let e2e_request_latency_seconds =
Family::new_with_constructor(request_latency_histogram as fn() -> Histogram);
registry.register(
"vllm:e2e_request_latency_seconds",
"Histogram of e2e request latency in seconds.",
e2e_request_latency_seconds.clone(),
);
let request_queue_time_seconds =
Family::new_with_constructor(request_latency_histogram as fn() -> Histogram);
registry.register(
"vllm:request_queue_time_seconds",
"Histogram of time spent in WAITING phase for request.",
request_queue_time_seconds.clone(),
);
let request_prefill_time_seconds =
Family::new_with_constructor(request_latency_histogram as fn() -> Histogram);
registry.register(
"vllm:request_prefill_time_seconds",
"Histogram of time spent in PREFILL phase for request.",
request_prefill_time_seconds.clone(),
);
let request_decode_time_seconds =
Family::new_with_constructor(request_latency_histogram as fn() -> Histogram);
registry.register(
"vllm:request_decode_time_seconds",
"Histogram of time spent in DECODE phase for request.",
request_decode_time_seconds.clone(),
);
let request_inference_time_seconds =
Family::new_with_constructor(request_latency_histogram as fn() -> Histogram);
registry.register(
"vllm:request_inference_time_seconds",
"Histogram of time spent in RUNNING phase for request.",
request_inference_time_seconds.clone(),
);
let request_time_per_output_token_seconds = Family::new_with_constructor(
request_time_per_output_token_histogram as fn() -> Histogram,
);
registry.register(
"vllm:request_time_per_output_token_seconds",
"Histogram of time_per_output_token_seconds per request.",
request_time_per_output_token_seconds.clone(),
);
Self {
num_preemptions,
prompt_tokens,
prompt_tokens_by_source,
prompt_tokens_cached,
generation_tokens,
request_success,
request_prompt_tokens,
request_generation_tokens,
request_max_num_generation_tokens,
request_params_max_tokens,
request_params_n,
request_prefill_kv_computed_tokens,
time_to_first_token_seconds,
inter_token_latency_seconds,
e2e_request_latency_seconds,
request_queue_time_seconds,
request_prefill_time_seconds,
request_decode_time_seconds,
request_inference_time_seconds,
request_time_per_output_token_seconds,
}
}
}
+272
View File
@@ -0,0 +1,272 @@
use prometheus_client::encoding::EncodeLabelSet;
use prometheus_client::metrics::family::Family;
use prometheus_client::metrics::histogram::Histogram;
use prometheus_client::registry::Registry;
use crate::{F64Gauge, HistogramFamily, U64Counter, U64Gauge};
const KV_CACHE_RESIDENCY_BUCKETS: [f64; 21] = [
0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 60.0,
120.0, 300.0, 600.0, 1200.0, 1800.0,
];
fn kv_block_lifetime_histogram() -> Histogram {
Histogram::new(KV_CACHE_RESIDENCY_BUCKETS.iter().copied())
}
fn kv_block_idle_before_evict_histogram() -> Histogram {
Histogram::new(KV_CACHE_RESIDENCY_BUCKETS.iter().copied())
}
fn kv_block_reuse_gap_histogram() -> Histogram {
Histogram::new(KV_CACHE_RESIDENCY_BUCKETS.iter().copied())
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct EngineLabels {
pub model_name: String,
pub engine: u32,
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct EnginePositionLabels {
pub model_name: String,
pub engine: u32,
pub position: u32,
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct WaitingReasonLabels {
pub model_name: String,
pub engine: u32,
pub reason: &'static str,
}
/// Scheduler/batch-scoped Prometheus families exported from `SchedulerStats`.
pub struct SchedulerMetrics {
// Scheduler state gauges.
pub scheduler_running: Family<EngineLabels, U64Gauge>,
pub scheduler_waiting: Family<EngineLabels, U64Gauge>,
pub scheduler_waiting_by_reason: Family<WaitingReasonLabels, U64Gauge>,
pub kv_cache_usage: Family<EngineLabels, F64Gauge>,
// Prefix-cache counters, including the connector-backed external cache path.
pub prefix_cache_queries: Family<EngineLabels, U64Counter>,
pub prefix_cache_hits: Family<EngineLabels, U64Counter>,
pub external_prefix_cache_queries: Family<EngineLabels, U64Counter>,
pub external_prefix_cache_hits: Family<EngineLabels, U64Counter>,
// Speculative decoding counters.
pub spec_decode_num_drafts: Family<EngineLabels, U64Counter>,
pub spec_decode_num_draft_tokens: Family<EngineLabels, U64Counter>,
pub spec_decode_num_accepted_tokens: Family<EngineLabels, U64Counter>,
pub spec_decode_num_accepted_tokens_per_pos: Family<EnginePositionLabels, U64Counter>,
// Per-engine performance / MFU counters.
pub estimated_flops_per_gpu: Family<EngineLabels, U64Counter>,
pub estimated_read_bytes_per_gpu: Family<EngineLabels, U64Counter>,
pub estimated_write_bytes_per_gpu: Family<EngineLabels, U64Counter>,
// Sampled KV-cache residency histograms.
pub kv_block_lifetime_seconds: HistogramFamily,
pub kv_block_idle_before_evict_seconds: HistogramFamily,
pub kv_block_reuse_gap_seconds: HistogramFamily,
}
impl SchedulerMetrics {
/// Register the scheduler-oriented metric families into the shared
/// registry.
pub(crate) fn register(registry: &mut Registry) -> Self {
// Scheduler state gauges.
let scheduler_running = Family::default();
registry.register(
"vllm:num_requests_running",
"Number of requests in model execution batches",
scheduler_running.clone(),
);
let scheduler_waiting = Family::default();
registry.register(
"vllm:num_requests_waiting",
"Number of requests waiting to be processed",
scheduler_waiting.clone(),
);
let scheduler_waiting_by_reason = Family::default();
registry.register(
"vllm:num_requests_waiting_by_reason",
"Number of waiting requests by reason. \
Reason labels: 'capacity' = waiting for scheduling capacity; \
'deferred' = deferred by transient constraints (LoRA budget, KV transfer, \
blocked status). Sum of all reasons equals vllm:num_requests_waiting.",
scheduler_waiting_by_reason.clone(),
);
let kv_cache_usage = Family::default();
registry.register(
"vllm:kv_cache_usage_perc",
"KV-cache usage. 1 means 100 percent usage",
kv_cache_usage.clone(),
);
// Prefix-cache counters, including the connector-backed external cache path.
let prefix_cache_queries = Family::default();
registry.register(
"vllm:prefix_cache_queries",
"Prefix cache queries, in terms of number of queried tokens",
prefix_cache_queries.clone(),
);
let prefix_cache_hits = Family::default();
registry.register(
"vllm:prefix_cache_hits",
"Prefix cache hits, in terms of number of cached tokens.",
prefix_cache_hits.clone(),
);
let external_prefix_cache_queries = Family::default();
registry.register(
"vllm:external_prefix_cache_queries",
"External prefix cache queries from KV connector cross-instance cache sharing, in terms of number of queried tokens.",
external_prefix_cache_queries.clone(),
);
let external_prefix_cache_hits = Family::default();
registry.register(
"vllm:external_prefix_cache_hits",
"External prefix cache hits from KV connector cross-instance cache sharing, in terms of number of cached tokens.",
external_prefix_cache_hits.clone(),
);
// Speculative decoding counters.
let spec_decode_num_drafts = Family::default();
registry.register(
"vllm:spec_decode_num_drafts",
"Number of spec decoding drafts.",
spec_decode_num_drafts.clone(),
);
let spec_decode_num_draft_tokens = Family::default();
registry.register(
"vllm:spec_decode_num_draft_tokens",
"Number of draft tokens.",
spec_decode_num_draft_tokens.clone(),
);
let spec_decode_num_accepted_tokens = Family::default();
registry.register(
"vllm:spec_decode_num_accepted_tokens",
"Number of accepted tokens.",
spec_decode_num_accepted_tokens.clone(),
);
let spec_decode_num_accepted_tokens_per_pos = Family::default();
registry.register(
"vllm:spec_decode_num_accepted_tokens_per_pos",
"Accepted tokens per draft position.",
spec_decode_num_accepted_tokens_per_pos.clone(),
);
// Per-engine performance / MFU counters.
let estimated_flops_per_gpu = Family::default();
registry.register(
"vllm:estimated_flops_per_gpu",
"Estimated number of floating point operations per GPU (for Model Flops Utilization calculations).",
estimated_flops_per_gpu.clone(),
);
let estimated_read_bytes_per_gpu = Family::default();
registry.register(
"vllm:estimated_read_bytes_per_gpu",
"Estimated number of bytes read from memory per GPU (for Model Flops Utilization calculations).",
estimated_read_bytes_per_gpu.clone(),
);
let estimated_write_bytes_per_gpu = Family::default();
registry.register(
"vllm:estimated_write_bytes_per_gpu",
"Estimated number of bytes written to memory per GPU (for Model Flops Utilization calculations).",
estimated_write_bytes_per_gpu.clone(),
);
// Sampled KV-cache residency histograms.
let kv_block_lifetime_seconds =
Family::new_with_constructor(kv_block_lifetime_histogram as fn() -> Histogram);
registry.register(
"vllm:kv_block_lifetime_seconds",
"Histogram of KV cache block lifetime from allocation to eviction. Sampled metrics (controlled by --kv-cache-metrics-sample).",
kv_block_lifetime_seconds.clone(),
);
let kv_block_idle_before_evict_seconds =
Family::new_with_constructor(kv_block_idle_before_evict_histogram as fn() -> Histogram);
registry.register(
"vllm:kv_block_idle_before_evict_seconds",
"Histogram of idle time before KV cache block eviction. Sampled metrics (controlled by --kv-cache-metrics-sample).",
kv_block_idle_before_evict_seconds.clone(),
);
let kv_block_reuse_gap_seconds =
Family::new_with_constructor(kv_block_reuse_gap_histogram as fn() -> Histogram);
registry.register(
"vllm:kv_block_reuse_gap_seconds",
"Histogram of time gaps between consecutive KV cache block accesses. Only the most recent accesses are recorded (ring buffer). Sampled metrics (controlled by --kv-cache-metrics-sample).",
kv_block_reuse_gap_seconds.clone(),
);
Self {
scheduler_running,
scheduler_waiting,
scheduler_waiting_by_reason,
kv_cache_usage,
prefix_cache_queries,
prefix_cache_hits,
external_prefix_cache_queries,
external_prefix_cache_hits,
spec_decode_num_drafts,
spec_decode_num_draft_tokens,
spec_decode_num_accepted_tokens,
spec_decode_num_accepted_tokens_per_pos,
estimated_flops_per_gpu,
estimated_read_bytes_per_gpu,
estimated_write_bytes_per_gpu,
kv_block_lifetime_seconds,
kv_block_idle_before_evict_seconds,
kv_block_reuse_gap_seconds,
}
}
}
#[cfg(test)]
mod tests {
use crate::{EngineLabels, Metrics};
#[test]
fn perf_counters_render_with_a_single_total_suffix() {
let metrics = Metrics::new();
let labels = EngineLabels {
model_name: "model".to_string(),
engine: 0,
};
metrics.scheduler.estimated_flops_per_gpu.get_or_create(&labels).inc();
metrics.scheduler.estimated_read_bytes_per_gpu.get_or_create(&labels).inc();
metrics.scheduler.estimated_write_bytes_per_gpu.get_or_create(&labels).inc();
let rendered = metrics.render().unwrap();
assert!(
rendered.contains(
"vllm:estimated_flops_per_gpu_total{model_name=\"model\",engine=\"0\"} 1"
)
);
assert!(rendered.contains(
"vllm:estimated_read_bytes_per_gpu_total{model_name=\"model\",engine=\"0\"} 1"
));
assert!(rendered.contains(
"vllm:estimated_write_bytes_per_gpu_total{model_name=\"model\",engine=\"0\"} 1"
));
assert!(!rendered.contains("vllm:estimated_flops_per_gpu_total_total"));
assert!(!rendered.contains("vllm:estimated_read_bytes_per_gpu_total_total"));
assert!(!rendered.contains("vllm:estimated_write_bytes_per_gpu_total_total"));
}
}