[Rust Frontend] Cache metric handles for scheduler & request stats (#47444)
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
8b79971bb9
commit
8f4c69b222
@@ -512,6 +512,7 @@ impl EngineCoreClient {
|
||||
|
||||
Ok(EngineCoreOutputStream::new(
|
||||
request_id,
|
||||
engine_id.engine_index().unwrap_or(0),
|
||||
self.abort_tx.clone(),
|
||||
rx,
|
||||
))
|
||||
|
||||
@@ -15,7 +15,7 @@ use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, Uti
|
||||
use crate::client::stream::EngineCoreStreamOutput;
|
||||
use crate::client::{AbortCause, AbortRequest};
|
||||
use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
|
||||
use crate::metrics::{LoraInfoExporter, record_scheduler_stats};
|
||||
use crate::metrics::{LoraInfoExporter, SchedulerStatsRecorder};
|
||||
use crate::protocol::encode_msgpack;
|
||||
use crate::protocol::output::{EngineCoreOutput, EngineCoreOutputs};
|
||||
use crate::protocol::request::EngineCoreRequestType;
|
||||
@@ -29,6 +29,7 @@ pub(crate) struct ClientInner {
|
||||
/// The runtime handle used for sending messages to the engine.
|
||||
handle: Handle,
|
||||
model_name: String,
|
||||
scheduler_stats_recorder: SchedulerStatsRecorder,
|
||||
request_reg: Mutex<RequestRegistry>,
|
||||
utility_reg: Mutex<UtilityRegistry>,
|
||||
health_error: ArcSwapOption<Error>,
|
||||
@@ -43,10 +44,13 @@ impl ClientInner {
|
||||
model_name: String,
|
||||
engines: &[ConnectedEngine],
|
||||
) -> Self {
|
||||
let scheduler_stats_recorder =
|
||||
SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines);
|
||||
Self {
|
||||
input_send,
|
||||
handle,
|
||||
model_name,
|
||||
scheduler_stats_recorder,
|
||||
request_reg: Mutex::new(RequestRegistry::new(engines)),
|
||||
utility_reg: Mutex::new(UtilityRegistry::default()),
|
||||
health_error: ArcSwapOption::empty(),
|
||||
@@ -389,12 +393,7 @@ pub(crate) async fn run_output_dispatcher_loop(
|
||||
"dropping scheduler stats for unknown engine"
|
||||
);
|
||||
}
|
||||
record_scheduler_stats(
|
||||
&METRICS.scheduler,
|
||||
inner.model_name(),
|
||||
batch.engine_index,
|
||||
scheduler_stats,
|
||||
);
|
||||
inner.scheduler_stats_recorder.record(batch.engine_index, scheduler_stats);
|
||||
}
|
||||
|
||||
// The engine's scheduler stats never carry adapter names;
|
||||
|
||||
@@ -45,6 +45,7 @@ impl Deref for EngineCoreStreamOutput {
|
||||
/// `finish_reason` is non-`None`.
|
||||
pub struct EngineCoreOutputStream {
|
||||
request_id: String,
|
||||
engine_index: u32,
|
||||
abort_tx: mpsc::UnboundedSender<AbortRequest>,
|
||||
state: State,
|
||||
rx: OutputReceiver,
|
||||
@@ -53,11 +54,13 @@ pub struct EngineCoreOutputStream {
|
||||
impl EngineCoreOutputStream {
|
||||
pub(crate) fn new(
|
||||
request_id: String,
|
||||
engine_index: u32,
|
||||
abort_tx: mpsc::UnboundedSender<AbortRequest>,
|
||||
rx: OutputReceiver,
|
||||
) -> Self {
|
||||
Self {
|
||||
request_id,
|
||||
engine_index,
|
||||
abort_tx,
|
||||
state: State::Running,
|
||||
rx,
|
||||
@@ -68,6 +71,11 @@ impl EngineCoreOutputStream {
|
||||
pub fn request_id(&self) -> &str {
|
||||
&self.request_id
|
||||
}
|
||||
|
||||
/// Return the index of the engine that owns this request.
|
||||
pub fn engine_index(&self) -> u32 {
|
||||
self.engine_index
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for EngineCoreOutputStream {
|
||||
|
||||
@@ -1,90 +1,190 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::BTreeSet;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use vllm_metrics::{
|
||||
EngineLabels, EnginePositionLabels, LoraAdapterNames, LoraInfoLabels, SchedulerMetrics,
|
||||
EngineLabels, EnginePositionLabels, F64Gauge, Family, HistogramMetric, LoraAdapterNames,
|
||||
LoraInfoLabels, SchedulerLogStatsAccumulator, SchedulerMetrics, U64Counter, U64Gauge,
|
||||
WaitingReasonLabels,
|
||||
};
|
||||
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::transport::ConnectedEngine;
|
||||
|
||||
const WAITING_REASON_CAPACITY: &str = "capacity";
|
||||
const WAITING_REASON_DEFERRED: &str = "deferred";
|
||||
|
||||
/// Record the scheduler-stats-backed metrics for one engine at one point in
|
||||
/// time.
|
||||
pub(crate) fn record_scheduler_stats(
|
||||
metrics: &SchedulerMetrics,
|
||||
model_name: impl Into<String>,
|
||||
engine: u32,
|
||||
stats: &SchedulerStats,
|
||||
) {
|
||||
let model_name = model_name.into();
|
||||
let labels = EngineLabels {
|
||||
model_name: model_name.clone(),
|
||||
engine,
|
||||
};
|
||||
/// Cached scheduler-stats metric handles for all engines connected to one
|
||||
/// frontend client.
|
||||
pub(crate) struct SchedulerStatsRecorder {
|
||||
engines: BTreeMap<u32, SchedulerStatsHandles>,
|
||||
}
|
||||
|
||||
/// Per-engine cached metric handles used while recording `SchedulerStats`.
|
||||
struct SchedulerStatsHandles {
|
||||
// Base labels reused for dynamic child labels.
|
||||
labels: EngineLabels,
|
||||
|
||||
// Scheduler state gauges.
|
||||
metrics.scheduler_running.get_or_create(&labels).set(stats.num_running_reqs);
|
||||
metrics
|
||||
.scheduler_waiting
|
||||
.get_or_create(&labels)
|
||||
.set(stats.num_waiting_reqs + stats.num_skipped_waiting_reqs);
|
||||
metrics
|
||||
.scheduler_waiting_by_reason
|
||||
.get_or_create(&WaitingReasonLabels {
|
||||
model_name: model_name.clone(),
|
||||
engine,
|
||||
reason: WAITING_REASON_CAPACITY,
|
||||
})
|
||||
.set(stats.num_waiting_reqs);
|
||||
metrics
|
||||
.scheduler_waiting_by_reason
|
||||
.get_or_create(&WaitingReasonLabels {
|
||||
model_name: model_name.clone(),
|
||||
engine,
|
||||
reason: WAITING_REASON_DEFERRED,
|
||||
})
|
||||
.set(stats.num_skipped_waiting_reqs);
|
||||
metrics.kv_cache_usage.get_or_create(&labels).set(stats.kv_cache_usage);
|
||||
scheduler_running: U64Gauge,
|
||||
scheduler_waiting: U64Gauge,
|
||||
scheduler_waiting_capacity: U64Gauge,
|
||||
scheduler_waiting_deferred: U64Gauge,
|
||||
kv_cache_usage: F64Gauge,
|
||||
|
||||
// Prefix-cache counters, including the connector-backed external cache path.
|
||||
metrics
|
||||
.prefix_cache_queries
|
||||
.get_or_create(&labels)
|
||||
.inc_by(stats.prefix_cache_stats.base.queries);
|
||||
metrics
|
||||
.prefix_cache_hits
|
||||
.get_or_create(&labels)
|
||||
.inc_by(stats.prefix_cache_stats.base.hits);
|
||||
prefix_cache_queries: U64Counter,
|
||||
prefix_cache_hits: U64Counter,
|
||||
external_prefix_cache_queries: U64Counter,
|
||||
external_prefix_cache_hits: U64Counter,
|
||||
|
||||
// Speculative decoding counters.
|
||||
spec_decode_num_drafts: U64Counter,
|
||||
spec_decode_num_draft_tokens: U64Counter,
|
||||
spec_decode_num_accepted_tokens: U64Counter,
|
||||
spec_decode_num_accepted_tokens_per_pos: Family<EnginePositionLabels, U64Counter>,
|
||||
|
||||
// Per-engine performance / MFU counters.
|
||||
estimated_flops_per_gpu: U64Counter,
|
||||
estimated_read_bytes_per_gpu: U64Counter,
|
||||
estimated_write_bytes_per_gpu: U64Counter,
|
||||
|
||||
// Sampled KV-cache residency histograms.
|
||||
kv_block_lifetime_seconds: HistogramMetric,
|
||||
kv_block_idle_before_evict_seconds: HistogramMetric,
|
||||
kv_block_reuse_gap_seconds: HistogramMetric,
|
||||
|
||||
// Non-Prometheus interval accumulator for periodic text-log helpers.
|
||||
log_stats: SchedulerLogStatsAccumulator,
|
||||
}
|
||||
|
||||
impl SchedulerStatsRecorder {
|
||||
/// Resolve the fixed-label metric handles for the connected engines.
|
||||
pub(crate) fn new(
|
||||
metrics: &SchedulerMetrics,
|
||||
model_name: &str,
|
||||
engines: &[ConnectedEngine],
|
||||
) -> Self {
|
||||
let engines = engines
|
||||
.iter()
|
||||
.filter_map(|engine| {
|
||||
let engine = engine.engine_id.engine_index()?;
|
||||
Some((
|
||||
engine,
|
||||
resolve_scheduler_stats_handles(metrics, model_name, engine),
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
|
||||
Self { engines }
|
||||
}
|
||||
|
||||
/// Record one scheduler-stats payload for the given engine index.
|
||||
pub(crate) fn record(&self, engine_index: u32, stats: &SchedulerStats) {
|
||||
if let Some(handles) = self.engines.get(&engine_index) {
|
||||
record_scheduler_stats_with_handles(handles, stats);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve all fixed-label scheduler metrics for one engine.
|
||||
fn resolve_scheduler_stats_handles(
|
||||
metrics: &SchedulerMetrics,
|
||||
model_name: &str,
|
||||
engine: u32,
|
||||
) -> SchedulerStatsHandles {
|
||||
let labels = EngineLabels {
|
||||
model_name: model_name.to_string(),
|
||||
engine,
|
||||
};
|
||||
let capacity = WaitingReasonLabels {
|
||||
model_name: model_name.to_string(),
|
||||
engine,
|
||||
reason: WAITING_REASON_CAPACITY,
|
||||
};
|
||||
let deferred = WaitingReasonLabels {
|
||||
model_name: model_name.to_string(),
|
||||
engine,
|
||||
reason: WAITING_REASON_DEFERRED,
|
||||
};
|
||||
|
||||
SchedulerStatsHandles {
|
||||
scheduler_running: metrics.scheduler_running.get_or_create_owned(&labels),
|
||||
scheduler_waiting: metrics.scheduler_waiting.get_or_create_owned(&labels),
|
||||
scheduler_waiting_capacity: metrics
|
||||
.scheduler_waiting_by_reason
|
||||
.get_or_create_owned(&capacity),
|
||||
scheduler_waiting_deferred: metrics
|
||||
.scheduler_waiting_by_reason
|
||||
.get_or_create_owned(&deferred),
|
||||
kv_cache_usage: metrics.kv_cache_usage.get_or_create_owned(&labels),
|
||||
prefix_cache_queries: metrics.prefix_cache_queries.get_or_create_owned(&labels),
|
||||
prefix_cache_hits: metrics.prefix_cache_hits.get_or_create_owned(&labels),
|
||||
external_prefix_cache_queries: metrics
|
||||
.external_prefix_cache_queries
|
||||
.get_or_create_owned(&labels),
|
||||
external_prefix_cache_hits: metrics.external_prefix_cache_hits.get_or_create_owned(&labels),
|
||||
spec_decode_num_drafts: metrics.spec_decode_num_drafts.get_or_create_owned(&labels),
|
||||
spec_decode_num_draft_tokens: metrics
|
||||
.spec_decode_num_draft_tokens
|
||||
.get_or_create_owned(&labels),
|
||||
spec_decode_num_accepted_tokens: metrics
|
||||
.spec_decode_num_accepted_tokens
|
||||
.get_or_create_owned(&labels),
|
||||
spec_decode_num_accepted_tokens_per_pos: metrics
|
||||
.spec_decode_num_accepted_tokens_per_pos
|
||||
.clone(),
|
||||
log_stats: metrics.log_stats.get_or_create_owned(&labels),
|
||||
estimated_flops_per_gpu: metrics.estimated_flops_per_gpu.get_or_create_owned(&labels),
|
||||
estimated_read_bytes_per_gpu: metrics
|
||||
.estimated_read_bytes_per_gpu
|
||||
.get_or_create_owned(&labels),
|
||||
estimated_write_bytes_per_gpu: metrics
|
||||
.estimated_write_bytes_per_gpu
|
||||
.get_or_create_owned(&labels),
|
||||
kv_block_lifetime_seconds: metrics.kv_block_lifetime_seconds.get_or_create_owned(&labels),
|
||||
kv_block_idle_before_evict_seconds: metrics
|
||||
.kv_block_idle_before_evict_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
kv_block_reuse_gap_seconds: metrics.kv_block_reuse_gap_seconds.get_or_create_owned(&labels),
|
||||
labels,
|
||||
}
|
||||
}
|
||||
|
||||
/// Record scheduler-stats values through pre-resolved metric handles.
|
||||
fn record_scheduler_stats_with_handles(handles: &SchedulerStatsHandles, stats: &SchedulerStats) {
|
||||
// Scheduler state gauges.
|
||||
handles.scheduler_running.set(stats.num_running_reqs);
|
||||
handles
|
||||
.scheduler_waiting
|
||||
.set(stats.num_waiting_reqs + stats.num_skipped_waiting_reqs);
|
||||
handles.scheduler_waiting_capacity.set(stats.num_waiting_reqs);
|
||||
handles.scheduler_waiting_deferred.set(stats.num_skipped_waiting_reqs);
|
||||
handles.kv_cache_usage.set(stats.kv_cache_usage);
|
||||
|
||||
// Prefix-cache counters, including the connector-backed external cache path.
|
||||
handles.prefix_cache_queries.inc_by(stats.prefix_cache_stats.base.queries);
|
||||
handles.prefix_cache_hits.inc_by(stats.prefix_cache_stats.base.hits);
|
||||
|
||||
if let Some(connector_prefix_cache_stats) = &stats.connector_prefix_cache_stats {
|
||||
metrics
|
||||
handles
|
||||
.external_prefix_cache_queries
|
||||
.get_or_create(&labels)
|
||||
.inc_by(connector_prefix_cache_stats.base.queries);
|
||||
metrics
|
||||
handles
|
||||
.external_prefix_cache_hits
|
||||
.get_or_create(&labels)
|
||||
.inc_by(connector_prefix_cache_stats.base.hits);
|
||||
}
|
||||
|
||||
// Speculative decoding counters.
|
||||
if let Some(spec_decoding_stats) = &stats.spec_decoding_stats {
|
||||
metrics
|
||||
.spec_decode_num_drafts
|
||||
.get_or_create(&labels)
|
||||
.inc_by(spec_decoding_stats.num_drafts);
|
||||
metrics
|
||||
handles.spec_decode_num_drafts.inc_by(spec_decoding_stats.num_drafts);
|
||||
handles
|
||||
.spec_decode_num_draft_tokens
|
||||
.get_or_create(&labels)
|
||||
.inc_by(spec_decoding_stats.num_draft_tokens);
|
||||
metrics
|
||||
handles
|
||||
.spec_decode_num_accepted_tokens
|
||||
.get_or_create(&labels)
|
||||
.inc_by(spec_decoding_stats.num_accepted_tokens);
|
||||
metrics.log_stats.get_or_create(&labels).observe_spec_decode(
|
||||
handles.log_stats.observe_spec_decode(
|
||||
spec_decoding_stats.num_drafts,
|
||||
&spec_decoding_stats.num_accepted_tokens_per_pos,
|
||||
);
|
||||
@@ -92,11 +192,11 @@ pub(crate) fn record_scheduler_stats(
|
||||
for (position, accepted_tokens) in
|
||||
spec_decoding_stats.num_accepted_tokens_per_pos.iter().copied().enumerate()
|
||||
{
|
||||
metrics
|
||||
handles
|
||||
.spec_decode_num_accepted_tokens_per_pos
|
||||
.get_or_create(&EnginePositionLabels {
|
||||
model_name: model_name.clone(),
|
||||
engine,
|
||||
model_name: handles.labels.model_name.clone(),
|
||||
engine: handles.labels.engine,
|
||||
position: position as u32,
|
||||
})
|
||||
.inc_by(accepted_tokens);
|
||||
@@ -109,22 +209,13 @@ pub(crate) fn record_scheduler_stats(
|
||||
|| perf_stats.num_read_bytes_per_gpu != 0
|
||||
|| perf_stats.num_write_bytes_per_gpu != 0)
|
||||
{
|
||||
metrics
|
||||
.estimated_flops_per_gpu
|
||||
.get_or_create(&labels)
|
||||
.inc_by(perf_stats.num_flops_per_gpu);
|
||||
metrics
|
||||
.estimated_read_bytes_per_gpu
|
||||
.get_or_create(&labels)
|
||||
.inc_by(perf_stats.num_read_bytes_per_gpu);
|
||||
metrics
|
||||
.estimated_write_bytes_per_gpu
|
||||
.get_or_create(&labels)
|
||||
.inc_by(perf_stats.num_write_bytes_per_gpu);
|
||||
handles.estimated_flops_per_gpu.inc_by(perf_stats.num_flops_per_gpu);
|
||||
handles.estimated_read_bytes_per_gpu.inc_by(perf_stats.num_read_bytes_per_gpu);
|
||||
handles.estimated_write_bytes_per_gpu.inc_by(perf_stats.num_write_bytes_per_gpu);
|
||||
}
|
||||
|
||||
if let Some(cudagraph_stats) = &stats.cudagraph_stats {
|
||||
metrics.log_stats.get_or_create(&labels).observe_cudagraph(
|
||||
handles.log_stats.observe_cudagraph(
|
||||
cudagraph_stats.num_unpadded_tokens,
|
||||
cudagraph_stats.num_padded_tokens,
|
||||
cudagraph_stats.num_paddings,
|
||||
@@ -134,16 +225,11 @@ pub(crate) fn record_scheduler_stats(
|
||||
|
||||
// Sampled KV-cache residency histograms.
|
||||
if !stats.kv_cache_eviction_events.is_empty() {
|
||||
let kv_block_lifetime_seconds = metrics.kv_block_lifetime_seconds.get_or_create(&labels);
|
||||
let kv_block_idle_before_evict_seconds =
|
||||
metrics.kv_block_idle_before_evict_seconds.get_or_create(&labels);
|
||||
let kv_block_reuse_gap_seconds = metrics.kv_block_reuse_gap_seconds.get_or_create(&labels);
|
||||
|
||||
for event in &stats.kv_cache_eviction_events {
|
||||
kv_block_lifetime_seconds.observe(event.lifetime_seconds);
|
||||
kv_block_idle_before_evict_seconds.observe(event.idle_seconds);
|
||||
handles.kv_block_lifetime_seconds.observe(event.lifetime_seconds);
|
||||
handles.kv_block_idle_before_evict_seconds.observe(event.idle_seconds);
|
||||
for reuse_gap_seconds in &event.reuse_gaps_seconds {
|
||||
kv_block_reuse_gap_seconds.observe(*reuse_gap_seconds);
|
||||
handles.kv_block_reuse_gap_seconds.observe(*reuse_gap_seconds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+11
-4
@@ -88,14 +88,21 @@ impl Llm {
|
||||
// Record internal engine-core request ID in the current tracing span.
|
||||
Span::current().record("engine_request_id", &internal_request_id);
|
||||
|
||||
let arrival_time = prepared.engine_request.arrival_time;
|
||||
let max_tokens_param =
|
||||
(prepared.engine_request.sampling_params.as_ref()).map(|p| p.max_tokens);
|
||||
let prompt_len = prepared.prompt_token_ids().len() as u32;
|
||||
|
||||
let stream = self.client.call(prepared.engine_request).await?;
|
||||
|
||||
let request_metrics = RequestMetricsTracker::new(
|
||||
self.client.model_name().to_string(),
|
||||
prepared.engine_request.arrival_time,
|
||||
prepared.prompt_token_ids().len() as u32,
|
||||
(prepared.engine_request.sampling_params.as_ref()).map(|p| p.max_tokens),
|
||||
stream.engine_index(),
|
||||
arrival_time,
|
||||
prompt_len,
|
||||
max_tokens_param,
|
||||
1,
|
||||
);
|
||||
let stream = self.client.call(prepared.engine_request).await?;
|
||||
let guard = self.inflight.track(external_request_id, internal_request_id);
|
||||
|
||||
Ok(GenerateOutputStream::new(
|
||||
|
||||
@@ -248,12 +248,7 @@ impl Stream for GenerateOutputStream {
|
||||
};
|
||||
|
||||
let received_at = current_unix_timestamp_secs();
|
||||
self.request_metrics.observe_output(
|
||||
raw.engine_index,
|
||||
raw.timestamp,
|
||||
received_at,
|
||||
&raw.output,
|
||||
);
|
||||
self.request_metrics.observe_output(raw.timestamp, received_at, &raw.output);
|
||||
|
||||
let raw = raw.output;
|
||||
|
||||
|
||||
+150
-142
@@ -5,15 +5,12 @@ use vllm_engine_core_client::protocol::output::{
|
||||
};
|
||||
use vllm_engine_core_client::protocol::stats::PrefillStats;
|
||||
use vllm_metrics::{
|
||||
EngineLabels, FinishedReasonLabels, METRICS, PromptTokenSourceLabels, RequestMetrics,
|
||||
EngineLabels, Family, FinishedReasonLabels, HistogramMetric, METRICS, PromptTokenSourceLabels,
|
||||
U64Counter,
|
||||
};
|
||||
|
||||
use crate::FinishReason;
|
||||
|
||||
fn metrics() -> &'static RequestMetrics {
|
||||
&METRICS.request
|
||||
}
|
||||
|
||||
const PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE: &str = "local_compute";
|
||||
const PROMPT_TOKEN_SOURCE_LOCAL_CACHE_HIT: &str = "local_cache_hit";
|
||||
const PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER: &str = "external_kv_transfer";
|
||||
@@ -29,9 +26,11 @@ const PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER: &str = "external_kv_transfer";
|
||||
///
|
||||
/// Original Python update flow:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/engine/output_processor.py#L600-L677>
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct RequestMetricsTracker {
|
||||
model_name: String,
|
||||
/// Cached request metric handles for this request's model and engine index.
|
||||
handles: RequestMetricHandles,
|
||||
|
||||
arrival_time: f64,
|
||||
prompt_len: u32,
|
||||
max_tokens_param: Option<u32>,
|
||||
@@ -44,7 +43,38 @@ pub(crate) struct RequestMetricsTracker {
|
||||
first_token_latency: f64,
|
||||
num_generation_tokens: u32,
|
||||
latest_num_cached_tokens: u32,
|
||||
last_seen_engine_index: u32,
|
||||
}
|
||||
|
||||
/// Cached request metric handles for one model and engine index.
|
||||
#[derive(Clone)]
|
||||
struct RequestMetricHandles {
|
||||
labels: EngineLabels,
|
||||
|
||||
// Request-derived counters.
|
||||
num_preemptions: U64Counter,
|
||||
prompt_tokens: U64Counter,
|
||||
prompt_tokens_local_compute: U64Counter,
|
||||
prompt_tokens_local_cache_hit: U64Counter,
|
||||
prompt_tokens_external_kv_transfer: U64Counter,
|
||||
prompt_tokens_cached: U64Counter,
|
||||
generation_tokens: U64Counter,
|
||||
|
||||
// Request lifecycle counters and histograms.
|
||||
request_success: Family<FinishedReasonLabels, U64Counter>,
|
||||
request_prompt_tokens: HistogramMetric,
|
||||
request_generation_tokens: HistogramMetric,
|
||||
request_max_num_generation_tokens: HistogramMetric,
|
||||
request_params_max_tokens: HistogramMetric,
|
||||
request_params_n: HistogramMetric,
|
||||
request_prefill_kv_computed_tokens: HistogramMetric,
|
||||
time_to_first_token_seconds: HistogramMetric,
|
||||
inter_token_latency_seconds: HistogramMetric,
|
||||
e2e_request_latency_seconds: HistogramMetric,
|
||||
request_queue_time_seconds: HistogramMetric,
|
||||
request_prefill_time_seconds: HistogramMetric,
|
||||
request_decode_time_seconds: HistogramMetric,
|
||||
request_inference_time_seconds: HistogramMetric,
|
||||
request_time_per_output_token_seconds: HistogramMetric,
|
||||
}
|
||||
|
||||
impl RequestMetricsTracker {
|
||||
@@ -52,13 +82,14 @@ impl RequestMetricsTracker {
|
||||
/// context.
|
||||
pub(crate) fn new(
|
||||
model_name: String,
|
||||
engine_index: u32,
|
||||
arrival_time: f64,
|
||||
prompt_len: u32,
|
||||
max_tokens_param: Option<u32>,
|
||||
n_param: u32,
|
||||
) -> Self {
|
||||
Self {
|
||||
model_name,
|
||||
handles: resolve_request_metric_handles(&model_name, engine_index),
|
||||
arrival_time,
|
||||
prompt_len,
|
||||
max_tokens_param,
|
||||
@@ -71,7 +102,6 @@ impl RequestMetricsTracker {
|
||||
first_token_latency: 0.0,
|
||||
num_generation_tokens: 0,
|
||||
latest_num_cached_tokens: 0,
|
||||
last_seen_engine_index: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,23 +111,18 @@ impl RequestMetricsTracker {
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L331-L384>
|
||||
pub(crate) fn observe_output(
|
||||
&mut self,
|
||||
engine_index: u32,
|
||||
batch_timestamp: f64,
|
||||
received_at: f64,
|
||||
output: &EngineCoreOutput,
|
||||
) {
|
||||
self.last_seen_engine_index = engine_index;
|
||||
if let Some(prefill_stats) = &output.prefill_stats {
|
||||
self.latest_num_cached_tokens = prefill_stats.num_cached_tokens;
|
||||
}
|
||||
self.num_generation_tokens += output.new_token_ids.len() as u32;
|
||||
metrics()
|
||||
.generation_tokens
|
||||
.get_or_create(&engine_labels(&self.model_name, engine_index))
|
||||
.inc_by(output.new_token_ids.len() as u64);
|
||||
self.handles.generation_tokens.inc_by(output.new_token_ids.len() as u64);
|
||||
|
||||
if let Some(events) = &output.events {
|
||||
self.observe_events(engine_index, events);
|
||||
self.observe_events(events);
|
||||
}
|
||||
|
||||
// Only outputs that actually carry tokens drive token-timing metrics.
|
||||
@@ -107,22 +132,16 @@ impl RequestMetricsTracker {
|
||||
if !output.new_token_ids.is_empty() {
|
||||
if self.is_prefilling {
|
||||
if let Some(prefill_stats) = &output.prefill_stats {
|
||||
record_prompt_tokens(&self.model_name, engine_index, prefill_stats);
|
||||
self.record_prompt_tokens(prefill_stats);
|
||||
}
|
||||
self.first_token_latency = received_at - self.arrival_time;
|
||||
observe_time_to_first_token_seconds(
|
||||
&self.model_name,
|
||||
engine_index,
|
||||
self.first_token_latency,
|
||||
);
|
||||
self.handles.time_to_first_token_seconds.observe(self.first_token_latency);
|
||||
self.first_token_ts = batch_timestamp;
|
||||
self.is_prefilling = false;
|
||||
} else if self.last_token_ts > 0.0 {
|
||||
observe_inter_token_latency_seconds(
|
||||
&self.model_name,
|
||||
engine_index,
|
||||
batch_timestamp - self.last_token_ts,
|
||||
);
|
||||
self.handles
|
||||
.inter_token_latency_seconds
|
||||
.observe(batch_timestamp - self.last_token_ts);
|
||||
}
|
||||
|
||||
self.last_token_ts = batch_timestamp;
|
||||
@@ -135,7 +154,6 @@ impl RequestMetricsTracker {
|
||||
/// Original Python finished-request stats:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L222-L237>
|
||||
pub(crate) fn record_finished(&self, received_at: f64, finish_reason: FinishReason) {
|
||||
let labels = engine_labels(&self.model_name, self.last_seen_engine_index);
|
||||
let prefill_kv_computed_tokens =
|
||||
self.prompt_len.saturating_sub(self.latest_num_cached_tokens);
|
||||
let e2e_latency_seconds = received_at - self.arrival_time;
|
||||
@@ -150,57 +168,47 @@ impl RequestMetricsTracker {
|
||||
0.0
|
||||
};
|
||||
|
||||
record_request_success(&self.model_name, self.last_seen_engine_index, finish_reason);
|
||||
metrics()
|
||||
.request_prompt_tokens
|
||||
.get_or_create(&labels)
|
||||
.observe(self.prompt_len as f64);
|
||||
metrics()
|
||||
self.record_request_success(finish_reason);
|
||||
|
||||
self.handles.request_prompt_tokens.observe(self.prompt_len as f64);
|
||||
self.handles
|
||||
.request_generation_tokens
|
||||
.get_or_create(&labels)
|
||||
.observe(self.num_generation_tokens as f64);
|
||||
metrics()
|
||||
self.handles
|
||||
.request_max_num_generation_tokens
|
||||
.get_or_create(&labels)
|
||||
.observe(self.num_generation_tokens as f64);
|
||||
if let Some(max_tokens_param) = self.max_tokens_param {
|
||||
metrics()
|
||||
.request_params_max_tokens
|
||||
.get_or_create(&labels)
|
||||
.observe(max_tokens_param as f64);
|
||||
self.handles.request_params_max_tokens.observe(max_tokens_param as f64);
|
||||
}
|
||||
metrics().request_params_n.get_or_create(&labels).observe(self.n_param as f64);
|
||||
metrics()
|
||||
self.handles.request_params_n.observe(self.n_param as f64);
|
||||
self.handles
|
||||
.request_prefill_kv_computed_tokens
|
||||
.get_or_create(&labels)
|
||||
.observe(prefill_kv_computed_tokens as f64);
|
||||
metrics()
|
||||
.e2e_request_latency_seconds
|
||||
.get_or_create(&labels)
|
||||
.observe(e2e_latency_seconds);
|
||||
metrics()
|
||||
.request_queue_time_seconds
|
||||
.get_or_create(&labels)
|
||||
.observe(queue_time_seconds);
|
||||
metrics()
|
||||
.request_prefill_time_seconds
|
||||
.get_or_create(&labels)
|
||||
.observe(prefill_time_seconds);
|
||||
metrics()
|
||||
.request_decode_time_seconds
|
||||
.get_or_create(&labels)
|
||||
.observe(decode_time_seconds);
|
||||
metrics()
|
||||
.request_inference_time_seconds
|
||||
.get_or_create(&labels)
|
||||
.observe(inference_time_seconds);
|
||||
metrics()
|
||||
self.handles.e2e_request_latency_seconds.observe(e2e_latency_seconds);
|
||||
self.handles.request_queue_time_seconds.observe(queue_time_seconds);
|
||||
self.handles.request_prefill_time_seconds.observe(prefill_time_seconds);
|
||||
self.handles.request_decode_time_seconds.observe(decode_time_seconds);
|
||||
self.handles.request_inference_time_seconds.observe(inference_time_seconds);
|
||||
self.handles
|
||||
.request_time_per_output_token_seconds
|
||||
.get_or_create(&labels)
|
||||
.observe(time_per_output_token_seconds);
|
||||
}
|
||||
|
||||
fn observe_events(&mut self, engine_index: u32, events: &[EngineCoreEvent]) {
|
||||
/// Record prompt token counters through cached metric handles.
|
||||
fn record_prompt_tokens(&self, prefill_stats: &PrefillStats) {
|
||||
let computed = prefill_stats.num_computed_tokens as u64;
|
||||
let local_cache_hit = prefill_stats.num_local_cached_tokens as u64;
|
||||
let external_kv_transfer = prefill_stats.num_external_cached_tokens as u64;
|
||||
|
||||
self.handles.prompt_tokens.inc_by(prefill_stats.num_prompt_tokens as u64);
|
||||
self.handles.prompt_tokens_local_compute.inc_by(computed);
|
||||
self.handles.prompt_tokens_local_cache_hit.inc_by(local_cache_hit);
|
||||
self.handles.prompt_tokens_external_kv_transfer.inc_by(external_kv_transfer);
|
||||
self.handles.prompt_tokens_cached.inc_by(prefill_stats.num_cached_tokens as u64);
|
||||
}
|
||||
|
||||
/// Record request event counters through cached metric handles.
|
||||
fn observe_events(&mut self, events: &[EngineCoreEvent]) {
|
||||
for event in events {
|
||||
match event.r#type {
|
||||
EngineCoreEventType::Queued => {
|
||||
@@ -212,46 +220,86 @@ impl RequestMetricsTracker {
|
||||
}
|
||||
}
|
||||
EngineCoreEventType::Preempted => {
|
||||
metrics()
|
||||
.num_preemptions
|
||||
.get_or_create(&engine_labels(&self.model_name, engine_index))
|
||||
.inc();
|
||||
self.handles.num_preemptions.inc();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn engine_labels(model_name: &str, engine: u32) -> EngineLabels {
|
||||
EngineLabels {
|
||||
model_name: model_name.to_string(),
|
||||
engine,
|
||||
/// Increment the request-success counter for the terminal finish reason.
|
||||
fn record_request_success(&self, finish_reason: FinishReason) {
|
||||
self.handles
|
||||
.request_success
|
||||
.get_or_create(&FinishedReasonLabels {
|
||||
model_name: self.handles.labels.model_name.clone(),
|
||||
engine: self.handles.labels.engine,
|
||||
finished_reason: finish_reason.as_str(),
|
||||
})
|
||||
.inc();
|
||||
}
|
||||
}
|
||||
|
||||
fn observe_time_to_first_token_seconds(model_name: &str, engine: u32, seconds: f64) {
|
||||
metrics()
|
||||
.time_to_first_token_seconds
|
||||
.get_or_create(&engine_labels(model_name, engine))
|
||||
.observe(seconds);
|
||||
}
|
||||
/// Resolve fixed request metric handles for one model and engine index.
|
||||
fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetricHandles {
|
||||
let metrics = &METRICS.request;
|
||||
let labels = EngineLabels {
|
||||
model_name: model_name.to_string(),
|
||||
engine,
|
||||
};
|
||||
|
||||
fn observe_inter_token_latency_seconds(model_name: &str, engine: u32, seconds: f64) {
|
||||
metrics()
|
||||
.inter_token_latency_seconds
|
||||
.get_or_create(&engine_labels(model_name, engine))
|
||||
.observe(seconds);
|
||||
}
|
||||
|
||||
fn record_request_success(model_name: &str, engine: u32, finish_reason: FinishReason) {
|
||||
metrics()
|
||||
.request_success
|
||||
.get_or_create(&FinishedReasonLabels {
|
||||
model_name: model_name.to_string(),
|
||||
engine,
|
||||
finished_reason: finish_reason.as_str(),
|
||||
})
|
||||
.inc();
|
||||
RequestMetricHandles {
|
||||
num_preemptions: metrics.num_preemptions.get_or_create_owned(&labels),
|
||||
prompt_tokens: metrics.prompt_tokens.get_or_create_owned(&labels),
|
||||
prompt_tokens_local_compute: metrics.prompt_tokens_by_source.get_or_create_owned(
|
||||
&prompt_token_source_labels(model_name, engine, PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE),
|
||||
),
|
||||
prompt_tokens_local_cache_hit: metrics.prompt_tokens_by_source.get_or_create_owned(
|
||||
&prompt_token_source_labels(model_name, engine, PROMPT_TOKEN_SOURCE_LOCAL_CACHE_HIT),
|
||||
),
|
||||
prompt_tokens_external_kv_transfer: metrics.prompt_tokens_by_source.get_or_create_owned(
|
||||
&prompt_token_source_labels(
|
||||
model_name,
|
||||
engine,
|
||||
PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER,
|
||||
),
|
||||
),
|
||||
prompt_tokens_cached: metrics.prompt_tokens_cached.get_or_create_owned(&labels),
|
||||
generation_tokens: metrics.generation_tokens.get_or_create_owned(&labels),
|
||||
request_success: metrics.request_success.clone(),
|
||||
request_prompt_tokens: metrics.request_prompt_tokens.get_or_create_owned(&labels),
|
||||
request_generation_tokens: metrics.request_generation_tokens.get_or_create_owned(&labels),
|
||||
request_max_num_generation_tokens: metrics
|
||||
.request_max_num_generation_tokens
|
||||
.get_or_create_owned(&labels),
|
||||
request_params_max_tokens: metrics.request_params_max_tokens.get_or_create_owned(&labels),
|
||||
request_params_n: metrics.request_params_n.get_or_create_owned(&labels),
|
||||
request_prefill_kv_computed_tokens: metrics
|
||||
.request_prefill_kv_computed_tokens
|
||||
.get_or_create_owned(&labels),
|
||||
time_to_first_token_seconds: metrics
|
||||
.time_to_first_token_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
inter_token_latency_seconds: metrics
|
||||
.inter_token_latency_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
e2e_request_latency_seconds: metrics
|
||||
.e2e_request_latency_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
request_queue_time_seconds: metrics.request_queue_time_seconds.get_or_create_owned(&labels),
|
||||
request_prefill_time_seconds: metrics
|
||||
.request_prefill_time_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
request_decode_time_seconds: metrics
|
||||
.request_decode_time_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
request_inference_time_seconds: metrics
|
||||
.request_inference_time_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
request_time_per_output_token_seconds: metrics
|
||||
.request_time_per_output_token_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
labels,
|
||||
}
|
||||
}
|
||||
|
||||
fn prompt_token_source_labels(
|
||||
@@ -266,45 +314,6 @@ fn prompt_token_source_labels(
|
||||
}
|
||||
}
|
||||
|
||||
fn record_prompt_tokens(model_name: &str, engine: u32, prefill_stats: &PrefillStats) {
|
||||
let computed = prefill_stats.num_computed_tokens as u64;
|
||||
let local_cache_hit = prefill_stats.num_local_cached_tokens as u64;
|
||||
let external_kv_transfer = prefill_stats.num_external_cached_tokens as u64;
|
||||
|
||||
metrics()
|
||||
.prompt_tokens
|
||||
.get_or_create(&engine_labels(model_name, engine))
|
||||
.inc_by(prefill_stats.num_prompt_tokens as u64);
|
||||
metrics()
|
||||
.prompt_tokens_by_source
|
||||
.get_or_create(&prompt_token_source_labels(
|
||||
model_name,
|
||||
engine,
|
||||
PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE,
|
||||
))
|
||||
.inc_by(computed);
|
||||
metrics()
|
||||
.prompt_tokens_by_source
|
||||
.get_or_create(&prompt_token_source_labels(
|
||||
model_name,
|
||||
engine,
|
||||
PROMPT_TOKEN_SOURCE_LOCAL_CACHE_HIT,
|
||||
))
|
||||
.inc_by(local_cache_hit);
|
||||
metrics()
|
||||
.prompt_tokens_by_source
|
||||
.get_or_create(&prompt_token_source_labels(
|
||||
model_name,
|
||||
engine,
|
||||
PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER,
|
||||
))
|
||||
.inc_by(external_kv_transfer);
|
||||
metrics()
|
||||
.prompt_tokens_cached
|
||||
.get_or_create(&engine_labels(model_name, engine))
|
||||
.inc_by(prefill_stats.num_cached_tokens as u64);
|
||||
}
|
||||
|
||||
fn diff_or_zero(end: f64, start: f64) -> f64 {
|
||||
if end > 0.0 && start > 0.0 && end >= start {
|
||||
end - start
|
||||
@@ -337,10 +346,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn tracker_updates_timing_state_across_prefill_decode_and_finish() {
|
||||
let mut tracker = RequestMetricsTracker::new("model".to_string(), 100.0, 64, Some(128), 1);
|
||||
let mut tracker =
|
||||
RequestMetricsTracker::new("model".to_string(), 2, 100.0, 64, Some(128), 1);
|
||||
|
||||
tracker.observe_output(
|
||||
2,
|
||||
10.0,
|
||||
100.2,
|
||||
&vllm_engine_core_client::protocol::output::EngineCoreOutput {
|
||||
@@ -368,7 +377,6 @@ mod tests {
|
||||
},
|
||||
);
|
||||
tracker.observe_output(
|
||||
2,
|
||||
11.5,
|
||||
100.4,
|
||||
&vllm_engine_core_client::protocol::output::EngineCoreOutput {
|
||||
@@ -384,7 +392,7 @@ mod tests {
|
||||
);
|
||||
|
||||
assert!(!tracker.is_prefilling);
|
||||
assert_eq!(tracker.last_seen_engine_index, 2);
|
||||
assert_eq!(tracker.handles.labels.engine, 2);
|
||||
assert_eq!(tracker.num_generation_tokens, 3);
|
||||
assert_eq!(tracker.queued_ts, 8.0);
|
||||
assert_eq!(tracker.scheduled_ts, 9.0);
|
||||
|
||||
@@ -17,7 +17,7 @@ use vllm_engine_core_client::protocol::request::EngineCoreRequest;
|
||||
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
|
||||
use vllm_engine_core_client::protocol::stats::PrefillStats;
|
||||
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
|
||||
use vllm_llm::{
|
||||
Error, FinishReason, GenerateOutputStreamExt as _, GeneratePromptInfo, GenerateRequest, Llm,
|
||||
};
|
||||
@@ -699,7 +699,7 @@ async fn abort_by_external_id_aborts_all_internal_requests() {
|
||||
async fn generate_records_request_metrics_in_prometheus_output() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-metrics".to_vec();
|
||||
let engine_id = EngineId::from_engine_index(4);
|
||||
let model_name = request_metrics_model_name("metrics-model");
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
@@ -832,7 +832,7 @@ async fn generate_records_request_metrics_in_prometheus_output() {
|
||||
async fn dropping_stream_records_abort_terminal_request_metrics() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-metrics-drop".to_vec();
|
||||
let engine_id = EngineId::from_engine_index(5);
|
||||
let model_name = request_metrics_model_name("metrics-drop-model");
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
|
||||
@@ -4,7 +4,7 @@ use std::sync::atomic::AtomicU64;
|
||||
|
||||
use prometheus_client::encoding::text::encode;
|
||||
use prometheus_client::metrics::counter::Counter;
|
||||
use prometheus_client::metrics::family::Family;
|
||||
pub use prometheus_client::metrics::family::Family;
|
||||
use prometheus_client::metrics::gauge::Gauge;
|
||||
use prometheus_client::metrics::histogram::Histogram;
|
||||
use prometheus_client::registry::Registry;
|
||||
@@ -23,6 +23,8 @@ pub use scheduler::*;
|
||||
pub type U64Counter = Counter<u64, AtomicU64>;
|
||||
pub type U64Gauge = Gauge<u64, AtomicU64>;
|
||||
pub type F64Gauge = Gauge<f64, AtomicU64>;
|
||||
/// Histogram metric handle cloned out of a Prometheus family.
|
||||
pub type HistogramMetric = Histogram;
|
||||
pub(crate) type HistogramFamily = Family<EngineLabels, Histogram, fn() -> Histogram>;
|
||||
|
||||
/// Shared Prometheus registry for frontend metrics.
|
||||
|
||||
Reference in New Issue
Block a user