From cc640ee8bc1e61d333ecec039b2e3f143f9d4066 Mon Sep 17 00:00:00 2001 From: Will Eaton Date: Thu, 11 Jun 2026 09:45:03 -0400 Subject: [PATCH] [Rust Frontend][Metrics] Export `vllm:lora_requests_info` from frontend (#45030) Signed-off-by: Will Eaton Signed-off-by: Bugen Zhao Co-authored-by: Bugen Zhao Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- rust/Cargo.lock | 1 + rust/src/engine-core-client/src/client.rs | 3 +- rust/src/engine-core-client/src/client/imp.rs | 24 +- .../engine-core-client/src/client/state.rs | 235 ++++++++++++++++-- rust/src/engine-core-client/src/metrics.rs | 117 ++++++++- .../engine-core-client/src/protocol/stats.rs | 4 - rust/src/metrics/Cargo.toml | 1 + rust/src/metrics/src/scheduler.rs | 34 ++- 8 files changed, 383 insertions(+), 36 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index e6011ddf5c..c1477092b9 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5834,6 +5834,7 @@ dependencies = [ name = "vllm-metrics" version = "0.1.0" dependencies = [ + "itertools 0.14.0", "prometheus-client", ] diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index 7186dfe240..73ebe9ef40 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -442,9 +442,10 @@ impl EngineCoreClient { ); let request_id = req.request_id.clone(); + let lora_name = req.lora_request.as_ref().map(|lora| lora.lora_name.clone()); let data_parallel_rank = req.data_parallel_rank; let (engine_id, rx) = - self.inner.register_request(request_id.clone(), data_parallel_rank)?; + self.inner.register_request(request_id.clone(), lora_name, data_parallel_rank)?; let result: Result<()> = async { if let Some(coordinator) = self.coordinator.as_ref() { diff --git a/rust/src/engine-core-client/src/client/imp.rs b/rust/src/engine-core-client/src/client/imp.rs index 44eae7e3e5..6f218717ed 100644 --- a/rust/src/engine-core-client/src/client/imp.rs +++ b/rust/src/engine-core-client/src/client/imp.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use arc_swap::ArcSwapOption; @@ -13,7 +13,7 @@ use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, Uti use crate::client::stream::EngineCoreStreamOutput; use crate::client::{AbortCause, AbortRequest}; use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output}; -use crate::metrics::record_scheduler_stats; +use crate::metrics::{LoraInfoExporter, record_scheduler_stats}; use crate::protocol::stats::SchedulerStats; use crate::protocol::utility::UtilityOutput; use crate::protocol::{ @@ -58,17 +58,19 @@ impl ClientInner { /// per-request output channel bound to its `request_id`. /// /// When `data_parallel_rank` is provided, the request is routed to that - /// specific engine rank, bypassing load balancing. + /// specific engine rank, bypassing load balancing. `lora_name` is the + /// request's LoRA adapter, tracked for `vllm:lora_requests_info`. pub fn register_request( &self, request_id: String, + lora_name: Option, data_parallel_rank: Option, ) -> Result<(EngineId, OutputReceiver)> { let mut registry = self.request_reg.lock(); if registry.is_closed() { return Err(self.closed_error()); } - registry.register(request_id, data_parallel_rank) + registry.register(request_id, lora_name, data_parallel_rank) } /// Allocate the next utility `call_id` and register its waiting receiver. @@ -131,6 +133,12 @@ impl ClientInner { self.request_reg.lock().apply_scheduler_stats(engine_index, stats) } + /// Snapshot the adapter names of tracked LoRA requests as + /// (running, waiting) sets. + pub fn lora_adapter_states(&self) -> (BTreeSet, BTreeSet) { + self.request_reg.lock().lora_adapter_states() + } + /// Close all active request streams and utility calls with the first /// persistent health error. pub fn close_registries(&self, error: Arc) { @@ -303,6 +311,8 @@ pub(crate) async fn run_output_dispatcher_loop( inner: Arc, mut output_rx: mpsc::Receiver>, ) { + let mut lora_info = LoraInfoExporter::default(); + let result: Result<()> = async { loop { let outputs = match output_rx.recv().await { @@ -357,6 +367,12 @@ pub(crate) async fn run_output_dispatcher_loop( scheduler_stats, ); } + + // The engine's scheduler stats never carry adapter names; + // the gauge is derived from the registry's frontend-side + // request tracking instead. + let (running, waiting) = inner.lora_adapter_states(); + lora_info.update(&METRICS.scheduler, running, waiting); } ClassifiedEngineCoreOutputs::Utility(utility) => { let call_id = utility.output.call_id; diff --git a/rust/src/engine-core-client/src/client/state.rs b/rust/src/engine-core-client/src/client/state.rs index 99302e4f8c..062f284d90 100644 --- a/rust/src/engine-core-client/src/client/state.rs +++ b/rust/src/engine-core-client/src/client/state.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::atomic::{AtomicU64, Ordering}; use tokio::sync::{mpsc, oneshot}; @@ -7,9 +7,9 @@ use tracing::trace; use crate::EngineId; use crate::client::stream::EngineCoreStreamOutput; use crate::error::{Error, Result}; -use crate::protocol::EngineCoreOutput; use crate::protocol::stats::SchedulerStats; use crate::protocol::utility::UtilityOutput; +use crate::protocol::{EngineCoreEventType, EngineCoreOutput}; use crate::transport::ConnectedEngine; pub type OutputSender = mpsc::UnboundedSender>; @@ -21,6 +21,25 @@ pub type UtilityReceiver = oneshot::Receiver>; struct TrackedRequest { sender: OutputSender, engine_id: EngineId, + lora: Option, +} + +/// Frontend-side view of one LoRA request's scheduling phase. +/// +/// The engine's `SchedulerStats` does not carry adapter names, so +/// `vllm:lora_requests_info` must be derived from per-request lifecycle events +/// observed by this client, mirroring `LoRARequestStates` in the Python +/// frontend (`vllm/v1/engine/output_processor.py`). +#[derive(Debug)] +struct LoraRequestState { + adapter_name: String, + phase: LoraPhase, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LoraPhase { + Waiting, + Running, } /// The latest real scheduler-side load snapshot observed from one engine. @@ -105,6 +124,7 @@ impl RequestRegistry { pub fn register( &mut self, request_id: String, + lora_name: Option, data_parallel_rank: Option, ) -> Result<(EngineId, OutputReceiver)> { if self.requests.contains_key(&request_id) { @@ -118,6 +138,10 @@ impl RequestRegistry { TrackedRequest { sender: tx, engine_id: engine_id.clone(), + lora: lora_name.map(|adapter_name| LoraRequestState { + adapter_name, + phase: LoraPhase::Waiting, + }), }, ); @@ -171,6 +195,7 @@ impl RequestRegistry { /// Obtain the stream sender for one output. If it indicates the request is /// finished, it will be removed from the registry. pub fn sender_for_output(&mut self, output: &EngineCoreOutput) -> Option { + self.apply_lora_events(output); if output.finished() { self.remove(output.request_id.as_str()).map(|tracked| tracked.0) } else { @@ -180,6 +205,43 @@ impl RequestRegistry { } } + /// Advance the request's LoRA scheduling phase from the engine-core events + /// attached to one output, mirroring the Python frontend's + /// `LoRARequestStates.update_from_events`. + fn apply_lora_events(&mut self, output: &EngineCoreOutput) { + let Some(events) = output.events.as_ref() else { + return; + }; + let Some(lora) = self + .requests + .get_mut(output.request_id.as_str()) + .and_then(|tracked| tracked.lora.as_mut()) + else { + return; + }; + for event in events { + lora.phase = match event.r#type { + EngineCoreEventType::Queued | EngineCoreEventType::Preempted => LoraPhase::Waiting, + EngineCoreEventType::Scheduled => LoraPhase::Running, + }; + } + } + + /// Snapshot the adapter names of tracked LoRA requests as + /// (running, waiting) sets. Feeds the `vllm:lora_requests_info` gauge. + pub fn lora_adapter_states(&self) -> (BTreeSet, BTreeSet) { + let mut running = BTreeSet::new(); + let mut waiting = BTreeSet::new(); + for lora in self.requests.values().filter_map(|tracked| tracked.lora.as_ref()) { + let set = match lora.phase { + LoraPhase::Running => &mut running, + LoraPhase::Waiting => &mut waiting, + }; + set.insert(lora.adapter_name.clone()); + } + (running, waiting) + } + /// Obtain stream senders for a whole engine output batch under one /// registry lock. Finished outputs are removed before returning. pub fn senders_for_outputs<'a>( @@ -336,11 +398,16 @@ impl UtilityRegistry { #[cfg(test)] mod tests { - use super::{EngineRoutingState, RequestRegistry, UtilityRegistry}; + use std::collections::BTreeSet; + use crate::EngineId; - use crate::client::state::EngineLoadSnapshot; + use crate::client::state::{ + EngineLoadSnapshot, EngineRoutingState, RequestRegistry, UtilityRegistry, + }; use crate::mock_engine::default_ready_response; - use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput}; + use crate::protocol::{ + EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput, + }; use crate::transport::ConnectedEngine; fn connected_engine(engine_id: EngineId) -> ConnectedEngine { @@ -350,11 +417,36 @@ mod tests { } } + fn output_with_events( + request_id: &str, + events: &[EngineCoreEventType], + finish_reason: Option, + ) -> EngineCoreOutput { + EngineCoreOutput { + request_id: request_id.to_string(), + events: Some( + events + .iter() + .map(|event_type| EngineCoreEvent { + r#type: *event_type, + timestamp: 0.0, + }) + .collect(), + ), + finish_reason, + ..Default::default() + } + } + + fn adapter_names(values: &[&str]) -> BTreeSet { + values.iter().map(|name| (*name).to_string()).collect() + } + #[test] fn registry_rejects_duplicate_request_ids() { let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]); - registry.register("req-1".to_string(), None).unwrap(); - let error = registry.register("req-1".to_string(), None).unwrap_err(); + registry.register("req-1".to_string(), None, None).unwrap(); + let error = registry.register("req-1".to_string(), None, None).unwrap_err(); assert!(matches!( error, crate::error::Error::DuplicateRequestId { request_id } if request_id == "req-1" @@ -364,7 +456,7 @@ mod tests { #[test] fn registry_removes_finished_request_on_output() { let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]); - registry.register("req-1".to_string(), None).unwrap(); + registry.register("req-1".to_string(), None, None).unwrap(); let sender = registry.sender_for_output(&EngineCoreOutput { request_id: "req-1".to_string(), @@ -376,11 +468,104 @@ mod tests { assert!(!registry.contains("req-1")); } + #[test] + fn registry_tracks_lora_phases_from_engine_events() { + let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]); + registry + .register("req-lora".to_string(), Some("adapter-a".to_string()), None) + .unwrap(); + registry.register("req-plain".to_string(), None, None).unwrap(); + + // Registered but not yet scheduled: counted as waiting. The non-LoRA + // request never shows up. + assert_eq!( + registry.lora_adapter_states(), + (adapter_names(&[]), adapter_names(&["adapter-a"])) + ); + + // Queued then scheduled in one output: running. + drop(registry.sender_for_output(&output_with_events( + "req-lora", + &[EngineCoreEventType::Queued, EngineCoreEventType::Scheduled], + None, + ))); + assert_eq!( + registry.lora_adapter_states(), + (adapter_names(&["adapter-a"]), adapter_names(&[])) + ); + + // Preempted: back to waiting. + drop(registry.sender_for_output(&output_with_events( + "req-lora", + &[EngineCoreEventType::Preempted], + None, + ))); + assert_eq!( + registry.lora_adapter_states(), + (adapter_names(&[]), adapter_names(&["adapter-a"])) + ); + + // Finished: dropped from tracking entirely. + drop(registry.sender_for_output(&output_with_events( + "req-lora", + &[EngineCoreEventType::Scheduled], + Some(EngineCoreFinishReason::Stop), + ))); + assert_eq!( + registry.lora_adapter_states(), + (adapter_names(&[]), adapter_names(&[])) + ); + } + + #[test] + fn registry_unions_lora_adapters_across_requests() { + let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]); + registry + .register("req-a1".to_string(), Some("adapter-a".to_string()), None) + .unwrap(); + registry + .register("req-a2".to_string(), Some("adapter-a".to_string()), None) + .unwrap(); + registry + .register("req-b".to_string(), Some("adapter-b".to_string()), None) + .unwrap(); + + // One of adapter-a's requests starts running while the other waits: + // the adapter appears in both sets. + drop(registry.sender_for_output(&output_with_events( + "req-a1", + &[EngineCoreEventType::Scheduled], + None, + ))); + assert_eq!( + registry.lora_adapter_states(), + ( + adapter_names(&["adapter-a"]), + adapter_names(&["adapter-a", "adapter-b"]) + ) + ); + } + + #[test] + fn registry_drops_lora_tracking_on_abort() { + let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]); + registry + .register("req-lora".to_string(), Some("adapter-a".to_string()), None) + .unwrap(); + + drop(registry.finish_many(&["req-lora".to_string()])); + + assert_eq!( + registry.lora_adapter_states(), + (adapter_names(&[]), adapter_names(&[])) + ); + } + #[test] fn registry_closes_all_requests_on_failure() { let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]); - registry.register("req-1".to_string(), None).unwrap(); - registry.register("req-2".to_string(), None).unwrap(); + registry.register("req-1".to_string(), None, None).unwrap(); + registry.register("req-2".to_string(), None, None).unwrap(); let senders = registry.close(); @@ -396,9 +581,9 @@ mod tests { connected_engine(engine_0.clone()), connected_engine(engine_1.clone()), ]); - let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap(); - let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap(); - let (chosen_0_again, _) = registry.register("req-3".to_string(), None).unwrap(); + let (chosen_0, _) = registry.register("req-1".to_string(), None, None).unwrap(); + let (chosen_1, _) = registry.register("req-2".to_string(), None, None).unwrap(); + let (chosen_0_again, _) = registry.register("req-3".to_string(), None, None).unwrap(); assert_eq!(chosen_0, engine_0); assert_eq!(chosen_1, engine_1); @@ -425,9 +610,9 @@ mod tests { connected_engine(engine_1.clone()), ]); - let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap(); - let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap(); - let (chosen_0_again, _) = registry.register("req-3".to_string(), None).unwrap(); + let (chosen_0, _) = registry.register("req-1".to_string(), None, None).unwrap(); + let (chosen_1, _) = registry.register("req-2".to_string(), None, None).unwrap(); + let (chosen_0_again, _) = registry.register("req-3".to_string(), None, None).unwrap(); assert_eq!(chosen_0, engine_0); assert_eq!(chosen_1, engine_1); @@ -494,7 +679,7 @@ mod tests { } )); - let (chosen, _) = registry.register("req-stats".to_string(), None).unwrap(); + let (chosen, _) = registry.register("req-stats".to_string(), None, None).unwrap(); assert_eq!(chosen, engine_1); } @@ -510,15 +695,15 @@ mod tests { ]); // Explicitly target rank 2 (third engine). - let (chosen, _) = registry.register("req-1".to_string(), Some(2)).unwrap(); + let (chosen, _) = registry.register("req-1".to_string(), None, Some(2)).unwrap(); assert_eq!(chosen, engine_2); // Explicitly target rank 0 (first engine). - let (chosen, _) = registry.register("req-2".to_string(), Some(0)).unwrap(); + let (chosen, _) = registry.register("req-2".to_string(), None, Some(0)).unwrap(); assert_eq!(chosen, engine_0); // Explicitly target rank 1. - let (chosen, _) = registry.register("req-3".to_string(), Some(1)).unwrap(); + let (chosen, _) = registry.register("req-3".to_string(), None, Some(1)).unwrap(); assert_eq!(chosen, engine_1); } @@ -532,11 +717,11 @@ mod tests { ]); // Load-balance: first two go to engine_0 and engine_1. - registry.register("req-lb-0".to_string(), None).unwrap(); + registry.register("req-lb-0".to_string(), None, None).unwrap(); // Now engine_0 has 1 in-flight. Without dp_rank, next would go to engine_1. // But with dp_rank=0, it should still go to engine_0. - let (chosen, _) = registry.register("req-dp".to_string(), Some(0)).unwrap(); + let (chosen, _) = registry.register("req-dp".to_string(), None, Some(0)).unwrap(); assert_eq!(chosen, engine_0); } @@ -547,7 +732,7 @@ mod tests { connected_engine(EngineId::from_engine_index(1)), ]); - let error = registry.register("req-1".to_string(), Some(2)).unwrap_err(); + let error = registry.register("req-1".to_string(), None, Some(2)).unwrap_err(); assert!(matches!( error, crate::error::Error::InvalidDataParallelRank { @@ -562,10 +747,10 @@ mod tests { let engine_0 = EngineId::from_engine_index(0); let mut registry = RequestRegistry::new(&[connected_engine(engine_0.clone())]); - let (chosen, _) = registry.register("req-ok".to_string(), Some(0)).unwrap(); + let (chosen, _) = registry.register("req-ok".to_string(), None, Some(0)).unwrap(); assert_eq!(chosen, engine_0); - let error = registry.register("req-bad".to_string(), Some(1)).unwrap_err(); + let error = registry.register("req-bad".to_string(), None, Some(1)).unwrap_err(); assert!(matches!( error, crate::error::Error::InvalidDataParallelRank { diff --git a/rust/src/engine-core-client/src/metrics.rs b/rust/src/engine-core-client/src/metrics.rs index 8f45939619..a939b02f65 100644 --- a/rust/src/engine-core-client/src/metrics.rs +++ b/rust/src/engine-core-client/src/metrics.rs @@ -1,4 +1,10 @@ -use vllm_metrics::{EngineLabels, EnginePositionLabels, SchedulerMetrics, WaitingReasonLabels}; +use std::collections::BTreeSet; +use std::time::{SystemTime, UNIX_EPOCH}; + +use vllm_metrics::{ + EngineLabels, EnginePositionLabels, LoraAdapterNames, LoraInfoLabels, SchedulerMetrics, + WaitingReasonLabels, +}; use crate::protocol::stats::SchedulerStats; @@ -129,3 +135,112 @@ pub(crate) fn record_scheduler_stats( } } } + +/// Exports `vllm:lora_requests_info` as a single series covering all LoRA +/// requests tracked by this client across every engine in the replica. +/// +/// The engine's `SchedulerStats` never carries adapter names: the Python +/// frontend fills them in from per-request lifecycle events tracked by +/// `LoRARequestStates` in `vllm/v1/engine/output_processor.py`. The Rust +/// frontend mirrors that, deriving the sets from the request registry. +#[derive(Default)] +pub(crate) struct LoraInfoExporter { + current: Option, +} + +impl LoraInfoExporter { + pub(crate) fn update( + &mut self, + metrics: &SchedulerMetrics, + running: BTreeSet, + waiting: BTreeSet, + ) { + let next = (!running.is_empty() || !waiting.is_empty()).then_some(LoraInfoLabels { + running_lora_adapters: LoraAdapterNames(running), + waiting_lora_adapters: LoraAdapterNames(waiting), + }); + + if self.current != next + && let Some(prev) = &self.current + { + metrics.lora_info.remove(prev); + } + + // Python sets this gauge to the current time on every record. + if let Some(labels) = &next { + metrics.lora_info.get_or_create(labels).set(now_unix_secs()); + } + + self.current = next; + } +} + +fn now_unix_secs() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use expect_test::expect; + use vllm_metrics::Metrics; + + use crate::metrics::LoraInfoExporter; + + fn names(values: &[&str]) -> BTreeSet { + values.iter().map(|name| (*name).to_string()).collect() + } + + /// The `lora_requests_info` series with the non-deterministic timestamp + /// value replaced by ``, one line per series. + fn lora_series(rendered: &str) -> String { + rendered + .lines() + .filter(|l| l.starts_with("vllm:lora_requests_info{")) + .map(|l| match l.rsplit_once("} ") { + Some((labels, _value)) => format!("{labels}}} "), + None => l.to_string(), + }) + .collect::>() + .join("\n") + } + + #[test] + fn lora_info_emits_clears_stale_and_drains() { + let metrics = Metrics::new(); + let mut exporter = LoraInfoExporter::default(); + + // No adapters: nothing emitted. + exporter.update(&metrics.scheduler, names(&[]), names(&[])); + expect![[""]].assert_eq(&lora_series(&metrics.render().unwrap())); + + // Two running (sorted), one waiting. + exporter.update(&metrics.scheduler, names(&["b", "a"]), names(&["c"])); + expect![[ + r#"vllm:lora_requests_info{running_lora_adapters="a,b",waiting_lora_adapters="c"} "# + ]] + .assert_eq(&lora_series(&metrics.render().unwrap())); + + // "c" gets scheduled and "d" arrives: the stale series is replaced. + exporter.update(&metrics.scheduler, names(&["a", "b", "c"]), names(&["d"])); + expect![[ + r#"vllm:lora_requests_info{running_lora_adapters="a,b,c",waiting_lora_adapters="d"} "# + ]] + .assert_eq(&lora_series(&metrics.render().unwrap())); + + // Everything but "d" finishes. + exporter.update(&metrics.scheduler, names(&["d"]), names(&[])); + expect![[ + r#"vllm:lora_requests_info{running_lora_adapters="d",waiting_lora_adapters=""} "# + ]] + .assert_eq(&lora_series(&metrics.render().unwrap())); + + // All requests done: series removed entirely. + exporter.update(&metrics.scheduler, names(&[]), names(&[])); + expect![[""]].assert_eq(&lora_series(&metrics.render().unwrap())); + } +} diff --git a/rust/src/engine-core-client/src/protocol/stats.rs b/rust/src/engine-core-client/src/protocol/stats.rs index 254efc31b2..9f35f8301e 100644 --- a/rust/src/engine-core-client/src/protocol/stats.rs +++ b/rust/src/engine-core-client/src/protocol/stats.rs @@ -181,10 +181,6 @@ pub struct SchedulerStats { pub spec_decoding_stats: Option, /// Connector-specific KV transfer stats, kept opaque for now. pub kv_connector_stats: Option>, - /// Waiting request counts per LoRA adapter. - pub waiting_lora_adapters: BTreeMap, - /// Running request counts per LoRA adapter. - pub running_lora_adapters: BTreeMap, /// CUDA graph runtime stats when graph metrics are enabled. pub cudagraph_stats: Option, /// Estimated MFU/performance stats, when enabled. diff --git a/rust/src/metrics/Cargo.toml b/rust/src/metrics/Cargo.toml index e6b579b97a..ab1a72098b 100644 --- a/rust/src/metrics/Cargo.toml +++ b/rust/src/metrics/Cargo.toml @@ -5,6 +5,7 @@ edition.workspace = true license.workspace = true [dependencies] +itertools.workspace = true prometheus-client.workspace = true [lints] diff --git a/rust/src/metrics/src/scheduler.rs b/rust/src/metrics/src/scheduler.rs index 0acbdf0fa7..ec5f8d4e9f 100644 --- a/rust/src/metrics/src/scheduler.rs +++ b/rust/src/metrics/src/scheduler.rs @@ -1,4 +1,7 @@ -use prometheus_client::encoding::EncodeLabelSet; +use std::collections::BTreeSet; + +use itertools::Itertools as _; +use prometheus_client::encoding::{EncodeLabelSet, EncodeLabelValue, LabelValueEncoder}; use prometheus_client::metrics::family::Family; use prometheus_client::metrics::histogram::Histogram; use prometheus_client::registry::Registry; @@ -42,6 +45,23 @@ pub struct WaitingReasonLabels { pub reason: &'static str, } +/// Adapter names encoded as a deterministic comma-joined Prometheus label value. +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub struct LoraAdapterNames(pub BTreeSet); + +impl EncodeLabelValue for LoraAdapterNames { + fn encode(&self, encoder: &mut LabelValueEncoder) -> Result<(), std::fmt::Error> { + EncodeLabelValue::encode(&self.0.iter().join(","), encoder) + } +} + +/// Labels for `vllm:lora_requests_info`. +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +pub struct LoraInfoLabels { + pub running_lora_adapters: LoraAdapterNames, + pub waiting_lora_adapters: LoraAdapterNames, +} + /// Scheduler/batch-scoped Prometheus families exported from `SchedulerStats`. pub struct SchedulerMetrics { // Scheduler state gauges. @@ -50,6 +70,10 @@ pub struct SchedulerMetrics { pub scheduler_waiting_by_reason: Family, pub kv_cache_usage: Family, + /// `vllm:lora_requests_info`. Value is the emit-time unix timestamp in + /// seconds. + pub lora_info: Family, + // Prefix-cache counters, including the connector-backed external cache path. pub prefix_cache_queries: Family, pub prefix_cache_hits: Family, @@ -109,6 +133,13 @@ impl SchedulerMetrics { kv_cache_usage.clone(), ); + let lora_info = Family::default(); + registry.register( + "vllm:lora_requests_info", + "Running stats on lora requests.", + lora_info.clone(), + ); + // Prefix-cache counters, including the connector-backed external cache path. let prefix_cache_queries = Family::default(); registry.register( @@ -219,6 +250,7 @@ impl SchedulerMetrics { scheduler_waiting, scheduler_waiting_by_reason, kv_cache_usage, + lora_info, prefix_cache_queries, prefix_cache_hits, external_prefix_cache_queries,