[Rust Frontend][Metrics] Export vllm:lora_requests_info from frontend (#45030)

Signed-off-by: Will Eaton <weaton@redhat.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Will Eaton
2026-06-11 06:45:03 -07:00
committed by GitHub
co-authored by Bugen Zhao mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ebc6ef971a
commit cc640ee8bc
8 changed files with 383 additions and 36 deletions
+2 -1
View File
@@ -442,9 +442,10 @@ impl EngineCoreClient {
);
let request_id = req.request_id.clone();
let lora_name = req.lora_request.as_ref().map(|lora| lora.lora_name.clone());
let data_parallel_rank = req.data_parallel_rank;
let (engine_id, rx) =
self.inner.register_request(request_id.clone(), data_parallel_rank)?;
self.inner.register_request(request_id.clone(), lora_name, data_parallel_rank)?;
let result: Result<()> = async {
if let Some(coordinator) = self.coordinator.as_ref() {
+20 -4
View File
@@ -1,4 +1,4 @@
use std::collections::BTreeMap;
use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
use arc_swap::ArcSwapOption;
@@ -13,7 +13,7 @@ use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, Uti
use crate::client::stream::EngineCoreStreamOutput;
use crate::client::{AbortCause, AbortRequest};
use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
use crate::metrics::record_scheduler_stats;
use crate::metrics::{LoraInfoExporter, record_scheduler_stats};
use crate::protocol::stats::SchedulerStats;
use crate::protocol::utility::UtilityOutput;
use crate::protocol::{
@@ -58,17 +58,19 @@ impl ClientInner {
/// per-request output channel bound to its `request_id`.
///
/// When `data_parallel_rank` is provided, the request is routed to that
/// specific engine rank, bypassing load balancing.
/// specific engine rank, bypassing load balancing. `lora_name` is the
/// request's LoRA adapter, tracked for `vllm:lora_requests_info`.
pub fn register_request(
&self,
request_id: String,
lora_name: Option<String>,
data_parallel_rank: Option<u32>,
) -> Result<(EngineId, OutputReceiver)> {
let mut registry = self.request_reg.lock();
if registry.is_closed() {
return Err(self.closed_error());
}
registry.register(request_id, data_parallel_rank)
registry.register(request_id, lora_name, data_parallel_rank)
}
/// Allocate the next utility `call_id` and register its waiting receiver.
@@ -131,6 +133,12 @@ impl ClientInner {
self.request_reg.lock().apply_scheduler_stats(engine_index, stats)
}
/// Snapshot the adapter names of tracked LoRA requests as
/// (running, waiting) sets.
pub fn lora_adapter_states(&self) -> (BTreeSet<String>, BTreeSet<String>) {
self.request_reg.lock().lora_adapter_states()
}
/// Close all active request streams and utility calls with the first
/// persistent health error.
pub fn close_registries(&self, error: Arc<Error>) {
@@ -303,6 +311,8 @@ pub(crate) async fn run_output_dispatcher_loop(
inner: Arc<ClientInner>,
mut output_rx: mpsc::Receiver<Result<EngineCoreOutputs>>,
) {
let mut lora_info = LoraInfoExporter::default();
let result: Result<()> = async {
loop {
let outputs = match output_rx.recv().await {
@@ -357,6 +367,12 @@ pub(crate) async fn run_output_dispatcher_loop(
scheduler_stats,
);
}
// The engine's scheduler stats never carry adapter names;
// the gauge is derived from the registry's frontend-side
// request tracking instead.
let (running, waiting) = inner.lora_adapter_states();
lora_info.update(&METRICS.scheduler, running, waiting);
}
ClassifiedEngineCoreOutputs::Utility(utility) => {
let call_id = utility.output.call_id;
+210 -25
View File
@@ -1,4 +1,4 @@
use std::collections::{BTreeMap, HashMap};
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::atomic::{AtomicU64, Ordering};
use tokio::sync::{mpsc, oneshot};
@@ -7,9 +7,9 @@ use tracing::trace;
use crate::EngineId;
use crate::client::stream::EngineCoreStreamOutput;
use crate::error::{Error, Result};
use crate::protocol::EngineCoreOutput;
use crate::protocol::stats::SchedulerStats;
use crate::protocol::utility::UtilityOutput;
use crate::protocol::{EngineCoreEventType, EngineCoreOutput};
use crate::transport::ConnectedEngine;
pub type OutputSender = mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>;
@@ -21,6 +21,25 @@ pub type UtilityReceiver = oneshot::Receiver<Result<UtilityOutput>>;
struct TrackedRequest {
sender: OutputSender,
engine_id: EngineId,
lora: Option<LoraRequestState>,
}
/// Frontend-side view of one LoRA request's scheduling phase.
///
/// The engine's `SchedulerStats` does not carry adapter names, so
/// `vllm:lora_requests_info` must be derived from per-request lifecycle events
/// observed by this client, mirroring `LoRARequestStates` in the Python
/// frontend (`vllm/v1/engine/output_processor.py`).
#[derive(Debug)]
struct LoraRequestState {
adapter_name: String,
phase: LoraPhase,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LoraPhase {
Waiting,
Running,
}
/// The latest real scheduler-side load snapshot observed from one engine.
@@ -105,6 +124,7 @@ impl RequestRegistry {
pub fn register(
&mut self,
request_id: String,
lora_name: Option<String>,
data_parallel_rank: Option<u32>,
) -> Result<(EngineId, OutputReceiver)> {
if self.requests.contains_key(&request_id) {
@@ -118,6 +138,10 @@ impl RequestRegistry {
TrackedRequest {
sender: tx,
engine_id: engine_id.clone(),
lora: lora_name.map(|adapter_name| LoraRequestState {
adapter_name,
phase: LoraPhase::Waiting,
}),
},
);
@@ -171,6 +195,7 @@ impl RequestRegistry {
/// Obtain the stream sender for one output. If it indicates the request is
/// finished, it will be removed from the registry.
pub fn sender_for_output(&mut self, output: &EngineCoreOutput) -> Option<OutputSender> {
self.apply_lora_events(output);
if output.finished() {
self.remove(output.request_id.as_str()).map(|tracked| tracked.0)
} else {
@@ -180,6 +205,43 @@ impl RequestRegistry {
}
}
/// Advance the request's LoRA scheduling phase from the engine-core events
/// attached to one output, mirroring the Python frontend's
/// `LoRARequestStates.update_from_events`.
fn apply_lora_events(&mut self, output: &EngineCoreOutput) {
let Some(events) = output.events.as_ref() else {
return;
};
let Some(lora) = self
.requests
.get_mut(output.request_id.as_str())
.and_then(|tracked| tracked.lora.as_mut())
else {
return;
};
for event in events {
lora.phase = match event.r#type {
EngineCoreEventType::Queued | EngineCoreEventType::Preempted => LoraPhase::Waiting,
EngineCoreEventType::Scheduled => LoraPhase::Running,
};
}
}
/// Snapshot the adapter names of tracked LoRA requests as
/// (running, waiting) sets. Feeds the `vllm:lora_requests_info` gauge.
pub fn lora_adapter_states(&self) -> (BTreeSet<String>, BTreeSet<String>) {
let mut running = BTreeSet::new();
let mut waiting = BTreeSet::new();
for lora in self.requests.values().filter_map(|tracked| tracked.lora.as_ref()) {
let set = match lora.phase {
LoraPhase::Running => &mut running,
LoraPhase::Waiting => &mut waiting,
};
set.insert(lora.adapter_name.clone());
}
(running, waiting)
}
/// Obtain stream senders for a whole engine output batch under one
/// registry lock. Finished outputs are removed before returning.
pub fn senders_for_outputs<'a>(
@@ -336,11 +398,16 @@ impl UtilityRegistry {
#[cfg(test)]
mod tests {
use super::{EngineRoutingState, RequestRegistry, UtilityRegistry};
use std::collections::BTreeSet;
use crate::EngineId;
use crate::client::state::EngineLoadSnapshot;
use crate::client::state::{
EngineLoadSnapshot, EngineRoutingState, RequestRegistry, UtilityRegistry,
};
use crate::mock_engine::default_ready_response;
use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput};
use crate::protocol::{
EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
};
use crate::transport::ConnectedEngine;
fn connected_engine(engine_id: EngineId) -> ConnectedEngine {
@@ -350,11 +417,36 @@ mod tests {
}
}
fn output_with_events(
request_id: &str,
events: &[EngineCoreEventType],
finish_reason: Option<EngineCoreFinishReason>,
) -> EngineCoreOutput {
EngineCoreOutput {
request_id: request_id.to_string(),
events: Some(
events
.iter()
.map(|event_type| EngineCoreEvent {
r#type: *event_type,
timestamp: 0.0,
})
.collect(),
),
finish_reason,
..Default::default()
}
}
fn adapter_names(values: &[&str]) -> BTreeSet<String> {
values.iter().map(|name| (*name).to_string()).collect()
}
#[test]
fn registry_rejects_duplicate_request_ids() {
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
registry.register("req-1".to_string(), None).unwrap();
let error = registry.register("req-1".to_string(), None).unwrap_err();
registry.register("req-1".to_string(), None, None).unwrap();
let error = registry.register("req-1".to_string(), None, None).unwrap_err();
assert!(matches!(
error,
crate::error::Error::DuplicateRequestId { request_id } if request_id == "req-1"
@@ -364,7 +456,7 @@ mod tests {
#[test]
fn registry_removes_finished_request_on_output() {
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
registry.register("req-1".to_string(), None).unwrap();
registry.register("req-1".to_string(), None, None).unwrap();
let sender = registry.sender_for_output(&EngineCoreOutput {
request_id: "req-1".to_string(),
@@ -376,11 +468,104 @@ mod tests {
assert!(!registry.contains("req-1"));
}
#[test]
fn registry_tracks_lora_phases_from_engine_events() {
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
registry
.register("req-lora".to_string(), Some("adapter-a".to_string()), None)
.unwrap();
registry.register("req-plain".to_string(), None, None).unwrap();
// Registered but not yet scheduled: counted as waiting. The non-LoRA
// request never shows up.
assert_eq!(
registry.lora_adapter_states(),
(adapter_names(&[]), adapter_names(&["adapter-a"]))
);
// Queued then scheduled in one output: running.
drop(registry.sender_for_output(&output_with_events(
"req-lora",
&[EngineCoreEventType::Queued, EngineCoreEventType::Scheduled],
None,
)));
assert_eq!(
registry.lora_adapter_states(),
(adapter_names(&["adapter-a"]), adapter_names(&[]))
);
// Preempted: back to waiting.
drop(registry.sender_for_output(&output_with_events(
"req-lora",
&[EngineCoreEventType::Preempted],
None,
)));
assert_eq!(
registry.lora_adapter_states(),
(adapter_names(&[]), adapter_names(&["adapter-a"]))
);
// Finished: dropped from tracking entirely.
drop(registry.sender_for_output(&output_with_events(
"req-lora",
&[EngineCoreEventType::Scheduled],
Some(EngineCoreFinishReason::Stop),
)));
assert_eq!(
registry.lora_adapter_states(),
(adapter_names(&[]), adapter_names(&[]))
);
}
#[test]
fn registry_unions_lora_adapters_across_requests() {
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
registry
.register("req-a1".to_string(), Some("adapter-a".to_string()), None)
.unwrap();
registry
.register("req-a2".to_string(), Some("adapter-a".to_string()), None)
.unwrap();
registry
.register("req-b".to_string(), Some("adapter-b".to_string()), None)
.unwrap();
// One of adapter-a's requests starts running while the other waits:
// the adapter appears in both sets.
drop(registry.sender_for_output(&output_with_events(
"req-a1",
&[EngineCoreEventType::Scheduled],
None,
)));
assert_eq!(
registry.lora_adapter_states(),
(
adapter_names(&["adapter-a"]),
adapter_names(&["adapter-a", "adapter-b"])
)
);
}
#[test]
fn registry_drops_lora_tracking_on_abort() {
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
registry
.register("req-lora".to_string(), Some("adapter-a".to_string()), None)
.unwrap();
drop(registry.finish_many(&["req-lora".to_string()]));
assert_eq!(
registry.lora_adapter_states(),
(adapter_names(&[]), adapter_names(&[]))
);
}
#[test]
fn registry_closes_all_requests_on_failure() {
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
registry.register("req-1".to_string(), None).unwrap();
registry.register("req-2".to_string(), None).unwrap();
registry.register("req-1".to_string(), None, None).unwrap();
registry.register("req-2".to_string(), None, None).unwrap();
let senders = registry.close();
@@ -396,9 +581,9 @@ mod tests {
connected_engine(engine_0.clone()),
connected_engine(engine_1.clone()),
]);
let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap();
let (chosen_0_again, _) = registry.register("req-3".to_string(), None).unwrap();
let (chosen_0, _) = registry.register("req-1".to_string(), None, None).unwrap();
let (chosen_1, _) = registry.register("req-2".to_string(), None, None).unwrap();
let (chosen_0_again, _) = registry.register("req-3".to_string(), None, None).unwrap();
assert_eq!(chosen_0, engine_0);
assert_eq!(chosen_1, engine_1);
@@ -425,9 +610,9 @@ mod tests {
connected_engine(engine_1.clone()),
]);
let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap();
let (chosen_0_again, _) = registry.register("req-3".to_string(), None).unwrap();
let (chosen_0, _) = registry.register("req-1".to_string(), None, None).unwrap();
let (chosen_1, _) = registry.register("req-2".to_string(), None, None).unwrap();
let (chosen_0_again, _) = registry.register("req-3".to_string(), None, None).unwrap();
assert_eq!(chosen_0, engine_0);
assert_eq!(chosen_1, engine_1);
@@ -494,7 +679,7 @@ mod tests {
}
));
let (chosen, _) = registry.register("req-stats".to_string(), None).unwrap();
let (chosen, _) = registry.register("req-stats".to_string(), None, None).unwrap();
assert_eq!(chosen, engine_1);
}
@@ -510,15 +695,15 @@ mod tests {
]);
// Explicitly target rank 2 (third engine).
let (chosen, _) = registry.register("req-1".to_string(), Some(2)).unwrap();
let (chosen, _) = registry.register("req-1".to_string(), None, Some(2)).unwrap();
assert_eq!(chosen, engine_2);
// Explicitly target rank 0 (first engine).
let (chosen, _) = registry.register("req-2".to_string(), Some(0)).unwrap();
let (chosen, _) = registry.register("req-2".to_string(), None, Some(0)).unwrap();
assert_eq!(chosen, engine_0);
// Explicitly target rank 1.
let (chosen, _) = registry.register("req-3".to_string(), Some(1)).unwrap();
let (chosen, _) = registry.register("req-3".to_string(), None, Some(1)).unwrap();
assert_eq!(chosen, engine_1);
}
@@ -532,11 +717,11 @@ mod tests {
]);
// Load-balance: first two go to engine_0 and engine_1.
registry.register("req-lb-0".to_string(), None).unwrap();
registry.register("req-lb-0".to_string(), None, None).unwrap();
// Now engine_0 has 1 in-flight. Without dp_rank, next would go to engine_1.
// But with dp_rank=0, it should still go to engine_0.
let (chosen, _) = registry.register("req-dp".to_string(), Some(0)).unwrap();
let (chosen, _) = registry.register("req-dp".to_string(), None, Some(0)).unwrap();
assert_eq!(chosen, engine_0);
}
@@ -547,7 +732,7 @@ mod tests {
connected_engine(EngineId::from_engine_index(1)),
]);
let error = registry.register("req-1".to_string(), Some(2)).unwrap_err();
let error = registry.register("req-1".to_string(), None, Some(2)).unwrap_err();
assert!(matches!(
error,
crate::error::Error::InvalidDataParallelRank {
@@ -562,10 +747,10 @@ mod tests {
let engine_0 = EngineId::from_engine_index(0);
let mut registry = RequestRegistry::new(&[connected_engine(engine_0.clone())]);
let (chosen, _) = registry.register("req-ok".to_string(), Some(0)).unwrap();
let (chosen, _) = registry.register("req-ok".to_string(), None, Some(0)).unwrap();
assert_eq!(chosen, engine_0);
let error = registry.register("req-bad".to_string(), Some(1)).unwrap_err();
let error = registry.register("req-bad".to_string(), None, Some(1)).unwrap_err();
assert!(matches!(
error,
crate::error::Error::InvalidDataParallelRank {
+116 -1
View File
@@ -1,4 +1,10 @@
use vllm_metrics::{EngineLabels, EnginePositionLabels, SchedulerMetrics, WaitingReasonLabels};
use std::collections::BTreeSet;
use std::time::{SystemTime, UNIX_EPOCH};
use vllm_metrics::{
EngineLabels, EnginePositionLabels, LoraAdapterNames, LoraInfoLabels, SchedulerMetrics,
WaitingReasonLabels,
};
use crate::protocol::stats::SchedulerStats;
@@ -129,3 +135,112 @@ pub(crate) fn record_scheduler_stats(
}
}
}
/// Exports `vllm:lora_requests_info` as a single series covering all LoRA
/// requests tracked by this client across every engine in the replica.
///
/// The engine's `SchedulerStats` never carries adapter names: the Python
/// frontend fills them in from per-request lifecycle events tracked by
/// `LoRARequestStates` in `vllm/v1/engine/output_processor.py`. The Rust
/// frontend mirrors that, deriving the sets from the request registry.
#[derive(Default)]
pub(crate) struct LoraInfoExporter {
current: Option<LoraInfoLabels>,
}
impl LoraInfoExporter {
pub(crate) fn update(
&mut self,
metrics: &SchedulerMetrics,
running: BTreeSet<String>,
waiting: BTreeSet<String>,
) {
let next = (!running.is_empty() || !waiting.is_empty()).then_some(LoraInfoLabels {
running_lora_adapters: LoraAdapterNames(running),
waiting_lora_adapters: LoraAdapterNames(waiting),
});
if self.current != next
&& let Some(prev) = &self.current
{
metrics.lora_info.remove(prev);
}
// Python sets this gauge to the current time on every record.
if let Some(labels) = &next {
metrics.lora_info.get_or_create(labels).set(now_unix_secs());
}
self.current = next;
}
}
fn now_unix_secs() -> f64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs_f64())
.unwrap_or(0.0)
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use expect_test::expect;
use vllm_metrics::Metrics;
use crate::metrics::LoraInfoExporter;
fn names(values: &[&str]) -> BTreeSet<String> {
values.iter().map(|name| (*name).to_string()).collect()
}
/// The `lora_requests_info` series with the non-deterministic timestamp
/// value replaced by `<ts>`, one line per series.
fn lora_series(rendered: &str) -> String {
rendered
.lines()
.filter(|l| l.starts_with("vllm:lora_requests_info{"))
.map(|l| match l.rsplit_once("} ") {
Some((labels, _value)) => format!("{labels}}} <ts>"),
None => l.to_string(),
})
.collect::<Vec<_>>()
.join("\n")
}
#[test]
fn lora_info_emits_clears_stale_and_drains() {
let metrics = Metrics::new();
let mut exporter = LoraInfoExporter::default();
// No adapters: nothing emitted.
exporter.update(&metrics.scheduler, names(&[]), names(&[]));
expect![[""]].assert_eq(&lora_series(&metrics.render().unwrap()));
// Two running (sorted), one waiting.
exporter.update(&metrics.scheduler, names(&["b", "a"]), names(&["c"]));
expect![[
r#"vllm:lora_requests_info{running_lora_adapters="a,b",waiting_lora_adapters="c"} <ts>"#
]]
.assert_eq(&lora_series(&metrics.render().unwrap()));
// "c" gets scheduled and "d" arrives: the stale series is replaced.
exporter.update(&metrics.scheduler, names(&["a", "b", "c"]), names(&["d"]));
expect![[
r#"vllm:lora_requests_info{running_lora_adapters="a,b,c",waiting_lora_adapters="d"} <ts>"#
]]
.assert_eq(&lora_series(&metrics.render().unwrap()));
// Everything but "d" finishes.
exporter.update(&metrics.scheduler, names(&["d"]), names(&[]));
expect![[
r#"vllm:lora_requests_info{running_lora_adapters="d",waiting_lora_adapters=""} <ts>"#
]]
.assert_eq(&lora_series(&metrics.render().unwrap()));
// All requests done: series removed entirely.
exporter.update(&metrics.scheduler, names(&[]), names(&[]));
expect![[""]].assert_eq(&lora_series(&metrics.render().unwrap()));
}
}
@@ -181,10 +181,6 @@ pub struct SchedulerStats {
pub spec_decoding_stats: Option<SpecDecodingStats>,
/// Connector-specific KV transfer stats, kept opaque for now.
pub kv_connector_stats: Option<BTreeMap<String, OpaqueValue>>,
/// Waiting request counts per LoRA adapter.
pub waiting_lora_adapters: BTreeMap<String, u64>,
/// Running request counts per LoRA adapter.
pub running_lora_adapters: BTreeMap<String, u64>,
/// CUDA graph runtime stats when graph metrics are enabled.
pub cudagraph_stats: Option<CudagraphStat>,
/// Estimated MFU/performance stats, when enabled.
+1
View File
@@ -5,6 +5,7 @@ edition.workspace = true
license.workspace = true
[dependencies]
itertools.workspace = true
prometheus-client.workspace = true
[lints]
+33 -1
View File
@@ -1,4 +1,7 @@
use prometheus_client::encoding::EncodeLabelSet;
use std::collections::BTreeSet;
use itertools::Itertools as _;
use prometheus_client::encoding::{EncodeLabelSet, EncodeLabelValue, LabelValueEncoder};
use prometheus_client::metrics::family::Family;
use prometheus_client::metrics::histogram::Histogram;
use prometheus_client::registry::Registry;
@@ -42,6 +45,23 @@ pub struct WaitingReasonLabels {
pub reason: &'static str,
}
/// Adapter names encoded as a deterministic comma-joined Prometheus label value.
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct LoraAdapterNames(pub BTreeSet<String>);
impl EncodeLabelValue for LoraAdapterNames {
fn encode(&self, encoder: &mut LabelValueEncoder) -> Result<(), std::fmt::Error> {
EncodeLabelValue::encode(&self.0.iter().join(","), encoder)
}
}
/// Labels for `vllm:lora_requests_info`.
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct LoraInfoLabels {
pub running_lora_adapters: LoraAdapterNames,
pub waiting_lora_adapters: LoraAdapterNames,
}
/// Scheduler/batch-scoped Prometheus families exported from `SchedulerStats`.
pub struct SchedulerMetrics {
// Scheduler state gauges.
@@ -50,6 +70,10 @@ pub struct SchedulerMetrics {
pub scheduler_waiting_by_reason: Family<WaitingReasonLabels, U64Gauge>,
pub kv_cache_usage: Family<EngineLabels, F64Gauge>,
/// `vllm:lora_requests_info`. Value is the emit-time unix timestamp in
/// seconds.
pub lora_info: Family<LoraInfoLabels, F64Gauge>,
// Prefix-cache counters, including the connector-backed external cache path.
pub prefix_cache_queries: Family<EngineLabels, U64Counter>,
pub prefix_cache_hits: Family<EngineLabels, U64Counter>,
@@ -109,6 +133,13 @@ impl SchedulerMetrics {
kv_cache_usage.clone(),
);
let lora_info = Family::default();
registry.register(
"vllm:lora_requests_info",
"Running stats on lora requests.",
lora_info.clone(),
);
// Prefix-cache counters, including the connector-backed external cache path.
let prefix_cache_queries = Family::default();
registry.register(
@@ -219,6 +250,7 @@ impl SchedulerMetrics {
scheduler_waiting,
scheduler_waiting_by_reason,
kv_cache_usage,
lora_info,
prefix_cache_queries,
prefix_cache_hits,
external_prefix_cache_queries,