[Rust Frontend][Metrics] Export vllm:lora_requests_info from frontend (#45030)
Signed-off-by: Will Eaton <weaton@redhat.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Bugen Zhao
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ebc6ef971a
commit
cc640ee8bc
@@ -442,9 +442,10 @@ impl EngineCoreClient {
|
||||
);
|
||||
|
||||
let request_id = req.request_id.clone();
|
||||
let lora_name = req.lora_request.as_ref().map(|lora| lora.lora_name.clone());
|
||||
let data_parallel_rank = req.data_parallel_rank;
|
||||
let (engine_id, rx) =
|
||||
self.inner.register_request(request_id.clone(), data_parallel_rank)?;
|
||||
self.inner.register_request(request_id.clone(), lora_name, data_parallel_rank)?;
|
||||
|
||||
let result: Result<()> = async {
|
||||
if let Some(coordinator) = self.coordinator.as_ref() {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::Arc;
|
||||
|
||||
use arc_swap::ArcSwapOption;
|
||||
@@ -13,7 +13,7 @@ use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, Uti
|
||||
use crate::client::stream::EngineCoreStreamOutput;
|
||||
use crate::client::{AbortCause, AbortRequest};
|
||||
use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
|
||||
use crate::metrics::record_scheduler_stats;
|
||||
use crate::metrics::{LoraInfoExporter, record_scheduler_stats};
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::protocol::utility::UtilityOutput;
|
||||
use crate::protocol::{
|
||||
@@ -58,17 +58,19 @@ impl ClientInner {
|
||||
/// per-request output channel bound to its `request_id`.
|
||||
///
|
||||
/// When `data_parallel_rank` is provided, the request is routed to that
|
||||
/// specific engine rank, bypassing load balancing.
|
||||
/// specific engine rank, bypassing load balancing. `lora_name` is the
|
||||
/// request's LoRA adapter, tracked for `vllm:lora_requests_info`.
|
||||
pub fn register_request(
|
||||
&self,
|
||||
request_id: String,
|
||||
lora_name: Option<String>,
|
||||
data_parallel_rank: Option<u32>,
|
||||
) -> Result<(EngineId, OutputReceiver)> {
|
||||
let mut registry = self.request_reg.lock();
|
||||
if registry.is_closed() {
|
||||
return Err(self.closed_error());
|
||||
}
|
||||
registry.register(request_id, data_parallel_rank)
|
||||
registry.register(request_id, lora_name, data_parallel_rank)
|
||||
}
|
||||
|
||||
/// Allocate the next utility `call_id` and register its waiting receiver.
|
||||
@@ -131,6 +133,12 @@ impl ClientInner {
|
||||
self.request_reg.lock().apply_scheduler_stats(engine_index, stats)
|
||||
}
|
||||
|
||||
/// Snapshot the adapter names of tracked LoRA requests as
|
||||
/// (running, waiting) sets.
|
||||
pub fn lora_adapter_states(&self) -> (BTreeSet<String>, BTreeSet<String>) {
|
||||
self.request_reg.lock().lora_adapter_states()
|
||||
}
|
||||
|
||||
/// Close all active request streams and utility calls with the first
|
||||
/// persistent health error.
|
||||
pub fn close_registries(&self, error: Arc<Error>) {
|
||||
@@ -303,6 +311,8 @@ pub(crate) async fn run_output_dispatcher_loop(
|
||||
inner: Arc<ClientInner>,
|
||||
mut output_rx: mpsc::Receiver<Result<EngineCoreOutputs>>,
|
||||
) {
|
||||
let mut lora_info = LoraInfoExporter::default();
|
||||
|
||||
let result: Result<()> = async {
|
||||
loop {
|
||||
let outputs = match output_rx.recv().await {
|
||||
@@ -357,6 +367,12 @@ pub(crate) async fn run_output_dispatcher_loop(
|
||||
scheduler_stats,
|
||||
);
|
||||
}
|
||||
|
||||
// The engine's scheduler stats never carry adapter names;
|
||||
// the gauge is derived from the registry's frontend-side
|
||||
// request tracking instead.
|
||||
let (running, waiting) = inner.lora_adapter_states();
|
||||
lora_info.update(&METRICS.scheduler, running, waiting);
|
||||
}
|
||||
ClassifiedEngineCoreOutputs::Utility(utility) => {
|
||||
let call_id = utility.output.call_id;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
@@ -7,9 +7,9 @@ use tracing::trace;
|
||||
use crate::EngineId;
|
||||
use crate::client::stream::EngineCoreStreamOutput;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::protocol::EngineCoreOutput;
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::protocol::utility::UtilityOutput;
|
||||
use crate::protocol::{EngineCoreEventType, EngineCoreOutput};
|
||||
use crate::transport::ConnectedEngine;
|
||||
|
||||
pub type OutputSender = mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>;
|
||||
@@ -21,6 +21,25 @@ pub type UtilityReceiver = oneshot::Receiver<Result<UtilityOutput>>;
|
||||
struct TrackedRequest {
|
||||
sender: OutputSender,
|
||||
engine_id: EngineId,
|
||||
lora: Option<LoraRequestState>,
|
||||
}
|
||||
|
||||
/// Frontend-side view of one LoRA request's scheduling phase.
|
||||
///
|
||||
/// The engine's `SchedulerStats` does not carry adapter names, so
|
||||
/// `vllm:lora_requests_info` must be derived from per-request lifecycle events
|
||||
/// observed by this client, mirroring `LoRARequestStates` in the Python
|
||||
/// frontend (`vllm/v1/engine/output_processor.py`).
|
||||
#[derive(Debug)]
|
||||
struct LoraRequestState {
|
||||
adapter_name: String,
|
||||
phase: LoraPhase,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum LoraPhase {
|
||||
Waiting,
|
||||
Running,
|
||||
}
|
||||
|
||||
/// The latest real scheduler-side load snapshot observed from one engine.
|
||||
@@ -105,6 +124,7 @@ impl RequestRegistry {
|
||||
pub fn register(
|
||||
&mut self,
|
||||
request_id: String,
|
||||
lora_name: Option<String>,
|
||||
data_parallel_rank: Option<u32>,
|
||||
) -> Result<(EngineId, OutputReceiver)> {
|
||||
if self.requests.contains_key(&request_id) {
|
||||
@@ -118,6 +138,10 @@ impl RequestRegistry {
|
||||
TrackedRequest {
|
||||
sender: tx,
|
||||
engine_id: engine_id.clone(),
|
||||
lora: lora_name.map(|adapter_name| LoraRequestState {
|
||||
adapter_name,
|
||||
phase: LoraPhase::Waiting,
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
@@ -171,6 +195,7 @@ impl RequestRegistry {
|
||||
/// Obtain the stream sender for one output. If it indicates the request is
|
||||
/// finished, it will be removed from the registry.
|
||||
pub fn sender_for_output(&mut self, output: &EngineCoreOutput) -> Option<OutputSender> {
|
||||
self.apply_lora_events(output);
|
||||
if output.finished() {
|
||||
self.remove(output.request_id.as_str()).map(|tracked| tracked.0)
|
||||
} else {
|
||||
@@ -180,6 +205,43 @@ impl RequestRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
/// Advance the request's LoRA scheduling phase from the engine-core events
|
||||
/// attached to one output, mirroring the Python frontend's
|
||||
/// `LoRARequestStates.update_from_events`.
|
||||
fn apply_lora_events(&mut self, output: &EngineCoreOutput) {
|
||||
let Some(events) = output.events.as_ref() else {
|
||||
return;
|
||||
};
|
||||
let Some(lora) = self
|
||||
.requests
|
||||
.get_mut(output.request_id.as_str())
|
||||
.and_then(|tracked| tracked.lora.as_mut())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
for event in events {
|
||||
lora.phase = match event.r#type {
|
||||
EngineCoreEventType::Queued | EngineCoreEventType::Preempted => LoraPhase::Waiting,
|
||||
EngineCoreEventType::Scheduled => LoraPhase::Running,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot the adapter names of tracked LoRA requests as
|
||||
/// (running, waiting) sets. Feeds the `vllm:lora_requests_info` gauge.
|
||||
pub fn lora_adapter_states(&self) -> (BTreeSet<String>, BTreeSet<String>) {
|
||||
let mut running = BTreeSet::new();
|
||||
let mut waiting = BTreeSet::new();
|
||||
for lora in self.requests.values().filter_map(|tracked| tracked.lora.as_ref()) {
|
||||
let set = match lora.phase {
|
||||
LoraPhase::Running => &mut running,
|
||||
LoraPhase::Waiting => &mut waiting,
|
||||
};
|
||||
set.insert(lora.adapter_name.clone());
|
||||
}
|
||||
(running, waiting)
|
||||
}
|
||||
|
||||
/// Obtain stream senders for a whole engine output batch under one
|
||||
/// registry lock. Finished outputs are removed before returning.
|
||||
pub fn senders_for_outputs<'a>(
|
||||
@@ -336,11 +398,16 @@ impl UtilityRegistry {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{EngineRoutingState, RequestRegistry, UtilityRegistry};
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use crate::EngineId;
|
||||
use crate::client::state::EngineLoadSnapshot;
|
||||
use crate::client::state::{
|
||||
EngineLoadSnapshot, EngineRoutingState, RequestRegistry, UtilityRegistry,
|
||||
};
|
||||
use crate::mock_engine::default_ready_response;
|
||||
use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput};
|
||||
use crate::protocol::{
|
||||
EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
|
||||
};
|
||||
use crate::transport::ConnectedEngine;
|
||||
|
||||
fn connected_engine(engine_id: EngineId) -> ConnectedEngine {
|
||||
@@ -350,11 +417,36 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn output_with_events(
|
||||
request_id: &str,
|
||||
events: &[EngineCoreEventType],
|
||||
finish_reason: Option<EngineCoreFinishReason>,
|
||||
) -> EngineCoreOutput {
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
events: Some(
|
||||
events
|
||||
.iter()
|
||||
.map(|event_type| EngineCoreEvent {
|
||||
r#type: *event_type,
|
||||
timestamp: 0.0,
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
finish_reason,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn adapter_names(values: &[&str]) -> BTreeSet<String> {
|
||||
values.iter().map(|name| (*name).to_string()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_rejects_duplicate_request_ids() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
registry.register("req-1".to_string(), None).unwrap();
|
||||
let error = registry.register("req-1".to_string(), None).unwrap_err();
|
||||
registry.register("req-1".to_string(), None, None).unwrap();
|
||||
let error = registry.register("req-1".to_string(), None, None).unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::error::Error::DuplicateRequestId { request_id } if request_id == "req-1"
|
||||
@@ -364,7 +456,7 @@ mod tests {
|
||||
#[test]
|
||||
fn registry_removes_finished_request_on_output() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
registry.register("req-1".to_string(), None).unwrap();
|
||||
registry.register("req-1".to_string(), None, None).unwrap();
|
||||
|
||||
let sender = registry.sender_for_output(&EngineCoreOutput {
|
||||
request_id: "req-1".to_string(),
|
||||
@@ -376,11 +468,104 @@ mod tests {
|
||||
assert!(!registry.contains("req-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_tracks_lora_phases_from_engine_events() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
registry
|
||||
.register("req-lora".to_string(), Some("adapter-a".to_string()), None)
|
||||
.unwrap();
|
||||
registry.register("req-plain".to_string(), None, None).unwrap();
|
||||
|
||||
// Registered but not yet scheduled: counted as waiting. The non-LoRA
|
||||
// request never shows up.
|
||||
assert_eq!(
|
||||
registry.lora_adapter_states(),
|
||||
(adapter_names(&[]), adapter_names(&["adapter-a"]))
|
||||
);
|
||||
|
||||
// Queued then scheduled in one output: running.
|
||||
drop(registry.sender_for_output(&output_with_events(
|
||||
"req-lora",
|
||||
&[EngineCoreEventType::Queued, EngineCoreEventType::Scheduled],
|
||||
None,
|
||||
)));
|
||||
assert_eq!(
|
||||
registry.lora_adapter_states(),
|
||||
(adapter_names(&["adapter-a"]), adapter_names(&[]))
|
||||
);
|
||||
|
||||
// Preempted: back to waiting.
|
||||
drop(registry.sender_for_output(&output_with_events(
|
||||
"req-lora",
|
||||
&[EngineCoreEventType::Preempted],
|
||||
None,
|
||||
)));
|
||||
assert_eq!(
|
||||
registry.lora_adapter_states(),
|
||||
(adapter_names(&[]), adapter_names(&["adapter-a"]))
|
||||
);
|
||||
|
||||
// Finished: dropped from tracking entirely.
|
||||
drop(registry.sender_for_output(&output_with_events(
|
||||
"req-lora",
|
||||
&[EngineCoreEventType::Scheduled],
|
||||
Some(EngineCoreFinishReason::Stop),
|
||||
)));
|
||||
assert_eq!(
|
||||
registry.lora_adapter_states(),
|
||||
(adapter_names(&[]), adapter_names(&[]))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_unions_lora_adapters_across_requests() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
registry
|
||||
.register("req-a1".to_string(), Some("adapter-a".to_string()), None)
|
||||
.unwrap();
|
||||
registry
|
||||
.register("req-a2".to_string(), Some("adapter-a".to_string()), None)
|
||||
.unwrap();
|
||||
registry
|
||||
.register("req-b".to_string(), Some("adapter-b".to_string()), None)
|
||||
.unwrap();
|
||||
|
||||
// One of adapter-a's requests starts running while the other waits:
|
||||
// the adapter appears in both sets.
|
||||
drop(registry.sender_for_output(&output_with_events(
|
||||
"req-a1",
|
||||
&[EngineCoreEventType::Scheduled],
|
||||
None,
|
||||
)));
|
||||
assert_eq!(
|
||||
registry.lora_adapter_states(),
|
||||
(
|
||||
adapter_names(&["adapter-a"]),
|
||||
adapter_names(&["adapter-a", "adapter-b"])
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_drops_lora_tracking_on_abort() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
registry
|
||||
.register("req-lora".to_string(), Some("adapter-a".to_string()), None)
|
||||
.unwrap();
|
||||
|
||||
drop(registry.finish_many(&["req-lora".to_string()]));
|
||||
|
||||
assert_eq!(
|
||||
registry.lora_adapter_states(),
|
||||
(adapter_names(&[]), adapter_names(&[]))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_closes_all_requests_on_failure() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
registry.register("req-1".to_string(), None).unwrap();
|
||||
registry.register("req-2".to_string(), None).unwrap();
|
||||
registry.register("req-1".to_string(), None, None).unwrap();
|
||||
registry.register("req-2".to_string(), None, None).unwrap();
|
||||
|
||||
let senders = registry.close();
|
||||
|
||||
@@ -396,9 +581,9 @@ mod tests {
|
||||
connected_engine(engine_0.clone()),
|
||||
connected_engine(engine_1.clone()),
|
||||
]);
|
||||
let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
|
||||
let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap();
|
||||
let (chosen_0_again, _) = registry.register("req-3".to_string(), None).unwrap();
|
||||
let (chosen_0, _) = registry.register("req-1".to_string(), None, None).unwrap();
|
||||
let (chosen_1, _) = registry.register("req-2".to_string(), None, None).unwrap();
|
||||
let (chosen_0_again, _) = registry.register("req-3".to_string(), None, None).unwrap();
|
||||
|
||||
assert_eq!(chosen_0, engine_0);
|
||||
assert_eq!(chosen_1, engine_1);
|
||||
@@ -425,9 +610,9 @@ mod tests {
|
||||
connected_engine(engine_1.clone()),
|
||||
]);
|
||||
|
||||
let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
|
||||
let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap();
|
||||
let (chosen_0_again, _) = registry.register("req-3".to_string(), None).unwrap();
|
||||
let (chosen_0, _) = registry.register("req-1".to_string(), None, None).unwrap();
|
||||
let (chosen_1, _) = registry.register("req-2".to_string(), None, None).unwrap();
|
||||
let (chosen_0_again, _) = registry.register("req-3".to_string(), None, None).unwrap();
|
||||
|
||||
assert_eq!(chosen_0, engine_0);
|
||||
assert_eq!(chosen_1, engine_1);
|
||||
@@ -494,7 +679,7 @@ mod tests {
|
||||
}
|
||||
));
|
||||
|
||||
let (chosen, _) = registry.register("req-stats".to_string(), None).unwrap();
|
||||
let (chosen, _) = registry.register("req-stats".to_string(), None, None).unwrap();
|
||||
assert_eq!(chosen, engine_1);
|
||||
}
|
||||
|
||||
@@ -510,15 +695,15 @@ mod tests {
|
||||
]);
|
||||
|
||||
// Explicitly target rank 2 (third engine).
|
||||
let (chosen, _) = registry.register("req-1".to_string(), Some(2)).unwrap();
|
||||
let (chosen, _) = registry.register("req-1".to_string(), None, Some(2)).unwrap();
|
||||
assert_eq!(chosen, engine_2);
|
||||
|
||||
// Explicitly target rank 0 (first engine).
|
||||
let (chosen, _) = registry.register("req-2".to_string(), Some(0)).unwrap();
|
||||
let (chosen, _) = registry.register("req-2".to_string(), None, Some(0)).unwrap();
|
||||
assert_eq!(chosen, engine_0);
|
||||
|
||||
// Explicitly target rank 1.
|
||||
let (chosen, _) = registry.register("req-3".to_string(), Some(1)).unwrap();
|
||||
let (chosen, _) = registry.register("req-3".to_string(), None, Some(1)).unwrap();
|
||||
assert_eq!(chosen, engine_1);
|
||||
}
|
||||
|
||||
@@ -532,11 +717,11 @@ mod tests {
|
||||
]);
|
||||
|
||||
// Load-balance: first two go to engine_0 and engine_1.
|
||||
registry.register("req-lb-0".to_string(), None).unwrap();
|
||||
registry.register("req-lb-0".to_string(), None, None).unwrap();
|
||||
|
||||
// Now engine_0 has 1 in-flight. Without dp_rank, next would go to engine_1.
|
||||
// But with dp_rank=0, it should still go to engine_0.
|
||||
let (chosen, _) = registry.register("req-dp".to_string(), Some(0)).unwrap();
|
||||
let (chosen, _) = registry.register("req-dp".to_string(), None, Some(0)).unwrap();
|
||||
assert_eq!(chosen, engine_0);
|
||||
}
|
||||
|
||||
@@ -547,7 +732,7 @@ mod tests {
|
||||
connected_engine(EngineId::from_engine_index(1)),
|
||||
]);
|
||||
|
||||
let error = registry.register("req-1".to_string(), Some(2)).unwrap_err();
|
||||
let error = registry.register("req-1".to_string(), None, Some(2)).unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::error::Error::InvalidDataParallelRank {
|
||||
@@ -562,10 +747,10 @@ mod tests {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(engine_0.clone())]);
|
||||
|
||||
let (chosen, _) = registry.register("req-ok".to_string(), Some(0)).unwrap();
|
||||
let (chosen, _) = registry.register("req-ok".to_string(), None, Some(0)).unwrap();
|
||||
assert_eq!(chosen, engine_0);
|
||||
|
||||
let error = registry.register("req-bad".to_string(), Some(1)).unwrap_err();
|
||||
let error = registry.register("req-bad".to_string(), None, Some(1)).unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::error::Error::InvalidDataParallelRank {
|
||||
|
||||
@@ -1,4 +1,10 @@
|
||||
use vllm_metrics::{EngineLabels, EnginePositionLabels, SchedulerMetrics, WaitingReasonLabels};
|
||||
use std::collections::BTreeSet;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use vllm_metrics::{
|
||||
EngineLabels, EnginePositionLabels, LoraAdapterNames, LoraInfoLabels, SchedulerMetrics,
|
||||
WaitingReasonLabels,
|
||||
};
|
||||
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
|
||||
@@ -129,3 +135,112 @@ pub(crate) fn record_scheduler_stats(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Exports `vllm:lora_requests_info` as a single series covering all LoRA
|
||||
/// requests tracked by this client across every engine in the replica.
|
||||
///
|
||||
/// The engine's `SchedulerStats` never carries adapter names: the Python
|
||||
/// frontend fills them in from per-request lifecycle events tracked by
|
||||
/// `LoRARequestStates` in `vllm/v1/engine/output_processor.py`. The Rust
|
||||
/// frontend mirrors that, deriving the sets from the request registry.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct LoraInfoExporter {
|
||||
current: Option<LoraInfoLabels>,
|
||||
}
|
||||
|
||||
impl LoraInfoExporter {
|
||||
pub(crate) fn update(
|
||||
&mut self,
|
||||
metrics: &SchedulerMetrics,
|
||||
running: BTreeSet<String>,
|
||||
waiting: BTreeSet<String>,
|
||||
) {
|
||||
let next = (!running.is_empty() || !waiting.is_empty()).then_some(LoraInfoLabels {
|
||||
running_lora_adapters: LoraAdapterNames(running),
|
||||
waiting_lora_adapters: LoraAdapterNames(waiting),
|
||||
});
|
||||
|
||||
if self.current != next
|
||||
&& let Some(prev) = &self.current
|
||||
{
|
||||
metrics.lora_info.remove(prev);
|
||||
}
|
||||
|
||||
// Python sets this gauge to the current time on every record.
|
||||
if let Some(labels) = &next {
|
||||
metrics.lora_info.get_or_create(labels).set(now_unix_secs());
|
||||
}
|
||||
|
||||
self.current = next;
|
||||
}
|
||||
}
|
||||
|
||||
fn now_unix_secs() -> f64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs_f64())
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use expect_test::expect;
|
||||
use vllm_metrics::Metrics;
|
||||
|
||||
use crate::metrics::LoraInfoExporter;
|
||||
|
||||
fn names(values: &[&str]) -> BTreeSet<String> {
|
||||
values.iter().map(|name| (*name).to_string()).collect()
|
||||
}
|
||||
|
||||
/// The `lora_requests_info` series with the non-deterministic timestamp
|
||||
/// value replaced by `<ts>`, one line per series.
|
||||
fn lora_series(rendered: &str) -> String {
|
||||
rendered
|
||||
.lines()
|
||||
.filter(|l| l.starts_with("vllm:lora_requests_info{"))
|
||||
.map(|l| match l.rsplit_once("} ") {
|
||||
Some((labels, _value)) => format!("{labels}}} <ts>"),
|
||||
None => l.to_string(),
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lora_info_emits_clears_stale_and_drains() {
|
||||
let metrics = Metrics::new();
|
||||
let mut exporter = LoraInfoExporter::default();
|
||||
|
||||
// No adapters: nothing emitted.
|
||||
exporter.update(&metrics.scheduler, names(&[]), names(&[]));
|
||||
expect![[""]].assert_eq(&lora_series(&metrics.render().unwrap()));
|
||||
|
||||
// Two running (sorted), one waiting.
|
||||
exporter.update(&metrics.scheduler, names(&["b", "a"]), names(&["c"]));
|
||||
expect![[
|
||||
r#"vllm:lora_requests_info{running_lora_adapters="a,b",waiting_lora_adapters="c"} <ts>"#
|
||||
]]
|
||||
.assert_eq(&lora_series(&metrics.render().unwrap()));
|
||||
|
||||
// "c" gets scheduled and "d" arrives: the stale series is replaced.
|
||||
exporter.update(&metrics.scheduler, names(&["a", "b", "c"]), names(&["d"]));
|
||||
expect![[
|
||||
r#"vllm:lora_requests_info{running_lora_adapters="a,b,c",waiting_lora_adapters="d"} <ts>"#
|
||||
]]
|
||||
.assert_eq(&lora_series(&metrics.render().unwrap()));
|
||||
|
||||
// Everything but "d" finishes.
|
||||
exporter.update(&metrics.scheduler, names(&["d"]), names(&[]));
|
||||
expect![[
|
||||
r#"vllm:lora_requests_info{running_lora_adapters="d",waiting_lora_adapters=""} <ts>"#
|
||||
]]
|
||||
.assert_eq(&lora_series(&metrics.render().unwrap()));
|
||||
|
||||
// All requests done: series removed entirely.
|
||||
exporter.update(&metrics.scheduler, names(&[]), names(&[]));
|
||||
expect![[""]].assert_eq(&lora_series(&metrics.render().unwrap()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,10 +181,6 @@ pub struct SchedulerStats {
|
||||
pub spec_decoding_stats: Option<SpecDecodingStats>,
|
||||
/// Connector-specific KV transfer stats, kept opaque for now.
|
||||
pub kv_connector_stats: Option<BTreeMap<String, OpaqueValue>>,
|
||||
/// Waiting request counts per LoRA adapter.
|
||||
pub waiting_lora_adapters: BTreeMap<String, u64>,
|
||||
/// Running request counts per LoRA adapter.
|
||||
pub running_lora_adapters: BTreeMap<String, u64>,
|
||||
/// CUDA graph runtime stats when graph metrics are enabled.
|
||||
pub cudagraph_stats: Option<CudagraphStat>,
|
||||
/// Estimated MFU/performance stats, when enabled.
|
||||
|
||||
@@ -5,6 +5,7 @@ edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
itertools.workspace = true
|
||||
prometheus-client.workspace = true
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
use prometheus_client::encoding::EncodeLabelSet;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use itertools::Itertools as _;
|
||||
use prometheus_client::encoding::{EncodeLabelSet, EncodeLabelValue, LabelValueEncoder};
|
||||
use prometheus_client::metrics::family::Family;
|
||||
use prometheus_client::metrics::histogram::Histogram;
|
||||
use prometheus_client::registry::Registry;
|
||||
@@ -42,6 +45,23 @@ pub struct WaitingReasonLabels {
|
||||
pub reason: &'static str,
|
||||
}
|
||||
|
||||
/// Adapter names encoded as a deterministic comma-joined Prometheus label value.
|
||||
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
|
||||
pub struct LoraAdapterNames(pub BTreeSet<String>);
|
||||
|
||||
impl EncodeLabelValue for LoraAdapterNames {
|
||||
fn encode(&self, encoder: &mut LabelValueEncoder) -> Result<(), std::fmt::Error> {
|
||||
EncodeLabelValue::encode(&self.0.iter().join(","), encoder)
|
||||
}
|
||||
}
|
||||
|
||||
/// Labels for `vllm:lora_requests_info`.
|
||||
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
|
||||
pub struct LoraInfoLabels {
|
||||
pub running_lora_adapters: LoraAdapterNames,
|
||||
pub waiting_lora_adapters: LoraAdapterNames,
|
||||
}
|
||||
|
||||
/// Scheduler/batch-scoped Prometheus families exported from `SchedulerStats`.
|
||||
pub struct SchedulerMetrics {
|
||||
// Scheduler state gauges.
|
||||
@@ -50,6 +70,10 @@ pub struct SchedulerMetrics {
|
||||
pub scheduler_waiting_by_reason: Family<WaitingReasonLabels, U64Gauge>,
|
||||
pub kv_cache_usage: Family<EngineLabels, F64Gauge>,
|
||||
|
||||
/// `vllm:lora_requests_info`. Value is the emit-time unix timestamp in
|
||||
/// seconds.
|
||||
pub lora_info: Family<LoraInfoLabels, F64Gauge>,
|
||||
|
||||
// Prefix-cache counters, including the connector-backed external cache path.
|
||||
pub prefix_cache_queries: Family<EngineLabels, U64Counter>,
|
||||
pub prefix_cache_hits: Family<EngineLabels, U64Counter>,
|
||||
@@ -109,6 +133,13 @@ impl SchedulerMetrics {
|
||||
kv_cache_usage.clone(),
|
||||
);
|
||||
|
||||
let lora_info = Family::default();
|
||||
registry.register(
|
||||
"vllm:lora_requests_info",
|
||||
"Running stats on lora requests.",
|
||||
lora_info.clone(),
|
||||
);
|
||||
|
||||
// Prefix-cache counters, including the connector-backed external cache path.
|
||||
let prefix_cache_queries = Family::default();
|
||||
registry.register(
|
||||
@@ -219,6 +250,7 @@ impl SchedulerMetrics {
|
||||
scheduler_waiting,
|
||||
scheduler_waiting_by_reason,
|
||||
kv_cache_usage,
|
||||
lora_info,
|
||||
prefix_cache_queries,
|
||||
prefix_cache_hits,
|
||||
external_prefix_cache_queries,
|
||||
|
||||
Reference in New Issue
Block a user