[Rust Frontend] Avoid LoRA registry scans without active LoRA requests (#47040)
Signed-off-by: reidliu41 <reid201711@gmail.com>
This commit is contained in:
@@ -100,6 +100,7 @@ impl EngineRoutingState {
|
||||
pub struct RequestRegistry {
|
||||
closed: bool,
|
||||
requests: HashMap<String, TrackedRequest>,
|
||||
active_lora_requests: usize,
|
||||
routing_per_engine: BTreeMap<EngineId, EngineRoutingState>,
|
||||
}
|
||||
|
||||
@@ -108,6 +109,7 @@ impl RequestRegistry {
|
||||
Self {
|
||||
closed: false,
|
||||
requests: HashMap::default(),
|
||||
active_lora_requests: 0,
|
||||
routing_per_engine: engines
|
||||
.iter()
|
||||
.map(|engine| (engine.engine_id.clone(), EngineRoutingState::default()))
|
||||
@@ -133,15 +135,19 @@ impl RequestRegistry {
|
||||
|
||||
let engine_id = self.choose_engine_for_request(data_parallel_rank)?;
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
let lora = lora_name.map(|adapter_name| LoraRequestState {
|
||||
adapter_name,
|
||||
phase: LoraPhase::Waiting,
|
||||
});
|
||||
if lora.is_some() {
|
||||
self.active_lora_requests += 1;
|
||||
}
|
||||
self.requests.insert(
|
||||
request_id,
|
||||
TrackedRequest {
|
||||
sender: tx,
|
||||
engine_id: engine_id.clone(),
|
||||
lora: lora_name.map(|adapter_name| LoraRequestState {
|
||||
adapter_name,
|
||||
phase: LoraPhase::Waiting,
|
||||
}),
|
||||
lora,
|
||||
},
|
||||
);
|
||||
|
||||
@@ -230,6 +236,10 @@ impl RequestRegistry {
|
||||
/// Snapshot the adapter names of tracked LoRA requests as
|
||||
/// (running, waiting) sets. Feeds the `vllm:lora_requests_info` gauge.
|
||||
pub fn lora_adapter_states(&self) -> (BTreeSet<String>, BTreeSet<String>) {
|
||||
if self.active_lora_requests == 0 {
|
||||
return (BTreeSet::new(), BTreeSet::new());
|
||||
}
|
||||
|
||||
let mut running = BTreeSet::new();
|
||||
let mut waiting = BTreeSet::new();
|
||||
for lora in self.requests.values().filter_map(|tracked| tracked.lora.as_ref()) {
|
||||
@@ -283,6 +293,7 @@ impl RequestRegistry {
|
||||
}
|
||||
|
||||
self.closed = true;
|
||||
self.active_lora_requests = 0;
|
||||
std::mem::take(&mut self.requests)
|
||||
.into_values()
|
||||
.map(|tracked| tracked.sender)
|
||||
@@ -322,6 +333,9 @@ impl RequestRegistry {
|
||||
#[must_use]
|
||||
pub fn remove(&mut self, request_id: &str) -> Option<(OutputSender, EngineId)> {
|
||||
let tracked = self.requests.remove(request_id)?;
|
||||
if tracked.lora.is_some() {
|
||||
self.active_lora_requests -= 1;
|
||||
}
|
||||
self.routing_per_engine
|
||||
.get_mut(&tracked.engine_id)
|
||||
.expect("request registry must track all known engines")
|
||||
@@ -359,6 +373,11 @@ impl RequestRegistry {
|
||||
pub fn is_closed(&self) -> bool {
|
||||
self.closed
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn active_lora_requests(&self) -> usize {
|
||||
self.active_lora_requests
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal registry for tracking active utility calls and their waiting
|
||||
@@ -574,6 +593,63 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_counts_only_active_lora_requests() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
|
||||
registry.register("req-plain".to_string(), None, None).unwrap();
|
||||
assert_eq!(registry.active_lora_requests(), 0);
|
||||
assert_eq!(
|
||||
registry.lora_adapter_states(),
|
||||
(adapter_names(&[]), adapter_names(&[]))
|
||||
);
|
||||
|
||||
registry
|
||||
.register(
|
||||
"req-lora-a".to_string(),
|
||||
Some("adapter-a".to_string()),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
registry
|
||||
.register(
|
||||
"req-lora-b".to_string(),
|
||||
Some("adapter-b".to_string()),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(registry.active_lora_requests(), 2);
|
||||
|
||||
drop(registry.remove("req-plain"));
|
||||
assert_eq!(registry.active_lora_requests(), 2);
|
||||
|
||||
drop(registry.finish_many(&["req-lora-a".to_string()]));
|
||||
assert_eq!(registry.active_lora_requests(), 1);
|
||||
|
||||
drop(registry.abort_many(&["req-lora-b".to_string()], 0.0));
|
||||
assert_eq!(registry.active_lora_requests(), 0);
|
||||
assert_eq!(
|
||||
registry.lora_adapter_states(),
|
||||
(adapter_names(&[]), adapter_names(&[]))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_clears_lora_count_on_close() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
registry
|
||||
.register("req-lora".to_string(), Some("adapter-a".to_string()), None)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(registry.active_lora_requests(), 1);
|
||||
drop(registry.close());
|
||||
assert_eq!(registry.active_lora_requests(), 0);
|
||||
assert_eq!(
|
||||
registry.lora_adapter_states(),
|
||||
(adapter_names(&[]), adapter_names(&[]))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_drops_lora_tracking_on_abort() {
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
|
||||
|
||||
Reference in New Issue
Block a user