[Bugfix][Rust Frontend] Correct DP routing after request removal (#59631)
Signed-off-by: 0xsensei <prblmslvr.aditya@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Bugen Zhao
Claude Opus 5.5
parent
21445806bd
commit
3b9e462774
@@ -26,6 +26,7 @@ struct TrackedRequest {
|
||||
sender: OutputSender,
|
||||
engine_id: EngineId,
|
||||
lora: Option<LoraRequestState>,
|
||||
admission_stats_generation: Option<u64>,
|
||||
}
|
||||
|
||||
/// Frontend-side view of one LoRA request's scheduling phase.
|
||||
@@ -73,6 +74,8 @@ struct EngineRoutingState {
|
||||
/// Added to the snapshot so each admission raises the routing score before
|
||||
/// the next snapshot arrives. Only tracked once a snapshot exists.
|
||||
admitted_since_stats: usize,
|
||||
/// Identifies admissions made after the latest scheduler snapshot.
|
||||
stats_generation: u64,
|
||||
}
|
||||
|
||||
impl EngineRoutingState {
|
||||
@@ -90,10 +93,21 @@ impl EngineRoutingState {
|
||||
}
|
||||
|
||||
/// Record one request admitted to this engine.
|
||||
fn record_admission(&mut self) {
|
||||
fn record_admission(&mut self) -> Option<u64> {
|
||||
self.inflight += 1;
|
||||
if self.last_scheduler_stats.is_some() {
|
||||
self.admitted_since_stats += 1;
|
||||
return Some(self.stats_generation);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Record one request removed from this engine. Its admission is dropped
|
||||
/// from `admitted_since_stats` only if no newer snapshot has reset that count.
|
||||
fn record_removal(&mut self, admission_stats_generation: Option<u64>) {
|
||||
self.inflight -= 1;
|
||||
if admission_stats_generation == Some(self.stats_generation) {
|
||||
self.admitted_since_stats -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,6 +116,10 @@ impl EngineRoutingState {
|
||||
fn apply_scheduler_counts(&mut self, next: EngineLoadSnapshot) {
|
||||
self.last_scheduler_stats = Some(next);
|
||||
self.admitted_since_stats = 0;
|
||||
self.stats_generation = self
|
||||
.stats_generation
|
||||
.checked_add(1)
|
||||
.expect("scheduler stats generation overflow");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,21 +179,22 @@ impl RequestRegistry {
|
||||
if lora.is_some() {
|
||||
self.active_lora_requests += 1;
|
||||
}
|
||||
let state = self
|
||||
.routing_per_engine
|
||||
.get_mut(&engine_id)
|
||||
.expect("request registry must track all known engines");
|
||||
let admission_stats_generation = state.record_admission();
|
||||
|
||||
self.requests.insert(
|
||||
request_id,
|
||||
TrackedRequest {
|
||||
sender: tx,
|
||||
engine_id: engine_id.clone(),
|
||||
lora,
|
||||
admission_stats_generation,
|
||||
},
|
||||
);
|
||||
|
||||
let state = self
|
||||
.routing_per_engine
|
||||
.get_mut(&engine_id)
|
||||
.expect("request registry must track all known engines");
|
||||
state.record_admission();
|
||||
|
||||
Ok((engine_id, rx))
|
||||
}
|
||||
|
||||
@@ -368,7 +387,7 @@ impl RequestRegistry {
|
||||
self.routing_per_engine
|
||||
.get_mut(&tracked.engine_id)
|
||||
.expect("request registry must track all known engines")
|
||||
.inflight -= 1;
|
||||
.record_removal(tracked.admission_stats_generation);
|
||||
Some((tracked.sender, tracked.engine_id))
|
||||
}
|
||||
|
||||
@@ -1079,6 +1098,80 @@ mod tests {
|
||||
expect_test::expect!["[2, 5, 2]"].assert_eq(&format!("{chosen:?}"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_finished_requests_do_not_route_live_work_to_a_busier_engine() {
|
||||
let mut registry = RequestRegistry::new(
|
||||
&[0, 1].map(|rank| connected_engine(EngineId::from_engine_index(rank))),
|
||||
);
|
||||
registry.apply_scheduler_counts(
|
||||
0,
|
||||
EngineLoadSnapshot {
|
||||
waiting: 0,
|
||||
running: 0,
|
||||
},
|
||||
);
|
||||
registry.apply_scheduler_counts(
|
||||
1,
|
||||
EngineLoadSnapshot {
|
||||
waiting: 0,
|
||||
running: 1,
|
||||
},
|
||||
);
|
||||
|
||||
let mut finished_routes = Vec::new();
|
||||
for request_id in ["finished-1", "finished-2"].map(String::from) {
|
||||
let (engine_id, _) =
|
||||
registry.register(request_id.clone(), None, None, NonZeroU32::MIN).unwrap();
|
||||
|
||||
finished_routes.push(engine_id.engine_index());
|
||||
drop(registry.finish_many([&request_id]));
|
||||
}
|
||||
|
||||
let (live_engine, _) =
|
||||
registry.register("live-request".into(), None, None, NonZeroU32::MIN).unwrap();
|
||||
expect_test::expect!["([Some(0), Some(0)], Some(0))"].assert_eq(&format!(
|
||||
"{:?}",
|
||||
(finished_routes, live_engine.engine_index())
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_removal_keeps_admissions_counted_after_a_newer_snapshot() {
|
||||
let engine = EngineId::from_engine_index(0);
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(engine.clone())]);
|
||||
registry.apply_scheduler_counts(
|
||||
0,
|
||||
EngineLoadSnapshot {
|
||||
waiting: 0,
|
||||
running: 0,
|
||||
},
|
||||
);
|
||||
|
||||
// `old` is admitted before the next snapshot, which already counts it.
|
||||
registry.register("old".into(), None, None, NonZeroU32::MIN).unwrap();
|
||||
registry.apply_scheduler_counts(
|
||||
0,
|
||||
EngineLoadSnapshot {
|
||||
waiting: 0,
|
||||
running: 1,
|
||||
},
|
||||
);
|
||||
// `new` is admitted after that snapshot, so only it is counted on top.
|
||||
registry.register("new".into(), None, None, NonZeroU32::MIN).unwrap();
|
||||
|
||||
// Removing `old` must not take back the admission counted for `new`.
|
||||
drop(registry.remove("old"));
|
||||
let state = ®istry.routing_per_engine[&engine];
|
||||
expect_test::expect!["(1, 1, 2)"].assert_eq(&format!(
|
||||
"{:?}",
|
||||
(
|
||||
state.inflight,
|
||||
state.admitted_since_stats,
|
||||
state.routing_score()
|
||||
)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn register_with_data_parallel_rank_routes_to_specified_engine() {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
|
||||
Reference in New Issue
Block a user