From 3b9e46277407ed53317e78114ad50f1f9bd9d6a4 Mon Sep 17 00:00:00 2001 From: 0xsensei Date: Mon, 12 Oct 2026 16:42:35 +0530 Subject: [PATCH] [Bugfix][Rust Frontend] Correct DP routing after request removal (#59631) Signed-off-by: 0xsensei Signed-off-by: Bugen Zhao Co-authored-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 --- .../engine-core-client/src/client/state.rs | 109 ++++++++++++++++-- 1 file changed, 101 insertions(+), 8 deletions(-) diff --git a/rust/src/engine-core-client/src/client/state.rs b/rust/src/engine-core-client/src/client/state.rs index 9f6f8bbf5c..3d4844f3ad 100644 --- a/rust/src/engine-core-client/src/client/state.rs +++ b/rust/src/engine-core-client/src/client/state.rs @@ -26,6 +26,7 @@ struct TrackedRequest { sender: OutputSender, engine_id: EngineId, lora: Option, + admission_stats_generation: Option, } /// Frontend-side view of one LoRA request's scheduling phase. @@ -73,6 +74,8 @@ struct EngineRoutingState { /// Added to the snapshot so each admission raises the routing score before /// the next snapshot arrives. Only tracked once a snapshot exists. admitted_since_stats: usize, + /// Identifies admissions made after the latest scheduler snapshot. + stats_generation: u64, } impl EngineRoutingState { @@ -90,10 +93,21 @@ impl EngineRoutingState { } /// Record one request admitted to this engine. - fn record_admission(&mut self) { + fn record_admission(&mut self) -> Option { self.inflight += 1; if self.last_scheduler_stats.is_some() { self.admitted_since_stats += 1; + return Some(self.stats_generation); + } + None + } + + /// Record one request removed from this engine. Its admission is dropped + /// from `admitted_since_stats` only if no newer snapshot has reset that count. + fn record_removal(&mut self, admission_stats_generation: Option) { + self.inflight -= 1; + if admission_stats_generation == Some(self.stats_generation) { + self.admitted_since_stats -= 1; } } @@ -102,6 +116,10 @@ impl EngineRoutingState { fn apply_scheduler_counts(&mut self, next: EngineLoadSnapshot) { self.last_scheduler_stats = Some(next); self.admitted_since_stats = 0; + self.stats_generation = self + .stats_generation + .checked_add(1) + .expect("scheduler stats generation overflow"); } } @@ -161,21 +179,22 @@ impl RequestRegistry { if lora.is_some() { self.active_lora_requests += 1; } + let state = self + .routing_per_engine + .get_mut(&engine_id) + .expect("request registry must track all known engines"); + let admission_stats_generation = state.record_admission(); + self.requests.insert( request_id, TrackedRequest { sender: tx, engine_id: engine_id.clone(), lora, + admission_stats_generation, }, ); - let state = self - .routing_per_engine - .get_mut(&engine_id) - .expect("request registry must track all known engines"); - state.record_admission(); - Ok((engine_id, rx)) } @@ -368,7 +387,7 @@ impl RequestRegistry { self.routing_per_engine .get_mut(&tracked.engine_id) .expect("request registry must track all known engines") - .inflight -= 1; + .record_removal(tracked.admission_stats_generation); Some((tracked.sender, tracked.engine_id)) } @@ -1079,6 +1098,80 @@ mod tests { expect_test::expect!["[2, 5, 2]"].assert_eq(&format!("{chosen:?}")); } + #[test] + fn registry_finished_requests_do_not_route_live_work_to_a_busier_engine() { + let mut registry = RequestRegistry::new( + &[0, 1].map(|rank| connected_engine(EngineId::from_engine_index(rank))), + ); + registry.apply_scheduler_counts( + 0, + EngineLoadSnapshot { + waiting: 0, + running: 0, + }, + ); + registry.apply_scheduler_counts( + 1, + EngineLoadSnapshot { + waiting: 0, + running: 1, + }, + ); + + let mut finished_routes = Vec::new(); + for request_id in ["finished-1", "finished-2"].map(String::from) { + let (engine_id, _) = + registry.register(request_id.clone(), None, None, NonZeroU32::MIN).unwrap(); + + finished_routes.push(engine_id.engine_index()); + drop(registry.finish_many([&request_id])); + } + + let (live_engine, _) = + registry.register("live-request".into(), None, None, NonZeroU32::MIN).unwrap(); + expect_test::expect!["([Some(0), Some(0)], Some(0))"].assert_eq(&format!( + "{:?}", + (finished_routes, live_engine.engine_index()) + )); + } + + #[test] + fn registry_removal_keeps_admissions_counted_after_a_newer_snapshot() { + let engine = EngineId::from_engine_index(0); + let mut registry = RequestRegistry::new(&[connected_engine(engine.clone())]); + registry.apply_scheduler_counts( + 0, + EngineLoadSnapshot { + waiting: 0, + running: 0, + }, + ); + + // `old` is admitted before the next snapshot, which already counts it. + registry.register("old".into(), None, None, NonZeroU32::MIN).unwrap(); + registry.apply_scheduler_counts( + 0, + EngineLoadSnapshot { + waiting: 0, + running: 1, + }, + ); + // `new` is admitted after that snapshot, so only it is counted on top. + registry.register("new".into(), None, None, NonZeroU32::MIN).unwrap(); + + // Removing `old` must not take back the admission counted for `new`. + drop(registry.remove("old")); + let state = ®istry.routing_per_engine[&engine]; + expect_test::expect!["(1, 1, 2)"].assert_eq(&format!( + "{:?}", + ( + state.inflight, + state.admitted_since_stats, + state.routing_score() + ) + )); + } + #[test] fn register_with_data_parallel_rank_routes_to_specified_engine() { let engine_0 = EngineId::from_engine_index(0);