[Rust Frontend] Wait for all utility calls to finish (#52671)

Signed-off-by: Connor Carpenter <connorc@nvidia.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Connor Carpenter
2026-08-18 11:23:34 -07:00
committed by GitHub
co-authored by Bugen Zhao
parent 01e56caaf2
commit d75136c030
4 changed files with 188 additions and 86 deletions
+58 -62
View File
@@ -5,7 +5,7 @@ use std::collections::BTreeMap;
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
use futures::future::{join_all, try_join_all}; use futures::future::join_all;
use itertools::Itertools; use itertools::Itertools;
use serde::Serialize; use serde::Serialize;
use serde_json::Value as JsonValue; use serde_json::Value as JsonValue;
@@ -450,6 +450,11 @@ impl EngineCoreClient {
self.config.transport_mode.data_parallel_size() self.config.transport_mode.data_parallel_size()
} }
#[cfg(test)]
pub(crate) fn pending_utility_call_count(&self) -> usize {
self.inner.pending_utility_call_count()
}
/// Return the engine-side indices connected to this client. /// Return the engine-side indices connected to this client.
pub fn engine_indices(&self) -> Vec<u32> { pub fn engine_indices(&self) -> Vec<u32> {
self.engines self.engines
@@ -596,6 +601,16 @@ impl EngineCoreClient {
let (engine_id, rx) = let (engine_id, rx) =
self.inner.register_request(request_id.clone(), lora_name, data_parallel_rank)?; self.inner.register_request(request_id.clone(), lora_name, data_parallel_rank)?;
// Construct the output stream first before actually sending the request to the engine.
// This ensures that cancelling the future will properly clean up the request registration
// via `Drop` of the stream.
let stream = EngineCoreOutputStream::new(
request_id.clone(),
engine_id.engine_index().unwrap_or(0),
self.abort_tx.clone(),
rx,
);
let result: Result<()> = async { let result: Result<()> = async {
if let Some(coordinator) = self.coordinator.as_ref() { if let Some(coordinator) = self.coordinator.as_ref() {
let snapshot = coordinator.snapshot(); let snapshot = coordinator.snapshot();
@@ -622,12 +637,7 @@ impl EngineCoreClient {
return Err(error); return Err(error);
} }
Ok(EngineCoreOutputStream::new( Ok(stream)
request_id,
engine_id.engine_index().unwrap_or(0),
self.abort_tx.clone(),
rx,
))
} }
/// Abort currently in-flight requests by request ID. /// Abort currently in-flight requests by request ID.
@@ -651,8 +661,9 @@ impl EngineCoreClient {
} }
/// Call a typed utility method on all connected engines, returning one /// Call a typed utility method on all connected engines, returning one
/// decoded result per connected engine if all calls succeed or an error /// decoded result per connected engine if all calls succeed. The client
/// if any call fails. /// waits for every engine outcome before returning an error so callers can
/// safely compensate partially applied mutations.
/// ///
/// Callers should pass utility arguments using Rust tuple semantics so the /// Callers should pass utility arguments using Rust tuple semantics so the
/// encoded payload matches Python's `(client_index, call_id, /// encoded payload matches Python's `(client_index, call_id,
@@ -669,62 +680,47 @@ impl EngineCoreClient {
"sending utility request" "sending utility request"
); );
// Phase 1: allocate one call id per engine and build the per-engine /// Removes utility waiters if a call future is cancelled before completion.
// request payloads up-front. Any failure here (registry closed, encode struct UtilityCallGuard<'a> {
// error) must roll back the call ids already allocated so they do not inner: &'a ClientInner,
// leak in the utility registry until shutdown. call_ids: Vec<u64>,
let mut pending_calls = Vec::with_capacity(self.engines.len()); }
let mut prepared_sends = Vec::with_capacity(self.engines.len());
impl Drop for UtilityCallGuard<'_> {
fn drop(&mut self) {
self.inner.unregister_utility_calls(self.call_ids.drain(..));
}
}
let mut call_guard = UtilityCallGuard {
inner: self.inner.as_ref(),
call_ids: Vec::with_capacity(self.engines.len()),
};
let mut prepared_calls = Vec::with_capacity(self.engines.len());
for engine in &self.engines { for engine in &self.engines {
let (call_id, rx) = match self.inner.allocate_and_register_utility_call() { let (call_id, rx) = self.inner.allocate_and_register_utility_call()?;
Ok(pair) => pair, call_guard.call_ids.push(call_id);
Err(err) => { let request =
self.inner.unregister_utility_calls(pending_calls.iter().map(|(id, _)| *id)); EngineCoreUtilityRequest::new(self.config.client_index, call_id, method, &args)?;
return Err(err); prepared_calls.push((&engine.engine_id, call_id, rx, request));
}
};
let request = match EngineCoreUtilityRequest::new(
self.config.client_index,
call_id,
method,
&args,
) {
Ok(request) => request,
Err(err) => {
self.inner.unregister_utility_calls(
pending_calls.iter().map(|(id, _)| *id).chain(std::iter::once(call_id)),
);
return Err(err);
}
};
pending_calls.push((call_id, rx));
prepared_sends.push((&engine.engine_id, request));
} }
// Phase 2: dispatch every utility request concurrently. `try_join_all` let outcomes = join_all(prepared_calls.into_iter().map(
// fails fast on the first transport error and drops the remaining send |(engine_id, call_id, rx, request)| async move {
// futures; any engines that already received the request will reply, self.inner
// but those replies are simply dropped because we roll back the call .send_to_engine(engine_id, EngineCoreRequestType::Utility, &request)
// ids below. .await?;
let send_futures = prepared_sends.iter().map(|(engine_id, request)| { rx.await
self.inner.send_to_engine(engine_id, EngineCoreRequestType::Utility, request) .map_err(|_| Error::UtilityCallClosed {
}); method: method.to_string(),
if let Err(err) = try_join_all(send_futures).await { call_id,
self.inner.unregister_utility_calls(pending_calls.iter().map(|(id, _)| *id)); })??
return Err(err); .into_typed_result(method)
} },
))
// Phase 3: wait for all engines to respond and preserve the per-engine .await;
// result list. outcomes.into_iter().collect()
let futures = pending_calls.into_iter().map(|(call_id, rx)| async move {
rx.await
.map_err(|_| Error::UtilityCallClosed {
method: method.to_string(),
call_id,
})??
.into_typed_result(method)
});
try_join_all(futures).await
} }
/// Call a utility method on all connected engines and return the shared /// Call a utility method on all connected engines and return the shared
@@ -118,6 +118,11 @@ impl ClientInner {
self.utility_reg.lock().unregister_many(call_ids); self.utility_reg.lock().unregister_many(call_ids);
} }
#[cfg(test)]
pub fn pending_utility_call_count(&self) -> usize {
self.utility_reg.lock().len()
}
/// Undo a request registration when `add_request()` fails. /// Undo a request registration when `add_request()` fails.
pub fn rollback_request(&self, request_id: &str) { pub fn rollback_request(&self, request_id: &str) {
let _ = self.request_reg.lock().remove(request_id); let _ = self.request_reg.lock().remove(request_id);
@@ -441,6 +441,11 @@ impl UtilityRegistry {
self.utility_calls.contains_key(&call_id) self.utility_calls.contains_key(&call_id)
} }
#[cfg(test)]
pub fn len(&self) -> usize {
self.utility_calls.len()
}
pub fn is_closed(&self) -> bool { pub fn is_closed(&self) -> bool {
self.closed self.closed
} }
+120 -24
View File
@@ -1431,23 +1431,20 @@ async fn is_sleeping_wrapper_sends_typed_request_and_returns_typed_response() {
} }
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn call_utility_failure_message_surfaces_as_error() { async fn call_utility_waits_for_all_engines_before_returning_error() {
init_tracing(); init_tracing();
let ipc = IpcNamespace::new().unwrap(); let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint(); let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-utility-fail".to_vec(); let (failure_sent_tx, failure_sent_rx) = oneshot::channel();
let (shutdown_tx_0, engine_task_0) = spawn_mock_engine_task(
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
handshake_address.clone(), handshake_address.clone(),
engine_id.clone(), EngineId::from_engine_index(0).into_frame().to_vec(),
|dealer, push| { move |dealer, push| {
Box::pin(async move { Box::pin(async move {
let utility = recv_engine_message(dealer).await; let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]); let payload = decode_value(&utility[1]);
let call_id = let call_id =
payload.as_array().and_then(|array| array[1].as_u64()).expect("call_id"); payload.as_array().and_then(|array| array[1].as_u64()).expect("call_id");
send_outputs( send_outputs(
push, push,
UtilityCallOutput { UtilityCallOutput {
@@ -1462,35 +1459,134 @@ async fn call_utility_failure_message_surfaces_as_error() {
.into(), .into(),
) )
.await; .await;
let _ = failure_sent_tx.send(());
})
},
);
let (second_received_tx, second_received_rx) = oneshot::channel();
let (release_second_tx, release_second_rx) = oneshot::channel();
let (shutdown_tx_1, engine_task_1) = spawn_mock_engine_task(
handshake_address.clone(),
EngineId::from_engine_index(1).into_frame().to_vec(),
move |dealer, push| {
Box::pin(async move {
let utility = recv_engine_message(dealer).await;
let payload = decode_value(&utility[1]);
let call_id =
payload.as_array().and_then(|array| array[1].as_u64()).expect("call_id");
let _ = second_received_tx.send(());
let _ = release_second_rx.await;
send_outputs(
push,
UtilityCallOutput {
engine_index: 1,
timestamp: 0.0,
output: UtilityOutput {
call_id: call_id.into(),
failure_message: None,
result: Some(utility_result_value(true)),
},
}
.into(),
)
.await;
}) })
}, },
); );
let client = connect_client_with_ipc( let client = std::sync::Arc::new(
handshake_test_config( connect_client_with_ipc(
handshake_address, handshake_test_config(
1, handshake_address,
"test-model", 2,
Duration::from_secs(2), "test-model",
0, Duration::from_secs(2),
None, 0,
), None,
&ipc, ),
) &ipc,
.await; )
.await,
);
let call_client = client.clone();
let call =
tokio::spawn(async move { call_client.call_utility::<bool, _>("test_mutation", ()).await });
let error = client.call_utility::<bool, _>("is_sleeping", ()).await.unwrap_err(); failure_sent_rx.await.unwrap();
second_received_rx.await.unwrap();
timeout(Duration::from_secs(2), async {
while client.pending_utility_call_count() != 1 {
tokio::task::yield_now().await;
}
})
.await
.expect("wait for first engine failure");
assert!(!call.is_finished());
let _ = release_second_tx.send(());
let error = call.await.unwrap().unwrap_err();
assert!(matches!( assert!(matches!(
error, error,
Error::UtilityCallFailed { Error::UtilityCallFailed {
method, method,
message, message,
.. ..
} if method == "is_sleeping" && message == "boom" } if method == "test_mutation" && message == "boom"
)); ));
let _ = shutdown_tx.send(()); let _ = shutdown_tx_0.send(());
engine_task.await.unwrap(); let _ = shutdown_tx_1.send(());
engine_task_0.await.unwrap();
engine_task_1.await.unwrap();
let client = std::sync::Arc::try_unwrap(client)
.unwrap_or_else(|_| panic!("utility task retained client after completion"));
client.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn cancelled_utility_call_unregisters_waiter() {
init_tracing();
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let (received_tx, received_rx) = oneshot::channel();
let (_shutdown, engine_task) = spawn_mock_engine_task(
handshake_address.clone(),
vec![0x00, 0x00],
|dealer, _push| {
Box::pin(async move {
let _utility = recv_engine_message(dealer).await;
let _ = received_tx.send(());
std::future::pending::<()>().await;
})
},
);
let client = std::sync::Arc::new(
connect_client_with_ipc(
handshake_test_config(
handshake_address,
1,
"test-model",
Duration::from_secs(2),
0,
None,
),
&ipc,
)
.await,
);
let call_client = client.clone();
let call =
tokio::spawn(async move { call_client.call_utility::<bool, _>("add_lora", ()).await });
received_rx.await.unwrap();
assert_eq!(client.pending_utility_call_count(), 1);
call.abort();
assert!(call.await.unwrap_err().is_cancelled());
assert_eq!(client.pending_utility_call_count(), 0);
engine_task.abort();
let client = std::sync::Arc::try_unwrap(client)
.unwrap_or_else(|_| panic!("utility task retained client after cancellation"));
client.shutdown().await.unwrap(); client.shutdown().await.unwrap();
} }