diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index 4e71405253..dd3000f3c6 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -595,9 +595,27 @@ impl EngineCoreClient { } /// Return whether the engine is currently sleeping at any level. + /// + /// Under data parallel, all engines should agree on the sleep state: a + /// divergence signals a control-plane bug. Returns + /// `Error::InconsistentUtilityResults` if engines disagree. pub async fn is_sleeping(&self) -> Result { - // TODO: we only return the result of the first engine here. - Ok(self.call_utility("is_sleeping", ()).await?[0]) + let results: Vec = self.call_utility("is_sleeping", ()).await?; + // `engine_count >= 1` is enforced during startup handshake, so `results` + // is normally non-empty; fall back to a fail-loud error rather than + // indexing in case that invariant is ever bypassed. + let first = *results.first().ok_or_else(|| Error::InconsistentUtilityResults { + method: "is_sleeping".to_string(), + values: "[]".to_string(), + })?; + if results.iter().all(|&v| v == first) { + Ok(first) + } else { + Err(Error::InconsistentUtilityResults { + method: "is_sleeping".to_string(), + values: format!("{results:?}"), + }) + } } /// Reset the multi-modal cache. @@ -613,18 +631,30 @@ impl EngineCoreClient { } /// Reset the prefix cache and optionally the external connector cache. + /// + /// Under data parallel, returns `true` only when every engine confirms the + /// reset (AND aggregation). pub async fn reset_prefix_cache( &self, reset_running_requests: bool, reset_connector: bool, ) -> Result { - // TODO: we only return the result of the first engine here. - Ok(self + let results: Vec = self .call_utility( "reset_prefix_cache", (reset_running_requests, reset_connector), ) - .await?[0]) + .await?; + // `engine_count >= 1` is enforced during startup handshake, so `results` + // is normally non-empty; fail loud rather than reporting a vacuous + // success (`[].all() == true`) in case that invariant is ever bypassed. + if results.is_empty() { + return Err(Error::InconsistentUtilityResults { + method: "reset_prefix_cache".to_string(), + values: "[]".to_string(), + }); + } + Ok(results.into_iter().all(|ok| ok)) } /// Put the engine to sleep. diff --git a/rust/src/engine-core-client/src/error.rs b/rust/src/engine-core-client/src/error.rs index 1ce03c4078..0493732b03 100644 --- a/rust/src/engine-core-client/src/error.rs +++ b/rust/src/engine-core-client/src/error.rs @@ -83,6 +83,8 @@ pub enum Error { }, #[error("utility call `{method}` closed unexpectedly (call_id={call_id})")] UtilityCallClosed { method: String, call_id: u64 }, + #[error("utility call `{method}` returned inconsistent results across engines: {values}")] + InconsistentUtilityResults { method: String, values: String }, /// A special variant to allow cloning the same error. #[error(transparent)] diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index d1e0b03cc7..af6cf3ea9a 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -2119,6 +2119,226 @@ async fn collective_rpc_flattens_results_from_all_engines() { client.shutdown().await.unwrap(); } +/// Spawn a mock engine that handles a single utility call, asserts the method +/// name and serialized args match, and replies with `result`. +fn spawn_mock_utility_engine( + handshake_address: String, + engine_id: Vec, + expected_method: &'static str, + expected_args: Value, + result: bool, +) -> ( + tokio::sync::oneshot::Sender<()>, + tokio::task::JoinHandle<()>, +) { + spawn_mock_engine_task(handshake_address, engine_id, move |dealer, push| { + Box::pin(async move { + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + let payload = decode_value(&utility[1]); + let array = match payload { + Value::Array(array) => array, + other => panic!("expected utility payload array, got {other:?}"), + }; + // Utility requests serialize as `(client_index, call_id, method, args)`. + let call_id = array[1].as_u64().expect("call_id"); + assert_eq!(array[2], Value::from(expected_method)); + assert_eq!(array[3], expected_args, "unexpected utility args"); + send_outputs( + push, + EngineCoreOutputs { + utility_output: Some(UtilityOutput { + call_id: call_id.into(), + failure_message: None, + result: Some(utility_result_value(result)), + }), + ..Default::default() + }, + ) + .await; + }) + }) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn is_sleeping_returns_error_when_engines_disagree() { + init_tracing(); + let ipc = IpcNamespace::new().unwrap(); + let handshake_address = ipc.handshake_endpoint(); + + let (shutdown_tx_0, engine_task_0) = spawn_mock_utility_engine( + handshake_address.clone(), + b"engine-0".to_vec(), + "is_sleeping", + Value::Array(vec![]), + true, + ); + let (shutdown_tx_1, engine_task_1) = spawn_mock_utility_engine( + handshake_address.clone(), + b"engine-1".to_vec(), + "is_sleeping", + Value::Array(vec![]), + false, + ); + + let client = connect_client_with_ipc( + handshake_test_config( + handshake_address, + 2, + "test-model", + Duration::from_secs(2), + 5, + None, + ), + &ipc, + ) + .await; + + let error = client.is_sleeping().await.unwrap_err(); + assert!( + matches!( + &error, + Error::InconsistentUtilityResults { method, .. } if method == "is_sleeping" + ), + "expected InconsistentUtilityResults, got {error:?}", + ); + + let _ = shutdown_tx_0.send(()); + let _ = shutdown_tx_1.send(()); + engine_task_0.await.unwrap(); + engine_task_1.await.unwrap(); + client.shutdown().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn is_sleeping_returns_value_when_all_engines_agree() { + init_tracing(); + let ipc = IpcNamespace::new().unwrap(); + let handshake_address = ipc.handshake_endpoint(); + + let (shutdown_tx_0, engine_task_0) = spawn_mock_utility_engine( + handshake_address.clone(), + b"engine-0".to_vec(), + "is_sleeping", + Value::Array(vec![]), + true, + ); + let (shutdown_tx_1, engine_task_1) = spawn_mock_utility_engine( + handshake_address.clone(), + b"engine-1".to_vec(), + "is_sleeping", + Value::Array(vec![]), + true, + ); + + let client = connect_client_with_ipc( + handshake_test_config( + handshake_address, + 2, + "test-model", + Duration::from_secs(2), + 5, + None, + ), + &ipc, + ) + .await; + + assert!(client.is_sleeping().await.unwrap()); + + let _ = shutdown_tx_0.send(()); + let _ = shutdown_tx_1.send(()); + engine_task_0.await.unwrap(); + engine_task_1.await.unwrap(); + client.shutdown().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn reset_prefix_cache_returns_true_when_all_engines_succeed() { + init_tracing(); + let ipc = IpcNamespace::new().unwrap(); + let handshake_address = ipc.handshake_endpoint(); + + let (shutdown_tx_0, engine_task_0) = spawn_mock_utility_engine( + handshake_address.clone(), + b"engine-0".to_vec(), + "reset_prefix_cache", + Value::Array(vec![Value::from(false), Value::from(false)]), + true, + ); + let (shutdown_tx_1, engine_task_1) = spawn_mock_utility_engine( + handshake_address.clone(), + b"engine-1".to_vec(), + "reset_prefix_cache", + Value::Array(vec![Value::from(false), Value::from(false)]), + true, + ); + + let client = connect_client_with_ipc( + handshake_test_config( + handshake_address, + 2, + "test-model", + Duration::from_secs(2), + 5, + None, + ), + &ipc, + ) + .await; + + assert!(client.reset_prefix_cache(false, false).await.unwrap()); + + let _ = shutdown_tx_0.send(()); + let _ = shutdown_tx_1.send(()); + engine_task_0.await.unwrap(); + engine_task_1.await.unwrap(); + client.shutdown().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn reset_prefix_cache_returns_false_when_any_engine_fails() { + init_tracing(); + let ipc = IpcNamespace::new().unwrap(); + let handshake_address = ipc.handshake_endpoint(); + + let (shutdown_tx_0, engine_task_0) = spawn_mock_utility_engine( + handshake_address.clone(), + b"engine-0".to_vec(), + "reset_prefix_cache", + Value::Array(vec![Value::from(false), Value::from(false)]), + true, + ); + let (shutdown_tx_1, engine_task_1) = spawn_mock_utility_engine( + handshake_address.clone(), + b"engine-1".to_vec(), + "reset_prefix_cache", + Value::Array(vec![Value::from(false), Value::from(false)]), + false, + ); + + let client = connect_client_with_ipc( + handshake_test_config( + handshake_address, + 2, + "test-model", + Duration::from_secs(2), + 5, + None, + ), + &ipc, + ) + .await; + + assert!(!client.reset_prefix_cache(false, false).await.unwrap()); + + let _ = shutdown_tx_0.send(()); + let _ = shutdown_tx_1.send(()); + engine_task_0.await.unwrap(); + engine_task_1.await.unwrap(); + client.shutdown().await.unwrap(); +} + #[test] fn python_msgpack_fixtures_match_rust_encoding() { init_tracing();