[rust] fix: aggregate is_sleeping and reset_prefix_cache across DP engines (#43429)

Signed-off-by: Will.hou <1205157517@qq.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Will.hou
2026-05-28 07:56:56 -07:00
committed by GitHub
co-authored by Claude Opus 4.7
parent 61a1e30473
commit 577d693838
3 changed files with 257 additions and 5 deletions
+35 -5
View File
@@ -595,9 +595,27 @@ impl EngineCoreClient {
}
/// Return whether the engine is currently sleeping at any level.
///
/// Under data parallel, all engines should agree on the sleep state: a
/// divergence signals a control-plane bug. Returns
/// `Error::InconsistentUtilityResults` if engines disagree.
pub async fn is_sleeping(&self) -> Result<bool> {
// TODO: we only return the result of the first engine here.
Ok(self.call_utility("is_sleeping", ()).await?[0])
let results: Vec<bool> = self.call_utility("is_sleeping", ()).await?;
// `engine_count >= 1` is enforced during startup handshake, so `results`
// is normally non-empty; fall back to a fail-loud error rather than
// indexing in case that invariant is ever bypassed.
let first = *results.first().ok_or_else(|| Error::InconsistentUtilityResults {
method: "is_sleeping".to_string(),
values: "[]".to_string(),
})?;
if results.iter().all(|&v| v == first) {
Ok(first)
} else {
Err(Error::InconsistentUtilityResults {
method: "is_sleeping".to_string(),
values: format!("{results:?}"),
})
}
}
/// Reset the multi-modal cache.
@@ -613,18 +631,30 @@ impl EngineCoreClient {
}
/// Reset the prefix cache and optionally the external connector cache.
///
/// Under data parallel, returns `true` only when every engine confirms the
/// reset (AND aggregation).
pub async fn reset_prefix_cache(
&self,
reset_running_requests: bool,
reset_connector: bool,
) -> Result<bool> {
// TODO: we only return the result of the first engine here.
Ok(self
let results: Vec<bool> = self
.call_utility(
"reset_prefix_cache",
(reset_running_requests, reset_connector),
)
.await?[0])
.await?;
// `engine_count >= 1` is enforced during startup handshake, so `results`
// is normally non-empty; fail loud rather than reporting a vacuous
// success (`[].all() == true`) in case that invariant is ever bypassed.
if results.is_empty() {
return Err(Error::InconsistentUtilityResults {
method: "reset_prefix_cache".to_string(),
values: "[]".to_string(),
});
}
Ok(results.into_iter().all(|ok| ok))
}
/// Put the engine to sleep.
+2
View File
@@ -83,6 +83,8 @@ pub enum Error {
},
#[error("utility call `{method}` closed unexpectedly (call_id={call_id})")]
UtilityCallClosed { method: String, call_id: u64 },
#[error("utility call `{method}` returned inconsistent results across engines: {values}")]
InconsistentUtilityResults { method: String, values: String },
/// A special variant to allow cloning the same error.
#[error(transparent)]
@@ -2119,6 +2119,226 @@ async fn collective_rpc_flattens_results_from_all_engines() {
client.shutdown().await.unwrap();
}
/// Spawn a mock engine that handles a single utility call, asserts the method
/// name and serialized args match, and replies with `result`.
fn spawn_mock_utility_engine(
handshake_address: String,
engine_id: Vec<u8>,
expected_method: &'static str,
expected_args: Value,
result: bool,
) -> (
tokio::sync::oneshot::Sender<()>,
tokio::task::JoinHandle<()>,
) {
spawn_mock_engine_task(handshake_address, engine_id, move |dealer, push| {
Box::pin(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]);
let array = match payload {
Value::Array(array) => array,
other => panic!("expected utility payload array, got {other:?}"),
};
// Utility requests serialize as `(client_index, call_id, method, args)`.
let call_id = array[1].as_u64().expect("call_id");
assert_eq!(array[2], Value::from(expected_method));
assert_eq!(array[3], expected_args, "unexpected utility args");
send_outputs(
push,
EngineCoreOutputs {
utility_output: Some(UtilityOutput {
call_id: call_id.into(),
failure_message: None,
result: Some(utility_result_value(result)),
}),
..Default::default()
},
)
.await;
})
})
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn is_sleeping_returns_error_when_engines_disagree() {
init_tracing();
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let (shutdown_tx_0, engine_task_0) = spawn_mock_utility_engine(
handshake_address.clone(),
b"engine-0".to_vec(),
"is_sleeping",
Value::Array(vec![]),
true,
);
let (shutdown_tx_1, engine_task_1) = spawn_mock_utility_engine(
handshake_address.clone(),
b"engine-1".to_vec(),
"is_sleeping",
Value::Array(vec![]),
false,
);
let client = connect_client_with_ipc(
handshake_test_config(
handshake_address,
2,
"test-model",
Duration::from_secs(2),
5,
None,
),
&ipc,
)
.await;
let error = client.is_sleeping().await.unwrap_err();
assert!(
matches!(
&error,
Error::InconsistentUtilityResults { method, .. } if method == "is_sleeping"
),
"expected InconsistentUtilityResults, got {error:?}",
);
let _ = shutdown_tx_0.send(());
let _ = shutdown_tx_1.send(());
engine_task_0.await.unwrap();
engine_task_1.await.unwrap();
client.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn is_sleeping_returns_value_when_all_engines_agree() {
init_tracing();
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let (shutdown_tx_0, engine_task_0) = spawn_mock_utility_engine(
handshake_address.clone(),
b"engine-0".to_vec(),
"is_sleeping",
Value::Array(vec![]),
true,
);
let (shutdown_tx_1, engine_task_1) = spawn_mock_utility_engine(
handshake_address.clone(),
b"engine-1".to_vec(),
"is_sleeping",
Value::Array(vec![]),
true,
);
let client = connect_client_with_ipc(
handshake_test_config(
handshake_address,
2,
"test-model",
Duration::from_secs(2),
5,
None,
),
&ipc,
)
.await;
assert!(client.is_sleeping().await.unwrap());
let _ = shutdown_tx_0.send(());
let _ = shutdown_tx_1.send(());
engine_task_0.await.unwrap();
engine_task_1.await.unwrap();
client.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn reset_prefix_cache_returns_true_when_all_engines_succeed() {
init_tracing();
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let (shutdown_tx_0, engine_task_0) = spawn_mock_utility_engine(
handshake_address.clone(),
b"engine-0".to_vec(),
"reset_prefix_cache",
Value::Array(vec![Value::from(false), Value::from(false)]),
true,
);
let (shutdown_tx_1, engine_task_1) = spawn_mock_utility_engine(
handshake_address.clone(),
b"engine-1".to_vec(),
"reset_prefix_cache",
Value::Array(vec![Value::from(false), Value::from(false)]),
true,
);
let client = connect_client_with_ipc(
handshake_test_config(
handshake_address,
2,
"test-model",
Duration::from_secs(2),
5,
None,
),
&ipc,
)
.await;
assert!(client.reset_prefix_cache(false, false).await.unwrap());
let _ = shutdown_tx_0.send(());
let _ = shutdown_tx_1.send(());
engine_task_0.await.unwrap();
engine_task_1.await.unwrap();
client.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn reset_prefix_cache_returns_false_when_any_engine_fails() {
init_tracing();
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let (shutdown_tx_0, engine_task_0) = spawn_mock_utility_engine(
handshake_address.clone(),
b"engine-0".to_vec(),
"reset_prefix_cache",
Value::Array(vec![Value::from(false), Value::from(false)]),
true,
);
let (shutdown_tx_1, engine_task_1) = spawn_mock_utility_engine(
handshake_address.clone(),
b"engine-1".to_vec(),
"reset_prefix_cache",
Value::Array(vec![Value::from(false), Value::from(false)]),
false,
);
let client = connect_client_with_ipc(
handshake_test_config(
handshake_address,
2,
"test-model",
Duration::from_secs(2),
5,
None,
),
&ipc,
)
.await;
assert!(!client.reset_prefix_cache(false, false).await.unwrap());
let _ = shutdown_tx_0.send(());
let _ = shutdown_tx_1.send(());
engine_task_0.await.unwrap();
engine_task_1.await.unwrap();
client.shutdown().await.unwrap();
}
#[test]
fn python_msgpack_fixtures_match_rust_encoding() {
init_tracing();