[rust] fix: aggregate is_sleeping and reset_prefix_cache across DP engines (#43429)
Signed-off-by: Will.hou <1205157517@qq.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
61a1e30473
commit
577d693838
@@ -595,9 +595,27 @@ impl EngineCoreClient {
|
||||
}
|
||||
|
||||
/// Return whether the engine is currently sleeping at any level.
|
||||
///
|
||||
/// Under data parallel, all engines should agree on the sleep state: a
|
||||
/// divergence signals a control-plane bug. Returns
|
||||
/// `Error::InconsistentUtilityResults` if engines disagree.
|
||||
pub async fn is_sleeping(&self) -> Result<bool> {
|
||||
// TODO: we only return the result of the first engine here.
|
||||
Ok(self.call_utility("is_sleeping", ()).await?[0])
|
||||
let results: Vec<bool> = self.call_utility("is_sleeping", ()).await?;
|
||||
// `engine_count >= 1` is enforced during startup handshake, so `results`
|
||||
// is normally non-empty; fall back to a fail-loud error rather than
|
||||
// indexing in case that invariant is ever bypassed.
|
||||
let first = *results.first().ok_or_else(|| Error::InconsistentUtilityResults {
|
||||
method: "is_sleeping".to_string(),
|
||||
values: "[]".to_string(),
|
||||
})?;
|
||||
if results.iter().all(|&v| v == first) {
|
||||
Ok(first)
|
||||
} else {
|
||||
Err(Error::InconsistentUtilityResults {
|
||||
method: "is_sleeping".to_string(),
|
||||
values: format!("{results:?}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset the multi-modal cache.
|
||||
@@ -613,18 +631,30 @@ impl EngineCoreClient {
|
||||
}
|
||||
|
||||
/// Reset the prefix cache and optionally the external connector cache.
|
||||
///
|
||||
/// Under data parallel, returns `true` only when every engine confirms the
|
||||
/// reset (AND aggregation).
|
||||
pub async fn reset_prefix_cache(
|
||||
&self,
|
||||
reset_running_requests: bool,
|
||||
reset_connector: bool,
|
||||
) -> Result<bool> {
|
||||
// TODO: we only return the result of the first engine here.
|
||||
Ok(self
|
||||
let results: Vec<bool> = self
|
||||
.call_utility(
|
||||
"reset_prefix_cache",
|
||||
(reset_running_requests, reset_connector),
|
||||
)
|
||||
.await?[0])
|
||||
.await?;
|
||||
// `engine_count >= 1` is enforced during startup handshake, so `results`
|
||||
// is normally non-empty; fail loud rather than reporting a vacuous
|
||||
// success (`[].all() == true`) in case that invariant is ever bypassed.
|
||||
if results.is_empty() {
|
||||
return Err(Error::InconsistentUtilityResults {
|
||||
method: "reset_prefix_cache".to_string(),
|
||||
values: "[]".to_string(),
|
||||
});
|
||||
}
|
||||
Ok(results.into_iter().all(|ok| ok))
|
||||
}
|
||||
|
||||
/// Put the engine to sleep.
|
||||
|
||||
@@ -83,6 +83,8 @@ pub enum Error {
|
||||
},
|
||||
#[error("utility call `{method}` closed unexpectedly (call_id={call_id})")]
|
||||
UtilityCallClosed { method: String, call_id: u64 },
|
||||
#[error("utility call `{method}` returned inconsistent results across engines: {values}")]
|
||||
InconsistentUtilityResults { method: String, values: String },
|
||||
|
||||
/// A special variant to allow cloning the same error.
|
||||
#[error(transparent)]
|
||||
|
||||
@@ -2119,6 +2119,226 @@ async fn collective_rpc_flattens_results_from_all_engines() {
|
||||
client.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
/// Spawn a mock engine that handles a single utility call, asserts the method
|
||||
/// name and serialized args match, and replies with `result`.
|
||||
fn spawn_mock_utility_engine(
|
||||
handshake_address: String,
|
||||
engine_id: Vec<u8>,
|
||||
expected_method: &'static str,
|
||||
expected_args: Value,
|
||||
result: bool,
|
||||
) -> (
|
||||
tokio::sync::oneshot::Sender<()>,
|
||||
tokio::task::JoinHandle<()>,
|
||||
) {
|
||||
spawn_mock_engine_task(handshake_address, engine_id, move |dealer, push| {
|
||||
Box::pin(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
let payload = decode_value(&utility[1]);
|
||||
let array = match payload {
|
||||
Value::Array(array) => array,
|
||||
other => panic!("expected utility payload array, got {other:?}"),
|
||||
};
|
||||
// Utility requests serialize as `(client_index, call_id, method, args)`.
|
||||
let call_id = array[1].as_u64().expect("call_id");
|
||||
assert_eq!(array[2], Value::from(expected_method));
|
||||
assert_eq!(array[3], expected_args, "unexpected utility args");
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
utility_output: Some(UtilityOutput {
|
||||
call_id: call_id.into(),
|
||||
failure_message: None,
|
||||
result: Some(utility_result_value(result)),
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn is_sleeping_returns_error_when_engines_disagree() {
|
||||
init_tracing();
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
|
||||
let (shutdown_tx_0, engine_task_0) = spawn_mock_utility_engine(
|
||||
handshake_address.clone(),
|
||||
b"engine-0".to_vec(),
|
||||
"is_sleeping",
|
||||
Value::Array(vec![]),
|
||||
true,
|
||||
);
|
||||
let (shutdown_tx_1, engine_task_1) = spawn_mock_utility_engine(
|
||||
handshake_address.clone(),
|
||||
b"engine-1".to_vec(),
|
||||
"is_sleeping",
|
||||
Value::Array(vec![]),
|
||||
false,
|
||||
);
|
||||
|
||||
let client = connect_client_with_ipc(
|
||||
handshake_test_config(
|
||||
handshake_address,
|
||||
2,
|
||||
"test-model",
|
||||
Duration::from_secs(2),
|
||||
5,
|
||||
None,
|
||||
),
|
||||
&ipc,
|
||||
)
|
||||
.await;
|
||||
|
||||
let error = client.is_sleeping().await.unwrap_err();
|
||||
assert!(
|
||||
matches!(
|
||||
&error,
|
||||
Error::InconsistentUtilityResults { method, .. } if method == "is_sleeping"
|
||||
),
|
||||
"expected InconsistentUtilityResults, got {error:?}",
|
||||
);
|
||||
|
||||
let _ = shutdown_tx_0.send(());
|
||||
let _ = shutdown_tx_1.send(());
|
||||
engine_task_0.await.unwrap();
|
||||
engine_task_1.await.unwrap();
|
||||
client.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn is_sleeping_returns_value_when_all_engines_agree() {
|
||||
init_tracing();
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
|
||||
let (shutdown_tx_0, engine_task_0) = spawn_mock_utility_engine(
|
||||
handshake_address.clone(),
|
||||
b"engine-0".to_vec(),
|
||||
"is_sleeping",
|
||||
Value::Array(vec![]),
|
||||
true,
|
||||
);
|
||||
let (shutdown_tx_1, engine_task_1) = spawn_mock_utility_engine(
|
||||
handshake_address.clone(),
|
||||
b"engine-1".to_vec(),
|
||||
"is_sleeping",
|
||||
Value::Array(vec![]),
|
||||
true,
|
||||
);
|
||||
|
||||
let client = connect_client_with_ipc(
|
||||
handshake_test_config(
|
||||
handshake_address,
|
||||
2,
|
||||
"test-model",
|
||||
Duration::from_secs(2),
|
||||
5,
|
||||
None,
|
||||
),
|
||||
&ipc,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(client.is_sleeping().await.unwrap());
|
||||
|
||||
let _ = shutdown_tx_0.send(());
|
||||
let _ = shutdown_tx_1.send(());
|
||||
engine_task_0.await.unwrap();
|
||||
engine_task_1.await.unwrap();
|
||||
client.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn reset_prefix_cache_returns_true_when_all_engines_succeed() {
|
||||
init_tracing();
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
|
||||
let (shutdown_tx_0, engine_task_0) = spawn_mock_utility_engine(
|
||||
handshake_address.clone(),
|
||||
b"engine-0".to_vec(),
|
||||
"reset_prefix_cache",
|
||||
Value::Array(vec![Value::from(false), Value::from(false)]),
|
||||
true,
|
||||
);
|
||||
let (shutdown_tx_1, engine_task_1) = spawn_mock_utility_engine(
|
||||
handshake_address.clone(),
|
||||
b"engine-1".to_vec(),
|
||||
"reset_prefix_cache",
|
||||
Value::Array(vec![Value::from(false), Value::from(false)]),
|
||||
true,
|
||||
);
|
||||
|
||||
let client = connect_client_with_ipc(
|
||||
handshake_test_config(
|
||||
handshake_address,
|
||||
2,
|
||||
"test-model",
|
||||
Duration::from_secs(2),
|
||||
5,
|
||||
None,
|
||||
),
|
||||
&ipc,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(client.reset_prefix_cache(false, false).await.unwrap());
|
||||
|
||||
let _ = shutdown_tx_0.send(());
|
||||
let _ = shutdown_tx_1.send(());
|
||||
engine_task_0.await.unwrap();
|
||||
engine_task_1.await.unwrap();
|
||||
client.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn reset_prefix_cache_returns_false_when_any_engine_fails() {
|
||||
init_tracing();
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
|
||||
let (shutdown_tx_0, engine_task_0) = spawn_mock_utility_engine(
|
||||
handshake_address.clone(),
|
||||
b"engine-0".to_vec(),
|
||||
"reset_prefix_cache",
|
||||
Value::Array(vec![Value::from(false), Value::from(false)]),
|
||||
true,
|
||||
);
|
||||
let (shutdown_tx_1, engine_task_1) = spawn_mock_utility_engine(
|
||||
handshake_address.clone(),
|
||||
b"engine-1".to_vec(),
|
||||
"reset_prefix_cache",
|
||||
Value::Array(vec![Value::from(false), Value::from(false)]),
|
||||
false,
|
||||
);
|
||||
|
||||
let client = connect_client_with_ipc(
|
||||
handshake_test_config(
|
||||
handshake_address,
|
||||
2,
|
||||
"test-model",
|
||||
Duration::from_secs(2),
|
||||
5,
|
||||
None,
|
||||
),
|
||||
&ipc,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(!client.reset_prefix_cache(false, false).await.unwrap());
|
||||
|
||||
let _ = shutdown_tx_0.send(());
|
||||
let _ = shutdown_tx_1.send(());
|
||||
engine_task_0.await.unwrap();
|
||||
engine_task_1.await.unwrap();
|
||||
client.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
init_tracing();
|
||||
|
||||
Reference in New Issue
Block a user