[Rust Frontend] Introduce mock engine for benchmark baseline (#43469)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-05-28 01:40:35 +00:00
committed by GitHub
parent 1223732dda
commit c87f62ccf8
19 changed files with 1462 additions and 204 deletions
@@ -107,13 +107,13 @@ impl ClientInner {
Ok(registry.abortable_request_ids(request_ids))
}
/// Obtain the stream sender for one output. If it indicates the request is
/// finished, it will be removed from the registry.
pub fn take_sender_for_output(
/// Obtain stream senders for a whole engine output batch with one registry
/// lock acquisition.
pub fn take_senders_for_outputs<'a>(
&self,
output: &EngineCoreOutput,
) -> Option<mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>> {
self.request_reg.lock().sender_for_output(output)
outputs: impl IntoIterator<Item = &'a EngineCoreOutput>,
) -> Vec<Option<mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>>> {
self.request_reg.lock().senders_for_outputs(outputs)
}
/// Remove a batch of requests that have finished or aborted, returning
@@ -301,9 +301,10 @@ pub(crate) async fn run_output_dispatcher_loop(
match outputs.classify() {
ClassifiedEngineCoreOutputs::RequestBatch(batch) => {
for output in batch.outputs {
let senders = inner.take_senders_for_outputs(&batch.outputs);
for (output, sender) in batch.outputs.into_iter().zip(senders) {
let request_id = output.request_id.clone();
let Some(sender) = inner.take_sender_for_output(&output) else {
let Some(sender) = sender else {
debug!(request_id, "dropping output for inactive request");
continue;
};
@@ -1,4 +1,4 @@
use std::collections::BTreeMap;
use std::collections::{BTreeMap, HashMap};
use std::sync::atomic::{AtomicU64, Ordering};
use tokio::sync::{mpsc, oneshot};
@@ -80,7 +80,7 @@ impl EngineRoutingState {
#[derive(Debug)]
pub struct RequestRegistry {
closed: bool,
requests: BTreeMap<String, TrackedRequest>,
requests: HashMap<String, TrackedRequest>,
routing_per_engine: BTreeMap<EngineId, EngineRoutingState>,
}
@@ -88,7 +88,7 @@ impl RequestRegistry {
pub fn new(engines: &[ConnectedEngine]) -> Self {
Self {
closed: false,
requests: BTreeMap::default(),
requests: HashMap::default(),
routing_per_engine: engines
.iter()
.map(|engine| (engine.engine_id.clone(), EngineRoutingState::default()))
@@ -180,6 +180,15 @@ impl RequestRegistry {
}
}
/// Obtain stream senders for a whole engine output batch under one
/// registry lock. Finished outputs are removed before returning.
pub fn senders_for_outputs<'a>(
&mut self,
outputs: impl IntoIterator<Item = &'a EngineCoreOutput>,
) -> Vec<Option<OutputSender>> {
outputs.into_iter().map(|output| self.sender_for_output(output)).collect()
}
/// Remove a batch of requests that have finished or aborted, returning
/// their stream senders.
pub fn finish_many<'a>(
+1
View File
@@ -2,6 +2,7 @@ mod client;
mod coordinator;
mod error;
mod metrics;
pub mod mock_engine;
pub mod protocol;
#[cfg(any(test, feature = "test-util"))]
pub mod test_utils;
@@ -0,0 +1,265 @@
use std::path::Path;
use std::time::Duration;
use tokio::time::timeout;
use zeromq::prelude::{Socket, SocketRecv, SocketSend};
use zeromq::util::PeerIdentity;
use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, ZmqMessage};
use crate::EngineId;
use crate::error::{Error, Result, bail_unexpected_handshake_message};
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
use crate::protocol::{ModelDtype, decode_msgpack, encode_msgpack};
/// Default model length advertised by reusable mock engine helpers.
pub const DEFAULT_MOCK_MAX_MODEL_LEN: u64 = 1024 * 1024;
/// Default KV block count advertised by reusable mock engine helpers.
pub const DEFAULT_MOCK_NUM_GPU_BLOCKS: u64 = 0;
/// Startup behavior for one mock engine joining a frontend.
#[derive(Debug, Clone)]
pub struct MockEngineConfig {
/// Whether the engine should advertise itself as local to the frontend.
pub local: bool,
/// Whether the engine should advertise itself as headless.
pub headless: bool,
/// Engine-ready payload reported after INIT, including max model length,
/// KV block count, and dtype.
pub ready_response: EngineCoreReadyResponse,
/// Maximum time to wait for IPC endpoints to appear before connecting.
pub connect_timeout: Duration,
}
impl Default for MockEngineConfig {
fn default() -> Self {
Self {
local: false,
headless: true,
ready_response: default_ready_response(),
connect_timeout: Duration::from_secs(5),
}
}
}
/// Construct the ready response used by the standalone mock engine CLI.
pub fn default_ready_response() -> EngineCoreReadyResponse {
EngineCoreReadyResponse {
max_model_len: DEFAULT_MOCK_MAX_MODEL_LEN,
num_gpu_blocks: DEFAULT_MOCK_NUM_GPU_BLOCKS,
dp_stats_address: None,
dtype: Some(ModelDtype::Float32),
}
}
/// Coordinator-side sockets used by one mock engine when coordinator mode
/// is enabled.
pub struct MockCoordinatorSockets {
/// Subscription socket that receives coordinator broadcasts such as
/// `START_DP_WAVE`.
pub input_sub: SubSocket,
/// Push socket used to send coordinator-only `EngineCoreOutputs` back to
/// the frontend.
pub output_push: PushSocket,
}
/// One mock engine's connection to one frontend client.
///
/// vLLM launches one engine-client pair per API server process. A remote
/// engine connects to every advertised input/output pair and uses the request's
/// `client_index` to route outputs back to the originating API server.
pub struct MockEngineDataSockets {
/// Socket used to receive frontend requests.
pub dealer: DealerSocket,
/// Socket used to publish normal request outputs back to the frontend.
pub push: PushSocket,
}
/// Frontend-facing sockets owned by one mock engine.
pub struct MockEngineSockets {
/// Decoded INIT message sent by the frontend during handshake.
pub init: HandshakeInitMessage,
/// Data sockets for all frontend clients in client-index order.
///
/// For Rust frontend this will always be one socket, while for Python frontend
/// this may be multiple sockets if there are multiple API server processes.
pub data_sockets: Vec<MockEngineDataSockets>,
/// Optional coordinator sockets when the client enabled the in-process
/// coordinator.
pub coordinator: Option<MockCoordinatorSockets>,
}
/// Build a HELLO or READY handshake status payload.
fn ready_message(status: &str, config: &MockEngineConfig) -> ReadyMessage {
ReadyMessage {
status: Some(status.to_string()),
local: Some(config.local),
headless: Some(config.headless),
parallel_config_hash: None,
}
}
/// Convert an engine id into a ZMQ DEALER identity.
fn peer_identity(engine_id: impl Into<EngineId>) -> Result<PeerIdentity> {
let engine_id = engine_id.into();
PeerIdentity::try_from(engine_id.clone()).map_err(|error| Error::UnexpectedHandshakeMessage {
message: format!(
"invalid mock engine identity {:?}: {error}",
engine_id.to_vec()
),
})
}
/// Wait for an IPC endpoint path to appear before attempting to connect.
async fn wait_for_ipc_endpoint(endpoint: &str, connect_timeout: Duration) -> Result<()> {
let Some(socket_path) = endpoint.strip_prefix("ipc://") else {
return Ok(());
};
timeout(connect_timeout, async {
while !Path::new(socket_path).exists() {
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.map_err(|_| Error::HandshakeTimeout {
stage: "mock engine IPC endpoint",
timeout: connect_timeout,
})
}
/// Encode the engine-ready response sent on input socket registration.
fn ready_response_payload(config: &MockEngineConfig) -> Result<Vec<u8>> {
encode_msgpack(&config.ready_response)
}
/// Join a frontend-owned handshake endpoint and open mock engine sockets.
pub async fn connect_to_frontend(
engine_handshake: impl AsRef<str>,
engine_id: impl Into<EngineId>,
config: MockEngineConfig,
) -> Result<MockEngineSockets> {
let engine_handshake = engine_handshake.as_ref();
wait_for_ipc_endpoint(engine_handshake, config.connect_timeout).await?;
let peer_identity = peer_identity(engine_id)?;
let mut options = SocketOptions::default();
options.peer_identity(peer_identity.clone());
let mut handshake = DealerSocket::with_options(options);
handshake.connect(engine_handshake).await?;
handshake
.send(ZmqMessage::from(encode_msgpack(&ready_message(
"HELLO", &config,
))?))
.await?;
let init_frames = handshake.recv().await?.into_vec();
if init_frames.len() != 1 {
bail_unexpected_handshake_message!(
"expected one INIT frame from frontend, got {}",
init_frames.len()
);
}
let init: HandshakeInitMessage = decode_msgpack(init_frames[0].as_ref())?;
if init.addresses.inputs.is_empty() {
return Err(Error::UnexpectedHandshakeMessage {
message: "frontend INIT did not include an input address".to_string(),
});
}
if init.addresses.inputs.len() != init.addresses.outputs.len() {
return Err(Error::UnexpectedHandshakeMessage {
message: format!(
"frontend INIT input/output address count mismatch: {} inputs, {} outputs",
init.addresses.inputs.len(),
init.addresses.outputs.len()
),
});
}
let mut data_sockets = Vec::with_capacity(init.addresses.inputs.len());
for (input_address, output_address) in
init.addresses.inputs.iter().zip(init.addresses.outputs.iter())
{
wait_for_ipc_endpoint(input_address, config.connect_timeout).await?;
wait_for_ipc_endpoint(output_address, config.connect_timeout).await?;
let mut input_options = SocketOptions::default();
input_options.peer_identity(peer_identity.clone());
let mut dealer = DealerSocket::with_options(input_options);
dealer.connect(input_address).await?;
dealer.send(ZmqMessage::from(ready_response_payload(&config)?)).await?;
let mut push = PushSocket::new();
push.connect(output_address).await?;
data_sockets.push(MockEngineDataSockets { dealer, push });
}
let coordinator = match (
init.addresses.coordinator_input.as_deref(),
init.addresses.coordinator_output.as_deref(),
) {
(Some(coordinator_input), Some(coordinator_output)) => {
let mut input_sub = SubSocket::new();
input_sub.connect(coordinator_input).await?;
input_sub.subscribe("").await?;
let mut output_push = PushSocket::new();
output_push.connect(coordinator_output).await?;
let ready = input_sub.recv().await?.into_vec();
if ready.len() != 1 || ready[0].as_ref() != b"READY" {
bail_unexpected_handshake_message!(
"expected coordinator READY marker, got {:?}",
ready
);
}
Some(MockCoordinatorSockets {
input_sub,
output_push,
})
}
(None, None) => None,
_ => bail_unexpected_handshake_message!(
"coordinator handshake addresses must be both present or both absent"
),
};
handshake
.send(ZmqMessage::from(encode_msgpack(&ready_message(
"READY", &config,
))?))
.await?;
Ok(MockEngineSockets {
init,
data_sockets,
coordinator,
})
}
/// Join already-bootstrapped frontend input/output sockets directly.
pub async fn connect_to_bootstrapped_frontend(
input_address: impl AsRef<str>,
output_address: impl AsRef<str>,
engine_id: impl Into<EngineId>,
config: MockEngineConfig,
) -> Result<(DealerSocket, PushSocket)> {
let input_address = input_address.as_ref();
let output_address = output_address.as_ref();
wait_for_ipc_endpoint(input_address, config.connect_timeout).await?;
wait_for_ipc_endpoint(output_address, config.connect_timeout).await?;
let peer_identity = peer_identity(engine_id)?;
let mut input_options = SocketOptions::default();
input_options.peer_identity(peer_identity);
let mut dealer = DealerSocket::with_options(input_options);
dealer.connect(input_address).await?;
dealer.send(ZmqMessage::from(ready_response_payload(&config)?)).await?;
let mut push = PushSocket::new();
push.connect(output_address).await?;
Ok((dealer, push))
}
@@ -36,6 +36,14 @@ fn is_false(v: &bool) -> bool {
!v
}
fn default_top_p() -> f32 {
1.0
}
fn default_repetition_penalty() -> f32 {
1.0
}
mod classified_outputs;
pub mod dtype;
pub mod handshake;
@@ -65,6 +73,24 @@ pub enum EngineCoreRequestType {
}
impl EngineCoreRequestType {
/// Decode the single-byte request type frame used on the engine input
/// socket. Returns `None` for unrecognized values.
pub fn from_frame(frame: &[u8]) -> Option<Self> {
let [value] = frame else {
return None;
};
match value {
0 => Some(Self::Add),
1 => Some(Self::Abort),
2 => Some(Self::StartDpWave),
3 => Some(Self::Utility),
_ => None,
}
}
/// Encode the request type as the single-byte frame used on the engine
/// input socket.
pub fn to_frame(self) -> Bytes {
Bytes::from_static(match self {
Self::Add => b"\x00",
@@ -200,14 +226,17 @@ pub struct EngineCoreSamplingParams {
/// greedy sampling.
pub temperature: f32,
/// Cumulative probability threshold for nucleus sampling.
#[serde(default = "default_top_p")]
pub top_p: f32,
/// Maximum number of top tokens to consider. `0` means all tokens.
#[serde(default)]
pub top_k: u32,
/// Random seed used by the sampler when present.
pub seed: Option<i64>,
/// Maximum number of tokens to generate per output sequence.
pub max_tokens: u32,
/// Minimum number of tokens to generate before EOS or stop-token handling.
#[serde(default)]
pub min_tokens: u32,
/// Number of log probabilities to return per generated token.
///
@@ -218,12 +247,14 @@ pub struct EngineCoreSamplingParams {
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Minimum probability threshold for token sampling.
#[serde(default)]
pub min_p: f32,
/// Frequency penalty applied by the sampler.
pub frequency_penalty: f32,
/// Presence penalty applied by the sampler.
pub presence_penalty: f32,
/// Repetition penalty applied by the sampler.
#[serde(default = "default_repetition_penalty")]
pub repetition_penalty: f32,
/// Token IDs that stop generation.
pub stop_token_ids: Vec<u32>,
@@ -102,7 +102,7 @@ impl<'de> Deserialize<'de> for UtilityCallId {
///
/// Original Python payload shape:
/// `(client_index, call_id, method_name, args)`
#[derive(Debug, Clone, PartialEq, Serialize_tuple)]
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
pub struct EngineCoreUtilityRequest {
pub client_index: u32,
pub call_id: UtilityCallId,
+34 -175
View File
@@ -1,17 +1,18 @@
use std::future::Future;
use std::path::Path;
use std::pin::Pin;
use std::time::Duration;
use tempfile::TempDir;
use tokio::sync::oneshot;
use zeromq::prelude::{Socket, SocketRecv, SocketSend};
use zeromq::util::PeerIdentity;
use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, ZmqMessage};
use zeromq::{DealerSocket, PushSocket};
use crate::EngineId;
pub use crate::mock_engine::{MockCoordinatorSockets, MockEngineSockets};
use crate::mock_engine::{
MockEngineConfig, MockEngineDataSockets, connect_to_bootstrapped_frontend, connect_to_frontend,
};
use crate::protocol::ModelDtype;
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage};
/// Per-test IPC endpoint namespace backed by a unique temporary directory.
///
@@ -52,156 +53,29 @@ impl IpcNamespace {
}
}
/// Construct a standard local READY message used by mock engines in tests.
fn ready_message(status: &str) -> ReadyMessage {
ReadyMessage {
status: Some(status.to_string()),
local: Some(true),
headless: Some(true),
parallel_config_hash: None,
fn test_mock_engine_config() -> MockEngineConfig {
MockEngineConfig {
local: true,
headless: true,
ready_response: EngineCoreReadyResponse {
max_model_len: 4096,
num_gpu_blocks: 0,
dp_stats_address: None,
dtype: Some(ModelDtype::Float32),
},
..Default::default()
}
}
/// Construct a default ready response payload for mock engine input
/// registration.
fn ready_response_payload() -> Vec<u8> {
rmp_serde::to_vec_named(&EngineCoreReadyResponse {
max_model_len: 4096,
num_gpu_blocks: 0,
dp_stats_address: None,
dtype: Some(ModelDtype::Float32),
})
.expect("encode ready response payload")
}
/// Coordinator-side sockets connected by one mock engine when coordinator mode
/// is enabled.
pub struct MockCoordinatorConnections {
/// Subscription socket that receives coordinator broadcasts such as
/// `START_DP_WAVE`.
pub input_sub: SubSocket,
/// Push socket used to send coordinator-only `EngineCoreOutputs` back to
/// the frontend.
pub output_push: PushSocket,
}
/// Fully connected mock engine transport state used by tests.
pub struct MockEngineConnections {
/// Decoded INIT message sent by the frontend during handshake.
pub init: HandshakeInitMessage,
/// Socket used to receive frontend requests.
pub dealer: DealerSocket,
/// Socket used to publish normal request outputs back to the frontend.
pub push: PushSocket,
/// Optional coordinator sockets when the client enabled the in-process
/// coordinator.
pub coordinator: Option<MockCoordinatorConnections>,
}
/// Complete the engine-core handshake and connect mock input/output sockets
/// plus optional coordinator sockets.
pub async fn setup_mock_engine_connections(
pub async fn setup_mock_engine_sockets(
engine_handshake: String,
engine_id: impl Into<EngineId>,
) -> MockEngineConnections {
// Wait for the client to bind the handshake socket before connecting.
// A fixed sleep is racy under CI load; instead poll for the socket file.
let socket_path = engine_handshake
.strip_prefix("ipc://")
.expect("handshake address must be ipc://");
for _ in 0..100 {
if Path::new(socket_path).exists() {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
let peer_identity = PeerIdentity::try_from(engine_id.into()).expect("peer id");
let mut options = SocketOptions::default();
options.peer_identity(peer_identity.clone());
let mut handshake = DealerSocket::with_options(options);
handshake
.connect(&engine_handshake)
) -> MockEngineSockets {
connect_to_frontend(engine_handshake, engine_id, test_mock_engine_config())
.await
.expect("connect mock engine handshake socket");
handshake
.send(ZmqMessage::from(
rmp_serde::to_vec_named(&ready_message("HELLO")).expect("encode HELLO ready message"),
))
.await
.expect("send HELLO ready message");
let init_frames = handshake.recv().await.expect("receive handshake init message").into_vec();
assert_eq!(init_frames.len(), 1);
let init: HandshakeInitMessage =
rmp_serde::from_slice(init_frames[0].as_ref()).expect("decode handshake init message");
let mut input_options = SocketOptions::default();
input_options.peer_identity(peer_identity);
let mut dealer = DealerSocket::with_options(input_options);
dealer
.connect(&init.addresses.inputs[0])
.await
.expect("connect mock engine input socket");
dealer
.send(ZmqMessage::from(ready_response_payload()))
.await
.expect("send mock engine input ready frame");
let mut push = PushSocket::new();
push.connect(&init.addresses.outputs[0])
.await
.expect("connect mock engine output socket");
let coordinator = match (
init.addresses.coordinator_input.as_deref(),
init.addresses.coordinator_output.as_deref(),
) {
(Some(coordinator_input), Some(coordinator_output)) => {
let mut input_sub = SubSocket::new();
input_sub
.connect(coordinator_input)
.await
.expect("connect mock engine coordinator input socket");
input_sub
.subscribe("")
.await
.expect("subscribe mock engine coordinator input socket");
let mut output_push = PushSocket::new();
output_push
.connect(coordinator_output)
.await
.expect("connect mock engine coordinator output socket");
let ready =
input_sub.recv().await.expect("receive coordinator READY marker").into_vec();
assert_eq!(ready.len(), 1);
assert_eq!(ready[0].as_ref(), b"READY");
Some(MockCoordinatorConnections {
input_sub,
output_push,
})
}
(None, None) => None,
_ => panic!("coordinator handshake addresses must be both present or both absent"),
};
handshake
.send(ZmqMessage::from(
rmp_serde::to_vec_named(&ready_message("READY")).expect("encode READY ready message"),
))
.await
.expect("send READY ready message");
MockEngineConnections {
init,
dealer,
push,
coordinator,
}
.expect("connect mock engine")
}
/// Connect one mock engine directly to already-bootstrapped frontend
@@ -211,31 +85,14 @@ pub async fn setup_bootstrapped_mock_engine(
output_address: String,
engine_id: impl Into<EngineId>,
) -> (DealerSocket, PushSocket) {
for endpoint in [&input_address, &output_address] {
if let Some(socket_path) = endpoint.strip_prefix("ipc://") {
for _ in 0..100 {
if Path::new(socket_path).exists() {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
}
let peer_identity = PeerIdentity::try_from(engine_id.into()).expect("peer id");
let mut input_options = SocketOptions::default();
input_options.peer_identity(peer_identity);
let mut dealer = DealerSocket::with_options(input_options);
dealer.connect(&input_address).await.expect("connect mock engine input socket");
dealer
.send(ZmqMessage::from(ready_response_payload()))
.await
.expect("send mock engine input ready frame");
let mut push = PushSocket::new();
push.connect(&output_address).await.expect("connect mock engine output socket");
(dealer, push)
connect_to_bootstrapped_frontend(
input_address,
output_address,
engine_id,
test_mock_engine_config(),
)
.await
.expect("connect bootstrapped mock engine")
}
/// Complete the engine-core handshake and connect mock input/output sockets.
@@ -247,9 +104,11 @@ pub async fn setup_mock_engine_with_init(
engine_handshake: String,
engine_id: impl Into<EngineId>,
) -> (HandshakeInitMessage, DealerSocket, PushSocket) {
let MockEngineConnections {
init, dealer, push, ..
} = setup_mock_engine_connections(engine_handshake, engine_id).await;
let MockEngineSockets {
init, data_sockets, ..
} = setup_mock_engine_sockets(engine_handshake, engine_id).await;
let MockEngineDataSockets { dealer, push } =
data_sockets.into_iter().next().expect("mock engine data socket");
(init, dealer, push)
}
+20 -17
View File
@@ -30,7 +30,7 @@ use crate::protocol::{
EngineCoreRequestType, EngineCoreSamplingParams, decode_engine_core_outputs,
};
use crate::test_utils::{
IpcNamespace, setup_bootstrapped_mock_engine, setup_mock_engine_connections,
IpcNamespace, setup_bootstrapped_mock_engine, setup_mock_engine_sockets,
setup_mock_engine_with_init, spawn_mock_engine_task,
};
use crate::{
@@ -477,8 +477,8 @@ async fn coordinator_handshake_includes_engine_control_addresses() {
let (init_tx, init_rx) = oneshot::channel();
let (shutdown_tx, shutdown_rx) = oneshot::channel();
let engine_task = tokio::spawn(async move {
let connections = setup_mock_engine_connections(handshake_address, &engine_id).await;
let _ = init_tx.send(connections.init.clone());
let sockets = setup_mock_engine_sockets(handshake_address, &engine_id).await;
let _ = init_tx.send(sockets.init.clone());
let _ = shutdown_rx.await;
});
@@ -515,14 +515,15 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
let engine0_task = tokio::spawn({
let handshake_address = handshake_address.clone();
async move {
let mut engine = setup_mock_engine_connections(handshake_address, &[0x00, 0x00]).await;
let mut engine = setup_mock_engine_sockets(handshake_address, &[0x00, 0x00]).await;
let mut coordinator =
engine.coordinator.take().expect("coordinator sockets should be present");
let data_socket = engine.data_sockets.first_mut().expect("data socket");
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (0, 0));
let add = recv_engine_message(&mut engine.dealer).await;
let add = recv_engine_message(&mut data_socket.dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
assert_eq!(request.request_id, "req-1");
@@ -538,7 +539,7 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
);
send_outputs(
&mut engine.push,
&mut data_socket.push,
EngineCoreOutputs {
engine_index: 0,
outputs: vec![request_output(
@@ -565,14 +566,14 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (1, 0));
let add = recv_engine_message(&mut engine.dealer).await;
let add = recv_engine_message(&mut data_socket.dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
assert_eq!(request.request_id, "req-3");
assert_eq!(request.current_wave, 1);
send_outputs(
&mut engine.push,
&mut data_socket.push,
EngineCoreOutputs {
engine_index: 0,
outputs: vec![request_output(
@@ -594,14 +595,15 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
let engine1_task = tokio::spawn({
let handshake_address = handshake_address.clone();
async move {
let mut engine = setup_mock_engine_connections(handshake_address, &[0x01, 0x00]).await;
let mut engine = setup_mock_engine_sockets(handshake_address, &[0x01, 0x00]).await;
let mut coordinator =
engine.coordinator.take().expect("coordinator sockets should be present");
let data_socket = engine.data_sockets.first_mut().expect("data socket");
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (0, 0));
let add = recv_engine_message(&mut engine.dealer).await;
let add = recv_engine_message(&mut data_socket.dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
assert_eq!(request.request_id, "req-2");
@@ -617,7 +619,7 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
);
send_outputs(
&mut engine.push,
&mut data_socket.push,
EngineCoreOutputs {
engine_index: 1,
outputs: vec![request_output(
@@ -637,7 +639,7 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
assert!(
timeout(
Duration::from_millis(200),
recv_engine_message(&mut engine.dealer)
recv_engine_message(&mut data_socket.dealer)
)
.await
.is_err()
@@ -712,7 +714,7 @@ async fn coordinator_rebroadcasts_engine_start_wave_control() {
let engine0_task = tokio::spawn({
let handshake_address = handshake_address.clone();
async move {
let mut engine = setup_mock_engine_connections(handshake_address, &[0x00, 0x00]).await;
let mut engine = setup_mock_engine_sockets(handshake_address, &[0x00, 0x00]).await;
let mut coordinator =
engine.coordinator.take().expect("coordinator sockets should be present");
@@ -727,7 +729,7 @@ async fn coordinator_rebroadcasts_engine_start_wave_control() {
let engine1_task = tokio::spawn({
let handshake_address = handshake_address.clone();
async move {
let mut engine = setup_mock_engine_connections(handshake_address, &[0x01, 0x00]).await;
let mut engine = setup_mock_engine_sockets(handshake_address, &[0x01, 0x00]).await;
let mut coordinator =
engine.coordinator.take().expect("coordinator sockets should be present");
@@ -778,9 +780,10 @@ async fn coordinator_accepts_stats_only_outputs() {
let (shutdown_tx, shutdown_rx) = oneshot::channel();
let engine_task = tokio::spawn(async move {
let mut engine = setup_mock_engine_connections(handshake_address, &[0x00, 0x00]).await;
let mut engine = setup_mock_engine_sockets(handshake_address, &[0x00, 0x00]).await;
let mut coordinator =
engine.coordinator.take().expect("coordinator sockets should be present");
let data_socket = engine.data_sockets.first_mut().expect("data socket");
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (0, 0));
@@ -799,13 +802,13 @@ async fn coordinator_accepts_stats_only_outputs() {
)
.await;
let add = recv_engine_message(&mut engine.dealer).await;
let add = recv_engine_message(&mut data_socket.dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
assert_eq!(request.request_id, "req-stats");
send_outputs(
&mut engine.push,
&mut data_socket.push,
EngineCoreOutputs {
engine_index: 0,
outputs: vec![request_output(