diff --git a/rust/src/engine-core-client/src/coordinator/bootstrap.rs b/rust/src/engine-core-client/src/coordinator/bootstrap.rs index 68229085d2..17c346c933 100644 --- a/rust/src/engine-core-client/src/coordinator/bootstrap.rs +++ b/rust/src/engine-core-client/src/coordinator/bootstrap.rs @@ -63,22 +63,9 @@ async fn wait_for_engine_subscriptions( timeout: ready_timeout, } })??; - if message.len() != 1 { + if !is_engine_subscription(&message) { bail_unexpected_handshake_message!( - "expected 1 frame for coordinator subscription, got {}", - message.len() - ); - } - - let frame = message - .into_vec() - .into_iter() - .next() - .expect("single-frame coordinator subscription message"); - if frame.as_ref() != [0x01] { - bail_unexpected_handshake_message!( - "expected coordinator subscription frame [0x01], got {:?}", - frame.as_ref() + "expected coordinator subscription [0x01], got {message:?}" ); } received += 1; @@ -87,6 +74,13 @@ async fn wait_for_engine_subscriptions( Ok(()) } +/// Whether `message` is the single-frame subscribe-all notification (`[0x01]`) +/// that an engine's XSUB sends whenever it connects or reconnects to the +/// coordinator broadcast socket. +pub(super) fn is_engine_subscription(message: &ZmqMessage) -> bool { + message.len() == 1 && message.get(0).is_some_and(|frame| frame.as_ref() == [0x01]) +} + /// Send the coordinator READY marker to all subscribed engines. async fn send_ready_to_engines(input_socket: &mut XPubSocket) -> Result<()> { input_socket.send(ZmqMessage::from(Bytes::from_static(b"READY"))).await?; diff --git a/rust/src/engine-core-client/src/coordinator/inproc.rs b/rust/src/engine-core-client/src/coordinator/inproc.rs index bf4c23cc13..cbd69095f8 100644 --- a/rust/src/engine-core-client/src/coordinator/inproc.rs +++ b/rust/src/engine-core-client/src/coordinator/inproc.rs @@ -7,10 +7,11 @@ use serde_tuple::{Deserialize_tuple, Serialize_tuple}; use thiserror_ext::AsReport; use tokio::sync::mpsc; use tracing::{debug, warn}; -use zeromq::prelude::SocketSend; -use zeromq::{XPubSocket, ZmqMessage}; +use zeromq::prelude::{SocketRecv, SocketSend}; +use zeromq::{XPubSocket, ZmqError, ZmqMessage}; use crate::client::imp::ClientInner; +use crate::coordinator::bootstrap::is_engine_subscription; use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState}; use crate::error::{Error, Result, bail_unexpected_coordinator_output}; use crate::protocol::encode_msgpack; @@ -187,6 +188,34 @@ impl InProcCoordinatorRunner { let result: Result<()> = async { loop { tokio::select! { + // Keep receiving engine subscriptions: zmq.rs XPUB registers a + // subscription only when `recv()` yields it (unlike libzmq), so an + // engine whose XSUB reconnects gets no broadcasts until we read its + // subscribe-all. If a wave is running, replay it so that engine does + // not miss the `START_DP_WAVE` sent while it was disconnected. + subscription = self.coordinator_input.recv() => { + match subscription { + Ok(message) => { + if is_engine_subscription(&message) { + let state = *self.state.lock(); + if state.engines_running { + debug!( + current_wave = state.current_wave, + "replaying active DP wave after engine subscription" + ); + self.broadcast_start_wave(state.current_wave, None).await?; + } + } + } + Err(error @ (ZmqError::Codec(_) | ZmqError::Network(_))) => { + warn!( + error = %error, + "coordinator subscription peer disconnected" + ); + } + Err(error) => return Err(error.into()), + } + } // Received frontend-originated command from the handle. command = self.command_rx.recv() => { let Some(command) = command else { diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index 43eb88db79..7b7a880bde 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -257,7 +257,7 @@ async fn recv_engine_message(dealer: &mut DealerSocket) -> Vec { dealer.recv().await.unwrap().into_vec() } -async fn recv_start_dp_wave(sub: &mut SubSocket) -> (u32, u32) { +async fn recv_start_dp_wave(sub: &mut SubSocket) -> (u32, Option) { let frames = sub.recv().await.unwrap().into_vec(); assert_eq!(frames.len(), 2); assert_eq!( @@ -607,12 +607,27 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() { let handshake_address = handshake_address.clone(); async move { let mut engine = setup_mock_engine_sockets(handshake_address, &[0x00, 0x00]).await; + let coordinator_address = engine + .init + .addresses + .coordinator_input + .clone() + .expect("coordinator input address should be present"); let mut coordinator = engine.coordinator.take().expect("coordinator sockets should be present"); let data_socket = engine.data_sockets.first_mut().expect("data socket"); let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await; - assert_eq!((wave, exclude_engine), (0, 0)); + assert_eq!((wave, exclude_engine), (0, Some(0))); + + let mut late_subscriber = SubSocket::new(); + late_subscriber.connect(&coordinator_address).await.unwrap(); + late_subscriber.subscribe("").await.unwrap(); + + let replay = recv_start_dp_wave(&mut late_subscriber).await; + assert_eq!(replay, (0, None)); + let replay = recv_start_dp_wave(&mut coordinator.input_sub).await; + assert_eq!(replay, (0, None)); let add = recv_engine_message(&mut data_socket.dealer).await; assert_eq!(add[0].as_ref(), &[0x00]); @@ -656,7 +671,7 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() { .await; let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await; - assert_eq!((wave, exclude_engine), (1, 0)); + assert_eq!((wave, exclude_engine), (1, Some(0))); let add = recv_engine_message(&mut data_socket.dealer).await; assert_eq!(add[0].as_ref(), &[0x00]); @@ -693,7 +708,9 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() { let data_socket = engine.data_sockets.first_mut().expect("data socket"); let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await; - assert_eq!((wave, exclude_engine), (0, 0)); + assert_eq!((wave, exclude_engine), (0, Some(0))); + let replay = recv_start_dp_wave(&mut coordinator.input_sub).await; + assert_eq!(replay, (0, None)); let add = recv_engine_message(&mut data_socket.dealer).await; assert_eq!(add[0].as_ref(), &[0x00]); @@ -727,7 +744,7 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() { .await; let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await; - assert_eq!((wave, exclude_engine), (1, 0)); + assert_eq!((wave, exclude_engine), (1, Some(0))); assert!( timeout( @@ -812,7 +829,7 @@ async fn coordinator_rebroadcasts_engine_start_wave_control() { engine.coordinator.take().expect("coordinator sockets should be present"); let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await; - assert_eq!((wave, exclude_engine), (4, 1)); + assert_eq!((wave, exclude_engine), (4, Some(1))); let _ = shutdown0_rx.await; } @@ -838,7 +855,7 @@ async fn coordinator_rebroadcasts_engine_start_wave_control() { .await; let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await; - assert_eq!((wave, exclude_engine), (4, 1)); + assert_eq!((wave, exclude_engine), (4, Some(1))); let _ = shutdown1_rx.await; } @@ -880,7 +897,7 @@ async fn coordinator_accepts_stats_only_outputs() { let data_socket = engine.data_sockets.first_mut().expect("data socket"); let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await; - assert_eq!((wave, exclude_engine), (0, 0)); + assert_eq!((wave, exclude_engine), (0, Some(0))); send_outputs( &mut coordinator.output_push,