[Bugfix][Rust Frontend] Restore DP wave delivery after XPUB reconnects (#60249)

Signed-off-by: reidliu41 <reid201711@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Reid
2026-10-12 19:12:43 +08:00
committed by GitHub
co-authored by Bugen Zhao Claude Opus 5.5
parent f5cda48c90
commit 32b05addc1
3 changed files with 65 additions and 25 deletions
@@ -63,22 +63,9 @@ async fn wait_for_engine_subscriptions(
timeout: ready_timeout,
}
})??;
if message.len() != 1 {
if !is_engine_subscription(&message) {
bail_unexpected_handshake_message!(
"expected 1 frame for coordinator subscription, got {}",
message.len()
);
}
let frame = message
.into_vec()
.into_iter()
.next()
.expect("single-frame coordinator subscription message");
if frame.as_ref() != [0x01] {
bail_unexpected_handshake_message!(
"expected coordinator subscription frame [0x01], got {:?}",
frame.as_ref()
"expected coordinator subscription [0x01], got {message:?}"
);
}
received += 1;
@@ -87,6 +74,13 @@ async fn wait_for_engine_subscriptions(
Ok(())
}
/// Whether `message` is the single-frame subscribe-all notification (`[0x01]`)
/// that an engine's XSUB sends whenever it connects or reconnects to the
/// coordinator broadcast socket.
pub(super) fn is_engine_subscription(message: &ZmqMessage) -> bool {
message.len() == 1 && message.get(0).is_some_and(|frame| frame.as_ref() == [0x01])
}
/// Send the coordinator READY marker to all subscribed engines.
async fn send_ready_to_engines(input_socket: &mut XPubSocket) -> Result<()> {
input_socket.send(ZmqMessage::from(Bytes::from_static(b"READY"))).await?;
@@ -7,10 +7,11 @@ use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use thiserror_ext::AsReport;
use tokio::sync::mpsc;
use tracing::{debug, warn};
use zeromq::prelude::SocketSend;
use zeromq::{XPubSocket, ZmqMessage};
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{XPubSocket, ZmqError, ZmqMessage};
use crate::client::imp::ClientInner;
use crate::coordinator::bootstrap::is_engine_subscription;
use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
use crate::error::{Error, Result, bail_unexpected_coordinator_output};
use crate::protocol::encode_msgpack;
@@ -187,6 +188,34 @@ impl InProcCoordinatorRunner {
let result: Result<()> = async {
loop {
tokio::select! {
// Keep receiving engine subscriptions: zmq.rs XPUB registers a
// subscription only when `recv()` yields it (unlike libzmq), so an
// engine whose XSUB reconnects gets no broadcasts until we read its
// subscribe-all. If a wave is running, replay it so that engine does
// not miss the `START_DP_WAVE` sent while it was disconnected.
subscription = self.coordinator_input.recv() => {
match subscription {
Ok(message) => {
if is_engine_subscription(&message) {
let state = *self.state.lock();
if state.engines_running {
debug!(
current_wave = state.current_wave,
"replaying active DP wave after engine subscription"
);
self.broadcast_start_wave(state.current_wave, None).await?;
}
}
}
Err(error @ (ZmqError::Codec(_) | ZmqError::Network(_))) => {
warn!(
error = %error,
"coordinator subscription peer disconnected"
);
}
Err(error) => return Err(error.into()),
}
}
// Received frontend-originated command from the handle.
command = self.command_rx.recv() => {
let Some(command) = command else {
@@ -257,7 +257,7 @@ async fn recv_engine_message(dealer: &mut DealerSocket) -> Vec<bytes::Bytes> {
dealer.recv().await.unwrap().into_vec()
}
async fn recv_start_dp_wave(sub: &mut SubSocket) -> (u32, u32) {
async fn recv_start_dp_wave(sub: &mut SubSocket) -> (u32, Option<u32>) {
let frames = sub.recv().await.unwrap().into_vec();
assert_eq!(frames.len(), 2);
assert_eq!(
@@ -607,12 +607,27 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
let handshake_address = handshake_address.clone();
async move {
let mut engine = setup_mock_engine_sockets(handshake_address, &[0x00, 0x00]).await;
let coordinator_address = engine
.init
.addresses
.coordinator_input
.clone()
.expect("coordinator input address should be present");
let mut coordinator =
engine.coordinator.take().expect("coordinator sockets should be present");
let data_socket = engine.data_sockets.first_mut().expect("data socket");
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (0, 0));
assert_eq!((wave, exclude_engine), (0, Some(0)));
let mut late_subscriber = SubSocket::new();
late_subscriber.connect(&coordinator_address).await.unwrap();
late_subscriber.subscribe("").await.unwrap();
let replay = recv_start_dp_wave(&mut late_subscriber).await;
assert_eq!(replay, (0, None));
let replay = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!(replay, (0, None));
let add = recv_engine_message(&mut data_socket.dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
@@ -656,7 +671,7 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
.await;
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (1, 0));
assert_eq!((wave, exclude_engine), (1, Some(0)));
let add = recv_engine_message(&mut data_socket.dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
@@ -693,7 +708,9 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
let data_socket = engine.data_sockets.first_mut().expect("data socket");
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (0, 0));
assert_eq!((wave, exclude_engine), (0, Some(0)));
let replay = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!(replay, (0, None));
let add = recv_engine_message(&mut data_socket.dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
@@ -727,7 +744,7 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
.await;
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (1, 0));
assert_eq!((wave, exclude_engine), (1, Some(0)));
assert!(
timeout(
@@ -812,7 +829,7 @@ async fn coordinator_rebroadcasts_engine_start_wave_control() {
engine.coordinator.take().expect("coordinator sockets should be present");
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (4, 1));
assert_eq!((wave, exclude_engine), (4, Some(1)));
let _ = shutdown0_rx.await;
}
@@ -838,7 +855,7 @@ async fn coordinator_rebroadcasts_engine_start_wave_control() {
.await;
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (4, 1));
assert_eq!((wave, exclude_engine), (4, Some(1)));
let _ = shutdown1_rx.await;
}
@@ -880,7 +897,7 @@ async fn coordinator_accepts_stats_only_outputs() {
let data_socket = engine.data_sockets.first_mut().expect("data socket");
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (0, 0));
assert_eq!((wave, exclude_engine), (0, Some(0)));
send_outputs(
&mut coordinator.output_push,