[Bugfix][Rust Frontend] Restore DP wave delivery after XPUB reconnects (#60249)
Signed-off-by: reidliu41 <reid201711@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Bugen Zhao
Claude Opus 5.5
parent
f5cda48c90
commit
32b05addc1
@@ -63,22 +63,9 @@ async fn wait_for_engine_subscriptions(
|
||||
timeout: ready_timeout,
|
||||
}
|
||||
})??;
|
||||
if message.len() != 1 {
|
||||
if !is_engine_subscription(&message) {
|
||||
bail_unexpected_handshake_message!(
|
||||
"expected 1 frame for coordinator subscription, got {}",
|
||||
message.len()
|
||||
);
|
||||
}
|
||||
|
||||
let frame = message
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("single-frame coordinator subscription message");
|
||||
if frame.as_ref() != [0x01] {
|
||||
bail_unexpected_handshake_message!(
|
||||
"expected coordinator subscription frame [0x01], got {:?}",
|
||||
frame.as_ref()
|
||||
"expected coordinator subscription [0x01], got {message:?}"
|
||||
);
|
||||
}
|
||||
received += 1;
|
||||
@@ -87,6 +74,13 @@ async fn wait_for_engine_subscriptions(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether `message` is the single-frame subscribe-all notification (`[0x01]`)
|
||||
/// that an engine's XSUB sends whenever it connects or reconnects to the
|
||||
/// coordinator broadcast socket.
|
||||
pub(super) fn is_engine_subscription(message: &ZmqMessage) -> bool {
|
||||
message.len() == 1 && message.get(0).is_some_and(|frame| frame.as_ref() == [0x01])
|
||||
}
|
||||
|
||||
/// Send the coordinator READY marker to all subscribed engines.
|
||||
async fn send_ready_to_engines(input_socket: &mut XPubSocket) -> Result<()> {
|
||||
input_socket.send(ZmqMessage::from(Bytes::from_static(b"READY"))).await?;
|
||||
|
||||
@@ -7,10 +7,11 @@ use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
use thiserror_ext::AsReport;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, warn};
|
||||
use zeromq::prelude::SocketSend;
|
||||
use zeromq::{XPubSocket, ZmqMessage};
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{XPubSocket, ZmqError, ZmqMessage};
|
||||
|
||||
use crate::client::imp::ClientInner;
|
||||
use crate::coordinator::bootstrap::is_engine_subscription;
|
||||
use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
|
||||
use crate::error::{Error, Result, bail_unexpected_coordinator_output};
|
||||
use crate::protocol::encode_msgpack;
|
||||
@@ -187,6 +188,34 @@ impl InProcCoordinatorRunner {
|
||||
let result: Result<()> = async {
|
||||
loop {
|
||||
tokio::select! {
|
||||
// Keep receiving engine subscriptions: zmq.rs XPUB registers a
|
||||
// subscription only when `recv()` yields it (unlike libzmq), so an
|
||||
// engine whose XSUB reconnects gets no broadcasts until we read its
|
||||
// subscribe-all. If a wave is running, replay it so that engine does
|
||||
// not miss the `START_DP_WAVE` sent while it was disconnected.
|
||||
subscription = self.coordinator_input.recv() => {
|
||||
match subscription {
|
||||
Ok(message) => {
|
||||
if is_engine_subscription(&message) {
|
||||
let state = *self.state.lock();
|
||||
if state.engines_running {
|
||||
debug!(
|
||||
current_wave = state.current_wave,
|
||||
"replaying active DP wave after engine subscription"
|
||||
);
|
||||
self.broadcast_start_wave(state.current_wave, None).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error @ (ZmqError::Codec(_) | ZmqError::Network(_))) => {
|
||||
warn!(
|
||||
error = %error,
|
||||
"coordinator subscription peer disconnected"
|
||||
);
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
// Received frontend-originated command from the handle.
|
||||
command = self.command_rx.recv() => {
|
||||
let Some(command) = command else {
|
||||
|
||||
@@ -257,7 +257,7 @@ async fn recv_engine_message(dealer: &mut DealerSocket) -> Vec<bytes::Bytes> {
|
||||
dealer.recv().await.unwrap().into_vec()
|
||||
}
|
||||
|
||||
async fn recv_start_dp_wave(sub: &mut SubSocket) -> (u32, u32) {
|
||||
async fn recv_start_dp_wave(sub: &mut SubSocket) -> (u32, Option<u32>) {
|
||||
let frames = sub.recv().await.unwrap().into_vec();
|
||||
assert_eq!(frames.len(), 2);
|
||||
assert_eq!(
|
||||
@@ -607,12 +607,27 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
|
||||
let handshake_address = handshake_address.clone();
|
||||
async move {
|
||||
let mut engine = setup_mock_engine_sockets(handshake_address, &[0x00, 0x00]).await;
|
||||
let coordinator_address = engine
|
||||
.init
|
||||
.addresses
|
||||
.coordinator_input
|
||||
.clone()
|
||||
.expect("coordinator input address should be present");
|
||||
let mut coordinator =
|
||||
engine.coordinator.take().expect("coordinator sockets should be present");
|
||||
let data_socket = engine.data_sockets.first_mut().expect("data socket");
|
||||
|
||||
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
|
||||
assert_eq!((wave, exclude_engine), (0, 0));
|
||||
assert_eq!((wave, exclude_engine), (0, Some(0)));
|
||||
|
||||
let mut late_subscriber = SubSocket::new();
|
||||
late_subscriber.connect(&coordinator_address).await.unwrap();
|
||||
late_subscriber.subscribe("").await.unwrap();
|
||||
|
||||
let replay = recv_start_dp_wave(&mut late_subscriber).await;
|
||||
assert_eq!(replay, (0, None));
|
||||
let replay = recv_start_dp_wave(&mut coordinator.input_sub).await;
|
||||
assert_eq!(replay, (0, None));
|
||||
|
||||
let add = recv_engine_message(&mut data_socket.dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
@@ -656,7 +671,7 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
|
||||
.await;
|
||||
|
||||
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
|
||||
assert_eq!((wave, exclude_engine), (1, 0));
|
||||
assert_eq!((wave, exclude_engine), (1, Some(0)));
|
||||
|
||||
let add = recv_engine_message(&mut data_socket.dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
@@ -693,7 +708,9 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
|
||||
let data_socket = engine.data_sockets.first_mut().expect("data socket");
|
||||
|
||||
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
|
||||
assert_eq!((wave, exclude_engine), (0, 0));
|
||||
assert_eq!((wave, exclude_engine), (0, Some(0)));
|
||||
let replay = recv_start_dp_wave(&mut coordinator.input_sub).await;
|
||||
assert_eq!(replay, (0, None));
|
||||
|
||||
let add = recv_engine_message(&mut data_socket.dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
@@ -727,7 +744,7 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
|
||||
.await;
|
||||
|
||||
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
|
||||
assert_eq!((wave, exclude_engine), (1, 0));
|
||||
assert_eq!((wave, exclude_engine), (1, Some(0)));
|
||||
|
||||
assert!(
|
||||
timeout(
|
||||
@@ -812,7 +829,7 @@ async fn coordinator_rebroadcasts_engine_start_wave_control() {
|
||||
engine.coordinator.take().expect("coordinator sockets should be present");
|
||||
|
||||
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
|
||||
assert_eq!((wave, exclude_engine), (4, 1));
|
||||
assert_eq!((wave, exclude_engine), (4, Some(1)));
|
||||
|
||||
let _ = shutdown0_rx.await;
|
||||
}
|
||||
@@ -838,7 +855,7 @@ async fn coordinator_rebroadcasts_engine_start_wave_control() {
|
||||
.await;
|
||||
|
||||
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
|
||||
assert_eq!((wave, exclude_engine), (4, 1));
|
||||
assert_eq!((wave, exclude_engine), (4, Some(1)));
|
||||
|
||||
let _ = shutdown1_rx.await;
|
||||
}
|
||||
@@ -880,7 +897,7 @@ async fn coordinator_accepts_stats_only_outputs() {
|
||||
let data_socket = engine.data_sockets.first_mut().expect("data socket");
|
||||
|
||||
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
|
||||
assert_eq!((wave, exclude_engine), (0, 0));
|
||||
assert_eq!((wave, exclude_engine), (0, Some(0)));
|
||||
|
||||
send_outputs(
|
||||
&mut coordinator.output_push,
|
||||
|
||||
Reference in New Issue
Block a user