[DP] Engines own wave wake-ups; drop the front-end FIRST_REQ (#59559)
Signed-off-by: fh3 <a@b> Co-authored-by: fh3 <a@b> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
fh3
Claude Opus 5.5
parent
b544587408
commit
a94c315711
@@ -657,12 +657,9 @@ impl EngineCoreClient {
|
||||
);
|
||||
|
||||
let result: Result<()> = async {
|
||||
// The engine that receives it wakes the other ranks if they are idle.
|
||||
if let Some(coordinator) = self.coordinator.as_ref() {
|
||||
let snapshot = coordinator.snapshot();
|
||||
req.current_wave = snapshot.current_wave;
|
||||
if !snapshot.engines_running {
|
||||
coordinator.notify_first_request(engine_id.clone())?;
|
||||
}
|
||||
req.current_wave = coordinator.snapshot().current_wave;
|
||||
}
|
||||
|
||||
debug!(
|
||||
|
||||
@@ -3,34 +3,16 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
use serde_tuple::Deserialize_tuple;
|
||||
use thiserror_ext::AsReport;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, warn};
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::prelude::SocketRecv;
|
||||
use zeromq::{XSubSocket, ZmqMessage};
|
||||
|
||||
use crate::client::imp::ClientInner;
|
||||
use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
|
||||
use crate::coordinator::handle::CoordinatorState;
|
||||
use crate::error::{Error, Result, bail_unexpected_coordinator_output};
|
||||
use crate::protocol::{OpaqueValue, decode_msgpack, encode_msgpack};
|
||||
|
||||
/// Frontend-to-coordinator wakeup message sent when the first request arrives
|
||||
/// while all engines are paused.
|
||||
///
|
||||
/// This matches the frontend-side msgpack tuple sent by Python
|
||||
/// `DPAsyncMPClient._ensure_stats_update_task` to the coordinator front socket.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/694449050f8dac3d9853e97e518b4a43ec52106a/vllm/v1/engine/core_client.py#L1230-L1236>
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]
|
||||
struct CoordinatorWakeupMessage {
|
||||
/// Engine index that already has the triggering request and should be
|
||||
/// excluded from the coordinator's `START_DP_WAVE` rebroadcast.
|
||||
exclude_engine_index: u32,
|
||||
/// DP wave number observed by the frontend when the request was admitted.
|
||||
wave: u32,
|
||||
}
|
||||
use crate::protocol::{OpaqueValue, decode_msgpack};
|
||||
|
||||
/// Coordinator-to-frontend state publish received on the front-side coordinator
|
||||
/// socket.
|
||||
@@ -56,56 +38,17 @@ struct CoordinatorStateUpdate {
|
||||
|
||||
/// Background half of an external Python-owned coordinator connection.
|
||||
///
|
||||
/// This owns the command receiver and one frontend-facing XSUB socket. It
|
||||
/// mirrors the subset of Python's coordinator protocol needed by the Rust
|
||||
/// bootstrapped frontend: receive `(counts, wave, running)` publishes, ignore
|
||||
/// `counts`, and send `(exclude_engine_index, wave)` wakeup messages when the
|
||||
/// first request arrives while engines are paused.
|
||||
/// This owns one frontend-facing XSUB socket. It mirrors the subset of Python's
|
||||
/// coordinator protocol needed by the Rust bootstrapped frontend: receive
|
||||
/// `(counts, wave, running)` publishes and ignore `counts`.
|
||||
pub(crate) struct ExternalCoordinatorService {
|
||||
state: Arc<CoordinatorState>,
|
||||
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
|
||||
socket: XSubSocket,
|
||||
}
|
||||
|
||||
impl ExternalCoordinatorService {
|
||||
pub(super) fn new(
|
||||
state: Arc<CoordinatorState>,
|
||||
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
|
||||
socket: XSubSocket,
|
||||
) -> Self {
|
||||
Self {
|
||||
state,
|
||||
command_rx,
|
||||
socket,
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply one frontend-originated command to the external coordinator state
|
||||
/// machine.
|
||||
async fn handle_command(&mut self, command: CoordinatorCommand) -> Result<()> {
|
||||
match command {
|
||||
CoordinatorCommand::FirstRequest {
|
||||
target_engine_id,
|
||||
wave,
|
||||
} => {
|
||||
let target_engine_index = target_engine_id.engine_index().ok_or_else(|| {
|
||||
Error::UnsupportedCoordinatorEngineId {
|
||||
engine_id: target_engine_id.to_vec(),
|
||||
}
|
||||
})?;
|
||||
debug!(
|
||||
wave,
|
||||
exclude_engine_index = target_engine_index,
|
||||
"notifying external coordinator about first request while engines were paused"
|
||||
);
|
||||
let payload = encode_msgpack(&CoordinatorWakeupMessage {
|
||||
exclude_engine_index: target_engine_index,
|
||||
wave,
|
||||
})?;
|
||||
self.socket.send(ZmqMessage::from(payload)).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
pub(super) fn new(state: Arc<CoordinatorState>, socket: XSubSocket) -> Self {
|
||||
Self { state, socket }
|
||||
}
|
||||
|
||||
/// Apply one publish received from the xsub socket containing a coordinator
|
||||
@@ -136,26 +79,14 @@ impl ExternalCoordinatorService {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drive the coordinator event loop until either side of the control plane
|
||||
/// is closed or a fatal error is observed.
|
||||
/// Drive the coordinator event loop until the task is aborted or a fatal
|
||||
/// error is observed.
|
||||
pub(crate) async fn run(mut self, inner: Arc<ClientInner>) {
|
||||
let result: Result<()> = async {
|
||||
loop {
|
||||
tokio::select! {
|
||||
// Received frontend-originated command from the handle.
|
||||
command = self.command_rx.recv() => {
|
||||
let Some(command) = command else {
|
||||
warn!("external coordinator command channel closed, shutting down service");
|
||||
return Ok(());
|
||||
};
|
||||
self.handle_command(command).await?;
|
||||
}
|
||||
// Received publish from the external coordinator socket.
|
||||
publish = self.socket.recv() => {
|
||||
let publish = publish.map_err(Error::from)?;
|
||||
self.handle_publish(publish).await?;
|
||||
}
|
||||
}
|
||||
// Received publish from the external coordinator socket.
|
||||
let publish = self.socket.recv().await.map_err(Error::from)?;
|
||||
self.handle_publish(publish).await?;
|
||||
}
|
||||
}
|
||||
.await;
|
||||
|
||||
@@ -4,101 +4,56 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use tokio::sync::mpsc;
|
||||
use zeromq::prelude::Socket;
|
||||
use zeromq::{XPubSocket, XSubSocket};
|
||||
|
||||
use crate::coordinator::external::ExternalCoordinatorService;
|
||||
use crate::coordinator::inproc::InProcCoordinatorRunner;
|
||||
use crate::error::{Error, Result, bail_control_closed};
|
||||
use crate::transport::EngineId;
|
||||
use crate::error::Result;
|
||||
|
||||
/// Snapshot to the coordinator state for request routing and stamping.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) struct CoordinatorStateSnapshot {
|
||||
/// The current DP wave, which will be stamped on outgoing requests.
|
||||
pub current_wave: u32,
|
||||
/// Whether the engines are currently running or paused, which determines if
|
||||
/// the frontend must trigger a new wave on the next request.
|
||||
/// Whether the engines are currently running or paused, as last reported
|
||||
/// by the engines themselves.
|
||||
pub engines_running: bool,
|
||||
}
|
||||
|
||||
impl CoordinatorStateSnapshot {
|
||||
/// Resume the engines for a `FirstRequest` and return the wave to broadcast
|
||||
/// and the engine to exclude from the wakeup.
|
||||
///
|
||||
/// The request may have been stamped with a `request_wave` older than
|
||||
/// `current_wave` if a `WaveComplete` advanced it after the command was
|
||||
/// enqueued. Such a request still needs serving, so the current wave is
|
||||
/// broadcast to every engine (`exclude = None`); the wave is never rewound.
|
||||
/// A non-stale request excludes the engine that already received it. Mirrors
|
||||
/// the Python coordinator's front-end path.
|
||||
pub(crate) fn start_wave_for_first_request(
|
||||
&mut self,
|
||||
request_wave: u32,
|
||||
target_engine_index: u32,
|
||||
) -> (u32, Option<u32>) {
|
||||
self.engines_running = true;
|
||||
let exclude = (request_wave >= self.current_wave).then_some(target_engine_index);
|
||||
(self.current_wave, exclude)
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared in-process coordinator state.
|
||||
pub(crate) type CoordinatorState = Mutex<CoordinatorStateSnapshot>;
|
||||
|
||||
/// Commands sent from the frontend request path into the background runner.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum CoordinatorCommand {
|
||||
/// The first request arrived while all engines were paused.
|
||||
///
|
||||
/// The coordinator should broadcast `START_DP_WAVE` with the current wave
|
||||
/// and the target engine index as the excluded engine.
|
||||
FirstRequest {
|
||||
target_engine_id: EngineId,
|
||||
wave: u32,
|
||||
},
|
||||
}
|
||||
|
||||
/// Frontend-facing coordinator handle used by `EngineCoreClient::call()`.
|
||||
///
|
||||
/// This side stays intentionally small: it can read the latest wave snapshot
|
||||
/// and enqueue a `FirstRequest` transition when the request path observes the
|
||||
/// system in the paused state.
|
||||
/// The frontend only reads the latest wave snapshot: waves are started by the
|
||||
/// engines, which notify the coordinator when an idle engine receives a request.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct CoordinatorHandle {
|
||||
state: Arc<CoordinatorState>,
|
||||
command_tx: mpsc::UnboundedSender<CoordinatorCommand>,
|
||||
}
|
||||
|
||||
impl CoordinatorHandle {
|
||||
fn new_parts() -> (
|
||||
Self,
|
||||
Arc<CoordinatorState>,
|
||||
mpsc::UnboundedReceiver<CoordinatorCommand>,
|
||||
) {
|
||||
fn new_parts() -> (Self, Arc<CoordinatorState>) {
|
||||
let state = Arc::new(Mutex::new(CoordinatorStateSnapshot {
|
||||
current_wave: 0,
|
||||
engines_running: false,
|
||||
}));
|
||||
let (command_tx, command_rx) = mpsc::unbounded_channel();
|
||||
(
|
||||
Self {
|
||||
state: state.clone(),
|
||||
command_tx,
|
||||
},
|
||||
state,
|
||||
command_rx,
|
||||
)
|
||||
}
|
||||
|
||||
/// Build the paired frontend handle and background runner around one
|
||||
/// engine-facing coordinator broadcast socket.
|
||||
pub(crate) fn new_inproc(coordinator_input: XPubSocket) -> (Self, InProcCoordinatorRunner) {
|
||||
let (handle, state, command_rx) = Self::new_parts();
|
||||
let (handle, state) = Self::new_parts();
|
||||
(
|
||||
handle,
|
||||
InProcCoordinatorRunner::new(state, command_rx, coordinator_input),
|
||||
InProcCoordinatorRunner::new(state, coordinator_input),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -107,41 +62,15 @@ impl CoordinatorHandle {
|
||||
pub(crate) async fn connect_external(
|
||||
coordinator_address: &str,
|
||||
) -> Result<(Self, ExternalCoordinatorService)> {
|
||||
let (handle, state, command_rx) = Self::new_parts();
|
||||
let (handle, state) = Self::new_parts();
|
||||
let mut socket = XSubSocket::new();
|
||||
socket.connect(coordinator_address).await?;
|
||||
socket.subscribe("").await?;
|
||||
Ok((
|
||||
handle,
|
||||
ExternalCoordinatorService::new(state, command_rx, socket),
|
||||
))
|
||||
Ok((handle, ExternalCoordinatorService::new(state, socket)))
|
||||
}
|
||||
|
||||
/// Snapshot the coordinator state for request routing and stamping.
|
||||
pub(crate) fn snapshot(&self) -> CoordinatorStateSnapshot {
|
||||
*self.state.lock()
|
||||
}
|
||||
|
||||
/// Notify the runner that a new request arrived while engines were paused.
|
||||
///
|
||||
/// The handle flips `engines_running` optimistically so concurrent request
|
||||
/// submissions coalesce behind one `START_DP_WAVE` broadcast instead of all
|
||||
/// trying to trigger the wave independently.
|
||||
pub(crate) fn notify_first_request(&self, target_engine_id: EngineId) -> Result<()> {
|
||||
let mut state = self.state.lock();
|
||||
if state.engines_running {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let command = CoordinatorCommand::FirstRequest {
|
||||
target_engine_id,
|
||||
wave: state.current_wave,
|
||||
};
|
||||
if self.command_tx.send(command).is_err() {
|
||||
bail_control_closed!("in-process coordinator command channel already shut down");
|
||||
}
|
||||
|
||||
state.engines_running = true;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ use zeromq::{XPubSocket, ZmqError, ZmqMessage};
|
||||
|
||||
use crate::client::imp::ClientInner;
|
||||
use crate::coordinator::bootstrap::is_engine_subscription;
|
||||
use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
|
||||
use crate::coordinator::handle::CoordinatorState;
|
||||
use crate::error::{Error, Result, bail_unexpected_coordinator_output};
|
||||
use crate::protocol::encode_msgpack;
|
||||
use crate::protocol::output::{DpControlMessage, DpControlOutput, EngineCoreOutputs};
|
||||
@@ -30,32 +30,25 @@ use crate::protocol::request::EngineCoreRequestType;
|
||||
struct StartDpWaveMessage {
|
||||
/// DP wave number that all engines should start processing.
|
||||
wave: u32,
|
||||
/// Engine index that already received the triggering request and so does not
|
||||
/// need an extra wakeup. `None` wakes every engine (used when the triggering
|
||||
/// request was for a stale wave).
|
||||
/// Engine index that started the wave and so does not need an extra wakeup.
|
||||
/// `None` wakes every engine.
|
||||
exclude_engine_index: Option<u32>,
|
||||
}
|
||||
|
||||
/// Background half of the in-process coordinator.
|
||||
///
|
||||
/// This owns the command receiver and the engine-facing coordinator input
|
||||
/// socket. It is the single place where wave transitions are serialized and
|
||||
/// where `START_DP_WAVE` broadcasts are emitted.
|
||||
/// This owns the engine-facing coordinator input socket. It is the single place
|
||||
/// where wave transitions are serialized and where `START_DP_WAVE` broadcasts
|
||||
/// are emitted.
|
||||
pub(crate) struct InProcCoordinatorRunner {
|
||||
state: Arc<CoordinatorState>,
|
||||
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
|
||||
coordinator_input: XPubSocket,
|
||||
}
|
||||
|
||||
impl InProcCoordinatorRunner {
|
||||
pub(super) fn new(
|
||||
state: Arc<CoordinatorState>,
|
||||
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
|
||||
coordinator_input: XPubSocket,
|
||||
) -> Self {
|
||||
pub(super) fn new(state: Arc<CoordinatorState>, coordinator_input: XPubSocket) -> Self {
|
||||
Self {
|
||||
state,
|
||||
command_rx,
|
||||
coordinator_input,
|
||||
}
|
||||
}
|
||||
@@ -82,34 +75,6 @@ impl InProcCoordinatorRunner {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Apply one frontend-originated command to the coordinator state machine.
|
||||
async fn handle_command(&mut self, command: CoordinatorCommand) -> Result<()> {
|
||||
match command {
|
||||
CoordinatorCommand::FirstRequest {
|
||||
target_engine_id,
|
||||
wave,
|
||||
} => {
|
||||
let target_engine_index = target_engine_id.engine_index().ok_or_else(|| {
|
||||
Error::UnsupportedCoordinatorEngineId {
|
||||
engine_id: target_engine_id.to_vec(),
|
||||
}
|
||||
})?;
|
||||
let (current_wave, exclude) = {
|
||||
let mut state = self.state.lock();
|
||||
state.start_wave_for_first_request(wave, target_engine_index)
|
||||
};
|
||||
debug!(
|
||||
current_wave,
|
||||
request_wave = wave,
|
||||
?exclude,
|
||||
"starting DP wave after first request while engines were paused"
|
||||
);
|
||||
self.broadcast_start_wave(current_wave, exclude).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Apply one engine-originated control output to the coordinator state
|
||||
/// machine.
|
||||
async fn handle_outputs(&mut self, outputs: EngineCoreOutputs) -> Result<()> {
|
||||
@@ -160,7 +125,7 @@ impl InProcCoordinatorRunner {
|
||||
debug!(
|
||||
wave,
|
||||
exclude_engine_index = engine_index,
|
||||
"starting DP wave after stale-wave notification from engine"
|
||||
"starting DP wave on notification from engine"
|
||||
);
|
||||
self.broadcast_start_wave(wave, Some(engine_index)).await?;
|
||||
}
|
||||
@@ -175,8 +140,8 @@ impl InProcCoordinatorRunner {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drive the coordinator event loop until either side of the control plane
|
||||
/// is closed or a fatal error is observed.
|
||||
/// Drive the coordinator event loop until the engine control output is
|
||||
/// closed or a fatal error is observed.
|
||||
///
|
||||
/// Any fatal error closes the main client registries so request streams and
|
||||
/// future calls observe a stable shutdown cause.
|
||||
@@ -216,14 +181,6 @@ impl InProcCoordinatorRunner {
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
// Received frontend-originated command from the handle.
|
||||
command = self.command_rx.recv() => {
|
||||
let Some(command) = command else {
|
||||
warn!("coordinator command channel closed, shutting down coordinator runner");
|
||||
return Ok(());
|
||||
};
|
||||
self.handle_command(command).await?;
|
||||
}
|
||||
// Received engine-originated control output from the coordinator socket.
|
||||
outputs = output_rx.recv() => {
|
||||
let Some(outputs) = outputs else {
|
||||
@@ -242,48 +199,3 @@ impl InProcCoordinatorRunner {
|
||||
inner.close_registries(Arc::new(error));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::coordinator::handle::CoordinatorStateSnapshot;
|
||||
|
||||
/// A `FirstRequest` for the current wave starts that wave and excludes the
|
||||
/// engine that already received the triggering request.
|
||||
#[test]
|
||||
fn first_request_for_current_wave_excludes_target() {
|
||||
let mut state = CoordinatorStateSnapshot {
|
||||
current_wave: 3,
|
||||
engines_running: false,
|
||||
};
|
||||
|
||||
let (wave, exclude) = state.start_wave_for_first_request(3, 2);
|
||||
|
||||
assert_eq!(wave, 3);
|
||||
assert_eq!(exclude, Some(2));
|
||||
assert!(state.engines_running);
|
||||
assert_eq!(state.current_wave, 3);
|
||||
}
|
||||
|
||||
/// A `FirstRequest` whose wave was superseded by a racing `WaveComplete`
|
||||
/// (`request_wave < current_wave`) must still start the request's wave: it
|
||||
/// broadcasts the current wave and wakes every engine (`exclude = None`)
|
||||
/// rather than rewinding the wave or dropping the request.
|
||||
#[test]
|
||||
fn stale_first_request_starts_current_wave_for_all_engines() {
|
||||
let mut state = CoordinatorStateSnapshot {
|
||||
current_wave: 4,
|
||||
engines_running: false,
|
||||
};
|
||||
|
||||
// Request stamped with wave 3 while the coordinator already advanced to 4.
|
||||
let (wave, exclude) = state.start_wave_for_first_request(3, 2);
|
||||
|
||||
assert_eq!(
|
||||
wave, 4,
|
||||
"must broadcast the current wave, not the stale one"
|
||||
);
|
||||
assert_eq!(exclude, None, "a stale request must wake every engine");
|
||||
assert!(state.engines_running);
|
||||
assert_eq!(state.current_wave, 4, "wave must not be rewound");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,8 +55,6 @@ pub enum Error {
|
||||
UnexpectedCoordinatorOutput { message: String },
|
||||
#[error("unexpected output on main dispatcher path: {message}")]
|
||||
UnexpectedDispatcherOutput { message: String },
|
||||
#[error("coordinator requires a Python-compatible two-byte engine id, got {engine_id:?}")]
|
||||
UnsupportedCoordinatorEngineId { engine_id: Vec<u8> },
|
||||
#[error("unsupported auxiliary frame(s): expected 1 frame, got {frame_count}")]
|
||||
UnsupportedAuxFrames { frame_count: usize },
|
||||
#[error("external coordinator mode is not implemented yet")]
|
||||
@@ -66,8 +64,6 @@ pub enum Error {
|
||||
context: &'static str,
|
||||
field: &'static str,
|
||||
},
|
||||
#[error("engine control channel closed unexpectedly: {message}")]
|
||||
ControlClosed { message: String },
|
||||
#[error("request `{request_id}` is already in flight")]
|
||||
DuplicateRequestId { request_id: String },
|
||||
#[error(
|
||||
|
||||
@@ -267,6 +267,19 @@ async fn recv_start_dp_wave(sub: &mut SubSocket) -> (u32, Option<u32>) {
|
||||
rmp_serde::from_slice(&frames[1]).expect("decode START_DP_WAVE payload")
|
||||
}
|
||||
|
||||
async fn send_start_wave(push: &mut PushSocket, engine_index: u32, wave: u32) {
|
||||
send_outputs(
|
||||
push,
|
||||
DpControlOutput {
|
||||
engine_index,
|
||||
timestamp: 0.0,
|
||||
control: DpControlMessage::StartWave(wave),
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn connect_client_with_ipc(
|
||||
config: EngineCoreClientConfig,
|
||||
ipc: &IpcNamespace,
|
||||
@@ -413,12 +426,6 @@ async fn recv_xpub_subscription(xpub: &mut XPubSocket) {
|
||||
assert_eq!(frames[0].as_ref(), b"\x01");
|
||||
}
|
||||
|
||||
async fn recv_external_coordinator_wakeup(xpub: &mut XPubSocket) -> (u32, u32) {
|
||||
let frames = recv_xpub_message(xpub).await;
|
||||
assert_eq!(frames.len(), 1);
|
||||
rmp_serde::from_slice(&frames[0]).expect("decode external coordinator wakeup")
|
||||
}
|
||||
|
||||
async fn send_external_coordinator_publish<T: serde::Serialize>(
|
||||
xpub: &mut XPubSocket,
|
||||
payload: &T,
|
||||
@@ -617,6 +624,14 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
|
||||
engine.coordinator.take().expect("coordinator sockets should be present");
|
||||
let data_socket = engine.data_sockets.first_mut().expect("data socket");
|
||||
|
||||
// An idle engine handed a request starts the wave itself.
|
||||
let add = recv_engine_message(&mut data_socket.dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
assert_eq!(request.request_id, "req-1");
|
||||
assert_eq!(request.current_wave, 0);
|
||||
send_start_wave(&mut coordinator.output_push, 0, 0).await;
|
||||
|
||||
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
|
||||
assert_eq!((wave, exclude_engine), (0, Some(0)));
|
||||
|
||||
@@ -629,12 +644,6 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
|
||||
let replay = recv_start_dp_wave(&mut coordinator.input_sub).await;
|
||||
assert_eq!(replay, (0, None));
|
||||
|
||||
let add = recv_engine_message(&mut data_socket.dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
assert_eq!(request.request_id, "req-1");
|
||||
assert_eq!(request.current_wave, 0);
|
||||
|
||||
assert!(
|
||||
timeout(
|
||||
Duration::from_millis(200),
|
||||
@@ -670,14 +679,15 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
|
||||
)
|
||||
.await;
|
||||
|
||||
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
|
||||
assert_eq!((wave, exclude_engine), (1, Some(0)));
|
||||
|
||||
let add = recv_engine_message(&mut data_socket.dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
assert_eq!(request.request_id, "req-3");
|
||||
assert_eq!(request.current_wave, 1);
|
||||
send_start_wave(&mut coordinator.output_push, 0, 1).await;
|
||||
|
||||
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
|
||||
assert_eq!((wave, exclude_engine), (1, Some(0)));
|
||||
|
||||
send_outputs(
|
||||
&mut data_socket.push,
|
||||
@@ -896,9 +906,6 @@ async fn coordinator_accepts_stats_only_outputs() {
|
||||
engine.coordinator.take().expect("coordinator sockets should be present");
|
||||
let data_socket = engine.data_sockets.first_mut().expect("data socket");
|
||||
|
||||
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
|
||||
assert_eq!((wave, exclude_engine), (0, Some(0)));
|
||||
|
||||
send_outputs(
|
||||
&mut coordinator.output_push,
|
||||
RequestBatchOutputs {
|
||||
@@ -3248,7 +3255,7 @@ async fn bootstrapped_external_coordinator_connects_and_subscribes() {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn bootstrapped_external_coordinator_updates_wave_ignores_counts_and_sends_one_wakeup() {
|
||||
async fn bootstrapped_external_coordinator_updates_wave_ignores_counts_and_sends_no_wakeup() {
|
||||
init_tracing();
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let input_address = ipc.input_endpoint();
|
||||
@@ -3289,18 +3296,11 @@ async fn bootstrapped_external_coordinator_updates_wave_ignores_counts_and_sends
|
||||
|
||||
let mut stream = client.call(sample_request()).await.unwrap().into_outputs();
|
||||
|
||||
let wakeup = timeout(
|
||||
Duration::from_secs(1),
|
||||
recv_external_coordinator_wakeup(&mut stats_socket),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(wakeup, (0, 7));
|
||||
|
||||
// The engine that receives the request starts the wave, not the frontend.
|
||||
assert!(
|
||||
timeout(
|
||||
Duration::from_millis(200),
|
||||
recv_external_coordinator_wakeup(&mut stats_socket)
|
||||
recv_xpub_message(&mut stats_socket)
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
@@ -3333,78 +3333,3 @@ async fn bootstrapped_external_coordinator_updates_wave_ignores_counts_and_sends
|
||||
|
||||
client.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn bootstrapped_external_coordinator_running_state_suppresses_wakeup() {
|
||||
init_tracing();
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let input_address = ipc.input_endpoint();
|
||||
let output_address = ipc.output_endpoint();
|
||||
let coordinator_address = ipc.endpoint("stats.sock");
|
||||
|
||||
let mut stats_socket = XPubSocket::new();
|
||||
stats_socket.bind(&coordinator_address).await.unwrap();
|
||||
|
||||
let client_task = tokio::spawn({
|
||||
let input_address = input_address.clone();
|
||||
let output_address = output_address.clone();
|
||||
let coordinator_address = coordinator_address.clone();
|
||||
async move {
|
||||
EngineCoreClient::connect(bootstrapped_test_config(
|
||||
input_address,
|
||||
output_address,
|
||||
1,
|
||||
Duration::from_secs(2),
|
||||
0,
|
||||
Some(CoordinatorMode::External {
|
||||
address: coordinator_address,
|
||||
}),
|
||||
))
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
});
|
||||
|
||||
let (mut dealer, mut push) =
|
||||
setup_bootstrapped_mock_engine(input_address, output_address, &[0x00, 0x00]).await;
|
||||
let client = client_task.await.unwrap();
|
||||
recv_xpub_subscription(&mut stats_socket).await;
|
||||
|
||||
send_external_coordinator_publish(&mut stats_socket, &(Value::Nil, 5_u32, true)).await;
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
|
||||
let mut stream = client.call(sample_request()).await.unwrap().into_outputs();
|
||||
|
||||
assert!(
|
||||
timeout(
|
||||
Duration::from_millis(200),
|
||||
recv_external_coordinator_wakeup(&mut stats_socket)
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let add = recv_engine_message(&mut dealer).await;
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
assert_eq!(request.current_wave, 5);
|
||||
|
||||
send_outputs(
|
||||
&mut push,
|
||||
RequestBatchOutputs {
|
||||
outputs: vec![request_output(
|
||||
"req-1",
|
||||
vec![],
|
||||
Some(EngineCoreFinishReason::Length),
|
||||
)],
|
||||
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let final_output = timeout(Duration::from_secs(1), stream.next()).await.unwrap();
|
||||
assert!(final_output.is_some());
|
||||
|
||||
client.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user