[DP] Engines own wave wake-ups; drop the front-end FIRST_REQ (#59559)

Signed-off-by: fh3 <a@b>
Co-authored-by: fh3 <a@b>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-16 19:41:54 -04:00
committed by GitHub
co-authored by fh3 Claude Opus 5.5
parent b544587408
commit a94c315711
13 changed files with 226 additions and 523 deletions
+2 -5
View File
@@ -657,12 +657,9 @@ impl EngineCoreClient {
);
let result: Result<()> = async {
// The engine that receives it wakes the other ranks if they are idle.
if let Some(coordinator) = self.coordinator.as_ref() {
let snapshot = coordinator.snapshot();
req.current_wave = snapshot.current_wave;
if !snapshot.engines_running {
coordinator.notify_first_request(engine_id.clone())?;
}
req.current_wave = coordinator.snapshot().current_wave;
}
debug!(
@@ -3,34 +3,16 @@
use std::sync::Arc;
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use serde_tuple::Deserialize_tuple;
use thiserror_ext::AsReport;
use tokio::sync::mpsc;
use tracing::{debug, warn};
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::prelude::SocketRecv;
use zeromq::{XSubSocket, ZmqMessage};
use crate::client::imp::ClientInner;
use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
use crate::coordinator::handle::CoordinatorState;
use crate::error::{Error, Result, bail_unexpected_coordinator_output};
use crate::protocol::{OpaqueValue, decode_msgpack, encode_msgpack};
/// Frontend-to-coordinator wakeup message sent when the first request arrives
/// while all engines are paused.
///
/// This matches the frontend-side msgpack tuple sent by Python
/// `DPAsyncMPClient._ensure_stats_update_task` to the coordinator front socket.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/694449050f8dac3d9853e97e518b4a43ec52106a/vllm/v1/engine/core_client.py#L1230-L1236>
#[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]
struct CoordinatorWakeupMessage {
/// Engine index that already has the triggering request and should be
/// excluded from the coordinator's `START_DP_WAVE` rebroadcast.
exclude_engine_index: u32,
/// DP wave number observed by the frontend when the request was admitted.
wave: u32,
}
use crate::protocol::{OpaqueValue, decode_msgpack};
/// Coordinator-to-frontend state publish received on the front-side coordinator
/// socket.
@@ -56,56 +38,17 @@ struct CoordinatorStateUpdate {
/// Background half of an external Python-owned coordinator connection.
///
/// This owns the command receiver and one frontend-facing XSUB socket. It
/// mirrors the subset of Python's coordinator protocol needed by the Rust
/// bootstrapped frontend: receive `(counts, wave, running)` publishes, ignore
/// `counts`, and send `(exclude_engine_index, wave)` wakeup messages when the
/// first request arrives while engines are paused.
/// This owns one frontend-facing XSUB socket. It mirrors the subset of Python's
/// coordinator protocol needed by the Rust bootstrapped frontend: receive
/// `(counts, wave, running)` publishes and ignore `counts`.
pub(crate) struct ExternalCoordinatorService {
state: Arc<CoordinatorState>,
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
socket: XSubSocket,
}
impl ExternalCoordinatorService {
pub(super) fn new(
state: Arc<CoordinatorState>,
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
socket: XSubSocket,
) -> Self {
Self {
state,
command_rx,
socket,
}
}
/// Apply one frontend-originated command to the external coordinator state
/// machine.
async fn handle_command(&mut self, command: CoordinatorCommand) -> Result<()> {
match command {
CoordinatorCommand::FirstRequest {
target_engine_id,
wave,
} => {
let target_engine_index = target_engine_id.engine_index().ok_or_else(|| {
Error::UnsupportedCoordinatorEngineId {
engine_id: target_engine_id.to_vec(),
}
})?;
debug!(
wave,
exclude_engine_index = target_engine_index,
"notifying external coordinator about first request while engines were paused"
);
let payload = encode_msgpack(&CoordinatorWakeupMessage {
exclude_engine_index: target_engine_index,
wave,
})?;
self.socket.send(ZmqMessage::from(payload)).await?;
}
}
Ok(())
pub(super) fn new(state: Arc<CoordinatorState>, socket: XSubSocket) -> Self {
Self { state, socket }
}
/// Apply one publish received from the xsub socket containing a coordinator
@@ -136,26 +79,14 @@ impl ExternalCoordinatorService {
Ok(())
}
/// Drive the coordinator event loop until either side of the control plane
/// is closed or a fatal error is observed.
/// Drive the coordinator event loop until the task is aborted or a fatal
/// error is observed.
pub(crate) async fn run(mut self, inner: Arc<ClientInner>) {
let result: Result<()> = async {
loop {
tokio::select! {
// Received frontend-originated command from the handle.
command = self.command_rx.recv() => {
let Some(command) = command else {
warn!("external coordinator command channel closed, shutting down service");
return Ok(());
};
self.handle_command(command).await?;
}
// Received publish from the external coordinator socket.
publish = self.socket.recv() => {
let publish = publish.map_err(Error::from)?;
self.handle_publish(publish).await?;
}
}
// Received publish from the external coordinator socket.
let publish = self.socket.recv().await.map_err(Error::from)?;
self.handle_publish(publish).await?;
}
}
.await;
@@ -4,101 +4,56 @@
use std::sync::Arc;
use parking_lot::Mutex;
use tokio::sync::mpsc;
use zeromq::prelude::Socket;
use zeromq::{XPubSocket, XSubSocket};
use crate::coordinator::external::ExternalCoordinatorService;
use crate::coordinator::inproc::InProcCoordinatorRunner;
use crate::error::{Error, Result, bail_control_closed};
use crate::transport::EngineId;
use crate::error::Result;
/// Snapshot to the coordinator state for request routing and stamping.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct CoordinatorStateSnapshot {
/// The current DP wave, which will be stamped on outgoing requests.
pub current_wave: u32,
/// Whether the engines are currently running or paused, which determines if
/// the frontend must trigger a new wave on the next request.
/// Whether the engines are currently running or paused, as last reported
/// by the engines themselves.
pub engines_running: bool,
}
impl CoordinatorStateSnapshot {
/// Resume the engines for a `FirstRequest` and return the wave to broadcast
/// and the engine to exclude from the wakeup.
///
/// The request may have been stamped with a `request_wave` older than
/// `current_wave` if a `WaveComplete` advanced it after the command was
/// enqueued. Such a request still needs serving, so the current wave is
/// broadcast to every engine (`exclude = None`); the wave is never rewound.
/// A non-stale request excludes the engine that already received it. Mirrors
/// the Python coordinator's front-end path.
pub(crate) fn start_wave_for_first_request(
&mut self,
request_wave: u32,
target_engine_index: u32,
) -> (u32, Option<u32>) {
self.engines_running = true;
let exclude = (request_wave >= self.current_wave).then_some(target_engine_index);
(self.current_wave, exclude)
}
}
/// Shared in-process coordinator state.
pub(crate) type CoordinatorState = Mutex<CoordinatorStateSnapshot>;
/// Commands sent from the frontend request path into the background runner.
#[derive(Debug)]
pub(crate) enum CoordinatorCommand {
/// The first request arrived while all engines were paused.
///
/// The coordinator should broadcast `START_DP_WAVE` with the current wave
/// and the target engine index as the excluded engine.
FirstRequest {
target_engine_id: EngineId,
wave: u32,
},
}
/// Frontend-facing coordinator handle used by `EngineCoreClient::call()`.
///
/// This side stays intentionally small: it can read the latest wave snapshot
/// and enqueue a `FirstRequest` transition when the request path observes the
/// system in the paused state.
/// The frontend only reads the latest wave snapshot: waves are started by the
/// engines, which notify the coordinator when an idle engine receives a request.
#[derive(Clone)]
pub(crate) struct CoordinatorHandle {
state: Arc<CoordinatorState>,
command_tx: mpsc::UnboundedSender<CoordinatorCommand>,
}
impl CoordinatorHandle {
fn new_parts() -> (
Self,
Arc<CoordinatorState>,
mpsc::UnboundedReceiver<CoordinatorCommand>,
) {
fn new_parts() -> (Self, Arc<CoordinatorState>) {
let state = Arc::new(Mutex::new(CoordinatorStateSnapshot {
current_wave: 0,
engines_running: false,
}));
let (command_tx, command_rx) = mpsc::unbounded_channel();
(
Self {
state: state.clone(),
command_tx,
},
state,
command_rx,
)
}
/// Build the paired frontend handle and background runner around one
/// engine-facing coordinator broadcast socket.
pub(crate) fn new_inproc(coordinator_input: XPubSocket) -> (Self, InProcCoordinatorRunner) {
let (handle, state, command_rx) = Self::new_parts();
let (handle, state) = Self::new_parts();
(
handle,
InProcCoordinatorRunner::new(state, command_rx, coordinator_input),
InProcCoordinatorRunner::new(state, coordinator_input),
)
}
@@ -107,41 +62,15 @@ impl CoordinatorHandle {
pub(crate) async fn connect_external(
coordinator_address: &str,
) -> Result<(Self, ExternalCoordinatorService)> {
let (handle, state, command_rx) = Self::new_parts();
let (handle, state) = Self::new_parts();
let mut socket = XSubSocket::new();
socket.connect(coordinator_address).await?;
socket.subscribe("").await?;
Ok((
handle,
ExternalCoordinatorService::new(state, command_rx, socket),
))
Ok((handle, ExternalCoordinatorService::new(state, socket)))
}
/// Snapshot the coordinator state for request routing and stamping.
pub(crate) fn snapshot(&self) -> CoordinatorStateSnapshot {
*self.state.lock()
}
/// Notify the runner that a new request arrived while engines were paused.
///
/// The handle flips `engines_running` optimistically so concurrent request
/// submissions coalesce behind one `START_DP_WAVE` broadcast instead of all
/// trying to trigger the wave independently.
pub(crate) fn notify_first_request(&self, target_engine_id: EngineId) -> Result<()> {
let mut state = self.state.lock();
if state.engines_running {
return Ok(());
}
let command = CoordinatorCommand::FirstRequest {
target_engine_id,
wave: state.current_wave,
};
if self.command_tx.send(command).is_err() {
bail_control_closed!("in-process coordinator command channel already shut down");
}
state.engines_running = true;
Ok(())
}
}
@@ -12,7 +12,7 @@ use zeromq::{XPubSocket, ZmqError, ZmqMessage};
use crate::client::imp::ClientInner;
use crate::coordinator::bootstrap::is_engine_subscription;
use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
use crate::coordinator::handle::CoordinatorState;
use crate::error::{Error, Result, bail_unexpected_coordinator_output};
use crate::protocol::encode_msgpack;
use crate::protocol::output::{DpControlMessage, DpControlOutput, EngineCoreOutputs};
@@ -30,32 +30,25 @@ use crate::protocol::request::EngineCoreRequestType;
struct StartDpWaveMessage {
/// DP wave number that all engines should start processing.
wave: u32,
/// Engine index that already received the triggering request and so does not
/// need an extra wakeup. `None` wakes every engine (used when the triggering
/// request was for a stale wave).
/// Engine index that started the wave and so does not need an extra wakeup.
/// `None` wakes every engine.
exclude_engine_index: Option<u32>,
}
/// Background half of the in-process coordinator.
///
/// This owns the command receiver and the engine-facing coordinator input
/// socket. It is the single place where wave transitions are serialized and
/// where `START_DP_WAVE` broadcasts are emitted.
/// This owns the engine-facing coordinator input socket. It is the single place
/// where wave transitions are serialized and where `START_DP_WAVE` broadcasts
/// are emitted.
pub(crate) struct InProcCoordinatorRunner {
state: Arc<CoordinatorState>,
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
coordinator_input: XPubSocket,
}
impl InProcCoordinatorRunner {
pub(super) fn new(
state: Arc<CoordinatorState>,
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
coordinator_input: XPubSocket,
) -> Self {
pub(super) fn new(state: Arc<CoordinatorState>, coordinator_input: XPubSocket) -> Self {
Self {
state,
command_rx,
coordinator_input,
}
}
@@ -82,34 +75,6 @@ impl InProcCoordinatorRunner {
Ok(())
}
/// Apply one frontend-originated command to the coordinator state machine.
async fn handle_command(&mut self, command: CoordinatorCommand) -> Result<()> {
match command {
CoordinatorCommand::FirstRequest {
target_engine_id,
wave,
} => {
let target_engine_index = target_engine_id.engine_index().ok_or_else(|| {
Error::UnsupportedCoordinatorEngineId {
engine_id: target_engine_id.to_vec(),
}
})?;
let (current_wave, exclude) = {
let mut state = self.state.lock();
state.start_wave_for_first_request(wave, target_engine_index)
};
debug!(
current_wave,
request_wave = wave,
?exclude,
"starting DP wave after first request while engines were paused"
);
self.broadcast_start_wave(current_wave, exclude).await?;
}
}
Ok(())
}
/// Apply one engine-originated control output to the coordinator state
/// machine.
async fn handle_outputs(&mut self, outputs: EngineCoreOutputs) -> Result<()> {
@@ -160,7 +125,7 @@ impl InProcCoordinatorRunner {
debug!(
wave,
exclude_engine_index = engine_index,
"starting DP wave after stale-wave notification from engine"
"starting DP wave on notification from engine"
);
self.broadcast_start_wave(wave, Some(engine_index)).await?;
}
@@ -175,8 +140,8 @@ impl InProcCoordinatorRunner {
Ok(())
}
/// Drive the coordinator event loop until either side of the control plane
/// is closed or a fatal error is observed.
/// Drive the coordinator event loop until the engine control output is
/// closed or a fatal error is observed.
///
/// Any fatal error closes the main client registries so request streams and
/// future calls observe a stable shutdown cause.
@@ -216,14 +181,6 @@ impl InProcCoordinatorRunner {
Err(error) => return Err(error.into()),
}
}
// Received frontend-originated command from the handle.
command = self.command_rx.recv() => {
let Some(command) = command else {
warn!("coordinator command channel closed, shutting down coordinator runner");
return Ok(());
};
self.handle_command(command).await?;
}
// Received engine-originated control output from the coordinator socket.
outputs = output_rx.recv() => {
let Some(outputs) = outputs else {
@@ -242,48 +199,3 @@ impl InProcCoordinatorRunner {
inner.close_registries(Arc::new(error));
}
}
#[cfg(test)]
mod tests {
use crate::coordinator::handle::CoordinatorStateSnapshot;
/// A `FirstRequest` for the current wave starts that wave and excludes the
/// engine that already received the triggering request.
#[test]
fn first_request_for_current_wave_excludes_target() {
let mut state = CoordinatorStateSnapshot {
current_wave: 3,
engines_running: false,
};
let (wave, exclude) = state.start_wave_for_first_request(3, 2);
assert_eq!(wave, 3);
assert_eq!(exclude, Some(2));
assert!(state.engines_running);
assert_eq!(state.current_wave, 3);
}
/// A `FirstRequest` whose wave was superseded by a racing `WaveComplete`
/// (`request_wave < current_wave`) must still start the request's wave: it
/// broadcasts the current wave and wakes every engine (`exclude = None`)
/// rather than rewinding the wave or dropping the request.
#[test]
fn stale_first_request_starts_current_wave_for_all_engines() {
let mut state = CoordinatorStateSnapshot {
current_wave: 4,
engines_running: false,
};
// Request stamped with wave 3 while the coordinator already advanced to 4.
let (wave, exclude) = state.start_wave_for_first_request(3, 2);
assert_eq!(
wave, 4,
"must broadcast the current wave, not the stale one"
);
assert_eq!(exclude, None, "a stale request must wake every engine");
assert!(state.engines_running);
assert_eq!(state.current_wave, 4, "wave must not be rewound");
}
}
-4
View File
@@ -55,8 +55,6 @@ pub enum Error {
UnexpectedCoordinatorOutput { message: String },
#[error("unexpected output on main dispatcher path: {message}")]
UnexpectedDispatcherOutput { message: String },
#[error("coordinator requires a Python-compatible two-byte engine id, got {engine_id:?}")]
UnsupportedCoordinatorEngineId { engine_id: Vec<u8> },
#[error("unsupported auxiliary frame(s): expected 1 frame, got {frame_count}")]
UnsupportedAuxFrames { frame_count: usize },
#[error("external coordinator mode is not implemented yet")]
@@ -66,8 +64,6 @@ pub enum Error {
context: &'static str,
field: &'static str,
},
#[error("engine control channel closed unexpectedly: {message}")]
ControlClosed { message: String },
#[error("request `{request_id}` is already in flight")]
DuplicateRequestId { request_id: String },
#[error(
+28 -103
View File
@@ -267,6 +267,19 @@ async fn recv_start_dp_wave(sub: &mut SubSocket) -> (u32, Option<u32>) {
rmp_serde::from_slice(&frames[1]).expect("decode START_DP_WAVE payload")
}
async fn send_start_wave(push: &mut PushSocket, engine_index: u32, wave: u32) {
send_outputs(
push,
DpControlOutput {
engine_index,
timestamp: 0.0,
control: DpControlMessage::StartWave(wave),
}
.into(),
)
.await;
}
async fn connect_client_with_ipc(
config: EngineCoreClientConfig,
ipc: &IpcNamespace,
@@ -413,12 +426,6 @@ async fn recv_xpub_subscription(xpub: &mut XPubSocket) {
assert_eq!(frames[0].as_ref(), b"\x01");
}
async fn recv_external_coordinator_wakeup(xpub: &mut XPubSocket) -> (u32, u32) {
let frames = recv_xpub_message(xpub).await;
assert_eq!(frames.len(), 1);
rmp_serde::from_slice(&frames[0]).expect("decode external coordinator wakeup")
}
async fn send_external_coordinator_publish<T: serde::Serialize>(
xpub: &mut XPubSocket,
payload: &T,
@@ -617,6 +624,14 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
engine.coordinator.take().expect("coordinator sockets should be present");
let data_socket = engine.data_sockets.first_mut().expect("data socket");
// An idle engine handed a request starts the wave itself.
let add = recv_engine_message(&mut data_socket.dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
assert_eq!(request.request_id, "req-1");
assert_eq!(request.current_wave, 0);
send_start_wave(&mut coordinator.output_push, 0, 0).await;
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (0, Some(0)));
@@ -629,12 +644,6 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
let replay = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!(replay, (0, None));
let add = recv_engine_message(&mut data_socket.dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
assert_eq!(request.request_id, "req-1");
assert_eq!(request.current_wave, 0);
assert!(
timeout(
Duration::from_millis(200),
@@ -670,14 +679,15 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
)
.await;
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (1, Some(0)));
let add = recv_engine_message(&mut data_socket.dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
assert_eq!(request.request_id, "req-3");
assert_eq!(request.current_wave, 1);
send_start_wave(&mut coordinator.output_push, 0, 1).await;
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (1, Some(0)));
send_outputs(
&mut data_socket.push,
@@ -896,9 +906,6 @@ async fn coordinator_accepts_stats_only_outputs() {
engine.coordinator.take().expect("coordinator sockets should be present");
let data_socket = engine.data_sockets.first_mut().expect("data socket");
let (wave, exclude_engine) = recv_start_dp_wave(&mut coordinator.input_sub).await;
assert_eq!((wave, exclude_engine), (0, Some(0)));
send_outputs(
&mut coordinator.output_push,
RequestBatchOutputs {
@@ -3248,7 +3255,7 @@ async fn bootstrapped_external_coordinator_connects_and_subscribes() {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn bootstrapped_external_coordinator_updates_wave_ignores_counts_and_sends_one_wakeup() {
async fn bootstrapped_external_coordinator_updates_wave_ignores_counts_and_sends_no_wakeup() {
init_tracing();
let ipc = IpcNamespace::new().unwrap();
let input_address = ipc.input_endpoint();
@@ -3289,18 +3296,11 @@ async fn bootstrapped_external_coordinator_updates_wave_ignores_counts_and_sends
let mut stream = client.call(sample_request()).await.unwrap().into_outputs();
let wakeup = timeout(
Duration::from_secs(1),
recv_external_coordinator_wakeup(&mut stats_socket),
)
.await
.unwrap();
assert_eq!(wakeup, (0, 7));
// The engine that receives the request starts the wave, not the frontend.
assert!(
timeout(
Duration::from_millis(200),
recv_external_coordinator_wakeup(&mut stats_socket)
recv_xpub_message(&mut stats_socket)
)
.await
.is_err()
@@ -3333,78 +3333,3 @@ async fn bootstrapped_external_coordinator_updates_wave_ignores_counts_and_sends
client.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn bootstrapped_external_coordinator_running_state_suppresses_wakeup() {
init_tracing();
let ipc = IpcNamespace::new().unwrap();
let input_address = ipc.input_endpoint();
let output_address = ipc.output_endpoint();
let coordinator_address = ipc.endpoint("stats.sock");
let mut stats_socket = XPubSocket::new();
stats_socket.bind(&coordinator_address).await.unwrap();
let client_task = tokio::spawn({
let input_address = input_address.clone();
let output_address = output_address.clone();
let coordinator_address = coordinator_address.clone();
async move {
EngineCoreClient::connect(bootstrapped_test_config(
input_address,
output_address,
1,
Duration::from_secs(2),
0,
Some(CoordinatorMode::External {
address: coordinator_address,
}),
))
.await
.unwrap()
}
});
let (mut dealer, mut push) =
setup_bootstrapped_mock_engine(input_address, output_address, &[0x00, 0x00]).await;
let client = client_task.await.unwrap();
recv_xpub_subscription(&mut stats_socket).await;
send_external_coordinator_publish(&mut stats_socket, &(Value::Nil, 5_u32, true)).await;
tokio::time::sleep(Duration::from_millis(50)).await;
let mut stream = client.call(sample_request()).await.unwrap().into_outputs();
assert!(
timeout(
Duration::from_millis(200),
recv_external_coordinator_wakeup(&mut stats_socket)
)
.await
.is_err()
);
let add = recv_engine_message(&mut dealer).await;
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
assert_eq!(request.current_wave, 5);
send_outputs(
&mut push,
RequestBatchOutputs {
outputs: vec![request_output(
"req-1",
vec![],
Some(EngineCoreFinishReason::Length),
)],
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
..Default::default()
}
.into(),
)
.await;
let final_output = timeout(Duration::from_secs(1), stream.next()).await.unwrap();
assert!(final_output.is_some());
client.shutdown().await.unwrap();
}