[DP] Engines own wave wake-ups; drop the front-end FIRST_REQ (#59559)
Signed-off-by: fh3 <a@b> Co-authored-by: fh3 <a@b> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
fh3
Claude Opus 5.5
parent
b544587408
commit
a94c315711
@@ -41,12 +41,9 @@ class DPCoordinator:
|
||||
DPEngineCoreProc._has_global_unfinished_reqs method.
|
||||
|
||||
* Broadcasts the START_DP_WAVE message to engines to move them from paused
|
||||
to running state when one engine receives a new request. This can happen
|
||||
in two cases:
|
||||
1) A front-end sending a new request while the engines are paused will
|
||||
concurrently notify the coordinator.
|
||||
2) An engine receiving a request for a stale request wave while in paused
|
||||
state will notify the coordinator.
|
||||
to running state when an idle engine receives a new request and notifies
|
||||
the coordinator. Only the engines start waves: a front-end's view of the
|
||||
running state can be stale, and a paused engine discards START_DP_WAVE.
|
||||
|
||||
Engines will move into running state when receiving a new request or
|
||||
START_DP_WAVE message.
|
||||
@@ -353,27 +350,9 @@ class DPCoordinatorProc:
|
||||
)
|
||||
continue # Skip normal engine notification processing
|
||||
|
||||
# Wave coordination: handle new-request messages from front-end.
|
||||
# Only process these when wave coordination is enabled
|
||||
if self.enable_wave_coordination:
|
||||
# We received a message on the front-end XPUB socket,
|
||||
# from an API server sending a new request while the
|
||||
# engines are paused, so that we can wake the other
|
||||
# engines.
|
||||
engine_to_exclude, wave = decoded
|
||||
if not engines_running:
|
||||
if wave < current_wave:
|
||||
# If the wave number is stale, ensure the message
|
||||
# is handled by all the engines.
|
||||
engine_to_exclude = None
|
||||
|
||||
# engines_running is only set from the engines'
|
||||
# own notifications; a paused engine discards
|
||||
# START_DP_WAVE, so sending it is not evidence
|
||||
# that the engines are running.
|
||||
self._send_start_wave(
|
||||
publish_back, current_wave, engine_to_exclude
|
||||
)
|
||||
logger.error(
|
||||
"DP Coordinator received unexpected message from front-end"
|
||||
)
|
||||
|
||||
if output_back in events:
|
||||
# We received a message from one of the engines.
|
||||
@@ -448,13 +427,12 @@ class DPCoordinatorProc:
|
||||
wave > current_wave
|
||||
or (wave == current_wave and not engines_running)
|
||||
):
|
||||
# 3. The engine received request for a non-current wave
|
||||
# so we must ensure that other engines progress to the
|
||||
# next wave (race condition handling).
|
||||
# 3. An engine started a wave (e.g. an idle engine
|
||||
# received a request), so the others must join it.
|
||||
logger.debug(
|
||||
"Starting wave %d after notification of "
|
||||
"stale wave request from engine.",
|
||||
"Starting wave %d on notification from engine %d.",
|
||||
wave,
|
||||
eng_index,
|
||||
)
|
||||
current_wave = wave
|
||||
engines_running = True
|
||||
|
||||
Reference in New Issue
Block a user