[DP] Engines own wave wake-ups; drop the front-end FIRST_REQ (#59559)

Signed-off-by: fh3 <a@b>
Co-authored-by: fh3 <a@b>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-16 19:41:54 -04:00
committed by GitHub
co-authored by fh3 Claude Opus 5.5
parent b544587408
commit a94c315711
13 changed files with 226 additions and 523 deletions
@@ -838,7 +838,7 @@ class ExternalElasticEPScaleCoordinator:
if prepared.dp_rank == 0:
self.client._coord_store = self.reconfig_store_ref
self.client._ensure_stats_update_task()
await self.client.first_req_send_socket.send(
await self.client.scale_send_socket.send(
msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", bootstrap.new_data_parallel_size)
)
+10 -32
View File
@@ -41,12 +41,9 @@ class DPCoordinator:
DPEngineCoreProc._has_global_unfinished_reqs method.
* Broadcasts the START_DP_WAVE message to engines to move them from paused
to running state when one engine receives a new request. This can happen
in two cases:
1) A front-end sending a new request while the engines are paused will
concurrently notify the coordinator.
2) An engine receiving a request for a stale request wave while in paused
state will notify the coordinator.
to running state when an idle engine receives a new request and notifies
the coordinator. Only the engines start waves: a front-end's view of the
running state can be stale, and a paused engine discards START_DP_WAVE.
Engines will move into running state when receiving a new request or
START_DP_WAVE message.
@@ -353,27 +350,9 @@ class DPCoordinatorProc:
)
continue # Skip normal engine notification processing
# Wave coordination: handle new-request messages from front-end.
# Only process these when wave coordination is enabled
if self.enable_wave_coordination:
# We received a message on the front-end XPUB socket,
# from an API server sending a new request while the
# engines are paused, so that we can wake the other
# engines.
engine_to_exclude, wave = decoded
if not engines_running:
if wave < current_wave:
# If the wave number is stale, ensure the message
# is handled by all the engines.
engine_to_exclude = None
# engines_running is only set from the engines'
# own notifications; a paused engine discards
# START_DP_WAVE, so sending it is not evidence
# that the engines are running.
self._send_start_wave(
publish_back, current_wave, engine_to_exclude
)
logger.error(
"DP Coordinator received unexpected message from front-end"
)
if output_back in events:
# We received a message from one of the engines.
@@ -448,13 +427,12 @@ class DPCoordinatorProc:
wave > current_wave
or (wave == current_wave and not engines_running)
):
# 3. The engine received request for a non-current wave
# so we must ensure that other engines progress to the
# next wave (race condition handling).
# 3. An engine started a wave (e.g. an idle engine
# received a request), so the others must join it.
logger.debug(
"Starting wave %d after notification of "
"stale wave request from engine.",
"Starting wave %d on notification from engine %d.",
wave,
eng_index,
)
current_wave = wave
engines_running = True
+3 -4
View File
@@ -2269,15 +2269,14 @@ class DPEngineCoreProc(EngineCoreProc):
def add_request(self, request: Request, request_wave: int = 0):
super().add_request(request, request_wave)
if self.has_coordinator and request_wave != self.current_wave:
if self.has_coordinator:
if request_wave > self.current_wave:
self.current_wave = request_wave
elif (
if (
not self.engines_running
and self.scheduler.pause_state == PauseState.UNPAUSED
):
# Request received for an already-completed wave, notify
# front-end that we need to start the next one.
# Engines alone start waves: have the coordinator wake the others.
self.engines_running = True
self.output_queue.put_nowait(
(-1, EngineCoreOutputs(start_wave=self.current_wave))
+43 -75
View File
@@ -506,8 +506,8 @@ class BackgroundResources:
coordinator: DPCoordinator | None = None
output_socket: zmq.Socket | zmq.asyncio.Socket | None = None
input_socket: zmq.Socket | zmq.asyncio.Socket | None = None
first_req_send_socket: zmq.asyncio.Socket | None = None
first_req_rcv_socket: zmq.asyncio.Socket | None = None
scale_send_socket: zmq.asyncio.Socket | None = None
scale_rcv_socket: zmq.asyncio.Socket | None = None
stats_update_socket: zmq.asyncio.Socket | None = None
output_queue_task: asyncio.Task | None = None
stats_update_task: asyncio.Task | None = None
@@ -535,8 +535,8 @@ class BackgroundResources:
sockets = (
self.output_socket,
self.input_socket,
self.first_req_send_socket,
self.first_req_rcv_socket,
self.scale_send_socket,
self.scale_rcv_socket,
self.stats_update_socket,
)
@@ -1493,9 +1493,9 @@ class DPAsyncMPClient(AsyncMPClient):
self.eep_scaling_cache: ElasticScalingCache | None = None
self.first_req_sock_addr = get_open_zmq_inproc_path()
self.first_req_send_socket = self.resources.first_req_send_socket = (
make_zmq_socket(self.ctx, self.first_req_sock_addr, zmq.PAIR, bind=True)
self.scale_sock_addr = get_open_zmq_inproc_path()
self.scale_send_socket = self.resources.scale_send_socket = make_zmq_socket(
self.ctx, self.scale_sock_addr, zmq.PAIR, bind=True
)
try:
# If we are running in an asyncio event loop, start the stats task.
@@ -1518,75 +1518,48 @@ class DPAsyncMPClient(AsyncMPClient):
with (
make_zmq_socket(self.ctx, stats_addr, zmq.XSUB, linger=0) as socket,
make_zmq_socket(
self.ctx, self.first_req_sock_addr, zmq.PAIR, bind=False, linger=0
) as first_req_rcv_socket,
self.ctx, self.scale_sock_addr, zmq.PAIR, bind=False, linger=0
) as scale_rcv_socket,
):
assert isinstance(socket, zmq.asyncio.Socket)
assert isinstance(first_req_rcv_socket, zmq.asyncio.Socket)
assert isinstance(scale_rcv_socket, zmq.asyncio.Socket)
self.resources.stats_update_socket = socket
self.resources.first_req_rcv_socket = first_req_rcv_socket
self.resources.scale_rcv_socket = scale_rcv_socket
# Send subscription message.
await socket.send(b"\x01")
poller = zmq.asyncio.Poller()
poller.register(socket, zmq.POLLIN)
poller.register(first_req_rcv_socket, zmq.POLLIN)
poller.register(scale_rcv_socket, zmq.POLLIN)
while True:
events = await poller.poll()
if (
not self.engines_running
and len(events) == 2
or (events[0][0] == first_req_rcv_socket)
):
# Check if this is a regular request notification or
# scale up notification
buf = first_req_rcv_socket.recv(flags=zmq.NOBLOCK).result()
decoded = msgspec.msgpack.decode(buf)
if (
isinstance(decoded, (list, tuple))
and len(decoded) == 2
and decoded[0] == "SCALE_ELASTIC_EP"
):
# Extract new engine count from the decoded message
new_engine_count = decoded[1]
# Update engine_ranks_managed and count_slice
parallel_config = self.vllm_config.parallel_config
dp_size = parallel_config.data_parallel_size
dp_rank = parallel_config.data_parallel_rank
assert dp_rank == 0
assert dp_size == new_engine_count
assert not parallel_config.data_parallel_hybrid_lb
# External LB clients continue to manage only local ranks.
if not parallel_config.data_parallel_external_lb:
self.engine_ranks_managed = list(range(dp_size))
if len(self.lb_engines) < new_engine_count:
self.lb_engines += [
[0, 0, 0.0]
for _ in range(
new_engine_count - len(self.lb_engines)
)
]
else:
self.lb_engines = self.lb_engines[:new_engine_count]
# Send scale up notification to coordinator
scale_msg = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_engine_count)
)
await socket.send(scale_msg)
continue
# we're sending a request while the engines are
# paused, so that it can wake the others up
# (to run dummy EP loop).
assert decoded[0] == "FIRST_REQ"
target_eng_index = decoded[1]
self.engines_running = True
msg = msgspec.msgpack.encode(
(target_eng_index, self.current_wave)
)
await socket.send(msg)
events = dict(await poller.poll())
if scale_rcv_socket in events:
buf = scale_rcv_socket.recv(flags=zmq.NOBLOCK).result()
msg_type, new_engine_count = msgspec.msgpack.decode(buf)
assert msg_type == "SCALE_ELASTIC_EP"
# Update engine_ranks_managed and count_slice
parallel_config = self.vllm_config.parallel_config
dp_size = parallel_config.data_parallel_size
dp_rank = parallel_config.data_parallel_rank
assert dp_rank == 0
assert dp_size == new_engine_count
assert not parallel_config.data_parallel_hybrid_lb
# External LB clients continue to manage only local ranks.
if not parallel_config.data_parallel_external_lb:
self.engine_ranks_managed = list(range(dp_size))
if len(self.lb_engines) < new_engine_count:
self.lb_engines += [
[0, 0, 0.0]
for _ in range(
new_engine_count - len(self.lb_engines)
)
]
else:
self.lb_engines = self.lb_engines[:new_engine_count]
# Forward the scale notification to the coordinator.
await socket.send(buf)
continue
buf = None
while True:
@@ -1624,14 +1597,9 @@ class DPAsyncMPClient(AsyncMPClient):
request.current_wave = self.current_wave
request.client_index = self.client_index
# The engine that receives it wakes the other ranks if they are idle.
chosen_engine = self.get_core_engine_for_request(request)
to_await = self._send_input(EngineCoreRequestType.ADD, request, chosen_engine)
if not self.engines_running:
# Notify coordinator that we're sending a request
req_msg = msgspec.msgpack.encode(("FIRST_REQ", chosen_engine))
await self.first_req_send_socket.send(req_msg)
await to_await
await self._send_input(EngineCoreRequestType.ADD, request, chosen_engine)
self._ensure_output_queue_task()
@@ -2100,7 +2068,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
scale_up_marker = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_data_parallel_size)
)
await self.first_req_send_socket.send(scale_up_marker)
await self.scale_send_socket.send(scale_up_marker)
logger.info(
"[Elastic EP] Scale up completed, new data parallel size: %s",
@@ -2176,7 +2144,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
scale_down_marker = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_data_parallel_size)
)
await self.first_req_send_socket.send(scale_down_marker)
await self.scale_send_socket.send(scale_down_marker)
await wait_future
await self.resume_scheduler_async()
except Exception: