[DP] Engines own wave wake-ups; drop the front-end FIRST_REQ (#59559)
Signed-off-by: fh3 <a@b> Co-authored-by: fh3 <a@b> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
fh3
Claude Opus 5.5
parent
b544587408
commit
a94c315711
@@ -838,7 +838,7 @@ class ExternalElasticEPScaleCoordinator:
|
||||
if prepared.dp_rank == 0:
|
||||
self.client._coord_store = self.reconfig_store_ref
|
||||
self.client._ensure_stats_update_task()
|
||||
await self.client.first_req_send_socket.send(
|
||||
await self.client.scale_send_socket.send(
|
||||
msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", bootstrap.new_data_parallel_size)
|
||||
)
|
||||
|
||||
@@ -41,12 +41,9 @@ class DPCoordinator:
|
||||
DPEngineCoreProc._has_global_unfinished_reqs method.
|
||||
|
||||
* Broadcasts the START_DP_WAVE message to engines to move them from paused
|
||||
to running state when one engine receives a new request. This can happen
|
||||
in two cases:
|
||||
1) A front-end sending a new request while the engines are paused will
|
||||
concurrently notify the coordinator.
|
||||
2) An engine receiving a request for a stale request wave while in paused
|
||||
state will notify the coordinator.
|
||||
to running state when an idle engine receives a new request and notifies
|
||||
the coordinator. Only the engines start waves: a front-end's view of the
|
||||
running state can be stale, and a paused engine discards START_DP_WAVE.
|
||||
|
||||
Engines will move into running state when receiving a new request or
|
||||
START_DP_WAVE message.
|
||||
@@ -353,27 +350,9 @@ class DPCoordinatorProc:
|
||||
)
|
||||
continue # Skip normal engine notification processing
|
||||
|
||||
# Wave coordination: handle new-request messages from front-end.
|
||||
# Only process these when wave coordination is enabled
|
||||
if self.enable_wave_coordination:
|
||||
# We received a message on the front-end XPUB socket,
|
||||
# from an API server sending a new request while the
|
||||
# engines are paused, so that we can wake the other
|
||||
# engines.
|
||||
engine_to_exclude, wave = decoded
|
||||
if not engines_running:
|
||||
if wave < current_wave:
|
||||
# If the wave number is stale, ensure the message
|
||||
# is handled by all the engines.
|
||||
engine_to_exclude = None
|
||||
|
||||
# engines_running is only set from the engines'
|
||||
# own notifications; a paused engine discards
|
||||
# START_DP_WAVE, so sending it is not evidence
|
||||
# that the engines are running.
|
||||
self._send_start_wave(
|
||||
publish_back, current_wave, engine_to_exclude
|
||||
)
|
||||
logger.error(
|
||||
"DP Coordinator received unexpected message from front-end"
|
||||
)
|
||||
|
||||
if output_back in events:
|
||||
# We received a message from one of the engines.
|
||||
@@ -448,13 +427,12 @@ class DPCoordinatorProc:
|
||||
wave > current_wave
|
||||
or (wave == current_wave and not engines_running)
|
||||
):
|
||||
# 3. The engine received request for a non-current wave
|
||||
# so we must ensure that other engines progress to the
|
||||
# next wave (race condition handling).
|
||||
# 3. An engine started a wave (e.g. an idle engine
|
||||
# received a request), so the others must join it.
|
||||
logger.debug(
|
||||
"Starting wave %d after notification of "
|
||||
"stale wave request from engine.",
|
||||
"Starting wave %d on notification from engine %d.",
|
||||
wave,
|
||||
eng_index,
|
||||
)
|
||||
current_wave = wave
|
||||
engines_running = True
|
||||
|
||||
@@ -2269,15 +2269,14 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
|
||||
def add_request(self, request: Request, request_wave: int = 0):
|
||||
super().add_request(request, request_wave)
|
||||
if self.has_coordinator and request_wave != self.current_wave:
|
||||
if self.has_coordinator:
|
||||
if request_wave > self.current_wave:
|
||||
self.current_wave = request_wave
|
||||
elif (
|
||||
if (
|
||||
not self.engines_running
|
||||
and self.scheduler.pause_state == PauseState.UNPAUSED
|
||||
):
|
||||
# Request received for an already-completed wave, notify
|
||||
# front-end that we need to start the next one.
|
||||
# Engines alone start waves: have the coordinator wake the others.
|
||||
self.engines_running = True
|
||||
self.output_queue.put_nowait(
|
||||
(-1, EngineCoreOutputs(start_wave=self.current_wave))
|
||||
|
||||
@@ -506,8 +506,8 @@ class BackgroundResources:
|
||||
coordinator: DPCoordinator | None = None
|
||||
output_socket: zmq.Socket | zmq.asyncio.Socket | None = None
|
||||
input_socket: zmq.Socket | zmq.asyncio.Socket | None = None
|
||||
first_req_send_socket: zmq.asyncio.Socket | None = None
|
||||
first_req_rcv_socket: zmq.asyncio.Socket | None = None
|
||||
scale_send_socket: zmq.asyncio.Socket | None = None
|
||||
scale_rcv_socket: zmq.asyncio.Socket | None = None
|
||||
stats_update_socket: zmq.asyncio.Socket | None = None
|
||||
output_queue_task: asyncio.Task | None = None
|
||||
stats_update_task: asyncio.Task | None = None
|
||||
@@ -535,8 +535,8 @@ class BackgroundResources:
|
||||
sockets = (
|
||||
self.output_socket,
|
||||
self.input_socket,
|
||||
self.first_req_send_socket,
|
||||
self.first_req_rcv_socket,
|
||||
self.scale_send_socket,
|
||||
self.scale_rcv_socket,
|
||||
self.stats_update_socket,
|
||||
)
|
||||
|
||||
@@ -1493,9 +1493,9 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
|
||||
self.eep_scaling_cache: ElasticScalingCache | None = None
|
||||
|
||||
self.first_req_sock_addr = get_open_zmq_inproc_path()
|
||||
self.first_req_send_socket = self.resources.first_req_send_socket = (
|
||||
make_zmq_socket(self.ctx, self.first_req_sock_addr, zmq.PAIR, bind=True)
|
||||
self.scale_sock_addr = get_open_zmq_inproc_path()
|
||||
self.scale_send_socket = self.resources.scale_send_socket = make_zmq_socket(
|
||||
self.ctx, self.scale_sock_addr, zmq.PAIR, bind=True
|
||||
)
|
||||
try:
|
||||
# If we are running in an asyncio event loop, start the stats task.
|
||||
@@ -1518,75 +1518,48 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
with (
|
||||
make_zmq_socket(self.ctx, stats_addr, zmq.XSUB, linger=0) as socket,
|
||||
make_zmq_socket(
|
||||
self.ctx, self.first_req_sock_addr, zmq.PAIR, bind=False, linger=0
|
||||
) as first_req_rcv_socket,
|
||||
self.ctx, self.scale_sock_addr, zmq.PAIR, bind=False, linger=0
|
||||
) as scale_rcv_socket,
|
||||
):
|
||||
assert isinstance(socket, zmq.asyncio.Socket)
|
||||
assert isinstance(first_req_rcv_socket, zmq.asyncio.Socket)
|
||||
assert isinstance(scale_rcv_socket, zmq.asyncio.Socket)
|
||||
self.resources.stats_update_socket = socket
|
||||
self.resources.first_req_rcv_socket = first_req_rcv_socket
|
||||
self.resources.scale_rcv_socket = scale_rcv_socket
|
||||
# Send subscription message.
|
||||
await socket.send(b"\x01")
|
||||
|
||||
poller = zmq.asyncio.Poller()
|
||||
poller.register(socket, zmq.POLLIN)
|
||||
poller.register(first_req_rcv_socket, zmq.POLLIN)
|
||||
poller.register(scale_rcv_socket, zmq.POLLIN)
|
||||
|
||||
while True:
|
||||
events = await poller.poll()
|
||||
if (
|
||||
not self.engines_running
|
||||
and len(events) == 2
|
||||
or (events[0][0] == first_req_rcv_socket)
|
||||
):
|
||||
# Check if this is a regular request notification or
|
||||
# scale up notification
|
||||
buf = first_req_rcv_socket.recv(flags=zmq.NOBLOCK).result()
|
||||
|
||||
decoded = msgspec.msgpack.decode(buf)
|
||||
if (
|
||||
isinstance(decoded, (list, tuple))
|
||||
and len(decoded) == 2
|
||||
and decoded[0] == "SCALE_ELASTIC_EP"
|
||||
):
|
||||
# Extract new engine count from the decoded message
|
||||
new_engine_count = decoded[1]
|
||||
# Update engine_ranks_managed and count_slice
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
dp_rank = parallel_config.data_parallel_rank
|
||||
assert dp_rank == 0
|
||||
assert dp_size == new_engine_count
|
||||
assert not parallel_config.data_parallel_hybrid_lb
|
||||
# External LB clients continue to manage only local ranks.
|
||||
if not parallel_config.data_parallel_external_lb:
|
||||
self.engine_ranks_managed = list(range(dp_size))
|
||||
if len(self.lb_engines) < new_engine_count:
|
||||
self.lb_engines += [
|
||||
[0, 0, 0.0]
|
||||
for _ in range(
|
||||
new_engine_count - len(self.lb_engines)
|
||||
)
|
||||
]
|
||||
else:
|
||||
self.lb_engines = self.lb_engines[:new_engine_count]
|
||||
# Send scale up notification to coordinator
|
||||
scale_msg = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_engine_count)
|
||||
)
|
||||
await socket.send(scale_msg)
|
||||
continue
|
||||
|
||||
# we're sending a request while the engines are
|
||||
# paused, so that it can wake the others up
|
||||
# (to run dummy EP loop).
|
||||
assert decoded[0] == "FIRST_REQ"
|
||||
target_eng_index = decoded[1]
|
||||
self.engines_running = True
|
||||
msg = msgspec.msgpack.encode(
|
||||
(target_eng_index, self.current_wave)
|
||||
)
|
||||
await socket.send(msg)
|
||||
events = dict(await poller.poll())
|
||||
if scale_rcv_socket in events:
|
||||
buf = scale_rcv_socket.recv(flags=zmq.NOBLOCK).result()
|
||||
msg_type, new_engine_count = msgspec.msgpack.decode(buf)
|
||||
assert msg_type == "SCALE_ELASTIC_EP"
|
||||
# Update engine_ranks_managed and count_slice
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
dp_rank = parallel_config.data_parallel_rank
|
||||
assert dp_rank == 0
|
||||
assert dp_size == new_engine_count
|
||||
assert not parallel_config.data_parallel_hybrid_lb
|
||||
# External LB clients continue to manage only local ranks.
|
||||
if not parallel_config.data_parallel_external_lb:
|
||||
self.engine_ranks_managed = list(range(dp_size))
|
||||
if len(self.lb_engines) < new_engine_count:
|
||||
self.lb_engines += [
|
||||
[0, 0, 0.0]
|
||||
for _ in range(
|
||||
new_engine_count - len(self.lb_engines)
|
||||
)
|
||||
]
|
||||
else:
|
||||
self.lb_engines = self.lb_engines[:new_engine_count]
|
||||
# Forward the scale notification to the coordinator.
|
||||
await socket.send(buf)
|
||||
continue
|
||||
|
||||
buf = None
|
||||
while True:
|
||||
@@ -1624,14 +1597,9 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
request.current_wave = self.current_wave
|
||||
request.client_index = self.client_index
|
||||
|
||||
# The engine that receives it wakes the other ranks if they are idle.
|
||||
chosen_engine = self.get_core_engine_for_request(request)
|
||||
to_await = self._send_input(EngineCoreRequestType.ADD, request, chosen_engine)
|
||||
if not self.engines_running:
|
||||
# Notify coordinator that we're sending a request
|
||||
req_msg = msgspec.msgpack.encode(("FIRST_REQ", chosen_engine))
|
||||
await self.first_req_send_socket.send(req_msg)
|
||||
|
||||
await to_await
|
||||
await self._send_input(EngineCoreRequestType.ADD, request, chosen_engine)
|
||||
|
||||
self._ensure_output_queue_task()
|
||||
|
||||
@@ -2100,7 +2068,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
scale_up_marker = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_data_parallel_size)
|
||||
)
|
||||
await self.first_req_send_socket.send(scale_up_marker)
|
||||
await self.scale_send_socket.send(scale_up_marker)
|
||||
|
||||
logger.info(
|
||||
"[Elastic EP] Scale up completed, new data parallel size: %s",
|
||||
@@ -2176,7 +2144,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
scale_down_marker = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_data_parallel_size)
|
||||
)
|
||||
await self.first_req_send_socket.send(scale_down_marker)
|
||||
await self.scale_send_socket.send(scale_down_marker)
|
||||
await wait_future
|
||||
await self.resume_scheduler_async()
|
||||
except Exception:
|
||||
|
||||
Reference in New Issue
Block a user