[KVConnector] Aggregate finished requests on the scheduler (#19555)
Signed-off-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -190,7 +190,9 @@ class KVConnectorBase_V1(ABC):
|
||||
) -> tuple[Optional[set[str]], Optional[set[str]]]:
|
||||
"""
|
||||
Notifies worker-side connector ids of requests that have
|
||||
finished generating tokens.
|
||||
finished generating tokens on the worker.
|
||||
The scheduler process (via the MultiprocExecutor) will use this output
|
||||
to track which workers are done.
|
||||
|
||||
Returns:
|
||||
ids of requests that have finished asynchronous transfer
|
||||
|
||||
@@ -408,14 +408,6 @@ class NixlConnectorWorker:
|
||||
# Track the expiration time of requests that are waiting to be sent.
|
||||
self._reqs_to_send: dict[ReqId, float] = {}
|
||||
|
||||
# Complete transfer tracker. Used by the rank 0 to track finished
|
||||
# transactions on ranks 1 to N-1.
|
||||
# [req_id -> count]
|
||||
self._done_recving_count: defaultdict[ReqId,
|
||||
int] = defaultdict(lambda: 0)
|
||||
self._done_sending_count: defaultdict[ReqId,
|
||||
int] = defaultdict(lambda: 0)
|
||||
|
||||
# Background thread for handling new handshake requests.
|
||||
self._nixl_handshake_listener_t: Optional[threading.Thread] = None
|
||||
# Background thread for initializing new NIXL handshakes.
|
||||
@@ -830,15 +822,9 @@ class NixlConnectorWorker:
|
||||
|
||||
def get_finished(self) -> tuple[set[str], set[str]]:
|
||||
"""
|
||||
Get requests that are done sending or recving.
|
||||
|
||||
In TP>1 setup, each rank exchanges KVs with its counterpart
|
||||
ranks independently. get_finished() runs in a worker creates
|
||||
the done_sending and done_recving sets that are sent to the
|
||||
scheduler via ModelRunnerOutput by Rank 0. To ensure trnxs
|
||||
are done before adding to finished, Ranks 1 to N-1 communicate
|
||||
to Rank 0 once their transaction is done + Rank 0 returns
|
||||
finished sets to Scheduler only once all ranks are done.
|
||||
Get requests that are done sending or recving on this specific worker.
|
||||
The scheduler process (via the MultiprocExecutor) will use this output
|
||||
to track which workers are done.
|
||||
"""
|
||||
done_sending = self._get_new_notifs()
|
||||
done_recving = self._pop_done_transfers(self._recving_transfers)
|
||||
@@ -858,50 +844,7 @@ class NixlConnectorWorker:
|
||||
del self._reqs_to_send[req_id]
|
||||
done_sending.add(req_id)
|
||||
|
||||
if self.world_size == 1:
|
||||
return done_sending, done_recving
|
||||
|
||||
# Rank 0: get finished from all other ranks.
|
||||
if self.tp_rank == 0:
|
||||
for req_id in done_sending:
|
||||
self._done_sending_count[req_id] += 1
|
||||
for req_id in done_recving:
|
||||
self._done_recving_count[req_id] += 1
|
||||
|
||||
# Keep track of how many other ranks have finished.
|
||||
other_ranks_finished_ids: list[str] = []
|
||||
for i in range(1, self.world_size):
|
||||
other_ranks_finished_ids.extend(
|
||||
self.tp_group.recv_object(src=i))
|
||||
for req_id in other_ranks_finished_ids:
|
||||
if (req_id in self._done_recving_count
|
||||
or req_id in self._recving_transfers):
|
||||
self._done_recving_count[req_id] += 1
|
||||
else:
|
||||
self._done_sending_count[req_id] += 1
|
||||
|
||||
# Return ids that finished on all ranks to the scheduler.
|
||||
all_done_recving: set[str] = set()
|
||||
for req_id in list(self._done_recving_count.keys()):
|
||||
if self._done_recving_count[req_id] == self.world_size:
|
||||
del self._done_recving_count[req_id]
|
||||
all_done_recving.add(req_id)
|
||||
|
||||
all_done_sending: set[str] = set()
|
||||
for req_id in list(self._done_sending_count.keys()):
|
||||
if self._done_sending_count[req_id] >= self.world_size:
|
||||
del self._done_sending_count[req_id]
|
||||
all_done_sending.add(req_id)
|
||||
|
||||
return all_done_sending, all_done_recving
|
||||
|
||||
# Ranks 1 to N-1: send finished ids to Rank 0.
|
||||
else:
|
||||
finished_req_ids = list(done_recving.union(done_sending))
|
||||
self.tp_group.send_object(finished_req_ids, dst=0)
|
||||
|
||||
# Unused as only Rank 0 results are sent to scheduler.
|
||||
return done_sending, done_recving
|
||||
return done_sending, done_recving
|
||||
|
||||
def _get_new_notifs(self) -> set[str]:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user