[BugFix] Prevent orphaned process on NCCL destroy (#39846)
Signed-off-by: Jeffrey Wang <jeffreywang@anyscale.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
parent
639cbfd274
commit
56e5810ff1
@@ -290,6 +290,12 @@ class NCCLLibrary:
|
||||
# it is better not to call it at all.
|
||||
# ncclResult_t ncclCommDestroy(ncclComm_t comm);
|
||||
Function("ncclCommDestroy", ncclResult_t, [ncclComm_t]),
|
||||
# ncclCommAbort frees resources associated with the communicator
|
||||
# without requiring a collective synchronization. Unlike
|
||||
# ncclCommDestroy, it is safe to call during an uncoordinated
|
||||
# shutdown when peer ranks may already be gone.
|
||||
# ncclResult_t ncclCommAbort(ncclComm_t comm);
|
||||
Function("ncclCommAbort", ncclResult_t, [ncclComm_t]),
|
||||
# ncclResult_t ncclGroupStart();
|
||||
Function("ncclGroupStart", ncclResult_t, []),
|
||||
# ncclResult_t ncclGroupEnd();
|
||||
@@ -548,6 +554,9 @@ class NCCLLibrary:
|
||||
def ncclCommDestroy(self, comm: ncclComm_t) -> None:
|
||||
self.NCCL_CHECK(self._funcs["ncclCommDestroy"](comm))
|
||||
|
||||
def ncclCommAbort(self, comm: ncclComm_t) -> None:
|
||||
self.NCCL_CHECK(self._funcs["ncclCommAbort"](comm))
|
||||
|
||||
def ncclGroupStart(self) -> None:
|
||||
self.NCCL_CHECK(self._funcs["ncclGroupStart"]())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user