[BugFix] Prevent orphaned process on NCCL destroy (#39846)

Signed-off-by: Jeffrey Wang <jeffreywang@anyscale.com>
Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
This commit is contained in:
Jeffrey Wang
2026-05-11 15:25:26 -04:00
committed by GitHub
co-authored by Tyler Michael Smith
parent 639cbfd274
commit 56e5810ff1
2 changed files with 24 additions and 2 deletions
@@ -290,6 +290,12 @@ class NCCLLibrary:
# it is better not to call it at all.
# ncclResult_t ncclCommDestroy(ncclComm_t comm);
Function("ncclCommDestroy", ncclResult_t, [ncclComm_t]),
# ncclCommAbort frees resources associated with the communicator
# without requiring a collective synchronization. Unlike
# ncclCommDestroy, it is safe to call during an uncoordinated
# shutdown when peer ranks may already be gone.
# ncclResult_t ncclCommAbort(ncclComm_t comm);
Function("ncclCommAbort", ncclResult_t, [ncclComm_t]),
# ncclResult_t ncclGroupStart();
Function("ncclGroupStart", ncclResult_t, []),
# ncclResult_t ncclGroupEnd();
@@ -548,6 +554,9 @@ class NCCLLibrary:
def ncclCommDestroy(self, comm: ncclComm_t) -> None:
self.NCCL_CHECK(self._funcs["ncclCommDestroy"](comm))
def ncclCommAbort(self, comm: ncclComm_t) -> None:
self.NCCL_CHECK(self._funcs["ncclCommAbort"](comm))
def ncclGroupStart(self) -> None:
self.NCCL_CHECK(self._funcs["ncclGroupStart"]())