[BugFix] Prevent orphaned process on NCCL destroy (#39846)
Signed-off-by: Jeffrey Wang <jeffreywang@anyscale.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
parent
639cbfd274
commit
56e5810ff1
@@ -3,6 +3,8 @@
|
||||
|
||||
|
||||
# ===================== import region =====================
|
||||
import threading
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from torch.distributed import ProcessGroup, ReduceOp
|
||||
@@ -145,8 +147,19 @@ class PyNcclCommunicator:
|
||||
|
||||
def destroy(self):
|
||||
if self.available and not self.disabled:
|
||||
with torch.accelerator.device_index(self.device.index):
|
||||
self.nccl.ncclCommDestroy(self.comm)
|
||||
# ncclCommAbort can block until all CUDA graphs that
|
||||
# captured NCCL ops on this comm are destroyed — and
|
||||
# those graphs are released later in this same main-
|
||||
# thread teardown, so a direct call here self-deadlocks.
|
||||
# Run it in a daemon thread with a timeout: the main
|
||||
# thread proceeds, the graphs drop, and the abort returns.
|
||||
def _abort():
|
||||
with torch.accelerator.device_index(self.device.index):
|
||||
self.nccl.ncclCommAbort(self.comm)
|
||||
|
||||
abort_thread = threading.Thread(target=_abort, daemon=True)
|
||||
abort_thread.start()
|
||||
abort_thread.join(timeout=5.0)
|
||||
self.available = False
|
||||
self.disabled = True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user