[BugFix] Prevent orphaned process on NCCL destroy (#39846)

Signed-off-by: Jeffrey Wang <jeffreywang@anyscale.com>
Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
This commit is contained in:
Jeffrey Wang
2026-05-11 15:25:26 -04:00
committed by GitHub
co-authored by Tyler Michael Smith
parent 639cbfd274
commit 56e5810ff1
2 changed files with 24 additions and 2 deletions
@@ -3,6 +3,8 @@
# ===================== import region =====================
import threading
import torch
import torch.distributed as dist
from torch.distributed import ProcessGroup, ReduceOp
@@ -145,8 +147,19 @@ class PyNcclCommunicator:
def destroy(self):
if self.available and not self.disabled:
with torch.accelerator.device_index(self.device.index):
self.nccl.ncclCommDestroy(self.comm)
# ncclCommAbort can block until all CUDA graphs that
# captured NCCL ops on this comm are destroyed — and
# those graphs are released later in this same main-
# thread teardown, so a direct call here self-deadlocks.
# Run it in a daemon thread with a timeout: the main
# thread proceeds, the graphs drop, and the abort returns.
def _abort():
with torch.accelerator.device_index(self.device.index):
self.nccl.ncclCommAbort(self.comm)
abort_thread = threading.Thread(target=_abort, daemon=True)
abort_thread.start()
abort_thread.join(timeout=5.0)
self.available = False
self.disabled = True