elastic_ep: Fix issues with repeated scale up/down cycles (#37131)

Signed-off-by: Itay Alroy <ialroy@nvidia.com>
Co-authored-by: Ron Tourgeman <rtourgeman@nvidia.com>
This commit is contained in:
Itay Alroy
2026-03-20 23:13:02 +00:00
committed by GitHub
co-authored by Ron Tourgeman
parent e5ed6c6c13
commit c57d38d603
10 changed files with 129 additions and 90 deletions
@@ -338,6 +338,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
def destroy(self):
if self.pynccl_comm is not None:
self.pynccl_comm.destroy()
self.pynccl_comm = None
if self.ca_comm is not None:
self.ca_comm = None
@@ -145,6 +145,13 @@ class PyNcclCommunicator:
stream.synchronize()
del data
def destroy(self):
if self.available and not self.disabled:
with torch.accelerator.device_index(self.device.index):
self.nccl.ncclCommDestroy(self.comm)
self.available = False
self.disabled = True
def all_reduce(
self,
in_tensor: torch.Tensor,