elastic_ep: Fix issues with repeated scale up/down cycles (#37131)
Signed-off-by: Itay Alroy <ialroy@nvidia.com> Co-authored-by: Ron Tourgeman <rtourgeman@nvidia.com>
This commit is contained in:
co-authored by
Ron Tourgeman
parent
e5ed6c6c13
commit
c57d38d603
@@ -338,6 +338,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
|
||||
def destroy(self):
|
||||
if self.pynccl_comm is not None:
|
||||
self.pynccl_comm.destroy()
|
||||
self.pynccl_comm = None
|
||||
if self.ca_comm is not None:
|
||||
self.ca_comm = None
|
||||
|
||||
@@ -145,6 +145,13 @@ class PyNcclCommunicator:
|
||||
stream.synchronize()
|
||||
del data
|
||||
|
||||
def destroy(self):
|
||||
if self.available and not self.disabled:
|
||||
with torch.accelerator.device_index(self.device.index):
|
||||
self.nccl.ncclCommDestroy(self.comm)
|
||||
self.available = False
|
||||
self.disabled = True
|
||||
|
||||
def all_reduce(
|
||||
self,
|
||||
in_tensor: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user