[Core] Pipeline Parallel Support (#4412)

Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>
This commit is contained in:
Murali Andoorveedu
2024-07-02 10:58:08 -07:00
committed by GitHub
parent 15aba081f3
commit c5832d2ae9
82 changed files with 1096 additions and 400 deletions
+28 -22
View File
@@ -416,7 +416,7 @@ class GroupCoordinator:
assert dst < self.world_size, f"Invalid dst rank ({dst})"
assert dst != self.rank, (
assert dst != self.rank_in_group, (
"Invalid destination rank. Destination rank is the same "
"as the current rank.")
@@ -446,7 +446,7 @@ class GroupCoordinator:
assert src < self.world_size, f"Invalid src rank ({src})"
assert src != self.rank, (
assert src != self.rank_in_group, (
"Invalid source rank. Source rank is the same as the current rank."
)
@@ -454,7 +454,7 @@ class GroupCoordinator:
# Receive object size
rank_size = torch.distributed.recv(size_tensor,
src=src,
src=self.ranks[src],
group=self.cpu_group)
# Tensor to receive serialized objects into.
@@ -464,7 +464,7 @@ class GroupCoordinator:
device="cpu")
rank_object = torch.distributed.recv(object_tensor,
src=src,
src=self.ranks[src],
group=self.cpu_group)
assert rank_object == rank_size, (
@@ -491,10 +491,9 @@ class GroupCoordinator:
group = self.device_group
metadata_group = self.cpu_group
assert src < self.world_size, f"Invalid src rank ({src})"
src = self.ranks[src]
rank = self.rank
if rank == src:
rank_in_group = self.rank_in_group
if rank_in_group == src:
metadata_list: List[Tuple[Any, Any]] = []
assert isinstance(
tensor_dict,
@@ -512,13 +511,13 @@ class GroupCoordinator:
if tensor.is_cpu:
# use metadata_group for CPU tensors
handle = torch.distributed.broadcast(tensor,
src=src,
src=self.ranks[src],
group=metadata_group,
async_op=True)
else:
# use group for GPU tensors
handle = torch.distributed.broadcast(tensor,
src=src,
src=self.ranks[src],
group=group,
async_op=True)
async_handles.append(handle)
@@ -542,15 +541,16 @@ class GroupCoordinator:
# use metadata_group for CPU tensors
handle = torch.distributed.broadcast(
tensor,
src=src,
src=self.ranks[src],
group=metadata_group,
async_op=True)
else:
# use group for GPU tensors
handle = torch.distributed.broadcast(tensor,
src=src,
group=group,
async_op=True)
handle = torch.distributed.broadcast(
tensor,
src=self.ranks[src],
group=group,
async_op=True)
async_handles.append(handle)
_update_nested_dict(tensor_dict, key, tensor)
else:
@@ -575,7 +575,7 @@ class GroupCoordinator:
metadata_group = self.cpu_group
if dst is None:
dst = self.next_rank
dst = (self.rank_in_group + 1) % self.world_size
assert dst < self.world_size, f"Invalid dst rank ({dst})"
metadata_list: List[Tuple[Any, Any]] = []
@@ -593,10 +593,14 @@ class GroupCoordinator:
continue
if tensor.is_cpu:
# use metadata_group for CPU tensors
torch.distributed.send(tensor, dst=dst, group=metadata_group)
torch.distributed.send(tensor,
dst=self.ranks[dst],
group=metadata_group)
else:
# use group for GPU tensors
torch.distributed.send(tensor, dst=dst, group=group)
torch.distributed.send(tensor,
dst=self.ranks[dst],
group=group)
return None
def recv_tensor_dict(
@@ -614,7 +618,7 @@ class GroupCoordinator:
metadata_group = self.cpu_group
if src is None:
src = self.prev_rank
src = (self.rank_in_group - 1) % self.world_size
assert src < self.world_size, f"Invalid src rank ({src})"
recv_metadata_list = self.recv_object(src=src)
@@ -631,11 +635,13 @@ class GroupCoordinator:
if tensor.is_cpu:
# use metadata_group for CPU tensors
torch.distributed.recv(tensor,
src=src,
src=self.ranks[src],
group=metadata_group)
else:
# use group for GPU tensors
torch.distributed.recv(tensor, src=src, group=group)
torch.distributed.recv(tensor,
src=self.ranks[src],
group=group)
_update_nested_dict(tensor_dict, key, tensor)
else:
_update_nested_dict(tensor_dict, key, value)
@@ -654,7 +660,7 @@ class GroupCoordinator:
"""Sends a tensor to the destination rank in a non-blocking way"""
"""NOTE: `dst` is the local rank of the destination rank."""
if dst is None:
dst = self.next_rank
dst = (self.rank_in_group + 1) % self.world_size
pynccl_comm = self.pynccl_comm
if pynccl_comm is not None and not pynccl_comm.disabled:
@@ -669,7 +675,7 @@ class GroupCoordinator:
"""Receives a tensor from the src rank."""
"""NOTE: `src` is the local rank of the destination rank."""
if src is None:
src = self.prev_rank
src = (self.rank_in_group - 1) % self.world_size
tensor = torch.empty(size, dtype=dtype, device=self.device)
pynccl_comm = self.pynccl_comm