[ModelRunnerV2] Avoid pipeline parallel bubbles (#42187)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -359,6 +359,9 @@ class GroupCoordinator:
|
||||
assert self_cpu_group is not None
|
||||
assert self_device_group is not None
|
||||
|
||||
self.group_ranks = group_ranks
|
||||
self.torch_distributed_backend = torch_distributed_backend
|
||||
|
||||
self.cpu_group = self_cpu_group
|
||||
self.device_group = self_device_group
|
||||
|
||||
@@ -406,6 +409,22 @@ class GroupCoordinator:
|
||||
and getattr(self.device_communicator, "supports_tensor_dict", False)
|
||||
)
|
||||
|
||||
def make_sibling_device_group(self, group_desc: str | None = None) -> ProcessGroup:
|
||||
"""Create a new device-side ProcessGroup with the same per-rank membership
|
||||
as this coordinator's `device_group`, but backed by a distinct communicator.
|
||||
This is a collective call: every world rank must invoke it. Used where we
|
||||
want to issue ops that can run concurrently with ops on `device_group`.
|
||||
"""
|
||||
sibling: ProcessGroup | None = None
|
||||
for ranks in self.group_ranks:
|
||||
pg = torch.distributed.new_group(
|
||||
ranks, backend=self.torch_distributed_backend, group_desc=group_desc
|
||||
)
|
||||
if self.rank in ranks:
|
||||
sibling = pg
|
||||
assert sibling is not None
|
||||
return sibling
|
||||
|
||||
def create_mq_broadcaster(
|
||||
self, writer_rank=0, external_writer_handle=None, blocking=True
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user