[ModelRunnerV2] Avoid pipeline parallel bubbles (#42187)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-02 14:02:01 -07:00
committed by GitHub
parent 557781131a
commit e15f20258b
19 changed files with 615 additions and 144 deletions
+19
View File
@@ -359,6 +359,9 @@ class GroupCoordinator:
assert self_cpu_group is not None
assert self_device_group is not None
self.group_ranks = group_ranks
self.torch_distributed_backend = torch_distributed_backend
self.cpu_group = self_cpu_group
self.device_group = self_device_group
@@ -406,6 +409,22 @@ class GroupCoordinator:
and getattr(self.device_communicator, "supports_tensor_dict", False)
)
def make_sibling_device_group(self, group_desc: str | None = None) -> ProcessGroup:
"""Create a new device-side ProcessGroup with the same per-rank membership
as this coordinator's `device_group`, but backed by a distinct communicator.
This is a collective call: every world rank must invoke it. Used where we
want to issue ops that can run concurrently with ops on `device_group`.
"""
sibling: ProcessGroup | None = None
for ranks in self.group_ranks:
pg = torch.distributed.new_group(
ranks, backend=self.torch_distributed_backend, group_desc=group_desc
)
if self.rank in ranks:
sibling = pg
assert sibling is not None
return sibling
def create_mq_broadcaster(
self, writer_rank=0, external_writer_handle=None, blocking=True
):