[Bugfix][EPLB] Use group-local ranks for torch P2P transfers (#55804)
Signed-off-by: LostFox11 <wangziyue17@huawei.com> Co-authored-by: LostFox11 <wangziyue17@huawei.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: YangShuai52 <178856220+YangShuai52@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
LostFox11
Codex
YangShuai52
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
6ad2e06c4a
commit
15c1cc12ac
@@ -36,7 +36,11 @@ def test_gloo_receive_staging_does_not_force_gpu_sync(monkeypatch):
|
||||
gsd._install_copy_checkers()
|
||||
monkeypatch.setattr(eplb_comm, "is_local_first_rank", lambda: False)
|
||||
|
||||
monkeypatch.setattr(eplb_comm, "P2POp", lambda op, tensor, peer, group: tensor)
|
||||
monkeypatch.setattr(
|
||||
eplb_comm,
|
||||
"P2POp",
|
||||
lambda op, tensor, peer=None, group=None, tag=0, group_peer=None: tensor,
|
||||
)
|
||||
|
||||
def receive(tensors):
|
||||
for tensor in tensors:
|
||||
@@ -849,3 +853,42 @@ def test_rearrange_expert_weights_profile_mode(world_size):
|
||||
_test_rearrange_expert_weights_profile_mode,
|
||||
world_size,
|
||||
)
|
||||
|
||||
|
||||
def _test_eplb_communicator_pipeline_parallel(env, world_size, backend):
|
||||
set_env_vars_and_device(env)
|
||||
config = VllmConfig()
|
||||
config.parallel_config.tensor_parallel_size = 2
|
||||
config.parallel_config.pipeline_parallel_size = world_size // 2
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=2,
|
||||
pipeline_model_parallel_size=world_size // 2,
|
||||
)
|
||||
ep_group = get_tp_group()
|
||||
peer_rank = 1 - ep_group.rank_in_group
|
||||
rank = torch.distributed.get_rank()
|
||||
send_tensor = torch.full((16,), rank, device="cuda", dtype=torch.int32)
|
||||
recv_tensor = torch.empty_like(send_tensor)
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group,
|
||||
backend=backend,
|
||||
expert_weights=[[send_tensor]],
|
||||
expert_buffer=[recv_tensor],
|
||||
)
|
||||
communicator.add_send([send_tensor], peer_rank, expert_id=0)
|
||||
communicator.add_recv([recv_tensor], peer_rank, expert_id=0)
|
||||
communicator.execute()
|
||||
|
||||
expected = torch.full_like(recv_tensor, ep_group.ranks[peer_rank])
|
||||
torch.testing.assert_close(recv_tensor, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
@pytest.mark.parametrize("backend", ["torch_nccl", "torch_gloo"])
|
||||
def test_eplb_communicator_pipeline_parallel(world_size, backend):
|
||||
"""Transfer weights within each PP stage, including nonzero global ranks."""
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(_test_eplb_communicator_pipeline_parallel, world_size, backend)
|
||||
|
||||
@@ -120,8 +120,8 @@ class TorchDistNcclEplbCommunicator(EplbCommunicator):
|
||||
P2POp(
|
||||
torch.distributed.isend,
|
||||
tensor,
|
||||
dst_rank,
|
||||
self._ep_group,
|
||||
group=self._ep_group,
|
||||
group_peer=dst_rank,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -136,8 +136,8 @@ class TorchDistNcclEplbCommunicator(EplbCommunicator):
|
||||
P2POp(
|
||||
torch.distributed.irecv,
|
||||
tensor,
|
||||
src_rank,
|
||||
self._ep_group,
|
||||
group=self._ep_group,
|
||||
group_peer=src_rank,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -199,8 +199,8 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator):
|
||||
P2POp(
|
||||
torch.distributed.isend,
|
||||
cpu_tensor,
|
||||
peer_rank,
|
||||
self._cpu_group,
|
||||
group=self._cpu_group,
|
||||
group_peer=peer_rank,
|
||||
)
|
||||
)
|
||||
continue
|
||||
@@ -211,8 +211,8 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator):
|
||||
P2POp(
|
||||
torch.distributed.irecv,
|
||||
cpu_tensor,
|
||||
peer_rank,
|
||||
self._cpu_group,
|
||||
group=self._cpu_group,
|
||||
group_peer=peer_rank,
|
||||
)
|
||||
)
|
||||
recv_staging.append((tensor, cpu_tensor))
|
||||
|
||||
Reference in New Issue
Block a user