diff --git a/tests/distributed/test_eplb_execute.py b/tests/distributed/test_eplb_execute.py index 2037535e3d..acdb9f39fc 100644 --- a/tests/distributed/test_eplb_execute.py +++ b/tests/distributed/test_eplb_execute.py @@ -36,7 +36,11 @@ def test_gloo_receive_staging_does_not_force_gpu_sync(monkeypatch): gsd._install_copy_checkers() monkeypatch.setattr(eplb_comm, "is_local_first_rank", lambda: False) - monkeypatch.setattr(eplb_comm, "P2POp", lambda op, tensor, peer, group: tensor) + monkeypatch.setattr( + eplb_comm, + "P2POp", + lambda op, tensor, peer=None, group=None, tag=0, group_peer=None: tensor, + ) def receive(tensors): for tensor in tensors: @@ -849,3 +853,42 @@ def test_rearrange_expert_weights_profile_mode(world_size): _test_rearrange_expert_weights_profile_mode, world_size, ) + + +def _test_eplb_communicator_pipeline_parallel(env, world_size, backend): + set_env_vars_and_device(env) + config = VllmConfig() + config.parallel_config.tensor_parallel_size = 2 + config.parallel_config.pipeline_parallel_size = world_size // 2 + + with set_current_vllm_config(config): + ensure_model_parallel_initialized( + tensor_model_parallel_size=2, + pipeline_model_parallel_size=world_size // 2, + ) + ep_group = get_tp_group() + peer_rank = 1 - ep_group.rank_in_group + rank = torch.distributed.get_rank() + send_tensor = torch.full((16,), rank, device="cuda", dtype=torch.int32) + recv_tensor = torch.empty_like(send_tensor) + communicator = create_eplb_communicator_or_raise( + group_coordinator=ep_group, + backend=backend, + expert_weights=[[send_tensor]], + expert_buffer=[recv_tensor], + ) + communicator.add_send([send_tensor], peer_rank, expert_id=0) + communicator.add_recv([recv_tensor], peer_rank, expert_id=0) + communicator.execute() + + expected = torch.full_like(recv_tensor, ep_group.ranks[peer_rank]) + torch.testing.assert_close(recv_tensor, expected) + + +@pytest.mark.parametrize("world_size", [2, 4]) +@pytest.mark.parametrize("backend", ["torch_nccl", "torch_gloo"]) +def test_eplb_communicator_pipeline_parallel(world_size, backend): + """Transfer weights within each PP stage, including nonzero global ranks.""" + if torch.accelerator.device_count() < world_size: + pytest.skip(f"Need at least {world_size} GPUs to run the test") + distributed_run(_test_eplb_communicator_pipeline_parallel, world_size, backend) diff --git a/vllm/distributed/eplb/eplb_communicator.py b/vllm/distributed/eplb/eplb_communicator.py index 730c01a388..473058d525 100644 --- a/vllm/distributed/eplb/eplb_communicator.py +++ b/vllm/distributed/eplb/eplb_communicator.py @@ -120,8 +120,8 @@ class TorchDistNcclEplbCommunicator(EplbCommunicator): P2POp( torch.distributed.isend, tensor, - dst_rank, - self._ep_group, + group=self._ep_group, + group_peer=dst_rank, ) ) @@ -136,8 +136,8 @@ class TorchDistNcclEplbCommunicator(EplbCommunicator): P2POp( torch.distributed.irecv, tensor, - src_rank, - self._ep_group, + group=self._ep_group, + group_peer=src_rank, ) ) @@ -199,8 +199,8 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator): P2POp( torch.distributed.isend, cpu_tensor, - peer_rank, - self._cpu_group, + group=self._cpu_group, + group_peer=peer_rank, ) ) continue @@ -211,8 +211,8 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator): P2POp( torch.distributed.irecv, cpu_tensor, - peer_rank, - self._cpu_group, + group=self._cpu_group, + group_peer=peer_rank, ) ) recv_staging.append((tensor, cpu_tensor))