[core][distributed] add ep group and all2all interface (#18077)

Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
youkaichao
2025-05-14 10:46:49 +08:00
committed by GitHub
parent 754b699cbe
commit 6266c57bae
8 changed files with 234 additions and 41 deletions
+59 -2
View File
@@ -757,6 +757,22 @@ class GroupCoordinator:
if self.mq_broadcaster is not None:
self.mq_broadcaster = None
def prepare_communication_buffer_for_model(self, model: torch.nn.Module):
if self.device_communicator is not None:
self.device_communicator.prepare_communication_buffer_for_model(
model)
def dispatch(
self, hidden_states: torch.Tensor,
router_logits: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
if self.device_communicator is not None:
return self.device_communicator.dispatch(hidden_states,
router_logits)
def combine(self, hidden_states) -> torch.Tensor:
if self.device_communicator is not None:
return self.device_communicator.combine(hidden_states)
_WORLD: Optional[GroupCoordinator] = None
@@ -816,6 +832,14 @@ def get_dp_group() -> GroupCoordinator:
return _DP
_EP: Optional[GroupCoordinator] = None
def get_ep_group() -> GroupCoordinator:
assert _EP is not None, ("expert parallel group is not initialized")
return _EP
def get_pp_group() -> GroupCoordinator:
assert _PP is not None, (
"pipeline model parallel group is not initialized")
@@ -1001,10 +1025,21 @@ def initialize_model_parallel(
backend,
group_name="dp")
global _EP
assert _EP is None, ("expert parallel group is already initialized")
group_ranks = all_ranks.transpose(1, 2).reshape(
-1, data_parallel_size * tensor_model_parallel_size).unbind(0)
group_ranks = [x.tolist() for x in group_ranks]
_EP = init_model_parallel_group(group_ranks,
get_world_group().local_rank,
backend,
group_name="ep")
logger.info(
"rank %s in world size %s is assigned as "
"DP rank %s, PP rank %s, TP rank %s", rank, world_size,
_DP.rank_in_group, _PP.rank_in_group, _TP.rank_in_group)
"DP rank %s, PP rank %s, TP rank %s, EP rank %s", rank, world_size,
_DP.rank_in_group, _PP.rank_in_group, _TP.rank_in_group,
_EP.rank_in_group)
def ensure_model_parallel_initialized(
@@ -1035,6 +1070,23 @@ def ensure_model_parallel_initialized(
f"{pipeline_model_parallel_size=}")
def prepare_communication_buffer_for_model(model: torch.nn.Module):
"""Prepare the communication buffer for the model.
Traditional communication libraries like NCCL are almost
model agnostic. However, emerging new communication libraries like
MoE all2all (DeepEP) usually allocate the communication buffer
based on the model shape for optimal performance.
"""
if _TP is not None:
_TP.prepare_communication_buffer_for_model(model)
if _PP is not None:
_PP.prepare_communication_buffer_for_model(model)
if _DP is not None:
_DP.prepare_communication_buffer_for_model(model)
if _EP is not None:
_EP.prepare_communication_buffer_for_model(model)
def model_parallel_is_initialized():
"""Check if tensor and pipeline parallel groups are initialized."""
return (_TP is not None and _PP is not None)
@@ -1095,6 +1147,11 @@ def destroy_model_parallel():
_DP.destroy()
_DP = None
global _EP
if _EP:
_EP.destroy()
_EP = None
def destroy_distributed_environment():
global _WORLD