[Elastic EP] Async preparation (#47288)
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
@@ -175,6 +175,7 @@ class DeviceCommunicatorBase:
|
||||
unique_name: str = "",
|
||||
global_ranks: list[int] | None = None,
|
||||
global_world_size: int | None = None,
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
self.device = device or torch.device("cpu")
|
||||
self.cpu_group = cpu_group
|
||||
@@ -204,26 +205,15 @@ class DeviceCommunicatorBase:
|
||||
self.global_world_size = dist.get_world_size()
|
||||
self.rank_in_group = dist.get_group_rank(self.cpu_group, self.global_rank)
|
||||
|
||||
use_ep = False
|
||||
all2all_backend = None
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
|
||||
config = get_current_vllm_config_or_none()
|
||||
if config is not None:
|
||||
# initialize the all2all manager for DP or sequence-parallel EP.
|
||||
parallel_config = config.parallel_config
|
||||
use_ep = (
|
||||
parallel_config.data_parallel_size > 1
|
||||
or parallel_config.use_sequence_parallel_moe
|
||||
or (
|
||||
parallel_config.enable_expert_parallel
|
||||
and parallel_config.prefill_context_parallel_size > 1
|
||||
)
|
||||
)
|
||||
all2all_backend = parallel_config.all2all_backend
|
||||
all2all_backend = config.parallel_config.all2all_backend
|
||||
|
||||
self.is_ep_communicator = unique_name.split(":")[0] == "ep"
|
||||
self.use_all2all = self.is_ep_communicator and use_ep
|
||||
self.use_all2all = self.is_ep_communicator and use_all2all
|
||||
self.all2all_backend = all2all_backend
|
||||
self.all2all_manager: All2AllManagerBase | None = None
|
||||
|
||||
|
||||
@@ -24,8 +24,11 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
device: torch.device | None = None,
|
||||
device_group: ProcessGroup | None = None,
|
||||
unique_name: str = "",
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
super().__init__(cpu_group, device, device_group, unique_name)
|
||||
super().__init__(
|
||||
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
|
||||
)
|
||||
self.dist_module = torch.distributed
|
||||
|
||||
if (
|
||||
|
||||
@@ -36,6 +36,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
global_ranks: list[int] | None = None,
|
||||
global_world_size: int | None = None,
|
||||
tcp_store_group: StatelessProcessGroup | None = None,
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
cpu_group,
|
||||
@@ -44,6 +45,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
unique_name,
|
||||
global_ranks,
|
||||
global_world_size,
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
if "tp" not in unique_name:
|
||||
# custom allreduce or torch symm mem can be used only by tp
|
||||
|
||||
@@ -20,8 +20,11 @@ class XpuCommunicator(DeviceCommunicatorBase):
|
||||
device: torch.device | None = None,
|
||||
device_group: ProcessGroup | None = None,
|
||||
unique_name: str = "",
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
super().__init__(cpu_group, device, device_group, unique_name)
|
||||
super().__init__(
|
||||
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
|
||||
)
|
||||
self.ca_comm: None = None
|
||||
if self.use_all2all:
|
||||
if self.all2all_backend in ("naive", "allgather_reducescatter"):
|
||||
|
||||
Reference in New Issue
Block a user