[Core][Distributed] refactor custom allreduce to support multiple tp groups (#4754)

This commit is contained in:
youkaichao
2024-05-12 17:47:59 -07:00
committed by GitHub
parent a7be4d0072
commit 702bee461f
10 changed files with 327 additions and 226 deletions
+8 -9
View File
@@ -6,24 +6,24 @@ from vllm.utils import get_open_port
def init_test_distributed_environment(
pipeline_parallel_size: int,
tensor_parallel_size: int,
tp_size: int,
pp_size: int,
rank: int,
distributed_init_port: str,
local_rank: int = -1,
) -> None:
distributed_init_method = f"tcp://localhost:{distributed_init_port}"
init_distributed_environment(
world_size=pipeline_parallel_size * tensor_parallel_size,
world_size=pp_size * tp_size,
rank=rank,
distributed_init_method=distributed_init_method,
local_rank=local_rank)
ensure_model_parallel_initialized(tensor_parallel_size,
pipeline_parallel_size)
ensure_model_parallel_initialized(tp_size, pp_size)
def multi_process_tensor_parallel(
tensor_parallel_size: int,
tp_size: int,
pp_size: int,
test_target,
) -> None:
# Using ray helps debugging the error when it failed
@@ -32,10 +32,9 @@ def multi_process_tensor_parallel(
distributed_init_port = get_open_port()
refs = []
for rank in range(tensor_parallel_size):
for rank in range(tp_size * pp_size):
refs.append(
test_target.remote(tensor_parallel_size, rank,
distributed_init_port))
test_target.remote(tp_size, pp_size, rank, distributed_init_port))
ray.get(refs)
ray.shutdown()