[Perf][ROCm] Add AITER custom AG/RS (DP only) (#48247)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
This commit is contained in:
+52
-10
@@ -1416,6 +1416,8 @@ def init_test_distributed_environment(
|
||||
rank: int,
|
||||
distributed_init_port: str,
|
||||
local_rank: int = -1,
|
||||
data_parallel_size: int = 1,
|
||||
data_parallel_master_port: int | None = None,
|
||||
) -> None:
|
||||
# Note: This function is often called from Ray worker processes, so we
|
||||
# can't rely on pytest fixtures to set the config. We check if the config
|
||||
@@ -1428,6 +1430,29 @@ def init_test_distributed_environment(
|
||||
|
||||
distributed_init_method = f"tcp://localhost:{distributed_init_port}"
|
||||
|
||||
if data_parallel_size > 1:
|
||||
# For DP we need to set a common DP master port
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
|
||||
assert data_parallel_master_port is not None, (
|
||||
"data_parallel_master_port is required when data_parallel_size > 1"
|
||||
)
|
||||
tp_pp_world = tp_size * pp_size
|
||||
parallel_config = ParallelConfig(
|
||||
data_parallel_size=data_parallel_size,
|
||||
data_parallel_rank=rank // tp_pp_world,
|
||||
_data_parallel_master_port_list=[int(data_parallel_master_port)],
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig(parallel_config=parallel_config)):
|
||||
init_distributed_environment(
|
||||
world_size=tp_pp_world,
|
||||
rank=rank % tp_pp_world,
|
||||
distributed_init_method=distributed_init_method,
|
||||
local_rank=local_rank if local_rank >= 0 else rank,
|
||||
)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
return
|
||||
|
||||
if get_current_vllm_config_or_none() is not None:
|
||||
# Config already set, use it directly
|
||||
init_distributed_environment(
|
||||
@@ -1454,6 +1479,7 @@ def multi_process_parallel(
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
test_target: Any,
|
||||
data_parallel_size: int = 1,
|
||||
) -> None:
|
||||
import ray
|
||||
|
||||
@@ -1474,18 +1500,34 @@ def multi_process_parallel(
|
||||
)
|
||||
|
||||
distributed_init_port = get_open_port()
|
||||
# Separate port for the DP master group; only used when data_parallel_size > 1.
|
||||
data_parallel_master_port = get_open_port() if data_parallel_size > 1 else None
|
||||
world_size = data_parallel_size * tp_size * pp_size
|
||||
try:
|
||||
refs = []
|
||||
for rank in range(tp_size * pp_size):
|
||||
refs.append(
|
||||
test_target.remote(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
),
|
||||
)
|
||||
for rank in range(world_size):
|
||||
if data_parallel_size > 1:
|
||||
refs.append(
|
||||
test_target.remote(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
data_parallel_size,
|
||||
data_parallel_master_port,
|
||||
),
|
||||
)
|
||||
else:
|
||||
refs.append(
|
||||
test_target.remote(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
),
|
||||
)
|
||||
ray.get(refs)
|
||||
finally:
|
||||
ray.shutdown()
|
||||
|
||||
Reference in New Issue
Block a user