[Perf][ROCm] Add AITER custom AG/RS (DP only) (#48247)

Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
This commit is contained in:
Simon Danielsson
2026-09-11 13:19:57 +08:00
committed by GitHub
parent 980c16c8e4
commit ae48466cf3
8 changed files with 549 additions and 74 deletions
+52 -10
View File
@@ -1416,6 +1416,8 @@ def init_test_distributed_environment(
rank: int,
distributed_init_port: str,
local_rank: int = -1,
data_parallel_size: int = 1,
data_parallel_master_port: int | None = None,
) -> None:
# Note: This function is often called from Ray worker processes, so we
# can't rely on pytest fixtures to set the config. We check if the config
@@ -1428,6 +1430,29 @@ def init_test_distributed_environment(
distributed_init_method = f"tcp://localhost:{distributed_init_port}"
if data_parallel_size > 1:
# For DP we need to set a common DP master port
from vllm.config.parallel import ParallelConfig
assert data_parallel_master_port is not None, (
"data_parallel_master_port is required when data_parallel_size > 1"
)
tp_pp_world = tp_size * pp_size
parallel_config = ParallelConfig(
data_parallel_size=data_parallel_size,
data_parallel_rank=rank // tp_pp_world,
_data_parallel_master_port_list=[int(data_parallel_master_port)],
)
with set_current_vllm_config(VllmConfig(parallel_config=parallel_config)):
init_distributed_environment(
world_size=tp_pp_world,
rank=rank % tp_pp_world,
distributed_init_method=distributed_init_method,
local_rank=local_rank if local_rank >= 0 else rank,
)
ensure_model_parallel_initialized(tp_size, pp_size)
return
if get_current_vllm_config_or_none() is not None:
# Config already set, use it directly
init_distributed_environment(
@@ -1454,6 +1479,7 @@ def multi_process_parallel(
tp_size: int,
pp_size: int,
test_target: Any,
data_parallel_size: int = 1,
) -> None:
import ray
@@ -1474,18 +1500,34 @@ def multi_process_parallel(
)
distributed_init_port = get_open_port()
# Separate port for the DP master group; only used when data_parallel_size > 1.
data_parallel_master_port = get_open_port() if data_parallel_size > 1 else None
world_size = data_parallel_size * tp_size * pp_size
try:
refs = []
for rank in range(tp_size * pp_size):
refs.append(
test_target.remote(
monkeypatch,
tp_size,
pp_size,
rank,
distributed_init_port,
),
)
for rank in range(world_size):
if data_parallel_size > 1:
refs.append(
test_target.remote(
monkeypatch,
tp_size,
pp_size,
rank,
distributed_init_port,
data_parallel_size,
data_parallel_master_port,
),
)
else:
refs.append(
test_target.remote(
monkeypatch,
tp_size,
pp_size,
rank,
distributed_init_port,
),
)
ray.get(refs)
finally:
ray.shutdown()