[Perf][ROCm] Add AITER custom AG/RS (DP only) (#48247)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
This commit is contained in:
@@ -29,6 +29,10 @@ import torch
|
||||
import torch.distributed as dist
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.distributed.device_communicators.aiter_custom_all_reduce import (
|
||||
AiterCustomAllreduce,
|
||||
)
|
||||
from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce
|
||||
from vllm.distributed.device_communicators.flashinfer_all_reduce import (
|
||||
FlashInferAllReduce,
|
||||
@@ -42,6 +46,7 @@ from vllm.distributed.device_communicators.pynccl_allocator import (
|
||||
)
|
||||
from vllm.distributed.device_communicators.symm_mem import SymmMemCommunicator
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -56,6 +61,30 @@ BENCHMARK_DTYPE = torch.bfloat16
|
||||
# CUDA graph settings
|
||||
CUDA_GRAPH_CAPTURE_CYCLES = 10
|
||||
|
||||
# All-gather and reduce-scatter are distinct collectives from all-reduce, so
|
||||
# results are compared within a collective (each against its own NCCL/RCCL
|
||||
# baseline) rather than across collectives.
|
||||
COMM_COLLECTIVE = {
|
||||
"ca_1stage": "all-reduce",
|
||||
"ca_2stage": "all-reduce",
|
||||
"pynccl": "all-reduce",
|
||||
"pynccl-symm": "all-reduce",
|
||||
"symm_mem_multimem": "all-reduce",
|
||||
"symm_mem_two_shot": "all-reduce",
|
||||
"flashinfer_trtllm": "all-reduce",
|
||||
"flashinfer_mnnvl": "all-reduce",
|
||||
"aiter_ag": "all-gather",
|
||||
"pynccl_ag": "all-gather",
|
||||
"aiter_rs": "reduce-scatter",
|
||||
"pynccl_rs": "reduce-scatter",
|
||||
}
|
||||
COLLECTIVE_ORDER = ["all-reduce", "all-gather", "reduce-scatter"]
|
||||
COLLECTIVE_BASELINE = {
|
||||
"all-reduce": "pynccl",
|
||||
"all-gather": "pynccl_ag",
|
||||
"reduce-scatter": "pynccl_rs",
|
||||
}
|
||||
|
||||
|
||||
class CommunicatorBenchmark:
|
||||
"""Benchmark class for testing device communicators."""
|
||||
@@ -77,6 +106,11 @@ class CommunicatorBenchmark:
|
||||
max_seq_len = max(sequence_lengths)
|
||||
max_tensor_elements = max_seq_len * HIDDEN_SIZE
|
||||
self.max_size_override = max_tensor_elements * BENCHMARK_DTYPE.itemsize + 1
|
||||
# AITER has effective max size of half the max size, so we double it, and
|
||||
# account for AG output size (which is world_size larger)
|
||||
self.aiter_max_size = (
|
||||
2 * world_size * max_tensor_elements * BENCHMARK_DTYPE.itemsize + 1
|
||||
)
|
||||
|
||||
# Initialize communicators
|
||||
self.custom_allreduce = None
|
||||
@@ -85,6 +119,7 @@ class CommunicatorBenchmark:
|
||||
self.symm_mem_comm_multimem = None
|
||||
self.symm_mem_comm_two_shot = None
|
||||
self.fi_ar_comm = None
|
||||
self.aiter_comm = None
|
||||
|
||||
self._init_communicators()
|
||||
|
||||
@@ -181,6 +216,27 @@ class CommunicatorBenchmark:
|
||||
)
|
||||
self.fi_ar_comm = None
|
||||
|
||||
if current_platform.is_rocm() and rocm_aiter_ops.is_custom_all_reduce_enabled():
|
||||
try:
|
||||
# Also used for AG/RS for DP
|
||||
self.aiter_comm = AiterCustomAllreduce(
|
||||
group=self.cpu_group,
|
||||
device=self.device,
|
||||
max_size=self.aiter_max_size,
|
||||
)
|
||||
if not self.aiter_comm.disabled:
|
||||
logger.info("Rank %s: AITER custom AG/RS initialized", self.rank)
|
||||
else:
|
||||
logger.info("Rank %s: AITER custom AG/RS disabled", self.rank)
|
||||
self.aiter_comm = None
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Rank %s: Failed to initialize AITER custom AG/RS: %s",
|
||||
self.rank,
|
||||
e,
|
||||
)
|
||||
self.aiter_comm = None
|
||||
|
||||
def benchmark_allreduce(
|
||||
self, sequence_length: int, num_warmup: int, num_trials: int
|
||||
) -> dict[str, float]:
|
||||
@@ -385,29 +441,116 @@ class CommunicatorBenchmark:
|
||||
f"CUDA graph benchmark failed for communicator: {e}"
|
||||
) from e
|
||||
|
||||
def benchmark_ag_rs(
|
||||
self, sequence_length: int, num_warmup: int, num_trials: int
|
||||
) -> dict[str, float]:
|
||||
"""Benchmark all-gather and reduce-scatter: custom (AITER) vs NCCL/RCCL.
|
||||
|
||||
def _calculate_speedup_info(comm_results: dict[str, float]) -> str:
|
||||
"""Calculate speedup information for a single tensor size."""
|
||||
Each collective is measured for both the AITER custom kernel and the
|
||||
PyNccl baseline so they compare like-for-like (print_results groups
|
||||
columns by collective). All-gather maps ``(seq, hidden)`` ->
|
||||
``(world_size * seq, hidden)``; reduce-scatter maps ``(seq, hidden)`` ->
|
||||
``(seq // world_size, hidden)`` and so only runs when ``seq`` divides
|
||||
evenly across ranks. Both reuse ``benchmark_allreduce_single`` timed on a
|
||||
``(seq, hidden)`` input; baselines write into a pre-allocated buffer.
|
||||
"""
|
||||
results: dict[str, float] = {}
|
||||
entries = []
|
||||
|
||||
# ---- all-gather: (seq, hidden) -> (world_size * seq, hidden) ----
|
||||
if self.aiter_comm is not None:
|
||||
c = self.aiter_comm
|
||||
entries.append(
|
||||
(
|
||||
"aiter_ag",
|
||||
lambda t, c=c: c.custom_all_gather(t, dim=0),
|
||||
lambda t, c=c: c.should_custom_ag(t),
|
||||
c.capture(),
|
||||
)
|
||||
)
|
||||
if self.pynccl_comm is not None:
|
||||
c = self.pynccl_comm
|
||||
ag_out = torch.empty(
|
||||
sequence_length * self.world_size,
|
||||
HIDDEN_SIZE,
|
||||
dtype=BENCHMARK_DTYPE,
|
||||
device=self.device,
|
||||
)
|
||||
entries.append(
|
||||
(
|
||||
"pynccl_ag",
|
||||
lambda t, c=c, o=ag_out: c.all_gather(o, t),
|
||||
lambda t: True,
|
||||
nullcontext(),
|
||||
)
|
||||
)
|
||||
|
||||
# ---- reduce-scatter: (seq, hidden) -> (seq // world_size, hidden) ----
|
||||
if sequence_length % self.world_size == 0:
|
||||
rs_shape = (sequence_length // self.world_size, HIDDEN_SIZE)
|
||||
if self.aiter_comm is not None:
|
||||
c = self.aiter_comm
|
||||
rs_out = torch.empty(
|
||||
rs_shape, dtype=BENCHMARK_DTYPE, device=self.device
|
||||
)
|
||||
entries.append(
|
||||
(
|
||||
"aiter_rs",
|
||||
lambda t, c=c, o=rs_out: c.custom_reduce_scatter(t, o, dim=0),
|
||||
lambda t, c=c: c.should_custom_rs(t, dim=0),
|
||||
c.capture(),
|
||||
)
|
||||
)
|
||||
if self.pynccl_comm is not None:
|
||||
c = self.pynccl_comm
|
||||
rs_out = torch.empty(
|
||||
rs_shape, dtype=BENCHMARK_DTYPE, device=self.device
|
||||
)
|
||||
entries.append(
|
||||
(
|
||||
"pynccl_rs",
|
||||
lambda t, c=c, o=rs_out: c.reduce_scatter(o, t),
|
||||
lambda t: True,
|
||||
nullcontext(),
|
||||
)
|
||||
)
|
||||
|
||||
for name, fn, should_use_fn, context in entries:
|
||||
latency = self.benchmark_allreduce_single(
|
||||
sequence_length, fn, should_use_fn, context, num_warmup, num_trials
|
||||
)
|
||||
if latency is not None:
|
||||
results[name] = latency
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _calculate_speedup_info(comm_results: dict[str, float], baseline: str) -> str:
|
||||
"""Fastest comm in a collective group and its speedup vs ``baseline``."""
|
||||
if not comm_results:
|
||||
return "N/A"
|
||||
|
||||
# Find the fastest communicator
|
||||
fastest_comm = min(comm_results.keys(), key=lambda k: comm_results[k])
|
||||
fastest_time = comm_results[fastest_comm]
|
||||
|
||||
# Calculate speedup vs PyNccl if available
|
||||
if "pynccl" in comm_results:
|
||||
pynccl_time = comm_results["pynccl"]
|
||||
speedup = pynccl_time / fastest_time
|
||||
if baseline in comm_results and fastest_time > 0:
|
||||
speedup = comm_results[baseline] / fastest_time
|
||||
return f"{fastest_comm} ({speedup:.2f}x)"
|
||||
else:
|
||||
return f"{fastest_comm} (N/A)"
|
||||
return f"{fastest_comm} (N/A)"
|
||||
|
||||
|
||||
def print_results(
|
||||
results: dict[str, dict[str, float]], sequence_lengths: list[int], world_size: int
|
||||
results: dict[int, dict[str, float]], sequence_lengths: list[int], world_size: int
|
||||
):
|
||||
"""Print benchmark results in a formatted table."""
|
||||
"""Print benchmark results as one table per collective.
|
||||
|
||||
All-reduce, all-gather and reduce-scatter are different collectives, so each
|
||||
gets its own table and its speedup is computed against that collective's
|
||||
NCCL/RCCL baseline rather than across collectives.
|
||||
"""
|
||||
present: set[str] = set()
|
||||
for size_results in results.values():
|
||||
present.update(size_results.keys())
|
||||
|
||||
print(f"\n{'=' * 130}")
|
||||
print("Device Communicator Benchmark Results")
|
||||
@@ -415,53 +558,45 @@ def print_results(
|
||||
f"World Size: {world_size}, Data Type: {BENCHMARK_DTYPE}, "
|
||||
f"Hidden Size: {HIDDEN_SIZE}"
|
||||
)
|
||||
print("Tensor Shape is the per-rank input; times are ms per operation.")
|
||||
print(f"{'=' * 130}")
|
||||
|
||||
# Get all communicator names
|
||||
all_comms = set()
|
||||
for size_results in results.values():
|
||||
all_comms.update(size_results.keys())
|
||||
for collective in COLLECTIVE_ORDER:
|
||||
comms = sorted(c for c in present if COMM_COLLECTIVE.get(c) == collective)
|
||||
if not comms:
|
||||
continue
|
||||
baseline = COLLECTIVE_BASELINE[collective]
|
||||
|
||||
all_comms = sorted(list(all_comms))
|
||||
print(f"\n{collective} (speedup vs {baseline})")
|
||||
header = f"{'Tensor Shape':<20}{'Tensor Size':<15}"
|
||||
for comm in comms:
|
||||
header += f"{comm:<20}"
|
||||
header += f"{'Best':<30}"
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
|
||||
# Print header
|
||||
header = f"{'Tensor Shape':<20}{'Tensor Size':<15}"
|
||||
for comm in all_comms:
|
||||
header += f"{comm:<20}"
|
||||
header += f"{'Best (Speedup vs PyNccl)':<30}"
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
for seq_len in sequence_lengths:
|
||||
if seq_len not in results:
|
||||
continue
|
||||
row_results = {
|
||||
c: results[seq_len][c] for c in comms if c in results[seq_len]
|
||||
}
|
||||
|
||||
# Print results for each sequence length
|
||||
for seq_len in sequence_lengths:
|
||||
if seq_len in results:
|
||||
# Calculate tensor size in elements and bytes
|
||||
tensor_elements = seq_len * HIDDEN_SIZE
|
||||
tensor_bytes = tensor_elements * BENCHMARK_DTYPE.itemsize
|
||||
|
||||
# Format tensor size (MB)
|
||||
tensor_size_mb = tensor_bytes / (1024 * 1024)
|
||||
tensor_size_str = f"{tensor_size_mb:.2f} MB"
|
||||
|
||||
# Format tensor shape
|
||||
tensor_bytes = seq_len * HIDDEN_SIZE * BENCHMARK_DTYPE.itemsize
|
||||
tensor_size_str = f"{tensor_bytes / (1024 * 1024):.2f} MB"
|
||||
tensor_shape = f"({seq_len}, {HIDDEN_SIZE})"
|
||||
|
||||
row = f"{tensor_shape:<20}{tensor_size_str:<15}"
|
||||
for comm in all_comms:
|
||||
if comm in results[seq_len]:
|
||||
row += f"{results[seq_len][comm]:<20.3f}"
|
||||
for comm in comms:
|
||||
if comm in row_results:
|
||||
row += f"{row_results[comm]:<20.3f}"
|
||||
else:
|
||||
row += f"{'N/A':<20}"
|
||||
|
||||
# Calculate speedup information
|
||||
speedup_info = _calculate_speedup_info(results[seq_len])
|
||||
row += f"{speedup_info:<30}"
|
||||
|
||||
row += f"{_calculate_speedup_info(row_results, baseline):<30}"
|
||||
print(row)
|
||||
|
||||
print(f"{'=' * 130}")
|
||||
print("All times are in milliseconds (ms) per allreduce operation")
|
||||
print("Speedup column shows: fastest_algorithm (speedup_vs_pynccl)")
|
||||
print(f"\n{'=' * 130}")
|
||||
print("All times are in milliseconds (ms) per operation")
|
||||
|
||||
|
||||
def main():
|
||||
@@ -526,6 +661,13 @@ def main():
|
||||
num_warmup=args.num_warmup,
|
||||
num_trials=args.num_trials,
|
||||
)
|
||||
results.update(
|
||||
benchmark.benchmark_ag_rs(
|
||||
sequence_length=seq_len,
|
||||
num_warmup=args.num_warmup,
|
||||
num_trials=args.num_trials,
|
||||
)
|
||||
)
|
||||
|
||||
all_results[seq_len] = results
|
||||
|
||||
@@ -538,12 +680,23 @@ def main():
|
||||
|
||||
# Save to JSON if requested
|
||||
if args.output_json:
|
||||
# Add speedup information to results
|
||||
# Add per-collective speedup information to results.
|
||||
enhanced_results = {}
|
||||
for seq_len, comm_results in all_results.items():
|
||||
speedup_info = {}
|
||||
for collective in COLLECTIVE_ORDER:
|
||||
group = {
|
||||
k: v
|
||||
for k, v in comm_results.items()
|
||||
if COMM_COLLECTIVE.get(k) == collective
|
||||
}
|
||||
if group:
|
||||
speedup_info[collective] = _calculate_speedup_info(
|
||||
group, COLLECTIVE_BASELINE[collective]
|
||||
)
|
||||
enhanced_results[seq_len] = {
|
||||
"timings": comm_results,
|
||||
"speedup_info": _calculate_speedup_info(comm_results),
|
||||
"speedup_info": speedup_info,
|
||||
}
|
||||
|
||||
output_data = {
|
||||
|
||||
@@ -410,6 +410,21 @@ def test_flashinfer_all_reduce_precedes_nccl(monkeypatch: pytest.MonkeyPatch) ->
|
||||
nccl_selector.assert_not_called()
|
||||
|
||||
|
||||
def test_aiter_all_gather_precedes_pynccl(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Ensure can use aiter all_gather if available even if pynccl is not available."""
|
||||
output = torch.empty(2)
|
||||
aiter_comm = Mock()
|
||||
aiter_comm.should_custom_ag.return_value = True
|
||||
aiter_comm.custom_all_gather.return_value = output
|
||||
communicator = CudaCommunicator.__new__(CudaCommunicator)
|
||||
communicator.world_size = 2
|
||||
communicator.aiter_ar_comm = aiter_comm
|
||||
communicator.pynccl_comm = Mock(disabled=True)
|
||||
monkeypatch.setattr(communicator, "_can_use_aiter_ag_rs", Mock(return_value=True))
|
||||
|
||||
assert communicator.all_gatherv(torch.empty(1)) is output
|
||||
|
||||
|
||||
def test_isend_object_posts_size_then_object_and_releases_on_wait(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
||||
@@ -8,7 +8,7 @@ import torch.distributed as dist
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found, rocm_aiter_ops
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
|
||||
from vllm.distributed.parallel_state import get_tp_group, graph_capture
|
||||
from vllm.distributed.parallel_state import get_dp_group, get_tp_group, graph_capture
|
||||
from vllm.envs import disable_envs_cache
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -132,3 +132,173 @@ def test_rocm_aiter_custom_allreduce(
|
||||
test_target,
|
||||
):
|
||||
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
|
||||
|
||||
|
||||
def _get_aiter_ag_rs_comm():
|
||||
device_communicator = get_dp_group().device_communicator
|
||||
assert device_communicator.use_aiter_ag_rs, (
|
||||
"AITER custom AG/RS was not enabled on the DP group."
|
||||
)
|
||||
aiter_comm = device_communicator.aiter_ar_comm
|
||||
assert aiter_comm is not None, "AITER custom AG/RS was not initialized."
|
||||
assert not aiter_comm.disabled, "AITER custom AG/RS is disabled."
|
||||
return aiter_comm
|
||||
|
||||
|
||||
def _assert_aiter_handles_ag(aiter_comm, inp: torch.Tensor) -> None:
|
||||
assert aiter_comm.should_custom_ag(inp), (
|
||||
f"AITER custom all-gather does not support input shape {inp.shape}."
|
||||
)
|
||||
|
||||
|
||||
def _assert_aiter_handles_rs(aiter_comm, inp: torch.Tensor) -> None:
|
||||
assert aiter_comm.should_custom_rs(inp, dim=0), (
|
||||
f"AITER custom reduce-scatter does not support input shape {inp.shape}."
|
||||
)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def eager_ag_rs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
data_parallel_size,
|
||||
data_parallel_master_port,
|
||||
) -> None:
|
||||
with monkeypatch.context() as m:
|
||||
_configure_aiter_custom_ar_env(m)
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
data_parallel_size=data_parallel_size,
|
||||
data_parallel_master_port=data_parallel_master_port,
|
||||
)
|
||||
|
||||
dp_group = get_dp_group()
|
||||
group = dp_group.device_group
|
||||
dp_world = dp_group.world_size
|
||||
aiter_comm = _get_aiter_ag_rs_comm()
|
||||
|
||||
for shape, dtype in test_cases:
|
||||
num_tokens, hidden = shape
|
||||
|
||||
# all-gather: each rank contributes (num_tokens, hidden).
|
||||
inp = torch.ones(shape, dtype=dtype, device=device) * (rank + 1)
|
||||
_assert_aiter_handles_ag(aiter_comm, inp)
|
||||
expected = torch.empty(
|
||||
(num_tokens * dp_world, hidden), dtype=dtype, device=device
|
||||
)
|
||||
dist.all_gather_into_tensor(expected, inp, group=group)
|
||||
out = aiter_comm.custom_all_gather(inp, dim=0)
|
||||
assert out is not None
|
||||
torch.testing.assert_close(out, expected)
|
||||
|
||||
# reduce-scatter: each rank contributes (num_tokens * dp, hidden).
|
||||
rs_in = torch.ones(
|
||||
(num_tokens * dp_world, hidden), dtype=dtype, device=device
|
||||
) * (rank + 1)
|
||||
_assert_aiter_handles_rs(aiter_comm, rs_in)
|
||||
rs_expected = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
|
||||
dist.reduce_scatter_tensor(rs_expected, rs_in, group=group)
|
||||
rs_out = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
|
||||
aiter_comm.custom_reduce_scatter(rs_in, rs_out, dim=0)
|
||||
torch.testing.assert_close(rs_out, rs_expected)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def graph_ag_rs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
data_parallel_size,
|
||||
data_parallel_master_port,
|
||||
) -> None:
|
||||
with monkeypatch.context() as m:
|
||||
_configure_aiter_custom_ar_env(m)
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
data_parallel_size=data_parallel_size,
|
||||
data_parallel_master_port=data_parallel_master_port,
|
||||
)
|
||||
|
||||
dp_group = get_dp_group()
|
||||
group = dp_group.device_group
|
||||
dp_world = dp_group.world_size
|
||||
aiter_comm = _get_aiter_ag_rs_comm()
|
||||
|
||||
# Warmup so DP comms is initialized before graph capture
|
||||
data = torch.zeros(1, device=device)
|
||||
dist.all_reduce(data, group=group)
|
||||
torch.accelerator.synchronize()
|
||||
del data
|
||||
|
||||
for shape, dtype in test_cases:
|
||||
num_tokens, hidden = shape
|
||||
|
||||
# all-gather under graph capture.
|
||||
inp = torch.ones(shape, dtype=dtype, device=device) * (rank + 1)
|
||||
_assert_aiter_handles_ag(aiter_comm, inp)
|
||||
ag_expected = torch.empty(
|
||||
(num_tokens * dp_world, hidden), dtype=dtype, device=device
|
||||
)
|
||||
dist.all_gather_into_tensor(ag_expected, inp, group=group)
|
||||
with graph_capture(device=device) as graph_capture_context:
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
|
||||
ag_out = aiter_comm.custom_all_gather(inp, dim=0)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(ag_out, ag_expected)
|
||||
|
||||
# reduce-scatter under graph capture.
|
||||
rs_in = torch.ones(
|
||||
(num_tokens * dp_world, hidden), dtype=dtype, device=device
|
||||
) * (rank + 1)
|
||||
_assert_aiter_handles_rs(aiter_comm, rs_in)
|
||||
rs_expected = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
|
||||
dist.reduce_scatter_tensor(rs_expected, rs_in, group=group)
|
||||
rs_out = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
|
||||
with graph_capture(device=device) as graph_capture_context:
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
|
||||
aiter_comm.custom_reduce_scatter(rs_in, rs_out, dim=0)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(rs_out, rs_expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_aiter_found(), reason="AITER is not installed")
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("tp_size", [1])
|
||||
@pytest.mark.parametrize("pipeline_parallel_size", [1])
|
||||
@pytest.mark.parametrize("data_parallel_size", [2])
|
||||
@pytest.mark.parametrize("test_target", [eager_ag_rs, graph_ag_rs])
|
||||
def test_rocm_aiter_custom_ag_rs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pipeline_parallel_size,
|
||||
data_parallel_size,
|
||||
test_target,
|
||||
):
|
||||
multi_process_parallel(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
pipeline_parallel_size,
|
||||
test_target,
|
||||
data_parallel_size=data_parallel_size,
|
||||
)
|
||||
|
||||
+52
-10
@@ -1416,6 +1416,8 @@ def init_test_distributed_environment(
|
||||
rank: int,
|
||||
distributed_init_port: str,
|
||||
local_rank: int = -1,
|
||||
data_parallel_size: int = 1,
|
||||
data_parallel_master_port: int | None = None,
|
||||
) -> None:
|
||||
# Note: This function is often called from Ray worker processes, so we
|
||||
# can't rely on pytest fixtures to set the config. We check if the config
|
||||
@@ -1428,6 +1430,29 @@ def init_test_distributed_environment(
|
||||
|
||||
distributed_init_method = f"tcp://localhost:{distributed_init_port}"
|
||||
|
||||
if data_parallel_size > 1:
|
||||
# For DP we need to set a common DP master port
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
|
||||
assert data_parallel_master_port is not None, (
|
||||
"data_parallel_master_port is required when data_parallel_size > 1"
|
||||
)
|
||||
tp_pp_world = tp_size * pp_size
|
||||
parallel_config = ParallelConfig(
|
||||
data_parallel_size=data_parallel_size,
|
||||
data_parallel_rank=rank // tp_pp_world,
|
||||
_data_parallel_master_port_list=[int(data_parallel_master_port)],
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig(parallel_config=parallel_config)):
|
||||
init_distributed_environment(
|
||||
world_size=tp_pp_world,
|
||||
rank=rank % tp_pp_world,
|
||||
distributed_init_method=distributed_init_method,
|
||||
local_rank=local_rank if local_rank >= 0 else rank,
|
||||
)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
return
|
||||
|
||||
if get_current_vllm_config_or_none() is not None:
|
||||
# Config already set, use it directly
|
||||
init_distributed_environment(
|
||||
@@ -1454,6 +1479,7 @@ def multi_process_parallel(
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
test_target: Any,
|
||||
data_parallel_size: int = 1,
|
||||
) -> None:
|
||||
import ray
|
||||
|
||||
@@ -1474,18 +1500,34 @@ def multi_process_parallel(
|
||||
)
|
||||
|
||||
distributed_init_port = get_open_port()
|
||||
# Separate port for the DP master group; only used when data_parallel_size > 1.
|
||||
data_parallel_master_port = get_open_port() if data_parallel_size > 1 else None
|
||||
world_size = data_parallel_size * tp_size * pp_size
|
||||
try:
|
||||
refs = []
|
||||
for rank in range(tp_size * pp_size):
|
||||
refs.append(
|
||||
test_target.remote(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
),
|
||||
)
|
||||
for rank in range(world_size):
|
||||
if data_parallel_size > 1:
|
||||
refs.append(
|
||||
test_target.remote(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
data_parallel_size,
|
||||
data_parallel_master_port,
|
||||
),
|
||||
)
|
||||
else:
|
||||
refs.append(
|
||||
test_target.remote(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
),
|
||||
)
|
||||
ray.get(refs)
|
||||
finally:
|
||||
ray.shutdown()
|
||||
|
||||
@@ -56,6 +56,20 @@ class AiterCustomAllreduce:
|
||||
def custom_all_reduce(self, inp: torch.Tensor) -> torch.Tensor | None:
|
||||
return self._impl.custom_all_reduce(inp)
|
||||
|
||||
def should_custom_ag(self, inp: torch.Tensor) -> bool:
|
||||
return self._impl.should_custom_ag(inp)
|
||||
|
||||
def should_custom_rs(self, inp: torch.Tensor, dim: int) -> bool:
|
||||
return self._impl.should_custom_rs(inp, dim)
|
||||
|
||||
def custom_all_gather(self, inp: torch.Tensor, dim: int = 0) -> torch.Tensor | None:
|
||||
return self._impl.custom_all_gather(inp, dim=dim)
|
||||
|
||||
def custom_reduce_scatter(
|
||||
self, inp: torch.Tensor, out: torch.Tensor, dim: int = 0
|
||||
) -> torch.Tensor | None:
|
||||
return self._impl.custom_reduce_scatter(inp, out, dim=dim)
|
||||
|
||||
def use_1stage_fused_ar_rms(self, inp: torch.Tensor) -> bool:
|
||||
"""Whether AITER's fused allreduce+RMSNorm runs as its one-stage kernel.
|
||||
|
||||
|
||||
@@ -109,6 +109,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.fi_ar_comm: FlashInferAllReduce | None = None
|
||||
self.fi_pcie_ipc_ar_comm: FlashInferPcieIpcAllReduce | None = None
|
||||
self.aiter_ar_comm: AiterCustomAllreduce | None = None
|
||||
self.use_aiter_ag_rs: bool = False
|
||||
|
||||
if use_torch_symm_mem and current_platform.is_cuda():
|
||||
self.symm_mem_comm = SymmMemCommunicator(
|
||||
@@ -149,6 +150,22 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
),
|
||||
)
|
||||
|
||||
# AITER custom all-gather/reduce-scatter DP-attention dispatch/combine
|
||||
if (
|
||||
"dp" in unique_name
|
||||
and self.world_size in (2, 4, 8)
|
||||
and current_platform.is_rocm()
|
||||
and rocm_aiter_ops.is_custom_all_reduce_enabled()
|
||||
):
|
||||
self.aiter_ar_comm = AiterCustomAllreduce(
|
||||
group=self.cpu_group,
|
||||
device=self.device,
|
||||
)
|
||||
if self.aiter_ar_comm.disabled:
|
||||
self.aiter_ar_comm = None
|
||||
else:
|
||||
self.use_aiter_ag_rs = True
|
||||
|
||||
if use_custom_allreduce and self.world_size > 1 and current_platform.is_rocm():
|
||||
# Initialize a custom quick all-reduce implementation for AMD.
|
||||
# Quick reduce is designed as a complement to custom allreduce
|
||||
@@ -285,7 +302,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
enabled_ar_backends.append("NCCL_SYMM_MEM")
|
||||
if self.qr_comm is not None and not self.qr_comm.disabled:
|
||||
enabled_ar_backends.append("QUICK_REDUCE")
|
||||
if self.aiter_ar_comm is not None and not self.aiter_ar_comm.disabled:
|
||||
if (
|
||||
self.use_aiter_allreduce
|
||||
and self.aiter_ar_comm is not None
|
||||
and not self.aiter_ar_comm.disabled
|
||||
):
|
||||
enabled_ar_backends.append("AITER_CUSTOM")
|
||||
if self.ca_comm is not None and not self.ca_comm.disabled:
|
||||
enabled_ar_backends.append("CUSTOM")
|
||||
@@ -340,7 +361,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
return out
|
||||
aiter_ar_comm = self.aiter_ar_comm
|
||||
if (
|
||||
aiter_ar_comm is not None
|
||||
self.use_aiter_allreduce
|
||||
and aiter_ar_comm is not None
|
||||
and not aiter_ar_comm.disabled
|
||||
and aiter_ar_comm.should_custom_ar(input_)
|
||||
):
|
||||
@@ -462,6 +484,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
# Convert negative dim to positive.
|
||||
dim += input_.dim()
|
||||
|
||||
# 'sizes' is not needed if all inputs in the same group have the same
|
||||
# shape
|
||||
if sizes is not None and all(s == sizes[0] for s in sizes):
|
||||
sizes = None
|
||||
|
||||
# Note: This will produce an incorrect answer if we don't make
|
||||
# the input_tensor contiguous. Possible bug in reduce_scatter_tensor?
|
||||
input_tensor = input_.movedim(0, dim).contiguous()
|
||||
@@ -475,6 +502,16 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
chunk_size = input_tensor.shape[0] // world_size
|
||||
output_shape = (chunk_size,) + input_tensor.shape[1:]
|
||||
|
||||
if self._can_use_aiter_ag_rs(sizes):
|
||||
aiter_comm = self.aiter_ar_comm
|
||||
assert aiter_comm is not None
|
||||
if aiter_comm.should_custom_rs(input_tensor, dim=0):
|
||||
output = torch.empty(
|
||||
output_shape, dtype=input_tensor.dtype, device=input_tensor.device
|
||||
)
|
||||
aiter_comm.custom_reduce_scatter(input_tensor, output, dim=0)
|
||||
return output.movedim(0, dim).contiguous()
|
||||
|
||||
# Symmetric memory is only used when all ranks have uniform sizes.
|
||||
# ncclCommWindowRegister is collective: asymmetric pool allocations
|
||||
# from variable per-rank sizes cause deadlocks.
|
||||
@@ -632,6 +669,34 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.all2all_manager.destroy()
|
||||
self.all2all_manager = None # type: ignore[assignment]
|
||||
|
||||
def _can_use_aiter_ag_rs(self, sizes: list[int] | None) -> bool:
|
||||
"""Whether the AITER custom AG/RS fast path may run for this collective.
|
||||
|
||||
Requires:
|
||||
- uniform batches
|
||||
- FULL CUDAgraphs
|
||||
"""
|
||||
if (
|
||||
not self.use_aiter_ag_rs
|
||||
or self.aiter_ar_comm is None
|
||||
or self.aiter_ar_comm.disabled
|
||||
):
|
||||
return False
|
||||
if sizes is not None:
|
||||
return False
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.forward_context import get_forward_context
|
||||
|
||||
try:
|
||||
ctx = get_forward_context()
|
||||
except AssertionError:
|
||||
return False
|
||||
if ctx.cudagraph_runtime_mode != CUDAGraphMode.FULL:
|
||||
return False
|
||||
bd = ctx.batch_descriptor
|
||||
return bd is not None and bd.uniform
|
||||
|
||||
def suspend(self) -> None:
|
||||
if self.pynccl_comm is not None:
|
||||
self.pynccl_comm.suspend()
|
||||
@@ -665,14 +730,28 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
if dim != 0:
|
||||
raise NotImplementedError("only dim 0 all-gatherv is supported")
|
||||
world_size = self.world_size
|
||||
pynccl_comm = self.pynccl_comm
|
||||
assert pynccl_comm is not None and not pynccl_comm.disabled
|
||||
|
||||
# 'sizes' is not needed if all inputs in the same group have the same
|
||||
# shape
|
||||
if sizes is not None and all(s == sizes[0] for s in sizes):
|
||||
sizes = None
|
||||
|
||||
if self._can_use_aiter_ag_rs(sizes):
|
||||
aiter_comm = self.aiter_ar_comm
|
||||
assert aiter_comm is not None
|
||||
if isinstance(input_, torch.Tensor):
|
||||
if aiter_comm.should_custom_ag(input_):
|
||||
out = aiter_comm.custom_all_gather(input_, dim=0)
|
||||
if out is not None:
|
||||
return out
|
||||
elif all(aiter_comm.should_custom_ag(inp) for inp in input_):
|
||||
outs = [aiter_comm.custom_all_gather(inp, dim=0) for inp in input_]
|
||||
if all(o is not None for o in outs):
|
||||
return outs
|
||||
|
||||
pynccl_comm = self.pynccl_comm
|
||||
assert pynccl_comm is not None and not pynccl_comm.disabled
|
||||
|
||||
# Symmetric memory is only used when all ranks have uniform sizes.
|
||||
# ncclCommWindowRegister is collective: asymmetric pool allocations
|
||||
# from variable per-rank sizes cause deadlocks.
|
||||
|
||||
@@ -673,11 +673,11 @@ class GroupCoordinator:
|
||||
else:
|
||||
stream = graph_capture_context.stream
|
||||
|
||||
# only cuda uses this function,
|
||||
# only cuda/rocm uses this function,
|
||||
# so we don't abstract it into the base class
|
||||
maybe_ca_context = nullcontext()
|
||||
maybe_fi_pcie_ipc_context: AbstractContextManager[Any] = nullcontext()
|
||||
maybe_aiter_context = nullcontext()
|
||||
maybe_aiter_ar_context = nullcontext()
|
||||
from vllm.distributed.device_communicators.cuda_communicator import (
|
||||
CudaCommunicator,
|
||||
)
|
||||
@@ -698,12 +698,10 @@ class GroupCoordinator:
|
||||
if fi_pcie_ipc_ar_comm is not None:
|
||||
maybe_fi_pcie_ipc_context = fi_pcie_ipc_ar_comm.capture()
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
if aiter_ar is not None:
|
||||
maybe_aiter_context = aiter_ar.capture() # type: ignore
|
||||
# Capture each group's own comm. A global lookup would double-capture
|
||||
aiter_ar_comm = getattr(self.device_communicator, "aiter_ar_comm", None)
|
||||
if aiter_ar_comm is not None:
|
||||
maybe_aiter_ar_context = aiter_ar_comm.capture() # type: ignore
|
||||
|
||||
# ensure all initialization operations complete before attempting to
|
||||
# capture the graph on another stream
|
||||
@@ -715,7 +713,7 @@ class GroupCoordinator:
|
||||
torch.cuda.stream(stream),
|
||||
maybe_ca_context,
|
||||
maybe_fi_pcie_ipc_context,
|
||||
maybe_aiter_context,
|
||||
maybe_aiter_ar_context,
|
||||
):
|
||||
yield graph_capture_context
|
||||
|
||||
@@ -1640,7 +1638,11 @@ def graph_capture(
|
||||
context = graph_capture_context or GraphCaptureContext(
|
||||
torch.cuda.Stream(device=device)
|
||||
)
|
||||
with get_tp_group().graph_capture(context), get_pp_group().graph_capture(context):
|
||||
with (
|
||||
get_tp_group().graph_capture(context),
|
||||
get_pp_group().graph_capture(context),
|
||||
get_dp_group().graph_capture(context),
|
||||
):
|
||||
yield context
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -1261,7 +1261,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
os.getenv("VLLM_ROCM_USE_AITER", "False").lower() in ("true", "1")
|
||||
),
|
||||
# Use AITER's CustomAllreduce as the custom-allreduce backend inside vLLM's
|
||||
# CudaCommunicator on ROCm.
|
||||
# CudaCommunicator on ROCm. Also enables AITER AG/RS for DP communication.
|
||||
"VLLM_ROCM_USE_AITER_CUSTOM_AR": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "True").lower() in ("true", "1")
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user