[Perf][ROCm] Add AITER custom AG/RS (DP only) (#48247)

Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
This commit is contained in:
Simon Danielsson
2026-09-11 13:19:57 +08:00
committed by GitHub
parent 980c16c8e4
commit ae48466cf3
8 changed files with 549 additions and 74 deletions
@@ -29,6 +29,10 @@ import torch
import torch.distributed as dist
from torch.distributed import ProcessGroup
from vllm._aiter_ops import rocm_aiter_ops
from vllm.distributed.device_communicators.aiter_custom_all_reduce import (
AiterCustomAllreduce,
)
from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce
from vllm.distributed.device_communicators.flashinfer_all_reduce import (
FlashInferAllReduce,
@@ -42,6 +46,7 @@ from vllm.distributed.device_communicators.pynccl_allocator import (
)
from vllm.distributed.device_communicators.symm_mem import SymmMemCommunicator
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.argparse_utils import FlexibleArgumentParser
logger = init_logger(__name__)
@@ -56,6 +61,30 @@ BENCHMARK_DTYPE = torch.bfloat16
# CUDA graph settings
CUDA_GRAPH_CAPTURE_CYCLES = 10
# All-gather and reduce-scatter are distinct collectives from all-reduce, so
# results are compared within a collective (each against its own NCCL/RCCL
# baseline) rather than across collectives.
COMM_COLLECTIVE = {
"ca_1stage": "all-reduce",
"ca_2stage": "all-reduce",
"pynccl": "all-reduce",
"pynccl-symm": "all-reduce",
"symm_mem_multimem": "all-reduce",
"symm_mem_two_shot": "all-reduce",
"flashinfer_trtllm": "all-reduce",
"flashinfer_mnnvl": "all-reduce",
"aiter_ag": "all-gather",
"pynccl_ag": "all-gather",
"aiter_rs": "reduce-scatter",
"pynccl_rs": "reduce-scatter",
}
COLLECTIVE_ORDER = ["all-reduce", "all-gather", "reduce-scatter"]
COLLECTIVE_BASELINE = {
"all-reduce": "pynccl",
"all-gather": "pynccl_ag",
"reduce-scatter": "pynccl_rs",
}
class CommunicatorBenchmark:
"""Benchmark class for testing device communicators."""
@@ -77,6 +106,11 @@ class CommunicatorBenchmark:
max_seq_len = max(sequence_lengths)
max_tensor_elements = max_seq_len * HIDDEN_SIZE
self.max_size_override = max_tensor_elements * BENCHMARK_DTYPE.itemsize + 1
# AITER has effective max size of half the max size, so we double it, and
# account for AG output size (which is world_size larger)
self.aiter_max_size = (
2 * world_size * max_tensor_elements * BENCHMARK_DTYPE.itemsize + 1
)
# Initialize communicators
self.custom_allreduce = None
@@ -85,6 +119,7 @@ class CommunicatorBenchmark:
self.symm_mem_comm_multimem = None
self.symm_mem_comm_two_shot = None
self.fi_ar_comm = None
self.aiter_comm = None
self._init_communicators()
@@ -181,6 +216,27 @@ class CommunicatorBenchmark:
)
self.fi_ar_comm = None
if current_platform.is_rocm() and rocm_aiter_ops.is_custom_all_reduce_enabled():
try:
# Also used for AG/RS for DP
self.aiter_comm = AiterCustomAllreduce(
group=self.cpu_group,
device=self.device,
max_size=self.aiter_max_size,
)
if not self.aiter_comm.disabled:
logger.info("Rank %s: AITER custom AG/RS initialized", self.rank)
else:
logger.info("Rank %s: AITER custom AG/RS disabled", self.rank)
self.aiter_comm = None
except Exception as e:
logger.warning(
"Rank %s: Failed to initialize AITER custom AG/RS: %s",
self.rank,
e,
)
self.aiter_comm = None
def benchmark_allreduce(
self, sequence_length: int, num_warmup: int, num_trials: int
) -> dict[str, float]:
@@ -385,29 +441,116 @@ class CommunicatorBenchmark:
f"CUDA graph benchmark failed for communicator: {e}"
) from e
def benchmark_ag_rs(
self, sequence_length: int, num_warmup: int, num_trials: int
) -> dict[str, float]:
"""Benchmark all-gather and reduce-scatter: custom (AITER) vs NCCL/RCCL.
def _calculate_speedup_info(comm_results: dict[str, float]) -> str:
"""Calculate speedup information for a single tensor size."""
Each collective is measured for both the AITER custom kernel and the
PyNccl baseline so they compare like-for-like (print_results groups
columns by collective). All-gather maps ``(seq, hidden)`` ->
``(world_size * seq, hidden)``; reduce-scatter maps ``(seq, hidden)`` ->
``(seq // world_size, hidden)`` and so only runs when ``seq`` divides
evenly across ranks. Both reuse ``benchmark_allreduce_single`` timed on a
``(seq, hidden)`` input; baselines write into a pre-allocated buffer.
"""
results: dict[str, float] = {}
entries = []
# ---- all-gather: (seq, hidden) -> (world_size * seq, hidden) ----
if self.aiter_comm is not None:
c = self.aiter_comm
entries.append(
(
"aiter_ag",
lambda t, c=c: c.custom_all_gather(t, dim=0),
lambda t, c=c: c.should_custom_ag(t),
c.capture(),
)
)
if self.pynccl_comm is not None:
c = self.pynccl_comm
ag_out = torch.empty(
sequence_length * self.world_size,
HIDDEN_SIZE,
dtype=BENCHMARK_DTYPE,
device=self.device,
)
entries.append(
(
"pynccl_ag",
lambda t, c=c, o=ag_out: c.all_gather(o, t),
lambda t: True,
nullcontext(),
)
)
# ---- reduce-scatter: (seq, hidden) -> (seq // world_size, hidden) ----
if sequence_length % self.world_size == 0:
rs_shape = (sequence_length // self.world_size, HIDDEN_SIZE)
if self.aiter_comm is not None:
c = self.aiter_comm
rs_out = torch.empty(
rs_shape, dtype=BENCHMARK_DTYPE, device=self.device
)
entries.append(
(
"aiter_rs",
lambda t, c=c, o=rs_out: c.custom_reduce_scatter(t, o, dim=0),
lambda t, c=c: c.should_custom_rs(t, dim=0),
c.capture(),
)
)
if self.pynccl_comm is not None:
c = self.pynccl_comm
rs_out = torch.empty(
rs_shape, dtype=BENCHMARK_DTYPE, device=self.device
)
entries.append(
(
"pynccl_rs",
lambda t, c=c, o=rs_out: c.reduce_scatter(o, t),
lambda t: True,
nullcontext(),
)
)
for name, fn, should_use_fn, context in entries:
latency = self.benchmark_allreduce_single(
sequence_length, fn, should_use_fn, context, num_warmup, num_trials
)
if latency is not None:
results[name] = latency
return results
def _calculate_speedup_info(comm_results: dict[str, float], baseline: str) -> str:
"""Fastest comm in a collective group and its speedup vs ``baseline``."""
if not comm_results:
return "N/A"
# Find the fastest communicator
fastest_comm = min(comm_results.keys(), key=lambda k: comm_results[k])
fastest_time = comm_results[fastest_comm]
# Calculate speedup vs PyNccl if available
if "pynccl" in comm_results:
pynccl_time = comm_results["pynccl"]
speedup = pynccl_time / fastest_time
if baseline in comm_results and fastest_time > 0:
speedup = comm_results[baseline] / fastest_time
return f"{fastest_comm} ({speedup:.2f}x)"
else:
return f"{fastest_comm} (N/A)"
return f"{fastest_comm} (N/A)"
def print_results(
results: dict[str, dict[str, float]], sequence_lengths: list[int], world_size: int
results: dict[int, dict[str, float]], sequence_lengths: list[int], world_size: int
):
"""Print benchmark results in a formatted table."""
"""Print benchmark results as one table per collective.
All-reduce, all-gather and reduce-scatter are different collectives, so each
gets its own table and its speedup is computed against that collective's
NCCL/RCCL baseline rather than across collectives.
"""
present: set[str] = set()
for size_results in results.values():
present.update(size_results.keys())
print(f"\n{'=' * 130}")
print("Device Communicator Benchmark Results")
@@ -415,53 +558,45 @@ def print_results(
f"World Size: {world_size}, Data Type: {BENCHMARK_DTYPE}, "
f"Hidden Size: {HIDDEN_SIZE}"
)
print("Tensor Shape is the per-rank input; times are ms per operation.")
print(f"{'=' * 130}")
# Get all communicator names
all_comms = set()
for size_results in results.values():
all_comms.update(size_results.keys())
for collective in COLLECTIVE_ORDER:
comms = sorted(c for c in present if COMM_COLLECTIVE.get(c) == collective)
if not comms:
continue
baseline = COLLECTIVE_BASELINE[collective]
all_comms = sorted(list(all_comms))
print(f"\n{collective} (speedup vs {baseline})")
header = f"{'Tensor Shape':<20}{'Tensor Size':<15}"
for comm in comms:
header += f"{comm:<20}"
header += f"{'Best':<30}"
print(header)
print("-" * len(header))
# Print header
header = f"{'Tensor Shape':<20}{'Tensor Size':<15}"
for comm in all_comms:
header += f"{comm:<20}"
header += f"{'Best (Speedup vs PyNccl)':<30}"
print(header)
print("-" * len(header))
for seq_len in sequence_lengths:
if seq_len not in results:
continue
row_results = {
c: results[seq_len][c] for c in comms if c in results[seq_len]
}
# Print results for each sequence length
for seq_len in sequence_lengths:
if seq_len in results:
# Calculate tensor size in elements and bytes
tensor_elements = seq_len * HIDDEN_SIZE
tensor_bytes = tensor_elements * BENCHMARK_DTYPE.itemsize
# Format tensor size (MB)
tensor_size_mb = tensor_bytes / (1024 * 1024)
tensor_size_str = f"{tensor_size_mb:.2f} MB"
# Format tensor shape
tensor_bytes = seq_len * HIDDEN_SIZE * BENCHMARK_DTYPE.itemsize
tensor_size_str = f"{tensor_bytes / (1024 * 1024):.2f} MB"
tensor_shape = f"({seq_len}, {HIDDEN_SIZE})"
row = f"{tensor_shape:<20}{tensor_size_str:<15}"
for comm in all_comms:
if comm in results[seq_len]:
row += f"{results[seq_len][comm]:<20.3f}"
for comm in comms:
if comm in row_results:
row += f"{row_results[comm]:<20.3f}"
else:
row += f"{'N/A':<20}"
# Calculate speedup information
speedup_info = _calculate_speedup_info(results[seq_len])
row += f"{speedup_info:<30}"
row += f"{_calculate_speedup_info(row_results, baseline):<30}"
print(row)
print(f"{'=' * 130}")
print("All times are in milliseconds (ms) per allreduce operation")
print("Speedup column shows: fastest_algorithm (speedup_vs_pynccl)")
print(f"\n{'=' * 130}")
print("All times are in milliseconds (ms) per operation")
def main():
@@ -526,6 +661,13 @@ def main():
num_warmup=args.num_warmup,
num_trials=args.num_trials,
)
results.update(
benchmark.benchmark_ag_rs(
sequence_length=seq_len,
num_warmup=args.num_warmup,
num_trials=args.num_trials,
)
)
all_results[seq_len] = results
@@ -538,12 +680,23 @@ def main():
# Save to JSON if requested
if args.output_json:
# Add speedup information to results
# Add per-collective speedup information to results.
enhanced_results = {}
for seq_len, comm_results in all_results.items():
speedup_info = {}
for collective in COLLECTIVE_ORDER:
group = {
k: v
for k, v in comm_results.items()
if COMM_COLLECTIVE.get(k) == collective
}
if group:
speedup_info[collective] = _calculate_speedup_info(
group, COLLECTIVE_BASELINE[collective]
)
enhanced_results[seq_len] = {
"timings": comm_results,
"speedup_info": _calculate_speedup_info(comm_results),
"speedup_info": speedup_info,
}
output_data = {
+15
View File
@@ -410,6 +410,21 @@ def test_flashinfer_all_reduce_precedes_nccl(monkeypatch: pytest.MonkeyPatch) ->
nccl_selector.assert_not_called()
def test_aiter_all_gather_precedes_pynccl(monkeypatch: pytest.MonkeyPatch) -> None:
"""Ensure can use aiter all_gather if available even if pynccl is not available."""
output = torch.empty(2)
aiter_comm = Mock()
aiter_comm.should_custom_ag.return_value = True
aiter_comm.custom_all_gather.return_value = output
communicator = CudaCommunicator.__new__(CudaCommunicator)
communicator.world_size = 2
communicator.aiter_ar_comm = aiter_comm
communicator.pynccl_comm = Mock(disabled=True)
monkeypatch.setattr(communicator, "_can_use_aiter_ag_rs", Mock(return_value=True))
assert communicator.all_gatherv(torch.empty(1)) is output
def test_isend_object_posts_size_then_object_and_releases_on_wait(
monkeypatch: pytest.MonkeyPatch,
) -> None:
+171 -1
View File
@@ -8,7 +8,7 @@ import torch.distributed as dist
from vllm._aiter_ops import is_aiter_found, rocm_aiter_ops
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
from vllm.distributed.parallel_state import get_tp_group, graph_capture
from vllm.distributed.parallel_state import get_dp_group, get_tp_group, graph_capture
from vllm.envs import disable_envs_cache
from vllm.platforms import current_platform
@@ -132,3 +132,173 @@ def test_rocm_aiter_custom_allreduce(
test_target,
):
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
def _get_aiter_ag_rs_comm():
device_communicator = get_dp_group().device_communicator
assert device_communicator.use_aiter_ag_rs, (
"AITER custom AG/RS was not enabled on the DP group."
)
aiter_comm = device_communicator.aiter_ar_comm
assert aiter_comm is not None, "AITER custom AG/RS was not initialized."
assert not aiter_comm.disabled, "AITER custom AG/RS is disabled."
return aiter_comm
def _assert_aiter_handles_ag(aiter_comm, inp: torch.Tensor) -> None:
assert aiter_comm.should_custom_ag(inp), (
f"AITER custom all-gather does not support input shape {inp.shape}."
)
def _assert_aiter_handles_rs(aiter_comm, inp: torch.Tensor) -> None:
assert aiter_comm.should_custom_rs(inp, dim=0), (
f"AITER custom reduce-scatter does not support input shape {inp.shape}."
)
@ray.remote(num_gpus=1, max_calls=1)
def eager_ag_rs(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
data_parallel_size,
data_parallel_master_port,
) -> None:
with monkeypatch.context() as m:
_configure_aiter_custom_ar_env(m)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(
tp_size,
pp_size,
rank,
distributed_init_port,
data_parallel_size=data_parallel_size,
data_parallel_master_port=data_parallel_master_port,
)
dp_group = get_dp_group()
group = dp_group.device_group
dp_world = dp_group.world_size
aiter_comm = _get_aiter_ag_rs_comm()
for shape, dtype in test_cases:
num_tokens, hidden = shape
# all-gather: each rank contributes (num_tokens, hidden).
inp = torch.ones(shape, dtype=dtype, device=device) * (rank + 1)
_assert_aiter_handles_ag(aiter_comm, inp)
expected = torch.empty(
(num_tokens * dp_world, hidden), dtype=dtype, device=device
)
dist.all_gather_into_tensor(expected, inp, group=group)
out = aiter_comm.custom_all_gather(inp, dim=0)
assert out is not None
torch.testing.assert_close(out, expected)
# reduce-scatter: each rank contributes (num_tokens * dp, hidden).
rs_in = torch.ones(
(num_tokens * dp_world, hidden), dtype=dtype, device=device
) * (rank + 1)
_assert_aiter_handles_rs(aiter_comm, rs_in)
rs_expected = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
dist.reduce_scatter_tensor(rs_expected, rs_in, group=group)
rs_out = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
aiter_comm.custom_reduce_scatter(rs_in, rs_out, dim=0)
torch.testing.assert_close(rs_out, rs_expected)
@ray.remote(num_gpus=1, max_calls=1)
def graph_ag_rs(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
data_parallel_size,
data_parallel_master_port,
) -> None:
with monkeypatch.context() as m:
_configure_aiter_custom_ar_env(m)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(
tp_size,
pp_size,
rank,
distributed_init_port,
data_parallel_size=data_parallel_size,
data_parallel_master_port=data_parallel_master_port,
)
dp_group = get_dp_group()
group = dp_group.device_group
dp_world = dp_group.world_size
aiter_comm = _get_aiter_ag_rs_comm()
# Warmup so DP comms is initialized before graph capture
data = torch.zeros(1, device=device)
dist.all_reduce(data, group=group)
torch.accelerator.synchronize()
del data
for shape, dtype in test_cases:
num_tokens, hidden = shape
# all-gather under graph capture.
inp = torch.ones(shape, dtype=dtype, device=device) * (rank + 1)
_assert_aiter_handles_ag(aiter_comm, inp)
ag_expected = torch.empty(
(num_tokens * dp_world, hidden), dtype=dtype, device=device
)
dist.all_gather_into_tensor(ag_expected, inp, group=group)
with graph_capture(device=device) as graph_capture_context:
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
ag_out = aiter_comm.custom_all_gather(inp, dim=0)
graph.replay()
torch.testing.assert_close(ag_out, ag_expected)
# reduce-scatter under graph capture.
rs_in = torch.ones(
(num_tokens * dp_world, hidden), dtype=dtype, device=device
) * (rank + 1)
_assert_aiter_handles_rs(aiter_comm, rs_in)
rs_expected = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
dist.reduce_scatter_tensor(rs_expected, rs_in, group=group)
rs_out = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
with graph_capture(device=device) as graph_capture_context:
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
aiter_comm.custom_reduce_scatter(rs_in, rs_out, dim=0)
graph.replay()
torch.testing.assert_close(rs_out, rs_expected)
@pytest.mark.skipif(not is_aiter_found(), reason="AITER is not installed")
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("tp_size", [1])
@pytest.mark.parametrize("pipeline_parallel_size", [1])
@pytest.mark.parametrize("data_parallel_size", [2])
@pytest.mark.parametrize("test_target", [eager_ag_rs, graph_ag_rs])
def test_rocm_aiter_custom_ag_rs(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pipeline_parallel_size,
data_parallel_size,
test_target,
):
multi_process_parallel(
monkeypatch,
tp_size,
pipeline_parallel_size,
test_target,
data_parallel_size=data_parallel_size,
)
+52 -10
View File
@@ -1416,6 +1416,8 @@ def init_test_distributed_environment(
rank: int,
distributed_init_port: str,
local_rank: int = -1,
data_parallel_size: int = 1,
data_parallel_master_port: int | None = None,
) -> None:
# Note: This function is often called from Ray worker processes, so we
# can't rely on pytest fixtures to set the config. We check if the config
@@ -1428,6 +1430,29 @@ def init_test_distributed_environment(
distributed_init_method = f"tcp://localhost:{distributed_init_port}"
if data_parallel_size > 1:
# For DP we need to set a common DP master port
from vllm.config.parallel import ParallelConfig
assert data_parallel_master_port is not None, (
"data_parallel_master_port is required when data_parallel_size > 1"
)
tp_pp_world = tp_size * pp_size
parallel_config = ParallelConfig(
data_parallel_size=data_parallel_size,
data_parallel_rank=rank // tp_pp_world,
_data_parallel_master_port_list=[int(data_parallel_master_port)],
)
with set_current_vllm_config(VllmConfig(parallel_config=parallel_config)):
init_distributed_environment(
world_size=tp_pp_world,
rank=rank % tp_pp_world,
distributed_init_method=distributed_init_method,
local_rank=local_rank if local_rank >= 0 else rank,
)
ensure_model_parallel_initialized(tp_size, pp_size)
return
if get_current_vllm_config_or_none() is not None:
# Config already set, use it directly
init_distributed_environment(
@@ -1454,6 +1479,7 @@ def multi_process_parallel(
tp_size: int,
pp_size: int,
test_target: Any,
data_parallel_size: int = 1,
) -> None:
import ray
@@ -1474,18 +1500,34 @@ def multi_process_parallel(
)
distributed_init_port = get_open_port()
# Separate port for the DP master group; only used when data_parallel_size > 1.
data_parallel_master_port = get_open_port() if data_parallel_size > 1 else None
world_size = data_parallel_size * tp_size * pp_size
try:
refs = []
for rank in range(tp_size * pp_size):
refs.append(
test_target.remote(
monkeypatch,
tp_size,
pp_size,
rank,
distributed_init_port,
),
)
for rank in range(world_size):
if data_parallel_size > 1:
refs.append(
test_target.remote(
monkeypatch,
tp_size,
pp_size,
rank,
distributed_init_port,
data_parallel_size,
data_parallel_master_port,
),
)
else:
refs.append(
test_target.remote(
monkeypatch,
tp_size,
pp_size,
rank,
distributed_init_port,
),
)
ray.get(refs)
finally:
ray.shutdown()
@@ -56,6 +56,20 @@ class AiterCustomAllreduce:
def custom_all_reduce(self, inp: torch.Tensor) -> torch.Tensor | None:
return self._impl.custom_all_reduce(inp)
def should_custom_ag(self, inp: torch.Tensor) -> bool:
return self._impl.should_custom_ag(inp)
def should_custom_rs(self, inp: torch.Tensor, dim: int) -> bool:
return self._impl.should_custom_rs(inp, dim)
def custom_all_gather(self, inp: torch.Tensor, dim: int = 0) -> torch.Tensor | None:
return self._impl.custom_all_gather(inp, dim=dim)
def custom_reduce_scatter(
self, inp: torch.Tensor, out: torch.Tensor, dim: int = 0
) -> torch.Tensor | None:
return self._impl.custom_reduce_scatter(inp, out, dim=dim)
def use_1stage_fused_ar_rms(self, inp: torch.Tensor) -> bool:
"""Whether AITER's fused allreduce+RMSNorm runs as its one-stage kernel.
@@ -109,6 +109,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.fi_ar_comm: FlashInferAllReduce | None = None
self.fi_pcie_ipc_ar_comm: FlashInferPcieIpcAllReduce | None = None
self.aiter_ar_comm: AiterCustomAllreduce | None = None
self.use_aiter_ag_rs: bool = False
if use_torch_symm_mem and current_platform.is_cuda():
self.symm_mem_comm = SymmMemCommunicator(
@@ -149,6 +150,22 @@ class CudaCommunicator(DeviceCommunicatorBase):
),
)
# AITER custom all-gather/reduce-scatter DP-attention dispatch/combine
if (
"dp" in unique_name
and self.world_size in (2, 4, 8)
and current_platform.is_rocm()
and rocm_aiter_ops.is_custom_all_reduce_enabled()
):
self.aiter_ar_comm = AiterCustomAllreduce(
group=self.cpu_group,
device=self.device,
)
if self.aiter_ar_comm.disabled:
self.aiter_ar_comm = None
else:
self.use_aiter_ag_rs = True
if use_custom_allreduce and self.world_size > 1 and current_platform.is_rocm():
# Initialize a custom quick all-reduce implementation for AMD.
# Quick reduce is designed as a complement to custom allreduce
@@ -285,7 +302,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
enabled_ar_backends.append("NCCL_SYMM_MEM")
if self.qr_comm is not None and not self.qr_comm.disabled:
enabled_ar_backends.append("QUICK_REDUCE")
if self.aiter_ar_comm is not None and not self.aiter_ar_comm.disabled:
if (
self.use_aiter_allreduce
and self.aiter_ar_comm is not None
and not self.aiter_ar_comm.disabled
):
enabled_ar_backends.append("AITER_CUSTOM")
if self.ca_comm is not None and not self.ca_comm.disabled:
enabled_ar_backends.append("CUSTOM")
@@ -340,7 +361,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
return out
aiter_ar_comm = self.aiter_ar_comm
if (
aiter_ar_comm is not None
self.use_aiter_allreduce
and aiter_ar_comm is not None
and not aiter_ar_comm.disabled
and aiter_ar_comm.should_custom_ar(input_)
):
@@ -462,6 +484,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
# Convert negative dim to positive.
dim += input_.dim()
# 'sizes' is not needed if all inputs in the same group have the same
# shape
if sizes is not None and all(s == sizes[0] for s in sizes):
sizes = None
# Note: This will produce an incorrect answer if we don't make
# the input_tensor contiguous. Possible bug in reduce_scatter_tensor?
input_tensor = input_.movedim(0, dim).contiguous()
@@ -475,6 +502,16 @@ class CudaCommunicator(DeviceCommunicatorBase):
chunk_size = input_tensor.shape[0] // world_size
output_shape = (chunk_size,) + input_tensor.shape[1:]
if self._can_use_aiter_ag_rs(sizes):
aiter_comm = self.aiter_ar_comm
assert aiter_comm is not None
if aiter_comm.should_custom_rs(input_tensor, dim=0):
output = torch.empty(
output_shape, dtype=input_tensor.dtype, device=input_tensor.device
)
aiter_comm.custom_reduce_scatter(input_tensor, output, dim=0)
return output.movedim(0, dim).contiguous()
# Symmetric memory is only used when all ranks have uniform sizes.
# ncclCommWindowRegister is collective: asymmetric pool allocations
# from variable per-rank sizes cause deadlocks.
@@ -632,6 +669,34 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.all2all_manager.destroy()
self.all2all_manager = None # type: ignore[assignment]
def _can_use_aiter_ag_rs(self, sizes: list[int] | None) -> bool:
"""Whether the AITER custom AG/RS fast path may run for this collective.
Requires:
- uniform batches
- FULL CUDAgraphs
"""
if (
not self.use_aiter_ag_rs
or self.aiter_ar_comm is None
or self.aiter_ar_comm.disabled
):
return False
if sizes is not None:
return False
from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import get_forward_context
try:
ctx = get_forward_context()
except AssertionError:
return False
if ctx.cudagraph_runtime_mode != CUDAGraphMode.FULL:
return False
bd = ctx.batch_descriptor
return bd is not None and bd.uniform
def suspend(self) -> None:
if self.pynccl_comm is not None:
self.pynccl_comm.suspend()
@@ -665,14 +730,28 @@ class CudaCommunicator(DeviceCommunicatorBase):
if dim != 0:
raise NotImplementedError("only dim 0 all-gatherv is supported")
world_size = self.world_size
pynccl_comm = self.pynccl_comm
assert pynccl_comm is not None and not pynccl_comm.disabled
# 'sizes' is not needed if all inputs in the same group have the same
# shape
if sizes is not None and all(s == sizes[0] for s in sizes):
sizes = None
if self._can_use_aiter_ag_rs(sizes):
aiter_comm = self.aiter_ar_comm
assert aiter_comm is not None
if isinstance(input_, torch.Tensor):
if aiter_comm.should_custom_ag(input_):
out = aiter_comm.custom_all_gather(input_, dim=0)
if out is not None:
return out
elif all(aiter_comm.should_custom_ag(inp) for inp in input_):
outs = [aiter_comm.custom_all_gather(inp, dim=0) for inp in input_]
if all(o is not None for o in outs):
return outs
pynccl_comm = self.pynccl_comm
assert pynccl_comm is not None and not pynccl_comm.disabled
# Symmetric memory is only used when all ranks have uniform sizes.
# ncclCommWindowRegister is collective: asymmetric pool allocations
# from variable per-rank sizes cause deadlocks.
+12 -10
View File
@@ -673,11 +673,11 @@ class GroupCoordinator:
else:
stream = graph_capture_context.stream
# only cuda uses this function,
# only cuda/rocm uses this function,
# so we don't abstract it into the base class
maybe_ca_context = nullcontext()
maybe_fi_pcie_ipc_context: AbstractContextManager[Any] = nullcontext()
maybe_aiter_context = nullcontext()
maybe_aiter_ar_context = nullcontext()
from vllm.distributed.device_communicators.cuda_communicator import (
CudaCommunicator,
)
@@ -698,12 +698,10 @@ class GroupCoordinator:
if fi_pcie_ipc_ar_comm is not None:
maybe_fi_pcie_ipc_context = fi_pcie_ipc_ar_comm.capture()
from vllm._aiter_ops import rocm_aiter_ops
if rocm_aiter_ops.is_enabled():
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
if aiter_ar is not None:
maybe_aiter_context = aiter_ar.capture() # type: ignore
# Capture each group's own comm. A global lookup would double-capture
aiter_ar_comm = getattr(self.device_communicator, "aiter_ar_comm", None)
if aiter_ar_comm is not None:
maybe_aiter_ar_context = aiter_ar_comm.capture() # type: ignore
# ensure all initialization operations complete before attempting to
# capture the graph on another stream
@@ -715,7 +713,7 @@ class GroupCoordinator:
torch.cuda.stream(stream),
maybe_ca_context,
maybe_fi_pcie_ipc_context,
maybe_aiter_context,
maybe_aiter_ar_context,
):
yield graph_capture_context
@@ -1640,7 +1638,11 @@ def graph_capture(
context = graph_capture_context or GraphCaptureContext(
torch.cuda.Stream(device=device)
)
with get_tp_group().graph_capture(context), get_pp_group().graph_capture(context):
with (
get_tp_group().graph_capture(context),
get_pp_group().graph_capture(context),
get_dp_group().graph_capture(context),
):
yield context
+1 -1
View File
@@ -1261,7 +1261,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
os.getenv("VLLM_ROCM_USE_AITER", "False").lower() in ("true", "1")
),
# Use AITER's CustomAllreduce as the custom-allreduce backend inside vLLM's
# CudaCommunicator on ROCm.
# CudaCommunicator on ROCm. Also enables AITER AG/RS for DP communication.
"VLLM_ROCM_USE_AITER_CUSTOM_AR": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "True").lower() in ("true", "1")
),