From ae48466cf33b4d7cfbe0f32370c425a3aa9fde61 Mon Sep 17 00:00:00 2001 From: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com> Date: Fri, 11 Sep 2026 07:19:57 +0200 Subject: [PATCH] [Perf][ROCm] Add AITER custom AG/RS (DP only) (#48247) Signed-off-by: simondanielsson --- .../kernels/benchmark_device_communicators.py | 249 ++++++++++++++---- tests/distributed/test_comm_ops.py | 15 ++ .../distributed/test_rocm_aiter_custom_ar.py | 172 +++++++++++- tests/utils.py | 62 ++++- .../aiter_custom_all_reduce.py | 14 + .../device_communicators/cuda_communicator.py | 87 +++++- vllm/distributed/parallel_state.py | 22 +- vllm/envs.py | 2 +- 8 files changed, 549 insertions(+), 74 deletions(-) diff --git a/benchmarks/kernels/benchmark_device_communicators.py b/benchmarks/kernels/benchmark_device_communicators.py index 24e22023b9..6c3a726a97 100644 --- a/benchmarks/kernels/benchmark_device_communicators.py +++ b/benchmarks/kernels/benchmark_device_communicators.py @@ -29,6 +29,10 @@ import torch import torch.distributed as dist from torch.distributed import ProcessGroup +from vllm._aiter_ops import rocm_aiter_ops +from vllm.distributed.device_communicators.aiter_custom_all_reduce import ( + AiterCustomAllreduce, +) from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce from vllm.distributed.device_communicators.flashinfer_all_reduce import ( FlashInferAllReduce, @@ -42,6 +46,7 @@ from vllm.distributed.device_communicators.pynccl_allocator import ( ) from vllm.distributed.device_communicators.symm_mem import SymmMemCommunicator from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm.utils.argparse_utils import FlexibleArgumentParser logger = init_logger(__name__) @@ -56,6 +61,30 @@ BENCHMARK_DTYPE = torch.bfloat16 # CUDA graph settings CUDA_GRAPH_CAPTURE_CYCLES = 10 +# All-gather and reduce-scatter are distinct collectives from all-reduce, so +# results are compared within a collective (each against its own NCCL/RCCL +# baseline) rather than across collectives. +COMM_COLLECTIVE = { + "ca_1stage": "all-reduce", + "ca_2stage": "all-reduce", + "pynccl": "all-reduce", + "pynccl-symm": "all-reduce", + "symm_mem_multimem": "all-reduce", + "symm_mem_two_shot": "all-reduce", + "flashinfer_trtllm": "all-reduce", + "flashinfer_mnnvl": "all-reduce", + "aiter_ag": "all-gather", + "pynccl_ag": "all-gather", + "aiter_rs": "reduce-scatter", + "pynccl_rs": "reduce-scatter", +} +COLLECTIVE_ORDER = ["all-reduce", "all-gather", "reduce-scatter"] +COLLECTIVE_BASELINE = { + "all-reduce": "pynccl", + "all-gather": "pynccl_ag", + "reduce-scatter": "pynccl_rs", +} + class CommunicatorBenchmark: """Benchmark class for testing device communicators.""" @@ -77,6 +106,11 @@ class CommunicatorBenchmark: max_seq_len = max(sequence_lengths) max_tensor_elements = max_seq_len * HIDDEN_SIZE self.max_size_override = max_tensor_elements * BENCHMARK_DTYPE.itemsize + 1 + # AITER has effective max size of half the max size, so we double it, and + # account for AG output size (which is world_size larger) + self.aiter_max_size = ( + 2 * world_size * max_tensor_elements * BENCHMARK_DTYPE.itemsize + 1 + ) # Initialize communicators self.custom_allreduce = None @@ -85,6 +119,7 @@ class CommunicatorBenchmark: self.symm_mem_comm_multimem = None self.symm_mem_comm_two_shot = None self.fi_ar_comm = None + self.aiter_comm = None self._init_communicators() @@ -181,6 +216,27 @@ class CommunicatorBenchmark: ) self.fi_ar_comm = None + if current_platform.is_rocm() and rocm_aiter_ops.is_custom_all_reduce_enabled(): + try: + # Also used for AG/RS for DP + self.aiter_comm = AiterCustomAllreduce( + group=self.cpu_group, + device=self.device, + max_size=self.aiter_max_size, + ) + if not self.aiter_comm.disabled: + logger.info("Rank %s: AITER custom AG/RS initialized", self.rank) + else: + logger.info("Rank %s: AITER custom AG/RS disabled", self.rank) + self.aiter_comm = None + except Exception as e: + logger.warning( + "Rank %s: Failed to initialize AITER custom AG/RS: %s", + self.rank, + e, + ) + self.aiter_comm = None + def benchmark_allreduce( self, sequence_length: int, num_warmup: int, num_trials: int ) -> dict[str, float]: @@ -385,29 +441,116 @@ class CommunicatorBenchmark: f"CUDA graph benchmark failed for communicator: {e}" ) from e + def benchmark_ag_rs( + self, sequence_length: int, num_warmup: int, num_trials: int + ) -> dict[str, float]: + """Benchmark all-gather and reduce-scatter: custom (AITER) vs NCCL/RCCL. -def _calculate_speedup_info(comm_results: dict[str, float]) -> str: - """Calculate speedup information for a single tensor size.""" + Each collective is measured for both the AITER custom kernel and the + PyNccl baseline so they compare like-for-like (print_results groups + columns by collective). All-gather maps ``(seq, hidden)`` -> + ``(world_size * seq, hidden)``; reduce-scatter maps ``(seq, hidden)`` -> + ``(seq // world_size, hidden)`` and so only runs when ``seq`` divides + evenly across ranks. Both reuse ``benchmark_allreduce_single`` timed on a + ``(seq, hidden)`` input; baselines write into a pre-allocated buffer. + """ + results: dict[str, float] = {} + entries = [] + + # ---- all-gather: (seq, hidden) -> (world_size * seq, hidden) ---- + if self.aiter_comm is not None: + c = self.aiter_comm + entries.append( + ( + "aiter_ag", + lambda t, c=c: c.custom_all_gather(t, dim=0), + lambda t, c=c: c.should_custom_ag(t), + c.capture(), + ) + ) + if self.pynccl_comm is not None: + c = self.pynccl_comm + ag_out = torch.empty( + sequence_length * self.world_size, + HIDDEN_SIZE, + dtype=BENCHMARK_DTYPE, + device=self.device, + ) + entries.append( + ( + "pynccl_ag", + lambda t, c=c, o=ag_out: c.all_gather(o, t), + lambda t: True, + nullcontext(), + ) + ) + + # ---- reduce-scatter: (seq, hidden) -> (seq // world_size, hidden) ---- + if sequence_length % self.world_size == 0: + rs_shape = (sequence_length // self.world_size, HIDDEN_SIZE) + if self.aiter_comm is not None: + c = self.aiter_comm + rs_out = torch.empty( + rs_shape, dtype=BENCHMARK_DTYPE, device=self.device + ) + entries.append( + ( + "aiter_rs", + lambda t, c=c, o=rs_out: c.custom_reduce_scatter(t, o, dim=0), + lambda t, c=c: c.should_custom_rs(t, dim=0), + c.capture(), + ) + ) + if self.pynccl_comm is not None: + c = self.pynccl_comm + rs_out = torch.empty( + rs_shape, dtype=BENCHMARK_DTYPE, device=self.device + ) + entries.append( + ( + "pynccl_rs", + lambda t, c=c, o=rs_out: c.reduce_scatter(o, t), + lambda t: True, + nullcontext(), + ) + ) + + for name, fn, should_use_fn, context in entries: + latency = self.benchmark_allreduce_single( + sequence_length, fn, should_use_fn, context, num_warmup, num_trials + ) + if latency is not None: + results[name] = latency + + return results + + +def _calculate_speedup_info(comm_results: dict[str, float], baseline: str) -> str: + """Fastest comm in a collective group and its speedup vs ``baseline``.""" if not comm_results: return "N/A" - # Find the fastest communicator fastest_comm = min(comm_results.keys(), key=lambda k: comm_results[k]) fastest_time = comm_results[fastest_comm] - # Calculate speedup vs PyNccl if available - if "pynccl" in comm_results: - pynccl_time = comm_results["pynccl"] - speedup = pynccl_time / fastest_time + if baseline in comm_results and fastest_time > 0: + speedup = comm_results[baseline] / fastest_time return f"{fastest_comm} ({speedup:.2f}x)" - else: - return f"{fastest_comm} (N/A)" + return f"{fastest_comm} (N/A)" def print_results( - results: dict[str, dict[str, float]], sequence_lengths: list[int], world_size: int + results: dict[int, dict[str, float]], sequence_lengths: list[int], world_size: int ): - """Print benchmark results in a formatted table.""" + """Print benchmark results as one table per collective. + + All-reduce, all-gather and reduce-scatter are different collectives, so each + gets its own table and its speedup is computed against that collective's + NCCL/RCCL baseline rather than across collectives. + """ + present: set[str] = set() + for size_results in results.values(): + present.update(size_results.keys()) print(f"\n{'=' * 130}") print("Device Communicator Benchmark Results") @@ -415,53 +558,45 @@ def print_results( f"World Size: {world_size}, Data Type: {BENCHMARK_DTYPE}, " f"Hidden Size: {HIDDEN_SIZE}" ) + print("Tensor Shape is the per-rank input; times are ms per operation.") print(f"{'=' * 130}") - # Get all communicator names - all_comms = set() - for size_results in results.values(): - all_comms.update(size_results.keys()) + for collective in COLLECTIVE_ORDER: + comms = sorted(c for c in present if COMM_COLLECTIVE.get(c) == collective) + if not comms: + continue + baseline = COLLECTIVE_BASELINE[collective] - all_comms = sorted(list(all_comms)) + print(f"\n{collective} (speedup vs {baseline})") + header = f"{'Tensor Shape':<20}{'Tensor Size':<15}" + for comm in comms: + header += f"{comm:<20}" + header += f"{'Best':<30}" + print(header) + print("-" * len(header)) - # Print header - header = f"{'Tensor Shape':<20}{'Tensor Size':<15}" - for comm in all_comms: - header += f"{comm:<20}" - header += f"{'Best (Speedup vs PyNccl)':<30}" - print(header) - print("-" * len(header)) + for seq_len in sequence_lengths: + if seq_len not in results: + continue + row_results = { + c: results[seq_len][c] for c in comms if c in results[seq_len] + } - # Print results for each sequence length - for seq_len in sequence_lengths: - if seq_len in results: - # Calculate tensor size in elements and bytes - tensor_elements = seq_len * HIDDEN_SIZE - tensor_bytes = tensor_elements * BENCHMARK_DTYPE.itemsize - - # Format tensor size (MB) - tensor_size_mb = tensor_bytes / (1024 * 1024) - tensor_size_str = f"{tensor_size_mb:.2f} MB" - - # Format tensor shape + tensor_bytes = seq_len * HIDDEN_SIZE * BENCHMARK_DTYPE.itemsize + tensor_size_str = f"{tensor_bytes / (1024 * 1024):.2f} MB" tensor_shape = f"({seq_len}, {HIDDEN_SIZE})" row = f"{tensor_shape:<20}{tensor_size_str:<15}" - for comm in all_comms: - if comm in results[seq_len]: - row += f"{results[seq_len][comm]:<20.3f}" + for comm in comms: + if comm in row_results: + row += f"{row_results[comm]:<20.3f}" else: row += f"{'N/A':<20}" - - # Calculate speedup information - speedup_info = _calculate_speedup_info(results[seq_len]) - row += f"{speedup_info:<30}" - + row += f"{_calculate_speedup_info(row_results, baseline):<30}" print(row) - print(f"{'=' * 130}") - print("All times are in milliseconds (ms) per allreduce operation") - print("Speedup column shows: fastest_algorithm (speedup_vs_pynccl)") + print(f"\n{'=' * 130}") + print("All times are in milliseconds (ms) per operation") def main(): @@ -526,6 +661,13 @@ def main(): num_warmup=args.num_warmup, num_trials=args.num_trials, ) + results.update( + benchmark.benchmark_ag_rs( + sequence_length=seq_len, + num_warmup=args.num_warmup, + num_trials=args.num_trials, + ) + ) all_results[seq_len] = results @@ -538,12 +680,23 @@ def main(): # Save to JSON if requested if args.output_json: - # Add speedup information to results + # Add per-collective speedup information to results. enhanced_results = {} for seq_len, comm_results in all_results.items(): + speedup_info = {} + for collective in COLLECTIVE_ORDER: + group = { + k: v + for k, v in comm_results.items() + if COMM_COLLECTIVE.get(k) == collective + } + if group: + speedup_info[collective] = _calculate_speedup_info( + group, COLLECTIVE_BASELINE[collective] + ) enhanced_results[seq_len] = { "timings": comm_results, - "speedup_info": _calculate_speedup_info(comm_results), + "speedup_info": speedup_info, } output_data = { diff --git a/tests/distributed/test_comm_ops.py b/tests/distributed/test_comm_ops.py index b5e552af49..f35ba14c1f 100644 --- a/tests/distributed/test_comm_ops.py +++ b/tests/distributed/test_comm_ops.py @@ -410,6 +410,21 @@ def test_flashinfer_all_reduce_precedes_nccl(monkeypatch: pytest.MonkeyPatch) -> nccl_selector.assert_not_called() +def test_aiter_all_gather_precedes_pynccl(monkeypatch: pytest.MonkeyPatch) -> None: + """Ensure can use aiter all_gather if available even if pynccl is not available.""" + output = torch.empty(2) + aiter_comm = Mock() + aiter_comm.should_custom_ag.return_value = True + aiter_comm.custom_all_gather.return_value = output + communicator = CudaCommunicator.__new__(CudaCommunicator) + communicator.world_size = 2 + communicator.aiter_ar_comm = aiter_comm + communicator.pynccl_comm = Mock(disabled=True) + monkeypatch.setattr(communicator, "_can_use_aiter_ag_rs", Mock(return_value=True)) + + assert communicator.all_gatherv(torch.empty(1)) is output + + def test_isend_object_posts_size_then_object_and_releases_on_wait( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/distributed/test_rocm_aiter_custom_ar.py b/tests/distributed/test_rocm_aiter_custom_ar.py index 0b85f36410..65018bfc4e 100644 --- a/tests/distributed/test_rocm_aiter_custom_ar.py +++ b/tests/distributed/test_rocm_aiter_custom_ar.py @@ -8,7 +8,7 @@ import torch.distributed as dist from vllm._aiter_ops import is_aiter_found, rocm_aiter_ops from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa -from vllm.distributed.parallel_state import get_tp_group, graph_capture +from vllm.distributed.parallel_state import get_dp_group, get_tp_group, graph_capture from vllm.envs import disable_envs_cache from vllm.platforms import current_platform @@ -132,3 +132,173 @@ def test_rocm_aiter_custom_allreduce( test_target, ): multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target) + + +def _get_aiter_ag_rs_comm(): + device_communicator = get_dp_group().device_communicator + assert device_communicator.use_aiter_ag_rs, ( + "AITER custom AG/RS was not enabled on the DP group." + ) + aiter_comm = device_communicator.aiter_ar_comm + assert aiter_comm is not None, "AITER custom AG/RS was not initialized." + assert not aiter_comm.disabled, "AITER custom AG/RS is disabled." + return aiter_comm + + +def _assert_aiter_handles_ag(aiter_comm, inp: torch.Tensor) -> None: + assert aiter_comm.should_custom_ag(inp), ( + f"AITER custom all-gather does not support input shape {inp.shape}." + ) + + +def _assert_aiter_handles_rs(aiter_comm, inp: torch.Tensor) -> None: + assert aiter_comm.should_custom_rs(inp, dim=0), ( + f"AITER custom reduce-scatter does not support input shape {inp.shape}." + ) + + +@ray.remote(num_gpus=1, max_calls=1) +def eager_ag_rs( + monkeypatch: pytest.MonkeyPatch, + tp_size, + pp_size, + rank, + distributed_init_port, + data_parallel_size, + data_parallel_master_port, +) -> None: + with monkeypatch.context() as m: + _configure_aiter_custom_ar_env(m) + + device = torch.device(f"cuda:{rank}") + torch.accelerator.set_device_index(device) + init_test_distributed_environment( + tp_size, + pp_size, + rank, + distributed_init_port, + data_parallel_size=data_parallel_size, + data_parallel_master_port=data_parallel_master_port, + ) + + dp_group = get_dp_group() + group = dp_group.device_group + dp_world = dp_group.world_size + aiter_comm = _get_aiter_ag_rs_comm() + + for shape, dtype in test_cases: + num_tokens, hidden = shape + + # all-gather: each rank contributes (num_tokens, hidden). + inp = torch.ones(shape, dtype=dtype, device=device) * (rank + 1) + _assert_aiter_handles_ag(aiter_comm, inp) + expected = torch.empty( + (num_tokens * dp_world, hidden), dtype=dtype, device=device + ) + dist.all_gather_into_tensor(expected, inp, group=group) + out = aiter_comm.custom_all_gather(inp, dim=0) + assert out is not None + torch.testing.assert_close(out, expected) + + # reduce-scatter: each rank contributes (num_tokens * dp, hidden). + rs_in = torch.ones( + (num_tokens * dp_world, hidden), dtype=dtype, device=device + ) * (rank + 1) + _assert_aiter_handles_rs(aiter_comm, rs_in) + rs_expected = torch.empty((num_tokens, hidden), dtype=dtype, device=device) + dist.reduce_scatter_tensor(rs_expected, rs_in, group=group) + rs_out = torch.empty((num_tokens, hidden), dtype=dtype, device=device) + aiter_comm.custom_reduce_scatter(rs_in, rs_out, dim=0) + torch.testing.assert_close(rs_out, rs_expected) + + +@ray.remote(num_gpus=1, max_calls=1) +def graph_ag_rs( + monkeypatch: pytest.MonkeyPatch, + tp_size, + pp_size, + rank, + distributed_init_port, + data_parallel_size, + data_parallel_master_port, +) -> None: + with monkeypatch.context() as m: + _configure_aiter_custom_ar_env(m) + + device = torch.device(f"cuda:{rank}") + torch.accelerator.set_device_index(device) + init_test_distributed_environment( + tp_size, + pp_size, + rank, + distributed_init_port, + data_parallel_size=data_parallel_size, + data_parallel_master_port=data_parallel_master_port, + ) + + dp_group = get_dp_group() + group = dp_group.device_group + dp_world = dp_group.world_size + aiter_comm = _get_aiter_ag_rs_comm() + + # Warmup so DP comms is initialized before graph capture + data = torch.zeros(1, device=device) + dist.all_reduce(data, group=group) + torch.accelerator.synchronize() + del data + + for shape, dtype in test_cases: + num_tokens, hidden = shape + + # all-gather under graph capture. + inp = torch.ones(shape, dtype=dtype, device=device) * (rank + 1) + _assert_aiter_handles_ag(aiter_comm, inp) + ag_expected = torch.empty( + (num_tokens * dp_world, hidden), dtype=dtype, device=device + ) + dist.all_gather_into_tensor(ag_expected, inp, group=group) + with graph_capture(device=device) as graph_capture_context: + torch.accelerator.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=graph_capture_context.stream): + ag_out = aiter_comm.custom_all_gather(inp, dim=0) + graph.replay() + torch.testing.assert_close(ag_out, ag_expected) + + # reduce-scatter under graph capture. + rs_in = torch.ones( + (num_tokens * dp_world, hidden), dtype=dtype, device=device + ) * (rank + 1) + _assert_aiter_handles_rs(aiter_comm, rs_in) + rs_expected = torch.empty((num_tokens, hidden), dtype=dtype, device=device) + dist.reduce_scatter_tensor(rs_expected, rs_in, group=group) + rs_out = torch.empty((num_tokens, hidden), dtype=dtype, device=device) + with graph_capture(device=device) as graph_capture_context: + torch.accelerator.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=graph_capture_context.stream): + aiter_comm.custom_reduce_scatter(rs_in, rs_out, dim=0) + graph.replay() + torch.testing.assert_close(rs_out, rs_expected) + + +@pytest.mark.skipif(not is_aiter_found(), reason="AITER is not installed") +@multi_gpu_test(num_gpus=2) +@pytest.mark.parametrize("tp_size", [1]) +@pytest.mark.parametrize("pipeline_parallel_size", [1]) +@pytest.mark.parametrize("data_parallel_size", [2]) +@pytest.mark.parametrize("test_target", [eager_ag_rs, graph_ag_rs]) +def test_rocm_aiter_custom_ag_rs( + monkeypatch: pytest.MonkeyPatch, + tp_size, + pipeline_parallel_size, + data_parallel_size, + test_target, +): + multi_process_parallel( + monkeypatch, + tp_size, + pipeline_parallel_size, + test_target, + data_parallel_size=data_parallel_size, + ) diff --git a/tests/utils.py b/tests/utils.py index 0c2cfeb6a0..e0166785ba 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1416,6 +1416,8 @@ def init_test_distributed_environment( rank: int, distributed_init_port: str, local_rank: int = -1, + data_parallel_size: int = 1, + data_parallel_master_port: int | None = None, ) -> None: # Note: This function is often called from Ray worker processes, so we # can't rely on pytest fixtures to set the config. We check if the config @@ -1428,6 +1430,29 @@ def init_test_distributed_environment( distributed_init_method = f"tcp://localhost:{distributed_init_port}" + if data_parallel_size > 1: + # For DP we need to set a common DP master port + from vllm.config.parallel import ParallelConfig + + assert data_parallel_master_port is not None, ( + "data_parallel_master_port is required when data_parallel_size > 1" + ) + tp_pp_world = tp_size * pp_size + parallel_config = ParallelConfig( + data_parallel_size=data_parallel_size, + data_parallel_rank=rank // tp_pp_world, + _data_parallel_master_port_list=[int(data_parallel_master_port)], + ) + with set_current_vllm_config(VllmConfig(parallel_config=parallel_config)): + init_distributed_environment( + world_size=tp_pp_world, + rank=rank % tp_pp_world, + distributed_init_method=distributed_init_method, + local_rank=local_rank if local_rank >= 0 else rank, + ) + ensure_model_parallel_initialized(tp_size, pp_size) + return + if get_current_vllm_config_or_none() is not None: # Config already set, use it directly init_distributed_environment( @@ -1454,6 +1479,7 @@ def multi_process_parallel( tp_size: int, pp_size: int, test_target: Any, + data_parallel_size: int = 1, ) -> None: import ray @@ -1474,18 +1500,34 @@ def multi_process_parallel( ) distributed_init_port = get_open_port() + # Separate port for the DP master group; only used when data_parallel_size > 1. + data_parallel_master_port = get_open_port() if data_parallel_size > 1 else None + world_size = data_parallel_size * tp_size * pp_size try: refs = [] - for rank in range(tp_size * pp_size): - refs.append( - test_target.remote( - monkeypatch, - tp_size, - pp_size, - rank, - distributed_init_port, - ), - ) + for rank in range(world_size): + if data_parallel_size > 1: + refs.append( + test_target.remote( + monkeypatch, + tp_size, + pp_size, + rank, + distributed_init_port, + data_parallel_size, + data_parallel_master_port, + ), + ) + else: + refs.append( + test_target.remote( + monkeypatch, + tp_size, + pp_size, + rank, + distributed_init_port, + ), + ) ray.get(refs) finally: ray.shutdown() diff --git a/vllm/distributed/device_communicators/aiter_custom_all_reduce.py b/vllm/distributed/device_communicators/aiter_custom_all_reduce.py index be2c901281..3247b33262 100644 --- a/vllm/distributed/device_communicators/aiter_custom_all_reduce.py +++ b/vllm/distributed/device_communicators/aiter_custom_all_reduce.py @@ -56,6 +56,20 @@ class AiterCustomAllreduce: def custom_all_reduce(self, inp: torch.Tensor) -> torch.Tensor | None: return self._impl.custom_all_reduce(inp) + def should_custom_ag(self, inp: torch.Tensor) -> bool: + return self._impl.should_custom_ag(inp) + + def should_custom_rs(self, inp: torch.Tensor, dim: int) -> bool: + return self._impl.should_custom_rs(inp, dim) + + def custom_all_gather(self, inp: torch.Tensor, dim: int = 0) -> torch.Tensor | None: + return self._impl.custom_all_gather(inp, dim=dim) + + def custom_reduce_scatter( + self, inp: torch.Tensor, out: torch.Tensor, dim: int = 0 + ) -> torch.Tensor | None: + return self._impl.custom_reduce_scatter(inp, out, dim=dim) + def use_1stage_fused_ar_rms(self, inp: torch.Tensor) -> bool: """Whether AITER's fused allreduce+RMSNorm runs as its one-stage kernel. diff --git a/vllm/distributed/device_communicators/cuda_communicator.py b/vllm/distributed/device_communicators/cuda_communicator.py index faa9fd0987..4afc28f652 100644 --- a/vllm/distributed/device_communicators/cuda_communicator.py +++ b/vllm/distributed/device_communicators/cuda_communicator.py @@ -109,6 +109,7 @@ class CudaCommunicator(DeviceCommunicatorBase): self.fi_ar_comm: FlashInferAllReduce | None = None self.fi_pcie_ipc_ar_comm: FlashInferPcieIpcAllReduce | None = None self.aiter_ar_comm: AiterCustomAllreduce | None = None + self.use_aiter_ag_rs: bool = False if use_torch_symm_mem and current_platform.is_cuda(): self.symm_mem_comm = SymmMemCommunicator( @@ -149,6 +150,22 @@ class CudaCommunicator(DeviceCommunicatorBase): ), ) + # AITER custom all-gather/reduce-scatter DP-attention dispatch/combine + if ( + "dp" in unique_name + and self.world_size in (2, 4, 8) + and current_platform.is_rocm() + and rocm_aiter_ops.is_custom_all_reduce_enabled() + ): + self.aiter_ar_comm = AiterCustomAllreduce( + group=self.cpu_group, + device=self.device, + ) + if self.aiter_ar_comm.disabled: + self.aiter_ar_comm = None + else: + self.use_aiter_ag_rs = True + if use_custom_allreduce and self.world_size > 1 and current_platform.is_rocm(): # Initialize a custom quick all-reduce implementation for AMD. # Quick reduce is designed as a complement to custom allreduce @@ -285,7 +302,11 @@ class CudaCommunicator(DeviceCommunicatorBase): enabled_ar_backends.append("NCCL_SYMM_MEM") if self.qr_comm is not None and not self.qr_comm.disabled: enabled_ar_backends.append("QUICK_REDUCE") - if self.aiter_ar_comm is not None and not self.aiter_ar_comm.disabled: + if ( + self.use_aiter_allreduce + and self.aiter_ar_comm is not None + and not self.aiter_ar_comm.disabled + ): enabled_ar_backends.append("AITER_CUSTOM") if self.ca_comm is not None and not self.ca_comm.disabled: enabled_ar_backends.append("CUSTOM") @@ -340,7 +361,8 @@ class CudaCommunicator(DeviceCommunicatorBase): return out aiter_ar_comm = self.aiter_ar_comm if ( - aiter_ar_comm is not None + self.use_aiter_allreduce + and aiter_ar_comm is not None and not aiter_ar_comm.disabled and aiter_ar_comm.should_custom_ar(input_) ): @@ -462,6 +484,11 @@ class CudaCommunicator(DeviceCommunicatorBase): # Convert negative dim to positive. dim += input_.dim() + # 'sizes' is not needed if all inputs in the same group have the same + # shape + if sizes is not None and all(s == sizes[0] for s in sizes): + sizes = None + # Note: This will produce an incorrect answer if we don't make # the input_tensor contiguous. Possible bug in reduce_scatter_tensor? input_tensor = input_.movedim(0, dim).contiguous() @@ -475,6 +502,16 @@ class CudaCommunicator(DeviceCommunicatorBase): chunk_size = input_tensor.shape[0] // world_size output_shape = (chunk_size,) + input_tensor.shape[1:] + if self._can_use_aiter_ag_rs(sizes): + aiter_comm = self.aiter_ar_comm + assert aiter_comm is not None + if aiter_comm.should_custom_rs(input_tensor, dim=0): + output = torch.empty( + output_shape, dtype=input_tensor.dtype, device=input_tensor.device + ) + aiter_comm.custom_reduce_scatter(input_tensor, output, dim=0) + return output.movedim(0, dim).contiguous() + # Symmetric memory is only used when all ranks have uniform sizes. # ncclCommWindowRegister is collective: asymmetric pool allocations # from variable per-rank sizes cause deadlocks. @@ -632,6 +669,34 @@ class CudaCommunicator(DeviceCommunicatorBase): self.all2all_manager.destroy() self.all2all_manager = None # type: ignore[assignment] + def _can_use_aiter_ag_rs(self, sizes: list[int] | None) -> bool: + """Whether the AITER custom AG/RS fast path may run for this collective. + + Requires: + - uniform batches + - FULL CUDAgraphs + """ + if ( + not self.use_aiter_ag_rs + or self.aiter_ar_comm is None + or self.aiter_ar_comm.disabled + ): + return False + if sizes is not None: + return False + + from vllm.config.compilation import CUDAGraphMode + from vllm.forward_context import get_forward_context + + try: + ctx = get_forward_context() + except AssertionError: + return False + if ctx.cudagraph_runtime_mode != CUDAGraphMode.FULL: + return False + bd = ctx.batch_descriptor + return bd is not None and bd.uniform + def suspend(self) -> None: if self.pynccl_comm is not None: self.pynccl_comm.suspend() @@ -665,14 +730,28 @@ class CudaCommunicator(DeviceCommunicatorBase): if dim != 0: raise NotImplementedError("only dim 0 all-gatherv is supported") world_size = self.world_size - pynccl_comm = self.pynccl_comm - assert pynccl_comm is not None and not pynccl_comm.disabled # 'sizes' is not needed if all inputs in the same group have the same # shape if sizes is not None and all(s == sizes[0] for s in sizes): sizes = None + if self._can_use_aiter_ag_rs(sizes): + aiter_comm = self.aiter_ar_comm + assert aiter_comm is not None + if isinstance(input_, torch.Tensor): + if aiter_comm.should_custom_ag(input_): + out = aiter_comm.custom_all_gather(input_, dim=0) + if out is not None: + return out + elif all(aiter_comm.should_custom_ag(inp) for inp in input_): + outs = [aiter_comm.custom_all_gather(inp, dim=0) for inp in input_] + if all(o is not None for o in outs): + return outs + + pynccl_comm = self.pynccl_comm + assert pynccl_comm is not None and not pynccl_comm.disabled + # Symmetric memory is only used when all ranks have uniform sizes. # ncclCommWindowRegister is collective: asymmetric pool allocations # from variable per-rank sizes cause deadlocks. diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index fd068dff1c..b24ff64d4d 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -673,11 +673,11 @@ class GroupCoordinator: else: stream = graph_capture_context.stream - # only cuda uses this function, + # only cuda/rocm uses this function, # so we don't abstract it into the base class maybe_ca_context = nullcontext() maybe_fi_pcie_ipc_context: AbstractContextManager[Any] = nullcontext() - maybe_aiter_context = nullcontext() + maybe_aiter_ar_context = nullcontext() from vllm.distributed.device_communicators.cuda_communicator import ( CudaCommunicator, ) @@ -698,12 +698,10 @@ class GroupCoordinator: if fi_pcie_ipc_ar_comm is not None: maybe_fi_pcie_ipc_context = fi_pcie_ipc_ar_comm.capture() - from vllm._aiter_ops import rocm_aiter_ops - - if rocm_aiter_ops.is_enabled(): - aiter_ar = rocm_aiter_ops.get_aiter_allreduce() - if aiter_ar is not None: - maybe_aiter_context = aiter_ar.capture() # type: ignore + # Capture each group's own comm. A global lookup would double-capture + aiter_ar_comm = getattr(self.device_communicator, "aiter_ar_comm", None) + if aiter_ar_comm is not None: + maybe_aiter_ar_context = aiter_ar_comm.capture() # type: ignore # ensure all initialization operations complete before attempting to # capture the graph on another stream @@ -715,7 +713,7 @@ class GroupCoordinator: torch.cuda.stream(stream), maybe_ca_context, maybe_fi_pcie_ipc_context, - maybe_aiter_context, + maybe_aiter_ar_context, ): yield graph_capture_context @@ -1640,7 +1638,11 @@ def graph_capture( context = graph_capture_context or GraphCaptureContext( torch.cuda.Stream(device=device) ) - with get_tp_group().graph_capture(context), get_pp_group().graph_capture(context): + with ( + get_tp_group().graph_capture(context), + get_pp_group().graph_capture(context), + get_dp_group().graph_capture(context), + ): yield context diff --git a/vllm/envs.py b/vllm/envs.py index d429686d9c..18e04e3dc6 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -1261,7 +1261,7 @@ environment_variables: dict[str, Callable[[], Any]] = { os.getenv("VLLM_ROCM_USE_AITER", "False").lower() in ("true", "1") ), # Use AITER's CustomAllreduce as the custom-allreduce backend inside vLLM's - # CudaCommunicator on ROCm. + # CudaCommunicator on ROCm. Also enables AITER AG/RS for DP communication. "VLLM_ROCM_USE_AITER_CUSTOM_AR": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "True").lower() in ("true", "1") ),