[Perf][ROCm] Add AITER custom AG/RS (DP only) (#48247)

Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
This commit is contained in:
Simon Danielsson
2026-09-11 13:19:57 +08:00
committed by GitHub
parent 980c16c8e4
commit ae48466cf3
8 changed files with 549 additions and 74 deletions
@@ -56,6 +56,20 @@ class AiterCustomAllreduce:
def custom_all_reduce(self, inp: torch.Tensor) -> torch.Tensor | None:
return self._impl.custom_all_reduce(inp)
def should_custom_ag(self, inp: torch.Tensor) -> bool:
return self._impl.should_custom_ag(inp)
def should_custom_rs(self, inp: torch.Tensor, dim: int) -> bool:
return self._impl.should_custom_rs(inp, dim)
def custom_all_gather(self, inp: torch.Tensor, dim: int = 0) -> torch.Tensor | None:
return self._impl.custom_all_gather(inp, dim=dim)
def custom_reduce_scatter(
self, inp: torch.Tensor, out: torch.Tensor, dim: int = 0
) -> torch.Tensor | None:
return self._impl.custom_reduce_scatter(inp, out, dim=dim)
def use_1stage_fused_ar_rms(self, inp: torch.Tensor) -> bool:
"""Whether AITER's fused allreduce+RMSNorm runs as its one-stage kernel.
@@ -109,6 +109,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.fi_ar_comm: FlashInferAllReduce | None = None
self.fi_pcie_ipc_ar_comm: FlashInferPcieIpcAllReduce | None = None
self.aiter_ar_comm: AiterCustomAllreduce | None = None
self.use_aiter_ag_rs: bool = False
if use_torch_symm_mem and current_platform.is_cuda():
self.symm_mem_comm = SymmMemCommunicator(
@@ -149,6 +150,22 @@ class CudaCommunicator(DeviceCommunicatorBase):
),
)
# AITER custom all-gather/reduce-scatter DP-attention dispatch/combine
if (
"dp" in unique_name
and self.world_size in (2, 4, 8)
and current_platform.is_rocm()
and rocm_aiter_ops.is_custom_all_reduce_enabled()
):
self.aiter_ar_comm = AiterCustomAllreduce(
group=self.cpu_group,
device=self.device,
)
if self.aiter_ar_comm.disabled:
self.aiter_ar_comm = None
else:
self.use_aiter_ag_rs = True
if use_custom_allreduce and self.world_size > 1 and current_platform.is_rocm():
# Initialize a custom quick all-reduce implementation for AMD.
# Quick reduce is designed as a complement to custom allreduce
@@ -285,7 +302,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
enabled_ar_backends.append("NCCL_SYMM_MEM")
if self.qr_comm is not None and not self.qr_comm.disabled:
enabled_ar_backends.append("QUICK_REDUCE")
if self.aiter_ar_comm is not None and not self.aiter_ar_comm.disabled:
if (
self.use_aiter_allreduce
and self.aiter_ar_comm is not None
and not self.aiter_ar_comm.disabled
):
enabled_ar_backends.append("AITER_CUSTOM")
if self.ca_comm is not None and not self.ca_comm.disabled:
enabled_ar_backends.append("CUSTOM")
@@ -340,7 +361,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
return out
aiter_ar_comm = self.aiter_ar_comm
if (
aiter_ar_comm is not None
self.use_aiter_allreduce
and aiter_ar_comm is not None
and not aiter_ar_comm.disabled
and aiter_ar_comm.should_custom_ar(input_)
):
@@ -462,6 +484,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
# Convert negative dim to positive.
dim += input_.dim()
# 'sizes' is not needed if all inputs in the same group have the same
# shape
if sizes is not None and all(s == sizes[0] for s in sizes):
sizes = None
# Note: This will produce an incorrect answer if we don't make
# the input_tensor contiguous. Possible bug in reduce_scatter_tensor?
input_tensor = input_.movedim(0, dim).contiguous()
@@ -475,6 +502,16 @@ class CudaCommunicator(DeviceCommunicatorBase):
chunk_size = input_tensor.shape[0] // world_size
output_shape = (chunk_size,) + input_tensor.shape[1:]
if self._can_use_aiter_ag_rs(sizes):
aiter_comm = self.aiter_ar_comm
assert aiter_comm is not None
if aiter_comm.should_custom_rs(input_tensor, dim=0):
output = torch.empty(
output_shape, dtype=input_tensor.dtype, device=input_tensor.device
)
aiter_comm.custom_reduce_scatter(input_tensor, output, dim=0)
return output.movedim(0, dim).contiguous()
# Symmetric memory is only used when all ranks have uniform sizes.
# ncclCommWindowRegister is collective: asymmetric pool allocations
# from variable per-rank sizes cause deadlocks.
@@ -632,6 +669,34 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.all2all_manager.destroy()
self.all2all_manager = None # type: ignore[assignment]
def _can_use_aiter_ag_rs(self, sizes: list[int] | None) -> bool:
"""Whether the AITER custom AG/RS fast path may run for this collective.
Requires:
- uniform batches
- FULL CUDAgraphs
"""
if (
not self.use_aiter_ag_rs
or self.aiter_ar_comm is None
or self.aiter_ar_comm.disabled
):
return False
if sizes is not None:
return False
from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import get_forward_context
try:
ctx = get_forward_context()
except AssertionError:
return False
if ctx.cudagraph_runtime_mode != CUDAGraphMode.FULL:
return False
bd = ctx.batch_descriptor
return bd is not None and bd.uniform
def suspend(self) -> None:
if self.pynccl_comm is not None:
self.pynccl_comm.suspend()
@@ -665,14 +730,28 @@ class CudaCommunicator(DeviceCommunicatorBase):
if dim != 0:
raise NotImplementedError("only dim 0 all-gatherv is supported")
world_size = self.world_size
pynccl_comm = self.pynccl_comm
assert pynccl_comm is not None and not pynccl_comm.disabled
# 'sizes' is not needed if all inputs in the same group have the same
# shape
if sizes is not None and all(s == sizes[0] for s in sizes):
sizes = None
if self._can_use_aiter_ag_rs(sizes):
aiter_comm = self.aiter_ar_comm
assert aiter_comm is not None
if isinstance(input_, torch.Tensor):
if aiter_comm.should_custom_ag(input_):
out = aiter_comm.custom_all_gather(input_, dim=0)
if out is not None:
return out
elif all(aiter_comm.should_custom_ag(inp) for inp in input_):
outs = [aiter_comm.custom_all_gather(inp, dim=0) for inp in input_]
if all(o is not None for o in outs):
return outs
pynccl_comm = self.pynccl_comm
assert pynccl_comm is not None and not pynccl_comm.disabled
# Symmetric memory is only used when all ranks have uniform sizes.
# ncclCommWindowRegister is collective: asymmetric pool allocations
# from variable per-rank sizes cause deadlocks.
+12 -10
View File
@@ -673,11 +673,11 @@ class GroupCoordinator:
else:
stream = graph_capture_context.stream
# only cuda uses this function,
# only cuda/rocm uses this function,
# so we don't abstract it into the base class
maybe_ca_context = nullcontext()
maybe_fi_pcie_ipc_context: AbstractContextManager[Any] = nullcontext()
maybe_aiter_context = nullcontext()
maybe_aiter_ar_context = nullcontext()
from vllm.distributed.device_communicators.cuda_communicator import (
CudaCommunicator,
)
@@ -698,12 +698,10 @@ class GroupCoordinator:
if fi_pcie_ipc_ar_comm is not None:
maybe_fi_pcie_ipc_context = fi_pcie_ipc_ar_comm.capture()
from vllm._aiter_ops import rocm_aiter_ops
if rocm_aiter_ops.is_enabled():
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
if aiter_ar is not None:
maybe_aiter_context = aiter_ar.capture() # type: ignore
# Capture each group's own comm. A global lookup would double-capture
aiter_ar_comm = getattr(self.device_communicator, "aiter_ar_comm", None)
if aiter_ar_comm is not None:
maybe_aiter_ar_context = aiter_ar_comm.capture() # type: ignore
# ensure all initialization operations complete before attempting to
# capture the graph on another stream
@@ -715,7 +713,7 @@ class GroupCoordinator:
torch.cuda.stream(stream),
maybe_ca_context,
maybe_fi_pcie_ipc_context,
maybe_aiter_context,
maybe_aiter_ar_context,
):
yield graph_capture_context
@@ -1640,7 +1638,11 @@ def graph_capture(
context = graph_capture_context or GraphCaptureContext(
torch.cuda.Stream(device=device)
)
with get_tp_group().graph_capture(context), get_pp_group().graph_capture(context):
with (
get_tp_group().graph_capture(context),
get_pp_group().graph_capture(context),
get_dp_group().graph_capture(context),
):
yield context