[Perf][ROCm] Add AITER custom AG/RS (DP only) (#48247)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
This commit is contained in:
@@ -56,6 +56,20 @@ class AiterCustomAllreduce:
|
||||
def custom_all_reduce(self, inp: torch.Tensor) -> torch.Tensor | None:
|
||||
return self._impl.custom_all_reduce(inp)
|
||||
|
||||
def should_custom_ag(self, inp: torch.Tensor) -> bool:
|
||||
return self._impl.should_custom_ag(inp)
|
||||
|
||||
def should_custom_rs(self, inp: torch.Tensor, dim: int) -> bool:
|
||||
return self._impl.should_custom_rs(inp, dim)
|
||||
|
||||
def custom_all_gather(self, inp: torch.Tensor, dim: int = 0) -> torch.Tensor | None:
|
||||
return self._impl.custom_all_gather(inp, dim=dim)
|
||||
|
||||
def custom_reduce_scatter(
|
||||
self, inp: torch.Tensor, out: torch.Tensor, dim: int = 0
|
||||
) -> torch.Tensor | None:
|
||||
return self._impl.custom_reduce_scatter(inp, out, dim=dim)
|
||||
|
||||
def use_1stage_fused_ar_rms(self, inp: torch.Tensor) -> bool:
|
||||
"""Whether AITER's fused allreduce+RMSNorm runs as its one-stage kernel.
|
||||
|
||||
|
||||
@@ -109,6 +109,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.fi_ar_comm: FlashInferAllReduce | None = None
|
||||
self.fi_pcie_ipc_ar_comm: FlashInferPcieIpcAllReduce | None = None
|
||||
self.aiter_ar_comm: AiterCustomAllreduce | None = None
|
||||
self.use_aiter_ag_rs: bool = False
|
||||
|
||||
if use_torch_symm_mem and current_platform.is_cuda():
|
||||
self.symm_mem_comm = SymmMemCommunicator(
|
||||
@@ -149,6 +150,22 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
),
|
||||
)
|
||||
|
||||
# AITER custom all-gather/reduce-scatter DP-attention dispatch/combine
|
||||
if (
|
||||
"dp" in unique_name
|
||||
and self.world_size in (2, 4, 8)
|
||||
and current_platform.is_rocm()
|
||||
and rocm_aiter_ops.is_custom_all_reduce_enabled()
|
||||
):
|
||||
self.aiter_ar_comm = AiterCustomAllreduce(
|
||||
group=self.cpu_group,
|
||||
device=self.device,
|
||||
)
|
||||
if self.aiter_ar_comm.disabled:
|
||||
self.aiter_ar_comm = None
|
||||
else:
|
||||
self.use_aiter_ag_rs = True
|
||||
|
||||
if use_custom_allreduce and self.world_size > 1 and current_platform.is_rocm():
|
||||
# Initialize a custom quick all-reduce implementation for AMD.
|
||||
# Quick reduce is designed as a complement to custom allreduce
|
||||
@@ -285,7 +302,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
enabled_ar_backends.append("NCCL_SYMM_MEM")
|
||||
if self.qr_comm is not None and not self.qr_comm.disabled:
|
||||
enabled_ar_backends.append("QUICK_REDUCE")
|
||||
if self.aiter_ar_comm is not None and not self.aiter_ar_comm.disabled:
|
||||
if (
|
||||
self.use_aiter_allreduce
|
||||
and self.aiter_ar_comm is not None
|
||||
and not self.aiter_ar_comm.disabled
|
||||
):
|
||||
enabled_ar_backends.append("AITER_CUSTOM")
|
||||
if self.ca_comm is not None and not self.ca_comm.disabled:
|
||||
enabled_ar_backends.append("CUSTOM")
|
||||
@@ -340,7 +361,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
return out
|
||||
aiter_ar_comm = self.aiter_ar_comm
|
||||
if (
|
||||
aiter_ar_comm is not None
|
||||
self.use_aiter_allreduce
|
||||
and aiter_ar_comm is not None
|
||||
and not aiter_ar_comm.disabled
|
||||
and aiter_ar_comm.should_custom_ar(input_)
|
||||
):
|
||||
@@ -462,6 +484,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
# Convert negative dim to positive.
|
||||
dim += input_.dim()
|
||||
|
||||
# 'sizes' is not needed if all inputs in the same group have the same
|
||||
# shape
|
||||
if sizes is not None and all(s == sizes[0] for s in sizes):
|
||||
sizes = None
|
||||
|
||||
# Note: This will produce an incorrect answer if we don't make
|
||||
# the input_tensor contiguous. Possible bug in reduce_scatter_tensor?
|
||||
input_tensor = input_.movedim(0, dim).contiguous()
|
||||
@@ -475,6 +502,16 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
chunk_size = input_tensor.shape[0] // world_size
|
||||
output_shape = (chunk_size,) + input_tensor.shape[1:]
|
||||
|
||||
if self._can_use_aiter_ag_rs(sizes):
|
||||
aiter_comm = self.aiter_ar_comm
|
||||
assert aiter_comm is not None
|
||||
if aiter_comm.should_custom_rs(input_tensor, dim=0):
|
||||
output = torch.empty(
|
||||
output_shape, dtype=input_tensor.dtype, device=input_tensor.device
|
||||
)
|
||||
aiter_comm.custom_reduce_scatter(input_tensor, output, dim=0)
|
||||
return output.movedim(0, dim).contiguous()
|
||||
|
||||
# Symmetric memory is only used when all ranks have uniform sizes.
|
||||
# ncclCommWindowRegister is collective: asymmetric pool allocations
|
||||
# from variable per-rank sizes cause deadlocks.
|
||||
@@ -632,6 +669,34 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.all2all_manager.destroy()
|
||||
self.all2all_manager = None # type: ignore[assignment]
|
||||
|
||||
def _can_use_aiter_ag_rs(self, sizes: list[int] | None) -> bool:
|
||||
"""Whether the AITER custom AG/RS fast path may run for this collective.
|
||||
|
||||
Requires:
|
||||
- uniform batches
|
||||
- FULL CUDAgraphs
|
||||
"""
|
||||
if (
|
||||
not self.use_aiter_ag_rs
|
||||
or self.aiter_ar_comm is None
|
||||
or self.aiter_ar_comm.disabled
|
||||
):
|
||||
return False
|
||||
if sizes is not None:
|
||||
return False
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.forward_context import get_forward_context
|
||||
|
||||
try:
|
||||
ctx = get_forward_context()
|
||||
except AssertionError:
|
||||
return False
|
||||
if ctx.cudagraph_runtime_mode != CUDAGraphMode.FULL:
|
||||
return False
|
||||
bd = ctx.batch_descriptor
|
||||
return bd is not None and bd.uniform
|
||||
|
||||
def suspend(self) -> None:
|
||||
if self.pynccl_comm is not None:
|
||||
self.pynccl_comm.suspend()
|
||||
@@ -665,14 +730,28 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
if dim != 0:
|
||||
raise NotImplementedError("only dim 0 all-gatherv is supported")
|
||||
world_size = self.world_size
|
||||
pynccl_comm = self.pynccl_comm
|
||||
assert pynccl_comm is not None and not pynccl_comm.disabled
|
||||
|
||||
# 'sizes' is not needed if all inputs in the same group have the same
|
||||
# shape
|
||||
if sizes is not None and all(s == sizes[0] for s in sizes):
|
||||
sizes = None
|
||||
|
||||
if self._can_use_aiter_ag_rs(sizes):
|
||||
aiter_comm = self.aiter_ar_comm
|
||||
assert aiter_comm is not None
|
||||
if isinstance(input_, torch.Tensor):
|
||||
if aiter_comm.should_custom_ag(input_):
|
||||
out = aiter_comm.custom_all_gather(input_, dim=0)
|
||||
if out is not None:
|
||||
return out
|
||||
elif all(aiter_comm.should_custom_ag(inp) for inp in input_):
|
||||
outs = [aiter_comm.custom_all_gather(inp, dim=0) for inp in input_]
|
||||
if all(o is not None for o in outs):
|
||||
return outs
|
||||
|
||||
pynccl_comm = self.pynccl_comm
|
||||
assert pynccl_comm is not None and not pynccl_comm.disabled
|
||||
|
||||
# Symmetric memory is only used when all ranks have uniform sizes.
|
||||
# ncclCommWindowRegister is collective: asymmetric pool allocations
|
||||
# from variable per-rank sizes cause deadlocks.
|
||||
|
||||
@@ -673,11 +673,11 @@ class GroupCoordinator:
|
||||
else:
|
||||
stream = graph_capture_context.stream
|
||||
|
||||
# only cuda uses this function,
|
||||
# only cuda/rocm uses this function,
|
||||
# so we don't abstract it into the base class
|
||||
maybe_ca_context = nullcontext()
|
||||
maybe_fi_pcie_ipc_context: AbstractContextManager[Any] = nullcontext()
|
||||
maybe_aiter_context = nullcontext()
|
||||
maybe_aiter_ar_context = nullcontext()
|
||||
from vllm.distributed.device_communicators.cuda_communicator import (
|
||||
CudaCommunicator,
|
||||
)
|
||||
@@ -698,12 +698,10 @@ class GroupCoordinator:
|
||||
if fi_pcie_ipc_ar_comm is not None:
|
||||
maybe_fi_pcie_ipc_context = fi_pcie_ipc_ar_comm.capture()
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
if aiter_ar is not None:
|
||||
maybe_aiter_context = aiter_ar.capture() # type: ignore
|
||||
# Capture each group's own comm. A global lookup would double-capture
|
||||
aiter_ar_comm = getattr(self.device_communicator, "aiter_ar_comm", None)
|
||||
if aiter_ar_comm is not None:
|
||||
maybe_aiter_ar_context = aiter_ar_comm.capture() # type: ignore
|
||||
|
||||
# ensure all initialization operations complete before attempting to
|
||||
# capture the graph on another stream
|
||||
@@ -715,7 +713,7 @@ class GroupCoordinator:
|
||||
torch.cuda.stream(stream),
|
||||
maybe_ca_context,
|
||||
maybe_fi_pcie_ipc_context,
|
||||
maybe_aiter_context,
|
||||
maybe_aiter_ar_context,
|
||||
):
|
||||
yield graph_capture_context
|
||||
|
||||
@@ -1640,7 +1638,11 @@ def graph_capture(
|
||||
context = graph_capture_context or GraphCaptureContext(
|
||||
torch.cuda.Stream(device=device)
|
||||
)
|
||||
with get_tp_group().graph_capture(context), get_pp_group().graph_capture(context):
|
||||
with (
|
||||
get_tp_group().graph_capture(context),
|
||||
get_pp_group().graph_capture(context),
|
||||
get_dp_group().graph_capture(context),
|
||||
):
|
||||
yield context
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user