[ROCm][AITER] Directly Implement AITER Custom All-reduce in CudaCommunicator (#46065)
Signed-off-by: BadrBasowid <badr.basowid@gmail.com>
This commit is contained in:
@@ -0,0 +1,95 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vLLM-owned wrapper over AITER's ``CustomAllreduce``.
|
||||
|
||||
vLLM's ``CudaCommunicator`` stores one of these as ``aiter_ar_comm`` (when
|
||||
``VLLM_ROCM_USE_AITER_CUSTOM_AR`` is set) so the plain allreduce and
|
||||
the fused allreduce+RMSNorm path share a single AITER instance with its IPC buffers.
|
||||
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class AiterCustomAllreduce:
|
||||
# Default IPC buffer size for AITER's CustomAllreduce.
|
||||
MAX_SIZE: int = 8192 * 1024 * 8 * 2
|
||||
|
||||
@classmethod
|
||||
def effective_max_size(cls) -> int:
|
||||
"""
|
||||
Max input byte size eligible for AITER custom allreduce.
|
||||
"""
|
||||
return cls.MAX_SIZE // 2
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group: ProcessGroup,
|
||||
device: int | str | torch.device,
|
||||
max_size: int | None = None,
|
||||
):
|
||||
from aiter.dist.device_communicators.custom_all_reduce import (
|
||||
CustomAllreduce as _AiterCustomAllreduce,
|
||||
)
|
||||
|
||||
if max_size is None:
|
||||
max_size = self.MAX_SIZE
|
||||
|
||||
self._impl = _AiterCustomAllreduce(group, device, max_size=max_size)
|
||||
|
||||
@property
|
||||
def aiter_ca(self):
|
||||
return self._impl
|
||||
|
||||
@property
|
||||
def disabled(self) -> bool:
|
||||
return self._impl.disabled
|
||||
|
||||
def should_custom_ar(self, inp: torch.Tensor) -> bool:
|
||||
return self._impl.should_custom_ar(inp)
|
||||
|
||||
def custom_all_reduce(self, inp: torch.Tensor) -> torch.Tensor | None:
|
||||
return self._impl.custom_all_reduce(inp)
|
||||
|
||||
def capture(self):
|
||||
return self._impl.capture()
|
||||
|
||||
def close(self) -> None:
|
||||
self._impl.close()
|
||||
|
||||
@property
|
||||
def supports_dynamic_hidden_dim(self) -> bool:
|
||||
"""Aiter's fused_allreduce_rmsnorm kernel dispatches on hidden_dim.
|
||||
Before aiter v0.1.12 the launcher was template-specialized on HIDDEN_DIM
|
||||
and silently no-op'd for sizes outside {512, 1024, 2048, 4096}. From v0.1.12
|
||||
hidden_dim is a runtime argument. Older builds are detected via
|
||||
AiterCustomAllreduce.supports_dynamic_hidden_dim; This function is used to
|
||||
skip fusion for unsupported sizes on them.
|
||||
Ref (old kernel): https://github.com/ROCm/aiter/blob/6a0e7b26ccf33164785531212cc2ec2cde0b9243/csrc/include/custom_all_reduce.cuh#L2590
|
||||
"""
|
||||
return hasattr(self._impl, "_pool")
|
||||
|
||||
@staticmethod
|
||||
def build_supports_per_group_quant() -> bool:
|
||||
"""True if the running AITER build exposes the per-group AR+RMS+quant
|
||||
kernel (added in ROCm/aiter PR #2823).
|
||||
|
||||
The pattern registration in ``RocmAiterAllReduceFusionPass`` keys off
|
||||
this so vLLM degrades to the AR+RMS-only fusion when run against an
|
||||
older aiter that lacks the per-group launcher.
|
||||
"""
|
||||
from aiter.dist.device_communicators.custom_all_reduce import (
|
||||
CustomAllreduce as _AiterCustomAllreduce,
|
||||
)
|
||||
|
||||
return hasattr(_AiterCustomAllreduce, "fused_ar_rms_per_group_quant")
|
||||
|
||||
# TODO(frida-andersson): drop once vLLM pins AITER >= 0.1.14 (ROCm/aiter#2823).
|
||||
@property
|
||||
def supports_per_group_quant(self) -> bool:
|
||||
return self.build_supports_per_group_quant()
|
||||
@@ -6,6 +6,7 @@ import torch
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.distributed.device_communicators.all_reduce_utils import (
|
||||
NCCL_SYMM_MEM_ALL_REDUCE_CONFIG,
|
||||
should_nccl_symm_mem_ag_rs,
|
||||
@@ -19,6 +20,7 @@ from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import StatelessProcessGroup
|
||||
from .aiter_custom_all_reduce import AiterCustomAllreduce
|
||||
from .base_device_communicator import DeviceCommunicatorBase
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -48,16 +50,21 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
use_custom_allreduce = False
|
||||
use_torch_symm_mem = False
|
||||
use_flashinfer_allreduce = False
|
||||
use_aiter_allreduce = False
|
||||
else:
|
||||
from vllm.distributed.parallel_state import _ENABLE_CUSTOM_ALL_REDUCE
|
||||
|
||||
use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
|
||||
use_torch_symm_mem = envs.VLLM_ALLREDUCE_USE_SYMM_MEM
|
||||
use_flashinfer_allreduce = envs.VLLM_ALLREDUCE_USE_FLASHINFER
|
||||
use_aiter_allreduce = use_custom_allreduce and bool(
|
||||
rocm_aiter_ops.is_custom_all_reduce_enabled()
|
||||
)
|
||||
|
||||
self.use_custom_allreduce = use_custom_allreduce
|
||||
self.use_torch_symm_mem = use_torch_symm_mem
|
||||
self.use_flashinfer_allreduce = use_flashinfer_allreduce
|
||||
self.use_aiter_allreduce = use_aiter_allreduce
|
||||
|
||||
# lazy import to avoid documentation build error
|
||||
from vllm.distributed.device_communicators.custom_all_reduce import (
|
||||
@@ -85,6 +92,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.qr_comm: QuickAllReduce | None = None
|
||||
self.symm_mem_comm: SymmMemCommunicator | None = None
|
||||
self.fi_ar_comm: FlashInferAllReduce | None = None
|
||||
self.aiter_ar_comm: AiterCustomAllreduce | None = None
|
||||
|
||||
if use_torch_symm_mem and current_platform.is_cuda():
|
||||
self.symm_mem_comm = SymmMemCommunicator(
|
||||
@@ -98,7 +106,13 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if use_custom_allreduce and self.world_size > 1:
|
||||
if self.use_aiter_allreduce and self.world_size > 1:
|
||||
self.aiter_ar_comm = AiterCustomAllreduce(
|
||||
group=self.cpu_group,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if use_custom_allreduce and self.aiter_ar_comm is None and self.world_size > 1:
|
||||
# Initialize a custom fast all-reduce implementation.
|
||||
self.ca_comm = CustomAllreduce(
|
||||
group=self.cpu_group,
|
||||
@@ -108,13 +122,14 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
),
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
# Initialize a custom quick all-reduce implementation for AMD.
|
||||
# Quick reduce is designed as a complement to custom allreduce.
|
||||
# Based on quickreduce (https://github.com/mk1-project/quickreduce).
|
||||
# If it's a rocm, 'use_custom_allreduce==True' means it must
|
||||
# currently be an MI300 series.
|
||||
self.qr_comm = QuickAllReduce(group=self.cpu_group, device=self.device)
|
||||
if use_custom_allreduce and self.world_size > 1 and current_platform.is_rocm():
|
||||
# Initialize a custom quick all-reduce implementation for AMD.
|
||||
# Quick reduce is designed as a complement to custom allreduce
|
||||
# (vLLM's or AITER's), so it is initialized for either backend.
|
||||
# Based on quickreduce (https://github.com/mk1-project/quickreduce).
|
||||
# On ROCm, 'use_custom_allreduce==True' means it must currently be
|
||||
# an MI300 series.
|
||||
self.qr_comm = QuickAllReduce(group=self.cpu_group, device=self.device)
|
||||
|
||||
if self.world_size > 1:
|
||||
self._log_all_reduce_backend_selection()
|
||||
@@ -203,6 +218,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
"NCCL_SYMM_MEM",
|
||||
"QUICK_REDUCE",
|
||||
"FLASHINFER",
|
||||
"AITER_CUSTOM",
|
||||
"CUSTOM",
|
||||
"SYMM_MEM",
|
||||
"PYNCCL",
|
||||
@@ -236,6 +252,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
enabled_ar_backends.append("QUICK_REDUCE")
|
||||
if self.fi_ar_comm is not None and not self.fi_ar_comm.disabled:
|
||||
enabled_ar_backends.append("FLASHINFER")
|
||||
if self.aiter_ar_comm is not None and not self.aiter_ar_comm.disabled:
|
||||
enabled_ar_backends.append("AITER_CUSTOM")
|
||||
if self.ca_comm is not None and not self.ca_comm.disabled:
|
||||
enabled_ar_backends.append("CUSTOM")
|
||||
if self.symm_mem_comm is not None and not self.symm_mem_comm.disabled:
|
||||
@@ -261,8 +279,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
out = torch.ops.vllm.all_reduce_symmetric_with_copy(input_)
|
||||
if out is not None:
|
||||
return out
|
||||
# always try quick reduce first, then flashinfer, then custom allreduce,
|
||||
# and then pynccl. (quick reduce just for ROCM MI3*)
|
||||
# always try quick reduce first, then flashinfer, then the AITER or vLLM
|
||||
# custom allreduce, and then pynccl. (quick reduce just for ROCM MI3*)
|
||||
qr_comm = self.qr_comm
|
||||
if (
|
||||
qr_comm is not None
|
||||
@@ -281,6 +299,15 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
out = fi_ar_comm.all_reduce(input_)
|
||||
assert out is not None
|
||||
return out
|
||||
aiter_ar_comm = self.aiter_ar_comm
|
||||
if (
|
||||
aiter_ar_comm is not None
|
||||
and not aiter_ar_comm.disabled
|
||||
and aiter_ar_comm.should_custom_ar(input_)
|
||||
):
|
||||
out = aiter_ar_comm.custom_all_reduce(input_)
|
||||
assert out is not None
|
||||
return out
|
||||
ca_comm = self.ca_comm
|
||||
if (
|
||||
ca_comm is not None
|
||||
@@ -509,6 +536,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.pynccl_comm = None
|
||||
if self.ca_comm is not None:
|
||||
self.ca_comm = None
|
||||
if self.aiter_ar_comm is not None:
|
||||
self.aiter_ar_comm.close()
|
||||
self.aiter_ar_comm = None
|
||||
if self.fi_ar_comm is not None:
|
||||
self.fi_ar_comm.destroy()
|
||||
self.fi_ar_comm = None
|
||||
|
||||
Reference in New Issue
Block a user