[Distributed] Add opt-in FlashInfer PCIe IPC all-reduce backend (#53576)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Yongye Zhu
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
e16b5e518d
commit
446c769482
@@ -52,6 +52,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
use_custom_allreduce = False
|
||||
use_torch_symm_mem = False
|
||||
use_flashinfer_allreduce = False
|
||||
use_flashinfer_pcie_ipc_allreduce = False
|
||||
use_aiter_allreduce = False
|
||||
else:
|
||||
from vllm.distributed.parallel_state import _ENABLE_CUSTOM_ALL_REDUCE
|
||||
@@ -62,6 +63,10 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
use_flashinfer_allreduce = (
|
||||
envs.VLLM_ALLREDUCE_USE_FLASHINFER and not envs.VLLM_BATCH_INVARIANT
|
||||
)
|
||||
use_flashinfer_pcie_ipc_allreduce = (
|
||||
envs.VLLM_ALLREDUCE_USE_FLASHINFER_PCIE_IPC
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
)
|
||||
use_aiter_allreduce = use_custom_allreduce and bool(
|
||||
rocm_aiter_ops.is_custom_all_reduce_enabled()
|
||||
)
|
||||
@@ -69,6 +74,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.use_custom_allreduce = use_custom_allreduce
|
||||
self.use_torch_symm_mem = use_torch_symm_mem
|
||||
self.use_flashinfer_allreduce = use_flashinfer_allreduce
|
||||
self.use_flashinfer_pcie_ipc_allreduce = use_flashinfer_pcie_ipc_allreduce
|
||||
self.use_aiter_allreduce = use_aiter_allreduce
|
||||
|
||||
# lazy import to avoid documentation build error
|
||||
@@ -78,6 +84,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
from vllm.distributed.device_communicators.flashinfer_all_reduce import (
|
||||
FlashInferAllReduce,
|
||||
)
|
||||
from vllm.distributed.device_communicators.flashinfer_pcie_ipc_all_reduce import ( # noqa: E501
|
||||
FlashInferPcieIpcAllReduce,
|
||||
)
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.device_communicators.quick_all_reduce import (
|
||||
QuickAllReduce,
|
||||
@@ -97,6 +106,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.qr_comm: QuickAllReduce | None = None
|
||||
self.symm_mem_comm: SymmMemCommunicator | None = None
|
||||
self.fi_ar_comm: FlashInferAllReduce | None = None
|
||||
self.fi_pcie_ipc_ar_comm: FlashInferPcieIpcAllReduce | None = None
|
||||
self.aiter_ar_comm: AiterCustomAllreduce | None = None
|
||||
|
||||
if use_torch_symm_mem and current_platform.is_cuda():
|
||||
@@ -111,6 +121,17 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if (
|
||||
self.use_flashinfer_pcie_ipc_allreduce
|
||||
and self.world_size > 1
|
||||
and self.device_group is not None
|
||||
):
|
||||
self.fi_pcie_ipc_ar_comm = FlashInferPcieIpcAllReduce(
|
||||
group=self.device_group,
|
||||
tune_group=self.cpu_group,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if self.use_aiter_allreduce and self.world_size > 1:
|
||||
self.aiter_ar_comm = AiterCustomAllreduce(
|
||||
group=self.cpu_group,
|
||||
@@ -220,6 +241,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
depends on the input tensor.
|
||||
"""
|
||||
all_potential_ar_backends = [
|
||||
"FLASHINFER_PCIE_IPC",
|
||||
"FLASHINFER",
|
||||
"NCCL_SYMM_MEM",
|
||||
"QUICK_REDUCE",
|
||||
@@ -229,6 +251,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
"PYNCCL",
|
||||
]
|
||||
enabled_ar_backends: list[str] = []
|
||||
if (
|
||||
self.fi_pcie_ipc_ar_comm is not None
|
||||
and not self.fi_pcie_ipc_ar_comm.disabled
|
||||
):
|
||||
enabled_ar_backends.append("FLASHINFER_PCIE_IPC")
|
||||
if self.fi_ar_comm is not None and not self.fi_ar_comm.disabled:
|
||||
enabled_ar_backends.append("FLASHINFER")
|
||||
# Mirror the static preconditions of `should_nccl_symm_mem_allreduce`:
|
||||
@@ -302,6 +329,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
out = qr_comm.quick_all_reduce(input_)
|
||||
assert out is not None
|
||||
return out
|
||||
fi_pcie_ipc_ar_comm = self.fi_pcie_ipc_ar_comm
|
||||
if fi_pcie_ipc_ar_comm is not None and fi_pcie_ipc_ar_comm.should_use(input_):
|
||||
return fi_pcie_ipc_ar_comm.all_reduce(input_)
|
||||
if use_fi_ar:
|
||||
assert fi_ar_comm is not None
|
||||
out = fi_ar_comm.all_reduce(input_)
|
||||
@@ -594,6 +624,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
if self.fi_ar_comm is not None:
|
||||
self.fi_ar_comm.destroy()
|
||||
self.fi_ar_comm = None
|
||||
if self.fi_pcie_ipc_ar_comm is not None:
|
||||
self.fi_pcie_ipc_ar_comm.destroy()
|
||||
self.fi_pcie_ipc_ar_comm = None
|
||||
if self.all2all_manager is not None:
|
||||
self.all2all_manager.destroy()
|
||||
self.all2all_manager = None # type: ignore[assignment]
|
||||
|
||||
Reference in New Issue
Block a user