[Distributed] Add opt-in FlashInfer PCIe IPC all-reduce backend (#53576)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Gabriel Wu
2026-09-01 03:34:56 +00:00
committed by GitHub
co-authored by Yongye Zhu mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent e16b5e518d
commit 446c769482
7 changed files with 450 additions and 4 deletions
@@ -52,6 +52,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
use_custom_allreduce = False
use_torch_symm_mem = False
use_flashinfer_allreduce = False
use_flashinfer_pcie_ipc_allreduce = False
use_aiter_allreduce = False
else:
from vllm.distributed.parallel_state import _ENABLE_CUSTOM_ALL_REDUCE
@@ -62,6 +63,10 @@ class CudaCommunicator(DeviceCommunicatorBase):
use_flashinfer_allreduce = (
envs.VLLM_ALLREDUCE_USE_FLASHINFER and not envs.VLLM_BATCH_INVARIANT
)
use_flashinfer_pcie_ipc_allreduce = (
envs.VLLM_ALLREDUCE_USE_FLASHINFER_PCIE_IPC
and not envs.VLLM_BATCH_INVARIANT
)
use_aiter_allreduce = use_custom_allreduce and bool(
rocm_aiter_ops.is_custom_all_reduce_enabled()
)
@@ -69,6 +74,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.use_custom_allreduce = use_custom_allreduce
self.use_torch_symm_mem = use_torch_symm_mem
self.use_flashinfer_allreduce = use_flashinfer_allreduce
self.use_flashinfer_pcie_ipc_allreduce = use_flashinfer_pcie_ipc_allreduce
self.use_aiter_allreduce = use_aiter_allreduce
# lazy import to avoid documentation build error
@@ -78,6 +84,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
from vllm.distributed.device_communicators.flashinfer_all_reduce import (
FlashInferAllReduce,
)
from vllm.distributed.device_communicators.flashinfer_pcie_ipc_all_reduce import ( # noqa: E501
FlashInferPcieIpcAllReduce,
)
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.device_communicators.quick_all_reduce import (
QuickAllReduce,
@@ -97,6 +106,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.qr_comm: QuickAllReduce | None = None
self.symm_mem_comm: SymmMemCommunicator | None = None
self.fi_ar_comm: FlashInferAllReduce | None = None
self.fi_pcie_ipc_ar_comm: FlashInferPcieIpcAllReduce | None = None
self.aiter_ar_comm: AiterCustomAllreduce | None = None
if use_torch_symm_mem and current_platform.is_cuda():
@@ -111,6 +121,17 @@ class CudaCommunicator(DeviceCommunicatorBase):
device=self.device,
)
if (
self.use_flashinfer_pcie_ipc_allreduce
and self.world_size > 1
and self.device_group is not None
):
self.fi_pcie_ipc_ar_comm = FlashInferPcieIpcAllReduce(
group=self.device_group,
tune_group=self.cpu_group,
device=self.device,
)
if self.use_aiter_allreduce and self.world_size > 1:
self.aiter_ar_comm = AiterCustomAllreduce(
group=self.cpu_group,
@@ -220,6 +241,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
depends on the input tensor.
"""
all_potential_ar_backends = [
"FLASHINFER_PCIE_IPC",
"FLASHINFER",
"NCCL_SYMM_MEM",
"QUICK_REDUCE",
@@ -229,6 +251,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
"PYNCCL",
]
enabled_ar_backends: list[str] = []
if (
self.fi_pcie_ipc_ar_comm is not None
and not self.fi_pcie_ipc_ar_comm.disabled
):
enabled_ar_backends.append("FLASHINFER_PCIE_IPC")
if self.fi_ar_comm is not None and not self.fi_ar_comm.disabled:
enabled_ar_backends.append("FLASHINFER")
# Mirror the static preconditions of `should_nccl_symm_mem_allreduce`:
@@ -302,6 +329,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
out = qr_comm.quick_all_reduce(input_)
assert out is not None
return out
fi_pcie_ipc_ar_comm = self.fi_pcie_ipc_ar_comm
if fi_pcie_ipc_ar_comm is not None and fi_pcie_ipc_ar_comm.should_use(input_):
return fi_pcie_ipc_ar_comm.all_reduce(input_)
if use_fi_ar:
assert fi_ar_comm is not None
out = fi_ar_comm.all_reduce(input_)
@@ -594,6 +624,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
if self.fi_ar_comm is not None:
self.fi_ar_comm.destroy()
self.fi_ar_comm = None
if self.fi_pcie_ipc_ar_comm is not None:
self.fi_pcie_ipc_ar_comm.destroy()
self.fi_pcie_ipc_ar_comm = None
if self.all2all_manager is not None:
self.all2all_manager.destroy()
self.all2all_manager = None # type: ignore[assignment]