[Distributed] Add opt-in FlashInfer PCIe IPC all-reduce backend (#53576)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Gabriel Wu
2026-09-01 03:34:56 +00:00
committed by GitHub
co-authored by Yongye Zhu mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent e16b5e518d
commit 446c769482
7 changed files with 450 additions and 4 deletions
+17 -4
View File
@@ -29,7 +29,7 @@ import pickle
import weakref
from collections import deque, namedtuple
from collections.abc import Callable
from contextlib import contextmanager, nullcontext
from contextlib import AbstractContextManager, contextmanager, nullcontext
from dataclasses import dataclass
from datetime import timedelta
from multiprocessing import shared_memory
@@ -666,6 +666,7 @@ class GroupCoordinator:
# only cuda uses this function,
# so we don't abstract it into the base class
maybe_ca_context = nullcontext()
maybe_fi_pcie_ipc_context: AbstractContextManager[Any] = nullcontext()
maybe_aiter_context = nullcontext()
from vllm.distributed.device_communicators.cuda_communicator import (
CudaCommunicator,
@@ -682,6 +683,10 @@ class GroupCoordinator:
ca_comm = self.device_communicator.ca_comm
if ca_comm is not None:
maybe_ca_context = ca_comm.capture() # type: ignore
if isinstance(self.device_communicator, CudaCommunicator):
fi_pcie_ipc_ar_comm = self.device_communicator.fi_pcie_ipc_ar_comm
if fi_pcie_ipc_ar_comm is not None:
maybe_fi_pcie_ipc_context = fi_pcie_ipc_ar_comm.capture()
from vllm._aiter_ops import rocm_aiter_ops
@@ -696,7 +701,12 @@ class GroupCoordinator:
if curr_stream != stream:
stream.wait_stream(curr_stream)
with torch.cuda.stream(stream), maybe_ca_context, maybe_aiter_context:
with (
torch.cuda.stream(stream),
maybe_ca_context,
maybe_fi_pcie_ipc_context,
maybe_aiter_context,
):
yield graph_capture_context
def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
@@ -1360,14 +1370,17 @@ class GroupCoordinator:
return self.device_communicator.recv(size, dtype, src)
def destroy(self):
# Device communicators can own collective workspaces whose teardown
# uses these process groups (for example FlashInfer PCIe IPC barriers).
# Release them before destroying the groups they depend on.
if self.device_communicator is not None:
self.device_communicator.destroy()
if hasattr(self, "device_group"):
torch.distributed.destroy_process_group(self.device_group)
del self.device_group
if hasattr(self, "cpu_group"):
torch.distributed.destroy_process_group(self.cpu_group)
del self.cpu_group
if self.device_communicator is not None:
self.device_communicator.destroy()
if self.mq_broadcaster is not None:
self.mq_broadcaster = None