[Distributed] Add opt-in FlashInfer PCIe IPC all-reduce backend (#53576)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Yongye Zhu
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
e16b5e518d
commit
446c769482
@@ -29,7 +29,7 @@ import pickle
|
||||
import weakref
|
||||
from collections import deque, namedtuple
|
||||
from collections.abc import Callable
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
from multiprocessing import shared_memory
|
||||
@@ -666,6 +666,7 @@ class GroupCoordinator:
|
||||
# only cuda uses this function,
|
||||
# so we don't abstract it into the base class
|
||||
maybe_ca_context = nullcontext()
|
||||
maybe_fi_pcie_ipc_context: AbstractContextManager[Any] = nullcontext()
|
||||
maybe_aiter_context = nullcontext()
|
||||
from vllm.distributed.device_communicators.cuda_communicator import (
|
||||
CudaCommunicator,
|
||||
@@ -682,6 +683,10 @@ class GroupCoordinator:
|
||||
ca_comm = self.device_communicator.ca_comm
|
||||
if ca_comm is not None:
|
||||
maybe_ca_context = ca_comm.capture() # type: ignore
|
||||
if isinstance(self.device_communicator, CudaCommunicator):
|
||||
fi_pcie_ipc_ar_comm = self.device_communicator.fi_pcie_ipc_ar_comm
|
||||
if fi_pcie_ipc_ar_comm is not None:
|
||||
maybe_fi_pcie_ipc_context = fi_pcie_ipc_ar_comm.capture()
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
@@ -696,7 +701,12 @@ class GroupCoordinator:
|
||||
if curr_stream != stream:
|
||||
stream.wait_stream(curr_stream)
|
||||
|
||||
with torch.cuda.stream(stream), maybe_ca_context, maybe_aiter_context:
|
||||
with (
|
||||
torch.cuda.stream(stream),
|
||||
maybe_ca_context,
|
||||
maybe_fi_pcie_ipc_context,
|
||||
maybe_aiter_context,
|
||||
):
|
||||
yield graph_capture_context
|
||||
|
||||
def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
|
||||
@@ -1360,14 +1370,17 @@ class GroupCoordinator:
|
||||
return self.device_communicator.recv(size, dtype, src)
|
||||
|
||||
def destroy(self):
|
||||
# Device communicators can own collective workspaces whose teardown
|
||||
# uses these process groups (for example FlashInfer PCIe IPC barriers).
|
||||
# Release them before destroying the groups they depend on.
|
||||
if self.device_communicator is not None:
|
||||
self.device_communicator.destroy()
|
||||
if hasattr(self, "device_group"):
|
||||
torch.distributed.destroy_process_group(self.device_group)
|
||||
del self.device_group
|
||||
if hasattr(self, "cpu_group"):
|
||||
torch.distributed.destroy_process_group(self.cpu_group)
|
||||
del self.cpu_group
|
||||
if self.device_communicator is not None:
|
||||
self.device_communicator.destroy()
|
||||
if self.mq_broadcaster is not None:
|
||||
self.mq_broadcaster = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user