[Distributed] Add opt-in FlashInfer PCIe IPC all-reduce backend (#53576)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Gabriel Wu
2026-09-01 03:34:56 +00:00
committed by GitHub
co-authored by Yongye Zhu mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent e16b5e518d
commit 446c769482
7 changed files with 450 additions and 4 deletions
@@ -52,6 +52,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
use_custom_allreduce = False
use_torch_symm_mem = False
use_flashinfer_allreduce = False
use_flashinfer_pcie_ipc_allreduce = False
use_aiter_allreduce = False
else:
from vllm.distributed.parallel_state import _ENABLE_CUSTOM_ALL_REDUCE
@@ -62,6 +63,10 @@ class CudaCommunicator(DeviceCommunicatorBase):
use_flashinfer_allreduce = (
envs.VLLM_ALLREDUCE_USE_FLASHINFER and not envs.VLLM_BATCH_INVARIANT
)
use_flashinfer_pcie_ipc_allreduce = (
envs.VLLM_ALLREDUCE_USE_FLASHINFER_PCIE_IPC
and not envs.VLLM_BATCH_INVARIANT
)
use_aiter_allreduce = use_custom_allreduce and bool(
rocm_aiter_ops.is_custom_all_reduce_enabled()
)
@@ -69,6 +74,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.use_custom_allreduce = use_custom_allreduce
self.use_torch_symm_mem = use_torch_symm_mem
self.use_flashinfer_allreduce = use_flashinfer_allreduce
self.use_flashinfer_pcie_ipc_allreduce = use_flashinfer_pcie_ipc_allreduce
self.use_aiter_allreduce = use_aiter_allreduce
# lazy import to avoid documentation build error
@@ -78,6 +84,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
from vllm.distributed.device_communicators.flashinfer_all_reduce import (
FlashInferAllReduce,
)
from vllm.distributed.device_communicators.flashinfer_pcie_ipc_all_reduce import ( # noqa: E501
FlashInferPcieIpcAllReduce,
)
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.device_communicators.quick_all_reduce import (
QuickAllReduce,
@@ -97,6 +106,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.qr_comm: QuickAllReduce | None = None
self.symm_mem_comm: SymmMemCommunicator | None = None
self.fi_ar_comm: FlashInferAllReduce | None = None
self.fi_pcie_ipc_ar_comm: FlashInferPcieIpcAllReduce | None = None
self.aiter_ar_comm: AiterCustomAllreduce | None = None
if use_torch_symm_mem and current_platform.is_cuda():
@@ -111,6 +121,17 @@ class CudaCommunicator(DeviceCommunicatorBase):
device=self.device,
)
if (
self.use_flashinfer_pcie_ipc_allreduce
and self.world_size > 1
and self.device_group is not None
):
self.fi_pcie_ipc_ar_comm = FlashInferPcieIpcAllReduce(
group=self.device_group,
tune_group=self.cpu_group,
device=self.device,
)
if self.use_aiter_allreduce and self.world_size > 1:
self.aiter_ar_comm = AiterCustomAllreduce(
group=self.cpu_group,
@@ -220,6 +241,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
depends on the input tensor.
"""
all_potential_ar_backends = [
"FLASHINFER_PCIE_IPC",
"FLASHINFER",
"NCCL_SYMM_MEM",
"QUICK_REDUCE",
@@ -229,6 +251,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
"PYNCCL",
]
enabled_ar_backends: list[str] = []
if (
self.fi_pcie_ipc_ar_comm is not None
and not self.fi_pcie_ipc_ar_comm.disabled
):
enabled_ar_backends.append("FLASHINFER_PCIE_IPC")
if self.fi_ar_comm is not None and not self.fi_ar_comm.disabled:
enabled_ar_backends.append("FLASHINFER")
# Mirror the static preconditions of `should_nccl_symm_mem_allreduce`:
@@ -302,6 +329,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
out = qr_comm.quick_all_reduce(input_)
assert out is not None
return out
fi_pcie_ipc_ar_comm = self.fi_pcie_ipc_ar_comm
if fi_pcie_ipc_ar_comm is not None and fi_pcie_ipc_ar_comm.should_use(input_):
return fi_pcie_ipc_ar_comm.all_reduce(input_)
if use_fi_ar:
assert fi_ar_comm is not None
out = fi_ar_comm.all_reduce(input_)
@@ -594,6 +624,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
if self.fi_ar_comm is not None:
self.fi_ar_comm.destroy()
self.fi_ar_comm = None
if self.fi_pcie_ipc_ar_comm is not None:
self.fi_pcie_ipc_ar_comm.destroy()
self.fi_pcie_ipc_ar_comm = None
if self.all2all_manager is not None:
self.all2all_manager.destroy()
self.all2all_manager = None # type: ignore[assignment]
@@ -0,0 +1,239 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FlashInfer PCIe CUDA-IPC all-reduce integration.
This backend targets small tensor-parallel all-reduces on a single PCIe-only
node. Its workspace has a stricter lifetime and stream contract than the
existing FlashInfer MNNVL/TRT-LLM all-reduce implementation, so it intentionally
lives behind a separate wrapper and an opt-in environment variable.
"""
from collections.abc import Sequence
from contextlib import contextmanager
from pathlib import Path
from typing import TYPE_CHECKING, Any
import torch
import torch.distributed as dist
from torch.distributed import ProcessGroup
from vllm.distributed.parallel_state import in_the_same_node_as
from vllm.logger import init_logger
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.v1.worker.gpu_worker import Worker
logger = init_logger(__name__)
_SUPPORTED_WORLD_SIZES = (2, 4, 8)
_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
try:
import flashinfer.comm as flashinfer_comm
_pcie_ipc_available = hasattr(flashinfer_comm, "PcieIpcAllReduceWorkspace")
except ImportError:
flashinfer_comm = None # type: ignore[assignment]
_pcie_ipc_available = False
class FlashInferPcieIpcAllReduce:
"""vLLM lifecycle wrapper for FlashInfer's PCIe IPC all-reduce."""
def __init__(
self,
group: ProcessGroup,
tune_group: ProcessGroup,
device: int | str | torch.device,
) -> None:
self.disabled = True
self.group = group
self.tune_group = tune_group
self.device = torch.device(device)
self.world_size = dist.get_world_size(group)
self.rank = dist.get_rank(group)
self.workspace: Any | None = None
self.hidden_dim = 0
self.dtype: torch.dtype | None = None
if not _pcie_ipc_available:
logger.warning_once(
"FlashInfer PCIe IPC all-reduce was requested but this "
"FlashInfer build does not provide PcieIpcAllReduceWorkspace; "
"falling back to another all-reduce backend."
)
return
if not current_platform.is_cuda():
logger.warning_once(
"FlashInfer PCIe IPC all-reduce requires the CUDA platform."
)
return
if self.world_size not in _SUPPORTED_WORLD_SIZES:
logger.warning_once(
"FlashInfer PCIe IPC all-reduce does not support world_size=%d; "
"supported sizes are %s.",
self.world_size,
_SUPPORTED_WORLD_SIZES,
)
return
if not all(in_the_same_node_as(tune_group, source_rank=0)):
logger.warning_once(
"FlashInfer PCIe IPC all-reduce requires every rank in the TP "
"group to be on one node."
)
return
# Setup is deferred until kernel_warmup, where the model hidden size and
# exact CUDA Graph buckets are known. Until then dispatch falls through.
self.disabled = False
@property
def initialized(self) -> bool:
return not self.disabled and self.workspace is not None
def setup(
self,
*,
hidden_dim: int,
dtype: torch.dtype,
capture_sizes: Sequence[int],
tune_cache: Path,
) -> None:
"""Allocate, tune, and prepare the exact graph-capture shapes."""
if self.disabled or self.workspace is not None:
return
if dtype not in _SUPPORTED_DTYPES:
logger.warning_once(
"FlashInfer PCIe IPC all-reduce does not support dtype=%s; "
"falling back to another all-reduce backend.",
dtype,
)
self.disabled = True
return
batches = tuple(sorted({int(size) for size in capture_sizes if size > 0}))
if not batches:
logger.warning_once(
"FlashInfer PCIe IPC all-reduce has no CUDA Graph capture sizes "
"to prepare; falling back to another all-reduce backend."
)
self.disabled = True
return
self.hidden_dim = int(hidden_dim)
self.dtype = dtype
max_numel = batches[-1] * self.hidden_dim
workspace = flashinfer_comm.PcieIpcAllReduceWorkspace(
group=self.group,
max_numel=max_numel,
dtype=dtype,
tune_batches=batches,
tune_cache=str(tune_cache),
)
self.workspace = workspace
# tune() reuses a complete persisted cache and profiles only cache
# misses. It is deliberately mandatory: FlashInfer's seed policy was
# substantially slower than NCCL for the target TP4 decode workload.
torch.accelerator.synchronize(self.device)
workspace.rebind_stream()
workspace.tune(
[self.hidden_dim],
dtype=dtype,
tune_group=self.tune_group,
)
workspace.prepare([(batch, self.hidden_dim) for batch in batches], dtype=dtype)
torch.accelerator.synchronize(self.device)
workspace.rebind_stream()
logger.info_once(
"Initialized FlashInfer PCIe IPC all-reduce for TP%d, hidden_dim=%d, "
"dtype=%s, max_tokens=%d.",
self.world_size,
self.hidden_dim,
dtype,
batches[-1],
)
def should_use(self, inp: torch.Tensor) -> bool:
workspace = self.workspace
return bool(
not self.disabled
and workspace is not None
and inp.is_cuda
and inp.is_contiguous()
and inp.dim() == 2
and inp.shape[1] == self.hidden_dim
and inp.dtype == self.dtype
and workspace.supports(inp)
)
def all_reduce(self, inp: torch.Tensor) -> torch.Tensor:
assert self.workspace is not None
return self.workspace.all_reduce(inp)
@contextmanager
def capture(self):
"""Move the single-stream workspace to and from the capture stream."""
workspace = self.workspace
if workspace is None:
yield
return
# The caller orders graph capture against normal execution. A device
# sync makes that assertion explicit before relaxing FlashInfer's
# single-stream guard. Captured launches themselves are exempt from the
# guard, while capture warmups execute and bind to the capture stream.
torch.accelerator.synchronize(self.device)
workspace.rebind_stream()
try:
yield
finally:
torch.accelerator.synchronize(self.device)
workspace.rebind_stream()
def destroy(self) -> None:
workspace = self.workspace
if workspace is not None:
workspace.destroy()
self.workspace = None
def warmup_flashinfer_pcie_ipc_allreduce(worker: "Worker") -> None:
"""Initialize the TP PCIe IPC backend immediately before graph capture."""
from vllm.distributed.device_communicators.cuda_communicator import (
CudaCommunicator,
)
from vllm.distributed.parallel_state import get_tp_group
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
)
tp_group = get_tp_group()
communicator = tp_group.device_communicator
if not isinstance(communicator, CudaCommunicator):
return
pcie_comm = communicator.fi_pcie_ipc_ar_comm
if pcie_comm is None or pcie_comm.disabled:
return
if worker.vllm_config.parallel_config.use_ubatching:
logger.warning_once(
"FlashInfer PCIe IPC all-reduce does not yet support DBO or "
"multi-ubatch execution; falling back to another backend."
)
pcie_comm.disabled = True
return
capture_sizes = worker.vllm_config.compilation_config.cudagraph_capture_sizes
if not capture_sizes:
return
base_cache = resolve_flashinfer_autotune_file(worker.model_runner)
ranks = "-".join(str(rank) for rank in tp_group.ranks)
tune_cache = base_cache.with_name(f"pcie_ipc_allreduce_tp_{ranks}.json")
pcie_comm.setup(
hidden_dim=worker.model_config.get_hidden_size(),
dtype=worker.model_config.dtype,
capture_sizes=capture_sizes,
tune_cache=tune_cache,
)