[Distributed] Enable custom all-reduce under VLLM_BATCH_INVARIANT (#58623)

Signed-off-by: Flora Feng <4florafeng@gmail.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-10-02 02:25:48 +00:00
committed by GitHub
parent e3cae8d2ac
commit 785b990116
7 changed files with 130 additions and 31 deletions
@@ -68,8 +68,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
envs.VLLM_ALLREDUCE_USE_FLASHINFER_PCIE_IPC
and not envs.VLLM_BATCH_INVARIANT
)
use_aiter_allreduce = use_custom_allreduce and bool(
rocm_aiter_ops.is_custom_all_reduce_enabled()
# Neither AITER nor QuickReduce all-reduce has a fixed reduction order.
use_aiter_allreduce = (
use_custom_allreduce
and not envs.VLLM_BATCH_INVARIANT
and bool(rocm_aiter_ops.is_custom_all_reduce_enabled())
)
self.use_custom_allreduce = use_custom_allreduce
@@ -166,7 +169,12 @@ class CudaCommunicator(DeviceCommunicatorBase):
else:
self.use_aiter_ag_rs = True
if use_custom_allreduce and self.world_size > 1 and current_platform.is_rocm():
if (
use_custom_allreduce
and self.world_size > 1
and current_platform.is_rocm()
and not envs.VLLM_BATCH_INVARIANT
):
# Initialize a custom quick all-reduce implementation for AMD.
# Quick reduce is designed as a complement to custom allreduce
# (vLLM's or AITER's), so it is initialized for either backend.
@@ -119,6 +119,7 @@ from vllm.distributed.utils import is_weak_contiguous # noqa: E402
class CustomAllreduce:
_SUPPORTED_WORLD_SIZES = [2, 4, 6, 8, 16]
batch_invariant = False
_DEFAULT_ALL_GATHER_MAX_SIZE = 2 * 1024 * 1024
_DEFAULT_MNNVL_ALL_GATHER_MAX_SIZES = {
2: 8 * 1024 * 1024,
@@ -179,6 +180,7 @@ class CustomAllreduce:
self.mnnvl_multimem_rs_local_ptr = 0
self.mnnvl_multimem_rs_multicast_ptr = 0
self.mnnvl_only = False
self.batch_invariant = envs.VLLM_BATCH_INVARIANT
if not custom_ar:
# disable because of missing custom allreduce library
@@ -504,7 +506,7 @@ class CustomAllreduce:
# for 4 or more non NVLink-capable GPUs, custom allreduce provides
# little performance improvement over NCCL.
if self.world_size == 2 or self.fully_connected:
return inp_size < self.max_size
return self.batch_invariant or inp_size < self.max_size
return False
def all_reduce(
@@ -518,13 +520,27 @@ class CustomAllreduce:
"""
if out is None:
out = torch.empty_like(inp)
chunk_numel = self.max_size // inp.element_size()
if inp.numel() <= chunk_numel:
self._all_reduce_chunk(inp, out, registered)
return out
# Storage-order views: the gate only admits one contiguous block.
flat_inp = inp.as_strided((inp.numel(),), (1,), inp.storage_offset())
flat_out = out.as_strided((out.numel(),), (1,), out.storage_offset())
for start in range(0, flat_inp.numel(), chunk_numel):
end = start + chunk_numel
self._all_reduce_chunk(flat_inp[start:end], flat_out[start:end], registered)
return out
def _all_reduce_chunk(
self, inp: torch.Tensor, out: torch.Tensor, registered: bool
) -> None:
if registered:
ops.all_reduce(self._ptr, inp, out, 0, 0)
else:
ops.all_reduce(
self._ptr, inp, out, self.buffer_ptrs[self.rank], self.max_size
)
return out
def custom_all_reduce(self, input: torch.Tensor) -> torch.Tensor | None:
"""The main allreduce API that provides support for cuda graph."""
@@ -603,6 +619,9 @@ class CustomAllreduce:
) -> _ReduceScatterBackend | None:
if self.disabled or not current_platform.is_cuda():
return None
# The size gates below would switch backends per batch.
if self.batch_invariant:
return None
if self.world_size == 16 and not self.mnnvl_only:
return None
inp_size = inp.nbytes