[Distributed] Enable custom all-reduce under VLLM_BATCH_INVARIANT (#58623)
Signed-off-by: Flora Feng <4florafeng@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -68,8 +68,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
envs.VLLM_ALLREDUCE_USE_FLASHINFER_PCIE_IPC
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
)
|
||||
use_aiter_allreduce = use_custom_allreduce and bool(
|
||||
rocm_aiter_ops.is_custom_all_reduce_enabled()
|
||||
# Neither AITER nor QuickReduce all-reduce has a fixed reduction order.
|
||||
use_aiter_allreduce = (
|
||||
use_custom_allreduce
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
and bool(rocm_aiter_ops.is_custom_all_reduce_enabled())
|
||||
)
|
||||
|
||||
self.use_custom_allreduce = use_custom_allreduce
|
||||
@@ -166,7 +169,12 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
else:
|
||||
self.use_aiter_ag_rs = True
|
||||
|
||||
if use_custom_allreduce and self.world_size > 1 and current_platform.is_rocm():
|
||||
if (
|
||||
use_custom_allreduce
|
||||
and self.world_size > 1
|
||||
and current_platform.is_rocm()
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
):
|
||||
# Initialize a custom quick all-reduce implementation for AMD.
|
||||
# Quick reduce is designed as a complement to custom allreduce
|
||||
# (vLLM's or AITER's), so it is initialized for either backend.
|
||||
|
||||
@@ -119,6 +119,7 @@ from vllm.distributed.utils import is_weak_contiguous # noqa: E402
|
||||
|
||||
class CustomAllreduce:
|
||||
_SUPPORTED_WORLD_SIZES = [2, 4, 6, 8, 16]
|
||||
batch_invariant = False
|
||||
_DEFAULT_ALL_GATHER_MAX_SIZE = 2 * 1024 * 1024
|
||||
_DEFAULT_MNNVL_ALL_GATHER_MAX_SIZES = {
|
||||
2: 8 * 1024 * 1024,
|
||||
@@ -179,6 +180,7 @@ class CustomAllreduce:
|
||||
self.mnnvl_multimem_rs_local_ptr = 0
|
||||
self.mnnvl_multimem_rs_multicast_ptr = 0
|
||||
self.mnnvl_only = False
|
||||
self.batch_invariant = envs.VLLM_BATCH_INVARIANT
|
||||
|
||||
if not custom_ar:
|
||||
# disable because of missing custom allreduce library
|
||||
@@ -504,7 +506,7 @@ class CustomAllreduce:
|
||||
# for 4 or more non NVLink-capable GPUs, custom allreduce provides
|
||||
# little performance improvement over NCCL.
|
||||
if self.world_size == 2 or self.fully_connected:
|
||||
return inp_size < self.max_size
|
||||
return self.batch_invariant or inp_size < self.max_size
|
||||
return False
|
||||
|
||||
def all_reduce(
|
||||
@@ -518,13 +520,27 @@ class CustomAllreduce:
|
||||
"""
|
||||
if out is None:
|
||||
out = torch.empty_like(inp)
|
||||
chunk_numel = self.max_size // inp.element_size()
|
||||
if inp.numel() <= chunk_numel:
|
||||
self._all_reduce_chunk(inp, out, registered)
|
||||
return out
|
||||
# Storage-order views: the gate only admits one contiguous block.
|
||||
flat_inp = inp.as_strided((inp.numel(),), (1,), inp.storage_offset())
|
||||
flat_out = out.as_strided((out.numel(),), (1,), out.storage_offset())
|
||||
for start in range(0, flat_inp.numel(), chunk_numel):
|
||||
end = start + chunk_numel
|
||||
self._all_reduce_chunk(flat_inp[start:end], flat_out[start:end], registered)
|
||||
return out
|
||||
|
||||
def _all_reduce_chunk(
|
||||
self, inp: torch.Tensor, out: torch.Tensor, registered: bool
|
||||
) -> None:
|
||||
if registered:
|
||||
ops.all_reduce(self._ptr, inp, out, 0, 0)
|
||||
else:
|
||||
ops.all_reduce(
|
||||
self._ptr, inp, out, self.buffer_ptrs[self.rank], self.max_size
|
||||
)
|
||||
return out
|
||||
|
||||
def custom_all_reduce(self, input: torch.Tensor) -> torch.Tensor | None:
|
||||
"""The main allreduce API that provides support for cuda graph."""
|
||||
@@ -603,6 +619,9 @@ class CustomAllreduce:
|
||||
) -> _ReduceScatterBackend | None:
|
||||
if self.disabled or not current_platform.is_cuda():
|
||||
return None
|
||||
# The size gates below would switch backends per batch.
|
||||
if self.batch_invariant:
|
||||
return None
|
||||
if self.world_size == 16 and not self.mnnvl_only:
|
||||
return None
|
||||
inp_size = inp.nbytes
|
||||
|
||||
Reference in New Issue
Block a user