[Kernel] Refactor FlashInfer allreduce for mnnvl backend (#34109)
Signed-off-by: hjjq <50634613+hjjq@users.noreply.github.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
wzhao18
Wei Zhao
parent
2aa4140402
commit
71dfce6aa6
@@ -34,19 +34,25 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
# custom allreduce or torch symm mem can be used only by tp
|
||||
use_custom_allreduce = False
|
||||
use_torch_symm_mem = False
|
||||
use_flashinfer_allreduce = False
|
||||
else:
|
||||
from vllm.distributed.parallel_state import _ENABLE_CUSTOM_ALL_REDUCE
|
||||
|
||||
use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
|
||||
use_torch_symm_mem = envs.VLLM_ALLREDUCE_USE_SYMM_MEM
|
||||
use_flashinfer_allreduce = envs.VLLM_ALLREDUCE_USE_FLASHINFER
|
||||
|
||||
self.use_custom_allreduce = use_custom_allreduce
|
||||
self.use_torch_symm_mem = use_torch_symm_mem
|
||||
self.use_flashinfer_allreduce = use_flashinfer_allreduce
|
||||
|
||||
# lazy import to avoid documentation build error
|
||||
from vllm.distributed.device_communicators.custom_all_reduce import (
|
||||
CustomAllreduce,
|
||||
)
|
||||
from vllm.distributed.device_communicators.flashinfer_all_reduce import (
|
||||
FlashInferAllReduce,
|
||||
)
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.device_communicators.quick_all_reduce import (
|
||||
QuickAllReduce,
|
||||
@@ -65,12 +71,20 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.ca_comm: CustomAllreduce | None = None
|
||||
self.qr_comm: QuickAllReduce | None = None
|
||||
self.symm_mem_comm: SymmMemCommunicator | None = None
|
||||
self.fi_ar_comm: FlashInferAllReduce | None = None
|
||||
|
||||
if use_torch_symm_mem and current_platform.is_cuda():
|
||||
self.symm_mem_comm = SymmMemCommunicator(
|
||||
group=self.cpu_group,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if self.use_flashinfer_allreduce and self.world_size > 1:
|
||||
self.fi_ar_comm = FlashInferAllReduce(
|
||||
group=self.cpu_group,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if use_custom_allreduce and self.world_size > 1:
|
||||
# Initialize a custom fast all-reduce implementation.
|
||||
self.ca_comm = CustomAllreduce(
|
||||
@@ -136,7 +150,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
out = torch.ops.vllm.all_reduce_symmetric_with_copy(input_)
|
||||
if out is not None:
|
||||
return out
|
||||
# always try quick reduce first, then custom allreduce,
|
||||
# always try quick reduce first, then flashinfer, then custom allreduce,
|
||||
# and then pynccl. (quick reduce just for ROCM MI3*)
|
||||
qr_comm = self.qr_comm
|
||||
if (
|
||||
@@ -147,6 +161,15 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
out = qr_comm.quick_all_reduce(input_)
|
||||
assert out is not None
|
||||
return out
|
||||
fi_ar_comm = self.fi_ar_comm
|
||||
if (
|
||||
fi_ar_comm is not None
|
||||
and not fi_ar_comm.disabled
|
||||
and fi_ar_comm.should_use_fi_ar(input_)
|
||||
):
|
||||
out = fi_ar_comm.all_reduce(input_)
|
||||
assert out is not None
|
||||
return out
|
||||
ca_comm = self.ca_comm
|
||||
if (
|
||||
ca_comm is not None
|
||||
@@ -270,6 +293,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.pynccl_comm = None
|
||||
if self.ca_comm is not None:
|
||||
self.ca_comm = None
|
||||
if self.fi_ar_comm is not None:
|
||||
self.fi_ar_comm.destroy()
|
||||
self.fi_ar_comm = None
|
||||
if self.all2all_manager is not None:
|
||||
self.all2all_manager.destroy()
|
||||
self.all2all_manager = None
|
||||
|
||||
Reference in New Issue
Block a user