[Hardware][Intel-Gaudi] Add Intel Gaudi (HPU) inference backend (#6143)
Signed-off-by: yuwenzho <yuwen.zhou@intel.com> Signed-off-by: Chendi.Xue <chendi.xue@intel.com> Signed-off-by: Bob Zhu <bob.zhu@intel.com> Signed-off-by: zehao-intel <zehao.huang@intel.com> Signed-off-by: Konrad Zawora <kzawora@habana.ai> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com> Co-authored-by: Sanju C Sudhakaran <scsudhakaran@habana.ai> Co-authored-by: Michal Adamczyk <madamczyk@habana.ai> Co-authored-by: Marceli Fylcek <mfylcek@habana.ai> Co-authored-by: Himangshu Lahkar <49579433+hlahkar@users.noreply.github.com> Co-authored-by: Vivek Goel <vgoel@habana.ai> Co-authored-by: yuwenzho <yuwen.zhou@intel.com> Co-authored-by: Dominika Olszewska <dolszewska@habana.ai> Co-authored-by: barak goldberg <149692267+bgoldberg-habana@users.noreply.github.com> Co-authored-by: Michal Szutenberg <37601244+szutenberg@users.noreply.github.com> Co-authored-by: Jan Kaniecki <jkaniecki@habana.ai> Co-authored-by: Agata Dobrzyniewicz <160237065+adobrzyniewicz-habana@users.noreply.github.com> Co-authored-by: Krzysztof Wisniewski <kwisniewski@habana.ai> Co-authored-by: Dudi Lester <160421192+dudilester@users.noreply.github.com> Co-authored-by: Ilia Taraban <tarabanil@gmail.com> Co-authored-by: Chendi.Xue <chendi.xue@intel.com> Co-authored-by: Michał Kuligowski <mkuligowski@habana.ai> Co-authored-by: Jakub Maksymczuk <jmaksymczuk@habana.ai> Co-authored-by: Tomasz Zielinski <85164140+tzielinski-habana@users.noreply.github.com> Co-authored-by: Sun Choi <schoi@habana.ai> Co-authored-by: Iryna Boiko <iboiko@habana.ai> Co-authored-by: Bob Zhu <41610754+czhu15@users.noreply.github.com> Co-authored-by: hlin99 <73271530+hlin99@users.noreply.github.com> Co-authored-by: Zehao Huang <zehao.huang@intel.com> Co-authored-by: Andrzej Kotłowski <Andrzej.Kotlowski@intel.com> Co-authored-by: Yan Tomsinsky <73292515+Yantom1@users.noreply.github.com> Co-authored-by: Nir David <ndavid@habana.ai> Co-authored-by: Yu-Zhou <yu.zhou@intel.com> Co-authored-by: Ruheena Suhani Shaik <rsshaik@habana.ai> Co-authored-by: Karol Damaszke <kdamaszke@habana.ai> Co-authored-by: Marcin Swiniarski <mswiniarski@habana.ai> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: Jacek Czaja <jacek.czaja@intel.com> Co-authored-by: Jacek Czaja <jczaja@habana.ai> Co-authored-by: Yuan <yuan.zhou@outlook.com>
This commit is contained in:
co-authored by
Kunshang Ji
Sanju C Sudhakaran
Michal Adamczyk
Marceli Fylcek
Himangshu Lahkar
Vivek Goel
yuwenzho
Dominika Olszewska
barak goldberg
Michal Szutenberg
Jan Kaniecki
Agata Dobrzyniewicz
Krzysztof Wisniewski
Dudi Lester
Ilia Taraban
Chendi.Xue
Michał Kuligowski
Jakub Maksymczuk
Tomasz Zielinski
Sun Choi
Iryna Boiko
Bob Zhu
hlin99
Zehao Huang
Andrzej Kotłowski
Yan Tomsinsky
Nir David
Yu-Zhou
Ruheena Suhani Shaik
Karol Damaszke
Marcin Swiniarski
Woosuk Kwon
Jacek Czaja
Jacek Czaja
Yuan
parent
a5fda50a10
commit
a02a50e6e5
@@ -177,6 +177,7 @@ class GroupCoordinator:
|
||||
use_pynccl: bool,
|
||||
use_custom_allreduce: bool,
|
||||
use_tpu_communicator: bool,
|
||||
use_hpu_communicator: bool,
|
||||
use_message_queue_broadcaster: bool = False,
|
||||
group_name: Optional[str] = None,
|
||||
):
|
||||
@@ -213,6 +214,7 @@ class GroupCoordinator:
|
||||
self.use_pynccl = use_pynccl
|
||||
self.use_custom_allreduce = use_custom_allreduce
|
||||
self.use_tpu_communicator = use_tpu_communicator
|
||||
self.use_hpu_communicator = use_hpu_communicator
|
||||
|
||||
# lazy import to avoid documentation build error
|
||||
from vllm.distributed.device_communicators.custom_all_reduce import (
|
||||
@@ -241,6 +243,12 @@ class GroupCoordinator:
|
||||
if use_tpu_communicator and self.world_size > 1:
|
||||
self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
|
||||
|
||||
from vllm.distributed.device_communicators.hpu_communicator import (
|
||||
HpuCommunicator)
|
||||
self.hpu_communicator: Optional[HpuCommunicator]
|
||||
if use_hpu_communicator and self.world_size > 1:
|
||||
self.hpu_communicator = HpuCommunicator(group=self.device_group)
|
||||
|
||||
from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
MessageQueue)
|
||||
self.mq_broadcaster: Optional[MessageQueue] = None
|
||||
@@ -362,6 +370,10 @@ class GroupCoordinator:
|
||||
# TPU handles Dynamo with its own logic.
|
||||
return self.tpu_communicator.all_reduce(input_)
|
||||
|
||||
if self.hpu_communicator is not None and \
|
||||
not self.hpu_communicator.disabled:
|
||||
return self.hpu_communicator.all_reduce(input_)
|
||||
|
||||
if self.ca_comm is not None and \
|
||||
not self.ca_comm.disabled and \
|
||||
self.ca_comm.should_custom_ar(input_):
|
||||
@@ -400,6 +412,11 @@ class GroupCoordinator:
|
||||
if tpu_comm is not None and not tpu_comm.disabled:
|
||||
return tpu_comm.all_gather(input_, dim)
|
||||
|
||||
# For HPUs, use HPU communicator.
|
||||
hpu_comm = self.hpu_communicator
|
||||
if hpu_comm is not None and not hpu_comm.disabled:
|
||||
return hpu_comm.all_gather(input_, dim)
|
||||
|
||||
if dim < 0:
|
||||
# Convert negative dim to positive.
|
||||
dim += input_.dim()
|
||||
@@ -879,6 +896,7 @@ def init_world_group(ranks: List[int], local_rank: int,
|
||||
use_pynccl=False,
|
||||
use_custom_allreduce=False,
|
||||
use_tpu_communicator=False,
|
||||
use_hpu_communicator=False,
|
||||
group_name="world",
|
||||
)
|
||||
|
||||
@@ -900,6 +918,7 @@ def init_model_parallel_group(
|
||||
use_pynccl=True,
|
||||
use_custom_allreduce=use_custom_allreduce,
|
||||
use_tpu_communicator=True,
|
||||
use_hpu_communicator=True,
|
||||
use_message_queue_broadcaster=use_message_queue_broadcaster,
|
||||
group_name=group_name,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user