[Feature] Add fault tolerance framework (simplified) for DP+EP external LB deployments (#44428)
Signed-off-by: fangyuchu <fangyuchu@qq.com> Signed-off-by: a798347923 <2645302020@qq.com> Signed-off-by: TianZhuo <2770730562@qq.com> Signed-off-by: a798347923 <39047817+a798347923@users.noreply.github.com> Signed-off-by: 205150940 <112750056+205150940@users.noreply.github.com> Signed-off-by: w00689259 <wangzhuo66@huawei.com> Signed-off-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com> Signed-off-by: zWaNg3 <389750525@qq.com> Signed-off-by: yzchang-plus <1078477584@qq.com> Signed-off-by: Jade Zheng <zheng.shoujian@outlook.com> Co-authored-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com> Co-authored-by: a798347923 <2645302020@qq.com> Co-authored-by: TianZhuo <2770730562@qq.com> Co-authored-by: 205150940 <112750056+205150940@users.noreply.github.com> Co-authored-by: a798347923 <39047817+a798347923@users.noreply.github.com> Co-authored-by: w00689259 <wangzhuo66@huawei.com> Co-authored-by: zWaNg3 <389750525@qq.com> Co-authored-by: yzchang-plus <1078477584@qq.com> Co-authored-by: Jade Zheng <zheng.shoujian@outlook.com>
This commit is contained in:
co-authored by
zWaNg3
a798347923
TianZhuo
205150940
a798347923
w00689259
zWaNg3
yzchang-plus
Jade Zheng
parent
33ef67e9fb
commit
0b0bd2b5f6
@@ -8,6 +8,7 @@ import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_dp_group, get_ep_group, get_pcp_group
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
from vllm.forward_context import get_forward_context
|
||||
@@ -278,7 +279,9 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
|
||||
def __init__(self, cpu_group, tcp_store_group=None):
|
||||
super().__init__(cpu_group, tcp_store_group)
|
||||
self.support_fault_tolerance = False # TODO: set to True when FT is supported.
|
||||
self.support_fault_tolerance = (
|
||||
get_current_vllm_config().parallel_config.enable_fault_tolerance
|
||||
)
|
||||
|
||||
def _make_all2all_kwargs(
|
||||
self,
|
||||
@@ -360,6 +363,16 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
has_fault = (current != DeepEPLLAll2AllManager._last_mask).any()
|
||||
return has_fault
|
||||
|
||||
def clean_buffers(self) -> None:
|
||||
buf = DeepEPLLAll2AllManager._buffer
|
||||
if buf is None:
|
||||
return
|
||||
buf.get_local_buffer_tensor(dtype=torch.int8, use_rdma_buffer=True).zero_()
|
||||
torch.accelerator.synchronize()
|
||||
buf.low_latency_clean_mask_buffer()
|
||||
torch.accelerator.synchronize()
|
||||
DeepEPLLAll2AllManager._last_mask = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _NixlEPBufferState:
|
||||
@@ -565,6 +578,18 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
has_fault = (current != last).any()
|
||||
return has_fault
|
||||
|
||||
def clean_buffers(self) -> None:
|
||||
if NixlEPAll2AllManager._buffer is None:
|
||||
return
|
||||
state = NixlEPAll2AllManager._buffer
|
||||
state.buffer.get_local_buffer_tensor(
|
||||
dtype=torch.int8, use_rdma_buffer=True
|
||||
).zero_()
|
||||
torch.accelerator.synchronize()
|
||||
state.buffer.clean_mask_buffer()
|
||||
torch.accelerator.synchronize()
|
||||
NixlEPAll2AllManager._last_mask = None
|
||||
|
||||
|
||||
class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user