[Feature] Add fault tolerance framework (simplified) for DP+EP external LB deployments (#44428)
Signed-off-by: fangyuchu <fangyuchu@qq.com> Signed-off-by: a798347923 <2645302020@qq.com> Signed-off-by: TianZhuo <2770730562@qq.com> Signed-off-by: a798347923 <39047817+a798347923@users.noreply.github.com> Signed-off-by: 205150940 <112750056+205150940@users.noreply.github.com> Signed-off-by: w00689259 <wangzhuo66@huawei.com> Signed-off-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com> Signed-off-by: zWaNg3 <389750525@qq.com> Signed-off-by: yzchang-plus <1078477584@qq.com> Signed-off-by: Jade Zheng <zheng.shoujian@outlook.com> Co-authored-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com> Co-authored-by: a798347923 <2645302020@qq.com> Co-authored-by: TianZhuo <2770730562@qq.com> Co-authored-by: 205150940 <112750056+205150940@users.noreply.github.com> Co-authored-by: a798347923 <39047817+a798347923@users.noreply.github.com> Co-authored-by: w00689259 <wangzhuo66@huawei.com> Co-authored-by: zWaNg3 <389750525@qq.com> Co-authored-by: yzchang-plus <1078477584@qq.com> Co-authored-by: Jade Zheng <zheng.shoujian@outlook.com>
This commit is contained in:
co-authored by
zWaNg3
a798347923
TianZhuo
205150940
a798347923
w00689259
zWaNg3
yzchang-plus
Jade Zheng
parent
33ef67e9fb
commit
0b0bd2b5f6
@@ -8,6 +8,7 @@ import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_dp_group, get_ep_group, get_pcp_group
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
from vllm.forward_context import get_forward_context
|
||||
@@ -278,7 +279,9 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
|
||||
def __init__(self, cpu_group, tcp_store_group=None):
|
||||
super().__init__(cpu_group, tcp_store_group)
|
||||
self.support_fault_tolerance = False # TODO: set to True when FT is supported.
|
||||
self.support_fault_tolerance = (
|
||||
get_current_vllm_config().parallel_config.enable_fault_tolerance
|
||||
)
|
||||
|
||||
def _make_all2all_kwargs(
|
||||
self,
|
||||
@@ -360,6 +363,16 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
has_fault = (current != DeepEPLLAll2AllManager._last_mask).any()
|
||||
return has_fault
|
||||
|
||||
def clean_buffers(self) -> None:
|
||||
buf = DeepEPLLAll2AllManager._buffer
|
||||
if buf is None:
|
||||
return
|
||||
buf.get_local_buffer_tensor(dtype=torch.int8, use_rdma_buffer=True).zero_()
|
||||
torch.accelerator.synchronize()
|
||||
buf.low_latency_clean_mask_buffer()
|
||||
torch.accelerator.synchronize()
|
||||
DeepEPLLAll2AllManager._last_mask = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _NixlEPBufferState:
|
||||
@@ -565,6 +578,18 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
has_fault = (current != last).any()
|
||||
return has_fault
|
||||
|
||||
def clean_buffers(self) -> None:
|
||||
if NixlEPAll2AllManager._buffer is None:
|
||||
return
|
||||
state = NixlEPAll2AllManager._buffer
|
||||
state.buffer.get_local_buffer_tensor(
|
||||
dtype=torch.int8, use_rdma_buffer=True
|
||||
).zero_()
|
||||
torch.accelerator.synchronize()
|
||||
state.buffer.clean_mask_buffer()
|
||||
torch.accelerator.synchronize()
|
||||
NixlEPAll2AllManager._last_mask = None
|
||||
|
||||
|
||||
class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
"""
|
||||
|
||||
@@ -105,10 +105,30 @@ class All2AllManagerBase:
|
||||
raise NotImplementedError
|
||||
|
||||
def query_active_mask(self) -> torch.Tensor:
|
||||
"""Return the all2all liveness mask for the EP ranks.
|
||||
|
||||
Returns:
|
||||
An int32 device tensor where 0 marks a live rank and 1 marks a
|
||||
masked (dead/unreachable) rank.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def query_fault(self) -> torch.Tensor:
|
||||
"""Returns has_fault scalar."""
|
||||
"""Return a scalar bool tensor, True if a new fault appeared.
|
||||
|
||||
Compares the current mask against the baseline recorded at the last
|
||||
recovery point.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def clean_buffers(self) -> None:
|
||||
"""Reset this rank's RDMA buffers and all2all mask state (rank-local).
|
||||
|
||||
Post-fault cleanup: a dispatch/combine that hit a dead peer or timed
|
||||
out can leave partially-written or stale tokens in the RDMA receive
|
||||
buffer, so it is zeroed to stop the next forward from reading that
|
||||
contaminated data.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def set_num_sms(self, num_sms: int):
|
||||
|
||||
Reference in New Issue
Block a user