[Feature] Add fault tolerance framework (simplified) for DP+EP external LB deployments (#44428)

Signed-off-by: fangyuchu <fangyuchu@qq.com>
Signed-off-by: a798347923 <2645302020@qq.com>
Signed-off-by: TianZhuo <2770730562@qq.com>
Signed-off-by: a798347923 <39047817+a798347923@users.noreply.github.com>
Signed-off-by: 205150940 <112750056+205150940@users.noreply.github.com>
Signed-off-by: w00689259 <wangzhuo66@huawei.com>
Signed-off-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com>
Signed-off-by: zWaNg3 <389750525@qq.com>
Signed-off-by: yzchang-plus <1078477584@qq.com>
Signed-off-by: Jade Zheng <zheng.shoujian@outlook.com>
Co-authored-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com>
Co-authored-by: a798347923 <2645302020@qq.com>
Co-authored-by: TianZhuo <2770730562@qq.com>
Co-authored-by: 205150940 <112750056+205150940@users.noreply.github.com>
Co-authored-by: a798347923 <39047817+a798347923@users.noreply.github.com>
Co-authored-by: w00689259 <wangzhuo66@huawei.com>
Co-authored-by: zWaNg3 <389750525@qq.com>
Co-authored-by: yzchang-plus <1078477584@qq.com>
Co-authored-by: Jade Zheng <zheng.shoujian@outlook.com>
This commit is contained in:
fangyuchu
2026-07-25 11:49:09 -04:00
committed by GitHub
co-authored by zWaNg3 a798347923 TianZhuo 205150940 a798347923 w00689259 zWaNg3 yzchang-plus Jade Zheng
parent 33ef67e9fb
commit 0b0bd2b5f6
27 changed files with 1088 additions and 7 deletions
@@ -8,6 +8,7 @@ import torch
import torch.distributed as dist
import vllm.envs as envs
from vllm.config import get_current_vllm_config
from vllm.distributed import get_dp_group, get_ep_group, get_pcp_group
from vllm.distributed.utils import StatelessProcessGroup
from vllm.forward_context import get_forward_context
@@ -278,7 +279,9 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
def __init__(self, cpu_group, tcp_store_group=None):
super().__init__(cpu_group, tcp_store_group)
self.support_fault_tolerance = False # TODO: set to True when FT is supported.
self.support_fault_tolerance = (
get_current_vllm_config().parallel_config.enable_fault_tolerance
)
def _make_all2all_kwargs(
self,
@@ -360,6 +363,16 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
has_fault = (current != DeepEPLLAll2AllManager._last_mask).any()
return has_fault
def clean_buffers(self) -> None:
buf = DeepEPLLAll2AllManager._buffer
if buf is None:
return
buf.get_local_buffer_tensor(dtype=torch.int8, use_rdma_buffer=True).zero_()
torch.accelerator.synchronize()
buf.low_latency_clean_mask_buffer()
torch.accelerator.synchronize()
DeepEPLLAll2AllManager._last_mask = None
@dataclass
class _NixlEPBufferState:
@@ -565,6 +578,18 @@ class NixlEPAll2AllManager(All2AllManagerBase):
has_fault = (current != last).any()
return has_fault
def clean_buffers(self) -> None:
if NixlEPAll2AllManager._buffer is None:
return
state = NixlEPAll2AllManager._buffer
state.buffer.get_local_buffer_tensor(
dtype=torch.int8, use_rdma_buffer=True
).zero_()
torch.accelerator.synchronize()
state.buffer.clean_mask_buffer()
torch.accelerator.synchronize()
NixlEPAll2AllManager._last_mask = None
class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
"""