[Feature] Detect all2all peer fault with fault tolerance backend and prevent corrupted output (#43637)

Signed-off-by: fangyuchu <fangyuchu@qq.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
fangyuchu
2026-06-30 09:00:25 -07:00
committed by GitHub
co-authored by Tyler Michael Smith mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent e840f0d3f5
commit 245888ff77
5 changed files with 82 additions and 4 deletions
@@ -261,8 +261,13 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
All2All communication based on DeepEP Low-Latency kernels.
"""
_buffer: Any = None
_mask: torch.Tensor | None = None
_last_mask: torch.Tensor | None = None
def __init__(self, cpu_group, tcp_store_group=None):
super().__init__(cpu_group, tcp_store_group)
self.support_fault_tolerance = False # TODO: set to True when FT is supported.
def _make_all2all_kwargs(
self,
@@ -304,6 +309,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
allow_nvlink_for_low_latency_mode=True,
allow_mnnvl=envs.VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL,
explicitly_destroy=True,
enable_shrink=self.support_fault_tolerance,
)
return kwargs
@@ -319,12 +325,30 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
handle: deep_ep.Buffer = self.handle_cache.get_or_create(
buffer_kwargs, deep_ep.Buffer
)
DeepEPLLAll2AllManager._buffer = handle
return handle
# DeepEP LL uses RDMA so no SMs are used for communication
def max_sms_used(self) -> int | None:
return 0
def query_active_mask(self) -> torch.Tensor:
buf = DeepEPLLAll2AllManager._buffer
assert buf is not None
if DeepEPLLAll2AllManager._mask is None:
DeepEPLLAll2AllManager._mask = torch.zeros(
self.world_size, device="cuda", dtype=torch.int32
)
buf.low_latency_query_mask_buffer(DeepEPLLAll2AllManager._mask)
return DeepEPLLAll2AllManager._mask
def query_fault(self) -> torch.Tensor:
current = self.query_active_mask()
if DeepEPLLAll2AllManager._last_mask is None:
DeepEPLLAll2AllManager._last_mask = torch.zeros_like(current)
has_fault = (current != DeepEPLLAll2AllManager._last_mask).any()
return has_fault
@dataclass
class _NixlEPBufferState:
@@ -341,6 +365,8 @@ class NixlEPAll2AllManager(All2AllManagerBase):
_buffer: _NixlEPBufferState | None = None
_lock = threading.RLock()
_mask: torch.Tensor | None = None
_last_mask: torch.Tensor | None = None
def __init__(self, cpu_group, tcp_store_group=None):
if tcp_store_group is None:
@@ -350,6 +376,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
store=dist.PrefixStore("nixl_ep", cpu_group.get_group_store()),
)
super().__init__(cpu_group, tcp_store_group)
self.support_fault_tolerance = True
self.max_num_ep_ranks = envs.VLLM_NIXL_EP_MAX_NUM_RANKS
@@ -508,6 +535,25 @@ class NixlEPAll2AllManager(All2AllManagerBase):
def max_sms_used(self) -> int | None:
return 0
def query_active_mask(self) -> torch.Tensor:
state = NixlEPAll2AllManager._buffer
assert state is not None
if NixlEPAll2AllManager._mask is None:
NixlEPAll2AllManager._mask = torch.zeros(
self.max_num_ep_ranks, device="cuda", dtype=torch.int32
)
state.buffer.query_mask_buffer(NixlEPAll2AllManager._mask)
return NixlEPAll2AllManager._mask[: state.active_ep_size]
def query_fault(self) -> torch.Tensor:
current = self.query_active_mask()
last = NixlEPAll2AllManager._last_mask
if last is None or last.shape != current.shape:
NixlEPAll2AllManager._last_mask = torch.zeros_like(current)
last = NixlEPAll2AllManager._last_mask
has_fault = (current != last).any()
return has_fault
class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
"""
@@ -62,6 +62,8 @@ class All2AllManagerBase:
in_the_same_node_as(tcp_store_group, source_rank=0)
)
self.support_fault_tolerance = False
def get_handle(self, kwargs):
# get a handle for the all2all communication,
# based on the kwargs.
@@ -102,6 +104,13 @@ class All2AllManagerBase:
# - raise a clear error if extra_tensors is not supported.
raise NotImplementedError
def query_active_mask(self) -> torch.Tensor:
raise NotImplementedError
def query_fault(self) -> torch.Tensor:
"""Returns has_fault scalar."""
raise NotImplementedError
def set_num_sms(self, num_sms: int):
pass