[Feature] Detect all2all peer fault with fault tolerance backend and prevent corrupted output (#43637)
Signed-off-by: fangyuchu <fangyuchu@qq.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
e840f0d3f5
commit
245888ff77
@@ -261,8 +261,13 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
All2All communication based on DeepEP Low-Latency kernels.
|
||||
"""
|
||||
|
||||
_buffer: Any = None
|
||||
_mask: torch.Tensor | None = None
|
||||
_last_mask: torch.Tensor | None = None
|
||||
|
||||
def __init__(self, cpu_group, tcp_store_group=None):
|
||||
super().__init__(cpu_group, tcp_store_group)
|
||||
self.support_fault_tolerance = False # TODO: set to True when FT is supported.
|
||||
|
||||
def _make_all2all_kwargs(
|
||||
self,
|
||||
@@ -304,6 +309,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
allow_nvlink_for_low_latency_mode=True,
|
||||
allow_mnnvl=envs.VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL,
|
||||
explicitly_destroy=True,
|
||||
enable_shrink=self.support_fault_tolerance,
|
||||
)
|
||||
return kwargs
|
||||
|
||||
@@ -319,12 +325,30 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
handle: deep_ep.Buffer = self.handle_cache.get_or_create(
|
||||
buffer_kwargs, deep_ep.Buffer
|
||||
)
|
||||
DeepEPLLAll2AllManager._buffer = handle
|
||||
return handle
|
||||
|
||||
# DeepEP LL uses RDMA so no SMs are used for communication
|
||||
def max_sms_used(self) -> int | None:
|
||||
return 0
|
||||
|
||||
def query_active_mask(self) -> torch.Tensor:
|
||||
buf = DeepEPLLAll2AllManager._buffer
|
||||
assert buf is not None
|
||||
if DeepEPLLAll2AllManager._mask is None:
|
||||
DeepEPLLAll2AllManager._mask = torch.zeros(
|
||||
self.world_size, device="cuda", dtype=torch.int32
|
||||
)
|
||||
buf.low_latency_query_mask_buffer(DeepEPLLAll2AllManager._mask)
|
||||
return DeepEPLLAll2AllManager._mask
|
||||
|
||||
def query_fault(self) -> torch.Tensor:
|
||||
current = self.query_active_mask()
|
||||
if DeepEPLLAll2AllManager._last_mask is None:
|
||||
DeepEPLLAll2AllManager._last_mask = torch.zeros_like(current)
|
||||
has_fault = (current != DeepEPLLAll2AllManager._last_mask).any()
|
||||
return has_fault
|
||||
|
||||
|
||||
@dataclass
|
||||
class _NixlEPBufferState:
|
||||
@@ -341,6 +365,8 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
|
||||
_buffer: _NixlEPBufferState | None = None
|
||||
_lock = threading.RLock()
|
||||
_mask: torch.Tensor | None = None
|
||||
_last_mask: torch.Tensor | None = None
|
||||
|
||||
def __init__(self, cpu_group, tcp_store_group=None):
|
||||
if tcp_store_group is None:
|
||||
@@ -350,6 +376,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
store=dist.PrefixStore("nixl_ep", cpu_group.get_group_store()),
|
||||
)
|
||||
super().__init__(cpu_group, tcp_store_group)
|
||||
self.support_fault_tolerance = True
|
||||
|
||||
self.max_num_ep_ranks = envs.VLLM_NIXL_EP_MAX_NUM_RANKS
|
||||
|
||||
@@ -508,6 +535,25 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
def max_sms_used(self) -> int | None:
|
||||
return 0
|
||||
|
||||
def query_active_mask(self) -> torch.Tensor:
|
||||
state = NixlEPAll2AllManager._buffer
|
||||
assert state is not None
|
||||
if NixlEPAll2AllManager._mask is None:
|
||||
NixlEPAll2AllManager._mask = torch.zeros(
|
||||
self.max_num_ep_ranks, device="cuda", dtype=torch.int32
|
||||
)
|
||||
state.buffer.query_mask_buffer(NixlEPAll2AllManager._mask)
|
||||
return NixlEPAll2AllManager._mask[: state.active_ep_size]
|
||||
|
||||
def query_fault(self) -> torch.Tensor:
|
||||
current = self.query_active_mask()
|
||||
last = NixlEPAll2AllManager._last_mask
|
||||
if last is None or last.shape != current.shape:
|
||||
NixlEPAll2AllManager._last_mask = torch.zeros_like(current)
|
||||
last = NixlEPAll2AllManager._last_mask
|
||||
has_fault = (current != last).any()
|
||||
return has_fault
|
||||
|
||||
|
||||
class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user