[Bugfix][NIXL] Fix multiple handles xfer race (#56104)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
caf56ce9da
commit
45779ef99f
@@ -692,7 +692,7 @@ class NixlBaseConnectorWorker:
|
||||
# Uses Queue for thread-safe cross-thread coordination with the
|
||||
# background handshake thread, matching the _ready_requests pattern.
|
||||
self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue()
|
||||
self._failed_inflight_recvs: set[ReqId] = set()
|
||||
self._recv_failures: set[ReqId] = set()
|
||||
self._pending_recv_notifs: dict[ReqId, list[tuple[str, bytes]]] = {}
|
||||
|
||||
# Handshake metadata of this worker for NIXL transfers.
|
||||
@@ -1212,7 +1212,7 @@ class NixlBaseConnectorWorker:
|
||||
error=e,
|
||||
meta=meta,
|
||||
)
|
||||
self._handle_failed_transfer(req_id, None)
|
||||
self._failed_recv_reqs.put(req_id)
|
||||
|
||||
fut.add_done_callback(request_ready)
|
||||
|
||||
@@ -2618,22 +2618,32 @@ class NixlBaseConnectorWorker:
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
done_sending = self._get_new_notifs()
|
||||
done_recving = self._pop_done_transfers(self._recving_transfers)
|
||||
done_recving, newly_failed = self._pop_done_transfers(self._recving_transfers)
|
||||
if newly_failed:
|
||||
self._recv_failures.update(newly_failed)
|
||||
|
||||
done_sending.update(self._replicated_pcp_done_sending)
|
||||
self._replicated_pcp_done_sending.clear()
|
||||
|
||||
# Drain queue of requests where handshake or transfer setup failed.
|
||||
failed_recv_reqs = set[ReqId]()
|
||||
# Process receive failures reported by background threads.
|
||||
while not self._failed_recv_reqs.empty():
|
||||
try:
|
||||
failed_recv_reqs.add(self._failed_recv_reqs.get_nowait())
|
||||
req_id = self._failed_recv_reqs.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self._handle_failed_transfer(req_id, None, self._recv_failures)
|
||||
|
||||
failed_recv_reqs: set[ReqId] = set()
|
||||
if self._recv_failures:
|
||||
# Keep failure bookkeeping independent of healthy request count.
|
||||
for req_id in tuple(self._recv_failures):
|
||||
if req_id not in self._recving_metadata:
|
||||
self._recv_failures.remove(req_id)
|
||||
elif req_id not in self._recving_transfers:
|
||||
self._recv_failures.remove(req_id)
|
||||
failed_recv_reqs.add(req_id)
|
||||
done_recving.update(failed_recv_reqs)
|
||||
|
||||
# Add failed requests to done_recving for scheduler tracking
|
||||
# (blocks are already marked invalid, scheduler will handle recompute)
|
||||
done_recving.update(failed_recv_reqs)
|
||||
if len(done_sending) > 0 or len(done_recving) > 0:
|
||||
logger.debug(
|
||||
"Rank %s, get_finished: %s requests done sending "
|
||||
@@ -2654,12 +2664,17 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
# Skip KV sync and post-processing for failed requests
|
||||
if req_id in failed_recv_reqs:
|
||||
self._pending_recv_notifs.pop(req_id, None)
|
||||
# TODO (NickLucche) handle failed transfer for HMA.
|
||||
if not self._is_hma_required:
|
||||
self._invalid_block_ids.put(set(meta.local_block_ids[0]))
|
||||
logger.warning(
|
||||
"Skipping KV post-processing for failed request %s",
|
||||
req_id,
|
||||
)
|
||||
continue
|
||||
|
||||
self._send_pending_recv_notifs(req_id)
|
||||
assert meta.remote is not None
|
||||
if self.use_host_buffer:
|
||||
self.sync_recved_kv_to_device(req_id, meta)
|
||||
@@ -2783,22 +2798,26 @@ class NixlBaseConnectorWorker:
|
||||
new_expiry,
|
||||
)
|
||||
|
||||
def _pop_done_transfers(self, transfers: dict[str, list[int]]) -> set[str]:
|
||||
"""
|
||||
Pop completed xfers by checking for DONE state.
|
||||
def _pop_done_transfers(
|
||||
self, transfers: dict[str, list[int]]
|
||||
) -> tuple[set[str], set[str]]:
|
||||
"""Poll transfers, retaining handles until they can be released.
|
||||
|
||||
Args:
|
||||
transfers: dict of req_id -> list[running_xfer]
|
||||
transfers: Outstanding handles by request, updated in place.
|
||||
|
||||
Returns:
|
||||
set of req_ids that have all done xfers
|
||||
Requests with no outstanding handles and requests with observed
|
||||
failures. Failed requests may still have outstanding handles.
|
||||
"""
|
||||
done_req_ids: set[str] = set()
|
||||
failed_req_ids: set[str] = set()
|
||||
for req_id, handles in list(transfers.items()):
|
||||
in_progress = []
|
||||
for handle in handles:
|
||||
try:
|
||||
xfer_state = self.nixl_wrapper.check_xfer_state(handle)
|
||||
if xfer_state == "DONE":
|
||||
# Get telemetry from NIXL
|
||||
res = self.nixl_wrapper.get_xfer_telemetry(handle)
|
||||
self.xfer_stats.record_transfer(res)
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
@@ -2807,58 +2826,52 @@ class NixlBaseConnectorWorker:
|
||||
else:
|
||||
self._log_failure(
|
||||
failure_type="transfer_failed",
|
||||
msg="Deferring request completion until its last "
|
||||
"xfer is terminal",
|
||||
req_id=req_id,
|
||||
xfer_state=xfer_state,
|
||||
)
|
||||
self._handle_failed_transfer(req_id, handle)
|
||||
if not self._handle_failed_transfer(
|
||||
req_id, handle, failed_req_ids
|
||||
):
|
||||
in_progress.append(handle)
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="transfer_exception",
|
||||
msg="Handle is unpollable; treating it as terminal",
|
||||
req_id=req_id,
|
||||
error=e,
|
||||
)
|
||||
self._handle_failed_transfer(req_id, handle)
|
||||
if not self._handle_failed_transfer(req_id, handle, failed_req_ids):
|
||||
in_progress.append(handle)
|
||||
|
||||
if not in_progress:
|
||||
del transfers[req_id]
|
||||
done_req_ids.add(req_id)
|
||||
if req_id in self._failed_inflight_recvs:
|
||||
self._failed_inflight_recvs.remove(req_id)
|
||||
self._report_failed_recv(req_id)
|
||||
else:
|
||||
self._send_pending_recv_notifs(req_id)
|
||||
del transfers[req_id]
|
||||
else:
|
||||
transfers[req_id] = in_progress
|
||||
return done_req_ids
|
||||
return done_req_ids, failed_req_ids
|
||||
|
||||
def _handle_failed_transfer(self, req_id: str, handle: int | None):
|
||||
"""
|
||||
Handle a failed transfer by marking all (logical) blocks as invalid and
|
||||
recording the failure.
|
||||
|
||||
Args:
|
||||
req_id: The request ID.
|
||||
handle: The transfer handle.
|
||||
"""
|
||||
# A sibling READ may still be writing these blocks. Retain metadata
|
||||
# and defer invalidation until every handle is terminal.
|
||||
self._pending_recv_notifs.pop(req_id, None)
|
||||
if req_id in self._recving_transfers:
|
||||
self._failed_inflight_recvs.add(req_id)
|
||||
else:
|
||||
self._report_failed_recv(req_id)
|
||||
if handle is not None:
|
||||
def _try_release_xfer_handle(self, req_id: str, handle: int) -> bool:
|
||||
"""Release a handle, returning False if the caller must retain it."""
|
||||
try:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
except Exception as e:
|
||||
# A status error does not guarantee that the backend stopped DMA.
|
||||
self._log_failure(
|
||||
failure_type="transfer_release_failed",
|
||||
msg="Retaining handle and blocks until release succeeds",
|
||||
req_id=req_id,
|
||||
error=e,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _report_failed_recv(self, req_id: str) -> None:
|
||||
if (meta := self._recving_metadata.get(req_id)) is not None:
|
||||
if not self._is_hma_required:
|
||||
self._invalid_block_ids.put(set(meta.local_block_ids[0]))
|
||||
self._failed_recv_reqs.put(req_id)
|
||||
def _handle_failed_transfer(
|
||||
self, req_id: str, handle: int | None, failed_req_ids: set[str] | None = None
|
||||
) -> bool:
|
||||
"""Record a failure and release its handle, returning False to retain it."""
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
if failed_req_ids is not None:
|
||||
failed_req_ids.add(req_id)
|
||||
return handle is None or self._try_release_xfer_handle(req_id, handle)
|
||||
|
||||
def _send_pending_recv_notifs(self, req_id: str) -> None:
|
||||
"""Send notifications deferred by split DRAM/VRAM reads."""
|
||||
|
||||
@@ -149,7 +149,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
engine_id,
|
||||
)
|
||||
self.xfer_stats.record_kv_expired_req()
|
||||
self._handle_failed_transfer(req_id, None)
|
||||
self._handle_failed_transfer(req_id, None, self._recv_failures)
|
||||
return
|
||||
|
||||
if any(len(group) > 0 for group in meta.local_block_ids):
|
||||
@@ -493,16 +493,19 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
return True
|
||||
except Exception as e:
|
||||
# mark all (logical) blocks for this request as invalid
|
||||
self._log_failure(
|
||||
failure_type="transfer_setup_failed",
|
||||
req_id=request_id,
|
||||
msg="Marking blocks as invalid",
|
||||
msg="Deferring failure reporting until outstanding transfers finish",
|
||||
error=e,
|
||||
dst_engine_id=dst_engine_id,
|
||||
remote_rank=remote_rank,
|
||||
)
|
||||
self._handle_failed_transfer(request_id, handle)
|
||||
if not self._handle_failed_transfer(
|
||||
request_id, handle, self._recv_failures
|
||||
):
|
||||
assert handle is not None
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
return False
|
||||
|
||||
def _read_blocks_mixed(
|
||||
@@ -544,7 +547,8 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
)
|
||||
except Exception:
|
||||
for handle in handles:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
if not self._try_release_xfer_handle(request_id, handle):
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
raise
|
||||
|
||||
self._pending_recv_notifs.setdefault(request_id, []).append(
|
||||
@@ -555,7 +559,8 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
self.nixl_wrapper.transfer(handle)
|
||||
except Exception:
|
||||
for unstarted in handles[i:]:
|
||||
self.nixl_wrapper.release_xfer_handle(unstarted)
|
||||
if not self._try_release_xfer_handle(request_id, unstarted):
|
||||
self._recving_transfers[request_id].append(unstarted)
|
||||
raise
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
|
||||
|
||||
@@ -341,7 +341,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
self._log_failure(
|
||||
failure_type="push_reg_handshake_failed", req_id=rid, error=e
|
||||
)
|
||||
self._handle_failed_transfer(rid, None)
|
||||
self._failed_recv_reqs.put(rid)
|
||||
return
|
||||
# Re-queue for the writer to send now that the handshake is done.
|
||||
self._reg_send_inbox.put((rid, rd))
|
||||
@@ -363,7 +363,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
engine_id,
|
||||
req_id,
|
||||
)
|
||||
self._handle_failed_transfer(req_id, None)
|
||||
self._failed_recv_reqs.put(req_id)
|
||||
return
|
||||
for rank, agent_name in agents.items():
|
||||
try:
|
||||
@@ -722,9 +722,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
# don't have a ``_recving_metadata`` entry to invalidate, so
|
||||
# we just release the handle and let the engine reschedule
|
||||
# via the lease / watchdog.
|
||||
if handle is not None:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
if not self._handle_failed_transfer(request_id, handle):
|
||||
return handle
|
||||
return None
|
||||
|
||||
# --- Notification handling on engine main thread ------------------ #
|
||||
@@ -803,7 +802,16 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
# ``_pop_done_transfers`` mutates ``_sending_transfers``; the
|
||||
# writer thread also appends to it, so guard the pop.
|
||||
with self._sending_transfers_lock:
|
||||
done_pushing = self._pop_done_transfers(self._sending_transfers)
|
||||
done_pushing, failed_pushing = self._pop_done_transfers(
|
||||
self._sending_transfers
|
||||
)
|
||||
# A failed send must never be reported as done: its blocks
|
||||
# are freed via the lease / watchdog instead.
|
||||
done_pushing = {
|
||||
req_id
|
||||
for req_id in done_pushing - failed_pushing
|
||||
if req_id in self._recving_metadata
|
||||
}
|
||||
for req_id in done_pushing:
|
||||
self._reqs_to_send.pop(req_id, None)
|
||||
self._reqs_to_process.discard(req_id)
|
||||
|
||||
Reference in New Issue
Block a user