[Bugfix][NIXL] Fix multiple handles xfer race (#56104)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nicolò Lucchesi
2026-09-15 04:27:10 +00:00
committed by GitHub
co-authored by Nick Hill
parent caf56ce9da
commit 45779ef99f
5 changed files with 233 additions and 151 deletions
@@ -692,7 +692,7 @@ class NixlBaseConnectorWorker:
# Uses Queue for thread-safe cross-thread coordination with the
# background handshake thread, matching the _ready_requests pattern.
self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue()
self._failed_inflight_recvs: set[ReqId] = set()
self._recv_failures: set[ReqId] = set()
self._pending_recv_notifs: dict[ReqId, list[tuple[str, bytes]]] = {}
# Handshake metadata of this worker for NIXL transfers.
@@ -1212,7 +1212,7 @@ class NixlBaseConnectorWorker:
error=e,
meta=meta,
)
self._handle_failed_transfer(req_id, None)
self._failed_recv_reqs.put(req_id)
fut.add_done_callback(request_ready)
@@ -2618,22 +2618,32 @@ class NixlBaseConnectorWorker:
"""
assert self.transfer_topo is not None
done_sending = self._get_new_notifs()
done_recving = self._pop_done_transfers(self._recving_transfers)
done_recving, newly_failed = self._pop_done_transfers(self._recving_transfers)
if newly_failed:
self._recv_failures.update(newly_failed)
done_sending.update(self._replicated_pcp_done_sending)
self._replicated_pcp_done_sending.clear()
# Drain queue of requests where handshake or transfer setup failed.
failed_recv_reqs = set[ReqId]()
# Process receive failures reported by background threads.
while not self._failed_recv_reqs.empty():
try:
failed_recv_reqs.add(self._failed_recv_reqs.get_nowait())
req_id = self._failed_recv_reqs.get_nowait()
except queue.Empty:
break
self._handle_failed_transfer(req_id, None, self._recv_failures)
failed_recv_reqs: set[ReqId] = set()
if self._recv_failures:
# Keep failure bookkeeping independent of healthy request count.
for req_id in tuple(self._recv_failures):
if req_id not in self._recving_metadata:
self._recv_failures.remove(req_id)
elif req_id not in self._recving_transfers:
self._recv_failures.remove(req_id)
failed_recv_reqs.add(req_id)
done_recving.update(failed_recv_reqs)
# Add failed requests to done_recving for scheduler tracking
# (blocks are already marked invalid, scheduler will handle recompute)
done_recving.update(failed_recv_reqs)
if len(done_sending) > 0 or len(done_recving) > 0:
logger.debug(
"Rank %s, get_finished: %s requests done sending "
@@ -2654,12 +2664,17 @@ class NixlBaseConnectorWorker:
# Skip KV sync and post-processing for failed requests
if req_id in failed_recv_reqs:
self._pending_recv_notifs.pop(req_id, None)
# TODO (NickLucche) handle failed transfer for HMA.
if not self._is_hma_required:
self._invalid_block_ids.put(set(meta.local_block_ids[0]))
logger.warning(
"Skipping KV post-processing for failed request %s",
req_id,
)
continue
self._send_pending_recv_notifs(req_id)
assert meta.remote is not None
if self.use_host_buffer:
self.sync_recved_kv_to_device(req_id, meta)
@@ -2783,22 +2798,26 @@ class NixlBaseConnectorWorker:
new_expiry,
)
def _pop_done_transfers(self, transfers: dict[str, list[int]]) -> set[str]:
"""
Pop completed xfers by checking for DONE state.
def _pop_done_transfers(
self, transfers: dict[str, list[int]]
) -> tuple[set[str], set[str]]:
"""Poll transfers, retaining handles until they can be released.
Args:
transfers: dict of req_id -> list[running_xfer]
transfers: Outstanding handles by request, updated in place.
Returns:
set of req_ids that have all done xfers
Requests with no outstanding handles and requests with observed
failures. Failed requests may still have outstanding handles.
"""
done_req_ids: set[str] = set()
failed_req_ids: set[str] = set()
for req_id, handles in list(transfers.items()):
in_progress = []
for handle in handles:
try:
xfer_state = self.nixl_wrapper.check_xfer_state(handle)
if xfer_state == "DONE":
# Get telemetry from NIXL
res = self.nixl_wrapper.get_xfer_telemetry(handle)
self.xfer_stats.record_transfer(res)
self.nixl_wrapper.release_xfer_handle(handle)
@@ -2807,58 +2826,52 @@ class NixlBaseConnectorWorker:
else:
self._log_failure(
failure_type="transfer_failed",
msg="Deferring request completion until its last "
"xfer is terminal",
req_id=req_id,
xfer_state=xfer_state,
)
self._handle_failed_transfer(req_id, handle)
if not self._handle_failed_transfer(
req_id, handle, failed_req_ids
):
in_progress.append(handle)
except Exception as e:
self._log_failure(
failure_type="transfer_exception",
msg="Handle is unpollable; treating it as terminal",
req_id=req_id,
error=e,
)
self._handle_failed_transfer(req_id, handle)
if not self._handle_failed_transfer(req_id, handle, failed_req_ids):
in_progress.append(handle)
if not in_progress:
del transfers[req_id]
done_req_ids.add(req_id)
if req_id in self._failed_inflight_recvs:
self._failed_inflight_recvs.remove(req_id)
self._report_failed_recv(req_id)
else:
self._send_pending_recv_notifs(req_id)
del transfers[req_id]
else:
transfers[req_id] = in_progress
return done_req_ids
return done_req_ids, failed_req_ids
def _handle_failed_transfer(self, req_id: str, handle: int | None):
"""
Handle a failed transfer by marking all (logical) blocks as invalid and
recording the failure.
Args:
req_id: The request ID.
handle: The transfer handle.
"""
# A sibling READ may still be writing these blocks. Retain metadata
# and defer invalidation until every handle is terminal.
self._pending_recv_notifs.pop(req_id, None)
if req_id in self._recving_transfers:
self._failed_inflight_recvs.add(req_id)
else:
self._report_failed_recv(req_id)
if handle is not None:
def _try_release_xfer_handle(self, req_id: str, handle: int) -> bool:
"""Release a handle, returning False if the caller must retain it."""
try:
self.nixl_wrapper.release_xfer_handle(handle)
self.xfer_stats.record_failed_transfer()
except Exception as e:
# A status error does not guarantee that the backend stopped DMA.
self._log_failure(
failure_type="transfer_release_failed",
msg="Retaining handle and blocks until release succeeds",
req_id=req_id,
error=e,
)
return False
return True
def _report_failed_recv(self, req_id: str) -> None:
if (meta := self._recving_metadata.get(req_id)) is not None:
if not self._is_hma_required:
self._invalid_block_ids.put(set(meta.local_block_ids[0]))
self._failed_recv_reqs.put(req_id)
def _handle_failed_transfer(
self, req_id: str, handle: int | None, failed_req_ids: set[str] | None = None
) -> bool:
"""Record a failure and release its handle, returning False to retain it."""
self.xfer_stats.record_failed_transfer()
if failed_req_ids is not None:
failed_req_ids.add(req_id)
return handle is None or self._try_release_xfer_handle(req_id, handle)
def _send_pending_recv_notifs(self, req_id: str) -> None:
"""Send notifications deferred by split DRAM/VRAM reads."""
@@ -149,7 +149,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
engine_id,
)
self.xfer_stats.record_kv_expired_req()
self._handle_failed_transfer(req_id, None)
self._handle_failed_transfer(req_id, None, self._recv_failures)
return
if any(len(group) > 0 for group in meta.local_block_ids):
@@ -493,16 +493,19 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
self._recving_transfers[request_id].append(handle)
return True
except Exception as e:
# mark all (logical) blocks for this request as invalid
self._log_failure(
failure_type="transfer_setup_failed",
req_id=request_id,
msg="Marking blocks as invalid",
msg="Deferring failure reporting until outstanding transfers finish",
error=e,
dst_engine_id=dst_engine_id,
remote_rank=remote_rank,
)
self._handle_failed_transfer(request_id, handle)
if not self._handle_failed_transfer(
request_id, handle, self._recv_failures
):
assert handle is not None
self._recving_transfers[request_id].append(handle)
return False
def _read_blocks_mixed(
@@ -544,7 +547,8 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
)
except Exception:
for handle in handles:
self.nixl_wrapper.release_xfer_handle(handle)
if not self._try_release_xfer_handle(request_id, handle):
self._recving_transfers[request_id].append(handle)
raise
self._pending_recv_notifs.setdefault(request_id, []).append(
@@ -555,7 +559,8 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
self.nixl_wrapper.transfer(handle)
except Exception:
for unstarted in handles[i:]:
self.nixl_wrapper.release_xfer_handle(unstarted)
if not self._try_release_xfer_handle(request_id, unstarted):
self._recving_transfers[request_id].append(unstarted)
raise
self._recving_transfers[request_id].append(handle)
@@ -341,7 +341,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
self._log_failure(
failure_type="push_reg_handshake_failed", req_id=rid, error=e
)
self._handle_failed_transfer(rid, None)
self._failed_recv_reqs.put(rid)
return
# Re-queue for the writer to send now that the handshake is done.
self._reg_send_inbox.put((rid, rd))
@@ -363,7 +363,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
engine_id,
req_id,
)
self._handle_failed_transfer(req_id, None)
self._failed_recv_reqs.put(req_id)
return
for rank, agent_name in agents.items():
try:
@@ -722,9 +722,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
# don't have a ``_recving_metadata`` entry to invalidate, so
# we just release the handle and let the engine reschedule
# via the lease / watchdog.
if handle is not None:
self.nixl_wrapper.release_xfer_handle(handle)
self.xfer_stats.record_failed_transfer()
if not self._handle_failed_transfer(request_id, handle):
return handle
return None
# --- Notification handling on engine main thread ------------------ #
@@ -803,7 +802,16 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
# ``_pop_done_transfers`` mutates ``_sending_transfers``; the
# writer thread also appends to it, so guard the pop.
with self._sending_transfers_lock:
done_pushing = self._pop_done_transfers(self._sending_transfers)
done_pushing, failed_pushing = self._pop_done_transfers(
self._sending_transfers
)
# A failed send must never be reported as done: its blocks
# are freed via the lease / watchdog instead.
done_pushing = {
req_id
for req_id in done_pushing - failed_pushing
if req_id in self._recving_metadata
}
for req_id in done_pushing:
self._reqs_to_send.pop(req_id, None)
self._reqs_to_process.discard(req_id)