diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 298886d160..39fb0bb54f 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -4,6 +4,7 @@ import contextlib import inspect import os +import queue import tempfile import textwrap import time @@ -1952,17 +1953,43 @@ def test_mixed_memory_local_descriptors_split_by_memory_type(): assert memory_types == ["DRAM", "VRAM"] -def test_mixed_memory_read_notifies_after_both_transfers_finish(): +@pytest.fixture +def recv_worker(): + """Receive lifecycle state without distributed or device initialization.""" worker = object.__new__(NixlConnectorWorker) + worker.transfer_topo = MagicMock() + worker.transfer_topo.block_size_ratio.return_value = 1 + worker.transfer_topo.get_engine_info.return_value = SimpleNamespace( + remote_block_size=16, remote_physical_blocks_per_logical=1 + ) + worker._physical_blocks_per_logical_kv_block = 1 + worker._recving_metadata = {"request": MagicMock(local_block_ids=([1, 2, 3],))} + worker._recving_transfers = defaultdict(list) + worker._failed_recv_reqs = queue.Queue() + worker._recv_failures = set() + worker._replicated_pcp_done_sending = set() + worker._invalid_block_ids = queue.Queue() + worker._pending_recv_notifs = {} + worker._reqs_to_send = {} + worker._replicated_pcp_done_sending = set() + worker._is_hma_required = False + worker._has_mamba = False + worker.use_host_buffer = False + worker.enable_permute_local_kv = False + worker.enable_heterogeneous_attn_post_process = False + worker.tp_rank = 0 + worker._log_failure = MagicMock() # type: ignore[method-assign] + worker.xfer_stats = NixlKVConnectorStats() + worker.nixl_wrapper = MagicMock() + worker.nixl_wrapper.get_xfer_telemetry.return_value = get_default_xfer_telemetry() + return worker + + +def test_mixed_memory_read_notifies_after_both_transfers_finish(recv_worker): + worker = recv_worker worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])} worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])} worker._dram_src_handles_by_block_size = {16: 10} - worker._recving_metadata = {"request": MagicMock()} - worker._recving_transfers = defaultdict(list) - worker._pending_recv_notifs = {} - worker._failed_inflight_recvs = set() - worker.xfer_stats = MagicMock() - worker.nixl_wrapper = MagicMock() worker.nixl_wrapper.make_prepped_xfer.side_effect = [101, 102] worker.nixl_wrapper.check_xfer_state.return_value = "DONE" @@ -1989,27 +2016,20 @@ def test_mixed_memory_read_notifies_after_both_transfers_finish(): np.testing.assert_array_equal(device_read.args[4], [7]) worker.nixl_wrapper.send_notif.assert_not_called() - assert worker._pop_done_transfers(worker._recving_transfers) == {"request"} + assert worker.get_finished() == (set(), {"request"}) worker.nixl_wrapper.send_notif.assert_called_once_with( "prefill", notif_msg=b"request:1" ) -def test_mixed_memory_read_failure_does_not_notify_producer(): +def test_mixed_memory_read_failure_does_not_notify_producer(recv_worker): """A failed half of a split READ must suppress its success notification.""" - worker = object.__new__(NixlConnectorWorker) - worker._recving_metadata = {"request": MagicMock()} + worker = recv_worker worker._recving_transfers = {"request": [101, 102]} worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]} - worker._failed_inflight_recvs = set() - worker._is_hma_required = True - worker._failed_recv_reqs = MagicMock() - worker._log_failure = MagicMock() # type: ignore[method-assign] - worker.xfer_stats = MagicMock() - worker.nixl_wrapper = MagicMock() worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "DONE"] - worker._pop_done_transfers(worker._recving_transfers) + assert worker.get_finished() == (set(), {"request"}) assert "request" not in worker._pending_recv_notifs worker.nixl_wrapper.send_notif.assert_not_called() @@ -2969,13 +2989,10 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init): # Wait for handshake to fail time.sleep(0.3) - # Check that blocks were marked invalid - invalid_blocks = connector.get_block_ids_with_load_errors() - assert invalid_blocks == {1, 2, 3} - - # Check that request appears in get_finished + assert connector.get_block_ids_with_load_errors() == set() _, done_recving = connector.get_finished(finished_req_ids=set()) assert request_id in done_recving + assert connector.get_block_ids_with_load_errors() == {1, 2, 3} @patch( @@ -3026,6 +3043,7 @@ def test_transfer_setup_failure_returns_finished( time.sleep(0.1) connector.start_load_kv(dummy_ctx) + assert connector.get_block_ids_with_load_errors() == set() results = connector.get_transfer_results(finished_req_ids=set()) assert request_id in results.finished_recving assert results.failed_recving == {request_id} @@ -3248,74 +3266,49 @@ def test_failed_request_skips_kv_postprocessing( assert invalid_blocks == {1, 2, 3} -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FailingNixlWrapper, -) -def test_handles_failing_in_separate_polls_do_not_kill_the_engine( - default_vllm_config, dist_init -): - """Failures across polls report once, after the last writer is terminal.""" - vllm_config = create_vllm_config() - connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0 - ) - worker = connector.connector_worker +@pytest.mark.parametrize("sibling_state", ["ERR", "DONE"]) +def test_recv_failure_waits_for_sibling_transfer(recv_worker, sibling_state): + """Failure must not release blocks while a sibling can still write to them.""" + worker = recv_worker + worker._recving_transfers["request"] = [101, 102] + worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]} + worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "PROC", sibling_state] - request_id = "test_two_handles_failing_in_separate_polls" - metadata = NixlConnectorMetadata() - metadata.add_new_req_to_recv( - request_id=request_id, - local_block_ids=([1, 2, 3],), - kv_transfer_params={ - "remote_block_ids": ([4, 5, 6],), - "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - "remote_request_id": f"prefill-{request_id}", - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": 1, - }, - ) - connector.bind_connector_metadata(metadata) - dummy_ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={}) - connector.start_load_kv(dummy_ctx) - connector.bind_connector_metadata(NixlConnectorMetadata()) - time.sleep(0.1) - connector.start_load_kv(dummy_ctx) + results = worker.get_transfer_results() + assert results.finished_recving == results.failed_recving == set() + assert worker.get_block_ids_with_load_errors() == set() + assert "request" in worker._recving_metadata + worker.nixl_wrapper.release_xfer_handle.assert_called_once_with(101) - # Give the request a second handle, as a second remote rank would. - handles = worker._recving_transfers[request_id] - assert len(handles) == 1 - first, second = handles[0], object() - worker._recving_transfers[request_id] = [first, second] + results = worker.get_transfer_results() + assert results.finished_recving == results.failed_recving == {"request"} + assert worker.get_block_ids_with_load_errors() == {1, 2, 3} + worker.nixl_wrapper.send_notif.assert_not_called() - # Poll 1: the first handle has failed, the second is still running. - def first_failed(handle): - return "ERR" if handle is first else "PROC" + results = worker.get_transfer_results() + assert results.finished_recving == results.failed_recving == set() + assert worker.get_block_ids_with_load_errors() == set() - with patch.object( - worker.nixl_wrapper, "check_xfer_state", side_effect=first_failed - ): - _, done_recving = connector.get_finished(finished_req_ids=set()) - assert request_id not in done_recving - assert request_id in worker._recving_metadata - assert worker._recving_transfers[request_id] == [second] - assert connector.get_block_ids_with_load_errors() == set() +def test_recv_failure_waits_for_unpollable_handle_to_be_released(recv_worker): + """A polling exception and failed release do not prove that DMA has stopped.""" + worker = recv_worker + worker._recving_transfers["request"] = [101] + worker.nixl_wrapper.check_xfer_state.side_effect = [RuntimeError("poll"), "DONE"] + worker.nixl_wrapper.release_xfer_handle.side_effect = [ + RuntimeError("release"), + None, + ] - # Only the last terminal handle permits the scheduler to reuse the blocks. - with patch.object(worker.nixl_wrapper, "check_xfer_state", return_value="ERR"): - _, done_recving = connector.get_finished(finished_req_ids=set()) + results = worker.get_transfer_results() + assert results.finished_recving == results.failed_recving == set() + assert worker.get_block_ids_with_load_errors() == set() + assert "request" in worker._recving_metadata - assert done_recving == {request_id} - assert request_id not in worker._recving_transfers - assert request_id not in worker._recving_metadata - assert connector.get_block_ids_with_load_errors() == {1, 2, 3} - _, done_recving = connector.get_finished(finished_req_ids=set()) - assert done_recving == set() + results = worker.get_transfer_results() + assert results.finished_recving == results.failed_recving == {"request"} + assert worker.get_block_ids_with_load_errors() == {1, 2, 3} + assert worker.nixl_wrapper.release_xfer_handle.call_count == 2 def _set_test_speculative_config( diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index 2c7f0cb866..423b0e9dec 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -351,7 +351,7 @@ class _StubWriterWorker(NixlPushConnectorWorker): # Base worker fields touched by start_load_kv / _get_new_notifs. w._recving_metadata = {} w._pending_recv_notifs = {} - w._failed_inflight_recvs = set() + w._recv_failures = set() w._recving_transfers = defaultdict(list) w._reqs_to_process = set() w._reqs_to_send = {} @@ -877,6 +877,68 @@ class TestPushWriterNotifs: assert evicted == ["req-done"] assert w._push_writer_wake.is_set() + @staticmethod + def _pollable_worker() -> _StubWriterWorker: + """Stub worker with the extra base-worker state the real + ``get_transfer_results`` needs to poll ``_sending_transfers``.""" + w = _StubWriterWorker.fresh() + w.transfer_topo = MagicMock() + w.nixl_wrapper = MagicMock() + w.xfer_stats = MagicMock() + w._log_failure = MagicMock() # type: ignore[method-assign] + w._failed_recv_reqs = queue.Queue() + w._invalid_block_ids = queue.Queue() + w._pending_recv_notifs = {} + w._replicated_pcp_done_sending = set() + w.use_host_buffer = False + return w + + def _make_sending_req(self, w: _StubWriterWorker) -> str: + request_id = "req-send-1" + w._sending_transfers[request_id] = [101, 102] + w._reqs_to_send[request_id] = time.perf_counter() + 60 + w._reqs_to_process.add(request_id) + return request_id + + def test_failed_push_send_is_not_reported_done_sending(self): + """A failed P-side WRITE must not surface as finished sending. + + Reporting it would free P-side blocks as if the KV had been + delivered while D still waits for the data; the request must be + left to the lease / watchdog instead. + """ + w = self._pollable_worker() + request_id = self._make_sending_req(w) + w.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "DONE"] + + results = w.get_transfer_results() + + assert request_id not in results.finished_sending + # Lease tracking is preserved so the watchdog can reschedule. + assert request_id in w._reqs_to_send + assert request_id in w._reqs_to_process + # Both handles were still cleaned up. + assert request_id not in w._sending_transfers + + def test_completed_push_send_waits_for_consumer_notif(self): + """P-side sends are completed by consumer notifs / lease expiry, + not by local WRITE-handle completion.""" + w = self._pollable_worker() + request_id = self._make_sending_req(w) + w.nixl_wrapper.check_xfer_state.return_value = "DONE" + + results = w.get_transfer_results() + + assert request_id not in results.finished_sending + assert request_id in w._reqs_to_send + assert request_id not in w._sending_transfers + + # The consumer notif is what completes the send. + w._pending_completion_notifs.put(f"{request_id}:1".encode()) + results = w.get_transfer_results() + assert request_id in results.finished_sending + assert request_id not in w._reqs_to_send + # ----------------------------------------------------------------- # # Negative / error-path tests # @@ -1079,7 +1141,7 @@ class TestPushWriterNegative: # Wake set so the writer drains NIXL notifs even when idle. assert w._push_writer_wake.is_set() - def test_failed_push_transfer_finishes_sending(self): + def test_failed_push_transfer_does_not_evict_writer_state(self): w = _StubWriterWorker.fresh() w.nixl_wrapper = MagicMock() w.nixl_wrapper.check_xfer_state.return_value = "ERR" @@ -1096,11 +1158,12 @@ class TestPushWriterNegative: ): results = w.get_transfer_results() - assert results.finished_sending == {"req-failed"} + assert results.finished_sending == set() assert results.finished_recving == set() assert "req-failed" not in w._sending_transfers - assert "req-failed" not in w._reqs_to_send - assert "req-failed" not in w._reqs_to_process + assert "req-failed" in w._reqs_to_send + assert "req-failed" in w._reqs_to_process + assert w._evict_finished_inbox.empty() w.nixl_wrapper.release_xfer_handle.assert_called_once_with(17) def test_get_new_notifs_unknown_request_is_logged_and_skipped(self, caplog): diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 58ab5850ea..6f8f1e55dc 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -692,7 +692,7 @@ class NixlBaseConnectorWorker: # Uses Queue for thread-safe cross-thread coordination with the # background handshake thread, matching the _ready_requests pattern. self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue() - self._failed_inflight_recvs: set[ReqId] = set() + self._recv_failures: set[ReqId] = set() self._pending_recv_notifs: dict[ReqId, list[tuple[str, bytes]]] = {} # Handshake metadata of this worker for NIXL transfers. @@ -1212,7 +1212,7 @@ class NixlBaseConnectorWorker: error=e, meta=meta, ) - self._handle_failed_transfer(req_id, None) + self._failed_recv_reqs.put(req_id) fut.add_done_callback(request_ready) @@ -2618,22 +2618,32 @@ class NixlBaseConnectorWorker: """ assert self.transfer_topo is not None done_sending = self._get_new_notifs() - done_recving = self._pop_done_transfers(self._recving_transfers) + done_recving, newly_failed = self._pop_done_transfers(self._recving_transfers) + if newly_failed: + self._recv_failures.update(newly_failed) done_sending.update(self._replicated_pcp_done_sending) self._replicated_pcp_done_sending.clear() - # Drain queue of requests where handshake or transfer setup failed. - failed_recv_reqs = set[ReqId]() + # Process receive failures reported by background threads. while not self._failed_recv_reqs.empty(): try: - failed_recv_reqs.add(self._failed_recv_reqs.get_nowait()) + req_id = self._failed_recv_reqs.get_nowait() except queue.Empty: break + self._handle_failed_transfer(req_id, None, self._recv_failures) + + failed_recv_reqs: set[ReqId] = set() + if self._recv_failures: + # Keep failure bookkeeping independent of healthy request count. + for req_id in tuple(self._recv_failures): + if req_id not in self._recving_metadata: + self._recv_failures.remove(req_id) + elif req_id not in self._recving_transfers: + self._recv_failures.remove(req_id) + failed_recv_reqs.add(req_id) + done_recving.update(failed_recv_reqs) - # Add failed requests to done_recving for scheduler tracking - # (blocks are already marked invalid, scheduler will handle recompute) - done_recving.update(failed_recv_reqs) if len(done_sending) > 0 or len(done_recving) > 0: logger.debug( "Rank %s, get_finished: %s requests done sending " @@ -2654,12 +2664,17 @@ class NixlBaseConnectorWorker: # Skip KV sync and post-processing for failed requests if req_id in failed_recv_reqs: + self._pending_recv_notifs.pop(req_id, None) + # TODO (NickLucche) handle failed transfer for HMA. + if not self._is_hma_required: + self._invalid_block_ids.put(set(meta.local_block_ids[0])) logger.warning( "Skipping KV post-processing for failed request %s", req_id, ) continue + self._send_pending_recv_notifs(req_id) assert meta.remote is not None if self.use_host_buffer: self.sync_recved_kv_to_device(req_id, meta) @@ -2783,22 +2798,26 @@ class NixlBaseConnectorWorker: new_expiry, ) - def _pop_done_transfers(self, transfers: dict[str, list[int]]) -> set[str]: - """ - Pop completed xfers by checking for DONE state. + def _pop_done_transfers( + self, transfers: dict[str, list[int]] + ) -> tuple[set[str], set[str]]: + """Poll transfers, retaining handles until they can be released. + Args: - transfers: dict of req_id -> list[running_xfer] + transfers: Outstanding handles by request, updated in place. + Returns: - set of req_ids that have all done xfers + Requests with no outstanding handles and requests with observed + failures. Failed requests may still have outstanding handles. """ done_req_ids: set[str] = set() + failed_req_ids: set[str] = set() for req_id, handles in list(transfers.items()): in_progress = [] for handle in handles: try: xfer_state = self.nixl_wrapper.check_xfer_state(handle) if xfer_state == "DONE": - # Get telemetry from NIXL res = self.nixl_wrapper.get_xfer_telemetry(handle) self.xfer_stats.record_transfer(res) self.nixl_wrapper.release_xfer_handle(handle) @@ -2807,58 +2826,52 @@ class NixlBaseConnectorWorker: else: self._log_failure( failure_type="transfer_failed", - msg="Deferring request completion until its last " - "xfer is terminal", req_id=req_id, xfer_state=xfer_state, ) - self._handle_failed_transfer(req_id, handle) + if not self._handle_failed_transfer( + req_id, handle, failed_req_ids + ): + in_progress.append(handle) except Exception as e: self._log_failure( failure_type="transfer_exception", - msg="Handle is unpollable; treating it as terminal", req_id=req_id, error=e, ) - self._handle_failed_transfer(req_id, handle) + if not self._handle_failed_transfer(req_id, handle, failed_req_ids): + in_progress.append(handle) if not in_progress: - del transfers[req_id] done_req_ids.add(req_id) - if req_id in self._failed_inflight_recvs: - self._failed_inflight_recvs.remove(req_id) - self._report_failed_recv(req_id) - else: - self._send_pending_recv_notifs(req_id) + del transfers[req_id] else: transfers[req_id] = in_progress - return done_req_ids + return done_req_ids, failed_req_ids - def _handle_failed_transfer(self, req_id: str, handle: int | None): - """ - Handle a failed transfer by marking all (logical) blocks as invalid and - recording the failure. - - Args: - req_id: The request ID. - handle: The transfer handle. - """ - # A sibling READ may still be writing these blocks. Retain metadata - # and defer invalidation until every handle is terminal. - self._pending_recv_notifs.pop(req_id, None) - if req_id in self._recving_transfers: - self._failed_inflight_recvs.add(req_id) - else: - self._report_failed_recv(req_id) - if handle is not None: + def _try_release_xfer_handle(self, req_id: str, handle: int) -> bool: + """Release a handle, returning False if the caller must retain it.""" + try: self.nixl_wrapper.release_xfer_handle(handle) - self.xfer_stats.record_failed_transfer() + except Exception as e: + # A status error does not guarantee that the backend stopped DMA. + self._log_failure( + failure_type="transfer_release_failed", + msg="Retaining handle and blocks until release succeeds", + req_id=req_id, + error=e, + ) + return False + return True - def _report_failed_recv(self, req_id: str) -> None: - if (meta := self._recving_metadata.get(req_id)) is not None: - if not self._is_hma_required: - self._invalid_block_ids.put(set(meta.local_block_ids[0])) - self._failed_recv_reqs.put(req_id) + def _handle_failed_transfer( + self, req_id: str, handle: int | None, failed_req_ids: set[str] | None = None + ) -> bool: + """Record a failure and release its handle, returning False to retain it.""" + self.xfer_stats.record_failed_transfer() + if failed_req_ids is not None: + failed_req_ids.add(req_id) + return handle is None or self._try_release_xfer_handle(req_id, handle) def _send_pending_recv_notifs(self, req_id: str) -> None: """Send notifications deferred by split DRAM/VRAM reads.""" diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index f787639cfe..04caebbd1f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -149,7 +149,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): engine_id, ) self.xfer_stats.record_kv_expired_req() - self._handle_failed_transfer(req_id, None) + self._handle_failed_transfer(req_id, None, self._recv_failures) return if any(len(group) > 0 for group in meta.local_block_ids): @@ -493,16 +493,19 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): self._recving_transfers[request_id].append(handle) return True except Exception as e: - # mark all (logical) blocks for this request as invalid self._log_failure( failure_type="transfer_setup_failed", req_id=request_id, - msg="Marking blocks as invalid", + msg="Deferring failure reporting until outstanding transfers finish", error=e, dst_engine_id=dst_engine_id, remote_rank=remote_rank, ) - self._handle_failed_transfer(request_id, handle) + if not self._handle_failed_transfer( + request_id, handle, self._recv_failures + ): + assert handle is not None + self._recving_transfers[request_id].append(handle) return False def _read_blocks_mixed( @@ -544,7 +547,8 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): ) except Exception: for handle in handles: - self.nixl_wrapper.release_xfer_handle(handle) + if not self._try_release_xfer_handle(request_id, handle): + self._recving_transfers[request_id].append(handle) raise self._pending_recv_notifs.setdefault(request_id, []).append( @@ -555,7 +559,8 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): self.nixl_wrapper.transfer(handle) except Exception: for unstarted in handles[i:]: - self.nixl_wrapper.release_xfer_handle(unstarted) + if not self._try_release_xfer_handle(request_id, unstarted): + self._recving_transfers[request_id].append(unstarted) raise self._recving_transfers[request_id].append(handle) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py index 603321260d..8b34e7ffbb 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py @@ -341,7 +341,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): self._log_failure( failure_type="push_reg_handshake_failed", req_id=rid, error=e ) - self._handle_failed_transfer(rid, None) + self._failed_recv_reqs.put(rid) return # Re-queue for the writer to send now that the handshake is done. self._reg_send_inbox.put((rid, rd)) @@ -363,7 +363,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): engine_id, req_id, ) - self._handle_failed_transfer(req_id, None) + self._failed_recv_reqs.put(req_id) return for rank, agent_name in agents.items(): try: @@ -722,9 +722,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): # don't have a ``_recving_metadata`` entry to invalidate, so # we just release the handle and let the engine reschedule # via the lease / watchdog. - if handle is not None: - self.nixl_wrapper.release_xfer_handle(handle) - self.xfer_stats.record_failed_transfer() + if not self._handle_failed_transfer(request_id, handle): + return handle return None # --- Notification handling on engine main thread ------------------ # @@ -803,7 +802,16 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): # ``_pop_done_transfers`` mutates ``_sending_transfers``; the # writer thread also appends to it, so guard the pop. with self._sending_transfers_lock: - done_pushing = self._pop_done_transfers(self._sending_transfers) + done_pushing, failed_pushing = self._pop_done_transfers( + self._sending_transfers + ) + # A failed send must never be reported as done: its blocks + # are freed via the lease / watchdog instead. + done_pushing = { + req_id + for req_id in done_pushing - failed_pushing + if req_id in self._recving_metadata + } for req_id in done_pushing: self._reqs_to_send.pop(req_id, None) self._reqs_to_process.discard(req_id)