[Bugfix][NIXL] Fix multiple handles xfer race (#56104)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nicolò Lucchesi
2026-09-15 04:27:10 +00:00
committed by GitHub
co-authored by Nick Hill
parent caf56ce9da
commit 45779ef99f
5 changed files with 233 additions and 151 deletions
@@ -4,6 +4,7 @@
import contextlib
import inspect
import os
import queue
import tempfile
import textwrap
import time
@@ -1952,17 +1953,43 @@ def test_mixed_memory_local_descriptors_split_by_memory_type():
assert memory_types == ["DRAM", "VRAM"]
def test_mixed_memory_read_notifies_after_both_transfers_finish():
@pytest.fixture
def recv_worker():
"""Receive lifecycle state without distributed or device initialization."""
worker = object.__new__(NixlConnectorWorker)
worker.transfer_topo = MagicMock()
worker.transfer_topo.block_size_ratio.return_value = 1
worker.transfer_topo.get_engine_info.return_value = SimpleNamespace(
remote_block_size=16, remote_physical_blocks_per_logical=1
)
worker._physical_blocks_per_logical_kv_block = 1
worker._recving_metadata = {"request": MagicMock(local_block_ids=([1, 2, 3],))}
worker._recving_transfers = defaultdict(list)
worker._failed_recv_reqs = queue.Queue()
worker._recv_failures = set()
worker._replicated_pcp_done_sending = set()
worker._invalid_block_ids = queue.Queue()
worker._pending_recv_notifs = {}
worker._reqs_to_send = {}
worker._replicated_pcp_done_sending = set()
worker._is_hma_required = False
worker._has_mamba = False
worker.use_host_buffer = False
worker.enable_permute_local_kv = False
worker.enable_heterogeneous_attn_post_process = False
worker.tp_rank = 0
worker._log_failure = MagicMock() # type: ignore[method-assign]
worker.xfer_stats = NixlKVConnectorStats()
worker.nixl_wrapper = MagicMock()
worker.nixl_wrapper.get_xfer_telemetry.return_value = get_default_xfer_telemetry()
return worker
def test_mixed_memory_read_notifies_after_both_transfers_finish(recv_worker):
worker = recv_worker
worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])}
worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])}
worker._dram_src_handles_by_block_size = {16: 10}
worker._recving_metadata = {"request": MagicMock()}
worker._recving_transfers = defaultdict(list)
worker._pending_recv_notifs = {}
worker._failed_inflight_recvs = set()
worker.xfer_stats = MagicMock()
worker.nixl_wrapper = MagicMock()
worker.nixl_wrapper.make_prepped_xfer.side_effect = [101, 102]
worker.nixl_wrapper.check_xfer_state.return_value = "DONE"
@@ -1989,27 +2016,20 @@ def test_mixed_memory_read_notifies_after_both_transfers_finish():
np.testing.assert_array_equal(device_read.args[4], [7])
worker.nixl_wrapper.send_notif.assert_not_called()
assert worker._pop_done_transfers(worker._recving_transfers) == {"request"}
assert worker.get_finished() == (set(), {"request"})
worker.nixl_wrapper.send_notif.assert_called_once_with(
"prefill", notif_msg=b"request:1"
)
def test_mixed_memory_read_failure_does_not_notify_producer():
def test_mixed_memory_read_failure_does_not_notify_producer(recv_worker):
"""A failed half of a split READ must suppress its success notification."""
worker = object.__new__(NixlConnectorWorker)
worker._recving_metadata = {"request": MagicMock()}
worker = recv_worker
worker._recving_transfers = {"request": [101, 102]}
worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]}
worker._failed_inflight_recvs = set()
worker._is_hma_required = True
worker._failed_recv_reqs = MagicMock()
worker._log_failure = MagicMock() # type: ignore[method-assign]
worker.xfer_stats = MagicMock()
worker.nixl_wrapper = MagicMock()
worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "DONE"]
worker._pop_done_transfers(worker._recving_transfers)
assert worker.get_finished() == (set(), {"request"})
assert "request" not in worker._pending_recv_notifs
worker.nixl_wrapper.send_notif.assert_not_called()
@@ -2969,13 +2989,10 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init):
# Wait for handshake to fail
time.sleep(0.3)
# Check that blocks were marked invalid
invalid_blocks = connector.get_block_ids_with_load_errors()
assert invalid_blocks == {1, 2, 3}
# Check that request appears in get_finished
assert connector.get_block_ids_with_load_errors() == set()
_, done_recving = connector.get_finished(finished_req_ids=set())
assert request_id in done_recving
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
@patch(
@@ -3026,6 +3043,7 @@ def test_transfer_setup_failure_returns_finished(
time.sleep(0.1)
connector.start_load_kv(dummy_ctx)
assert connector.get_block_ids_with_load_errors() == set()
results = connector.get_transfer_results(finished_req_ids=set())
assert request_id in results.finished_recving
assert results.failed_recving == {request_id}
@@ -3248,74 +3266,49 @@ def test_failed_request_skips_kv_postprocessing(
assert invalid_blocks == {1, 2, 3}
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FailingNixlWrapper,
)
def test_handles_failing_in_separate_polls_do_not_kill_the_engine(
default_vllm_config, dist_init
):
"""Failures across polls report once, after the last writer is terminal."""
vllm_config = create_vllm_config()
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
)
worker = connector.connector_worker
@pytest.mark.parametrize("sibling_state", ["ERR", "DONE"])
def test_recv_failure_waits_for_sibling_transfer(recv_worker, sibling_state):
"""Failure must not release blocks while a sibling can still write to them."""
worker = recv_worker
worker._recving_transfers["request"] = [101, 102]
worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]}
worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "PROC", sibling_state]
request_id = "test_two_handles_failing_in_separate_polls"
metadata = NixlConnectorMetadata()
metadata.add_new_req_to_recv(
request_id=request_id,
local_block_ids=([1, 2, 3],),
kv_transfer_params={
"remote_block_ids": ([4, 5, 6],),
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
"remote_request_id": f"prefill-{request_id}",
"remote_host": "localhost",
"remote_port": 1234,
"remote_tp_size": 1,
},
)
connector.bind_connector_metadata(metadata)
dummy_ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={})
connector.start_load_kv(dummy_ctx)
connector.bind_connector_metadata(NixlConnectorMetadata())
time.sleep(0.1)
connector.start_load_kv(dummy_ctx)
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == set()
assert worker.get_block_ids_with_load_errors() == set()
assert "request" in worker._recving_metadata
worker.nixl_wrapper.release_xfer_handle.assert_called_once_with(101)
# Give the request a second handle, as a second remote rank would.
handles = worker._recving_transfers[request_id]
assert len(handles) == 1
first, second = handles[0], object()
worker._recving_transfers[request_id] = [first, second]
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == {"request"}
assert worker.get_block_ids_with_load_errors() == {1, 2, 3}
worker.nixl_wrapper.send_notif.assert_not_called()
# Poll 1: the first handle has failed, the second is still running.
def first_failed(handle):
return "ERR" if handle is first else "PROC"
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == set()
assert worker.get_block_ids_with_load_errors() == set()
with patch.object(
worker.nixl_wrapper, "check_xfer_state", side_effect=first_failed
):
_, done_recving = connector.get_finished(finished_req_ids=set())
assert request_id not in done_recving
assert request_id in worker._recving_metadata
assert worker._recving_transfers[request_id] == [second]
assert connector.get_block_ids_with_load_errors() == set()
def test_recv_failure_waits_for_unpollable_handle_to_be_released(recv_worker):
"""A polling exception and failed release do not prove that DMA has stopped."""
worker = recv_worker
worker._recving_transfers["request"] = [101]
worker.nixl_wrapper.check_xfer_state.side_effect = [RuntimeError("poll"), "DONE"]
worker.nixl_wrapper.release_xfer_handle.side_effect = [
RuntimeError("release"),
None,
]
# Only the last terminal handle permits the scheduler to reuse the blocks.
with patch.object(worker.nixl_wrapper, "check_xfer_state", return_value="ERR"):
_, done_recving = connector.get_finished(finished_req_ids=set())
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == set()
assert worker.get_block_ids_with_load_errors() == set()
assert "request" in worker._recving_metadata
assert done_recving == {request_id}
assert request_id not in worker._recving_transfers
assert request_id not in worker._recving_metadata
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
_, done_recving = connector.get_finished(finished_req_ids=set())
assert done_recving == set()
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == {"request"}
assert worker.get_block_ids_with_load_errors() == {1, 2, 3}
assert worker.nixl_wrapper.release_xfer_handle.call_count == 2
def _set_test_speculative_config(
@@ -351,7 +351,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
# Base worker fields touched by start_load_kv / _get_new_notifs.
w._recving_metadata = {}
w._pending_recv_notifs = {}
w._failed_inflight_recvs = set()
w._recv_failures = set()
w._recving_transfers = defaultdict(list)
w._reqs_to_process = set()
w._reqs_to_send = {}
@@ -877,6 +877,68 @@ class TestPushWriterNotifs:
assert evicted == ["req-done"]
assert w._push_writer_wake.is_set()
@staticmethod
def _pollable_worker() -> _StubWriterWorker:
"""Stub worker with the extra base-worker state the real
``get_transfer_results`` needs to poll ``_sending_transfers``."""
w = _StubWriterWorker.fresh()
w.transfer_topo = MagicMock()
w.nixl_wrapper = MagicMock()
w.xfer_stats = MagicMock()
w._log_failure = MagicMock() # type: ignore[method-assign]
w._failed_recv_reqs = queue.Queue()
w._invalid_block_ids = queue.Queue()
w._pending_recv_notifs = {}
w._replicated_pcp_done_sending = set()
w.use_host_buffer = False
return w
def _make_sending_req(self, w: _StubWriterWorker) -> str:
request_id = "req-send-1"
w._sending_transfers[request_id] = [101, 102]
w._reqs_to_send[request_id] = time.perf_counter() + 60
w._reqs_to_process.add(request_id)
return request_id
def test_failed_push_send_is_not_reported_done_sending(self):
"""A failed P-side WRITE must not surface as finished sending.
Reporting it would free P-side blocks as if the KV had been
delivered while D still waits for the data; the request must be
left to the lease / watchdog instead.
"""
w = self._pollable_worker()
request_id = self._make_sending_req(w)
w.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "DONE"]
results = w.get_transfer_results()
assert request_id not in results.finished_sending
# Lease tracking is preserved so the watchdog can reschedule.
assert request_id in w._reqs_to_send
assert request_id in w._reqs_to_process
# Both handles were still cleaned up.
assert request_id not in w._sending_transfers
def test_completed_push_send_waits_for_consumer_notif(self):
"""P-side sends are completed by consumer notifs / lease expiry,
not by local WRITE-handle completion."""
w = self._pollable_worker()
request_id = self._make_sending_req(w)
w.nixl_wrapper.check_xfer_state.return_value = "DONE"
results = w.get_transfer_results()
assert request_id not in results.finished_sending
assert request_id in w._reqs_to_send
assert request_id not in w._sending_transfers
# The consumer notif is what completes the send.
w._pending_completion_notifs.put(f"{request_id}:1".encode())
results = w.get_transfer_results()
assert request_id in results.finished_sending
assert request_id not in w._reqs_to_send
# ----------------------------------------------------------------- #
# Negative / error-path tests #
@@ -1079,7 +1141,7 @@ class TestPushWriterNegative:
# Wake set so the writer drains NIXL notifs even when idle.
assert w._push_writer_wake.is_set()
def test_failed_push_transfer_finishes_sending(self):
def test_failed_push_transfer_does_not_evict_writer_state(self):
w = _StubWriterWorker.fresh()
w.nixl_wrapper = MagicMock()
w.nixl_wrapper.check_xfer_state.return_value = "ERR"
@@ -1096,11 +1158,12 @@ class TestPushWriterNegative:
):
results = w.get_transfer_results()
assert results.finished_sending == {"req-failed"}
assert results.finished_sending == set()
assert results.finished_recving == set()
assert "req-failed" not in w._sending_transfers
assert "req-failed" not in w._reqs_to_send
assert "req-failed" not in w._reqs_to_process
assert "req-failed" in w._reqs_to_send
assert "req-failed" in w._reqs_to_process
assert w._evict_finished_inbox.empty()
w.nixl_wrapper.release_xfer_handle.assert_called_once_with(17)
def test_get_new_notifs_unknown_request_is_logged_and_skipped(self, caplog):
@@ -692,7 +692,7 @@ class NixlBaseConnectorWorker:
# Uses Queue for thread-safe cross-thread coordination with the
# background handshake thread, matching the _ready_requests pattern.
self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue()
self._failed_inflight_recvs: set[ReqId] = set()
self._recv_failures: set[ReqId] = set()
self._pending_recv_notifs: dict[ReqId, list[tuple[str, bytes]]] = {}
# Handshake metadata of this worker for NIXL transfers.
@@ -1212,7 +1212,7 @@ class NixlBaseConnectorWorker:
error=e,
meta=meta,
)
self._handle_failed_transfer(req_id, None)
self._failed_recv_reqs.put(req_id)
fut.add_done_callback(request_ready)
@@ -2618,22 +2618,32 @@ class NixlBaseConnectorWorker:
"""
assert self.transfer_topo is not None
done_sending = self._get_new_notifs()
done_recving = self._pop_done_transfers(self._recving_transfers)
done_recving, newly_failed = self._pop_done_transfers(self._recving_transfers)
if newly_failed:
self._recv_failures.update(newly_failed)
done_sending.update(self._replicated_pcp_done_sending)
self._replicated_pcp_done_sending.clear()
# Drain queue of requests where handshake or transfer setup failed.
failed_recv_reqs = set[ReqId]()
# Process receive failures reported by background threads.
while not self._failed_recv_reqs.empty():
try:
failed_recv_reqs.add(self._failed_recv_reqs.get_nowait())
req_id = self._failed_recv_reqs.get_nowait()
except queue.Empty:
break
self._handle_failed_transfer(req_id, None, self._recv_failures)
failed_recv_reqs: set[ReqId] = set()
if self._recv_failures:
# Keep failure bookkeeping independent of healthy request count.
for req_id in tuple(self._recv_failures):
if req_id not in self._recving_metadata:
self._recv_failures.remove(req_id)
elif req_id not in self._recving_transfers:
self._recv_failures.remove(req_id)
failed_recv_reqs.add(req_id)
done_recving.update(failed_recv_reqs)
# Add failed requests to done_recving for scheduler tracking
# (blocks are already marked invalid, scheduler will handle recompute)
done_recving.update(failed_recv_reqs)
if len(done_sending) > 0 or len(done_recving) > 0:
logger.debug(
"Rank %s, get_finished: %s requests done sending "
@@ -2654,12 +2664,17 @@ class NixlBaseConnectorWorker:
# Skip KV sync and post-processing for failed requests
if req_id in failed_recv_reqs:
self._pending_recv_notifs.pop(req_id, None)
# TODO (NickLucche) handle failed transfer for HMA.
if not self._is_hma_required:
self._invalid_block_ids.put(set(meta.local_block_ids[0]))
logger.warning(
"Skipping KV post-processing for failed request %s",
req_id,
)
continue
self._send_pending_recv_notifs(req_id)
assert meta.remote is not None
if self.use_host_buffer:
self.sync_recved_kv_to_device(req_id, meta)
@@ -2783,22 +2798,26 @@ class NixlBaseConnectorWorker:
new_expiry,
)
def _pop_done_transfers(self, transfers: dict[str, list[int]]) -> set[str]:
"""
Pop completed xfers by checking for DONE state.
def _pop_done_transfers(
self, transfers: dict[str, list[int]]
) -> tuple[set[str], set[str]]:
"""Poll transfers, retaining handles until they can be released.
Args:
transfers: dict of req_id -> list[running_xfer]
transfers: Outstanding handles by request, updated in place.
Returns:
set of req_ids that have all done xfers
Requests with no outstanding handles and requests with observed
failures. Failed requests may still have outstanding handles.
"""
done_req_ids: set[str] = set()
failed_req_ids: set[str] = set()
for req_id, handles in list(transfers.items()):
in_progress = []
for handle in handles:
try:
xfer_state = self.nixl_wrapper.check_xfer_state(handle)
if xfer_state == "DONE":
# Get telemetry from NIXL
res = self.nixl_wrapper.get_xfer_telemetry(handle)
self.xfer_stats.record_transfer(res)
self.nixl_wrapper.release_xfer_handle(handle)
@@ -2807,58 +2826,52 @@ class NixlBaseConnectorWorker:
else:
self._log_failure(
failure_type="transfer_failed",
msg="Deferring request completion until its last "
"xfer is terminal",
req_id=req_id,
xfer_state=xfer_state,
)
self._handle_failed_transfer(req_id, handle)
if not self._handle_failed_transfer(
req_id, handle, failed_req_ids
):
in_progress.append(handle)
except Exception as e:
self._log_failure(
failure_type="transfer_exception",
msg="Handle is unpollable; treating it as terminal",
req_id=req_id,
error=e,
)
self._handle_failed_transfer(req_id, handle)
if not self._handle_failed_transfer(req_id, handle, failed_req_ids):
in_progress.append(handle)
if not in_progress:
del transfers[req_id]
done_req_ids.add(req_id)
if req_id in self._failed_inflight_recvs:
self._failed_inflight_recvs.remove(req_id)
self._report_failed_recv(req_id)
else:
self._send_pending_recv_notifs(req_id)
del transfers[req_id]
else:
transfers[req_id] = in_progress
return done_req_ids
return done_req_ids, failed_req_ids
def _handle_failed_transfer(self, req_id: str, handle: int | None):
"""
Handle a failed transfer by marking all (logical) blocks as invalid and
recording the failure.
Args:
req_id: The request ID.
handle: The transfer handle.
"""
# A sibling READ may still be writing these blocks. Retain metadata
# and defer invalidation until every handle is terminal.
self._pending_recv_notifs.pop(req_id, None)
if req_id in self._recving_transfers:
self._failed_inflight_recvs.add(req_id)
else:
self._report_failed_recv(req_id)
if handle is not None:
def _try_release_xfer_handle(self, req_id: str, handle: int) -> bool:
"""Release a handle, returning False if the caller must retain it."""
try:
self.nixl_wrapper.release_xfer_handle(handle)
self.xfer_stats.record_failed_transfer()
except Exception as e:
# A status error does not guarantee that the backend stopped DMA.
self._log_failure(
failure_type="transfer_release_failed",
msg="Retaining handle and blocks until release succeeds",
req_id=req_id,
error=e,
)
return False
return True
def _report_failed_recv(self, req_id: str) -> None:
if (meta := self._recving_metadata.get(req_id)) is not None:
if not self._is_hma_required:
self._invalid_block_ids.put(set(meta.local_block_ids[0]))
self._failed_recv_reqs.put(req_id)
def _handle_failed_transfer(
self, req_id: str, handle: int | None, failed_req_ids: set[str] | None = None
) -> bool:
"""Record a failure and release its handle, returning False to retain it."""
self.xfer_stats.record_failed_transfer()
if failed_req_ids is not None:
failed_req_ids.add(req_id)
return handle is None or self._try_release_xfer_handle(req_id, handle)
def _send_pending_recv_notifs(self, req_id: str) -> None:
"""Send notifications deferred by split DRAM/VRAM reads."""
@@ -149,7 +149,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
engine_id,
)
self.xfer_stats.record_kv_expired_req()
self._handle_failed_transfer(req_id, None)
self._handle_failed_transfer(req_id, None, self._recv_failures)
return
if any(len(group) > 0 for group in meta.local_block_ids):
@@ -493,16 +493,19 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
self._recving_transfers[request_id].append(handle)
return True
except Exception as e:
# mark all (logical) blocks for this request as invalid
self._log_failure(
failure_type="transfer_setup_failed",
req_id=request_id,
msg="Marking blocks as invalid",
msg="Deferring failure reporting until outstanding transfers finish",
error=e,
dst_engine_id=dst_engine_id,
remote_rank=remote_rank,
)
self._handle_failed_transfer(request_id, handle)
if not self._handle_failed_transfer(
request_id, handle, self._recv_failures
):
assert handle is not None
self._recving_transfers[request_id].append(handle)
return False
def _read_blocks_mixed(
@@ -544,7 +547,8 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
)
except Exception:
for handle in handles:
self.nixl_wrapper.release_xfer_handle(handle)
if not self._try_release_xfer_handle(request_id, handle):
self._recving_transfers[request_id].append(handle)
raise
self._pending_recv_notifs.setdefault(request_id, []).append(
@@ -555,7 +559,8 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
self.nixl_wrapper.transfer(handle)
except Exception:
for unstarted in handles[i:]:
self.nixl_wrapper.release_xfer_handle(unstarted)
if not self._try_release_xfer_handle(request_id, unstarted):
self._recving_transfers[request_id].append(unstarted)
raise
self._recving_transfers[request_id].append(handle)
@@ -341,7 +341,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
self._log_failure(
failure_type="push_reg_handshake_failed", req_id=rid, error=e
)
self._handle_failed_transfer(rid, None)
self._failed_recv_reqs.put(rid)
return
# Re-queue for the writer to send now that the handshake is done.
self._reg_send_inbox.put((rid, rd))
@@ -363,7 +363,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
engine_id,
req_id,
)
self._handle_failed_transfer(req_id, None)
self._failed_recv_reqs.put(req_id)
return
for rank, agent_name in agents.items():
try:
@@ -722,9 +722,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
# don't have a ``_recving_metadata`` entry to invalidate, so
# we just release the handle and let the engine reschedule
# via the lease / watchdog.
if handle is not None:
self.nixl_wrapper.release_xfer_handle(handle)
self.xfer_stats.record_failed_transfer()
if not self._handle_failed_transfer(request_id, handle):
return handle
return None
# --- Notification handling on engine main thread ------------------ #
@@ -803,7 +802,16 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
# ``_pop_done_transfers`` mutates ``_sending_transfers``; the
# writer thread also appends to it, so guard the pop.
with self._sending_transfers_lock:
done_pushing = self._pop_done_transfers(self._sending_transfers)
done_pushing, failed_pushing = self._pop_done_transfers(
self._sending_transfers
)
# A failed send must never be reported as done: its blocks
# are freed via the lease / watchdog instead.
done_pushing = {
req_id
for req_id in done_pushing - failed_pushing
if req_id in self._recving_metadata
}
for req_id in done_pushing:
self._reqs_to_send.pop(req_id, None)
self._reqs_to_process.discard(req_id)