[Bugfix][NIXL] Fix multiple handles xfer race (#56104)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nicolò Lucchesi
2026-09-15 04:27:10 +00:00
committed by GitHub
co-authored by Nick Hill
parent caf56ce9da
commit 45779ef99f
5 changed files with 233 additions and 151 deletions
@@ -4,6 +4,7 @@
import contextlib
import inspect
import os
import queue
import tempfile
import textwrap
import time
@@ -1952,17 +1953,43 @@ def test_mixed_memory_local_descriptors_split_by_memory_type():
assert memory_types == ["DRAM", "VRAM"]
def test_mixed_memory_read_notifies_after_both_transfers_finish():
@pytest.fixture
def recv_worker():
"""Receive lifecycle state without distributed or device initialization."""
worker = object.__new__(NixlConnectorWorker)
worker.transfer_topo = MagicMock()
worker.transfer_topo.block_size_ratio.return_value = 1
worker.transfer_topo.get_engine_info.return_value = SimpleNamespace(
remote_block_size=16, remote_physical_blocks_per_logical=1
)
worker._physical_blocks_per_logical_kv_block = 1
worker._recving_metadata = {"request": MagicMock(local_block_ids=([1, 2, 3],))}
worker._recving_transfers = defaultdict(list)
worker._failed_recv_reqs = queue.Queue()
worker._recv_failures = set()
worker._replicated_pcp_done_sending = set()
worker._invalid_block_ids = queue.Queue()
worker._pending_recv_notifs = {}
worker._reqs_to_send = {}
worker._replicated_pcp_done_sending = set()
worker._is_hma_required = False
worker._has_mamba = False
worker.use_host_buffer = False
worker.enable_permute_local_kv = False
worker.enable_heterogeneous_attn_post_process = False
worker.tp_rank = 0
worker._log_failure = MagicMock() # type: ignore[method-assign]
worker.xfer_stats = NixlKVConnectorStats()
worker.nixl_wrapper = MagicMock()
worker.nixl_wrapper.get_xfer_telemetry.return_value = get_default_xfer_telemetry()
return worker
def test_mixed_memory_read_notifies_after_both_transfers_finish(recv_worker):
worker = recv_worker
worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])}
worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])}
worker._dram_src_handles_by_block_size = {16: 10}
worker._recving_metadata = {"request": MagicMock()}
worker._recving_transfers = defaultdict(list)
worker._pending_recv_notifs = {}
worker._failed_inflight_recvs = set()
worker.xfer_stats = MagicMock()
worker.nixl_wrapper = MagicMock()
worker.nixl_wrapper.make_prepped_xfer.side_effect = [101, 102]
worker.nixl_wrapper.check_xfer_state.return_value = "DONE"
@@ -1989,27 +2016,20 @@ def test_mixed_memory_read_notifies_after_both_transfers_finish():
np.testing.assert_array_equal(device_read.args[4], [7])
worker.nixl_wrapper.send_notif.assert_not_called()
assert worker._pop_done_transfers(worker._recving_transfers) == {"request"}
assert worker.get_finished() == (set(), {"request"})
worker.nixl_wrapper.send_notif.assert_called_once_with(
"prefill", notif_msg=b"request:1"
)
def test_mixed_memory_read_failure_does_not_notify_producer():
def test_mixed_memory_read_failure_does_not_notify_producer(recv_worker):
"""A failed half of a split READ must suppress its success notification."""
worker = object.__new__(NixlConnectorWorker)
worker._recving_metadata = {"request": MagicMock()}
worker = recv_worker
worker._recving_transfers = {"request": [101, 102]}
worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]}
worker._failed_inflight_recvs = set()
worker._is_hma_required = True
worker._failed_recv_reqs = MagicMock()
worker._log_failure = MagicMock() # type: ignore[method-assign]
worker.xfer_stats = MagicMock()
worker.nixl_wrapper = MagicMock()
worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "DONE"]
worker._pop_done_transfers(worker._recving_transfers)
assert worker.get_finished() == (set(), {"request"})
assert "request" not in worker._pending_recv_notifs
worker.nixl_wrapper.send_notif.assert_not_called()
@@ -2969,13 +2989,10 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init):
# Wait for handshake to fail
time.sleep(0.3)
# Check that blocks were marked invalid
invalid_blocks = connector.get_block_ids_with_load_errors()
assert invalid_blocks == {1, 2, 3}
# Check that request appears in get_finished
assert connector.get_block_ids_with_load_errors() == set()
_, done_recving = connector.get_finished(finished_req_ids=set())
assert request_id in done_recving
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
@patch(
@@ -3026,6 +3043,7 @@ def test_transfer_setup_failure_returns_finished(
time.sleep(0.1)
connector.start_load_kv(dummy_ctx)
assert connector.get_block_ids_with_load_errors() == set()
results = connector.get_transfer_results(finished_req_ids=set())
assert request_id in results.finished_recving
assert results.failed_recving == {request_id}
@@ -3248,74 +3266,49 @@ def test_failed_request_skips_kv_postprocessing(
assert invalid_blocks == {1, 2, 3}
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FailingNixlWrapper,
)
def test_handles_failing_in_separate_polls_do_not_kill_the_engine(
default_vllm_config, dist_init
):
"""Failures across polls report once, after the last writer is terminal."""
vllm_config = create_vllm_config()
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
)
worker = connector.connector_worker
@pytest.mark.parametrize("sibling_state", ["ERR", "DONE"])
def test_recv_failure_waits_for_sibling_transfer(recv_worker, sibling_state):
"""Failure must not release blocks while a sibling can still write to them."""
worker = recv_worker
worker._recving_transfers["request"] = [101, 102]
worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]}
worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "PROC", sibling_state]
request_id = "test_two_handles_failing_in_separate_polls"
metadata = NixlConnectorMetadata()
metadata.add_new_req_to_recv(
request_id=request_id,
local_block_ids=([1, 2, 3],),
kv_transfer_params={
"remote_block_ids": ([4, 5, 6],),
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
"remote_request_id": f"prefill-{request_id}",
"remote_host": "localhost",
"remote_port": 1234,
"remote_tp_size": 1,
},
)
connector.bind_connector_metadata(metadata)
dummy_ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={})
connector.start_load_kv(dummy_ctx)
connector.bind_connector_metadata(NixlConnectorMetadata())
time.sleep(0.1)
connector.start_load_kv(dummy_ctx)
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == set()
assert worker.get_block_ids_with_load_errors() == set()
assert "request" in worker._recving_metadata
worker.nixl_wrapper.release_xfer_handle.assert_called_once_with(101)
# Give the request a second handle, as a second remote rank would.
handles = worker._recving_transfers[request_id]
assert len(handles) == 1
first, second = handles[0], object()
worker._recving_transfers[request_id] = [first, second]
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == {"request"}
assert worker.get_block_ids_with_load_errors() == {1, 2, 3}
worker.nixl_wrapper.send_notif.assert_not_called()
# Poll 1: the first handle has failed, the second is still running.
def first_failed(handle):
return "ERR" if handle is first else "PROC"
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == set()
assert worker.get_block_ids_with_load_errors() == set()
with patch.object(
worker.nixl_wrapper, "check_xfer_state", side_effect=first_failed
):
_, done_recving = connector.get_finished(finished_req_ids=set())
assert request_id not in done_recving
assert request_id in worker._recving_metadata
assert worker._recving_transfers[request_id] == [second]
assert connector.get_block_ids_with_load_errors() == set()
def test_recv_failure_waits_for_unpollable_handle_to_be_released(recv_worker):
"""A polling exception and failed release do not prove that DMA has stopped."""
worker = recv_worker
worker._recving_transfers["request"] = [101]
worker.nixl_wrapper.check_xfer_state.side_effect = [RuntimeError("poll"), "DONE"]
worker.nixl_wrapper.release_xfer_handle.side_effect = [
RuntimeError("release"),
None,
]
# Only the last terminal handle permits the scheduler to reuse the blocks.
with patch.object(worker.nixl_wrapper, "check_xfer_state", return_value="ERR"):
_, done_recving = connector.get_finished(finished_req_ids=set())
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == set()
assert worker.get_block_ids_with_load_errors() == set()
assert "request" in worker._recving_metadata
assert done_recving == {request_id}
assert request_id not in worker._recving_transfers
assert request_id not in worker._recving_metadata
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
_, done_recving = connector.get_finished(finished_req_ids=set())
assert done_recving == set()
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == {"request"}
assert worker.get_block_ids_with_load_errors() == {1, 2, 3}
assert worker.nixl_wrapper.release_xfer_handle.call_count == 2
def _set_test_speculative_config(
@@ -351,7 +351,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
# Base worker fields touched by start_load_kv / _get_new_notifs.
w._recving_metadata = {}
w._pending_recv_notifs = {}
w._failed_inflight_recvs = set()
w._recv_failures = set()
w._recving_transfers = defaultdict(list)
w._reqs_to_process = set()
w._reqs_to_send = {}
@@ -877,6 +877,68 @@ class TestPushWriterNotifs:
assert evicted == ["req-done"]
assert w._push_writer_wake.is_set()
@staticmethod
def _pollable_worker() -> _StubWriterWorker:
"""Stub worker with the extra base-worker state the real
``get_transfer_results`` needs to poll ``_sending_transfers``."""
w = _StubWriterWorker.fresh()
w.transfer_topo = MagicMock()
w.nixl_wrapper = MagicMock()
w.xfer_stats = MagicMock()
w._log_failure = MagicMock() # type: ignore[method-assign]
w._failed_recv_reqs = queue.Queue()
w._invalid_block_ids = queue.Queue()
w._pending_recv_notifs = {}
w._replicated_pcp_done_sending = set()
w.use_host_buffer = False
return w
def _make_sending_req(self, w: _StubWriterWorker) -> str:
request_id = "req-send-1"
w._sending_transfers[request_id] = [101, 102]
w._reqs_to_send[request_id] = time.perf_counter() + 60
w._reqs_to_process.add(request_id)
return request_id
def test_failed_push_send_is_not_reported_done_sending(self):
"""A failed P-side WRITE must not surface as finished sending.
Reporting it would free P-side blocks as if the KV had been
delivered while D still waits for the data; the request must be
left to the lease / watchdog instead.
"""
w = self._pollable_worker()
request_id = self._make_sending_req(w)
w.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "DONE"]
results = w.get_transfer_results()
assert request_id not in results.finished_sending
# Lease tracking is preserved so the watchdog can reschedule.
assert request_id in w._reqs_to_send
assert request_id in w._reqs_to_process
# Both handles were still cleaned up.
assert request_id not in w._sending_transfers
def test_completed_push_send_waits_for_consumer_notif(self):
"""P-side sends are completed by consumer notifs / lease expiry,
not by local WRITE-handle completion."""
w = self._pollable_worker()
request_id = self._make_sending_req(w)
w.nixl_wrapper.check_xfer_state.return_value = "DONE"
results = w.get_transfer_results()
assert request_id not in results.finished_sending
assert request_id in w._reqs_to_send
assert request_id not in w._sending_transfers
# The consumer notif is what completes the send.
w._pending_completion_notifs.put(f"{request_id}:1".encode())
results = w.get_transfer_results()
assert request_id in results.finished_sending
assert request_id not in w._reqs_to_send
# ----------------------------------------------------------------- #
# Negative / error-path tests #
@@ -1079,7 +1141,7 @@ class TestPushWriterNegative:
# Wake set so the writer drains NIXL notifs even when idle.
assert w._push_writer_wake.is_set()
def test_failed_push_transfer_finishes_sending(self):
def test_failed_push_transfer_does_not_evict_writer_state(self):
w = _StubWriterWorker.fresh()
w.nixl_wrapper = MagicMock()
w.nixl_wrapper.check_xfer_state.return_value = "ERR"
@@ -1096,11 +1158,12 @@ class TestPushWriterNegative:
):
results = w.get_transfer_results()
assert results.finished_sending == {"req-failed"}
assert results.finished_sending == set()
assert results.finished_recving == set()
assert "req-failed" not in w._sending_transfers
assert "req-failed" not in w._reqs_to_send
assert "req-failed" not in w._reqs_to_process
assert "req-failed" in w._reqs_to_send
assert "req-failed" in w._reqs_to_process
assert w._evict_finished_inbox.empty()
w.nixl_wrapper.release_xfer_handle.assert_called_once_with(17)
def test_get_new_notifs_unknown_request_is_logged_and_skipped(self, caplog):