[Bugfix][NIXL] Fix multiple handles xfer race (#56104)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
caf56ce9da
commit
45779ef99f
@@ -4,6 +4,7 @@
|
||||
import contextlib
|
||||
import inspect
|
||||
import os
|
||||
import queue
|
||||
import tempfile
|
||||
import textwrap
|
||||
import time
|
||||
@@ -1952,17 +1953,43 @@ def test_mixed_memory_local_descriptors_split_by_memory_type():
|
||||
assert memory_types == ["DRAM", "VRAM"]
|
||||
|
||||
|
||||
def test_mixed_memory_read_notifies_after_both_transfers_finish():
|
||||
@pytest.fixture
|
||||
def recv_worker():
|
||||
"""Receive lifecycle state without distributed or device initialization."""
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker.transfer_topo = MagicMock()
|
||||
worker.transfer_topo.block_size_ratio.return_value = 1
|
||||
worker.transfer_topo.get_engine_info.return_value = SimpleNamespace(
|
||||
remote_block_size=16, remote_physical_blocks_per_logical=1
|
||||
)
|
||||
worker._physical_blocks_per_logical_kv_block = 1
|
||||
worker._recving_metadata = {"request": MagicMock(local_block_ids=([1, 2, 3],))}
|
||||
worker._recving_transfers = defaultdict(list)
|
||||
worker._failed_recv_reqs = queue.Queue()
|
||||
worker._recv_failures = set()
|
||||
worker._replicated_pcp_done_sending = set()
|
||||
worker._invalid_block_ids = queue.Queue()
|
||||
worker._pending_recv_notifs = {}
|
||||
worker._reqs_to_send = {}
|
||||
worker._replicated_pcp_done_sending = set()
|
||||
worker._is_hma_required = False
|
||||
worker._has_mamba = False
|
||||
worker.use_host_buffer = False
|
||||
worker.enable_permute_local_kv = False
|
||||
worker.enable_heterogeneous_attn_post_process = False
|
||||
worker.tp_rank = 0
|
||||
worker._log_failure = MagicMock() # type: ignore[method-assign]
|
||||
worker.xfer_stats = NixlKVConnectorStats()
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
worker.nixl_wrapper.get_xfer_telemetry.return_value = get_default_xfer_telemetry()
|
||||
return worker
|
||||
|
||||
|
||||
def test_mixed_memory_read_notifies_after_both_transfers_finish(recv_worker):
|
||||
worker = recv_worker
|
||||
worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])}
|
||||
worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])}
|
||||
worker._dram_src_handles_by_block_size = {16: 10}
|
||||
worker._recving_metadata = {"request": MagicMock()}
|
||||
worker._recving_transfers = defaultdict(list)
|
||||
worker._pending_recv_notifs = {}
|
||||
worker._failed_inflight_recvs = set()
|
||||
worker.xfer_stats = MagicMock()
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
worker.nixl_wrapper.make_prepped_xfer.side_effect = [101, 102]
|
||||
worker.nixl_wrapper.check_xfer_state.return_value = "DONE"
|
||||
|
||||
@@ -1989,27 +2016,20 @@ def test_mixed_memory_read_notifies_after_both_transfers_finish():
|
||||
np.testing.assert_array_equal(device_read.args[4], [7])
|
||||
worker.nixl_wrapper.send_notif.assert_not_called()
|
||||
|
||||
assert worker._pop_done_transfers(worker._recving_transfers) == {"request"}
|
||||
assert worker.get_finished() == (set(), {"request"})
|
||||
worker.nixl_wrapper.send_notif.assert_called_once_with(
|
||||
"prefill", notif_msg=b"request:1"
|
||||
)
|
||||
|
||||
|
||||
def test_mixed_memory_read_failure_does_not_notify_producer():
|
||||
def test_mixed_memory_read_failure_does_not_notify_producer(recv_worker):
|
||||
"""A failed half of a split READ must suppress its success notification."""
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker._recving_metadata = {"request": MagicMock()}
|
||||
worker = recv_worker
|
||||
worker._recving_transfers = {"request": [101, 102]}
|
||||
worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]}
|
||||
worker._failed_inflight_recvs = set()
|
||||
worker._is_hma_required = True
|
||||
worker._failed_recv_reqs = MagicMock()
|
||||
worker._log_failure = MagicMock() # type: ignore[method-assign]
|
||||
worker.xfer_stats = MagicMock()
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "DONE"]
|
||||
|
||||
worker._pop_done_transfers(worker._recving_transfers)
|
||||
assert worker.get_finished() == (set(), {"request"})
|
||||
|
||||
assert "request" not in worker._pending_recv_notifs
|
||||
worker.nixl_wrapper.send_notif.assert_not_called()
|
||||
@@ -2969,13 +2989,10 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init):
|
||||
# Wait for handshake to fail
|
||||
time.sleep(0.3)
|
||||
|
||||
# Check that blocks were marked invalid
|
||||
invalid_blocks = connector.get_block_ids_with_load_errors()
|
||||
assert invalid_blocks == {1, 2, 3}
|
||||
|
||||
# Check that request appears in get_finished
|
||||
assert connector.get_block_ids_with_load_errors() == set()
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
assert request_id in done_recving
|
||||
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
|
||||
|
||||
|
||||
@patch(
|
||||
@@ -3026,6 +3043,7 @@ def test_transfer_setup_failure_returns_finished(
|
||||
time.sleep(0.1)
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
|
||||
assert connector.get_block_ids_with_load_errors() == set()
|
||||
results = connector.get_transfer_results(finished_req_ids=set())
|
||||
assert request_id in results.finished_recving
|
||||
assert results.failed_recving == {request_id}
|
||||
@@ -3248,74 +3266,49 @@ def test_failed_request_skips_kv_postprocessing(
|
||||
assert invalid_blocks == {1, 2, 3}
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FailingNixlWrapper,
|
||||
)
|
||||
def test_handles_failing_in_separate_polls_do_not_kill_the_engine(
|
||||
default_vllm_config, dist_init
|
||||
):
|
||||
"""Failures across polls report once, after the last writer is terminal."""
|
||||
vllm_config = create_vllm_config()
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
)
|
||||
worker = connector.connector_worker
|
||||
@pytest.mark.parametrize("sibling_state", ["ERR", "DONE"])
|
||||
def test_recv_failure_waits_for_sibling_transfer(recv_worker, sibling_state):
|
||||
"""Failure must not release blocks while a sibling can still write to them."""
|
||||
worker = recv_worker
|
||||
worker._recving_transfers["request"] = [101, 102]
|
||||
worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]}
|
||||
worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "PROC", sibling_state]
|
||||
|
||||
request_id = "test_two_handles_failing_in_separate_polls"
|
||||
metadata = NixlConnectorMetadata()
|
||||
metadata.add_new_req_to_recv(
|
||||
request_id=request_id,
|
||||
local_block_ids=([1, 2, 3],),
|
||||
kv_transfer_params={
|
||||
"remote_block_ids": ([4, 5, 6],),
|
||||
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
|
||||
"remote_request_id": f"prefill-{request_id}",
|
||||
"remote_host": "localhost",
|
||||
"remote_port": 1234,
|
||||
"remote_tp_size": 1,
|
||||
},
|
||||
)
|
||||
connector.bind_connector_metadata(metadata)
|
||||
dummy_ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={})
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
connector.bind_connector_metadata(NixlConnectorMetadata())
|
||||
time.sleep(0.1)
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
results = worker.get_transfer_results()
|
||||
assert results.finished_recving == results.failed_recving == set()
|
||||
assert worker.get_block_ids_with_load_errors() == set()
|
||||
assert "request" in worker._recving_metadata
|
||||
worker.nixl_wrapper.release_xfer_handle.assert_called_once_with(101)
|
||||
|
||||
# Give the request a second handle, as a second remote rank would.
|
||||
handles = worker._recving_transfers[request_id]
|
||||
assert len(handles) == 1
|
||||
first, second = handles[0], object()
|
||||
worker._recving_transfers[request_id] = [first, second]
|
||||
results = worker.get_transfer_results()
|
||||
assert results.finished_recving == results.failed_recving == {"request"}
|
||||
assert worker.get_block_ids_with_load_errors() == {1, 2, 3}
|
||||
worker.nixl_wrapper.send_notif.assert_not_called()
|
||||
|
||||
# Poll 1: the first handle has failed, the second is still running.
|
||||
def first_failed(handle):
|
||||
return "ERR" if handle is first else "PROC"
|
||||
results = worker.get_transfer_results()
|
||||
assert results.finished_recving == results.failed_recving == set()
|
||||
assert worker.get_block_ids_with_load_errors() == set()
|
||||
|
||||
with patch.object(
|
||||
worker.nixl_wrapper, "check_xfer_state", side_effect=first_failed
|
||||
):
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
|
||||
assert request_id not in done_recving
|
||||
assert request_id in worker._recving_metadata
|
||||
assert worker._recving_transfers[request_id] == [second]
|
||||
assert connector.get_block_ids_with_load_errors() == set()
|
||||
def test_recv_failure_waits_for_unpollable_handle_to_be_released(recv_worker):
|
||||
"""A polling exception and failed release do not prove that DMA has stopped."""
|
||||
worker = recv_worker
|
||||
worker._recving_transfers["request"] = [101]
|
||||
worker.nixl_wrapper.check_xfer_state.side_effect = [RuntimeError("poll"), "DONE"]
|
||||
worker.nixl_wrapper.release_xfer_handle.side_effect = [
|
||||
RuntimeError("release"),
|
||||
None,
|
||||
]
|
||||
|
||||
# Only the last terminal handle permits the scheduler to reuse the blocks.
|
||||
with patch.object(worker.nixl_wrapper, "check_xfer_state", return_value="ERR"):
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
results = worker.get_transfer_results()
|
||||
assert results.finished_recving == results.failed_recving == set()
|
||||
assert worker.get_block_ids_with_load_errors() == set()
|
||||
assert "request" in worker._recving_metadata
|
||||
|
||||
assert done_recving == {request_id}
|
||||
assert request_id not in worker._recving_transfers
|
||||
assert request_id not in worker._recving_metadata
|
||||
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
assert done_recving == set()
|
||||
results = worker.get_transfer_results()
|
||||
assert results.finished_recving == results.failed_recving == {"request"}
|
||||
assert worker.get_block_ids_with_load_errors() == {1, 2, 3}
|
||||
assert worker.nixl_wrapper.release_xfer_handle.call_count == 2
|
||||
|
||||
|
||||
def _set_test_speculative_config(
|
||||
|
||||
@@ -351,7 +351,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
# Base worker fields touched by start_load_kv / _get_new_notifs.
|
||||
w._recving_metadata = {}
|
||||
w._pending_recv_notifs = {}
|
||||
w._failed_inflight_recvs = set()
|
||||
w._recv_failures = set()
|
||||
w._recving_transfers = defaultdict(list)
|
||||
w._reqs_to_process = set()
|
||||
w._reqs_to_send = {}
|
||||
@@ -877,6 +877,68 @@ class TestPushWriterNotifs:
|
||||
assert evicted == ["req-done"]
|
||||
assert w._push_writer_wake.is_set()
|
||||
|
||||
@staticmethod
|
||||
def _pollable_worker() -> _StubWriterWorker:
|
||||
"""Stub worker with the extra base-worker state the real
|
||||
``get_transfer_results`` needs to poll ``_sending_transfers``."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.transfer_topo = MagicMock()
|
||||
w.nixl_wrapper = MagicMock()
|
||||
w.xfer_stats = MagicMock()
|
||||
w._log_failure = MagicMock() # type: ignore[method-assign]
|
||||
w._failed_recv_reqs = queue.Queue()
|
||||
w._invalid_block_ids = queue.Queue()
|
||||
w._pending_recv_notifs = {}
|
||||
w._replicated_pcp_done_sending = set()
|
||||
w.use_host_buffer = False
|
||||
return w
|
||||
|
||||
def _make_sending_req(self, w: _StubWriterWorker) -> str:
|
||||
request_id = "req-send-1"
|
||||
w._sending_transfers[request_id] = [101, 102]
|
||||
w._reqs_to_send[request_id] = time.perf_counter() + 60
|
||||
w._reqs_to_process.add(request_id)
|
||||
return request_id
|
||||
|
||||
def test_failed_push_send_is_not_reported_done_sending(self):
|
||||
"""A failed P-side WRITE must not surface as finished sending.
|
||||
|
||||
Reporting it would free P-side blocks as if the KV had been
|
||||
delivered while D still waits for the data; the request must be
|
||||
left to the lease / watchdog instead.
|
||||
"""
|
||||
w = self._pollable_worker()
|
||||
request_id = self._make_sending_req(w)
|
||||
w.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "DONE"]
|
||||
|
||||
results = w.get_transfer_results()
|
||||
|
||||
assert request_id not in results.finished_sending
|
||||
# Lease tracking is preserved so the watchdog can reschedule.
|
||||
assert request_id in w._reqs_to_send
|
||||
assert request_id in w._reqs_to_process
|
||||
# Both handles were still cleaned up.
|
||||
assert request_id not in w._sending_transfers
|
||||
|
||||
def test_completed_push_send_waits_for_consumer_notif(self):
|
||||
"""P-side sends are completed by consumer notifs / lease expiry,
|
||||
not by local WRITE-handle completion."""
|
||||
w = self._pollable_worker()
|
||||
request_id = self._make_sending_req(w)
|
||||
w.nixl_wrapper.check_xfer_state.return_value = "DONE"
|
||||
|
||||
results = w.get_transfer_results()
|
||||
|
||||
assert request_id not in results.finished_sending
|
||||
assert request_id in w._reqs_to_send
|
||||
assert request_id not in w._sending_transfers
|
||||
|
||||
# The consumer notif is what completes the send.
|
||||
w._pending_completion_notifs.put(f"{request_id}:1".encode())
|
||||
results = w.get_transfer_results()
|
||||
assert request_id in results.finished_sending
|
||||
assert request_id not in w._reqs_to_send
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- #
|
||||
# Negative / error-path tests #
|
||||
@@ -1079,7 +1141,7 @@ class TestPushWriterNegative:
|
||||
# Wake set so the writer drains NIXL notifs even when idle.
|
||||
assert w._push_writer_wake.is_set()
|
||||
|
||||
def test_failed_push_transfer_finishes_sending(self):
|
||||
def test_failed_push_transfer_does_not_evict_writer_state(self):
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.nixl_wrapper = MagicMock()
|
||||
w.nixl_wrapper.check_xfer_state.return_value = "ERR"
|
||||
@@ -1096,11 +1158,12 @@ class TestPushWriterNegative:
|
||||
):
|
||||
results = w.get_transfer_results()
|
||||
|
||||
assert results.finished_sending == {"req-failed"}
|
||||
assert results.finished_sending == set()
|
||||
assert results.finished_recving == set()
|
||||
assert "req-failed" not in w._sending_transfers
|
||||
assert "req-failed" not in w._reqs_to_send
|
||||
assert "req-failed" not in w._reqs_to_process
|
||||
assert "req-failed" in w._reqs_to_send
|
||||
assert "req-failed" in w._reqs_to_process
|
||||
assert w._evict_finished_inbox.empty()
|
||||
w.nixl_wrapper.release_xfer_handle.assert_called_once_with(17)
|
||||
|
||||
def test_get_new_notifs_unknown_request_is_logged_and_skipped(self, caplog):
|
||||
|
||||
@@ -692,7 +692,7 @@ class NixlBaseConnectorWorker:
|
||||
# Uses Queue for thread-safe cross-thread coordination with the
|
||||
# background handshake thread, matching the _ready_requests pattern.
|
||||
self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue()
|
||||
self._failed_inflight_recvs: set[ReqId] = set()
|
||||
self._recv_failures: set[ReqId] = set()
|
||||
self._pending_recv_notifs: dict[ReqId, list[tuple[str, bytes]]] = {}
|
||||
|
||||
# Handshake metadata of this worker for NIXL transfers.
|
||||
@@ -1212,7 +1212,7 @@ class NixlBaseConnectorWorker:
|
||||
error=e,
|
||||
meta=meta,
|
||||
)
|
||||
self._handle_failed_transfer(req_id, None)
|
||||
self._failed_recv_reqs.put(req_id)
|
||||
|
||||
fut.add_done_callback(request_ready)
|
||||
|
||||
@@ -2618,22 +2618,32 @@ class NixlBaseConnectorWorker:
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
done_sending = self._get_new_notifs()
|
||||
done_recving = self._pop_done_transfers(self._recving_transfers)
|
||||
done_recving, newly_failed = self._pop_done_transfers(self._recving_transfers)
|
||||
if newly_failed:
|
||||
self._recv_failures.update(newly_failed)
|
||||
|
||||
done_sending.update(self._replicated_pcp_done_sending)
|
||||
self._replicated_pcp_done_sending.clear()
|
||||
|
||||
# Drain queue of requests where handshake or transfer setup failed.
|
||||
failed_recv_reqs = set[ReqId]()
|
||||
# Process receive failures reported by background threads.
|
||||
while not self._failed_recv_reqs.empty():
|
||||
try:
|
||||
failed_recv_reqs.add(self._failed_recv_reqs.get_nowait())
|
||||
req_id = self._failed_recv_reqs.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self._handle_failed_transfer(req_id, None, self._recv_failures)
|
||||
|
||||
failed_recv_reqs: set[ReqId] = set()
|
||||
if self._recv_failures:
|
||||
# Keep failure bookkeeping independent of healthy request count.
|
||||
for req_id in tuple(self._recv_failures):
|
||||
if req_id not in self._recving_metadata:
|
||||
self._recv_failures.remove(req_id)
|
||||
elif req_id not in self._recving_transfers:
|
||||
self._recv_failures.remove(req_id)
|
||||
failed_recv_reqs.add(req_id)
|
||||
done_recving.update(failed_recv_reqs)
|
||||
|
||||
# Add failed requests to done_recving for scheduler tracking
|
||||
# (blocks are already marked invalid, scheduler will handle recompute)
|
||||
done_recving.update(failed_recv_reqs)
|
||||
if len(done_sending) > 0 or len(done_recving) > 0:
|
||||
logger.debug(
|
||||
"Rank %s, get_finished: %s requests done sending "
|
||||
@@ -2654,12 +2664,17 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
# Skip KV sync and post-processing for failed requests
|
||||
if req_id in failed_recv_reqs:
|
||||
self._pending_recv_notifs.pop(req_id, None)
|
||||
# TODO (NickLucche) handle failed transfer for HMA.
|
||||
if not self._is_hma_required:
|
||||
self._invalid_block_ids.put(set(meta.local_block_ids[0]))
|
||||
logger.warning(
|
||||
"Skipping KV post-processing for failed request %s",
|
||||
req_id,
|
||||
)
|
||||
continue
|
||||
|
||||
self._send_pending_recv_notifs(req_id)
|
||||
assert meta.remote is not None
|
||||
if self.use_host_buffer:
|
||||
self.sync_recved_kv_to_device(req_id, meta)
|
||||
@@ -2783,22 +2798,26 @@ class NixlBaseConnectorWorker:
|
||||
new_expiry,
|
||||
)
|
||||
|
||||
def _pop_done_transfers(self, transfers: dict[str, list[int]]) -> set[str]:
|
||||
"""
|
||||
Pop completed xfers by checking for DONE state.
|
||||
def _pop_done_transfers(
|
||||
self, transfers: dict[str, list[int]]
|
||||
) -> tuple[set[str], set[str]]:
|
||||
"""Poll transfers, retaining handles until they can be released.
|
||||
|
||||
Args:
|
||||
transfers: dict of req_id -> list[running_xfer]
|
||||
transfers: Outstanding handles by request, updated in place.
|
||||
|
||||
Returns:
|
||||
set of req_ids that have all done xfers
|
||||
Requests with no outstanding handles and requests with observed
|
||||
failures. Failed requests may still have outstanding handles.
|
||||
"""
|
||||
done_req_ids: set[str] = set()
|
||||
failed_req_ids: set[str] = set()
|
||||
for req_id, handles in list(transfers.items()):
|
||||
in_progress = []
|
||||
for handle in handles:
|
||||
try:
|
||||
xfer_state = self.nixl_wrapper.check_xfer_state(handle)
|
||||
if xfer_state == "DONE":
|
||||
# Get telemetry from NIXL
|
||||
res = self.nixl_wrapper.get_xfer_telemetry(handle)
|
||||
self.xfer_stats.record_transfer(res)
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
@@ -2807,58 +2826,52 @@ class NixlBaseConnectorWorker:
|
||||
else:
|
||||
self._log_failure(
|
||||
failure_type="transfer_failed",
|
||||
msg="Deferring request completion until its last "
|
||||
"xfer is terminal",
|
||||
req_id=req_id,
|
||||
xfer_state=xfer_state,
|
||||
)
|
||||
self._handle_failed_transfer(req_id, handle)
|
||||
if not self._handle_failed_transfer(
|
||||
req_id, handle, failed_req_ids
|
||||
):
|
||||
in_progress.append(handle)
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="transfer_exception",
|
||||
msg="Handle is unpollable; treating it as terminal",
|
||||
req_id=req_id,
|
||||
error=e,
|
||||
)
|
||||
self._handle_failed_transfer(req_id, handle)
|
||||
if not self._handle_failed_transfer(req_id, handle, failed_req_ids):
|
||||
in_progress.append(handle)
|
||||
|
||||
if not in_progress:
|
||||
del transfers[req_id]
|
||||
done_req_ids.add(req_id)
|
||||
if req_id in self._failed_inflight_recvs:
|
||||
self._failed_inflight_recvs.remove(req_id)
|
||||
self._report_failed_recv(req_id)
|
||||
else:
|
||||
self._send_pending_recv_notifs(req_id)
|
||||
del transfers[req_id]
|
||||
else:
|
||||
transfers[req_id] = in_progress
|
||||
return done_req_ids
|
||||
return done_req_ids, failed_req_ids
|
||||
|
||||
def _handle_failed_transfer(self, req_id: str, handle: int | None):
|
||||
"""
|
||||
Handle a failed transfer by marking all (logical) blocks as invalid and
|
||||
recording the failure.
|
||||
|
||||
Args:
|
||||
req_id: The request ID.
|
||||
handle: The transfer handle.
|
||||
"""
|
||||
# A sibling READ may still be writing these blocks. Retain metadata
|
||||
# and defer invalidation until every handle is terminal.
|
||||
self._pending_recv_notifs.pop(req_id, None)
|
||||
if req_id in self._recving_transfers:
|
||||
self._failed_inflight_recvs.add(req_id)
|
||||
else:
|
||||
self._report_failed_recv(req_id)
|
||||
if handle is not None:
|
||||
def _try_release_xfer_handle(self, req_id: str, handle: int) -> bool:
|
||||
"""Release a handle, returning False if the caller must retain it."""
|
||||
try:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
except Exception as e:
|
||||
# A status error does not guarantee that the backend stopped DMA.
|
||||
self._log_failure(
|
||||
failure_type="transfer_release_failed",
|
||||
msg="Retaining handle and blocks until release succeeds",
|
||||
req_id=req_id,
|
||||
error=e,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _report_failed_recv(self, req_id: str) -> None:
|
||||
if (meta := self._recving_metadata.get(req_id)) is not None:
|
||||
if not self._is_hma_required:
|
||||
self._invalid_block_ids.put(set(meta.local_block_ids[0]))
|
||||
self._failed_recv_reqs.put(req_id)
|
||||
def _handle_failed_transfer(
|
||||
self, req_id: str, handle: int | None, failed_req_ids: set[str] | None = None
|
||||
) -> bool:
|
||||
"""Record a failure and release its handle, returning False to retain it."""
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
if failed_req_ids is not None:
|
||||
failed_req_ids.add(req_id)
|
||||
return handle is None or self._try_release_xfer_handle(req_id, handle)
|
||||
|
||||
def _send_pending_recv_notifs(self, req_id: str) -> None:
|
||||
"""Send notifications deferred by split DRAM/VRAM reads."""
|
||||
|
||||
@@ -149,7 +149,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
engine_id,
|
||||
)
|
||||
self.xfer_stats.record_kv_expired_req()
|
||||
self._handle_failed_transfer(req_id, None)
|
||||
self._handle_failed_transfer(req_id, None, self._recv_failures)
|
||||
return
|
||||
|
||||
if any(len(group) > 0 for group in meta.local_block_ids):
|
||||
@@ -493,16 +493,19 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
return True
|
||||
except Exception as e:
|
||||
# mark all (logical) blocks for this request as invalid
|
||||
self._log_failure(
|
||||
failure_type="transfer_setup_failed",
|
||||
req_id=request_id,
|
||||
msg="Marking blocks as invalid",
|
||||
msg="Deferring failure reporting until outstanding transfers finish",
|
||||
error=e,
|
||||
dst_engine_id=dst_engine_id,
|
||||
remote_rank=remote_rank,
|
||||
)
|
||||
self._handle_failed_transfer(request_id, handle)
|
||||
if not self._handle_failed_transfer(
|
||||
request_id, handle, self._recv_failures
|
||||
):
|
||||
assert handle is not None
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
return False
|
||||
|
||||
def _read_blocks_mixed(
|
||||
@@ -544,7 +547,8 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
)
|
||||
except Exception:
|
||||
for handle in handles:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
if not self._try_release_xfer_handle(request_id, handle):
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
raise
|
||||
|
||||
self._pending_recv_notifs.setdefault(request_id, []).append(
|
||||
@@ -555,7 +559,8 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
self.nixl_wrapper.transfer(handle)
|
||||
except Exception:
|
||||
for unstarted in handles[i:]:
|
||||
self.nixl_wrapper.release_xfer_handle(unstarted)
|
||||
if not self._try_release_xfer_handle(request_id, unstarted):
|
||||
self._recving_transfers[request_id].append(unstarted)
|
||||
raise
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
|
||||
|
||||
@@ -341,7 +341,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
self._log_failure(
|
||||
failure_type="push_reg_handshake_failed", req_id=rid, error=e
|
||||
)
|
||||
self._handle_failed_transfer(rid, None)
|
||||
self._failed_recv_reqs.put(rid)
|
||||
return
|
||||
# Re-queue for the writer to send now that the handshake is done.
|
||||
self._reg_send_inbox.put((rid, rd))
|
||||
@@ -363,7 +363,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
engine_id,
|
||||
req_id,
|
||||
)
|
||||
self._handle_failed_transfer(req_id, None)
|
||||
self._failed_recv_reqs.put(req_id)
|
||||
return
|
||||
for rank, agent_name in agents.items():
|
||||
try:
|
||||
@@ -722,9 +722,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
# don't have a ``_recving_metadata`` entry to invalidate, so
|
||||
# we just release the handle and let the engine reschedule
|
||||
# via the lease / watchdog.
|
||||
if handle is not None:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
if not self._handle_failed_transfer(request_id, handle):
|
||||
return handle
|
||||
return None
|
||||
|
||||
# --- Notification handling on engine main thread ------------------ #
|
||||
@@ -803,7 +802,16 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
# ``_pop_done_transfers`` mutates ``_sending_transfers``; the
|
||||
# writer thread also appends to it, so guard the pop.
|
||||
with self._sending_transfers_lock:
|
||||
done_pushing = self._pop_done_transfers(self._sending_transfers)
|
||||
done_pushing, failed_pushing = self._pop_done_transfers(
|
||||
self._sending_transfers
|
||||
)
|
||||
# A failed send must never be reported as done: its blocks
|
||||
# are freed via the lease / watchdog instead.
|
||||
done_pushing = {
|
||||
req_id
|
||||
for req_id in done_pushing - failed_pushing
|
||||
if req_id in self._recving_metadata
|
||||
}
|
||||
for req_id in done_pushing:
|
||||
self._reqs_to_send.pop(req_id, None)
|
||||
self._reqs_to_process.discard(req_id)
|
||||
|
||||
Reference in New Issue
Block a user