[Bugfix][KV Offload][P2P] Preserve aborted loads until abort completion (#52571)
Signed-off-by: Uttam Kumar <uttam.kumar912@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -534,6 +534,94 @@ class TestClientFlows:
|
|||||||
assert abort[TYPE_KEY] == AbortFetchMsg.TYPE
|
assert abort[TYPE_KEY] == AbortFetchMsg.TYPE
|
||||||
assert abort[AbortFetchMsg.KV_REQUEST_ID] == "req-1"
|
assert abort[AbortFetchMsg.KV_REQUEST_ID] == "req-1"
|
||||||
|
|
||||||
|
def test_finish_preserves_active_load_until_terminal_result(self):
|
||||||
|
session, conn, _ = _make_session()
|
||||||
|
_activate(session, conn)
|
||||||
|
session.request_blocks(
|
||||||
|
job_id=7, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
session.finish_request("req-1")
|
||||||
|
|
||||||
|
load = _client_load(session, "req-1")
|
||||||
|
assert load.job_id == 7
|
||||||
|
assert load.aborted_at is not None
|
||||||
|
aborted_at = load.aborted_at
|
||||||
|
assert session._client.has_active_loads is True
|
||||||
|
session.finish_request("req-1")
|
||||||
|
assert _client_load(session, "req-1").aborted_at == aborted_at
|
||||||
|
aborts = [m for m in conn._sent if m[TYPE_KEY] == AbortFetchMsg.TYPE]
|
||||||
|
assert len(aborts) == 1
|
||||||
|
assert aborts[0][AbortFetchMsg.ROUND_SEQ] == 0
|
||||||
|
|
||||||
|
def test_finish_abort_ack_emits_one_failure(self):
|
||||||
|
session, conn, _ = _make_session()
|
||||||
|
_activate(session, conn)
|
||||||
|
session.request_blocks(
|
||||||
|
job_id=8, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
|
||||||
|
)
|
||||||
|
session.finish_request("req-1")
|
||||||
|
|
||||||
|
conn.enqueue(
|
||||||
|
{
|
||||||
|
TYPE_KEY: AbortAckMsg.TYPE,
|
||||||
|
AbortAckMsg.ROUND_SEQ: 0,
|
||||||
|
AbortAckMsg.KV_REQUEST_ID: "req-1",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
loads = session.poll().loads
|
||||||
|
assert loads == [LoadResult(job_id=8, kv_request_id="req-1", success=False)]
|
||||||
|
assert session._client.has_active_loads is False
|
||||||
|
assert "req-1" not in session._client._requests
|
||||||
|
assert session.poll().loads == []
|
||||||
|
|
||||||
|
def test_finish_abort_timeout_emits_one_failure(self):
|
||||||
|
session, conn, _ = _make_session()
|
||||||
|
_activate(session, conn)
|
||||||
|
session.request_blocks(
|
||||||
|
job_id=9, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
|
||||||
|
)
|
||||||
|
session.finish_request("req-1")
|
||||||
|
_client_load(session, "req-1").aborted_at = (
|
||||||
|
time.monotonic() - _ABORT_ACK_TIMEOUT_S - 1.0
|
||||||
|
)
|
||||||
|
|
||||||
|
loads = session.poll().loads
|
||||||
|
assert loads == [LoadResult(job_id=9, kv_request_id="req-1", success=False)]
|
||||||
|
assert session._client.has_active_loads is False
|
||||||
|
assert "req-1" not in session._client._requests
|
||||||
|
assert session.poll().loads == []
|
||||||
|
|
||||||
|
def test_late_transfer_done_after_abort_ack_is_ignored(self):
|
||||||
|
session, conn, _ = _make_session()
|
||||||
|
_activate(session, conn)
|
||||||
|
session.request_blocks(
|
||||||
|
job_id=10, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
|
||||||
|
)
|
||||||
|
session.finish_request("req-1")
|
||||||
|
|
||||||
|
conn.enqueue(
|
||||||
|
{
|
||||||
|
TYPE_KEY: AbortAckMsg.TYPE,
|
||||||
|
AbortAckMsg.ROUND_SEQ: 0,
|
||||||
|
AbortAckMsg.KV_REQUEST_ID: "req-1",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
assert session.poll().loads == [
|
||||||
|
LoadResult(job_id=10, kv_request_id="req-1", success=False)
|
||||||
|
]
|
||||||
|
|
||||||
|
conn.enqueue(
|
||||||
|
{
|
||||||
|
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||||
|
TransferDoneMsg.ROUND_SEQ: 0,
|
||||||
|
TransferDoneMsg.KV_REQUEST_ID: "req-1",
|
||||||
|
TransferDoneMsg.SUCCESS: True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
assert session.poll().loads == []
|
||||||
|
assert session._client.has_active_loads is False
|
||||||
|
|
||||||
def test_active_loads_work_list_tracks_in_flight(self):
|
def test_active_loads_work_list_tracks_in_flight(self):
|
||||||
"""collect_results / has_active_loads use the _active_loads work-list,
|
"""collect_results / has_active_loads use the _active_loads work-list,
|
||||||
armed when a fetch is issued and discarded exactly when its load
|
armed when a fetch is issued and discarded exactly when its load
|
||||||
|
|||||||
@@ -130,10 +130,10 @@ class ClientRole:
|
|||||||
# _serve_pending. Populated by register_lookup, drained by
|
# _serve_pending. Populated by register_lookup, drained by
|
||||||
# flush_pending_lookups, and discarded on finish/close.
|
# flush_pending_lookups, and discarded on finish/close.
|
||||||
self._flush_pending: set[str] = set()
|
self._flush_pending: set[str] = set()
|
||||||
# kv_request_ids with at least one fetch in flight — the work-list
|
# kv_request_ids with at least one fetch in flight or aborting — the
|
||||||
# collect_results walks for timeouts, and the has_active_loads
|
# work-list collect_results walks for timeouts, and the
|
||||||
# predicate, instead of scanning every request. Kept in exact sync
|
# has_active_loads predicate, instead of scanning every request.
|
||||||
# with ``st.loads``.
|
# Kept in exact sync with ``st.loads``.
|
||||||
self._active_loads: set[str] = set()
|
self._active_loads: set[str] = set()
|
||||||
self._completed_loads: list[LoadResult] = []
|
self._completed_loads: list[LoadResult] = []
|
||||||
|
|
||||||
@@ -241,7 +241,8 @@ class ClientRole:
|
|||||||
while an earlier round's load is still in flight, so both can be
|
while an earlier round's load is still in flight, so both can be
|
||||||
owed at once.
|
owed at once.
|
||||||
|
|
||||||
Then drop all probe/lookup state and prune the entry.
|
Then drop all probe/lookup state. Keep aborted loads until their abort
|
||||||
|
acknowledgement or timeout produces a terminal result.
|
||||||
"""
|
"""
|
||||||
st = self._requests.get(kv_request_id)
|
st = self._requests.get(kv_request_id)
|
||||||
if st is None:
|
if st is None:
|
||||||
@@ -257,8 +258,7 @@ class ClientRole:
|
|||||||
AbortFetchMsg.ROUND_SEQ: round_seq,
|
AbortFetchMsg.ROUND_SEQ: round_seq,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
st.loads.clear()
|
load.aborted_at = time.monotonic()
|
||||||
self._active_loads.discard(kv_request_id)
|
|
||||||
if st.peer_lookup_open:
|
if st.peer_lookup_open:
|
||||||
st.peer_lookup_open = False
|
st.peer_lookup_open = False
|
||||||
self._send(
|
self._send(
|
||||||
@@ -311,10 +311,8 @@ class ClientRole:
|
|||||||
load = st.loads.pop(round_seq, None) if st is not None else None
|
load = st.loads.pop(round_seq, None) if st is not None else None
|
||||||
if st is not None and load is not None:
|
if st is not None and load is not None:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"P2PSession %s: load request %s (job_id=%d) timed out; "
|
"P2PSession %s: load request %s (job_id=%d) "
|
||||||
"load job completed with failure. If this recurs, ensure all "
|
"completed with failure after abort",
|
||||||
"nodes use the same prefix-cache hash seed (matching "
|
|
||||||
"PYTHONHASHSEED, if set) and hash algorithm.",
|
|
||||||
self._peer_id,
|
self._peer_id,
|
||||||
kv_request_id,
|
kv_request_id,
|
||||||
load.job_id,
|
load.job_id,
|
||||||
|
|||||||
Reference in New Issue
Block a user