[Bugfix][KV Offload][P2P] Preserve aborted loads until abort completion (#52571)

Signed-off-by: Uttam Kumar <uttam.kumar912@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Uttam Kumar
2026-08-31 14:34:48 +03:00
committed by GitHub
co-authored by Or Ozeri
parent e2c8eeac40
commit e0d27040dd
2 changed files with 97 additions and 11 deletions
@@ -534,6 +534,94 @@ class TestClientFlows:
assert abort[TYPE_KEY] == AbortFetchMsg.TYPE assert abort[TYPE_KEY] == AbortFetchMsg.TYPE
assert abort[AbortFetchMsg.KV_REQUEST_ID] == "req-1" assert abort[AbortFetchMsg.KV_REQUEST_ID] == "req-1"
def test_finish_preserves_active_load_until_terminal_result(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=7, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
session.finish_request("req-1")
load = _client_load(session, "req-1")
assert load.job_id == 7
assert load.aborted_at is not None
aborted_at = load.aborted_at
assert session._client.has_active_loads is True
session.finish_request("req-1")
assert _client_load(session, "req-1").aborted_at == aborted_at
aborts = [m for m in conn._sent if m[TYPE_KEY] == AbortFetchMsg.TYPE]
assert len(aborts) == 1
assert aborts[0][AbortFetchMsg.ROUND_SEQ] == 0
def test_finish_abort_ack_emits_one_failure(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=8, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
session.finish_request("req-1")
conn.enqueue(
{
TYPE_KEY: AbortAckMsg.TYPE,
AbortAckMsg.ROUND_SEQ: 0,
AbortAckMsg.KV_REQUEST_ID: "req-1",
}
)
loads = session.poll().loads
assert loads == [LoadResult(job_id=8, kv_request_id="req-1", success=False)]
assert session._client.has_active_loads is False
assert "req-1" not in session._client._requests
assert session.poll().loads == []
def test_finish_abort_timeout_emits_one_failure(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=9, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
session.finish_request("req-1")
_client_load(session, "req-1").aborted_at = (
time.monotonic() - _ABORT_ACK_TIMEOUT_S - 1.0
)
loads = session.poll().loads
assert loads == [LoadResult(job_id=9, kv_request_id="req-1", success=False)]
assert session._client.has_active_loads is False
assert "req-1" not in session._client._requests
assert session.poll().loads == []
def test_late_transfer_done_after_abort_ack_is_ignored(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=10, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
session.finish_request("req-1")
conn.enqueue(
{
TYPE_KEY: AbortAckMsg.TYPE,
AbortAckMsg.ROUND_SEQ: 0,
AbortAckMsg.KV_REQUEST_ID: "req-1",
}
)
assert session.poll().loads == [
LoadResult(job_id=10, kv_request_id="req-1", success=False)
]
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: True,
}
)
assert session.poll().loads == []
assert session._client.has_active_loads is False
def test_active_loads_work_list_tracks_in_flight(self): def test_active_loads_work_list_tracks_in_flight(self):
"""collect_results / has_active_loads use the _active_loads work-list, """collect_results / has_active_loads use the _active_loads work-list,
armed when a fetch is issued and discarded exactly when its load armed when a fetch is issued and discarded exactly when its load
@@ -130,10 +130,10 @@ class ClientRole:
# _serve_pending. Populated by register_lookup, drained by # _serve_pending. Populated by register_lookup, drained by
# flush_pending_lookups, and discarded on finish/close. # flush_pending_lookups, and discarded on finish/close.
self._flush_pending: set[str] = set() self._flush_pending: set[str] = set()
# kv_request_ids with at least one fetch in flight — the work-list # kv_request_ids with at least one fetch in flight or aborting — the
# collect_results walks for timeouts, and the has_active_loads # work-list collect_results walks for timeouts, and the
# predicate, instead of scanning every request. Kept in exact sync # has_active_loads predicate, instead of scanning every request.
# with ``st.loads``. # Kept in exact sync with ``st.loads``.
self._active_loads: set[str] = set() self._active_loads: set[str] = set()
self._completed_loads: list[LoadResult] = [] self._completed_loads: list[LoadResult] = []
@@ -241,7 +241,8 @@ class ClientRole:
while an earlier round's load is still in flight, so both can be while an earlier round's load is still in flight, so both can be
owed at once. owed at once.
Then drop all probe/lookup state and prune the entry. Then drop all probe/lookup state. Keep aborted loads until their abort
acknowledgement or timeout produces a terminal result.
""" """
st = self._requests.get(kv_request_id) st = self._requests.get(kv_request_id)
if st is None: if st is None:
@@ -257,8 +258,7 @@ class ClientRole:
AbortFetchMsg.ROUND_SEQ: round_seq, AbortFetchMsg.ROUND_SEQ: round_seq,
} }
) )
st.loads.clear() load.aborted_at = time.monotonic()
self._active_loads.discard(kv_request_id)
if st.peer_lookup_open: if st.peer_lookup_open:
st.peer_lookup_open = False st.peer_lookup_open = False
self._send( self._send(
@@ -311,10 +311,8 @@ class ClientRole:
load = st.loads.pop(round_seq, None) if st is not None else None load = st.loads.pop(round_seq, None) if st is not None else None
if st is not None and load is not None: if st is not None and load is not None:
logger.warning( logger.warning(
"P2PSession %s: load request %s (job_id=%d) timed out; " "P2PSession %s: load request %s (job_id=%d) "
"load job completed with failure. If this recurs, ensure all " "completed with failure after abort",
"nodes use the same prefix-cache hash seed (matching "
"PYTHONHASHSEED, if set) and hash algorithm.",
self._peer_id, self._peer_id,
kv_request_id, kv_request_id,
load.job_id, load.job_id,