[Bugfix][KV Offload][P2P] Scope serve state to fetch rounds (#49877)
Signed-off-by: Itay Etelis <itay.etelis@ibm.com> Co-authored-by: Itay Etelis <itay.etelis@ibm.com> Co-authored-by: Itay Etelis <Itay.etelis@gmail.com>
This commit is contained in:
co-authored by
Itay Etelis
Itay Etelis
parent
0d0504b54c
commit
30217b0e80
@@ -1612,16 +1612,17 @@ class TestBindHostPortDefaults:
|
||||
monkeypatch.setattr(
|
||||
manager_module,
|
||||
"NixlTransport",
|
||||
lambda agent_name, *a, **k: calls.update(nixl_name=agent_name)
|
||||
or SimpleNamespace(),
|
||||
lambda agent_name, *a, **k: (
|
||||
calls.update(nixl_name=agent_name) or SimpleNamespace()
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
manager_module,
|
||||
"ZmqTransport",
|
||||
lambda local_id, host, port, *a, **k: calls.update(
|
||||
zmq_id=local_id, zmq_host=host, zmq_port=port
|
||||
)
|
||||
or SimpleNamespace(),
|
||||
lambda local_id, host, port, *a, **k: (
|
||||
calls.update(zmq_id=local_id, zmq_host=host, zmq_port=port)
|
||||
or SimpleNamespace()
|
||||
),
|
||||
)
|
||||
spec = SimpleNamespace(
|
||||
blocks_per_chunk=1,
|
||||
|
||||
@@ -49,6 +49,7 @@ from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.server import (
|
||||
_CANCEL_DRAIN_TIMEOUT_S,
|
||||
_InflightXfer,
|
||||
_OutboundRequestState,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.session import (
|
||||
_MAX_CONSECUTIVE_DISPATCH_ERRORS,
|
||||
@@ -315,10 +316,26 @@ def _activate(
|
||||
# either a missing entry or a None field. These helpers paper over that.
|
||||
|
||||
|
||||
def _client_load(session: P2PSession, kv_request_id: str):
|
||||
"""The single in-flight load of a kv_request_id (loads are per-round)."""
|
||||
loads = session._client._requests[kv_request_id].loads
|
||||
assert len(loads) == 1
|
||||
return next(iter(loads.values()))
|
||||
|
||||
|
||||
def _srv_outbound(session: P2PSession, kv_request_id: str):
|
||||
"""Outbound serve state for a kv_request_id, or None (idle / GC'd)."""
|
||||
"""Serve-side round for a kv_request_id, or None (idle / GC'd).
|
||||
|
||||
Rounds are keyed by wire round_seq; surfaces the demanded round when
|
||||
a fetch has bound one, else any parked supply round.
|
||||
"""
|
||||
st = session._server._requests.get(kv_request_id)
|
||||
return st.outbound if st is not None else None
|
||||
if st is None or not st.outbound:
|
||||
return None
|
||||
for rnd in st.outbound.values():
|
||||
if rnd.demand_received:
|
||||
return rnd
|
||||
return next(iter(st.outbound.values()))
|
||||
|
||||
|
||||
def _srv_lookups(session: P2PSession) -> list:
|
||||
@@ -330,8 +347,10 @@ def _srv_lookups(session: P2PSession) -> list:
|
||||
|
||||
def _srv_abort_started(session: P2PSession, kv_request_id: str) -> float | None:
|
||||
"""Pending-abort start time for a kv_request_id, or None."""
|
||||
st = session._server._requests.get(kv_request_id)
|
||||
return st.abort_started_at if st is not None else None
|
||||
for (kv, _), started in session._server._pending_aborts.items():
|
||||
if kv == kv_request_id:
|
||||
return started
|
||||
return None
|
||||
|
||||
|
||||
def _srv_inflight_count(session: P2PSession, kv_request_id: str) -> int:
|
||||
@@ -479,6 +498,7 @@ class TestClientFlows:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||
TransferDoneMsg.ROUND_SEQ: 0,
|
||||
TransferDoneMsg.KV_REQUEST_ID: "req-1",
|
||||
TransferDoneMsg.SUCCESS: True,
|
||||
}
|
||||
@@ -495,6 +515,7 @@ class TestClientFlows:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||
TransferDoneMsg.ROUND_SEQ: 0,
|
||||
TransferDoneMsg.KV_REQUEST_ID: "req-1",
|
||||
TransferDoneMsg.SUCCESS: False,
|
||||
}
|
||||
@@ -538,6 +559,7 @@ class TestClientFlows:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||
TransferDoneMsg.ROUND_SEQ: 0,
|
||||
TransferDoneMsg.KV_REQUEST_ID: "req-1",
|
||||
TransferDoneMsg.SUCCESS: True,
|
||||
}
|
||||
@@ -552,7 +574,7 @@ class TestClientFlows:
|
||||
session.request_blocks(
|
||||
job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
|
||||
)
|
||||
session._client._requests["req-1"].load.submitted_at = time.monotonic() - 60.0
|
||||
_client_load(session, "req-1").submitted_at = time.monotonic() - 60.0
|
||||
session.poll()
|
||||
abort = conn._sent[-1]
|
||||
assert abort[TYPE_KEY] == AbortFetchMsg.TYPE
|
||||
@@ -569,7 +591,7 @@ class TestClientFlows:
|
||||
job_id=7, kv_request_id="req-7", keys=[b"k"], block_ids=[0]
|
||||
)
|
||||
# 1) Trip the load timeout to send AbortFetch and stamp aborted_at.
|
||||
session._client._requests["req-7"].load.submitted_at = (
|
||||
_client_load(session, "req-7").submitted_at = (
|
||||
time.monotonic() - _LOAD_TIMEOUT_S - 1.0
|
||||
)
|
||||
loads = session.poll().loads
|
||||
@@ -579,11 +601,11 @@ class TestClientFlows:
|
||||
and m[AbortFetchMsg.KV_REQUEST_ID] == "req-7"
|
||||
for m in conn._sent
|
||||
)
|
||||
assert session._client._requests["req-7"].load.aborted_at is not None
|
||||
assert _client_load(session, "req-7").aborted_at is not None
|
||||
|
||||
# 2) Now backdate aborted_at past the abort-ack timeout. No ack ever
|
||||
# arrived from the peer.
|
||||
session._client._requests["req-7"].load.aborted_at = (
|
||||
_client_load(session, "req-7").aborted_at = (
|
||||
time.monotonic() - _ABORT_ACK_TIMEOUT_S - 1.0
|
||||
)
|
||||
loads = session.poll().loads
|
||||
@@ -599,17 +621,18 @@ class TestClientFlows:
|
||||
session.request_blocks(
|
||||
job_id=8, kv_request_id="req-8", keys=[b"k"], block_ids=[0]
|
||||
)
|
||||
session._client._requests["req-8"].load.submitted_at = (
|
||||
_client_load(session, "req-8").submitted_at = (
|
||||
time.monotonic() - _LOAD_TIMEOUT_S - 1.0
|
||||
)
|
||||
# First poll: AbortFetch goes out.
|
||||
session.poll()
|
||||
assert session._client._requests["req-8"].load.aborted_at is not None
|
||||
assert _client_load(session, "req-8").aborted_at is not None
|
||||
|
||||
# Peer acks the abort.
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: AbortAckMsg.TYPE,
|
||||
AbortAckMsg.ROUND_SEQ: 0,
|
||||
AbortAckMsg.KV_REQUEST_ID: "req-8",
|
||||
}
|
||||
)
|
||||
@@ -915,6 +938,7 @@ class TestLookupFlow:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: LookupMsg.TYPE,
|
||||
LookupMsg.ROUND_SEQ: 0,
|
||||
LookupMsg.KV_REQUEST_ID: "req-1",
|
||||
LookupMsg.KEYS: [b"hX", b"hY", b"hZ"],
|
||||
}
|
||||
@@ -951,6 +975,7 @@ def _send_lookup(conn: FakeConnection, kv_request_id: str, keys: list[bytes]):
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: LookupMsg.TYPE,
|
||||
LookupMsg.ROUND_SEQ: 0,
|
||||
LookupMsg.KV_REQUEST_ID: kv_request_id,
|
||||
LookupMsg.KEYS: list(keys),
|
||||
}
|
||||
@@ -1201,6 +1226,7 @@ class TestServerLookupHandling:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [],
|
||||
FetchMsg.BLOCK_INDEXES: [],
|
||||
@@ -1240,6 +1266,7 @@ class TestServerLookupHandling:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"hA", b"hB"],
|
||||
FetchMsg.BLOCK_INDEXES: [20, 21],
|
||||
@@ -1277,6 +1304,7 @@ class TestServerFlows:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1", b"k2"],
|
||||
FetchMsg.BLOCK_INDEXES: [10, 11],
|
||||
@@ -1295,6 +1323,7 @@ class TestServerFlows:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1"],
|
||||
FetchMsg.BLOCK_INDEXES: [5],
|
||||
@@ -1313,6 +1342,7 @@ class TestServerFlows:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1"],
|
||||
FetchMsg.BLOCK_INDEXES: [5],
|
||||
@@ -1331,6 +1361,7 @@ class TestServerFlows:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.ROUND_SEQ: 0,
|
||||
AbortFetchMsg.KV_REQUEST_ID: "req-1",
|
||||
}
|
||||
)
|
||||
@@ -1349,13 +1380,19 @@ class TestServerFlows:
|
||||
tid = 42
|
||||
session._server._inflight_add(
|
||||
tid,
|
||||
_InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}),
|
||||
_InflightXfer(
|
||||
kv_request_id="req-1",
|
||||
block_count=1,
|
||||
job_ids={1},
|
||||
round=_OutboundRequestState(inflight=1),
|
||||
),
|
||||
)
|
||||
transport._cancel_still_inflight.add(tid)
|
||||
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.ROUND_SEQ: 0,
|
||||
AbortFetchMsg.KV_REQUEST_ID: "req-1",
|
||||
}
|
||||
)
|
||||
@@ -1376,13 +1413,19 @@ class TestServerFlows:
|
||||
tid = 42
|
||||
session._server._inflight_add(
|
||||
tid,
|
||||
_InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}),
|
||||
_InflightXfer(
|
||||
kv_request_id="req-1",
|
||||
block_count=1,
|
||||
job_ids={1},
|
||||
round=_OutboundRequestState(inflight=1),
|
||||
),
|
||||
)
|
||||
transport._cancel_still_inflight.add(tid)
|
||||
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.ROUND_SEQ: 0,
|
||||
AbortFetchMsg.KV_REQUEST_ID: "req-1",
|
||||
}
|
||||
)
|
||||
@@ -1409,20 +1452,26 @@ class TestServerFlows:
|
||||
tid = 42
|
||||
session._server._inflight_add(
|
||||
tid,
|
||||
_InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}),
|
||||
_InflightXfer(
|
||||
kv_request_id="req-1",
|
||||
block_count=1,
|
||||
job_ids={1},
|
||||
round=_OutboundRequestState(inflight=1),
|
||||
),
|
||||
)
|
||||
transport._cancel_still_inflight.add(tid)
|
||||
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.ROUND_SEQ: 0,
|
||||
AbortFetchMsg.KV_REQUEST_ID: "req-1",
|
||||
}
|
||||
)
|
||||
session.poll()
|
||||
assert _srv_abort_started(session, "req-1") is not None
|
||||
# Backdate past the drain deadline.
|
||||
session._server._requests["req-1"].abort_started_at = (
|
||||
session._server._pending_aborts[("req-1", 0)] = (
|
||||
time.monotonic() - _CANCEL_DRAIN_TIMEOUT_S - 1.0
|
||||
)
|
||||
# Even if the transport still claims it can't cancel, the
|
||||
@@ -1445,13 +1494,19 @@ class TestServerFlows:
|
||||
tid = 42
|
||||
session._server._inflight_add(
|
||||
tid,
|
||||
_InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}),
|
||||
_InflightXfer(
|
||||
kv_request_id="req-1",
|
||||
block_count=1,
|
||||
job_ids={1},
|
||||
round=_OutboundRequestState(inflight=1),
|
||||
),
|
||||
)
|
||||
transport._cancel_still_inflight.add(tid)
|
||||
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.ROUND_SEQ: 0,
|
||||
AbortFetchMsg.KV_REQUEST_ID: "req-1",
|
||||
}
|
||||
)
|
||||
@@ -1463,6 +1518,7 @@ class TestServerFlows:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.ROUND_SEQ: 0,
|
||||
AbortFetchMsg.KV_REQUEST_ID: "req-1",
|
||||
}
|
||||
)
|
||||
@@ -1497,6 +1553,7 @@ class TestServerFlows:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1"],
|
||||
FetchMsg.BLOCK_INDEXES: [5],
|
||||
@@ -1529,6 +1586,7 @@ class TestServerFlows:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1"],
|
||||
FetchMsg.BLOCK_INDEXES: [5],
|
||||
@@ -1571,6 +1629,7 @@ class TestFinishRequestServerSide:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1"],
|
||||
FetchMsg.BLOCK_INDEXES: [5],
|
||||
@@ -1596,6 +1655,7 @@ class TestFinishRequestServerSide:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1", b"k2"],
|
||||
FetchMsg.BLOCK_INDEXES: [10, 11],
|
||||
@@ -1631,6 +1691,7 @@ class TestFinishRequestServerSide:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1"],
|
||||
FetchMsg.BLOCK_INDEXES: [10],
|
||||
@@ -1665,6 +1726,7 @@ class TestFinishRequestServerSide:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1"],
|
||||
FetchMsg.BLOCK_INDEXES: [10],
|
||||
@@ -1686,6 +1748,7 @@ class TestFinishRequestServerSide:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-2",
|
||||
FetchMsg.KEYS: [b"k1", b"k2"],
|
||||
FetchMsg.BLOCK_INDEXES: [10, 11],
|
||||
@@ -1719,6 +1782,7 @@ class TestFinishRequestServerSide:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"demand"],
|
||||
FetchMsg.BLOCK_INDEXES: [5],
|
||||
@@ -1753,6 +1817,7 @@ class TestFinishRequestServerSide:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1"],
|
||||
FetchMsg.BLOCK_INDEXES: [10],
|
||||
@@ -1790,6 +1855,7 @@ class TestFinishRequestServerSide:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1"],
|
||||
FetchMsg.BLOCK_INDEXES: [10],
|
||||
@@ -1824,6 +1890,7 @@ class TestFinishRequestServerSide:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1", b"k2", b"k3"],
|
||||
FetchMsg.BLOCK_INDEXES: [10, 11, 12],
|
||||
@@ -1885,6 +1952,7 @@ class TestFinishRequestServerSide:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1", b"k2"],
|
||||
FetchMsg.BLOCK_INDEXES: [10, 11],
|
||||
@@ -1951,6 +2019,7 @@ class TestBidirectional:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-srv",
|
||||
FetchMsg.KEYS: [b"served"],
|
||||
FetchMsg.BLOCK_INDEXES: [7],
|
||||
@@ -1981,6 +2050,7 @@ class TestBidirectional:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||
TransferDoneMsg.ROUND_SEQ: 0,
|
||||
TransferDoneMsg.KV_REQUEST_ID: "req-cli",
|
||||
TransferDoneMsg.SUCCESS: True,
|
||||
}
|
||||
@@ -2168,6 +2238,7 @@ class TestAdversarial:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-bad",
|
||||
FetchMsg.KEYS: [b"k1", b"k2"],
|
||||
FetchMsg.BLOCK_INDEXES: [1],
|
||||
@@ -2216,6 +2287,7 @@ class TestDispatchErrorHandling:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-bad",
|
||||
FetchMsg.KEYS: [b"k1", b"k2"],
|
||||
FetchMsg.BLOCK_INDEXES: [1],
|
||||
@@ -2246,6 +2318,7 @@ class TestDispatchErrorHandling:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1"],
|
||||
FetchMsg.BLOCK_INDEXES: [0],
|
||||
@@ -2269,6 +2342,7 @@ class TestDispatchErrorHandling:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1"],
|
||||
FetchMsg.BLOCK_INDEXES: [0],
|
||||
@@ -2296,6 +2370,7 @@ class TestDispatchErrorHandling:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1"],
|
||||
FetchMsg.BLOCK_INDEXES: [0],
|
||||
@@ -2341,6 +2416,7 @@ class TestInflightPerReqInvariant:
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: kv_id,
|
||||
FetchMsg.KEYS: keys,
|
||||
FetchMsg.BLOCK_INDEXES: indexes,
|
||||
@@ -2394,7 +2470,12 @@ class TestInflightPerReqInvariant:
|
||||
tid = kv_id_idx * 10 + j
|
||||
session._server._inflight_add(
|
||||
tid,
|
||||
_InflightXfer(kv_request_id=kv_id, block_count=1, job_ids={tid}),
|
||||
_InflightXfer(
|
||||
kv_request_id=kv_id,
|
||||
block_count=1,
|
||||
job_ids={tid},
|
||||
round=_OutboundRequestState(inflight=1),
|
||||
),
|
||||
)
|
||||
assert _srv_total_inflight(session) == len(session._server._inflight)
|
||||
assert session._server._has_inflight_for("req-0")
|
||||
@@ -2488,6 +2569,7 @@ class TestFetchMsgValidation:
|
||||
def _valid_msg(self) -> dict:
|
||||
return {
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1", b"k2"],
|
||||
FetchMsg.BLOCK_INDEXES: [0, 1],
|
||||
@@ -2513,6 +2595,7 @@ class TestTransferDoneMsgValidation:
|
||||
def test_valid_message_passes(self):
|
||||
msg = {
|
||||
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||
TransferDoneMsg.ROUND_SEQ: 0,
|
||||
TransferDoneMsg.KV_REQUEST_ID: "req-1",
|
||||
TransferDoneMsg.SUCCESS: True,
|
||||
}
|
||||
@@ -2521,6 +2604,7 @@ class TestTransferDoneMsgValidation:
|
||||
def test_success_wrong_type(self):
|
||||
msg = {
|
||||
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||
TransferDoneMsg.ROUND_SEQ: 0,
|
||||
TransferDoneMsg.KV_REQUEST_ID: "req-1",
|
||||
TransferDoneMsg.SUCCESS: 1,
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ callback injected by the coordinator (which gates on ConnectAck).
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import time
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
@@ -35,26 +34,12 @@ _LOAD_TIMEOUT_S = 30.0
|
||||
_ABORT_ACK_TIMEOUT_S = 10.0
|
||||
|
||||
|
||||
class ClientPhase(enum.Enum):
|
||||
"""Lifecycle of a request's client-side lookup/fetch signalling.
|
||||
|
||||
Advances monotonically. Only ``finish`` reads it, to decide
|
||||
whether a terminal empty FetchMsg is owed to release the peer's
|
||||
lookup state (owed only from ``PROBING``: a LookupMsg went out but no
|
||||
FetchMsg has since closed the peer's lookup phase).
|
||||
"""
|
||||
|
||||
REGISTERED = enum.auto() # keys in probes/unsent, nothing sent yet
|
||||
PROBING = enum.auto() # LookupMsg flushed, awaiting responses
|
||||
FETCH_SENT = enum.auto() # FetchMsg sent (real or terminal empty)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _InboundLoadState:
|
||||
"""Client-role state for a single in-flight load request.
|
||||
|
||||
Lives on ``_ClientRequestState.load`` for the duration of a fetch;
|
||||
the owning kv_request_id is the dict key, so it isn't stored here.
|
||||
Lives in ``_ClientRequestState.loads`` keyed by round_seq for the
|
||||
duration of a fetch; the owning kv_request_id is the outer dict key.
|
||||
"""
|
||||
|
||||
job_id: int # opaque ID assigned by the manager to this load request
|
||||
@@ -68,7 +53,7 @@ class _ClientRequestState:
|
||||
|
||||
One entry per kv_request_id we're driving. Lookup-phase fields are
|
||||
used only by symmetric P2P (``do_p2p_fetch``); PD-only loads leave
|
||||
``probes``/``unsent`` empty and drive just ``phase`` and ``load``. An
|
||||
``probes``/``unsent`` empty and drive just ``phase`` and ``loads``. An
|
||||
entry is dropped once every field is idle — see ``ClientRole._maybe_prune``.
|
||||
"""
|
||||
|
||||
@@ -82,11 +67,23 @@ class _ClientRequestState:
|
||||
# OffloadKeys registered but not yet flushed onto the wire. Drained and
|
||||
# cleared by the next flush_pending_lookups.
|
||||
unsent: list[OffloadKey] = field(default_factory=list)
|
||||
# Current lookup round. LookupMsgs carry it, each fetch closes it and
|
||||
# advances it, so every round's supply/demand/completion is isolated
|
||||
# on the wire. PD clients never probe and stay on round 0.
|
||||
round_seq: int = 0
|
||||
# This id ran the symmetric lookup phase (register_lookup); a fetch
|
||||
# with keys then requires every key to be a confirmed probe. PD
|
||||
# loads never probe.
|
||||
probed: bool = False
|
||||
|
||||
# Monotonic lookup/fetch signalling phase; see ``ClientPhase``.
|
||||
phase: ClientPhase = ClientPhase.REGISTERED
|
||||
# Set while a fetch is in flight; cleared on completion/abort/timeout.
|
||||
load: _InboundLoadState | None = None
|
||||
# The peer holds lookup state no FetchMsg has closed: a LookupMsg
|
||||
# was flushed since the last fetch. finish owes a terminal empty
|
||||
# FetchMsg while set, so the peer releases parked supply.
|
||||
peer_lookup_open: bool = False
|
||||
# In-flight loads keyed by the round their fetch carried. The
|
||||
# scheduler submits loads incrementally as chunks resolve, so several
|
||||
# can be in flight at once; TransferDone/AbortAck match by round.
|
||||
loads: dict[int, _InboundLoadState] = field(default_factory=dict)
|
||||
|
||||
|
||||
class LoadResult(NamedTuple):
|
||||
@@ -133,11 +130,10 @@ class ClientRole:
|
||||
# _serve_pending. Populated by register_lookup, drained by
|
||||
# flush_pending_lookups, and discarded on finish/close.
|
||||
self._flush_pending: set[str] = set()
|
||||
# kv_request_ids with a fetch in flight (``st.load is not None``) —
|
||||
# the work-list collect_results walks for timeouts, and the
|
||||
# has_active_loads predicate, instead of scanning every request.
|
||||
# Kept in exact sync with ``st.load``: armed in request_blocks,
|
||||
# discarded wherever load is cleared, and cleared on close.
|
||||
# kv_request_ids with at least one fetch in flight — the work-list
|
||||
# collect_results walks for timeouts, and the has_active_loads
|
||||
# predicate, instead of scanning every request. Kept in exact sync
|
||||
# with ``st.loads``.
|
||||
self._active_loads: set[str] = set()
|
||||
self._completed_loads: list[LoadResult] = []
|
||||
|
||||
@@ -156,17 +152,21 @@ class ClientRole:
|
||||
def _maybe_prune(self, kv_request_id: str) -> None:
|
||||
"""Drop the entry once it holds no live load or lookup state.
|
||||
|
||||
The sticky ``phase`` is only read by ``finish``. A probe
|
||||
clears when its fetch is issued (``request_blocks``) or when the
|
||||
request finishes (``finish``/``close``); in the former case
|
||||
``load`` is set and keeps the entry alive, in the latter the phase
|
||||
is no longer needed — so dropping on emptiness never loses a phase
|
||||
still in use.
|
||||
``peer_lookup_open`` is only read by ``finish``, and every path
|
||||
that clears the last probe (fetch / finish / close) also settles
|
||||
it, so dropping on emptiness never loses a flag still in use.
|
||||
"""
|
||||
st = self._requests.get(kv_request_id)
|
||||
if st is not None and st.load is None and not st.probes and not st.unsent:
|
||||
if st is not None and not st.loads and not st.probes and not st.unsent:
|
||||
del self._requests[kv_request_id]
|
||||
|
||||
def _on_load_terminal(self, kv_request_id: str, st: _ClientRequestState) -> None:
|
||||
"""Wind down id-level state once no load remains in flight."""
|
||||
if st.loads:
|
||||
return
|
||||
self._active_loads.discard(kv_request_id)
|
||||
self._maybe_prune(kv_request_id)
|
||||
|
||||
@property
|
||||
def has_active_loads(self) -> bool:
|
||||
"""True if any kv_request_id has a fetch in flight."""
|
||||
@@ -184,7 +184,12 @@ class ClientRole:
|
||||
block_ids: Sequence[int],
|
||||
send_ready: bool,
|
||||
) -> None:
|
||||
"""Register a load request and send the FetchMsg."""
|
||||
"""Send the FetchMsg closing the current lookup round.
|
||||
|
||||
The scheduler may submit several loads per kv_request_id as its
|
||||
matched prefix resolves incrementally; each fetch carries the
|
||||
round it closes so the loads stay independent on the wire.
|
||||
"""
|
||||
logger.debug(
|
||||
"P2PSession %s: request_blocks job_id=%d kv_request_id=%s "
|
||||
"blocks=%d ready=%s",
|
||||
@@ -195,77 +200,74 @@ class ClientRole:
|
||||
send_ready,
|
||||
)
|
||||
st = self._get_or_create_request(kv_request_id)
|
||||
st.load = _InboundLoadState(
|
||||
round_seq = st.round_seq
|
||||
st.round_seq += 1
|
||||
st.loads[round_seq] = _InboundLoadState(
|
||||
job_id=job_id,
|
||||
submitted_at=time.monotonic(),
|
||||
)
|
||||
self._active_loads.add(kv_request_id)
|
||||
st.phase = ClientPhase.FETCH_SENT
|
||||
st.peer_lookup_open = False
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.KV_REQUEST_ID: kv_request_id,
|
||||
FetchMsg.KEYS: list(keys),
|
||||
FetchMsg.BLOCK_INDEXES: [int(idx) for idx in block_ids],
|
||||
FetchMsg.ROUND_SEQ: round_seq,
|
||||
}
|
||||
)
|
||||
# Issuing the fetch ends this request's lookup phase, so drop all
|
||||
# probe state. Once the peer serves this fetch both sides unpin, so
|
||||
# the producer may evict the block; a stale cached True would
|
||||
# otherwise let a re-scheduled lookup() return HIT without
|
||||
# re-probing, pointing at a block the producer no longer holds.
|
||||
# Clearing forces a fresh LookupMsg on re-schedule so the producer
|
||||
# answers from current state. For a symmetric-P2P request (probes
|
||||
# populated) every fetched block was a confirmed HIT; a PD-only load
|
||||
# never probes, so probes is empty and the clear is a no-op.
|
||||
if st.probes:
|
||||
# Issuing the fetch closes this lookup round, so drop all probe
|
||||
# state. Once the peer serves the fetch both sides unpin, so a
|
||||
# stale cached True would let a later lookup() return HIT for a
|
||||
# block the producer may have evicted; clearing forces a fresh
|
||||
# probe under the next round.
|
||||
if st.probed and keys:
|
||||
assert st.probes, (
|
||||
f"symmetric fetch for {kv_request_id} has keys but no probes"
|
||||
)
|
||||
assert all(st.probes.get(key) is True for key in keys)
|
||||
st.probes.clear()
|
||||
|
||||
def finish(self, kv_request_id: str) -> None:
|
||||
"""Finish a request: abort any in-flight load and release lookup state.
|
||||
"""Finish a request: abort in-flight loads and release lookup state.
|
||||
|
||||
Called from the session's ``finish_request``. The two branches are
|
||||
mutually exclusive: ``load`` is set only by ``request_blocks``, which
|
||||
also advances ``phase`` to ``FETCH_SENT``, and nothing moves it back
|
||||
to ``PROBING`` — so a fetch in flight never coexists with the
|
||||
``PROBING`` phase.
|
||||
|
||||
- Fetch in flight (``load`` set, phase ``FETCH_SENT``): send an
|
||||
AbortFetchMsg unless the load is already aborting, then drop it.
|
||||
- Outstanding lookups (``PROBING``): a LookupMsg was flushed but no
|
||||
FetchMsg has closed the peer's lookup phase. Every FetchMsg the
|
||||
server receives in p2p mode is its "request finished" signal (it
|
||||
releases lookup state and fires ``cb.finish_request``); when the
|
||||
client's lookups all missed no FetchMsg is otherwise sent, so emit
|
||||
a terminal empty one purely to trigger those semantics. In
|
||||
``REGISTERED`` the peer never received a LookupMsg and in
|
||||
``FETCH_SENT`` a FetchMsg already closed the phase, so neither owes
|
||||
a terminal FetchMsg.
|
||||
Called from the session's ``finish_request``. Sends an
|
||||
AbortFetchMsg per load not already aborting, and — independently —
|
||||
the terminal empty FetchMsg when the peer still holds lookup
|
||||
state no fetch has closed (its "request finished" signal: it
|
||||
releases lookup state, drains parked supply, and fires
|
||||
``cb.finish_request``). A later round's supply can be parked
|
||||
while an earlier round's load is still in flight, so both can be
|
||||
owed at once.
|
||||
|
||||
Then drop all probe/lookup state and prune the entry.
|
||||
"""
|
||||
st = self._requests.get(kv_request_id)
|
||||
if st is None:
|
||||
return
|
||||
if st.load is not None:
|
||||
if st.load.aborted_at is None:
|
||||
if st.loads:
|
||||
for round_seq, load in st.loads.items():
|
||||
if load.aborted_at is not None:
|
||||
continue
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.KV_REQUEST_ID: kv_request_id,
|
||||
AbortFetchMsg.ROUND_SEQ: round_seq,
|
||||
}
|
||||
)
|
||||
st.load = None
|
||||
st.loads.clear()
|
||||
self._active_loads.discard(kv_request_id)
|
||||
elif st.phase is ClientPhase.PROBING:
|
||||
st.phase = ClientPhase.FETCH_SENT
|
||||
if st.peer_lookup_open:
|
||||
st.peer_lookup_open = False
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.KV_REQUEST_ID: kv_request_id,
|
||||
FetchMsg.KEYS: [],
|
||||
FetchMsg.BLOCK_INDEXES: [],
|
||||
FetchMsg.ROUND_SEQ: st.round_seq,
|
||||
}
|
||||
)
|
||||
st.probes.clear()
|
||||
@@ -273,20 +275,21 @@ class ClientRole:
|
||||
self._flush_pending.discard(kv_request_id)
|
||||
self._maybe_prune(kv_request_id)
|
||||
|
||||
def on_transfer_done(self, kv_request_id: str, success: bool) -> None:
|
||||
def on_transfer_done(
|
||||
self, kv_request_id: str, success: bool, round_seq: int
|
||||
) -> None:
|
||||
"""Handle a TransferDoneMsg from the peer."""
|
||||
st = self._requests.get(kv_request_id)
|
||||
if st is not None and st.load is not None:
|
||||
load = st.loads.pop(round_seq, None) if st is not None else None
|
||||
if st is not None and load is not None:
|
||||
self._completed_loads.append(
|
||||
LoadResult(
|
||||
job_id=st.load.job_id,
|
||||
job_id=load.job_id,
|
||||
kv_request_id=kv_request_id,
|
||||
success=success,
|
||||
)
|
||||
)
|
||||
st.load = None
|
||||
self._active_loads.discard(kv_request_id)
|
||||
self._maybe_prune(kv_request_id)
|
||||
self._on_load_terminal(kv_request_id, st)
|
||||
else:
|
||||
# No matching in-flight load: either a duplicate
|
||||
# transfer_done from the peer (protocol violation) or a
|
||||
@@ -295,41 +298,44 @@ class ClientRole:
|
||||
# so we can't tell — log so it's findable.
|
||||
logger.warning(
|
||||
"P2PSession %s: transfer_done for unknown kv_request_id=%s "
|
||||
"(duplicate from peer, or raced with local cancel/timeout)",
|
||||
"round=%s (duplicate from peer, or raced with local "
|
||||
"cancel/timeout)",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
round_seq,
|
||||
)
|
||||
|
||||
def on_abort_ack(self, kv_request_id: str) -> None:
|
||||
def on_abort_ack(self, kv_request_id: str, round_seq: int) -> None:
|
||||
"""Handle an AbortAckMsg from the peer."""
|
||||
st = self._requests.get(kv_request_id)
|
||||
if st is not None and st.load is not None:
|
||||
load = st.loads.pop(round_seq, None) if st is not None else None
|
||||
if st is not None and load is not None:
|
||||
logger.warning(
|
||||
"P2PSession %s: load request %s (job_id=%d) timed out; "
|
||||
"load job completed with failure. If this recurs, ensure "
|
||||
"PYTHONHASHSEED is set to the same value on all nodes.",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
st.load.job_id,
|
||||
load.job_id,
|
||||
)
|
||||
self._completed_loads.append(
|
||||
LoadResult(
|
||||
job_id=st.load.job_id,
|
||||
job_id=load.job_id,
|
||||
kv_request_id=kv_request_id,
|
||||
success=False,
|
||||
)
|
||||
)
|
||||
st.load = None
|
||||
self._active_loads.discard(kv_request_id)
|
||||
self._maybe_prune(kv_request_id)
|
||||
self._on_load_terminal(kv_request_id, st)
|
||||
else:
|
||||
# See on_transfer_done: same ambiguity (duplicate ack
|
||||
# vs. raced with local cancel/timeout that already popped).
|
||||
logger.warning(
|
||||
"P2PSession %s: abort_ack for unknown kv_request_id=%s "
|
||||
"(duplicate from peer, or raced with local cancel/timeout)",
|
||||
"round=%s (duplicate from peer, or raced with local "
|
||||
"cancel/timeout)",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
round_seq,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
@@ -345,17 +351,18 @@ class ClientRole:
|
||||
- Once a LookupRespMsg has resolved the entry: returns the cached
|
||||
bool result on every call without popping it.
|
||||
|
||||
A resolved entry is retained until its fetch is issued
|
||||
(``request_blocks`` pops it) or the request finishes
|
||||
A resolved entry is retained until a fetch closes the round
|
||||
(``request_blocks`` clears all probes) or the request finishes
|
||||
(``finish`` clears all entries for the id). A request's
|
||||
block set can be re-probed across steps, so popping on read would
|
||||
make a repeat probe of an already-resolved key look brand-new and
|
||||
re-queue it, emitting a redundant LookupMsg for an answer we
|
||||
already hold. Keeping the entry until fetch makes repeat probes
|
||||
free; clearing it at fetch forces a fresh probe if the request is
|
||||
re-scheduled, since the block is unpinned once served.
|
||||
free; clearing at fetch forces a fresh probe under the next
|
||||
round, since the block is unpinned once served.
|
||||
"""
|
||||
st = self._get_or_create_request(kv_request_id)
|
||||
st.probed = True
|
||||
okey = OffloadKey(key)
|
||||
if okey in st.probes:
|
||||
return st.probes[okey]
|
||||
@@ -378,14 +385,11 @@ class ClientRole:
|
||||
``on_schedule_end()``. A request's block set may be discovered
|
||||
across several scheduler steps, so more than one LookupMsg can
|
||||
go out per kv_request_id — one per step that registered new
|
||||
keys. register_lookup() de-dups in-flight and already-resolved
|
||||
(req_id, key) pairs, so each LookupMsg carries only the keys
|
||||
first probed in that step. The peer's lookup phase for the id is
|
||||
still closed by exactly one FetchMsg, which the client contract
|
||||
guarantees is sent after every lookup for the id has resolved
|
||||
(see request_blocks / finish). Send-gating is handled by
|
||||
the injected ``_send`` callback (queues until ConnectAckMsg if
|
||||
needed).
|
||||
keys, all tagged with the current round. register_lookup()
|
||||
de-dups in-flight and already-resolved (req_id, key) pairs, so
|
||||
each LookupMsg carries only the keys first probed in that step.
|
||||
Send-gating is handled by the injected ``_send`` callback
|
||||
(queues until ConnectAckMsg if needed).
|
||||
|
||||
Only requests that registered new keys since the last flush are
|
||||
visited — the ``_flush_pending`` work-list avoids scanning every
|
||||
@@ -395,13 +399,9 @@ class ClientRole:
|
||||
st = self._requests.get(req_id)
|
||||
if st is None or not st.unsent:
|
||||
continue
|
||||
# Record that the peer now holds lookup state for this id so
|
||||
# finish knows a terminal empty FetchMsg may be owed.
|
||||
# Only promote from REGISTERED: once a fetch has gone out
|
||||
# (FETCH_SENT) a later LookupMsg must not regress the phase, as
|
||||
# no terminal FetchMsg is owed for an already-fetched request.
|
||||
if st.phase is ClientPhase.REGISTERED:
|
||||
st.phase = ClientPhase.PROBING
|
||||
# The peer now holds lookup state for this id; finish owes a
|
||||
# terminal empty FetchMsg until a fetch closes it.
|
||||
st.peer_lookup_open = True
|
||||
logger.debug(
|
||||
"P2P LOOKUP client %s: SEND LookupMsg kv_request_id=%s keys=%d",
|
||||
self._peer_id,
|
||||
@@ -413,6 +413,7 @@ class ClientRole:
|
||||
TYPE_KEY: LookupMsg.TYPE,
|
||||
LookupMsg.KV_REQUEST_ID: req_id,
|
||||
LookupMsg.KEYS: list(st.unsent),
|
||||
LookupMsg.ROUND_SEQ: st.round_seq,
|
||||
}
|
||||
)
|
||||
st.unsent = []
|
||||
@@ -451,52 +452,56 @@ class ClientRole:
|
||||
def collect_results(self) -> list[LoadResult]:
|
||||
"""Walk load timeouts and drain completed loads.
|
||||
|
||||
Active requests past ``_LOAD_TIMEOUT_S`` get an AbortFetchMsg
|
||||
sent and enter the aborting phase. Aborting requests past
|
||||
``_ABORT_ACK_TIMEOUT_S`` are surfaced as failed loads.
|
||||
Loads past ``_LOAD_TIMEOUT_S`` get an AbortFetchMsg sent and
|
||||
enter the aborting phase. Aborting loads past
|
||||
``_ABORT_ACK_TIMEOUT_S`` are surfaced as failed.
|
||||
|
||||
Lookups have no timeout: an unanswered probe stays None (RETRY)
|
||||
until finish_request clears it — see ``_ClientRequestState.probes``.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
to_remove: list[str] = []
|
||||
to_remove: list[tuple[str, int]] = []
|
||||
for req_id in self._active_loads:
|
||||
st = self._requests[req_id]
|
||||
assert st.load is not None
|
||||
load = st.load
|
||||
if load.aborted_at is None:
|
||||
if now - load.submitted_at >= _LOAD_TIMEOUT_S:
|
||||
load.aborted_at = now
|
||||
logger.warning(
|
||||
"P2PSession %s: %s timed out, sending abort",
|
||||
self._peer_id,
|
||||
req_id,
|
||||
)
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.KV_REQUEST_ID: req_id,
|
||||
}
|
||||
)
|
||||
else:
|
||||
if now - load.aborted_at >= _ABORT_ACK_TIMEOUT_S:
|
||||
to_remove.append(req_id)
|
||||
self._completed_loads.append(
|
||||
LoadResult(
|
||||
job_id=load.job_id,
|
||||
kv_request_id=req_id,
|
||||
success=False,
|
||||
assert st.loads
|
||||
for round_seq, load in st.loads.items():
|
||||
if load.aborted_at is None:
|
||||
if now - load.submitted_at >= _LOAD_TIMEOUT_S:
|
||||
load.aborted_at = now
|
||||
logger.warning(
|
||||
"P2PSession %s: %s round=%s timed out, sending abort",
|
||||
self._peer_id,
|
||||
req_id,
|
||||
round_seq,
|
||||
)
|
||||
)
|
||||
logger.warning(
|
||||
"P2PSession %s: abort_ack timed out for kv_request_id=%s",
|
||||
self._peer_id,
|
||||
req_id,
|
||||
)
|
||||
for req_id in to_remove:
|
||||
self._requests[req_id].load = None
|
||||
self._active_loads.discard(req_id)
|
||||
self._maybe_prune(req_id)
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.KV_REQUEST_ID: req_id,
|
||||
AbortFetchMsg.ROUND_SEQ: round_seq,
|
||||
}
|
||||
)
|
||||
else:
|
||||
if now - load.aborted_at >= _ABORT_ACK_TIMEOUT_S:
|
||||
to_remove.append((req_id, round_seq))
|
||||
self._completed_loads.append(
|
||||
LoadResult(
|
||||
job_id=load.job_id,
|
||||
kv_request_id=req_id,
|
||||
success=False,
|
||||
)
|
||||
)
|
||||
logger.warning(
|
||||
"P2PSession %s: abort_ack timed out for "
|
||||
"kv_request_id=%s round=%s",
|
||||
self._peer_id,
|
||||
req_id,
|
||||
round_seq,
|
||||
)
|
||||
for req_id, round_seq in to_remove:
|
||||
st = self._requests[req_id]
|
||||
st.loads.pop(round_seq)
|
||||
self._on_load_terminal(req_id, st)
|
||||
|
||||
results = self._completed_loads
|
||||
self._completed_loads = []
|
||||
@@ -510,12 +515,12 @@ class ClientRole:
|
||||
forever on an answer that can never arrive). See ``ClientCloseResult``.
|
||||
"""
|
||||
failed_jobs = [
|
||||
st.load.job_id for st in self._requests.values() if st.load is not None
|
||||
load.job_id for st in self._requests.values() for load in st.loads.values()
|
||||
]
|
||||
failed_req_ids = [
|
||||
req_id
|
||||
for req_id, st in self._requests.items()
|
||||
if st.load is not None or any(hit is None for hit in st.probes.values())
|
||||
if st.loads or any(hit is None for hit in st.probes.values())
|
||||
]
|
||||
self._requests.clear()
|
||||
self._flush_pending.clear()
|
||||
|
||||
@@ -26,14 +26,12 @@ Block Transfer Flow (happy path)
|
||||
|
||||
1. Client sends FetchMsg with a kv_request_id and lists of
|
||||
block keys + remote indexes where it wants the data written.
|
||||
In p2p mode FetchMsg is also the server-side "request finished"
|
||||
signal for the id: no further ``cb.create_store_job`` will fire
|
||||
(parked LookupMsg batches are popped, so pending-key resolution
|
||||
cannot promote a HIT after this point), all server-side lookup
|
||||
state for the id is released, and ``cb.finish_request`` fires on
|
||||
each dropped batch. The client emits exactly one FetchMsg per
|
||||
lookup-touched request, including an empty one when no blocks
|
||||
end up being fetched.
|
||||
A request may run several lookup→fetch rounds; symmetric-P2P
|
||||
messages carry ROUND_SEQ so each round's supply, demand, and
|
||||
completion stay isolated. The terminal empty FetchMsg is the
|
||||
server-side "request finished" signal for the id: parked
|
||||
LookupMsg batches are popped and ``cb.finish_request`` fires on
|
||||
each.
|
||||
2. Server matches requested blocks against locally stored blocks:
|
||||
- Blocks already available are transferred immediately via RDMA.
|
||||
- Blocks not yet available are recorded as "demanded" and
|
||||
@@ -178,33 +176,32 @@ class DisconnectMsg:
|
||||
|
||||
|
||||
class FetchMsg:
|
||||
"""Client → Server: request blocks by key and close the lookup phase.
|
||||
"""Client → Server: request blocks for one lookup round.
|
||||
|
||||
In p2p mode FetchMsg is also the server-side "request finished"
|
||||
signal for ``kv_request_id``: on receipt the server (a) fires no
|
||||
further ``cb.create_store_job`` for this id — parked LookupMsg
|
||||
batches are popped, so ``_resolve_pending_lookups`` cannot promote
|
||||
a HIT_PENDING / RETRY key into a fresh pin after this point — and
|
||||
(b) calls ``cb.finish_request(batch.ctx)`` on each dropped batch
|
||||
so the TieringManager can release per-batch bookkeeping. In the
|
||||
all-miss case the client emits an empty FetchMsg (``KEYS``
|
||||
and ``BLOCK_INDEXES`` both empty) purely to fire this signal.
|
||||
A non-empty fetch closes only its round. The terminal empty FetchMsg
|
||||
(``KEYS`` and ``BLOCK_INDEXES`` both empty) is the "request
|
||||
finished" signal: the server pops parked LookupMsg batches, calls
|
||||
``cb.finish_request`` on each, and drains any leftover supply.
|
||||
|
||||
Fields:
|
||||
KV_REQUEST_ID: Identifies this block transfer request.
|
||||
KEYS: List of block keys (OffloadKey bytes). May be empty.
|
||||
BLOCK_INDEXES: List of remote block indexes (same length as KEYS).
|
||||
ROUND_SEQ: Lookup round this fetch closes. PD clients never probe
|
||||
and stay on their single round 0.
|
||||
"""
|
||||
|
||||
TYPE = "fetch"
|
||||
KV_REQUEST_ID = "kv_request_id"
|
||||
KEYS = "keys"
|
||||
BLOCK_INDEXES = "block_indexes"
|
||||
ROUND_SEQ = "round_seq"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, FetchMsg.KV_REQUEST_ID, str)
|
||||
_require_non_neg_int(msg, FetchMsg.ROUND_SEQ)
|
||||
_require_list(msg, FetchMsg.KEYS)
|
||||
_require_list(msg, FetchMsg.BLOCK_INDEXES)
|
||||
keys = msg[FetchMsg.KEYS]
|
||||
@@ -229,17 +226,21 @@ class LookupMsg:
|
||||
Fields:
|
||||
KV_REQUEST_ID: Identifies this lookup transaction.
|
||||
KEYS: List of block keys (OffloadKey bytes) to probe.
|
||||
ROUND_SEQ: Lookup round these probes belong to; pinned supply is
|
||||
parked under it for that round's fetch.
|
||||
"""
|
||||
|
||||
TYPE = "lookup"
|
||||
KV_REQUEST_ID = "kv_request_id"
|
||||
KEYS = "keys"
|
||||
ROUND_SEQ = "round_seq"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, LookupMsg.KV_REQUEST_ID, str)
|
||||
_require_list(msg, LookupMsg.KEYS)
|
||||
_require_non_neg_int(msg, LookupMsg.ROUND_SEQ)
|
||||
|
||||
|
||||
class LookupRespMsg:
|
||||
@@ -284,17 +285,22 @@ class TransferDoneMsg:
|
||||
Fields:
|
||||
KV_REQUEST_ID: The request that completed.
|
||||
SUCCESS: Whether the transfer completed successfully.
|
||||
ROUND_SEQ: The fetch round that completed. Several loads can be
|
||||
in flight per id (the scheduler submits loads incrementally),
|
||||
so completions are matched by round.
|
||||
"""
|
||||
|
||||
TYPE = "transfer_done"
|
||||
KV_REQUEST_ID = "kv_request_id"
|
||||
SUCCESS = "success"
|
||||
ROUND_SEQ = "round_seq"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, TransferDoneMsg.KV_REQUEST_ID, str)
|
||||
_require(msg, TransferDoneMsg.SUCCESS, bool)
|
||||
_require_non_neg_int(msg, TransferDoneMsg.ROUND_SEQ)
|
||||
|
||||
|
||||
class AbortFetchMsg:
|
||||
@@ -302,15 +308,18 @@ class AbortFetchMsg:
|
||||
|
||||
Fields:
|
||||
KV_REQUEST_ID: The request to cancel.
|
||||
ROUND_SEQ: The fetch round to cancel.
|
||||
"""
|
||||
|
||||
TYPE = "abort_fetch"
|
||||
KV_REQUEST_ID = "kv_request_id"
|
||||
ROUND_SEQ = "round_seq"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, AbortFetchMsg.KV_REQUEST_ID, str)
|
||||
_require_non_neg_int(msg, AbortFetchMsg.ROUND_SEQ)
|
||||
|
||||
|
||||
class AbortAckMsg:
|
||||
@@ -318,12 +327,15 @@ class AbortAckMsg:
|
||||
|
||||
Fields:
|
||||
KV_REQUEST_ID: The request that was cancelled.
|
||||
ROUND_SEQ: The round that was cancelled; echoes AbortFetchMsg.
|
||||
"""
|
||||
|
||||
TYPE = "abort_ack"
|
||||
KV_REQUEST_ID = "kv_request_id"
|
||||
ROUND_SEQ = "round_seq"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, AbortAckMsg.KV_REQUEST_ID, str)
|
||||
_require_non_neg_int(msg, AbortAckMsg.ROUND_SEQ)
|
||||
|
||||
@@ -53,15 +53,6 @@ class StoreResult(NamedTuple):
|
||||
success: bool
|
||||
|
||||
|
||||
class _InflightXfer(NamedTuple):
|
||||
"""Metadata for a single inflight RDMA transfer, keyed by transfer_id."""
|
||||
|
||||
kv_request_id: str
|
||||
block_count: int
|
||||
# The set of store job IDs that contributed blocks to this transfer.
|
||||
job_ids: set[int]
|
||||
|
||||
|
||||
class _MatchResult(NamedTuple):
|
||||
"""Result of block matching: pairs ready for transfer."""
|
||||
|
||||
@@ -73,12 +64,17 @@ class _MatchResult(NamedTuple):
|
||||
|
||||
@dataclass
|
||||
class _OutboundRequestState:
|
||||
"""Server-role state for a single peer fetch request.
|
||||
"""Server-role state for a single fetch round of a peer request.
|
||||
|
||||
The owning ``kv_request_id`` is the ``ServerRole._requests`` dict key
|
||||
and is not duplicated on the value.
|
||||
A kv_request_id may run several lookup→fetch rounds; rounds live in
|
||||
``_ServerRequestState.outbound`` keyed by the wire ``round_seq``, so
|
||||
terminals touch only their own round.
|
||||
"""
|
||||
|
||||
# Supply came from inbound lookup pins (symmetric): no late
|
||||
# submit_store can arrive, so unmatched fetch demand fails fast. PD
|
||||
# rounds park demand for stores instead.
|
||||
lookup_supplied: bool = False
|
||||
demand_received: bool = False
|
||||
available: dict[OffloadKey, tuple[int, int]] = field(
|
||||
default_factory=dict
|
||||
@@ -88,7 +84,8 @@ class _OutboundRequestState:
|
||||
) # key → remote_block_idx: blocks peer wants, awaiting supply
|
||||
remaining: int = 0 # blocks that need to be transferred to client
|
||||
finishing: bool = False # Signal finish request ASAP
|
||||
# Job IDs that submit_store'd blocks for this request and have not
|
||||
inflight: int = 0 # transfers submitted for this round, not yet polled
|
||||
# Job IDs that submit_store'd blocks for this round and have not
|
||||
# yet emitted a StoreResult. The terminal-finalize helper drains
|
||||
# this set; poll-done and poll-failed discard entries as their
|
||||
# StoreResults fire.
|
||||
@@ -145,6 +142,21 @@ class _OutboundRequestState:
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _InflightXfer:
|
||||
"""Metadata for a single inflight RDMA transfer, keyed by transfer_id."""
|
||||
|
||||
kv_request_id: str
|
||||
block_count: int
|
||||
# The set of store job IDs that contributed blocks to this transfer.
|
||||
job_ids: set[int]
|
||||
# Round this transfer serves and its key in ``st.outbound``;
|
||||
# remaining/finalize apply only while the round is still registered.
|
||||
# Dummy default for test-seeded entries.
|
||||
round: _OutboundRequestState = field(default_factory=_OutboundRequestState)
|
||||
round_key: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ActiveLookup:
|
||||
"""In-flight state for one inbound LookupMsg.
|
||||
@@ -159,6 +171,8 @@ class _ActiveLookup:
|
||||
lookup_id: int
|
||||
kv_request_id: str
|
||||
ctx: ReqContext
|
||||
# Wire round these probes belong to; pins park under it.
|
||||
round_seq: int = 0
|
||||
# Keys from the inbound LookupMsg, preserved in wire order so
|
||||
# the aggregated response goes back in the same order.
|
||||
keys: list[OffloadKey] = field(default_factory=list)
|
||||
@@ -185,6 +199,7 @@ class _PendingLookup(NamedTuple):
|
||||
|
||||
keys: list[OffloadKey]
|
||||
enqueued_at: float
|
||||
round_seq: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -198,17 +213,15 @@ class _ServerRequestState:
|
||||
is idle — see ``ServerRole._maybe_prune``.
|
||||
"""
|
||||
|
||||
# Outbound serve state (PD + symmetric producer side). None until the
|
||||
# first add_stored_blocks / on_fetch; reset to None on finalize/abort.
|
||||
outbound: _OutboundRequestState | None = None
|
||||
# Fetch rounds keyed by wire round_seq. A round is created by its
|
||||
# first supply or its fetch and removed at its terminal (finalize /
|
||||
# failure / abort).
|
||||
outbound: dict[int, _OutboundRequestState] = field(default_factory=dict)
|
||||
# Raw inbound LookupMsgs not yet processed against the ParentManager.
|
||||
pending_lookups: list[_PendingLookup] = field(default_factory=list)
|
||||
# Per-LookupMsg state parked with HIT_PENDING / RETRY keys, keyed by
|
||||
# the (globally unique) lookup_id and re-polled each serve.
|
||||
lookups: dict[int, _ActiveLookup] = field(default_factory=dict)
|
||||
# Start time of a pending abort drain (``time.monotonic``); None when
|
||||
# no abort is in progress.
|
||||
abort_started_at: float | None = None
|
||||
# Transfer ids in ``ServerRole._inflight`` for this id. Kept in sync
|
||||
# via _inflight_add / _inflight_pop so a non-empty set is an exact
|
||||
# "has any inflight transfer" predicate and the abort drain can
|
||||
@@ -256,9 +269,9 @@ class ServerRole:
|
||||
# ``parent.on_request_finished`` in ``serve_external_requests``.
|
||||
self._finished_lookup_ctxs: list[ReqContext] = []
|
||||
self._lookup_id_counter: int = 0
|
||||
# kv_request_ids with a parked abort awaiting drain — work-list so
|
||||
# drain_pending_aborts doesn't scan every request each poll tick.
|
||||
self._parked_aborts: set[str] = set()
|
||||
# Parked aborts awaiting drain, keyed by (kv_request_id, round)
|
||||
# with the abort start time.
|
||||
self._pending_aborts: dict[tuple[str, int], float] = {}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# State helpers
|
||||
@@ -277,11 +290,11 @@ class ServerRole:
|
||||
st = self._requests.get(kv_request_id)
|
||||
if (
|
||||
st is not None
|
||||
and st.outbound is None
|
||||
and not st.outbound
|
||||
and not st.inflight_tids
|
||||
and not st.lookups
|
||||
and not st.pending_lookups
|
||||
and st.abort_started_at is None
|
||||
and not any(kv == kv_request_id for kv, _ in self._pending_aborts)
|
||||
):
|
||||
del self._requests[kv_request_id]
|
||||
|
||||
@@ -295,87 +308,100 @@ class ServerRole:
|
||||
keys: Sequence[OffloadKey],
|
||||
block_ids: Sequence[int],
|
||||
job_id: JobId,
|
||||
round_seq: int = 0,
|
||||
*,
|
||||
from_lookup: bool = False,
|
||||
) -> None:
|
||||
"""New blocks stored locally — match against pending fetch demand."""
|
||||
"""New blocks stored locally — match within their fetch round.
|
||||
|
||||
Lookup pins carry the round they were probed under; PD
|
||||
submit_store batches share PD's single round 0.
|
||||
"""
|
||||
self._store_jobs[job_id] = time.monotonic()
|
||||
st = self._get_or_create_request(kv_request_id)
|
||||
if st.outbound is None:
|
||||
st.outbound = _OutboundRequestState()
|
||||
result = st.outbound.add_stored_blocks(keys, block_ids, job_id)
|
||||
if result.local_idxs and st.outbound.demand_received:
|
||||
self._submit_transfer(kv_request_id, result)
|
||||
rnd = st.outbound.get(round_seq)
|
||||
if rnd is None:
|
||||
rnd = st.outbound[round_seq] = _OutboundRequestState()
|
||||
if from_lookup:
|
||||
rnd.lookup_supplied = True
|
||||
result = rnd.add_stored_blocks(keys, block_ids, job_id)
|
||||
if result.local_idxs and rnd.demand_received:
|
||||
self._submit_transfer(kv_request_id, result, rnd, round_seq)
|
||||
|
||||
def on_fetch(
|
||||
self,
|
||||
kv_request_id: str,
|
||||
keys: Sequence[OffloadKey],
|
||||
block_indexes: Sequence[int],
|
||||
round_seq: int = 0,
|
||||
) -> None:
|
||||
"""Handle a FetchMsg from the peer.
|
||||
|
||||
In p2p mode FetchMsg is the server-side "request finished"
|
||||
signal for ``kv_request_id``. Three consequences flow from that:
|
||||
|
||||
- No further ``parent.create_store_job`` will fire for this id:
|
||||
the request's parked ``lookups`` are popped here (via
|
||||
``_finish_inbound_lookups``) before the next
|
||||
``serve_external_requests`` runs ``_resolve_pending_lookups``,
|
||||
so any HIT_PENDING / RETRY key that would otherwise later
|
||||
promote to HIT and pin a slot is dropped instead. Any raw
|
||||
not-yet-processed LookupMsg for this id is dropped too.
|
||||
- All server-side lookup state for the id is cleaned up (the
|
||||
``lookups`` entries themselves).
|
||||
- The synthetic ctx is queued for ``parent.on_request_finished``
|
||||
(fired by the next ``serve_external_requests``) so the
|
||||
TieringManager can release per-lookup bookkeeping.
|
||||
|
||||
The client contract guarantees exactly one FetchMsg per
|
||||
lookup-touched request — including an empty one when no
|
||||
blocks end up being fetched.
|
||||
|
||||
Raises ``ValueError`` on a duplicate fetch for the same
|
||||
``kv_request_id``; the coordinator's dispatch loop turns that
|
||||
into a protocol-error disconnect.
|
||||
A non-empty fetch binds and closes its round, leaving lookup
|
||||
state alone (the next round's LookupMsg may already be in
|
||||
flight). The terminal empty fetch closes the id: parked lookups
|
||||
are popped and every remaining round drained. A second fetch for
|
||||
a round already holding demand raises ValueError
|
||||
(protocol-error disconnect).
|
||||
"""
|
||||
logger.debug(
|
||||
"P2PSession %s: fetch RECEIVED kv_request_id=%s blocks=%d",
|
||||
"P2PSession %s: fetch RECEIVED kv_request_id=%s round=%s blocks=%d",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
round_seq,
|
||||
len(keys),
|
||||
)
|
||||
st = self._requests.get(kv_request_id)
|
||||
existing = st.outbound if st is not None else None
|
||||
existing = st.outbound.get(round_seq) if st is not None else None
|
||||
if existing is not None and existing.demand_received:
|
||||
# A second fetch for the same kv_request_id would overwrite
|
||||
# `remaining` and leak inflight bookkeeping. Treat as a
|
||||
# protocol violation.
|
||||
raise ValueError(f"duplicate fetch for kv_request_id={kv_request_id}")
|
||||
raise ValueError(
|
||||
f"duplicate fetch for kv_request_id={kv_request_id} round={round_seq}"
|
||||
)
|
||||
st = self._get_or_create_request(kv_request_id)
|
||||
if st.outbound is None:
|
||||
st.outbound = _OutboundRequestState()
|
||||
req = st.outbound
|
||||
req = st.outbound.get(round_seq)
|
||||
if req is None:
|
||||
req = st.outbound[round_seq] = _OutboundRequestState()
|
||||
result = req.add_fetch_demand(keys, block_indexes)
|
||||
if not keys:
|
||||
# Terminal empty fetch: close the lookup phase and drain
|
||||
# every round with no TransferDoneMsg (nothing waits on it).
|
||||
self._finish_inbound_lookups(kv_request_id)
|
||||
for key in list(st.outbound):
|
||||
self._finalize_outbound(kv_request_id, key, send_done=False)
|
||||
return
|
||||
if req.lookup_supplied and req.demanded:
|
||||
# A symmetric round's supply always precedes its fetch, so
|
||||
# unmatched demand is unservable — fail now, not at the load
|
||||
# timeout. PD rounds keep parking demand for stores that
|
||||
# arrive later.
|
||||
logger.warning(
|
||||
"P2PSession %s: fetch kv_request_id=%s round=%s demanded %d "
|
||||
"blocks but %d have no pinned supply; failing fetch "
|
||||
"immediately",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
round_seq,
|
||||
len(keys),
|
||||
len(req.demanded),
|
||||
)
|
||||
self._finalize_outbound(kv_request_id, round_seq, success=False)
|
||||
return
|
||||
if result.local_idxs:
|
||||
self._submit_transfer(kv_request_id, result)
|
||||
# Close the peer's request as far as the server's lookup phase
|
||||
# is concerned: pop parked lookups so no further
|
||||
# ``parent.create_store_job`` fires for this id, and queue their
|
||||
# ctxs for ``parent.on_request_finished``. Done before the
|
||||
# finalize path below so bookkeeping releases in-order.
|
||||
self._finish_inbound_lookups(kv_request_id)
|
||||
self._submit_transfer(kv_request_id, result, req, round_seq)
|
||||
# Prefiller-first mode: finish_request may have run before
|
||||
# fetch arrived. If so, finalize once we know what was
|
||||
# demanded — fully satisfied → success, else early-fail.
|
||||
if req.finishing and not self._has_inflight_for(kv_request_id):
|
||||
self._finalize_outbound(kv_request_id)
|
||||
if req.finishing and req.inflight == 0:
|
||||
self._finalize_outbound(kv_request_id, round_seq)
|
||||
|
||||
def on_abort_fetch(self, kv_request_id: str) -> None:
|
||||
"""Handle an AbortFetchMsg from the peer."""
|
||||
def on_abort_fetch(self, kv_request_id: str, round_seq: int = 0) -> None:
|
||||
"""Handle an AbortFetchMsg from the peer, cancelling one round."""
|
||||
# Abort for an unknown id may be a benign race/duplicate or a
|
||||
# real protocol violation; we don't track completed ids, so warn.
|
||||
st = self._requests.get(kv_request_id)
|
||||
has_outbound = st is not None and st.outbound is not None
|
||||
if not has_outbound and not self._has_inflight_for(kv_request_id):
|
||||
if (st is None or not st.outbound) and not self._has_inflight_for(
|
||||
kv_request_id
|
||||
):
|
||||
logger.warning(
|
||||
"P2PSession %s: abort_fetch for unknown kv_request_id=%s "
|
||||
"(no outbound or inflight state); benign race or stale",
|
||||
@@ -385,16 +411,15 @@ class ServerRole:
|
||||
# Idempotent: receiving AbortFetchMsg again before we've sent the
|
||||
# ack just triggers another drain attempt without resetting the
|
||||
# deadline.
|
||||
st = self._get_or_create_request(kv_request_id)
|
||||
if st.abort_started_at is None:
|
||||
st.abort_started_at = time.monotonic()
|
||||
self._parked_aborts.add(kv_request_id)
|
||||
self._drain_abort(kv_request_id)
|
||||
self._get_or_create_request(kv_request_id)
|
||||
self._pending_aborts.setdefault((kv_request_id, round_seq), time.monotonic())
|
||||
self._drain_abort(kv_request_id, round_seq)
|
||||
|
||||
def on_lookup(
|
||||
self,
|
||||
kv_request_id: str,
|
||||
keys: Sequence[OffloadKey],
|
||||
round_seq: int = 0,
|
||||
) -> None:
|
||||
"""Enqueue a LookupMsg from a symmetric-P2P consumer.
|
||||
|
||||
@@ -406,13 +431,18 @@ class ServerRole:
|
||||
parent calls are valid.
|
||||
"""
|
||||
logger.debug(
|
||||
"P2P LOOKUP server %s: RECV LookupMsg kv_request_id=%s keys=%d",
|
||||
"P2P LOOKUP server %s: RECV LookupMsg kv_request_id=%s round=%s keys=%d",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
round_seq,
|
||||
len(keys),
|
||||
)
|
||||
self._get_or_create_request(kv_request_id).pending_lookups.append(
|
||||
_PendingLookup(keys=list(keys), enqueued_at=time.monotonic())
|
||||
_PendingLookup(
|
||||
keys=list(keys),
|
||||
enqueued_at=time.monotonic(),
|
||||
round_seq=round_seq,
|
||||
)
|
||||
)
|
||||
self._serve_pending.add(kv_request_id)
|
||||
|
||||
@@ -434,7 +464,7 @@ class ServerRole:
|
||||
st.pending_lookups = []
|
||||
for pl in pending:
|
||||
self._process_inbound_lookup(
|
||||
kv_request_id, pl.keys, pl.enqueued_at, parent
|
||||
kv_request_id, pl.keys, pl.enqueued_at, pl.round_seq, parent
|
||||
)
|
||||
self._resolve_pending_lookups(kv_request_id, parent)
|
||||
st = self._requests.get(kv_request_id)
|
||||
@@ -481,9 +511,7 @@ class ServerRole:
|
||||
else:
|
||||
lookup.pending.add(h)
|
||||
if new_hits:
|
||||
self._pin_and_register_hits(
|
||||
lookup.kv_request_id, new_hits, lookup.ctx, parent
|
||||
)
|
||||
self._pin_and_register_hits(lookup, new_hits, parent)
|
||||
return new_hits
|
||||
|
||||
def _process_inbound_lookup(
|
||||
@@ -491,6 +519,7 @@ class ServerRole:
|
||||
kv_request_id: str,
|
||||
keys: list[OffloadKey],
|
||||
enqueued_at: float,
|
||||
round_seq: int,
|
||||
parent: ParentManager,
|
||||
) -> None:
|
||||
"""Resolve one enqueued LookupMsg against ``parent``.
|
||||
@@ -515,6 +544,7 @@ class ServerRole:
|
||||
lookup_id=lookup_id,
|
||||
kv_request_id=kv_request_id,
|
||||
ctx=ctx,
|
||||
round_seq=round_seq,
|
||||
keys=list(keys),
|
||||
deadline=enqueued_at + _LOOKUP_PENDING_TIMEOUT_S,
|
||||
)
|
||||
@@ -547,25 +577,26 @@ class ServerRole:
|
||||
|
||||
def _pin_and_register_hits(
|
||||
self,
|
||||
kv_request_id: str,
|
||||
lookup: _ActiveLookup,
|
||||
keys: list[OffloadKey],
|
||||
ctx: ReqContext,
|
||||
parent: ParentManager,
|
||||
) -> None:
|
||||
"""Pin primary slots for HIT keys and feed them into the
|
||||
existing ``add_stored_blocks`` matching path.
|
||||
"""Pin primary slots for HIT keys and park them as the lookup's
|
||||
round supply via ``add_stored_blocks``.
|
||||
|
||||
Caller has already confirmed every key is HIT (single-threaded
|
||||
scheduler ⇒ no eviction race), so the JobMetadata returned by
|
||||
``parent.create_store_job`` carries parallel ``keys``/``block_ids``
|
||||
of length ``len(keys)``.
|
||||
"""
|
||||
meta = parent.create_store_job(keys, ctx)
|
||||
meta = parent.create_store_job(keys, lookup.ctx)
|
||||
self.add_stored_blocks(
|
||||
kv_request_id,
|
||||
lookup.kv_request_id,
|
||||
list(meta.keys),
|
||||
list(meta.block_ids),
|
||||
meta.job_id,
|
||||
round_seq=lookup.round_seq,
|
||||
from_lookup=True,
|
||||
)
|
||||
|
||||
def _resolve_pending_lookups(
|
||||
@@ -650,9 +681,8 @@ class ServerRole:
|
||||
|
||||
Called on the two events that mean "no more lookup traffic for
|
||||
``kv_request_id`` is expected on this session": the terminal
|
||||
FetchMsg from the peer (client contract: exactly one FetchMsg
|
||||
per lookup-touched request, even if empty), and a local
|
||||
``finish``. Whichever fires second is a no-op.
|
||||
empty FetchMsg from the peer and a local ``finish``. Whichever
|
||||
fires second is a no-op.
|
||||
"""
|
||||
st = self._requests.get(kv_request_id)
|
||||
if st is None:
|
||||
@@ -688,20 +718,15 @@ class ServerRole:
|
||||
self._finish_inbound_lookups(kv_request_id)
|
||||
|
||||
st = self._requests.get(kv_request_id)
|
||||
req = st.outbound if st is not None else None
|
||||
if req is None:
|
||||
if st is None:
|
||||
return
|
||||
req.finishing = True
|
||||
if not req.demand_received:
|
||||
return
|
||||
if self._has_inflight_for(kv_request_id):
|
||||
return
|
||||
# Remaining > 0 here: if it had hit 0, the poll-done success
|
||||
# branch would have already cleared outbound and we'd have
|
||||
# returned at `req is None` above. Helper derives success from
|
||||
# remaining and emits StoreResult(success=False) for any
|
||||
# leftover pending jobs.
|
||||
self._finalize_outbound(kv_request_id)
|
||||
for key, req in list(st.outbound.items()):
|
||||
req.finishing = True
|
||||
if not req.demand_received or req.inflight:
|
||||
# No demand yet (prefiller-first): on_fetch finalizes via
|
||||
# `finishing`. Inflight: the last completion finalizes.
|
||||
continue
|
||||
self._finalize_outbound(kv_request_id, key)
|
||||
|
||||
def collect_results(self) -> list[StoreResult]:
|
||||
"""Drain timeouts, deferred results, and transport completions.
|
||||
@@ -741,20 +766,24 @@ class ServerRole:
|
||||
)
|
||||
continue
|
||||
results.extend(self._settle_xfer_jobs(xfer, success=True))
|
||||
rnd = xfer.round
|
||||
st = self._requests.get(xfer.kv_request_id)
|
||||
req = st.outbound if st is not None else None
|
||||
if req is not None and req.demand_received:
|
||||
req.remaining -= xfer.block_count
|
||||
assert req.remaining >= 0, (
|
||||
if st is not None and st.outbound.get(xfer.round_key) is rnd:
|
||||
rnd.remaining -= xfer.block_count
|
||||
assert rnd.remaining >= 0, (
|
||||
f"remaining went negative for kv_request_id={xfer.kv_request_id}"
|
||||
)
|
||||
if req.remaining == 0:
|
||||
self._finalize_outbound(xfer.kv_request_id, success=True)
|
||||
elif req.finishing and not self._has_inflight_for(xfer.kv_request_id):
|
||||
self._finalize_outbound(xfer.kv_request_id, success=False)
|
||||
if rnd.remaining == 0:
|
||||
self._finalize_outbound(
|
||||
xfer.kv_request_id, xfer.round_key, success=True
|
||||
)
|
||||
elif rnd.finishing and rnd.inflight == 0:
|
||||
self._finalize_outbound(
|
||||
xfer.kv_request_id, xfer.round_key, success=False
|
||||
)
|
||||
self._maybe_prune(xfer.kv_request_id)
|
||||
|
||||
failed_kv_request_ids: set[str] | None = None
|
||||
failed_rounds: list[tuple[str, _OutboundRequestState]] | None = None
|
||||
for tid in poll_result.failed:
|
||||
xfer = self._inflight_pop(tid)
|
||||
if xfer is None:
|
||||
@@ -767,35 +796,36 @@ class ServerRole:
|
||||
tid,
|
||||
)
|
||||
continue
|
||||
if failed_kv_request_ids is None:
|
||||
failed_kv_request_ids = set()
|
||||
failed_kv_request_ids.add(xfer.kv_request_id)
|
||||
results.extend(self._settle_xfer_jobs(xfer, success=False))
|
||||
rnd = xfer.round
|
||||
st = self._requests.get(xfer.kv_request_id)
|
||||
req = st.outbound if st is not None else None
|
||||
if st is not None:
|
||||
st.outbound = None
|
||||
if req is not None and req.demand_received:
|
||||
if st is not None and st.outbound.get(xfer.round_key) is rnd:
|
||||
del st.outbound[xfer.round_key]
|
||||
if failed_rounds is None:
|
||||
failed_rounds = []
|
||||
failed_rounds.append((xfer.kv_request_id, rnd))
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||
TransferDoneMsg.KV_REQUEST_ID: xfer.kv_request_id,
|
||||
TransferDoneMsg.SUCCESS: False,
|
||||
TransferDoneMsg.ROUND_SEQ: xfer.round_key,
|
||||
}
|
||||
)
|
||||
self._maybe_prune(xfer.kv_request_id)
|
||||
|
||||
# Cancel other inflight for the same failed kv_request_ids
|
||||
if failed_kv_request_ids:
|
||||
ids_to_cancel = [
|
||||
tid
|
||||
for tid, xfer in self._inflight.items()
|
||||
if xfer.kv_request_id in failed_kv_request_ids
|
||||
]
|
||||
for tid in ids_to_cancel:
|
||||
self._inflight_pop(tid)
|
||||
self._transport.cancel(ids_to_cancel)
|
||||
for kv_request_id in failed_kv_request_ids:
|
||||
# Cancel each failed round's other inflight and fail its
|
||||
# remaining store jobs — nothing else will settle them.
|
||||
if failed_rounds:
|
||||
for kv_request_id, rnd in failed_rounds:
|
||||
ids_to_cancel = [
|
||||
tid for tid, x in self._inflight.items() if x.round is rnd
|
||||
]
|
||||
for tid in ids_to_cancel:
|
||||
self._inflight_pop(tid)
|
||||
if ids_to_cancel:
|
||||
self._transport.cancel(ids_to_cancel)
|
||||
results.extend(self._fail_round_jobs(rnd))
|
||||
self._maybe_prune(kv_request_id)
|
||||
|
||||
return results
|
||||
@@ -811,8 +841,8 @@ class ServerRole:
|
||||
|
||||
def drain_pending_aborts(self) -> None:
|
||||
"""Re-attempt every parked abort once per poll tick."""
|
||||
for kv_request_id in list(self._parked_aborts):
|
||||
self._drain_abort(kv_request_id)
|
||||
for kv_request_id, round_seq in list(self._pending_aborts):
|
||||
self._drain_abort(kv_request_id, round_seq)
|
||||
|
||||
def close(self) -> tuple[list[int], list[ReqContext]]:
|
||||
"""Tear down. Cancels inflight.
|
||||
@@ -839,7 +869,7 @@ class ServerRole:
|
||||
failed_serves.extend(self._finished_lookup_ctxs)
|
||||
self._requests.clear()
|
||||
self._serve_pending.clear()
|
||||
self._parked_aborts.clear()
|
||||
self._pending_aborts.clear()
|
||||
self._finished_lookup_ctxs.clear()
|
||||
return failed_stores, failed_serves
|
||||
|
||||
@@ -870,6 +900,8 @@ class ServerRole:
|
||||
xfer = self._inflight.pop(tid, None)
|
||||
if xfer is None:
|
||||
return None
|
||||
xfer.round.inflight -= 1
|
||||
assert xfer.round.inflight >= 0
|
||||
st = self._requests.get(xfer.kv_request_id)
|
||||
if st is not None:
|
||||
st.inflight_tids.discard(tid)
|
||||
@@ -880,80 +912,108 @@ class ServerRole:
|
||||
) -> list[StoreResult]:
|
||||
"""Emit StoreResults for a completed transfer's store jobs.
|
||||
|
||||
Pops each attached job from ``_store_jobs`` and clears it from the
|
||||
request's pending set. A job already popped (via timeout, cancel,
|
||||
etc.) is skipped so we never double-emit a contradictory result.
|
||||
Pops each attached job from ``_store_jobs`` and clears it from
|
||||
its round's pending set. A job already popped (via timeout,
|
||||
cancel, etc.) is skipped so we never double-emit a contradictory
|
||||
result.
|
||||
"""
|
||||
results: list[StoreResult] = []
|
||||
st = self._requests.get(xfer.kv_request_id)
|
||||
req = st.outbound if st is not None else None
|
||||
for job_id in xfer.job_ids:
|
||||
if self._store_jobs.pop(job_id, None) is None:
|
||||
continue
|
||||
results.append(StoreResult(job_id=job_id, success=success))
|
||||
if req is not None:
|
||||
req.pending_job_ids.discard(job_id)
|
||||
xfer.round.pending_job_ids.discard(job_id)
|
||||
return results
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal — finalize / abort drain
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _fail_round_jobs(self, rnd: _OutboundRequestState) -> list[StoreResult]:
|
||||
"""Fail a terminated round's still-pending store jobs (idempotent)."""
|
||||
results: list[StoreResult] = []
|
||||
for job_id in rnd.pending_job_ids:
|
||||
if self._store_jobs.pop(job_id, None) is None:
|
||||
continue
|
||||
results.append(StoreResult(job_id=job_id, success=False))
|
||||
rnd.pending_job_ids.clear()
|
||||
return results
|
||||
|
||||
def _finalize_outbound(
|
||||
self,
|
||||
kv_request_id: str,
|
||||
round_key: int,
|
||||
success: bool | None = None,
|
||||
send_done: bool = True,
|
||||
) -> None:
|
||||
"""Pop the outbound state and emit terminal results.
|
||||
|
||||
Called when no further work will happen for this kv_request_id
|
||||
on the server side: either request_finish has fired and there
|
||||
are no inflight transfers, or the last inflight just completed
|
||||
while finishing.
|
||||
"""Pop one round and emit its terminal results.
|
||||
|
||||
If ``success`` is None, derive it from ``req.remaining == 0``.
|
||||
The same flag is used for both the peer's TransferDoneMsg and
|
||||
the StoreResult(s) emitted for any leftover pending job_ids.
|
||||
``send_done=False`` skips the TransferDoneMsg (terminal empty
|
||||
fetch). Other rounds of the id are untouched.
|
||||
"""
|
||||
st = self._requests[kv_request_id]
|
||||
assert st.outbound is not None
|
||||
req = st.outbound
|
||||
st.outbound = None
|
||||
req = st.outbound.pop(round_key)
|
||||
if success is None:
|
||||
success = req.remaining == 0
|
||||
for job_id in req.pending_job_ids:
|
||||
self._store_jobs.pop(job_id, None)
|
||||
self._pending_store_results.append(
|
||||
StoreResult(job_id=job_id, success=success)
|
||||
)
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||
TransferDoneMsg.KV_REQUEST_ID: kv_request_id,
|
||||
TransferDoneMsg.SUCCESS: success,
|
||||
}
|
||||
success = req.demand_received and req.remaining == 0
|
||||
settled = self._fail_round_jobs(req) if not success else None
|
||||
if settled is not None:
|
||||
self._pending_store_results.extend(settled)
|
||||
else:
|
||||
for job_id in req.pending_job_ids:
|
||||
if self._store_jobs.pop(job_id, None) is None:
|
||||
continue
|
||||
self._pending_store_results.append(
|
||||
StoreResult(job_id=job_id, success=True)
|
||||
)
|
||||
req.pending_job_ids.clear()
|
||||
logger.debug(
|
||||
"P2PSession %s: finalize kv_request_id=%s round=%s success=%s "
|
||||
"remaining=%d leftover_available=%d send_done=%s",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
round_key,
|
||||
success,
|
||||
req.remaining,
|
||||
len(req.available),
|
||||
send_done,
|
||||
)
|
||||
if send_done and req.demand_received:
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||
TransferDoneMsg.KV_REQUEST_ID: kv_request_id,
|
||||
TransferDoneMsg.SUCCESS: success,
|
||||
TransferDoneMsg.ROUND_SEQ: round_key,
|
||||
}
|
||||
)
|
||||
self._maybe_prune(kv_request_id)
|
||||
|
||||
def _drain_abort(self, kv_request_id: str) -> None:
|
||||
def _drain_abort(self, kv_request_id: str, round_seq: int) -> None:
|
||||
"""One drain attempt for a pending abort.
|
||||
|
||||
Stops accepting more blocks for ``kv_request_id``, then asks the
|
||||
transport to cancel any matching inflight transfers in
|
||||
``mode="wait"``. Sends ``AbortAckMsg`` once nothing remains
|
||||
inflight, or after ``_CANCEL_DRAIN_TIMEOUT_S`` falls back to
|
||||
``mode="immediate"`` and acks anyway.
|
||||
Detaches the aborted round, then asks the transport to cancel its
|
||||
inflight transfers in ``mode="wait"``. Sends ``AbortAckMsg`` once
|
||||
nothing remains inflight, or after ``_CANCEL_DRAIN_TIMEOUT_S``
|
||||
falls back to ``mode="immediate"`` and acks anyway.
|
||||
"""
|
||||
st = self._requests[kv_request_id]
|
||||
st.outbound = None
|
||||
ids = list(st.inflight_tids)
|
||||
rnd = st.outbound.pop(round_seq, None)
|
||||
if rnd is not None:
|
||||
# Its transfers are being cancelled; fail its jobs now
|
||||
# instead of leaking them to the store timeout.
|
||||
self._pending_store_results.extend(self._fail_round_jobs(rnd))
|
||||
ids = [
|
||||
tid
|
||||
for tid, x in self._inflight.items()
|
||||
if x.kv_request_id == kv_request_id and x.round_key == round_seq
|
||||
]
|
||||
if not ids:
|
||||
self._finalize_abort(kv_request_id)
|
||||
self._finalize_abort(kv_request_id, round_seq)
|
||||
return
|
||||
|
||||
assert st.abort_started_at is not None
|
||||
expired = time.monotonic() - st.abort_started_at >= _CANCEL_DRAIN_TIMEOUT_S
|
||||
if expired:
|
||||
started_at = self._pending_aborts[(kv_request_id, round_seq)]
|
||||
if time.monotonic() - started_at >= _CANCEL_DRAIN_TIMEOUT_S:
|
||||
for tid in ids:
|
||||
self._inflight_pop(tid)
|
||||
self._transport.cancel(ids, mode="immediate")
|
||||
@@ -964,7 +1024,7 @@ class ServerRole:
|
||||
kv_request_id,
|
||||
len(ids),
|
||||
)
|
||||
self._finalize_abort(kv_request_id)
|
||||
self._finalize_abort(kv_request_id, round_seq)
|
||||
return
|
||||
|
||||
still = self._transport.cancel(ids, mode="wait")
|
||||
@@ -977,16 +1037,15 @@ class ServerRole:
|
||||
if tid not in still_set:
|
||||
self._inflight_pop(tid)
|
||||
if not still:
|
||||
self._finalize_abort(kv_request_id)
|
||||
self._finalize_abort(kv_request_id, round_seq)
|
||||
|
||||
def _finalize_abort(self, kv_request_id: str) -> None:
|
||||
st = self._requests[kv_request_id]
|
||||
st.abort_started_at = None
|
||||
self._parked_aborts.discard(kv_request_id)
|
||||
def _finalize_abort(self, kv_request_id: str, round_seq: int) -> None:
|
||||
self._pending_aborts.pop((kv_request_id, round_seq), None)
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: AbortAckMsg.TYPE,
|
||||
AbortAckMsg.KV_REQUEST_ID: kv_request_id,
|
||||
AbortAckMsg.ROUND_SEQ: round_seq,
|
||||
}
|
||||
)
|
||||
self._maybe_prune(kv_request_id)
|
||||
@@ -995,7 +1054,13 @@ class ServerRole:
|
||||
# Internal — transfers and store-job timeouts
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _submit_transfer(self, kv_request_id: str, result: _MatchResult) -> None:
|
||||
def _submit_transfer(
|
||||
self,
|
||||
kv_request_id: str,
|
||||
result: _MatchResult,
|
||||
rnd: _OutboundRequestState,
|
||||
round_key: int,
|
||||
) -> None:
|
||||
logger.debug(
|
||||
"P2PSession %s: NIXL write_blocks CALL kv_request_id=%s "
|
||||
"local_idxs=%d remote_idxs=%d",
|
||||
@@ -1016,12 +1081,15 @@ class ServerRole:
|
||||
transfer_id,
|
||||
len(result.local_idxs),
|
||||
)
|
||||
rnd.inflight += 1
|
||||
self._inflight_add(
|
||||
transfer_id,
|
||||
_InflightXfer(
|
||||
kv_request_id=kv_request_id,
|
||||
block_count=len(result.local_idxs),
|
||||
job_ids=result.job_ids,
|
||||
round=rnd,
|
||||
round_key=round_key,
|
||||
),
|
||||
)
|
||||
else:
|
||||
@@ -1031,22 +1099,24 @@ class ServerRole:
|
||||
kv_request_id,
|
||||
len(result.local_idxs),
|
||||
)
|
||||
# The matched blocks were popped from req.demanded /
|
||||
# req.available, but no inflight will satisfy them, so
|
||||
# remaining will never reach 0 on its own. Mark the
|
||||
# request as finishing so the existing terminal paths
|
||||
# clean up: if other inflight is in flight, the last one
|
||||
# to drain will fire _finalize_outbound(success=False)
|
||||
# via the elif branch in collect_results. If
|
||||
# nothing else is in flight, finalize now so the peer
|
||||
# and the local store jobs don't wait for finish_request
|
||||
# or for _STORE_TIMEOUT_S / _LOAD_TIMEOUT_S.
|
||||
# The matched blocks were popped from rnd.demanded /
|
||||
# rnd.available, but no inflight will satisfy them, so
|
||||
# remaining will never reach 0 on its own. Mark the round
|
||||
# as finishing so the existing terminal paths clean up: if
|
||||
# other transfers of this round are in flight, the last one
|
||||
# to drain will fire _finalize_outbound(success=False) via
|
||||
# the elif branch in collect_results. If nothing else is in
|
||||
# flight, finalize now so the peer and the local store jobs
|
||||
# don't wait for finish_request or for _STORE_TIMEOUT_S /
|
||||
# _LOAD_TIMEOUT_S.
|
||||
rnd.finishing = True
|
||||
st = self._requests.get(kv_request_id)
|
||||
req = st.outbound if st is not None else None
|
||||
if req is not None:
|
||||
req.finishing = True
|
||||
if not self._has_inflight_for(kv_request_id):
|
||||
self._finalize_outbound(kv_request_id, success=False)
|
||||
if (
|
||||
st is not None
|
||||
and st.outbound.get(round_key) is rnd
|
||||
and rnd.inflight == 0
|
||||
):
|
||||
self._finalize_outbound(kv_request_id, round_key, success=False)
|
||||
|
||||
def _timeout_pending_store_jobs(self) -> list[StoreResult]:
|
||||
if not self._store_jobs:
|
||||
|
||||
@@ -374,26 +374,34 @@ class P2PSession:
|
||||
for bh in msg[FetchMsg.KEYS]
|
||||
]
|
||||
block_indexes = msg[FetchMsg.BLOCK_INDEXES]
|
||||
round_seq = msg[FetchMsg.ROUND_SEQ]
|
||||
# Run the server-role state machine inline as today —
|
||||
# add_fetch_demand records demand against any blocks we've
|
||||
# already seen in `available`. Report the kv_request_id so
|
||||
# the manager (after poll() returns) can replay any parked
|
||||
# submit_store batches; their add_stored_blocks calls hit
|
||||
# the demand recorded here and submit transfers immediately.
|
||||
self._server.on_fetch(kv_request_id, keys, block_indexes)
|
||||
self._server.on_fetch(kv_request_id, keys, block_indexes, round_seq)
|
||||
self._new_fetch_ids.append(kv_request_id)
|
||||
elif msg_type == AbortFetchMsg.TYPE:
|
||||
AbortFetchMsg.validate(msg)
|
||||
self._server.on_abort_fetch(msg[AbortFetchMsg.KV_REQUEST_ID])
|
||||
self._server.on_abort_fetch(
|
||||
msg[AbortFetchMsg.KV_REQUEST_ID],
|
||||
msg[AbortFetchMsg.ROUND_SEQ],
|
||||
)
|
||||
elif msg_type == TransferDoneMsg.TYPE:
|
||||
TransferDoneMsg.validate(msg)
|
||||
self._client.on_transfer_done(
|
||||
msg[TransferDoneMsg.KV_REQUEST_ID],
|
||||
msg[TransferDoneMsg.SUCCESS],
|
||||
msg[TransferDoneMsg.ROUND_SEQ],
|
||||
)
|
||||
elif msg_type == AbortAckMsg.TYPE:
|
||||
AbortAckMsg.validate(msg)
|
||||
self._client.on_abort_ack(msg[AbortAckMsg.KV_REQUEST_ID])
|
||||
self._client.on_abort_ack(
|
||||
msg[AbortAckMsg.KV_REQUEST_ID],
|
||||
msg[AbortAckMsg.ROUND_SEQ],
|
||||
)
|
||||
elif msg_type == LookupMsg.TYPE:
|
||||
LookupMsg.validate(msg)
|
||||
kv_request_id = msg[LookupMsg.KV_REQUEST_ID]
|
||||
@@ -401,7 +409,7 @@ class P2PSession:
|
||||
OffloadKey(bh if isinstance(bh, bytes) else bytes(bh))
|
||||
for bh in msg[LookupMsg.KEYS]
|
||||
]
|
||||
self._server.on_lookup(kv_request_id, keys)
|
||||
self._server.on_lookup(kv_request_id, keys, msg[LookupMsg.ROUND_SEQ])
|
||||
elif msg_type == LookupRespMsg.TYPE:
|
||||
LookupRespMsg.validate(msg)
|
||||
kv_request_id = msg[LookupRespMsg.KV_REQUEST_ID]
|
||||
|
||||
Reference in New Issue
Block a user