[Bugfix][KV Offload][P2P] Scope serve state to fetch rounds (#49877)

Signed-off-by: Itay Etelis <itay.etelis@ibm.com>
Co-authored-by: Itay Etelis <itay.etelis@ibm.com>
Co-authored-by: Itay Etelis <Itay.etelis@gmail.com>
This commit is contained in:
Itay Etelis
2026-07-28 18:01:55 +03:00
committed by GitHub
co-authored by Itay Etelis Itay Etelis
parent 0d0504b54c
commit 30217b0e80
6 changed files with 570 additions and 390 deletions
@@ -1612,16 +1612,17 @@ class TestBindHostPortDefaults:
monkeypatch.setattr(
manager_module,
"NixlTransport",
lambda agent_name, *a, **k: calls.update(nixl_name=agent_name)
or SimpleNamespace(),
lambda agent_name, *a, **k: (
calls.update(nixl_name=agent_name) or SimpleNamespace()
),
)
monkeypatch.setattr(
manager_module,
"ZmqTransport",
lambda local_id, host, port, *a, **k: calls.update(
zmq_id=local_id, zmq_host=host, zmq_port=port
)
or SimpleNamespace(),
lambda local_id, host, port, *a, **k: (
calls.update(zmq_id=local_id, zmq_host=host, zmq_port=port)
or SimpleNamespace()
),
)
spec = SimpleNamespace(
blocks_per_chunk=1,
+100 -16
View File
@@ -49,6 +49,7 @@ from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
from vllm.v1.kv_offload.tiering.p2p.session.server import (
_CANCEL_DRAIN_TIMEOUT_S,
_InflightXfer,
_OutboundRequestState,
)
from vllm.v1.kv_offload.tiering.p2p.session.session import (
_MAX_CONSECUTIVE_DISPATCH_ERRORS,
@@ -315,10 +316,26 @@ def _activate(
# either a missing entry or a None field. These helpers paper over that.
def _client_load(session: P2PSession, kv_request_id: str):
"""The single in-flight load of a kv_request_id (loads are per-round)."""
loads = session._client._requests[kv_request_id].loads
assert len(loads) == 1
return next(iter(loads.values()))
def _srv_outbound(session: P2PSession, kv_request_id: str):
"""Outbound serve state for a kv_request_id, or None (idle / GC'd)."""
"""Serve-side round for a kv_request_id, or None (idle / GC'd).
Rounds are keyed by wire round_seq; surfaces the demanded round when
a fetch has bound one, else any parked supply round.
"""
st = session._server._requests.get(kv_request_id)
return st.outbound if st is not None else None
if st is None or not st.outbound:
return None
for rnd in st.outbound.values():
if rnd.demand_received:
return rnd
return next(iter(st.outbound.values()))
def _srv_lookups(session: P2PSession) -> list:
@@ -330,8 +347,10 @@ def _srv_lookups(session: P2PSession) -> list:
def _srv_abort_started(session: P2PSession, kv_request_id: str) -> float | None:
"""Pending-abort start time for a kv_request_id, or None."""
st = session._server._requests.get(kv_request_id)
return st.abort_started_at if st is not None else None
for (kv, _), started in session._server._pending_aborts.items():
if kv == kv_request_id:
return started
return None
def _srv_inflight_count(session: P2PSession, kv_request_id: str) -> int:
@@ -479,6 +498,7 @@ class TestClientFlows:
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: True,
}
@@ -495,6 +515,7 @@ class TestClientFlows:
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: False,
}
@@ -538,6 +559,7 @@ class TestClientFlows:
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: True,
}
@@ -552,7 +574,7 @@ class TestClientFlows:
session.request_blocks(
job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
session._client._requests["req-1"].load.submitted_at = time.monotonic() - 60.0
_client_load(session, "req-1").submitted_at = time.monotonic() - 60.0
session.poll()
abort = conn._sent[-1]
assert abort[TYPE_KEY] == AbortFetchMsg.TYPE
@@ -569,7 +591,7 @@ class TestClientFlows:
job_id=7, kv_request_id="req-7", keys=[b"k"], block_ids=[0]
)
# 1) Trip the load timeout to send AbortFetch and stamp aborted_at.
session._client._requests["req-7"].load.submitted_at = (
_client_load(session, "req-7").submitted_at = (
time.monotonic() - _LOAD_TIMEOUT_S - 1.0
)
loads = session.poll().loads
@@ -579,11 +601,11 @@ class TestClientFlows:
and m[AbortFetchMsg.KV_REQUEST_ID] == "req-7"
for m in conn._sent
)
assert session._client._requests["req-7"].load.aborted_at is not None
assert _client_load(session, "req-7").aborted_at is not None
# 2) Now backdate aborted_at past the abort-ack timeout. No ack ever
# arrived from the peer.
session._client._requests["req-7"].load.aborted_at = (
_client_load(session, "req-7").aborted_at = (
time.monotonic() - _ABORT_ACK_TIMEOUT_S - 1.0
)
loads = session.poll().loads
@@ -599,17 +621,18 @@ class TestClientFlows:
session.request_blocks(
job_id=8, kv_request_id="req-8", keys=[b"k"], block_ids=[0]
)
session._client._requests["req-8"].load.submitted_at = (
_client_load(session, "req-8").submitted_at = (
time.monotonic() - _LOAD_TIMEOUT_S - 1.0
)
# First poll: AbortFetch goes out.
session.poll()
assert session._client._requests["req-8"].load.aborted_at is not None
assert _client_load(session, "req-8").aborted_at is not None
# Peer acks the abort.
conn.enqueue(
{
TYPE_KEY: AbortAckMsg.TYPE,
AbortAckMsg.ROUND_SEQ: 0,
AbortAckMsg.KV_REQUEST_ID: "req-8",
}
)
@@ -915,6 +938,7 @@ class TestLookupFlow:
conn.enqueue(
{
TYPE_KEY: LookupMsg.TYPE,
LookupMsg.ROUND_SEQ: 0,
LookupMsg.KV_REQUEST_ID: "req-1",
LookupMsg.KEYS: [b"hX", b"hY", b"hZ"],
}
@@ -951,6 +975,7 @@ def _send_lookup(conn: FakeConnection, kv_request_id: str, keys: list[bytes]):
conn.enqueue(
{
TYPE_KEY: LookupMsg.TYPE,
LookupMsg.ROUND_SEQ: 0,
LookupMsg.KV_REQUEST_ID: kv_request_id,
LookupMsg.KEYS: list(keys),
}
@@ -1201,6 +1226,7 @@ class TestServerLookupHandling:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [],
FetchMsg.BLOCK_INDEXES: [],
@@ -1240,6 +1266,7 @@ class TestServerLookupHandling:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"hA", b"hB"],
FetchMsg.BLOCK_INDEXES: [20, 21],
@@ -1277,6 +1304,7 @@ class TestServerFlows:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [10, 11],
@@ -1295,6 +1323,7 @@ class TestServerFlows:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
@@ -1313,6 +1342,7 @@ class TestServerFlows:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
@@ -1331,6 +1361,7 @@ class TestServerFlows:
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
@@ -1349,13 +1380,19 @@ class TestServerFlows:
tid = 42
session._server._inflight_add(
tid,
_InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}),
_InflightXfer(
kv_request_id="req-1",
block_count=1,
job_ids={1},
round=_OutboundRequestState(inflight=1),
),
)
transport._cancel_still_inflight.add(tid)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
@@ -1376,13 +1413,19 @@ class TestServerFlows:
tid = 42
session._server._inflight_add(
tid,
_InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}),
_InflightXfer(
kv_request_id="req-1",
block_count=1,
job_ids={1},
round=_OutboundRequestState(inflight=1),
),
)
transport._cancel_still_inflight.add(tid)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
@@ -1409,20 +1452,26 @@ class TestServerFlows:
tid = 42
session._server._inflight_add(
tid,
_InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}),
_InflightXfer(
kv_request_id="req-1",
block_count=1,
job_ids={1},
round=_OutboundRequestState(inflight=1),
),
)
transport._cancel_still_inflight.add(tid)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
session.poll()
assert _srv_abort_started(session, "req-1") is not None
# Backdate past the drain deadline.
session._server._requests["req-1"].abort_started_at = (
session._server._pending_aborts[("req-1", 0)] = (
time.monotonic() - _CANCEL_DRAIN_TIMEOUT_S - 1.0
)
# Even if the transport still claims it can't cancel, the
@@ -1445,13 +1494,19 @@ class TestServerFlows:
tid = 42
session._server._inflight_add(
tid,
_InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}),
_InflightXfer(
kv_request_id="req-1",
block_count=1,
job_ids={1},
round=_OutboundRequestState(inflight=1),
),
)
transport._cancel_still_inflight.add(tid)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
@@ -1463,6 +1518,7 @@ class TestServerFlows:
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
@@ -1497,6 +1553,7 @@ class TestServerFlows:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
@@ -1529,6 +1586,7 @@ class TestServerFlows:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
@@ -1571,6 +1629,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
@@ -1596,6 +1655,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [10, 11],
@@ -1631,6 +1691,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [10],
@@ -1665,6 +1726,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [10],
@@ -1686,6 +1748,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-2",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [10, 11],
@@ -1719,6 +1782,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"demand"],
FetchMsg.BLOCK_INDEXES: [5],
@@ -1753,6 +1817,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [10],
@@ -1790,6 +1855,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [10],
@@ -1824,6 +1890,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2", b"k3"],
FetchMsg.BLOCK_INDEXES: [10, 11, 12],
@@ -1885,6 +1952,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [10, 11],
@@ -1951,6 +2019,7 @@ class TestBidirectional:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-srv",
FetchMsg.KEYS: [b"served"],
FetchMsg.BLOCK_INDEXES: [7],
@@ -1981,6 +2050,7 @@ class TestBidirectional:
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-cli",
TransferDoneMsg.SUCCESS: True,
}
@@ -2168,6 +2238,7 @@ class TestAdversarial:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-bad",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [1],
@@ -2216,6 +2287,7 @@ class TestDispatchErrorHandling:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-bad",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [1],
@@ -2246,6 +2318,7 @@ class TestDispatchErrorHandling:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [0],
@@ -2269,6 +2342,7 @@ class TestDispatchErrorHandling:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [0],
@@ -2296,6 +2370,7 @@ class TestDispatchErrorHandling:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [0],
@@ -2341,6 +2416,7 @@ class TestInflightPerReqInvariant:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: kv_id,
FetchMsg.KEYS: keys,
FetchMsg.BLOCK_INDEXES: indexes,
@@ -2394,7 +2470,12 @@ class TestInflightPerReqInvariant:
tid = kv_id_idx * 10 + j
session._server._inflight_add(
tid,
_InflightXfer(kv_request_id=kv_id, block_count=1, job_ids={tid}),
_InflightXfer(
kv_request_id=kv_id,
block_count=1,
job_ids={tid},
round=_OutboundRequestState(inflight=1),
),
)
assert _srv_total_inflight(session) == len(session._server._inflight)
assert session._server._has_inflight_for("req-0")
@@ -2488,6 +2569,7 @@ class TestFetchMsgValidation:
def _valid_msg(self) -> dict:
return {
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [0, 1],
@@ -2513,6 +2595,7 @@ class TestTransferDoneMsgValidation:
def test_valid_message_passes(self):
msg = {
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: True,
}
@@ -2521,6 +2604,7 @@ class TestTransferDoneMsgValidation:
def test_success_wrong_type(self):
msg = {
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: 1,
}
+149 -144
View File
@@ -11,7 +11,6 @@ callback injected by the coordinator (which gates on ConnectAck).
from __future__ import annotations
import enum
import time
from collections.abc import Callable, Sequence
from dataclasses import dataclass, field
@@ -35,26 +34,12 @@ _LOAD_TIMEOUT_S = 30.0
_ABORT_ACK_TIMEOUT_S = 10.0
class ClientPhase(enum.Enum):
"""Lifecycle of a request's client-side lookup/fetch signalling.
Advances monotonically. Only ``finish`` reads it, to decide
whether a terminal empty FetchMsg is owed to release the peer's
lookup state (owed only from ``PROBING``: a LookupMsg went out but no
FetchMsg has since closed the peer's lookup phase).
"""
REGISTERED = enum.auto() # keys in probes/unsent, nothing sent yet
PROBING = enum.auto() # LookupMsg flushed, awaiting responses
FETCH_SENT = enum.auto() # FetchMsg sent (real or terminal empty)
@dataclass
class _InboundLoadState:
"""Client-role state for a single in-flight load request.
Lives on ``_ClientRequestState.load`` for the duration of a fetch;
the owning kv_request_id is the dict key, so it isn't stored here.
Lives in ``_ClientRequestState.loads`` keyed by round_seq for the
duration of a fetch; the owning kv_request_id is the outer dict key.
"""
job_id: int # opaque ID assigned by the manager to this load request
@@ -68,7 +53,7 @@ class _ClientRequestState:
One entry per kv_request_id we're driving. Lookup-phase fields are
used only by symmetric P2P (``do_p2p_fetch``); PD-only loads leave
``probes``/``unsent`` empty and drive just ``phase`` and ``load``. An
``probes``/``unsent`` empty and drive just ``phase`` and ``loads``. An
entry is dropped once every field is idle — see ``ClientRole._maybe_prune``.
"""
@@ -82,11 +67,23 @@ class _ClientRequestState:
# OffloadKeys registered but not yet flushed onto the wire. Drained and
# cleared by the next flush_pending_lookups.
unsent: list[OffloadKey] = field(default_factory=list)
# Current lookup round. LookupMsgs carry it, each fetch closes it and
# advances it, so every round's supply/demand/completion is isolated
# on the wire. PD clients never probe and stay on round 0.
round_seq: int = 0
# This id ran the symmetric lookup phase (register_lookup); a fetch
# with keys then requires every key to be a confirmed probe. PD
# loads never probe.
probed: bool = False
# Monotonic lookup/fetch signalling phase; see ``ClientPhase``.
phase: ClientPhase = ClientPhase.REGISTERED
# Set while a fetch is in flight; cleared on completion/abort/timeout.
load: _InboundLoadState | None = None
# The peer holds lookup state no FetchMsg has closed: a LookupMsg
# was flushed since the last fetch. finish owes a terminal empty
# FetchMsg while set, so the peer releases parked supply.
peer_lookup_open: bool = False
# In-flight loads keyed by the round their fetch carried. The
# scheduler submits loads incrementally as chunks resolve, so several
# can be in flight at once; TransferDone/AbortAck match by round.
loads: dict[int, _InboundLoadState] = field(default_factory=dict)
class LoadResult(NamedTuple):
@@ -133,11 +130,10 @@ class ClientRole:
# _serve_pending. Populated by register_lookup, drained by
# flush_pending_lookups, and discarded on finish/close.
self._flush_pending: set[str] = set()
# kv_request_ids with a fetch in flight (``st.load is not None``) —
# the work-list collect_results walks for timeouts, and the
# has_active_loads predicate, instead of scanning every request.
# Kept in exact sync with ``st.load``: armed in request_blocks,
# discarded wherever load is cleared, and cleared on close.
# kv_request_ids with at least one fetch in flight — the work-list
# collect_results walks for timeouts, and the has_active_loads
# predicate, instead of scanning every request. Kept in exact sync
# with ``st.loads``.
self._active_loads: set[str] = set()
self._completed_loads: list[LoadResult] = []
@@ -156,17 +152,21 @@ class ClientRole:
def _maybe_prune(self, kv_request_id: str) -> None:
"""Drop the entry once it holds no live load or lookup state.
The sticky ``phase`` is only read by ``finish``. A probe
clears when its fetch is issued (``request_blocks``) or when the
request finishes (``finish``/``close``); in the former case
``load`` is set and keeps the entry alive, in the latter the phase
is no longer needed — so dropping on emptiness never loses a phase
still in use.
``peer_lookup_open`` is only read by ``finish``, and every path
that clears the last probe (fetch / finish / close) also settles
it, so dropping on emptiness never loses a flag still in use.
"""
st = self._requests.get(kv_request_id)
if st is not None and st.load is None and not st.probes and not st.unsent:
if st is not None and not st.loads and not st.probes and not st.unsent:
del self._requests[kv_request_id]
def _on_load_terminal(self, kv_request_id: str, st: _ClientRequestState) -> None:
"""Wind down id-level state once no load remains in flight."""
if st.loads:
return
self._active_loads.discard(kv_request_id)
self._maybe_prune(kv_request_id)
@property
def has_active_loads(self) -> bool:
"""True if any kv_request_id has a fetch in flight."""
@@ -184,7 +184,12 @@ class ClientRole:
block_ids: Sequence[int],
send_ready: bool,
) -> None:
"""Register a load request and send the FetchMsg."""
"""Send the FetchMsg closing the current lookup round.
The scheduler may submit several loads per kv_request_id as its
matched prefix resolves incrementally; each fetch carries the
round it closes so the loads stay independent on the wire.
"""
logger.debug(
"P2PSession %s: request_blocks job_id=%d kv_request_id=%s "
"blocks=%d ready=%s",
@@ -195,77 +200,74 @@ class ClientRole:
send_ready,
)
st = self._get_or_create_request(kv_request_id)
st.load = _InboundLoadState(
round_seq = st.round_seq
st.round_seq += 1
st.loads[round_seq] = _InboundLoadState(
job_id=job_id,
submitted_at=time.monotonic(),
)
self._active_loads.add(kv_request_id)
st.phase = ClientPhase.FETCH_SENT
st.peer_lookup_open = False
self._send(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.KV_REQUEST_ID: kv_request_id,
FetchMsg.KEYS: list(keys),
FetchMsg.BLOCK_INDEXES: [int(idx) for idx in block_ids],
FetchMsg.ROUND_SEQ: round_seq,
}
)
# Issuing the fetch ends this request's lookup phase, so drop all
# probe state. Once the peer serves this fetch both sides unpin, so
# the producer may evict the block; a stale cached True would
# otherwise let a re-scheduled lookup() return HIT without
# re-probing, pointing at a block the producer no longer holds.
# Clearing forces a fresh LookupMsg on re-schedule so the producer
# answers from current state. For a symmetric-P2P request (probes
# populated) every fetched block was a confirmed HIT; a PD-only load
# never probes, so probes is empty and the clear is a no-op.
if st.probes:
# Issuing the fetch closes this lookup round, so drop all probe
# state. Once the peer serves the fetch both sides unpin, so a
# stale cached True would let a later lookup() return HIT for a
# block the producer may have evicted; clearing forces a fresh
# probe under the next round.
if st.probed and keys:
assert st.probes, (
f"symmetric fetch for {kv_request_id} has keys but no probes"
)
assert all(st.probes.get(key) is True for key in keys)
st.probes.clear()
def finish(self, kv_request_id: str) -> None:
"""Finish a request: abort any in-flight load and release lookup state.
"""Finish a request: abort in-flight loads and release lookup state.
Called from the session's ``finish_request``. The two branches are
mutually exclusive: ``load`` is set only by ``request_blocks``, which
also advances ``phase`` to ``FETCH_SENT``, and nothing moves it back
to ``PROBING`` — so a fetch in flight never coexists with the
``PROBING`` phase.
- Fetch in flight (``load`` set, phase ``FETCH_SENT``): send an
AbortFetchMsg unless the load is already aborting, then drop it.
- Outstanding lookups (``PROBING``): a LookupMsg was flushed but no
FetchMsg has closed the peer's lookup phase. Every FetchMsg the
server receives in p2p mode is its "request finished" signal (it
releases lookup state and fires ``cb.finish_request``); when the
client's lookups all missed no FetchMsg is otherwise sent, so emit
a terminal empty one purely to trigger those semantics. In
``REGISTERED`` the peer never received a LookupMsg and in
``FETCH_SENT`` a FetchMsg already closed the phase, so neither owes
a terminal FetchMsg.
Called from the session's ``finish_request``. Sends an
AbortFetchMsg per load not already aborting, and — independently —
the terminal empty FetchMsg when the peer still holds lookup
state no fetch has closed (its "request finished" signal: it
releases lookup state, drains parked supply, and fires
``cb.finish_request``). A later round's supply can be parked
while an earlier round's load is still in flight, so both can be
owed at once.
Then drop all probe/lookup state and prune the entry.
"""
st = self._requests.get(kv_request_id)
if st is None:
return
if st.load is not None:
if st.load.aborted_at is None:
if st.loads:
for round_seq, load in st.loads.items():
if load.aborted_at is not None:
continue
self._send(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: kv_request_id,
AbortFetchMsg.ROUND_SEQ: round_seq,
}
)
st.load = None
st.loads.clear()
self._active_loads.discard(kv_request_id)
elif st.phase is ClientPhase.PROBING:
st.phase = ClientPhase.FETCH_SENT
if st.peer_lookup_open:
st.peer_lookup_open = False
self._send(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.KV_REQUEST_ID: kv_request_id,
FetchMsg.KEYS: [],
FetchMsg.BLOCK_INDEXES: [],
FetchMsg.ROUND_SEQ: st.round_seq,
}
)
st.probes.clear()
@@ -273,20 +275,21 @@ class ClientRole:
self._flush_pending.discard(kv_request_id)
self._maybe_prune(kv_request_id)
def on_transfer_done(self, kv_request_id: str, success: bool) -> None:
def on_transfer_done(
self, kv_request_id: str, success: bool, round_seq: int
) -> None:
"""Handle a TransferDoneMsg from the peer."""
st = self._requests.get(kv_request_id)
if st is not None and st.load is not None:
load = st.loads.pop(round_seq, None) if st is not None else None
if st is not None and load is not None:
self._completed_loads.append(
LoadResult(
job_id=st.load.job_id,
job_id=load.job_id,
kv_request_id=kv_request_id,
success=success,
)
)
st.load = None
self._active_loads.discard(kv_request_id)
self._maybe_prune(kv_request_id)
self._on_load_terminal(kv_request_id, st)
else:
# No matching in-flight load: either a duplicate
# transfer_done from the peer (protocol violation) or a
@@ -295,41 +298,44 @@ class ClientRole:
# so we can't tell — log so it's findable.
logger.warning(
"P2PSession %s: transfer_done for unknown kv_request_id=%s "
"(duplicate from peer, or raced with local cancel/timeout)",
"round=%s (duplicate from peer, or raced with local "
"cancel/timeout)",
self._peer_id,
kv_request_id,
round_seq,
)
def on_abort_ack(self, kv_request_id: str) -> None:
def on_abort_ack(self, kv_request_id: str, round_seq: int) -> None:
"""Handle an AbortAckMsg from the peer."""
st = self._requests.get(kv_request_id)
if st is not None and st.load is not None:
load = st.loads.pop(round_seq, None) if st is not None else None
if st is not None and load is not None:
logger.warning(
"P2PSession %s: load request %s (job_id=%d) timed out; "
"load job completed with failure. If this recurs, ensure "
"PYTHONHASHSEED is set to the same value on all nodes.",
self._peer_id,
kv_request_id,
st.load.job_id,
load.job_id,
)
self._completed_loads.append(
LoadResult(
job_id=st.load.job_id,
job_id=load.job_id,
kv_request_id=kv_request_id,
success=False,
)
)
st.load = None
self._active_loads.discard(kv_request_id)
self._maybe_prune(kv_request_id)
self._on_load_terminal(kv_request_id, st)
else:
# See on_transfer_done: same ambiguity (duplicate ack
# vs. raced with local cancel/timeout that already popped).
logger.warning(
"P2PSession %s: abort_ack for unknown kv_request_id=%s "
"(duplicate from peer, or raced with local cancel/timeout)",
"round=%s (duplicate from peer, or raced with local "
"cancel/timeout)",
self._peer_id,
kv_request_id,
round_seq,
)
# ------------------------------------------------------------------
@@ -345,17 +351,18 @@ class ClientRole:
- Once a LookupRespMsg has resolved the entry: returns the cached
bool result on every call without popping it.
A resolved entry is retained until its fetch is issued
(``request_blocks`` pops it) or the request finishes
A resolved entry is retained until a fetch closes the round
(``request_blocks`` clears all probes) or the request finishes
(``finish`` clears all entries for the id). A request's
block set can be re-probed across steps, so popping on read would
make a repeat probe of an already-resolved key look brand-new and
re-queue it, emitting a redundant LookupMsg for an answer we
already hold. Keeping the entry until fetch makes repeat probes
free; clearing it at fetch forces a fresh probe if the request is
re-scheduled, since the block is unpinned once served.
free; clearing at fetch forces a fresh probe under the next
round, since the block is unpinned once served.
"""
st = self._get_or_create_request(kv_request_id)
st.probed = True
okey = OffloadKey(key)
if okey in st.probes:
return st.probes[okey]
@@ -378,14 +385,11 @@ class ClientRole:
``on_schedule_end()``. A request's block set may be discovered
across several scheduler steps, so more than one LookupMsg can
go out per kv_request_id — one per step that registered new
keys. register_lookup() de-dups in-flight and already-resolved
(req_id, key) pairs, so each LookupMsg carries only the keys
first probed in that step. The peer's lookup phase for the id is
still closed by exactly one FetchMsg, which the client contract
guarantees is sent after every lookup for the id has resolved
(see request_blocks / finish). Send-gating is handled by
the injected ``_send`` callback (queues until ConnectAckMsg if
needed).
keys, all tagged with the current round. register_lookup()
de-dups in-flight and already-resolved (req_id, key) pairs, so
each LookupMsg carries only the keys first probed in that step.
Send-gating is handled by the injected ``_send`` callback
(queues until ConnectAckMsg if needed).
Only requests that registered new keys since the last flush are
visited — the ``_flush_pending`` work-list avoids scanning every
@@ -395,13 +399,9 @@ class ClientRole:
st = self._requests.get(req_id)
if st is None or not st.unsent:
continue
# Record that the peer now holds lookup state for this id so
# finish knows a terminal empty FetchMsg may be owed.
# Only promote from REGISTERED: once a fetch has gone out
# (FETCH_SENT) a later LookupMsg must not regress the phase, as
# no terminal FetchMsg is owed for an already-fetched request.
if st.phase is ClientPhase.REGISTERED:
st.phase = ClientPhase.PROBING
# The peer now holds lookup state for this id; finish owes a
# terminal empty FetchMsg until a fetch closes it.
st.peer_lookup_open = True
logger.debug(
"P2P LOOKUP client %s: SEND LookupMsg kv_request_id=%s keys=%d",
self._peer_id,
@@ -413,6 +413,7 @@ class ClientRole:
TYPE_KEY: LookupMsg.TYPE,
LookupMsg.KV_REQUEST_ID: req_id,
LookupMsg.KEYS: list(st.unsent),
LookupMsg.ROUND_SEQ: st.round_seq,
}
)
st.unsent = []
@@ -451,52 +452,56 @@ class ClientRole:
def collect_results(self) -> list[LoadResult]:
"""Walk load timeouts and drain completed loads.
Active requests past ``_LOAD_TIMEOUT_S`` get an AbortFetchMsg
sent and enter the aborting phase. Aborting requests past
``_ABORT_ACK_TIMEOUT_S`` are surfaced as failed loads.
Loads past ``_LOAD_TIMEOUT_S`` get an AbortFetchMsg sent and
enter the aborting phase. Aborting loads past
``_ABORT_ACK_TIMEOUT_S`` are surfaced as failed.
Lookups have no timeout: an unanswered probe stays None (RETRY)
until finish_request clears it — see ``_ClientRequestState.probes``.
"""
now = time.monotonic()
to_remove: list[str] = []
to_remove: list[tuple[str, int]] = []
for req_id in self._active_loads:
st = self._requests[req_id]
assert st.load is not None
load = st.load
if load.aborted_at is None:
if now - load.submitted_at >= _LOAD_TIMEOUT_S:
load.aborted_at = now
logger.warning(
"P2PSession %s: %s timed out, sending abort",
self._peer_id,
req_id,
)
self._send(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: req_id,
}
)
else:
if now - load.aborted_at >= _ABORT_ACK_TIMEOUT_S:
to_remove.append(req_id)
self._completed_loads.append(
LoadResult(
job_id=load.job_id,
kv_request_id=req_id,
success=False,
assert st.loads
for round_seq, load in st.loads.items():
if load.aborted_at is None:
if now - load.submitted_at >= _LOAD_TIMEOUT_S:
load.aborted_at = now
logger.warning(
"P2PSession %s: %s round=%s timed out, sending abort",
self._peer_id,
req_id,
round_seq,
)
)
logger.warning(
"P2PSession %s: abort_ack timed out for kv_request_id=%s",
self._peer_id,
req_id,
)
for req_id in to_remove:
self._requests[req_id].load = None
self._active_loads.discard(req_id)
self._maybe_prune(req_id)
self._send(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: req_id,
AbortFetchMsg.ROUND_SEQ: round_seq,
}
)
else:
if now - load.aborted_at >= _ABORT_ACK_TIMEOUT_S:
to_remove.append((req_id, round_seq))
self._completed_loads.append(
LoadResult(
job_id=load.job_id,
kv_request_id=req_id,
success=False,
)
)
logger.warning(
"P2PSession %s: abort_ack timed out for "
"kv_request_id=%s round=%s",
self._peer_id,
req_id,
round_seq,
)
for req_id, round_seq in to_remove:
st = self._requests[req_id]
st.loads.pop(round_seq)
self._on_load_terminal(req_id, st)
results = self._completed_loads
self._completed_loads = []
@@ -510,12 +515,12 @@ class ClientRole:
forever on an answer that can never arrive). See ``ClientCloseResult``.
"""
failed_jobs = [
st.load.job_id for st in self._requests.values() if st.load is not None
load.job_id for st in self._requests.values() for load in st.loads.values()
]
failed_req_ids = [
req_id
for req_id, st in self._requests.items()
if st.load is not None or any(hit is None for hit in st.probes.values())
if st.loads or any(hit is None for hit in st.probes.values())
]
self._requests.clear()
self._flush_pending.clear()
@@ -26,14 +26,12 @@ Block Transfer Flow (happy path)
1. Client sends FetchMsg with a kv_request_id and lists of
block keys + remote indexes where it wants the data written.
In p2p mode FetchMsg is also the server-side "request finished"
signal for the id: no further ``cb.create_store_job`` will fire
(parked LookupMsg batches are popped, so pending-key resolution
cannot promote a HIT after this point), all server-side lookup
state for the id is released, and ``cb.finish_request`` fires on
each dropped batch. The client emits exactly one FetchMsg per
lookup-touched request, including an empty one when no blocks
end up being fetched.
A request may run several lookup→fetch rounds; symmetric-P2P
messages carry ROUND_SEQ so each round's supply, demand, and
completion stay isolated. The terminal empty FetchMsg is the
server-side "request finished" signal for the id: parked
LookupMsg batches are popped and ``cb.finish_request`` fires on
each.
2. Server matches requested blocks against locally stored blocks:
- Blocks already available are transferred immediately via RDMA.
- Blocks not yet available are recorded as "demanded" and
@@ -178,33 +176,32 @@ class DisconnectMsg:
class FetchMsg:
"""Client → Server: request blocks by key and close the lookup phase.
"""Client → Server: request blocks for one lookup round.
In p2p mode FetchMsg is also the server-side "request finished"
signal for ``kv_request_id``: on receipt the server (a) fires no
further ``cb.create_store_job`` for this id — parked LookupMsg
batches are popped, so ``_resolve_pending_lookups`` cannot promote
a HIT_PENDING / RETRY key into a fresh pin after this point — and
(b) calls ``cb.finish_request(batch.ctx)`` on each dropped batch
so the TieringManager can release per-batch bookkeeping. In the
all-miss case the client emits an empty FetchMsg (``KEYS``
and ``BLOCK_INDEXES`` both empty) purely to fire this signal.
A non-empty fetch closes only its round. The terminal empty FetchMsg
(``KEYS`` and ``BLOCK_INDEXES`` both empty) is the "request
finished" signal: the server pops parked LookupMsg batches, calls
``cb.finish_request`` on each, and drains any leftover supply.
Fields:
KV_REQUEST_ID: Identifies this block transfer request.
KEYS: List of block keys (OffloadKey bytes). May be empty.
BLOCK_INDEXES: List of remote block indexes (same length as KEYS).
ROUND_SEQ: Lookup round this fetch closes. PD clients never probe
and stay on their single round 0.
"""
TYPE = "fetch"
KV_REQUEST_ID = "kv_request_id"
KEYS = "keys"
BLOCK_INDEXES = "block_indexes"
ROUND_SEQ = "round_seq"
@staticmethod
def validate(msg: dict) -> None:
"""Raise ValueError if any field has an invalid type or value."""
_require(msg, FetchMsg.KV_REQUEST_ID, str)
_require_non_neg_int(msg, FetchMsg.ROUND_SEQ)
_require_list(msg, FetchMsg.KEYS)
_require_list(msg, FetchMsg.BLOCK_INDEXES)
keys = msg[FetchMsg.KEYS]
@@ -229,17 +226,21 @@ class LookupMsg:
Fields:
KV_REQUEST_ID: Identifies this lookup transaction.
KEYS: List of block keys (OffloadKey bytes) to probe.
ROUND_SEQ: Lookup round these probes belong to; pinned supply is
parked under it for that round's fetch.
"""
TYPE = "lookup"
KV_REQUEST_ID = "kv_request_id"
KEYS = "keys"
ROUND_SEQ = "round_seq"
@staticmethod
def validate(msg: dict) -> None:
"""Raise ValueError if any field has an invalid type or value."""
_require(msg, LookupMsg.KV_REQUEST_ID, str)
_require_list(msg, LookupMsg.KEYS)
_require_non_neg_int(msg, LookupMsg.ROUND_SEQ)
class LookupRespMsg:
@@ -284,17 +285,22 @@ class TransferDoneMsg:
Fields:
KV_REQUEST_ID: The request that completed.
SUCCESS: Whether the transfer completed successfully.
ROUND_SEQ: The fetch round that completed. Several loads can be
in flight per id (the scheduler submits loads incrementally),
so completions are matched by round.
"""
TYPE = "transfer_done"
KV_REQUEST_ID = "kv_request_id"
SUCCESS = "success"
ROUND_SEQ = "round_seq"
@staticmethod
def validate(msg: dict) -> None:
"""Raise ValueError if any field has an invalid type or value."""
_require(msg, TransferDoneMsg.KV_REQUEST_ID, str)
_require(msg, TransferDoneMsg.SUCCESS, bool)
_require_non_neg_int(msg, TransferDoneMsg.ROUND_SEQ)
class AbortFetchMsg:
@@ -302,15 +308,18 @@ class AbortFetchMsg:
Fields:
KV_REQUEST_ID: The request to cancel.
ROUND_SEQ: The fetch round to cancel.
"""
TYPE = "abort_fetch"
KV_REQUEST_ID = "kv_request_id"
ROUND_SEQ = "round_seq"
@staticmethod
def validate(msg: dict) -> None:
"""Raise ValueError if any field has an invalid type or value."""
_require(msg, AbortFetchMsg.KV_REQUEST_ID, str)
_require_non_neg_int(msg, AbortFetchMsg.ROUND_SEQ)
class AbortAckMsg:
@@ -318,12 +327,15 @@ class AbortAckMsg:
Fields:
KV_REQUEST_ID: The request that was cancelled.
ROUND_SEQ: The round that was cancelled; echoes AbortFetchMsg.
"""
TYPE = "abort_ack"
KV_REQUEST_ID = "kv_request_id"
ROUND_SEQ = "round_seq"
@staticmethod
def validate(msg: dict) -> None:
"""Raise ValueError if any field has an invalid type or value."""
_require(msg, AbortAckMsg.KV_REQUEST_ID, str)
_require_non_neg_int(msg, AbortAckMsg.ROUND_SEQ)
+272 -202
View File
@@ -53,15 +53,6 @@ class StoreResult(NamedTuple):
success: bool
class _InflightXfer(NamedTuple):
"""Metadata for a single inflight RDMA transfer, keyed by transfer_id."""
kv_request_id: str
block_count: int
# The set of store job IDs that contributed blocks to this transfer.
job_ids: set[int]
class _MatchResult(NamedTuple):
"""Result of block matching: pairs ready for transfer."""
@@ -73,12 +64,17 @@ class _MatchResult(NamedTuple):
@dataclass
class _OutboundRequestState:
"""Server-role state for a single peer fetch request.
"""Server-role state for a single fetch round of a peer request.
The owning ``kv_request_id`` is the ``ServerRole._requests`` dict key
and is not duplicated on the value.
A kv_request_id may run several lookup→fetch rounds; rounds live in
``_ServerRequestState.outbound`` keyed by the wire ``round_seq``, so
terminals touch only their own round.
"""
# Supply came from inbound lookup pins (symmetric): no late
# submit_store can arrive, so unmatched fetch demand fails fast. PD
# rounds park demand for stores instead.
lookup_supplied: bool = False
demand_received: bool = False
available: dict[OffloadKey, tuple[int, int]] = field(
default_factory=dict
@@ -88,7 +84,8 @@ class _OutboundRequestState:
) # key → remote_block_idx: blocks peer wants, awaiting supply
remaining: int = 0 # blocks that need to be transferred to client
finishing: bool = False # Signal finish request ASAP
# Job IDs that submit_store'd blocks for this request and have not
inflight: int = 0 # transfers submitted for this round, not yet polled
# Job IDs that submit_store'd blocks for this round and have not
# yet emitted a StoreResult. The terminal-finalize helper drains
# this set; poll-done and poll-failed discard entries as their
# StoreResults fire.
@@ -145,6 +142,21 @@ class _OutboundRequestState:
)
@dataclass
class _InflightXfer:
"""Metadata for a single inflight RDMA transfer, keyed by transfer_id."""
kv_request_id: str
block_count: int
# The set of store job IDs that contributed blocks to this transfer.
job_ids: set[int]
# Round this transfer serves and its key in ``st.outbound``;
# remaining/finalize apply only while the round is still registered.
# Dummy default for test-seeded entries.
round: _OutboundRequestState = field(default_factory=_OutboundRequestState)
round_key: int = 0
@dataclass
class _ActiveLookup:
"""In-flight state for one inbound LookupMsg.
@@ -159,6 +171,8 @@ class _ActiveLookup:
lookup_id: int
kv_request_id: str
ctx: ReqContext
# Wire round these probes belong to; pins park under it.
round_seq: int = 0
# Keys from the inbound LookupMsg, preserved in wire order so
# the aggregated response goes back in the same order.
keys: list[OffloadKey] = field(default_factory=list)
@@ -185,6 +199,7 @@ class _PendingLookup(NamedTuple):
keys: list[OffloadKey]
enqueued_at: float
round_seq: int = 0
@dataclass
@@ -198,17 +213,15 @@ class _ServerRequestState:
is idle — see ``ServerRole._maybe_prune``.
"""
# Outbound serve state (PD + symmetric producer side). None until the
# first add_stored_blocks / on_fetch; reset to None on finalize/abort.
outbound: _OutboundRequestState | None = None
# Fetch rounds keyed by wire round_seq. A round is created by its
# first supply or its fetch and removed at its terminal (finalize /
# failure / abort).
outbound: dict[int, _OutboundRequestState] = field(default_factory=dict)
# Raw inbound LookupMsgs not yet processed against the ParentManager.
pending_lookups: list[_PendingLookup] = field(default_factory=list)
# Per-LookupMsg state parked with HIT_PENDING / RETRY keys, keyed by
# the (globally unique) lookup_id and re-polled each serve.
lookups: dict[int, _ActiveLookup] = field(default_factory=dict)
# Start time of a pending abort drain (``time.monotonic``); None when
# no abort is in progress.
abort_started_at: float | None = None
# Transfer ids in ``ServerRole._inflight`` for this id. Kept in sync
# via _inflight_add / _inflight_pop so a non-empty set is an exact
# "has any inflight transfer" predicate and the abort drain can
@@ -256,9 +269,9 @@ class ServerRole:
# ``parent.on_request_finished`` in ``serve_external_requests``.
self._finished_lookup_ctxs: list[ReqContext] = []
self._lookup_id_counter: int = 0
# kv_request_ids with a parked abort awaiting drain — work-list so
# drain_pending_aborts doesn't scan every request each poll tick.
self._parked_aborts: set[str] = set()
# Parked aborts awaiting drain, keyed by (kv_request_id, round)
# with the abort start time.
self._pending_aborts: dict[tuple[str, int], float] = {}
# ------------------------------------------------------------------
# State helpers
@@ -277,11 +290,11 @@ class ServerRole:
st = self._requests.get(kv_request_id)
if (
st is not None
and st.outbound is None
and not st.outbound
and not st.inflight_tids
and not st.lookups
and not st.pending_lookups
and st.abort_started_at is None
and not any(kv == kv_request_id for kv, _ in self._pending_aborts)
):
del self._requests[kv_request_id]
@@ -295,87 +308,100 @@ class ServerRole:
keys: Sequence[OffloadKey],
block_ids: Sequence[int],
job_id: JobId,
round_seq: int = 0,
*,
from_lookup: bool = False,
) -> None:
"""New blocks stored locally — match against pending fetch demand."""
"""New blocks stored locally — match within their fetch round.
Lookup pins carry the round they were probed under; PD
submit_store batches share PD's single round 0.
"""
self._store_jobs[job_id] = time.monotonic()
st = self._get_or_create_request(kv_request_id)
if st.outbound is None:
st.outbound = _OutboundRequestState()
result = st.outbound.add_stored_blocks(keys, block_ids, job_id)
if result.local_idxs and st.outbound.demand_received:
self._submit_transfer(kv_request_id, result)
rnd = st.outbound.get(round_seq)
if rnd is None:
rnd = st.outbound[round_seq] = _OutboundRequestState()
if from_lookup:
rnd.lookup_supplied = True
result = rnd.add_stored_blocks(keys, block_ids, job_id)
if result.local_idxs and rnd.demand_received:
self._submit_transfer(kv_request_id, result, rnd, round_seq)
def on_fetch(
self,
kv_request_id: str,
keys: Sequence[OffloadKey],
block_indexes: Sequence[int],
round_seq: int = 0,
) -> None:
"""Handle a FetchMsg from the peer.
In p2p mode FetchMsg is the server-side "request finished"
signal for ``kv_request_id``. Three consequences flow from that:
- No further ``parent.create_store_job`` will fire for this id:
the request's parked ``lookups`` are popped here (via
``_finish_inbound_lookups``) before the next
``serve_external_requests`` runs ``_resolve_pending_lookups``,
so any HIT_PENDING / RETRY key that would otherwise later
promote to HIT and pin a slot is dropped instead. Any raw
not-yet-processed LookupMsg for this id is dropped too.
- All server-side lookup state for the id is cleaned up (the
``lookups`` entries themselves).
- The synthetic ctx is queued for ``parent.on_request_finished``
(fired by the next ``serve_external_requests``) so the
TieringManager can release per-lookup bookkeeping.
The client contract guarantees exactly one FetchMsg per
lookup-touched request — including an empty one when no
blocks end up being fetched.
Raises ``ValueError`` on a duplicate fetch for the same
``kv_request_id``; the coordinator's dispatch loop turns that
into a protocol-error disconnect.
A non-empty fetch binds and closes its round, leaving lookup
state alone (the next round's LookupMsg may already be in
flight). The terminal empty fetch closes the id: parked lookups
are popped and every remaining round drained. A second fetch for
a round already holding demand raises ValueError
(protocol-error disconnect).
"""
logger.debug(
"P2PSession %s: fetch RECEIVED kv_request_id=%s blocks=%d",
"P2PSession %s: fetch RECEIVED kv_request_id=%s round=%s blocks=%d",
self._peer_id,
kv_request_id,
round_seq,
len(keys),
)
st = self._requests.get(kv_request_id)
existing = st.outbound if st is not None else None
existing = st.outbound.get(round_seq) if st is not None else None
if existing is not None and existing.demand_received:
# A second fetch for the same kv_request_id would overwrite
# `remaining` and leak inflight bookkeeping. Treat as a
# protocol violation.
raise ValueError(f"duplicate fetch for kv_request_id={kv_request_id}")
raise ValueError(
f"duplicate fetch for kv_request_id={kv_request_id} round={round_seq}"
)
st = self._get_or_create_request(kv_request_id)
if st.outbound is None:
st.outbound = _OutboundRequestState()
req = st.outbound
req = st.outbound.get(round_seq)
if req is None:
req = st.outbound[round_seq] = _OutboundRequestState()
result = req.add_fetch_demand(keys, block_indexes)
if not keys:
# Terminal empty fetch: close the lookup phase and drain
# every round with no TransferDoneMsg (nothing waits on it).
self._finish_inbound_lookups(kv_request_id)
for key in list(st.outbound):
self._finalize_outbound(kv_request_id, key, send_done=False)
return
if req.lookup_supplied and req.demanded:
# A symmetric round's supply always precedes its fetch, so
# unmatched demand is unservable — fail now, not at the load
# timeout. PD rounds keep parking demand for stores that
# arrive later.
logger.warning(
"P2PSession %s: fetch kv_request_id=%s round=%s demanded %d "
"blocks but %d have no pinned supply; failing fetch "
"immediately",
self._peer_id,
kv_request_id,
round_seq,
len(keys),
len(req.demanded),
)
self._finalize_outbound(kv_request_id, round_seq, success=False)
return
if result.local_idxs:
self._submit_transfer(kv_request_id, result)
# Close the peer's request as far as the server's lookup phase
# is concerned: pop parked lookups so no further
# ``parent.create_store_job`` fires for this id, and queue their
# ctxs for ``parent.on_request_finished``. Done before the
# finalize path below so bookkeeping releases in-order.
self._finish_inbound_lookups(kv_request_id)
self._submit_transfer(kv_request_id, result, req, round_seq)
# Prefiller-first mode: finish_request may have run before
# fetch arrived. If so, finalize once we know what was
# demanded — fully satisfied → success, else early-fail.
if req.finishing and not self._has_inflight_for(kv_request_id):
self._finalize_outbound(kv_request_id)
if req.finishing and req.inflight == 0:
self._finalize_outbound(kv_request_id, round_seq)
def on_abort_fetch(self, kv_request_id: str) -> None:
"""Handle an AbortFetchMsg from the peer."""
def on_abort_fetch(self, kv_request_id: str, round_seq: int = 0) -> None:
"""Handle an AbortFetchMsg from the peer, cancelling one round."""
# Abort for an unknown id may be a benign race/duplicate or a
# real protocol violation; we don't track completed ids, so warn.
st = self._requests.get(kv_request_id)
has_outbound = st is not None and st.outbound is not None
if not has_outbound and not self._has_inflight_for(kv_request_id):
if (st is None or not st.outbound) and not self._has_inflight_for(
kv_request_id
):
logger.warning(
"P2PSession %s: abort_fetch for unknown kv_request_id=%s "
"(no outbound or inflight state); benign race or stale",
@@ -385,16 +411,15 @@ class ServerRole:
# Idempotent: receiving AbortFetchMsg again before we've sent the
# ack just triggers another drain attempt without resetting the
# deadline.
st = self._get_or_create_request(kv_request_id)
if st.abort_started_at is None:
st.abort_started_at = time.monotonic()
self._parked_aborts.add(kv_request_id)
self._drain_abort(kv_request_id)
self._get_or_create_request(kv_request_id)
self._pending_aborts.setdefault((kv_request_id, round_seq), time.monotonic())
self._drain_abort(kv_request_id, round_seq)
def on_lookup(
self,
kv_request_id: str,
keys: Sequence[OffloadKey],
round_seq: int = 0,
) -> None:
"""Enqueue a LookupMsg from a symmetric-P2P consumer.
@@ -406,13 +431,18 @@ class ServerRole:
parent calls are valid.
"""
logger.debug(
"P2P LOOKUP server %s: RECV LookupMsg kv_request_id=%s keys=%d",
"P2P LOOKUP server %s: RECV LookupMsg kv_request_id=%s round=%s keys=%d",
self._peer_id,
kv_request_id,
round_seq,
len(keys),
)
self._get_or_create_request(kv_request_id).pending_lookups.append(
_PendingLookup(keys=list(keys), enqueued_at=time.monotonic())
_PendingLookup(
keys=list(keys),
enqueued_at=time.monotonic(),
round_seq=round_seq,
)
)
self._serve_pending.add(kv_request_id)
@@ -434,7 +464,7 @@ class ServerRole:
st.pending_lookups = []
for pl in pending:
self._process_inbound_lookup(
kv_request_id, pl.keys, pl.enqueued_at, parent
kv_request_id, pl.keys, pl.enqueued_at, pl.round_seq, parent
)
self._resolve_pending_lookups(kv_request_id, parent)
st = self._requests.get(kv_request_id)
@@ -481,9 +511,7 @@ class ServerRole:
else:
lookup.pending.add(h)
if new_hits:
self._pin_and_register_hits(
lookup.kv_request_id, new_hits, lookup.ctx, parent
)
self._pin_and_register_hits(lookup, new_hits, parent)
return new_hits
def _process_inbound_lookup(
@@ -491,6 +519,7 @@ class ServerRole:
kv_request_id: str,
keys: list[OffloadKey],
enqueued_at: float,
round_seq: int,
parent: ParentManager,
) -> None:
"""Resolve one enqueued LookupMsg against ``parent``.
@@ -515,6 +544,7 @@ class ServerRole:
lookup_id=lookup_id,
kv_request_id=kv_request_id,
ctx=ctx,
round_seq=round_seq,
keys=list(keys),
deadline=enqueued_at + _LOOKUP_PENDING_TIMEOUT_S,
)
@@ -547,25 +577,26 @@ class ServerRole:
def _pin_and_register_hits(
self,
kv_request_id: str,
lookup: _ActiveLookup,
keys: list[OffloadKey],
ctx: ReqContext,
parent: ParentManager,
) -> None:
"""Pin primary slots for HIT keys and feed them into the
existing ``add_stored_blocks`` matching path.
"""Pin primary slots for HIT keys and park them as the lookup's
round supply via ``add_stored_blocks``.
Caller has already confirmed every key is HIT (single-threaded
scheduler ⇒ no eviction race), so the JobMetadata returned by
``parent.create_store_job`` carries parallel ``keys``/``block_ids``
of length ``len(keys)``.
"""
meta = parent.create_store_job(keys, ctx)
meta = parent.create_store_job(keys, lookup.ctx)
self.add_stored_blocks(
kv_request_id,
lookup.kv_request_id,
list(meta.keys),
list(meta.block_ids),
meta.job_id,
round_seq=lookup.round_seq,
from_lookup=True,
)
def _resolve_pending_lookups(
@@ -650,9 +681,8 @@ class ServerRole:
Called on the two events that mean "no more lookup traffic for
``kv_request_id`` is expected on this session": the terminal
FetchMsg from the peer (client contract: exactly one FetchMsg
per lookup-touched request, even if empty), and a local
``finish``. Whichever fires second is a no-op.
empty FetchMsg from the peer and a local ``finish``. Whichever
fires second is a no-op.
"""
st = self._requests.get(kv_request_id)
if st is None:
@@ -688,20 +718,15 @@ class ServerRole:
self._finish_inbound_lookups(kv_request_id)
st = self._requests.get(kv_request_id)
req = st.outbound if st is not None else None
if req is None:
if st is None:
return
req.finishing = True
if not req.demand_received:
return
if self._has_inflight_for(kv_request_id):
return
# Remaining > 0 here: if it had hit 0, the poll-done success
# branch would have already cleared outbound and we'd have
# returned at `req is None` above. Helper derives success from
# remaining and emits StoreResult(success=False) for any
# leftover pending jobs.
self._finalize_outbound(kv_request_id)
for key, req in list(st.outbound.items()):
req.finishing = True
if not req.demand_received or req.inflight:
# No demand yet (prefiller-first): on_fetch finalizes via
# `finishing`. Inflight: the last completion finalizes.
continue
self._finalize_outbound(kv_request_id, key)
def collect_results(self) -> list[StoreResult]:
"""Drain timeouts, deferred results, and transport completions.
@@ -741,20 +766,24 @@ class ServerRole:
)
continue
results.extend(self._settle_xfer_jobs(xfer, success=True))
rnd = xfer.round
st = self._requests.get(xfer.kv_request_id)
req = st.outbound if st is not None else None
if req is not None and req.demand_received:
req.remaining -= xfer.block_count
assert req.remaining >= 0, (
if st is not None and st.outbound.get(xfer.round_key) is rnd:
rnd.remaining -= xfer.block_count
assert rnd.remaining >= 0, (
f"remaining went negative for kv_request_id={xfer.kv_request_id}"
)
if req.remaining == 0:
self._finalize_outbound(xfer.kv_request_id, success=True)
elif req.finishing and not self._has_inflight_for(xfer.kv_request_id):
self._finalize_outbound(xfer.kv_request_id, success=False)
if rnd.remaining == 0:
self._finalize_outbound(
xfer.kv_request_id, xfer.round_key, success=True
)
elif rnd.finishing and rnd.inflight == 0:
self._finalize_outbound(
xfer.kv_request_id, xfer.round_key, success=False
)
self._maybe_prune(xfer.kv_request_id)
failed_kv_request_ids: set[str] | None = None
failed_rounds: list[tuple[str, _OutboundRequestState]] | None = None
for tid in poll_result.failed:
xfer = self._inflight_pop(tid)
if xfer is None:
@@ -767,35 +796,36 @@ class ServerRole:
tid,
)
continue
if failed_kv_request_ids is None:
failed_kv_request_ids = set()
failed_kv_request_ids.add(xfer.kv_request_id)
results.extend(self._settle_xfer_jobs(xfer, success=False))
rnd = xfer.round
st = self._requests.get(xfer.kv_request_id)
req = st.outbound if st is not None else None
if st is not None:
st.outbound = None
if req is not None and req.demand_received:
if st is not None and st.outbound.get(xfer.round_key) is rnd:
del st.outbound[xfer.round_key]
if failed_rounds is None:
failed_rounds = []
failed_rounds.append((xfer.kv_request_id, rnd))
self._send(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.KV_REQUEST_ID: xfer.kv_request_id,
TransferDoneMsg.SUCCESS: False,
TransferDoneMsg.ROUND_SEQ: xfer.round_key,
}
)
self._maybe_prune(xfer.kv_request_id)
# Cancel other inflight for the same failed kv_request_ids
if failed_kv_request_ids:
ids_to_cancel = [
tid
for tid, xfer in self._inflight.items()
if xfer.kv_request_id in failed_kv_request_ids
]
for tid in ids_to_cancel:
self._inflight_pop(tid)
self._transport.cancel(ids_to_cancel)
for kv_request_id in failed_kv_request_ids:
# Cancel each failed round's other inflight and fail its
# remaining store jobs — nothing else will settle them.
if failed_rounds:
for kv_request_id, rnd in failed_rounds:
ids_to_cancel = [
tid for tid, x in self._inflight.items() if x.round is rnd
]
for tid in ids_to_cancel:
self._inflight_pop(tid)
if ids_to_cancel:
self._transport.cancel(ids_to_cancel)
results.extend(self._fail_round_jobs(rnd))
self._maybe_prune(kv_request_id)
return results
@@ -811,8 +841,8 @@ class ServerRole:
def drain_pending_aborts(self) -> None:
"""Re-attempt every parked abort once per poll tick."""
for kv_request_id in list(self._parked_aborts):
self._drain_abort(kv_request_id)
for kv_request_id, round_seq in list(self._pending_aborts):
self._drain_abort(kv_request_id, round_seq)
def close(self) -> tuple[list[int], list[ReqContext]]:
"""Tear down. Cancels inflight.
@@ -839,7 +869,7 @@ class ServerRole:
failed_serves.extend(self._finished_lookup_ctxs)
self._requests.clear()
self._serve_pending.clear()
self._parked_aborts.clear()
self._pending_aborts.clear()
self._finished_lookup_ctxs.clear()
return failed_stores, failed_serves
@@ -870,6 +900,8 @@ class ServerRole:
xfer = self._inflight.pop(tid, None)
if xfer is None:
return None
xfer.round.inflight -= 1
assert xfer.round.inflight >= 0
st = self._requests.get(xfer.kv_request_id)
if st is not None:
st.inflight_tids.discard(tid)
@@ -880,80 +912,108 @@ class ServerRole:
) -> list[StoreResult]:
"""Emit StoreResults for a completed transfer's store jobs.
Pops each attached job from ``_store_jobs`` and clears it from the
request's pending set. A job already popped (via timeout, cancel,
etc.) is skipped so we never double-emit a contradictory result.
Pops each attached job from ``_store_jobs`` and clears it from
its round's pending set. A job already popped (via timeout,
cancel, etc.) is skipped so we never double-emit a contradictory
result.
"""
results: list[StoreResult] = []
st = self._requests.get(xfer.kv_request_id)
req = st.outbound if st is not None else None
for job_id in xfer.job_ids:
if self._store_jobs.pop(job_id, None) is None:
continue
results.append(StoreResult(job_id=job_id, success=success))
if req is not None:
req.pending_job_ids.discard(job_id)
xfer.round.pending_job_ids.discard(job_id)
return results
# ------------------------------------------------------------------
# Internal — finalize / abort drain
# ------------------------------------------------------------------
def _fail_round_jobs(self, rnd: _OutboundRequestState) -> list[StoreResult]:
"""Fail a terminated round's still-pending store jobs (idempotent)."""
results: list[StoreResult] = []
for job_id in rnd.pending_job_ids:
if self._store_jobs.pop(job_id, None) is None:
continue
results.append(StoreResult(job_id=job_id, success=False))
rnd.pending_job_ids.clear()
return results
def _finalize_outbound(
self,
kv_request_id: str,
round_key: int,
success: bool | None = None,
send_done: bool = True,
) -> None:
"""Pop the outbound state and emit terminal results.
Called when no further work will happen for this kv_request_id
on the server side: either request_finish has fired and there
are no inflight transfers, or the last inflight just completed
while finishing.
"""Pop one round and emit its terminal results.
If ``success`` is None, derive it from ``req.remaining == 0``.
The same flag is used for both the peer's TransferDoneMsg and
the StoreResult(s) emitted for any leftover pending job_ids.
``send_done=False`` skips the TransferDoneMsg (terminal empty
fetch). Other rounds of the id are untouched.
"""
st = self._requests[kv_request_id]
assert st.outbound is not None
req = st.outbound
st.outbound = None
req = st.outbound.pop(round_key)
if success is None:
success = req.remaining == 0
for job_id in req.pending_job_ids:
self._store_jobs.pop(job_id, None)
self._pending_store_results.append(
StoreResult(job_id=job_id, success=success)
)
self._send(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.KV_REQUEST_ID: kv_request_id,
TransferDoneMsg.SUCCESS: success,
}
success = req.demand_received and req.remaining == 0
settled = self._fail_round_jobs(req) if not success else None
if settled is not None:
self._pending_store_results.extend(settled)
else:
for job_id in req.pending_job_ids:
if self._store_jobs.pop(job_id, None) is None:
continue
self._pending_store_results.append(
StoreResult(job_id=job_id, success=True)
)
req.pending_job_ids.clear()
logger.debug(
"P2PSession %s: finalize kv_request_id=%s round=%s success=%s "
"remaining=%d leftover_available=%d send_done=%s",
self._peer_id,
kv_request_id,
round_key,
success,
req.remaining,
len(req.available),
send_done,
)
if send_done and req.demand_received:
self._send(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.KV_REQUEST_ID: kv_request_id,
TransferDoneMsg.SUCCESS: success,
TransferDoneMsg.ROUND_SEQ: round_key,
}
)
self._maybe_prune(kv_request_id)
def _drain_abort(self, kv_request_id: str) -> None:
def _drain_abort(self, kv_request_id: str, round_seq: int) -> None:
"""One drain attempt for a pending abort.
Stops accepting more blocks for ``kv_request_id``, then asks the
transport to cancel any matching inflight transfers in
``mode="wait"``. Sends ``AbortAckMsg`` once nothing remains
inflight, or after ``_CANCEL_DRAIN_TIMEOUT_S`` falls back to
``mode="immediate"`` and acks anyway.
Detaches the aborted round, then asks the transport to cancel its
inflight transfers in ``mode="wait"``. Sends ``AbortAckMsg`` once
nothing remains inflight, or after ``_CANCEL_DRAIN_TIMEOUT_S``
falls back to ``mode="immediate"`` and acks anyway.
"""
st = self._requests[kv_request_id]
st.outbound = None
ids = list(st.inflight_tids)
rnd = st.outbound.pop(round_seq, None)
if rnd is not None:
# Its transfers are being cancelled; fail its jobs now
# instead of leaking them to the store timeout.
self._pending_store_results.extend(self._fail_round_jobs(rnd))
ids = [
tid
for tid, x in self._inflight.items()
if x.kv_request_id == kv_request_id and x.round_key == round_seq
]
if not ids:
self._finalize_abort(kv_request_id)
self._finalize_abort(kv_request_id, round_seq)
return
assert st.abort_started_at is not None
expired = time.monotonic() - st.abort_started_at >= _CANCEL_DRAIN_TIMEOUT_S
if expired:
started_at = self._pending_aborts[(kv_request_id, round_seq)]
if time.monotonic() - started_at >= _CANCEL_DRAIN_TIMEOUT_S:
for tid in ids:
self._inflight_pop(tid)
self._transport.cancel(ids, mode="immediate")
@@ -964,7 +1024,7 @@ class ServerRole:
kv_request_id,
len(ids),
)
self._finalize_abort(kv_request_id)
self._finalize_abort(kv_request_id, round_seq)
return
still = self._transport.cancel(ids, mode="wait")
@@ -977,16 +1037,15 @@ class ServerRole:
if tid not in still_set:
self._inflight_pop(tid)
if not still:
self._finalize_abort(kv_request_id)
self._finalize_abort(kv_request_id, round_seq)
def _finalize_abort(self, kv_request_id: str) -> None:
st = self._requests[kv_request_id]
st.abort_started_at = None
self._parked_aborts.discard(kv_request_id)
def _finalize_abort(self, kv_request_id: str, round_seq: int) -> None:
self._pending_aborts.pop((kv_request_id, round_seq), None)
self._send(
{
TYPE_KEY: AbortAckMsg.TYPE,
AbortAckMsg.KV_REQUEST_ID: kv_request_id,
AbortAckMsg.ROUND_SEQ: round_seq,
}
)
self._maybe_prune(kv_request_id)
@@ -995,7 +1054,13 @@ class ServerRole:
# Internal — transfers and store-job timeouts
# ------------------------------------------------------------------
def _submit_transfer(self, kv_request_id: str, result: _MatchResult) -> None:
def _submit_transfer(
self,
kv_request_id: str,
result: _MatchResult,
rnd: _OutboundRequestState,
round_key: int,
) -> None:
logger.debug(
"P2PSession %s: NIXL write_blocks CALL kv_request_id=%s "
"local_idxs=%d remote_idxs=%d",
@@ -1016,12 +1081,15 @@ class ServerRole:
transfer_id,
len(result.local_idxs),
)
rnd.inflight += 1
self._inflight_add(
transfer_id,
_InflightXfer(
kv_request_id=kv_request_id,
block_count=len(result.local_idxs),
job_ids=result.job_ids,
round=rnd,
round_key=round_key,
),
)
else:
@@ -1031,22 +1099,24 @@ class ServerRole:
kv_request_id,
len(result.local_idxs),
)
# The matched blocks were popped from req.demanded /
# req.available, but no inflight will satisfy them, so
# remaining will never reach 0 on its own. Mark the
# request as finishing so the existing terminal paths
# clean up: if other inflight is in flight, the last one
# to drain will fire _finalize_outbound(success=False)
# via the elif branch in collect_results. If
# nothing else is in flight, finalize now so the peer
# and the local store jobs don't wait for finish_request
# or for _STORE_TIMEOUT_S / _LOAD_TIMEOUT_S.
# The matched blocks were popped from rnd.demanded /
# rnd.available, but no inflight will satisfy them, so
# remaining will never reach 0 on its own. Mark the round
# as finishing so the existing terminal paths clean up: if
# other transfers of this round are in flight, the last one
# to drain will fire _finalize_outbound(success=False) via
# the elif branch in collect_results. If nothing else is in
# flight, finalize now so the peer and the local store jobs
# don't wait for finish_request or for _STORE_TIMEOUT_S /
# _LOAD_TIMEOUT_S.
rnd.finishing = True
st = self._requests.get(kv_request_id)
req = st.outbound if st is not None else None
if req is not None:
req.finishing = True
if not self._has_inflight_for(kv_request_id):
self._finalize_outbound(kv_request_id, success=False)
if (
st is not None
and st.outbound.get(round_key) is rnd
and rnd.inflight == 0
):
self._finalize_outbound(kv_request_id, round_key, success=False)
def _timeout_pending_store_jobs(self) -> list[StoreResult]:
if not self._store_jobs:
@@ -374,26 +374,34 @@ class P2PSession:
for bh in msg[FetchMsg.KEYS]
]
block_indexes = msg[FetchMsg.BLOCK_INDEXES]
round_seq = msg[FetchMsg.ROUND_SEQ]
# Run the server-role state machine inline as today —
# add_fetch_demand records demand against any blocks we've
# already seen in `available`. Report the kv_request_id so
# the manager (after poll() returns) can replay any parked
# submit_store batches; their add_stored_blocks calls hit
# the demand recorded here and submit transfers immediately.
self._server.on_fetch(kv_request_id, keys, block_indexes)
self._server.on_fetch(kv_request_id, keys, block_indexes, round_seq)
self._new_fetch_ids.append(kv_request_id)
elif msg_type == AbortFetchMsg.TYPE:
AbortFetchMsg.validate(msg)
self._server.on_abort_fetch(msg[AbortFetchMsg.KV_REQUEST_ID])
self._server.on_abort_fetch(
msg[AbortFetchMsg.KV_REQUEST_ID],
msg[AbortFetchMsg.ROUND_SEQ],
)
elif msg_type == TransferDoneMsg.TYPE:
TransferDoneMsg.validate(msg)
self._client.on_transfer_done(
msg[TransferDoneMsg.KV_REQUEST_ID],
msg[TransferDoneMsg.SUCCESS],
msg[TransferDoneMsg.ROUND_SEQ],
)
elif msg_type == AbortAckMsg.TYPE:
AbortAckMsg.validate(msg)
self._client.on_abort_ack(msg[AbortAckMsg.KV_REQUEST_ID])
self._client.on_abort_ack(
msg[AbortAckMsg.KV_REQUEST_ID],
msg[AbortAckMsg.ROUND_SEQ],
)
elif msg_type == LookupMsg.TYPE:
LookupMsg.validate(msg)
kv_request_id = msg[LookupMsg.KV_REQUEST_ID]
@@ -401,7 +409,7 @@ class P2PSession:
OffloadKey(bh if isinstance(bh, bytes) else bytes(bh))
for bh in msg[LookupMsg.KEYS]
]
self._server.on_lookup(kv_request_id, keys)
self._server.on_lookup(kv_request_id, keys, msg[LookupMsg.ROUND_SEQ])
elif msg_type == LookupRespMsg.TYPE:
LookupRespMsg.validate(msg)
kv_request_id = msg[LookupRespMsg.KV_REQUEST_ID]