From 30217b0e809ef045b3b4aa7c011d6c6d2b0b0362 Mon Sep 17 00:00:00 2001 From: Itay Etelis <92247226+Etelis@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:01:55 +0300 Subject: [PATCH] [Bugfix][KV Offload][P2P] Scope serve state to fetch rounds (#49877) Signed-off-by: Itay Etelis Co-authored-by: Itay Etelis Co-authored-by: Itay Etelis --- .../v1/kv_offload/tiering/p2p/test_manager.py | 13 +- .../kv_offload/tiering/p2p/test_sessions.py | 116 ++++- .../kv_offload/tiering/p2p/session/client.py | 293 +++++------ .../tiering/p2p/session/protocol.py | 48 +- .../kv_offload/tiering/p2p/session/server.py | 474 ++++++++++-------- .../kv_offload/tiering/p2p/session/session.py | 16 +- 6 files changed, 570 insertions(+), 390 deletions(-) diff --git a/tests/v1/kv_offload/tiering/p2p/test_manager.py b/tests/v1/kv_offload/tiering/p2p/test_manager.py index cd6e8f382a..c8a9d7f3ea 100644 --- a/tests/v1/kv_offload/tiering/p2p/test_manager.py +++ b/tests/v1/kv_offload/tiering/p2p/test_manager.py @@ -1612,16 +1612,17 @@ class TestBindHostPortDefaults: monkeypatch.setattr( manager_module, "NixlTransport", - lambda agent_name, *a, **k: calls.update(nixl_name=agent_name) - or SimpleNamespace(), + lambda agent_name, *a, **k: ( + calls.update(nixl_name=agent_name) or SimpleNamespace() + ), ) monkeypatch.setattr( manager_module, "ZmqTransport", - lambda local_id, host, port, *a, **k: calls.update( - zmq_id=local_id, zmq_host=host, zmq_port=port - ) - or SimpleNamespace(), + lambda local_id, host, port, *a, **k: ( + calls.update(zmq_id=local_id, zmq_host=host, zmq_port=port) + or SimpleNamespace() + ), ) spec = SimpleNamespace( blocks_per_chunk=1, diff --git a/tests/v1/kv_offload/tiering/p2p/test_sessions.py b/tests/v1/kv_offload/tiering/p2p/test_sessions.py index cab4ce6705..20e4bad218 100644 --- a/tests/v1/kv_offload/tiering/p2p/test_sessions.py +++ b/tests/v1/kv_offload/tiering/p2p/test_sessions.py @@ -49,6 +49,7 @@ from vllm.v1.kv_offload.tiering.p2p.session.protocol import ( from vllm.v1.kv_offload.tiering.p2p.session.server import ( _CANCEL_DRAIN_TIMEOUT_S, _InflightXfer, + _OutboundRequestState, ) from vllm.v1.kv_offload.tiering.p2p.session.session import ( _MAX_CONSECUTIVE_DISPATCH_ERRORS, @@ -315,10 +316,26 @@ def _activate( # either a missing entry or a None field. These helpers paper over that. +def _client_load(session: P2PSession, kv_request_id: str): + """The single in-flight load of a kv_request_id (loads are per-round).""" + loads = session._client._requests[kv_request_id].loads + assert len(loads) == 1 + return next(iter(loads.values())) + + def _srv_outbound(session: P2PSession, kv_request_id: str): - """Outbound serve state for a kv_request_id, or None (idle / GC'd).""" + """Serve-side round for a kv_request_id, or None (idle / GC'd). + + Rounds are keyed by wire round_seq; surfaces the demanded round when + a fetch has bound one, else any parked supply round. + """ st = session._server._requests.get(kv_request_id) - return st.outbound if st is not None else None + if st is None or not st.outbound: + return None + for rnd in st.outbound.values(): + if rnd.demand_received: + return rnd + return next(iter(st.outbound.values())) def _srv_lookups(session: P2PSession) -> list: @@ -330,8 +347,10 @@ def _srv_lookups(session: P2PSession) -> list: def _srv_abort_started(session: P2PSession, kv_request_id: str) -> float | None: """Pending-abort start time for a kv_request_id, or None.""" - st = session._server._requests.get(kv_request_id) - return st.abort_started_at if st is not None else None + for (kv, _), started in session._server._pending_aborts.items(): + if kv == kv_request_id: + return started + return None def _srv_inflight_count(session: P2PSession, kv_request_id: str) -> int: @@ -479,6 +498,7 @@ class TestClientFlows: conn.enqueue( { TYPE_KEY: TransferDoneMsg.TYPE, + TransferDoneMsg.ROUND_SEQ: 0, TransferDoneMsg.KV_REQUEST_ID: "req-1", TransferDoneMsg.SUCCESS: True, } @@ -495,6 +515,7 @@ class TestClientFlows: conn.enqueue( { TYPE_KEY: TransferDoneMsg.TYPE, + TransferDoneMsg.ROUND_SEQ: 0, TransferDoneMsg.KV_REQUEST_ID: "req-1", TransferDoneMsg.SUCCESS: False, } @@ -538,6 +559,7 @@ class TestClientFlows: conn.enqueue( { TYPE_KEY: TransferDoneMsg.TYPE, + TransferDoneMsg.ROUND_SEQ: 0, TransferDoneMsg.KV_REQUEST_ID: "req-1", TransferDoneMsg.SUCCESS: True, } @@ -552,7 +574,7 @@ class TestClientFlows: session.request_blocks( job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0] ) - session._client._requests["req-1"].load.submitted_at = time.monotonic() - 60.0 + _client_load(session, "req-1").submitted_at = time.monotonic() - 60.0 session.poll() abort = conn._sent[-1] assert abort[TYPE_KEY] == AbortFetchMsg.TYPE @@ -569,7 +591,7 @@ class TestClientFlows: job_id=7, kv_request_id="req-7", keys=[b"k"], block_ids=[0] ) # 1) Trip the load timeout to send AbortFetch and stamp aborted_at. - session._client._requests["req-7"].load.submitted_at = ( + _client_load(session, "req-7").submitted_at = ( time.monotonic() - _LOAD_TIMEOUT_S - 1.0 ) loads = session.poll().loads @@ -579,11 +601,11 @@ class TestClientFlows: and m[AbortFetchMsg.KV_REQUEST_ID] == "req-7" for m in conn._sent ) - assert session._client._requests["req-7"].load.aborted_at is not None + assert _client_load(session, "req-7").aborted_at is not None # 2) Now backdate aborted_at past the abort-ack timeout. No ack ever # arrived from the peer. - session._client._requests["req-7"].load.aborted_at = ( + _client_load(session, "req-7").aborted_at = ( time.monotonic() - _ABORT_ACK_TIMEOUT_S - 1.0 ) loads = session.poll().loads @@ -599,17 +621,18 @@ class TestClientFlows: session.request_blocks( job_id=8, kv_request_id="req-8", keys=[b"k"], block_ids=[0] ) - session._client._requests["req-8"].load.submitted_at = ( + _client_load(session, "req-8").submitted_at = ( time.monotonic() - _LOAD_TIMEOUT_S - 1.0 ) # First poll: AbortFetch goes out. session.poll() - assert session._client._requests["req-8"].load.aborted_at is not None + assert _client_load(session, "req-8").aborted_at is not None # Peer acks the abort. conn.enqueue( { TYPE_KEY: AbortAckMsg.TYPE, + AbortAckMsg.ROUND_SEQ: 0, AbortAckMsg.KV_REQUEST_ID: "req-8", } ) @@ -915,6 +938,7 @@ class TestLookupFlow: conn.enqueue( { TYPE_KEY: LookupMsg.TYPE, + LookupMsg.ROUND_SEQ: 0, LookupMsg.KV_REQUEST_ID: "req-1", LookupMsg.KEYS: [b"hX", b"hY", b"hZ"], } @@ -951,6 +975,7 @@ def _send_lookup(conn: FakeConnection, kv_request_id: str, keys: list[bytes]): conn.enqueue( { TYPE_KEY: LookupMsg.TYPE, + LookupMsg.ROUND_SEQ: 0, LookupMsg.KV_REQUEST_ID: kv_request_id, LookupMsg.KEYS: list(keys), } @@ -1201,6 +1226,7 @@ class TestServerLookupHandling: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [], FetchMsg.BLOCK_INDEXES: [], @@ -1240,6 +1266,7 @@ class TestServerLookupHandling: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"hA", b"hB"], FetchMsg.BLOCK_INDEXES: [20, 21], @@ -1277,6 +1304,7 @@ class TestServerFlows: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1", b"k2"], FetchMsg.BLOCK_INDEXES: [10, 11], @@ -1295,6 +1323,7 @@ class TestServerFlows: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1"], FetchMsg.BLOCK_INDEXES: [5], @@ -1313,6 +1342,7 @@ class TestServerFlows: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1"], FetchMsg.BLOCK_INDEXES: [5], @@ -1331,6 +1361,7 @@ class TestServerFlows: conn.enqueue( { TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.ROUND_SEQ: 0, AbortFetchMsg.KV_REQUEST_ID: "req-1", } ) @@ -1349,13 +1380,19 @@ class TestServerFlows: tid = 42 session._server._inflight_add( tid, - _InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}), + _InflightXfer( + kv_request_id="req-1", + block_count=1, + job_ids={1}, + round=_OutboundRequestState(inflight=1), + ), ) transport._cancel_still_inflight.add(tid) conn.enqueue( { TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.ROUND_SEQ: 0, AbortFetchMsg.KV_REQUEST_ID: "req-1", } ) @@ -1376,13 +1413,19 @@ class TestServerFlows: tid = 42 session._server._inflight_add( tid, - _InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}), + _InflightXfer( + kv_request_id="req-1", + block_count=1, + job_ids={1}, + round=_OutboundRequestState(inflight=1), + ), ) transport._cancel_still_inflight.add(tid) conn.enqueue( { TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.ROUND_SEQ: 0, AbortFetchMsg.KV_REQUEST_ID: "req-1", } ) @@ -1409,20 +1452,26 @@ class TestServerFlows: tid = 42 session._server._inflight_add( tid, - _InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}), + _InflightXfer( + kv_request_id="req-1", + block_count=1, + job_ids={1}, + round=_OutboundRequestState(inflight=1), + ), ) transport._cancel_still_inflight.add(tid) conn.enqueue( { TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.ROUND_SEQ: 0, AbortFetchMsg.KV_REQUEST_ID: "req-1", } ) session.poll() assert _srv_abort_started(session, "req-1") is not None # Backdate past the drain deadline. - session._server._requests["req-1"].abort_started_at = ( + session._server._pending_aborts[("req-1", 0)] = ( time.monotonic() - _CANCEL_DRAIN_TIMEOUT_S - 1.0 ) # Even if the transport still claims it can't cancel, the @@ -1445,13 +1494,19 @@ class TestServerFlows: tid = 42 session._server._inflight_add( tid, - _InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}), + _InflightXfer( + kv_request_id="req-1", + block_count=1, + job_ids={1}, + round=_OutboundRequestState(inflight=1), + ), ) transport._cancel_still_inflight.add(tid) conn.enqueue( { TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.ROUND_SEQ: 0, AbortFetchMsg.KV_REQUEST_ID: "req-1", } ) @@ -1463,6 +1518,7 @@ class TestServerFlows: conn.enqueue( { TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.ROUND_SEQ: 0, AbortFetchMsg.KV_REQUEST_ID: "req-1", } ) @@ -1497,6 +1553,7 @@ class TestServerFlows: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1"], FetchMsg.BLOCK_INDEXES: [5], @@ -1529,6 +1586,7 @@ class TestServerFlows: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1"], FetchMsg.BLOCK_INDEXES: [5], @@ -1571,6 +1629,7 @@ class TestFinishRequestServerSide: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1"], FetchMsg.BLOCK_INDEXES: [5], @@ -1596,6 +1655,7 @@ class TestFinishRequestServerSide: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1", b"k2"], FetchMsg.BLOCK_INDEXES: [10, 11], @@ -1631,6 +1691,7 @@ class TestFinishRequestServerSide: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1"], FetchMsg.BLOCK_INDEXES: [10], @@ -1665,6 +1726,7 @@ class TestFinishRequestServerSide: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1"], FetchMsg.BLOCK_INDEXES: [10], @@ -1686,6 +1748,7 @@ class TestFinishRequestServerSide: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-2", FetchMsg.KEYS: [b"k1", b"k2"], FetchMsg.BLOCK_INDEXES: [10, 11], @@ -1719,6 +1782,7 @@ class TestFinishRequestServerSide: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"demand"], FetchMsg.BLOCK_INDEXES: [5], @@ -1753,6 +1817,7 @@ class TestFinishRequestServerSide: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1"], FetchMsg.BLOCK_INDEXES: [10], @@ -1790,6 +1855,7 @@ class TestFinishRequestServerSide: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1"], FetchMsg.BLOCK_INDEXES: [10], @@ -1824,6 +1890,7 @@ class TestFinishRequestServerSide: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1", b"k2", b"k3"], FetchMsg.BLOCK_INDEXES: [10, 11, 12], @@ -1885,6 +1952,7 @@ class TestFinishRequestServerSide: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1", b"k2"], FetchMsg.BLOCK_INDEXES: [10, 11], @@ -1951,6 +2019,7 @@ class TestBidirectional: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-srv", FetchMsg.KEYS: [b"served"], FetchMsg.BLOCK_INDEXES: [7], @@ -1981,6 +2050,7 @@ class TestBidirectional: conn.enqueue( { TYPE_KEY: TransferDoneMsg.TYPE, + TransferDoneMsg.ROUND_SEQ: 0, TransferDoneMsg.KV_REQUEST_ID: "req-cli", TransferDoneMsg.SUCCESS: True, } @@ -2168,6 +2238,7 @@ class TestAdversarial: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-bad", FetchMsg.KEYS: [b"k1", b"k2"], FetchMsg.BLOCK_INDEXES: [1], @@ -2216,6 +2287,7 @@ class TestDispatchErrorHandling: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-bad", FetchMsg.KEYS: [b"k1", b"k2"], FetchMsg.BLOCK_INDEXES: [1], @@ -2246,6 +2318,7 @@ class TestDispatchErrorHandling: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1"], FetchMsg.BLOCK_INDEXES: [0], @@ -2269,6 +2342,7 @@ class TestDispatchErrorHandling: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1"], FetchMsg.BLOCK_INDEXES: [0], @@ -2296,6 +2370,7 @@ class TestDispatchErrorHandling: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1"], FetchMsg.BLOCK_INDEXES: [0], @@ -2341,6 +2416,7 @@ class TestInflightPerReqInvariant: conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: kv_id, FetchMsg.KEYS: keys, FetchMsg.BLOCK_INDEXES: indexes, @@ -2394,7 +2470,12 @@ class TestInflightPerReqInvariant: tid = kv_id_idx * 10 + j session._server._inflight_add( tid, - _InflightXfer(kv_request_id=kv_id, block_count=1, job_ids={tid}), + _InflightXfer( + kv_request_id=kv_id, + block_count=1, + job_ids={tid}, + round=_OutboundRequestState(inflight=1), + ), ) assert _srv_total_inflight(session) == len(session._server._inflight) assert session._server._has_inflight_for("req-0") @@ -2488,6 +2569,7 @@ class TestFetchMsgValidation: def _valid_msg(self) -> dict: return { TYPE_KEY: FetchMsg.TYPE, + FetchMsg.ROUND_SEQ: 0, FetchMsg.KV_REQUEST_ID: "req-1", FetchMsg.KEYS: [b"k1", b"k2"], FetchMsg.BLOCK_INDEXES: [0, 1], @@ -2513,6 +2595,7 @@ class TestTransferDoneMsgValidation: def test_valid_message_passes(self): msg = { TYPE_KEY: TransferDoneMsg.TYPE, + TransferDoneMsg.ROUND_SEQ: 0, TransferDoneMsg.KV_REQUEST_ID: "req-1", TransferDoneMsg.SUCCESS: True, } @@ -2521,6 +2604,7 @@ class TestTransferDoneMsgValidation: def test_success_wrong_type(self): msg = { TYPE_KEY: TransferDoneMsg.TYPE, + TransferDoneMsg.ROUND_SEQ: 0, TransferDoneMsg.KV_REQUEST_ID: "req-1", TransferDoneMsg.SUCCESS: 1, } diff --git a/vllm/v1/kv_offload/tiering/p2p/session/client.py b/vllm/v1/kv_offload/tiering/p2p/session/client.py index ed91f7f9b0..07a4620780 100644 --- a/vllm/v1/kv_offload/tiering/p2p/session/client.py +++ b/vllm/v1/kv_offload/tiering/p2p/session/client.py @@ -11,7 +11,6 @@ callback injected by the coordinator (which gates on ConnectAck). from __future__ import annotations -import enum import time from collections.abc import Callable, Sequence from dataclasses import dataclass, field @@ -35,26 +34,12 @@ _LOAD_TIMEOUT_S = 30.0 _ABORT_ACK_TIMEOUT_S = 10.0 -class ClientPhase(enum.Enum): - """Lifecycle of a request's client-side lookup/fetch signalling. - - Advances monotonically. Only ``finish`` reads it, to decide - whether a terminal empty FetchMsg is owed to release the peer's - lookup state (owed only from ``PROBING``: a LookupMsg went out but no - FetchMsg has since closed the peer's lookup phase). - """ - - REGISTERED = enum.auto() # keys in probes/unsent, nothing sent yet - PROBING = enum.auto() # LookupMsg flushed, awaiting responses - FETCH_SENT = enum.auto() # FetchMsg sent (real or terminal empty) - - @dataclass class _InboundLoadState: """Client-role state for a single in-flight load request. - Lives on ``_ClientRequestState.load`` for the duration of a fetch; - the owning kv_request_id is the dict key, so it isn't stored here. + Lives in ``_ClientRequestState.loads`` keyed by round_seq for the + duration of a fetch; the owning kv_request_id is the outer dict key. """ job_id: int # opaque ID assigned by the manager to this load request @@ -68,7 +53,7 @@ class _ClientRequestState: One entry per kv_request_id we're driving. Lookup-phase fields are used only by symmetric P2P (``do_p2p_fetch``); PD-only loads leave - ``probes``/``unsent`` empty and drive just ``phase`` and ``load``. An + ``probes``/``unsent`` empty and drive just ``phase`` and ``loads``. An entry is dropped once every field is idle — see ``ClientRole._maybe_prune``. """ @@ -82,11 +67,23 @@ class _ClientRequestState: # OffloadKeys registered but not yet flushed onto the wire. Drained and # cleared by the next flush_pending_lookups. unsent: list[OffloadKey] = field(default_factory=list) + # Current lookup round. LookupMsgs carry it, each fetch closes it and + # advances it, so every round's supply/demand/completion is isolated + # on the wire. PD clients never probe and stay on round 0. + round_seq: int = 0 + # This id ran the symmetric lookup phase (register_lookup); a fetch + # with keys then requires every key to be a confirmed probe. PD + # loads never probe. + probed: bool = False - # Monotonic lookup/fetch signalling phase; see ``ClientPhase``. - phase: ClientPhase = ClientPhase.REGISTERED - # Set while a fetch is in flight; cleared on completion/abort/timeout. - load: _InboundLoadState | None = None + # The peer holds lookup state no FetchMsg has closed: a LookupMsg + # was flushed since the last fetch. finish owes a terminal empty + # FetchMsg while set, so the peer releases parked supply. + peer_lookup_open: bool = False + # In-flight loads keyed by the round their fetch carried. The + # scheduler submits loads incrementally as chunks resolve, so several + # can be in flight at once; TransferDone/AbortAck match by round. + loads: dict[int, _InboundLoadState] = field(default_factory=dict) class LoadResult(NamedTuple): @@ -133,11 +130,10 @@ class ClientRole: # _serve_pending. Populated by register_lookup, drained by # flush_pending_lookups, and discarded on finish/close. self._flush_pending: set[str] = set() - # kv_request_ids with a fetch in flight (``st.load is not None``) — - # the work-list collect_results walks for timeouts, and the - # has_active_loads predicate, instead of scanning every request. - # Kept in exact sync with ``st.load``: armed in request_blocks, - # discarded wherever load is cleared, and cleared on close. + # kv_request_ids with at least one fetch in flight — the work-list + # collect_results walks for timeouts, and the has_active_loads + # predicate, instead of scanning every request. Kept in exact sync + # with ``st.loads``. self._active_loads: set[str] = set() self._completed_loads: list[LoadResult] = [] @@ -156,17 +152,21 @@ class ClientRole: def _maybe_prune(self, kv_request_id: str) -> None: """Drop the entry once it holds no live load or lookup state. - The sticky ``phase`` is only read by ``finish``. A probe - clears when its fetch is issued (``request_blocks``) or when the - request finishes (``finish``/``close``); in the former case - ``load`` is set and keeps the entry alive, in the latter the phase - is no longer needed — so dropping on emptiness never loses a phase - still in use. + ``peer_lookup_open`` is only read by ``finish``, and every path + that clears the last probe (fetch / finish / close) also settles + it, so dropping on emptiness never loses a flag still in use. """ st = self._requests.get(kv_request_id) - if st is not None and st.load is None and not st.probes and not st.unsent: + if st is not None and not st.loads and not st.probes and not st.unsent: del self._requests[kv_request_id] + def _on_load_terminal(self, kv_request_id: str, st: _ClientRequestState) -> None: + """Wind down id-level state once no load remains in flight.""" + if st.loads: + return + self._active_loads.discard(kv_request_id) + self._maybe_prune(kv_request_id) + @property def has_active_loads(self) -> bool: """True if any kv_request_id has a fetch in flight.""" @@ -184,7 +184,12 @@ class ClientRole: block_ids: Sequence[int], send_ready: bool, ) -> None: - """Register a load request and send the FetchMsg.""" + """Send the FetchMsg closing the current lookup round. + + The scheduler may submit several loads per kv_request_id as its + matched prefix resolves incrementally; each fetch carries the + round it closes so the loads stay independent on the wire. + """ logger.debug( "P2PSession %s: request_blocks job_id=%d kv_request_id=%s " "blocks=%d ready=%s", @@ -195,77 +200,74 @@ class ClientRole: send_ready, ) st = self._get_or_create_request(kv_request_id) - st.load = _InboundLoadState( + round_seq = st.round_seq + st.round_seq += 1 + st.loads[round_seq] = _InboundLoadState( job_id=job_id, submitted_at=time.monotonic(), ) self._active_loads.add(kv_request_id) - st.phase = ClientPhase.FETCH_SENT + st.peer_lookup_open = False self._send( { TYPE_KEY: FetchMsg.TYPE, FetchMsg.KV_REQUEST_ID: kv_request_id, FetchMsg.KEYS: list(keys), FetchMsg.BLOCK_INDEXES: [int(idx) for idx in block_ids], + FetchMsg.ROUND_SEQ: round_seq, } ) - # Issuing the fetch ends this request's lookup phase, so drop all - # probe state. Once the peer serves this fetch both sides unpin, so - # the producer may evict the block; a stale cached True would - # otherwise let a re-scheduled lookup() return HIT without - # re-probing, pointing at a block the producer no longer holds. - # Clearing forces a fresh LookupMsg on re-schedule so the producer - # answers from current state. For a symmetric-P2P request (probes - # populated) every fetched block was a confirmed HIT; a PD-only load - # never probes, so probes is empty and the clear is a no-op. - if st.probes: + # Issuing the fetch closes this lookup round, so drop all probe + # state. Once the peer serves the fetch both sides unpin, so a + # stale cached True would let a later lookup() return HIT for a + # block the producer may have evicted; clearing forces a fresh + # probe under the next round. + if st.probed and keys: + assert st.probes, ( + f"symmetric fetch for {kv_request_id} has keys but no probes" + ) assert all(st.probes.get(key) is True for key in keys) st.probes.clear() def finish(self, kv_request_id: str) -> None: - """Finish a request: abort any in-flight load and release lookup state. + """Finish a request: abort in-flight loads and release lookup state. - Called from the session's ``finish_request``. The two branches are - mutually exclusive: ``load`` is set only by ``request_blocks``, which - also advances ``phase`` to ``FETCH_SENT``, and nothing moves it back - to ``PROBING`` — so a fetch in flight never coexists with the - ``PROBING`` phase. - - - Fetch in flight (``load`` set, phase ``FETCH_SENT``): send an - AbortFetchMsg unless the load is already aborting, then drop it. - - Outstanding lookups (``PROBING``): a LookupMsg was flushed but no - FetchMsg has closed the peer's lookup phase. Every FetchMsg the - server receives in p2p mode is its "request finished" signal (it - releases lookup state and fires ``cb.finish_request``); when the - client's lookups all missed no FetchMsg is otherwise sent, so emit - a terminal empty one purely to trigger those semantics. In - ``REGISTERED`` the peer never received a LookupMsg and in - ``FETCH_SENT`` a FetchMsg already closed the phase, so neither owes - a terminal FetchMsg. + Called from the session's ``finish_request``. Sends an + AbortFetchMsg per load not already aborting, and — independently — + the terminal empty FetchMsg when the peer still holds lookup + state no fetch has closed (its "request finished" signal: it + releases lookup state, drains parked supply, and fires + ``cb.finish_request``). A later round's supply can be parked + while an earlier round's load is still in flight, so both can be + owed at once. Then drop all probe/lookup state and prune the entry. """ st = self._requests.get(kv_request_id) if st is None: return - if st.load is not None: - if st.load.aborted_at is None: + if st.loads: + for round_seq, load in st.loads.items(): + if load.aborted_at is not None: + continue self._send( { TYPE_KEY: AbortFetchMsg.TYPE, AbortFetchMsg.KV_REQUEST_ID: kv_request_id, + AbortFetchMsg.ROUND_SEQ: round_seq, } ) - st.load = None + st.loads.clear() self._active_loads.discard(kv_request_id) - elif st.phase is ClientPhase.PROBING: - st.phase = ClientPhase.FETCH_SENT + if st.peer_lookup_open: + st.peer_lookup_open = False self._send( { TYPE_KEY: FetchMsg.TYPE, FetchMsg.KV_REQUEST_ID: kv_request_id, FetchMsg.KEYS: [], FetchMsg.BLOCK_INDEXES: [], + FetchMsg.ROUND_SEQ: st.round_seq, } ) st.probes.clear() @@ -273,20 +275,21 @@ class ClientRole: self._flush_pending.discard(kv_request_id) self._maybe_prune(kv_request_id) - def on_transfer_done(self, kv_request_id: str, success: bool) -> None: + def on_transfer_done( + self, kv_request_id: str, success: bool, round_seq: int + ) -> None: """Handle a TransferDoneMsg from the peer.""" st = self._requests.get(kv_request_id) - if st is not None and st.load is not None: + load = st.loads.pop(round_seq, None) if st is not None else None + if st is not None and load is not None: self._completed_loads.append( LoadResult( - job_id=st.load.job_id, + job_id=load.job_id, kv_request_id=kv_request_id, success=success, ) ) - st.load = None - self._active_loads.discard(kv_request_id) - self._maybe_prune(kv_request_id) + self._on_load_terminal(kv_request_id, st) else: # No matching in-flight load: either a duplicate # transfer_done from the peer (protocol violation) or a @@ -295,41 +298,44 @@ class ClientRole: # so we can't tell — log so it's findable. logger.warning( "P2PSession %s: transfer_done for unknown kv_request_id=%s " - "(duplicate from peer, or raced with local cancel/timeout)", + "round=%s (duplicate from peer, or raced with local " + "cancel/timeout)", self._peer_id, kv_request_id, + round_seq, ) - def on_abort_ack(self, kv_request_id: str) -> None: + def on_abort_ack(self, kv_request_id: str, round_seq: int) -> None: """Handle an AbortAckMsg from the peer.""" st = self._requests.get(kv_request_id) - if st is not None and st.load is not None: + load = st.loads.pop(round_seq, None) if st is not None else None + if st is not None and load is not None: logger.warning( "P2PSession %s: load request %s (job_id=%d) timed out; " "load job completed with failure. If this recurs, ensure " "PYTHONHASHSEED is set to the same value on all nodes.", self._peer_id, kv_request_id, - st.load.job_id, + load.job_id, ) self._completed_loads.append( LoadResult( - job_id=st.load.job_id, + job_id=load.job_id, kv_request_id=kv_request_id, success=False, ) ) - st.load = None - self._active_loads.discard(kv_request_id) - self._maybe_prune(kv_request_id) + self._on_load_terminal(kv_request_id, st) else: # See on_transfer_done: same ambiguity (duplicate ack # vs. raced with local cancel/timeout that already popped). logger.warning( "P2PSession %s: abort_ack for unknown kv_request_id=%s " - "(duplicate from peer, or raced with local cancel/timeout)", + "round=%s (duplicate from peer, or raced with local " + "cancel/timeout)", self._peer_id, kv_request_id, + round_seq, ) # ------------------------------------------------------------------ @@ -345,17 +351,18 @@ class ClientRole: - Once a LookupRespMsg has resolved the entry: returns the cached bool result on every call without popping it. - A resolved entry is retained until its fetch is issued - (``request_blocks`` pops it) or the request finishes + A resolved entry is retained until a fetch closes the round + (``request_blocks`` clears all probes) or the request finishes (``finish`` clears all entries for the id). A request's block set can be re-probed across steps, so popping on read would make a repeat probe of an already-resolved key look brand-new and re-queue it, emitting a redundant LookupMsg for an answer we already hold. Keeping the entry until fetch makes repeat probes - free; clearing it at fetch forces a fresh probe if the request is - re-scheduled, since the block is unpinned once served. + free; clearing at fetch forces a fresh probe under the next + round, since the block is unpinned once served. """ st = self._get_or_create_request(kv_request_id) + st.probed = True okey = OffloadKey(key) if okey in st.probes: return st.probes[okey] @@ -378,14 +385,11 @@ class ClientRole: ``on_schedule_end()``. A request's block set may be discovered across several scheduler steps, so more than one LookupMsg can go out per kv_request_id — one per step that registered new - keys. register_lookup() de-dups in-flight and already-resolved - (req_id, key) pairs, so each LookupMsg carries only the keys - first probed in that step. The peer's lookup phase for the id is - still closed by exactly one FetchMsg, which the client contract - guarantees is sent after every lookup for the id has resolved - (see request_blocks / finish). Send-gating is handled by - the injected ``_send`` callback (queues until ConnectAckMsg if - needed). + keys, all tagged with the current round. register_lookup() + de-dups in-flight and already-resolved (req_id, key) pairs, so + each LookupMsg carries only the keys first probed in that step. + Send-gating is handled by the injected ``_send`` callback + (queues until ConnectAckMsg if needed). Only requests that registered new keys since the last flush are visited — the ``_flush_pending`` work-list avoids scanning every @@ -395,13 +399,9 @@ class ClientRole: st = self._requests.get(req_id) if st is None or not st.unsent: continue - # Record that the peer now holds lookup state for this id so - # finish knows a terminal empty FetchMsg may be owed. - # Only promote from REGISTERED: once a fetch has gone out - # (FETCH_SENT) a later LookupMsg must not regress the phase, as - # no terminal FetchMsg is owed for an already-fetched request. - if st.phase is ClientPhase.REGISTERED: - st.phase = ClientPhase.PROBING + # The peer now holds lookup state for this id; finish owes a + # terminal empty FetchMsg until a fetch closes it. + st.peer_lookup_open = True logger.debug( "P2P LOOKUP client %s: SEND LookupMsg kv_request_id=%s keys=%d", self._peer_id, @@ -413,6 +413,7 @@ class ClientRole: TYPE_KEY: LookupMsg.TYPE, LookupMsg.KV_REQUEST_ID: req_id, LookupMsg.KEYS: list(st.unsent), + LookupMsg.ROUND_SEQ: st.round_seq, } ) st.unsent = [] @@ -451,52 +452,56 @@ class ClientRole: def collect_results(self) -> list[LoadResult]: """Walk load timeouts and drain completed loads. - Active requests past ``_LOAD_TIMEOUT_S`` get an AbortFetchMsg - sent and enter the aborting phase. Aborting requests past - ``_ABORT_ACK_TIMEOUT_S`` are surfaced as failed loads. + Loads past ``_LOAD_TIMEOUT_S`` get an AbortFetchMsg sent and + enter the aborting phase. Aborting loads past + ``_ABORT_ACK_TIMEOUT_S`` are surfaced as failed. Lookups have no timeout: an unanswered probe stays None (RETRY) until finish_request clears it — see ``_ClientRequestState.probes``. """ now = time.monotonic() - to_remove: list[str] = [] + to_remove: list[tuple[str, int]] = [] for req_id in self._active_loads: st = self._requests[req_id] - assert st.load is not None - load = st.load - if load.aborted_at is None: - if now - load.submitted_at >= _LOAD_TIMEOUT_S: - load.aborted_at = now - logger.warning( - "P2PSession %s: %s timed out, sending abort", - self._peer_id, - req_id, - ) - self._send( - { - TYPE_KEY: AbortFetchMsg.TYPE, - AbortFetchMsg.KV_REQUEST_ID: req_id, - } - ) - else: - if now - load.aborted_at >= _ABORT_ACK_TIMEOUT_S: - to_remove.append(req_id) - self._completed_loads.append( - LoadResult( - job_id=load.job_id, - kv_request_id=req_id, - success=False, + assert st.loads + for round_seq, load in st.loads.items(): + if load.aborted_at is None: + if now - load.submitted_at >= _LOAD_TIMEOUT_S: + load.aborted_at = now + logger.warning( + "P2PSession %s: %s round=%s timed out, sending abort", + self._peer_id, + req_id, + round_seq, ) - ) - logger.warning( - "P2PSession %s: abort_ack timed out for kv_request_id=%s", - self._peer_id, - req_id, - ) - for req_id in to_remove: - self._requests[req_id].load = None - self._active_loads.discard(req_id) - self._maybe_prune(req_id) + self._send( + { + TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.KV_REQUEST_ID: req_id, + AbortFetchMsg.ROUND_SEQ: round_seq, + } + ) + else: + if now - load.aborted_at >= _ABORT_ACK_TIMEOUT_S: + to_remove.append((req_id, round_seq)) + self._completed_loads.append( + LoadResult( + job_id=load.job_id, + kv_request_id=req_id, + success=False, + ) + ) + logger.warning( + "P2PSession %s: abort_ack timed out for " + "kv_request_id=%s round=%s", + self._peer_id, + req_id, + round_seq, + ) + for req_id, round_seq in to_remove: + st = self._requests[req_id] + st.loads.pop(round_seq) + self._on_load_terminal(req_id, st) results = self._completed_loads self._completed_loads = [] @@ -510,12 +515,12 @@ class ClientRole: forever on an answer that can never arrive). See ``ClientCloseResult``. """ failed_jobs = [ - st.load.job_id for st in self._requests.values() if st.load is not None + load.job_id for st in self._requests.values() for load in st.loads.values() ] failed_req_ids = [ req_id for req_id, st in self._requests.items() - if st.load is not None or any(hit is None for hit in st.probes.values()) + if st.loads or any(hit is None for hit in st.probes.values()) ] self._requests.clear() self._flush_pending.clear() diff --git a/vllm/v1/kv_offload/tiering/p2p/session/protocol.py b/vllm/v1/kv_offload/tiering/p2p/session/protocol.py index 8988c7e79f..33ae5bf2f7 100644 --- a/vllm/v1/kv_offload/tiering/p2p/session/protocol.py +++ b/vllm/v1/kv_offload/tiering/p2p/session/protocol.py @@ -26,14 +26,12 @@ Block Transfer Flow (happy path) 1. Client sends FetchMsg with a kv_request_id and lists of block keys + remote indexes where it wants the data written. - In p2p mode FetchMsg is also the server-side "request finished" - signal for the id: no further ``cb.create_store_job`` will fire - (parked LookupMsg batches are popped, so pending-key resolution - cannot promote a HIT after this point), all server-side lookup - state for the id is released, and ``cb.finish_request`` fires on - each dropped batch. The client emits exactly one FetchMsg per - lookup-touched request, including an empty one when no blocks - end up being fetched. + A request may run several lookup→fetch rounds; symmetric-P2P + messages carry ROUND_SEQ so each round's supply, demand, and + completion stay isolated. The terminal empty FetchMsg is the + server-side "request finished" signal for the id: parked + LookupMsg batches are popped and ``cb.finish_request`` fires on + each. 2. Server matches requested blocks against locally stored blocks: - Blocks already available are transferred immediately via RDMA. - Blocks not yet available are recorded as "demanded" and @@ -178,33 +176,32 @@ class DisconnectMsg: class FetchMsg: - """Client → Server: request blocks by key and close the lookup phase. + """Client → Server: request blocks for one lookup round. - In p2p mode FetchMsg is also the server-side "request finished" - signal for ``kv_request_id``: on receipt the server (a) fires no - further ``cb.create_store_job`` for this id — parked LookupMsg - batches are popped, so ``_resolve_pending_lookups`` cannot promote - a HIT_PENDING / RETRY key into a fresh pin after this point — and - (b) calls ``cb.finish_request(batch.ctx)`` on each dropped batch - so the TieringManager can release per-batch bookkeeping. In the - all-miss case the client emits an empty FetchMsg (``KEYS`` - and ``BLOCK_INDEXES`` both empty) purely to fire this signal. + A non-empty fetch closes only its round. The terminal empty FetchMsg + (``KEYS`` and ``BLOCK_INDEXES`` both empty) is the "request + finished" signal: the server pops parked LookupMsg batches, calls + ``cb.finish_request`` on each, and drains any leftover supply. Fields: KV_REQUEST_ID: Identifies this block transfer request. KEYS: List of block keys (OffloadKey bytes). May be empty. BLOCK_INDEXES: List of remote block indexes (same length as KEYS). + ROUND_SEQ: Lookup round this fetch closes. PD clients never probe + and stay on their single round 0. """ TYPE = "fetch" KV_REQUEST_ID = "kv_request_id" KEYS = "keys" BLOCK_INDEXES = "block_indexes" + ROUND_SEQ = "round_seq" @staticmethod def validate(msg: dict) -> None: """Raise ValueError if any field has an invalid type or value.""" _require(msg, FetchMsg.KV_REQUEST_ID, str) + _require_non_neg_int(msg, FetchMsg.ROUND_SEQ) _require_list(msg, FetchMsg.KEYS) _require_list(msg, FetchMsg.BLOCK_INDEXES) keys = msg[FetchMsg.KEYS] @@ -229,17 +226,21 @@ class LookupMsg: Fields: KV_REQUEST_ID: Identifies this lookup transaction. KEYS: List of block keys (OffloadKey bytes) to probe. + ROUND_SEQ: Lookup round these probes belong to; pinned supply is + parked under it for that round's fetch. """ TYPE = "lookup" KV_REQUEST_ID = "kv_request_id" KEYS = "keys" + ROUND_SEQ = "round_seq" @staticmethod def validate(msg: dict) -> None: """Raise ValueError if any field has an invalid type or value.""" _require(msg, LookupMsg.KV_REQUEST_ID, str) _require_list(msg, LookupMsg.KEYS) + _require_non_neg_int(msg, LookupMsg.ROUND_SEQ) class LookupRespMsg: @@ -284,17 +285,22 @@ class TransferDoneMsg: Fields: KV_REQUEST_ID: The request that completed. SUCCESS: Whether the transfer completed successfully. + ROUND_SEQ: The fetch round that completed. Several loads can be + in flight per id (the scheduler submits loads incrementally), + so completions are matched by round. """ TYPE = "transfer_done" KV_REQUEST_ID = "kv_request_id" SUCCESS = "success" + ROUND_SEQ = "round_seq" @staticmethod def validate(msg: dict) -> None: """Raise ValueError if any field has an invalid type or value.""" _require(msg, TransferDoneMsg.KV_REQUEST_ID, str) _require(msg, TransferDoneMsg.SUCCESS, bool) + _require_non_neg_int(msg, TransferDoneMsg.ROUND_SEQ) class AbortFetchMsg: @@ -302,15 +308,18 @@ class AbortFetchMsg: Fields: KV_REQUEST_ID: The request to cancel. + ROUND_SEQ: The fetch round to cancel. """ TYPE = "abort_fetch" KV_REQUEST_ID = "kv_request_id" + ROUND_SEQ = "round_seq" @staticmethod def validate(msg: dict) -> None: """Raise ValueError if any field has an invalid type or value.""" _require(msg, AbortFetchMsg.KV_REQUEST_ID, str) + _require_non_neg_int(msg, AbortFetchMsg.ROUND_SEQ) class AbortAckMsg: @@ -318,12 +327,15 @@ class AbortAckMsg: Fields: KV_REQUEST_ID: The request that was cancelled. + ROUND_SEQ: The round that was cancelled; echoes AbortFetchMsg. """ TYPE = "abort_ack" KV_REQUEST_ID = "kv_request_id" + ROUND_SEQ = "round_seq" @staticmethod def validate(msg: dict) -> None: """Raise ValueError if any field has an invalid type or value.""" _require(msg, AbortAckMsg.KV_REQUEST_ID, str) + _require_non_neg_int(msg, AbortAckMsg.ROUND_SEQ) diff --git a/vllm/v1/kv_offload/tiering/p2p/session/server.py b/vllm/v1/kv_offload/tiering/p2p/session/server.py index 3709ecd9c5..8558acdf91 100644 --- a/vllm/v1/kv_offload/tiering/p2p/session/server.py +++ b/vllm/v1/kv_offload/tiering/p2p/session/server.py @@ -53,15 +53,6 @@ class StoreResult(NamedTuple): success: bool -class _InflightXfer(NamedTuple): - """Metadata for a single inflight RDMA transfer, keyed by transfer_id.""" - - kv_request_id: str - block_count: int - # The set of store job IDs that contributed blocks to this transfer. - job_ids: set[int] - - class _MatchResult(NamedTuple): """Result of block matching: pairs ready for transfer.""" @@ -73,12 +64,17 @@ class _MatchResult(NamedTuple): @dataclass class _OutboundRequestState: - """Server-role state for a single peer fetch request. + """Server-role state for a single fetch round of a peer request. - The owning ``kv_request_id`` is the ``ServerRole._requests`` dict key - and is not duplicated on the value. + A kv_request_id may run several lookup→fetch rounds; rounds live in + ``_ServerRequestState.outbound`` keyed by the wire ``round_seq``, so + terminals touch only their own round. """ + # Supply came from inbound lookup pins (symmetric): no late + # submit_store can arrive, so unmatched fetch demand fails fast. PD + # rounds park demand for stores instead. + lookup_supplied: bool = False demand_received: bool = False available: dict[OffloadKey, tuple[int, int]] = field( default_factory=dict @@ -88,7 +84,8 @@ class _OutboundRequestState: ) # key → remote_block_idx: blocks peer wants, awaiting supply remaining: int = 0 # blocks that need to be transferred to client finishing: bool = False # Signal finish request ASAP - # Job IDs that submit_store'd blocks for this request and have not + inflight: int = 0 # transfers submitted for this round, not yet polled + # Job IDs that submit_store'd blocks for this round and have not # yet emitted a StoreResult. The terminal-finalize helper drains # this set; poll-done and poll-failed discard entries as their # StoreResults fire. @@ -145,6 +142,21 @@ class _OutboundRequestState: ) +@dataclass +class _InflightXfer: + """Metadata for a single inflight RDMA transfer, keyed by transfer_id.""" + + kv_request_id: str + block_count: int + # The set of store job IDs that contributed blocks to this transfer. + job_ids: set[int] + # Round this transfer serves and its key in ``st.outbound``; + # remaining/finalize apply only while the round is still registered. + # Dummy default for test-seeded entries. + round: _OutboundRequestState = field(default_factory=_OutboundRequestState) + round_key: int = 0 + + @dataclass class _ActiveLookup: """In-flight state for one inbound LookupMsg. @@ -159,6 +171,8 @@ class _ActiveLookup: lookup_id: int kv_request_id: str ctx: ReqContext + # Wire round these probes belong to; pins park under it. + round_seq: int = 0 # Keys from the inbound LookupMsg, preserved in wire order so # the aggregated response goes back in the same order. keys: list[OffloadKey] = field(default_factory=list) @@ -185,6 +199,7 @@ class _PendingLookup(NamedTuple): keys: list[OffloadKey] enqueued_at: float + round_seq: int = 0 @dataclass @@ -198,17 +213,15 @@ class _ServerRequestState: is idle — see ``ServerRole._maybe_prune``. """ - # Outbound serve state (PD + symmetric producer side). None until the - # first add_stored_blocks / on_fetch; reset to None on finalize/abort. - outbound: _OutboundRequestState | None = None + # Fetch rounds keyed by wire round_seq. A round is created by its + # first supply or its fetch and removed at its terminal (finalize / + # failure / abort). + outbound: dict[int, _OutboundRequestState] = field(default_factory=dict) # Raw inbound LookupMsgs not yet processed against the ParentManager. pending_lookups: list[_PendingLookup] = field(default_factory=list) # Per-LookupMsg state parked with HIT_PENDING / RETRY keys, keyed by # the (globally unique) lookup_id and re-polled each serve. lookups: dict[int, _ActiveLookup] = field(default_factory=dict) - # Start time of a pending abort drain (``time.monotonic``); None when - # no abort is in progress. - abort_started_at: float | None = None # Transfer ids in ``ServerRole._inflight`` for this id. Kept in sync # via _inflight_add / _inflight_pop so a non-empty set is an exact # "has any inflight transfer" predicate and the abort drain can @@ -256,9 +269,9 @@ class ServerRole: # ``parent.on_request_finished`` in ``serve_external_requests``. self._finished_lookup_ctxs: list[ReqContext] = [] self._lookup_id_counter: int = 0 - # kv_request_ids with a parked abort awaiting drain — work-list so - # drain_pending_aborts doesn't scan every request each poll tick. - self._parked_aborts: set[str] = set() + # Parked aborts awaiting drain, keyed by (kv_request_id, round) + # with the abort start time. + self._pending_aborts: dict[tuple[str, int], float] = {} # ------------------------------------------------------------------ # State helpers @@ -277,11 +290,11 @@ class ServerRole: st = self._requests.get(kv_request_id) if ( st is not None - and st.outbound is None + and not st.outbound and not st.inflight_tids and not st.lookups and not st.pending_lookups - and st.abort_started_at is None + and not any(kv == kv_request_id for kv, _ in self._pending_aborts) ): del self._requests[kv_request_id] @@ -295,87 +308,100 @@ class ServerRole: keys: Sequence[OffloadKey], block_ids: Sequence[int], job_id: JobId, + round_seq: int = 0, + *, + from_lookup: bool = False, ) -> None: - """New blocks stored locally — match against pending fetch demand.""" + """New blocks stored locally — match within their fetch round. + + Lookup pins carry the round they were probed under; PD + submit_store batches share PD's single round 0. + """ self._store_jobs[job_id] = time.monotonic() st = self._get_or_create_request(kv_request_id) - if st.outbound is None: - st.outbound = _OutboundRequestState() - result = st.outbound.add_stored_blocks(keys, block_ids, job_id) - if result.local_idxs and st.outbound.demand_received: - self._submit_transfer(kv_request_id, result) + rnd = st.outbound.get(round_seq) + if rnd is None: + rnd = st.outbound[round_seq] = _OutboundRequestState() + if from_lookup: + rnd.lookup_supplied = True + result = rnd.add_stored_blocks(keys, block_ids, job_id) + if result.local_idxs and rnd.demand_received: + self._submit_transfer(kv_request_id, result, rnd, round_seq) def on_fetch( self, kv_request_id: str, keys: Sequence[OffloadKey], block_indexes: Sequence[int], + round_seq: int = 0, ) -> None: """Handle a FetchMsg from the peer. - In p2p mode FetchMsg is the server-side "request finished" - signal for ``kv_request_id``. Three consequences flow from that: - - - No further ``parent.create_store_job`` will fire for this id: - the request's parked ``lookups`` are popped here (via - ``_finish_inbound_lookups``) before the next - ``serve_external_requests`` runs ``_resolve_pending_lookups``, - so any HIT_PENDING / RETRY key that would otherwise later - promote to HIT and pin a slot is dropped instead. Any raw - not-yet-processed LookupMsg for this id is dropped too. - - All server-side lookup state for the id is cleaned up (the - ``lookups`` entries themselves). - - The synthetic ctx is queued for ``parent.on_request_finished`` - (fired by the next ``serve_external_requests``) so the - TieringManager can release per-lookup bookkeeping. - - The client contract guarantees exactly one FetchMsg per - lookup-touched request — including an empty one when no - blocks end up being fetched. - - Raises ``ValueError`` on a duplicate fetch for the same - ``kv_request_id``; the coordinator's dispatch loop turns that - into a protocol-error disconnect. + A non-empty fetch binds and closes its round, leaving lookup + state alone (the next round's LookupMsg may already be in + flight). The terminal empty fetch closes the id: parked lookups + are popped and every remaining round drained. A second fetch for + a round already holding demand raises ValueError + (protocol-error disconnect). """ logger.debug( - "P2PSession %s: fetch RECEIVED kv_request_id=%s blocks=%d", + "P2PSession %s: fetch RECEIVED kv_request_id=%s round=%s blocks=%d", self._peer_id, kv_request_id, + round_seq, len(keys), ) st = self._requests.get(kv_request_id) - existing = st.outbound if st is not None else None + existing = st.outbound.get(round_seq) if st is not None else None if existing is not None and existing.demand_received: - # A second fetch for the same kv_request_id would overwrite - # `remaining` and leak inflight bookkeeping. Treat as a - # protocol violation. - raise ValueError(f"duplicate fetch for kv_request_id={kv_request_id}") + raise ValueError( + f"duplicate fetch for kv_request_id={kv_request_id} round={round_seq}" + ) st = self._get_or_create_request(kv_request_id) - if st.outbound is None: - st.outbound = _OutboundRequestState() - req = st.outbound + req = st.outbound.get(round_seq) + if req is None: + req = st.outbound[round_seq] = _OutboundRequestState() result = req.add_fetch_demand(keys, block_indexes) + if not keys: + # Terminal empty fetch: close the lookup phase and drain + # every round with no TransferDoneMsg (nothing waits on it). + self._finish_inbound_lookups(kv_request_id) + for key in list(st.outbound): + self._finalize_outbound(kv_request_id, key, send_done=False) + return + if req.lookup_supplied and req.demanded: + # A symmetric round's supply always precedes its fetch, so + # unmatched demand is unservable — fail now, not at the load + # timeout. PD rounds keep parking demand for stores that + # arrive later. + logger.warning( + "P2PSession %s: fetch kv_request_id=%s round=%s demanded %d " + "blocks but %d have no pinned supply; failing fetch " + "immediately", + self._peer_id, + kv_request_id, + round_seq, + len(keys), + len(req.demanded), + ) + self._finalize_outbound(kv_request_id, round_seq, success=False) + return if result.local_idxs: - self._submit_transfer(kv_request_id, result) - # Close the peer's request as far as the server's lookup phase - # is concerned: pop parked lookups so no further - # ``parent.create_store_job`` fires for this id, and queue their - # ctxs for ``parent.on_request_finished``. Done before the - # finalize path below so bookkeeping releases in-order. - self._finish_inbound_lookups(kv_request_id) + self._submit_transfer(kv_request_id, result, req, round_seq) # Prefiller-first mode: finish_request may have run before # fetch arrived. If so, finalize once we know what was # demanded — fully satisfied → success, else early-fail. - if req.finishing and not self._has_inflight_for(kv_request_id): - self._finalize_outbound(kv_request_id) + if req.finishing and req.inflight == 0: + self._finalize_outbound(kv_request_id, round_seq) - def on_abort_fetch(self, kv_request_id: str) -> None: - """Handle an AbortFetchMsg from the peer.""" + def on_abort_fetch(self, kv_request_id: str, round_seq: int = 0) -> None: + """Handle an AbortFetchMsg from the peer, cancelling one round.""" # Abort for an unknown id may be a benign race/duplicate or a # real protocol violation; we don't track completed ids, so warn. st = self._requests.get(kv_request_id) - has_outbound = st is not None and st.outbound is not None - if not has_outbound and not self._has_inflight_for(kv_request_id): + if (st is None or not st.outbound) and not self._has_inflight_for( + kv_request_id + ): logger.warning( "P2PSession %s: abort_fetch for unknown kv_request_id=%s " "(no outbound or inflight state); benign race or stale", @@ -385,16 +411,15 @@ class ServerRole: # Idempotent: receiving AbortFetchMsg again before we've sent the # ack just triggers another drain attempt without resetting the # deadline. - st = self._get_or_create_request(kv_request_id) - if st.abort_started_at is None: - st.abort_started_at = time.monotonic() - self._parked_aborts.add(kv_request_id) - self._drain_abort(kv_request_id) + self._get_or_create_request(kv_request_id) + self._pending_aborts.setdefault((kv_request_id, round_seq), time.monotonic()) + self._drain_abort(kv_request_id, round_seq) def on_lookup( self, kv_request_id: str, keys: Sequence[OffloadKey], + round_seq: int = 0, ) -> None: """Enqueue a LookupMsg from a symmetric-P2P consumer. @@ -406,13 +431,18 @@ class ServerRole: parent calls are valid. """ logger.debug( - "P2P LOOKUP server %s: RECV LookupMsg kv_request_id=%s keys=%d", + "P2P LOOKUP server %s: RECV LookupMsg kv_request_id=%s round=%s keys=%d", self._peer_id, kv_request_id, + round_seq, len(keys), ) self._get_or_create_request(kv_request_id).pending_lookups.append( - _PendingLookup(keys=list(keys), enqueued_at=time.monotonic()) + _PendingLookup( + keys=list(keys), + enqueued_at=time.monotonic(), + round_seq=round_seq, + ) ) self._serve_pending.add(kv_request_id) @@ -434,7 +464,7 @@ class ServerRole: st.pending_lookups = [] for pl in pending: self._process_inbound_lookup( - kv_request_id, pl.keys, pl.enqueued_at, parent + kv_request_id, pl.keys, pl.enqueued_at, pl.round_seq, parent ) self._resolve_pending_lookups(kv_request_id, parent) st = self._requests.get(kv_request_id) @@ -481,9 +511,7 @@ class ServerRole: else: lookup.pending.add(h) if new_hits: - self._pin_and_register_hits( - lookup.kv_request_id, new_hits, lookup.ctx, parent - ) + self._pin_and_register_hits(lookup, new_hits, parent) return new_hits def _process_inbound_lookup( @@ -491,6 +519,7 @@ class ServerRole: kv_request_id: str, keys: list[OffloadKey], enqueued_at: float, + round_seq: int, parent: ParentManager, ) -> None: """Resolve one enqueued LookupMsg against ``parent``. @@ -515,6 +544,7 @@ class ServerRole: lookup_id=lookup_id, kv_request_id=kv_request_id, ctx=ctx, + round_seq=round_seq, keys=list(keys), deadline=enqueued_at + _LOOKUP_PENDING_TIMEOUT_S, ) @@ -547,25 +577,26 @@ class ServerRole: def _pin_and_register_hits( self, - kv_request_id: str, + lookup: _ActiveLookup, keys: list[OffloadKey], - ctx: ReqContext, parent: ParentManager, ) -> None: - """Pin primary slots for HIT keys and feed them into the - existing ``add_stored_blocks`` matching path. + """Pin primary slots for HIT keys and park them as the lookup's + round supply via ``add_stored_blocks``. Caller has already confirmed every key is HIT (single-threaded scheduler ⇒ no eviction race), so the JobMetadata returned by ``parent.create_store_job`` carries parallel ``keys``/``block_ids`` of length ``len(keys)``. """ - meta = parent.create_store_job(keys, ctx) + meta = parent.create_store_job(keys, lookup.ctx) self.add_stored_blocks( - kv_request_id, + lookup.kv_request_id, list(meta.keys), list(meta.block_ids), meta.job_id, + round_seq=lookup.round_seq, + from_lookup=True, ) def _resolve_pending_lookups( @@ -650,9 +681,8 @@ class ServerRole: Called on the two events that mean "no more lookup traffic for ``kv_request_id`` is expected on this session": the terminal - FetchMsg from the peer (client contract: exactly one FetchMsg - per lookup-touched request, even if empty), and a local - ``finish``. Whichever fires second is a no-op. + empty FetchMsg from the peer and a local ``finish``. Whichever + fires second is a no-op. """ st = self._requests.get(kv_request_id) if st is None: @@ -688,20 +718,15 @@ class ServerRole: self._finish_inbound_lookups(kv_request_id) st = self._requests.get(kv_request_id) - req = st.outbound if st is not None else None - if req is None: + if st is None: return - req.finishing = True - if not req.demand_received: - return - if self._has_inflight_for(kv_request_id): - return - # Remaining > 0 here: if it had hit 0, the poll-done success - # branch would have already cleared outbound and we'd have - # returned at `req is None` above. Helper derives success from - # remaining and emits StoreResult(success=False) for any - # leftover pending jobs. - self._finalize_outbound(kv_request_id) + for key, req in list(st.outbound.items()): + req.finishing = True + if not req.demand_received or req.inflight: + # No demand yet (prefiller-first): on_fetch finalizes via + # `finishing`. Inflight: the last completion finalizes. + continue + self._finalize_outbound(kv_request_id, key) def collect_results(self) -> list[StoreResult]: """Drain timeouts, deferred results, and transport completions. @@ -741,20 +766,24 @@ class ServerRole: ) continue results.extend(self._settle_xfer_jobs(xfer, success=True)) + rnd = xfer.round st = self._requests.get(xfer.kv_request_id) - req = st.outbound if st is not None else None - if req is not None and req.demand_received: - req.remaining -= xfer.block_count - assert req.remaining >= 0, ( + if st is not None and st.outbound.get(xfer.round_key) is rnd: + rnd.remaining -= xfer.block_count + assert rnd.remaining >= 0, ( f"remaining went negative for kv_request_id={xfer.kv_request_id}" ) - if req.remaining == 0: - self._finalize_outbound(xfer.kv_request_id, success=True) - elif req.finishing and not self._has_inflight_for(xfer.kv_request_id): - self._finalize_outbound(xfer.kv_request_id, success=False) + if rnd.remaining == 0: + self._finalize_outbound( + xfer.kv_request_id, xfer.round_key, success=True + ) + elif rnd.finishing and rnd.inflight == 0: + self._finalize_outbound( + xfer.kv_request_id, xfer.round_key, success=False + ) self._maybe_prune(xfer.kv_request_id) - failed_kv_request_ids: set[str] | None = None + failed_rounds: list[tuple[str, _OutboundRequestState]] | None = None for tid in poll_result.failed: xfer = self._inflight_pop(tid) if xfer is None: @@ -767,35 +796,36 @@ class ServerRole: tid, ) continue - if failed_kv_request_ids is None: - failed_kv_request_ids = set() - failed_kv_request_ids.add(xfer.kv_request_id) results.extend(self._settle_xfer_jobs(xfer, success=False)) + rnd = xfer.round st = self._requests.get(xfer.kv_request_id) - req = st.outbound if st is not None else None - if st is not None: - st.outbound = None - if req is not None and req.demand_received: + if st is not None and st.outbound.get(xfer.round_key) is rnd: + del st.outbound[xfer.round_key] + if failed_rounds is None: + failed_rounds = [] + failed_rounds.append((xfer.kv_request_id, rnd)) self._send( { TYPE_KEY: TransferDoneMsg.TYPE, TransferDoneMsg.KV_REQUEST_ID: xfer.kv_request_id, TransferDoneMsg.SUCCESS: False, + TransferDoneMsg.ROUND_SEQ: xfer.round_key, } ) self._maybe_prune(xfer.kv_request_id) - # Cancel other inflight for the same failed kv_request_ids - if failed_kv_request_ids: - ids_to_cancel = [ - tid - for tid, xfer in self._inflight.items() - if xfer.kv_request_id in failed_kv_request_ids - ] - for tid in ids_to_cancel: - self._inflight_pop(tid) - self._transport.cancel(ids_to_cancel) - for kv_request_id in failed_kv_request_ids: + # Cancel each failed round's other inflight and fail its + # remaining store jobs — nothing else will settle them. + if failed_rounds: + for kv_request_id, rnd in failed_rounds: + ids_to_cancel = [ + tid for tid, x in self._inflight.items() if x.round is rnd + ] + for tid in ids_to_cancel: + self._inflight_pop(tid) + if ids_to_cancel: + self._transport.cancel(ids_to_cancel) + results.extend(self._fail_round_jobs(rnd)) self._maybe_prune(kv_request_id) return results @@ -811,8 +841,8 @@ class ServerRole: def drain_pending_aborts(self) -> None: """Re-attempt every parked abort once per poll tick.""" - for kv_request_id in list(self._parked_aborts): - self._drain_abort(kv_request_id) + for kv_request_id, round_seq in list(self._pending_aborts): + self._drain_abort(kv_request_id, round_seq) def close(self) -> tuple[list[int], list[ReqContext]]: """Tear down. Cancels inflight. @@ -839,7 +869,7 @@ class ServerRole: failed_serves.extend(self._finished_lookup_ctxs) self._requests.clear() self._serve_pending.clear() - self._parked_aborts.clear() + self._pending_aborts.clear() self._finished_lookup_ctxs.clear() return failed_stores, failed_serves @@ -870,6 +900,8 @@ class ServerRole: xfer = self._inflight.pop(tid, None) if xfer is None: return None + xfer.round.inflight -= 1 + assert xfer.round.inflight >= 0 st = self._requests.get(xfer.kv_request_id) if st is not None: st.inflight_tids.discard(tid) @@ -880,80 +912,108 @@ class ServerRole: ) -> list[StoreResult]: """Emit StoreResults for a completed transfer's store jobs. - Pops each attached job from ``_store_jobs`` and clears it from the - request's pending set. A job already popped (via timeout, cancel, - etc.) is skipped so we never double-emit a contradictory result. + Pops each attached job from ``_store_jobs`` and clears it from + its round's pending set. A job already popped (via timeout, + cancel, etc.) is skipped so we never double-emit a contradictory + result. """ results: list[StoreResult] = [] - st = self._requests.get(xfer.kv_request_id) - req = st.outbound if st is not None else None for job_id in xfer.job_ids: if self._store_jobs.pop(job_id, None) is None: continue results.append(StoreResult(job_id=job_id, success=success)) - if req is not None: - req.pending_job_ids.discard(job_id) + xfer.round.pending_job_ids.discard(job_id) return results # ------------------------------------------------------------------ # Internal — finalize / abort drain # ------------------------------------------------------------------ + def _fail_round_jobs(self, rnd: _OutboundRequestState) -> list[StoreResult]: + """Fail a terminated round's still-pending store jobs (idempotent).""" + results: list[StoreResult] = [] + for job_id in rnd.pending_job_ids: + if self._store_jobs.pop(job_id, None) is None: + continue + results.append(StoreResult(job_id=job_id, success=False)) + rnd.pending_job_ids.clear() + return results + def _finalize_outbound( self, kv_request_id: str, + round_key: int, success: bool | None = None, + send_done: bool = True, ) -> None: - """Pop the outbound state and emit terminal results. - - Called when no further work will happen for this kv_request_id - on the server side: either request_finish has fired and there - are no inflight transfers, or the last inflight just completed - while finishing. + """Pop one round and emit its terminal results. If ``success`` is None, derive it from ``req.remaining == 0``. - The same flag is used for both the peer's TransferDoneMsg and - the StoreResult(s) emitted for any leftover pending job_ids. + ``send_done=False`` skips the TransferDoneMsg (terminal empty + fetch). Other rounds of the id are untouched. """ st = self._requests[kv_request_id] - assert st.outbound is not None - req = st.outbound - st.outbound = None + req = st.outbound.pop(round_key) if success is None: - success = req.remaining == 0 - for job_id in req.pending_job_ids: - self._store_jobs.pop(job_id, None) - self._pending_store_results.append( - StoreResult(job_id=job_id, success=success) - ) - self._send( - { - TYPE_KEY: TransferDoneMsg.TYPE, - TransferDoneMsg.KV_REQUEST_ID: kv_request_id, - TransferDoneMsg.SUCCESS: success, - } + success = req.demand_received and req.remaining == 0 + settled = self._fail_round_jobs(req) if not success else None + if settled is not None: + self._pending_store_results.extend(settled) + else: + for job_id in req.pending_job_ids: + if self._store_jobs.pop(job_id, None) is None: + continue + self._pending_store_results.append( + StoreResult(job_id=job_id, success=True) + ) + req.pending_job_ids.clear() + logger.debug( + "P2PSession %s: finalize kv_request_id=%s round=%s success=%s " + "remaining=%d leftover_available=%d send_done=%s", + self._peer_id, + kv_request_id, + round_key, + success, + req.remaining, + len(req.available), + send_done, ) + if send_done and req.demand_received: + self._send( + { + TYPE_KEY: TransferDoneMsg.TYPE, + TransferDoneMsg.KV_REQUEST_ID: kv_request_id, + TransferDoneMsg.SUCCESS: success, + TransferDoneMsg.ROUND_SEQ: round_key, + } + ) self._maybe_prune(kv_request_id) - def _drain_abort(self, kv_request_id: str) -> None: + def _drain_abort(self, kv_request_id: str, round_seq: int) -> None: """One drain attempt for a pending abort. - Stops accepting more blocks for ``kv_request_id``, then asks the - transport to cancel any matching inflight transfers in - ``mode="wait"``. Sends ``AbortAckMsg`` once nothing remains - inflight, or after ``_CANCEL_DRAIN_TIMEOUT_S`` falls back to - ``mode="immediate"`` and acks anyway. + Detaches the aborted round, then asks the transport to cancel its + inflight transfers in ``mode="wait"``. Sends ``AbortAckMsg`` once + nothing remains inflight, or after ``_CANCEL_DRAIN_TIMEOUT_S`` + falls back to ``mode="immediate"`` and acks anyway. """ st = self._requests[kv_request_id] - st.outbound = None - ids = list(st.inflight_tids) + rnd = st.outbound.pop(round_seq, None) + if rnd is not None: + # Its transfers are being cancelled; fail its jobs now + # instead of leaking them to the store timeout. + self._pending_store_results.extend(self._fail_round_jobs(rnd)) + ids = [ + tid + for tid, x in self._inflight.items() + if x.kv_request_id == kv_request_id and x.round_key == round_seq + ] if not ids: - self._finalize_abort(kv_request_id) + self._finalize_abort(kv_request_id, round_seq) return - assert st.abort_started_at is not None - expired = time.monotonic() - st.abort_started_at >= _CANCEL_DRAIN_TIMEOUT_S - if expired: + started_at = self._pending_aborts[(kv_request_id, round_seq)] + if time.monotonic() - started_at >= _CANCEL_DRAIN_TIMEOUT_S: for tid in ids: self._inflight_pop(tid) self._transport.cancel(ids, mode="immediate") @@ -964,7 +1024,7 @@ class ServerRole: kv_request_id, len(ids), ) - self._finalize_abort(kv_request_id) + self._finalize_abort(kv_request_id, round_seq) return still = self._transport.cancel(ids, mode="wait") @@ -977,16 +1037,15 @@ class ServerRole: if tid not in still_set: self._inflight_pop(tid) if not still: - self._finalize_abort(kv_request_id) + self._finalize_abort(kv_request_id, round_seq) - def _finalize_abort(self, kv_request_id: str) -> None: - st = self._requests[kv_request_id] - st.abort_started_at = None - self._parked_aborts.discard(kv_request_id) + def _finalize_abort(self, kv_request_id: str, round_seq: int) -> None: + self._pending_aborts.pop((kv_request_id, round_seq), None) self._send( { TYPE_KEY: AbortAckMsg.TYPE, AbortAckMsg.KV_REQUEST_ID: kv_request_id, + AbortAckMsg.ROUND_SEQ: round_seq, } ) self._maybe_prune(kv_request_id) @@ -995,7 +1054,13 @@ class ServerRole: # Internal — transfers and store-job timeouts # ------------------------------------------------------------------ - def _submit_transfer(self, kv_request_id: str, result: _MatchResult) -> None: + def _submit_transfer( + self, + kv_request_id: str, + result: _MatchResult, + rnd: _OutboundRequestState, + round_key: int, + ) -> None: logger.debug( "P2PSession %s: NIXL write_blocks CALL kv_request_id=%s " "local_idxs=%d remote_idxs=%d", @@ -1016,12 +1081,15 @@ class ServerRole: transfer_id, len(result.local_idxs), ) + rnd.inflight += 1 self._inflight_add( transfer_id, _InflightXfer( kv_request_id=kv_request_id, block_count=len(result.local_idxs), job_ids=result.job_ids, + round=rnd, + round_key=round_key, ), ) else: @@ -1031,22 +1099,24 @@ class ServerRole: kv_request_id, len(result.local_idxs), ) - # The matched blocks were popped from req.demanded / - # req.available, but no inflight will satisfy them, so - # remaining will never reach 0 on its own. Mark the - # request as finishing so the existing terminal paths - # clean up: if other inflight is in flight, the last one - # to drain will fire _finalize_outbound(success=False) - # via the elif branch in collect_results. If - # nothing else is in flight, finalize now so the peer - # and the local store jobs don't wait for finish_request - # or for _STORE_TIMEOUT_S / _LOAD_TIMEOUT_S. + # The matched blocks were popped from rnd.demanded / + # rnd.available, but no inflight will satisfy them, so + # remaining will never reach 0 on its own. Mark the round + # as finishing so the existing terminal paths clean up: if + # other transfers of this round are in flight, the last one + # to drain will fire _finalize_outbound(success=False) via + # the elif branch in collect_results. If nothing else is in + # flight, finalize now so the peer and the local store jobs + # don't wait for finish_request or for _STORE_TIMEOUT_S / + # _LOAD_TIMEOUT_S. + rnd.finishing = True st = self._requests.get(kv_request_id) - req = st.outbound if st is not None else None - if req is not None: - req.finishing = True - if not self._has_inflight_for(kv_request_id): - self._finalize_outbound(kv_request_id, success=False) + if ( + st is not None + and st.outbound.get(round_key) is rnd + and rnd.inflight == 0 + ): + self._finalize_outbound(kv_request_id, round_key, success=False) def _timeout_pending_store_jobs(self) -> list[StoreResult]: if not self._store_jobs: diff --git a/vllm/v1/kv_offload/tiering/p2p/session/session.py b/vllm/v1/kv_offload/tiering/p2p/session/session.py index 7d19913b0a..4bd780159a 100644 --- a/vllm/v1/kv_offload/tiering/p2p/session/session.py +++ b/vllm/v1/kv_offload/tiering/p2p/session/session.py @@ -374,26 +374,34 @@ class P2PSession: for bh in msg[FetchMsg.KEYS] ] block_indexes = msg[FetchMsg.BLOCK_INDEXES] + round_seq = msg[FetchMsg.ROUND_SEQ] # Run the server-role state machine inline as today — # add_fetch_demand records demand against any blocks we've # already seen in `available`. Report the kv_request_id so # the manager (after poll() returns) can replay any parked # submit_store batches; their add_stored_blocks calls hit # the demand recorded here and submit transfers immediately. - self._server.on_fetch(kv_request_id, keys, block_indexes) + self._server.on_fetch(kv_request_id, keys, block_indexes, round_seq) self._new_fetch_ids.append(kv_request_id) elif msg_type == AbortFetchMsg.TYPE: AbortFetchMsg.validate(msg) - self._server.on_abort_fetch(msg[AbortFetchMsg.KV_REQUEST_ID]) + self._server.on_abort_fetch( + msg[AbortFetchMsg.KV_REQUEST_ID], + msg[AbortFetchMsg.ROUND_SEQ], + ) elif msg_type == TransferDoneMsg.TYPE: TransferDoneMsg.validate(msg) self._client.on_transfer_done( msg[TransferDoneMsg.KV_REQUEST_ID], msg[TransferDoneMsg.SUCCESS], + msg[TransferDoneMsg.ROUND_SEQ], ) elif msg_type == AbortAckMsg.TYPE: AbortAckMsg.validate(msg) - self._client.on_abort_ack(msg[AbortAckMsg.KV_REQUEST_ID]) + self._client.on_abort_ack( + msg[AbortAckMsg.KV_REQUEST_ID], + msg[AbortAckMsg.ROUND_SEQ], + ) elif msg_type == LookupMsg.TYPE: LookupMsg.validate(msg) kv_request_id = msg[LookupMsg.KV_REQUEST_ID] @@ -401,7 +409,7 @@ class P2PSession: OffloadKey(bh if isinstance(bh, bytes) else bytes(bh)) for bh in msg[LookupMsg.KEYS] ] - self._server.on_lookup(kv_request_id, keys) + self._server.on_lookup(kv_request_id, keys, msg[LookupMsg.ROUND_SEQ]) elif msg_type == LookupRespMsg.TYPE: LookupRespMsg.validate(msg) kv_request_id = msg[LookupRespMsg.KV_REQUEST_ID]