[Bugfix][KV Offload][P2P] Scope serve state to fetch rounds (#49877)

Signed-off-by: Itay Etelis <itay.etelis@ibm.com>
Co-authored-by: Itay Etelis <itay.etelis@ibm.com>
Co-authored-by: Itay Etelis <Itay.etelis@gmail.com>
This commit is contained in:
Itay Etelis
2026-07-28 18:01:55 +03:00
committed by GitHub
co-authored by Itay Etelis Itay Etelis
parent 0d0504b54c
commit 30217b0e80
6 changed files with 570 additions and 390 deletions
@@ -1612,16 +1612,17 @@ class TestBindHostPortDefaults:
monkeypatch.setattr(
manager_module,
"NixlTransport",
lambda agent_name, *a, **k: calls.update(nixl_name=agent_name)
or SimpleNamespace(),
lambda agent_name, *a, **k: (
calls.update(nixl_name=agent_name) or SimpleNamespace()
),
)
monkeypatch.setattr(
manager_module,
"ZmqTransport",
lambda local_id, host, port, *a, **k: calls.update(
zmq_id=local_id, zmq_host=host, zmq_port=port
)
or SimpleNamespace(),
lambda local_id, host, port, *a, **k: (
calls.update(zmq_id=local_id, zmq_host=host, zmq_port=port)
or SimpleNamespace()
),
)
spec = SimpleNamespace(
blocks_per_chunk=1,
+100 -16
View File
@@ -49,6 +49,7 @@ from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
from vllm.v1.kv_offload.tiering.p2p.session.server import (
_CANCEL_DRAIN_TIMEOUT_S,
_InflightXfer,
_OutboundRequestState,
)
from vllm.v1.kv_offload.tiering.p2p.session.session import (
_MAX_CONSECUTIVE_DISPATCH_ERRORS,
@@ -315,10 +316,26 @@ def _activate(
# either a missing entry or a None field. These helpers paper over that.
def _client_load(session: P2PSession, kv_request_id: str):
"""The single in-flight load of a kv_request_id (loads are per-round)."""
loads = session._client._requests[kv_request_id].loads
assert len(loads) == 1
return next(iter(loads.values()))
def _srv_outbound(session: P2PSession, kv_request_id: str):
"""Outbound serve state for a kv_request_id, or None (idle / GC'd)."""
"""Serve-side round for a kv_request_id, or None (idle / GC'd).
Rounds are keyed by wire round_seq; surfaces the demanded round when
a fetch has bound one, else any parked supply round.
"""
st = session._server._requests.get(kv_request_id)
return st.outbound if st is not None else None
if st is None or not st.outbound:
return None
for rnd in st.outbound.values():
if rnd.demand_received:
return rnd
return next(iter(st.outbound.values()))
def _srv_lookups(session: P2PSession) -> list:
@@ -330,8 +347,10 @@ def _srv_lookups(session: P2PSession) -> list:
def _srv_abort_started(session: P2PSession, kv_request_id: str) -> float | None:
"""Pending-abort start time for a kv_request_id, or None."""
st = session._server._requests.get(kv_request_id)
return st.abort_started_at if st is not None else None
for (kv, _), started in session._server._pending_aborts.items():
if kv == kv_request_id:
return started
return None
def _srv_inflight_count(session: P2PSession, kv_request_id: str) -> int:
@@ -479,6 +498,7 @@ class TestClientFlows:
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: True,
}
@@ -495,6 +515,7 @@ class TestClientFlows:
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: False,
}
@@ -538,6 +559,7 @@ class TestClientFlows:
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: True,
}
@@ -552,7 +574,7 @@ class TestClientFlows:
session.request_blocks(
job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
session._client._requests["req-1"].load.submitted_at = time.monotonic() - 60.0
_client_load(session, "req-1").submitted_at = time.monotonic() - 60.0
session.poll()
abort = conn._sent[-1]
assert abort[TYPE_KEY] == AbortFetchMsg.TYPE
@@ -569,7 +591,7 @@ class TestClientFlows:
job_id=7, kv_request_id="req-7", keys=[b"k"], block_ids=[0]
)
# 1) Trip the load timeout to send AbortFetch and stamp aborted_at.
session._client._requests["req-7"].load.submitted_at = (
_client_load(session, "req-7").submitted_at = (
time.monotonic() - _LOAD_TIMEOUT_S - 1.0
)
loads = session.poll().loads
@@ -579,11 +601,11 @@ class TestClientFlows:
and m[AbortFetchMsg.KV_REQUEST_ID] == "req-7"
for m in conn._sent
)
assert session._client._requests["req-7"].load.aborted_at is not None
assert _client_load(session, "req-7").aborted_at is not None
# 2) Now backdate aborted_at past the abort-ack timeout. No ack ever
# arrived from the peer.
session._client._requests["req-7"].load.aborted_at = (
_client_load(session, "req-7").aborted_at = (
time.monotonic() - _ABORT_ACK_TIMEOUT_S - 1.0
)
loads = session.poll().loads
@@ -599,17 +621,18 @@ class TestClientFlows:
session.request_blocks(
job_id=8, kv_request_id="req-8", keys=[b"k"], block_ids=[0]
)
session._client._requests["req-8"].load.submitted_at = (
_client_load(session, "req-8").submitted_at = (
time.monotonic() - _LOAD_TIMEOUT_S - 1.0
)
# First poll: AbortFetch goes out.
session.poll()
assert session._client._requests["req-8"].load.aborted_at is not None
assert _client_load(session, "req-8").aborted_at is not None
# Peer acks the abort.
conn.enqueue(
{
TYPE_KEY: AbortAckMsg.TYPE,
AbortAckMsg.ROUND_SEQ: 0,
AbortAckMsg.KV_REQUEST_ID: "req-8",
}
)
@@ -915,6 +938,7 @@ class TestLookupFlow:
conn.enqueue(
{
TYPE_KEY: LookupMsg.TYPE,
LookupMsg.ROUND_SEQ: 0,
LookupMsg.KV_REQUEST_ID: "req-1",
LookupMsg.KEYS: [b"hX", b"hY", b"hZ"],
}
@@ -951,6 +975,7 @@ def _send_lookup(conn: FakeConnection, kv_request_id: str, keys: list[bytes]):
conn.enqueue(
{
TYPE_KEY: LookupMsg.TYPE,
LookupMsg.ROUND_SEQ: 0,
LookupMsg.KV_REQUEST_ID: kv_request_id,
LookupMsg.KEYS: list(keys),
}
@@ -1201,6 +1226,7 @@ class TestServerLookupHandling:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [],
FetchMsg.BLOCK_INDEXES: [],
@@ -1240,6 +1266,7 @@ class TestServerLookupHandling:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"hA", b"hB"],
FetchMsg.BLOCK_INDEXES: [20, 21],
@@ -1277,6 +1304,7 @@ class TestServerFlows:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [10, 11],
@@ -1295,6 +1323,7 @@ class TestServerFlows:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
@@ -1313,6 +1342,7 @@ class TestServerFlows:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
@@ -1331,6 +1361,7 @@ class TestServerFlows:
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
@@ -1349,13 +1380,19 @@ class TestServerFlows:
tid = 42
session._server._inflight_add(
tid,
_InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}),
_InflightXfer(
kv_request_id="req-1",
block_count=1,
job_ids={1},
round=_OutboundRequestState(inflight=1),
),
)
transport._cancel_still_inflight.add(tid)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
@@ -1376,13 +1413,19 @@ class TestServerFlows:
tid = 42
session._server._inflight_add(
tid,
_InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}),
_InflightXfer(
kv_request_id="req-1",
block_count=1,
job_ids={1},
round=_OutboundRequestState(inflight=1),
),
)
transport._cancel_still_inflight.add(tid)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
@@ -1409,20 +1452,26 @@ class TestServerFlows:
tid = 42
session._server._inflight_add(
tid,
_InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}),
_InflightXfer(
kv_request_id="req-1",
block_count=1,
job_ids={1},
round=_OutboundRequestState(inflight=1),
),
)
transport._cancel_still_inflight.add(tid)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
session.poll()
assert _srv_abort_started(session, "req-1") is not None
# Backdate past the drain deadline.
session._server._requests["req-1"].abort_started_at = (
session._server._pending_aborts[("req-1", 0)] = (
time.monotonic() - _CANCEL_DRAIN_TIMEOUT_S - 1.0
)
# Even if the transport still claims it can't cancel, the
@@ -1445,13 +1494,19 @@ class TestServerFlows:
tid = 42
session._server._inflight_add(
tid,
_InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}),
_InflightXfer(
kv_request_id="req-1",
block_count=1,
job_ids={1},
round=_OutboundRequestState(inflight=1),
),
)
transport._cancel_still_inflight.add(tid)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
@@ -1463,6 +1518,7 @@ class TestServerFlows:
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
@@ -1497,6 +1553,7 @@ class TestServerFlows:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
@@ -1529,6 +1586,7 @@ class TestServerFlows:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
@@ -1571,6 +1629,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
@@ -1596,6 +1655,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [10, 11],
@@ -1631,6 +1691,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [10],
@@ -1665,6 +1726,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [10],
@@ -1686,6 +1748,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-2",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [10, 11],
@@ -1719,6 +1782,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"demand"],
FetchMsg.BLOCK_INDEXES: [5],
@@ -1753,6 +1817,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [10],
@@ -1790,6 +1855,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [10],
@@ -1824,6 +1890,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2", b"k3"],
FetchMsg.BLOCK_INDEXES: [10, 11, 12],
@@ -1885,6 +1952,7 @@ class TestFinishRequestServerSide:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [10, 11],
@@ -1951,6 +2019,7 @@ class TestBidirectional:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-srv",
FetchMsg.KEYS: [b"served"],
FetchMsg.BLOCK_INDEXES: [7],
@@ -1981,6 +2050,7 @@ class TestBidirectional:
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-cli",
TransferDoneMsg.SUCCESS: True,
}
@@ -2168,6 +2238,7 @@ class TestAdversarial:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-bad",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [1],
@@ -2216,6 +2287,7 @@ class TestDispatchErrorHandling:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-bad",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [1],
@@ -2246,6 +2318,7 @@ class TestDispatchErrorHandling:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [0],
@@ -2269,6 +2342,7 @@ class TestDispatchErrorHandling:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [0],
@@ -2296,6 +2370,7 @@ class TestDispatchErrorHandling:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [0],
@@ -2341,6 +2416,7 @@ class TestInflightPerReqInvariant:
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: kv_id,
FetchMsg.KEYS: keys,
FetchMsg.BLOCK_INDEXES: indexes,
@@ -2394,7 +2470,12 @@ class TestInflightPerReqInvariant:
tid = kv_id_idx * 10 + j
session._server._inflight_add(
tid,
_InflightXfer(kv_request_id=kv_id, block_count=1, job_ids={tid}),
_InflightXfer(
kv_request_id=kv_id,
block_count=1,
job_ids={tid},
round=_OutboundRequestState(inflight=1),
),
)
assert _srv_total_inflight(session) == len(session._server._inflight)
assert session._server._has_inflight_for("req-0")
@@ -2488,6 +2569,7 @@ class TestFetchMsgValidation:
def _valid_msg(self) -> dict:
return {
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [0, 1],
@@ -2513,6 +2595,7 @@ class TestTransferDoneMsgValidation:
def test_valid_message_passes(self):
msg = {
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: True,
}
@@ -2521,6 +2604,7 @@ class TestTransferDoneMsgValidation:
def test_success_wrong_type(self):
msg = {
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: 1,
}