Files
Liqian LinCodexRobert ShawItay Etelismergify[bot] <37929162+mergify[bot]@users.noreply.github.com>Or OzeriKevin H. Luu
3f1bcfd82b [Bugfix][KV Offload] Bound unanswered P2P lookup waits (#56836)
Signed-off-by: jacklin78911-collab <jacklin78911@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
2026-10-15 08:23:34 +03:00

3510 lines
134 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the unified bidirectional P2PSession.
A P2PSession owns a single ControlConnection and dispatches every message
type — both client-role (FetchMsg / TransferDoneMsg / AbortAck) and
server-role (FetchMsg / TransferDoneMsg / AbortAck from the peer's
perspective). These tests exercise both flows independently and the
bidirectional case where one session simultaneously serves a fetch and
completes its own load.
"""
from __future__ import annotations
import time
from collections.abc import Callable, Sequence
from concurrent.futures import Future
from types import SimpleNamespace
import numpy as np
import pytest
import vllm.v1.kv_offload.tiering.p2p.session.server as server_mod
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadKey,
ReqContext,
RequestOffloadingContext,
)
from vllm.v1.kv_offload.tiering.base import TransferJob
from vllm.v1.kv_offload.tiering.p2p.session import (
LoadResult,
P2PSession,
StoreResult,
)
from vllm.v1.kv_offload.tiering.p2p.session.client import (
_ABORT_ACK_TIMEOUT_S,
_LOAD_TIMEOUT_S,
_LOOKUP_TIMEOUT_S,
)
from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
TYPE_KEY,
AbortAckMsg,
AbortFetchMsg,
ConnectAckMsg,
ConnectMsg,
DisconnectMsg,
FetchMsg,
LookupMsg,
LookupRespMsg,
TransferDoneMsg,
)
from vllm.v1.kv_offload.tiering.p2p.session.server import (
_CANCEL_DRAIN_TIMEOUT_S,
_InflightXfer,
_OutboundRequestState,
)
from vllm.v1.kv_offload.tiering.p2p.session.session import (
_MAX_CONSECUTIVE_DISPATCH_ERRORS,
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
# Shared PYTHONHASHSEED used by the session under test and the fake peer's
# ConnectMsg so the handshake succeeds unless a test overrides one side.
_DEFAULT_HASH_SEED = "0"
class FakeDataTransport:
"""Minimal fake DataTransport for testing sessions."""
def __init__(
self,
base_addr: int = 0x1000,
num_blocks: int = 16,
block_len: int = 4096,
config_fingerprint: str = "",
defer_registration: bool = False,
) -> None:
# When True, add_remote_peer_async parks the registration until
# complete_registrations() resolves it, standing in for the real
# transport's worker thread.
self.defer_registration = defer_registration
self._deferred: list[tuple[Future[None], Callable[[], None]]] = []
self._base_addr = base_addr
self._num_blocks = num_blocks
self._block_len = block_len
self._config_fingerprint = config_fingerprint
self._remote_peers: dict[str, dict] = {}
self._transfers: dict[int, tuple] = {}
self._next_id = 0
self._poll_done: list[int] = []
self._poll_failed: list[int] = []
self._cancel_still_inflight: set[int] = set()
self._cancel_calls: list[tuple[list[int], str]] = []
@property
def base_addr(self) -> int:
return self._base_addr
@property
def num_blocks(self) -> int:
return self._num_blocks
@property
def block_len(self) -> int:
return self._block_len
@property
def config_fingerprint(self) -> str:
return self._config_fingerprint
def get_agent_metadata(self) -> bytes:
return b"fake-metadata"
def add_remote_peer(
self, peer_id, agent_metadata, base_addr, num_blocks, block_len
) -> None:
self._remote_peers[peer_id] = {
ConnectMsg.AGENT_METADATA: agent_metadata,
ConnectMsg.BASE_ADDR: base_addr,
"num_blocks": num_blocks,
"block_len": block_len,
}
def add_remote_peer_async(
self, peer_id, agent_metadata, base_addr, num_blocks, block_len
) -> Future[None]:
future: Future[None] = Future()
def apply() -> None:
self.add_remote_peer(
peer_id, agent_metadata, base_addr, num_blocks, block_len
)
if self.defer_registration:
self._deferred.append((future, apply))
return future
apply()
future.set_result(None)
return future
def complete_registrations(self, exc: Exception | None = None) -> int:
"""Resolve parked registrations the way the worker thread would.
Returns the number resolved so callers can assert one was actually
in flight rather than silently testing nothing.
"""
deferred, self._deferred = self._deferred, []
for future, apply in deferred:
if exc is not None:
future.set_exception(exc)
else:
apply()
future.set_result(None)
return len(deferred)
def remove_remote_peer(self, peer_id: str) -> None:
self._remote_peers.pop(peer_id, None)
def write_blocks(self, peer_id, local_idxs, remote_idxs) -> int | None:
if peer_id not in self._remote_peers:
return None
tid = self._next_id
self._next_id += 1
self._transfers[tid] = (peer_id, local_idxs, remote_idxs)
return tid
def poll(self, peer_id=None):
from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult
result = PollResult(done=list(self._poll_done), failed=list(self._poll_failed))
self._poll_done.clear()
self._poll_failed.clear()
return result
def cancel(self, transfer_ids, mode: str = "immediate") -> list[int]:
ids = list(transfer_ids)
self._cancel_calls.append((ids, mode))
if mode == "wait":
still: list[int] = []
for tid in ids:
if tid in self._cancel_still_inflight:
still.append(tid)
else:
self._transfers.pop(tid, None)
return still
for tid in ids:
self._transfers.pop(tid, None)
self._cancel_still_inflight.discard(tid)
return []
def close(self) -> None:
pass
class FakeConnection:
"""Fake ControlConnection that captures sent messages."""
def __init__(self, peer_id: str = "peer:8000") -> None:
self.peer_id = peer_id
self._inbox: list[dict] = []
self._sent: list[dict] = []
self._closed = False
# When True, send() raises to simulate a broken/dead connection.
self.fail_send = False
@property
def alive(self) -> bool:
return not self._closed
def send(self, msg: dict) -> None:
if self.fail_send:
raise ConnectionError("simulated dead connection")
self._sent.append(msg)
def recv(self) -> list[dict]:
msgs = self._inbox
self._inbox = []
return msgs
def enqueue(self, msg: dict) -> None:
self._inbox.append(msg)
def mark_dead(self) -> None:
self._closed = True
def close(self) -> None:
self._closed = True
def _peer_connect_msg(
peer_id: str = "peer:8000",
block_len: int = 4096,
fingerprint: str | None = None,
hash_seed: str = _DEFAULT_HASH_SEED,
) -> dict:
"""Build a ConnectMsg as if the peer sent it."""
msg = {
TYPE_KEY: ConnectMsg.TYPE,
ConnectMsg.PEER_ID: peer_id,
ConnectMsg.AGENT_METADATA: b"peer-metadata",
ConnectMsg.BASE_ADDR: 0x2000,
ConnectMsg.NUM_BLOCKS: 16,
ConnectMsg.BLOCK_LEN: block_len,
ConnectMsg.HASH_SEED: hash_seed,
}
if fingerprint is not None:
msg[ConnectMsg.CONFIG_FINGERPRINT] = fingerprint
return msg
class FakeParent:
"""Configurable :class:`ParentManager` for server-role tests.
``stored`` is the dict of ready blocks (key → primary block_id).
``pending`` and ``retry`` script the first lookup() result for those
keys; subsequent lookups behave normally (a key that promised
HIT_PENDING / RETRY can later be promoted to HIT by adding it to
``stored`` and removing it from ``pending``/``retry``). ``calls``
captures every parent invocation in order for assertions.
Injected per-step via ``session.serve_external_requests(parent)`` —
not held by the session, matching how ``TieringOffloadingManager``
hands the tier a handle valid only for that call.
"""
def __init__(
self,
stored: dict[OffloadKey, int] | None = None,
pending: set[OffloadKey] | None = None,
retry: set[OffloadKey] | None = None,
) -> None:
self.stored: dict[OffloadKey, int] = dict(stored or {})
self.pending: set[OffloadKey] = set(pending or ())
self.retry: set[OffloadKey] = set(retry or ())
self._next_job_id: int = 1000
self.calls: list[tuple] = []
def on_new_request(self, ctx: ReqContext) -> RequestOffloadingContext:
self.calls.append(("on_new_request", ctx.req_id))
return RequestOffloadingContext()
def lookup(self, key: OffloadKey, ctx: ReqContext) -> LookupResult:
self.calls.append(("lookup", key, ctx.req_id))
if key in self.pending:
return LookupResult.HIT_PENDING
if key in self.retry:
return LookupResult.RETRY
if key in self.stored:
return LookupResult.HIT
return LookupResult.MISS
def create_store_job(
self,
keys: Sequence[OffloadKey],
ctx: ReqContext,
) -> TransferJob:
keys_list = list(keys)
self.calls.append(("create_store_job", tuple(keys_list), ctx.req_id))
chunk_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32)
job_id = self._next_job_id
self._next_job_id += 1
return TransferJob(
job_id=job_id,
keys=keys_list,
chunk_ids=chunk_ids,
is_promotion=False,
req_context=ctx,
)
def on_request_finished(self, ctx: ReqContext) -> None:
self.calls.append(("on_request_finished", ctx.req_id))
def _make_session(
conn: FakeConnection | None = None,
transport: FakeDataTransport | None = None,
peer_id: str = "peer:8000",
local_id: str = "local:9000",
local_hash_seed: str = _DEFAULT_HASH_SEED,
) -> tuple[P2PSession, FakeConnection, FakeDataTransport]:
if conn is None:
conn = FakeConnection(peer_id=peer_id)
if transport is None:
transport = FakeDataTransport()
session = P2PSession(
peer_id=peer_id,
local_id=local_id,
transport=transport,
local_block_len=transport.block_len,
local_hash_seed=local_hash_seed,
conn=conn,
)
return session, conn, transport
def _serve(session: P2PSession, parent: FakeParent) -> None:
"""Resolve enqueued inbound lookups, as the manager does each step."""
session.serve_external_requests(parent)
def _activate(
session: P2PSession, conn: FakeConnection, peer_id: str = "peer:8000"
) -> None:
"""Drive the handshake: peer sends ConnectMsg + ConnectAckMsg."""
conn.enqueue(_peer_connect_msg(peer_id=peer_id))
conn.enqueue({TYPE_KEY: ConnectAckMsg.TYPE, ConnectAckMsg.PEER_ID: peer_id})
session.poll()
# --- Accessors for the server role's consolidated per-request state.
# ServerRole keeps one _ServerRequestState per kv_request_id; entries are
# garbage-collected once fully idle, so "no outbound"/"no abort" reads as
# either a missing entry or a None field. These helpers paper over that.
def _client_load(session: P2PSession, kv_request_id: str):
"""The single in-flight load of a kv_request_id (loads are per-round)."""
loads = session._client._requests[kv_request_id].loads
assert len(loads) == 1
return next(iter(loads.values()))
def _srv_outbound(session: P2PSession, kv_request_id: str):
"""Serve-side round for a kv_request_id, or None (idle / GC'd).
Rounds are keyed by wire round_seq; surfaces the demanded round when
a fetch has bound one, else any parked supply round.
"""
st = session._server._requests.get(kv_request_id)
if st is None or not st.outbound:
return None
for rnd in st.outbound.values():
if rnd.demand_received:
return rnd
return next(iter(st.outbound.values()))
def _srv_lookups(session: P2PSession) -> list:
"""Every parked inbound _ActiveLookup across all requests."""
return [
lu for st in session._server._requests.values() for lu in st.lookups.values()
]
def _srv_abort_started(session: P2PSession, kv_request_id: str) -> float | None:
"""Pending-abort start time for a kv_request_id, or None."""
for (kv, _), started in session._server._pending_aborts.items():
if kv == kv_request_id:
return started
return None
def _srv_inflight_count(session: P2PSession, kv_request_id: str) -> int:
"""Inflight-transfer count tracked for a kv_request_id (0 if idle)."""
st = session._server._requests.get(kv_request_id)
return len(st.inflight_tids) if st is not None else 0
def _srv_total_inflight(session: P2PSession) -> int:
"""Sum of per-request inflight counts across all requests."""
return sum(len(st.inflight_tids) for st in session._server._requests.values())
# ---------------------------------------------------------------------------
# Connect / handshake
# ---------------------------------------------------------------------------
class TestConnectHandshake:
def test_connect_msg_sent_on_creation(self):
"""Session sends its own ConnectMsg on connection."""
session, conn, _ = _make_session()
assert len(conn._sent) == 1
msg = conn._sent[0]
assert msg[TYPE_KEY] == ConnectMsg.TYPE
assert msg[ConnectMsg.PEER_ID] == "local:9000"
assert msg[ConnectMsg.NUM_BLOCKS] == 16
assert msg[ConnectMsg.BLOCK_LEN] == 4096
assert ConnectMsg.AGENT_METADATA in msg
def test_peer_connect_triggers_add_remote_and_ack(self):
"""Receiving ConnectMsg registers peer and replies with ConnectAck."""
session, conn, transport = _make_session()
conn.enqueue(_peer_connect_msg())
session.poll()
assert "peer:8000" in transport._remote_peers
ack = next(m for m in conn._sent if m[TYPE_KEY] == ConnectAckMsg.TYPE)
assert ack[ConnectAckMsg.PEER_ID] == "local:9000"
def test_ack_withheld_until_registration_completes(self):
"""ConnectAck must not precede registration.
The peer treats ConnectAck as permission to send, so acking while
registration is still in flight would let a FetchMsg arrive for
blocks this side cannot yet write to.
"""
transport = FakeDataTransport(defer_registration=True)
session, conn, _ = _make_session(transport=transport)
conn.enqueue(_peer_connect_msg())
session.poll()
assert "peer:8000" not in transport._remote_peers
assert not any(m[TYPE_KEY] == ConnectAckMsg.TYPE for m in conn._sent)
assert transport.complete_registrations() == 1
session.poll()
assert "peer:8000" in transport._remote_peers
ack = next(m for m in conn._sent if m[TYPE_KEY] == ConnectAckMsg.TYPE)
assert ack[ConnectAckMsg.PEER_ID] == "local:9000"
def test_pending_registration_is_pending_work(self):
"""An in-flight registration keeps the engine ticking.
Nothing else drives poll(), so if the session reported idle the
ack would never be sent.
"""
transport = FakeDataTransport(defer_registration=True)
session, conn, _ = _make_session(transport=transport)
conn.enqueue(_peer_connect_msg())
session.poll()
assert session.has_pending_work
transport.complete_registrations()
session.poll()
assert not session.has_pending_work
def test_failed_registration_rejects_peer(self):
"""A registration that fails kills the connection instead of acking."""
transport = FakeDataTransport(defer_registration=True)
session, conn, _ = _make_session(transport=transport)
conn.enqueue(_peer_connect_msg())
session.poll()
transport.complete_registrations(exc=RuntimeError("nixl exploded"))
session.poll()
assert not conn.alive
assert not any(m[TYPE_KEY] == ConnectAckMsg.TYPE for m in conn._sent)
def test_duplicate_connect_during_registration_is_rejected(self):
"""A second ConnectMsg mid-registration is a protocol violation."""
transport = FakeDataTransport(defer_registration=True)
session, conn, _ = _make_session(transport=transport)
conn.enqueue(_peer_connect_msg())
session.poll()
conn.enqueue(_peer_connect_msg())
session.poll()
assert not conn.alive
def test_connect_ack_makes_session_ready(self):
"""Session.ready becomes True after ConnectAckMsg."""
session, conn, _ = _make_session()
assert not session.ready
conn.enqueue({TYPE_KEY: ConnectAckMsg.TYPE, ConnectAckMsg.PEER_ID: "peer:8000"})
session.poll()
assert session.ready
def test_messages_queued_before_ack_flush_on_ack(self):
"""Outgoing messages sent before ConnectAck are flushed after."""
session, conn, _ = _make_session()
session.request_blocks(
job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
# Before ack: only our ConnectMsg was sent.
assert len(conn._sent) == 1
assert conn._sent[0][TYPE_KEY] == ConnectMsg.TYPE
# Ack arrives.
conn.enqueue({TYPE_KEY: ConnectAckMsg.TYPE, ConnectAckMsg.PEER_ID: "peer:8000"})
session.poll()
# Queued fetch is now sent.
assert any(m[TYPE_KEY] == FetchMsg.TYPE for m in conn._sent)
def test_block_len_mismatch_marks_dead(self):
"""Mismatched block_len rejects peer and marks connection dead."""
session, conn, transport = _make_session()
conn.enqueue(_peer_connect_msg(block_len=8192)) # mismatch
session.poll()
assert "peer:8000" not in transport._remote_peers
assert not session.alive
def test_config_fingerprint_mismatch_marks_dead(self):
"""Mismatched config fingerprint rejects peer."""
transport = FakeDataTransport(config_fingerprint="abc123")
session, conn, _ = _make_session(transport=transport)
conn.enqueue(_peer_connect_msg(fingerprint="different"))
session.poll()
assert "peer:8000" not in transport._remote_peers
assert not session.alive
def test_config_fingerprint_match_succeeds(self):
"""Matching fingerprints register the peer."""
transport = FakeDataTransport(config_fingerprint="same_fp")
session, conn, _ = _make_session(transport=transport)
conn.enqueue(_peer_connect_msg(fingerprint="same_fp"))
session.poll()
assert "peer:8000" in transport._remote_peers
assert session.alive
def test_missing_fingerprint_allowed(self):
"""Missing fingerprint on either side is allowed."""
transport = FakeDataTransport(config_fingerprint="abc123")
session, conn, _ = _make_session(transport=transport)
conn.enqueue(_peer_connect_msg()) # no fingerprint
session.poll()
assert "peer:8000" in transport._remote_peers
def test_hash_seed_mismatch_marks_dead(self):
"""Mismatched PYTHONHASHSEED rejects peer and marks connection dead."""
session, conn, transport = _make_session(local_hash_seed="0")
conn.enqueue(_peer_connect_msg(hash_seed="12345")) # mismatch
session.poll()
assert "peer:8000" not in transport._remote_peers
assert not session.alive
assert not any(m[TYPE_KEY] == ConnectAckMsg.TYPE for m in conn._sent)
def test_hash_seed_match_succeeds(self):
"""Matching PYTHONHASHSEED registers the peer and acks."""
session, conn, transport = _make_session(local_hash_seed="12345")
conn.enqueue(_peer_connect_msg(hash_seed="12345"))
session.poll()
assert "peer:8000" in transport._remote_peers
assert session.alive
assert any(m[TYPE_KEY] == ConnectAckMsg.TYPE for m in conn._sent)
def test_hash_seed_advertised_in_connect_msg(self):
"""Session advertises its own PYTHONHASHSEED in the ConnectMsg."""
_, conn, _ = _make_session(local_hash_seed="777")
assert conn._sent[0][ConnectMsg.HASH_SEED] == "777"
# ---------------------------------------------------------------------------
# Client-role flows
# ---------------------------------------------------------------------------
class TestClientFlows:
def test_request_blocks_sends_fetch(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=1, kv_request_id="req-1", keys=[b"k1", b"k2"], block_ids=[0, 1]
)
lookup = conn._sent[-1]
assert lookup[TYPE_KEY] == FetchMsg.TYPE
assert lookup[FetchMsg.KV_REQUEST_ID] == "req-1"
assert lookup[FetchMsg.KEYS] == [b"k1", b"k2"]
assert lookup[FetchMsg.BLOCK_INDEXES] == [0, 1]
def test_transfer_done_success(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: True,
}
)
loads = session.poll().loads
assert loads == [LoadResult(job_id=1, kv_request_id="req-1", success=True)]
def test_transfer_done_failure(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: False,
}
)
loads = session.poll().loads
assert loads == [LoadResult(job_id=1, kv_request_id="req-1", success=False)]
def test_finish_request_sends_abort(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
session.finish_request("req-1")
abort = conn._sent[-1]
assert abort[TYPE_KEY] == AbortFetchMsg.TYPE
assert abort[AbortFetchMsg.KV_REQUEST_ID] == "req-1"
def test_finish_preserves_active_load_until_terminal_result(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=7, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
session.finish_request("req-1")
load = _client_load(session, "req-1")
assert load.job_id == 7
assert load.aborted_at is not None
aborted_at = load.aborted_at
assert session._client.has_active_loads is True
session.finish_request("req-1")
assert _client_load(session, "req-1").aborted_at == aborted_at
aborts = [m for m in conn._sent if m[TYPE_KEY] == AbortFetchMsg.TYPE]
assert len(aborts) == 1
assert aborts[0][AbortFetchMsg.ROUND_SEQ] == 0
def test_finish_abort_ack_emits_one_failure(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=8, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
session.finish_request("req-1")
conn.enqueue(
{
TYPE_KEY: AbortAckMsg.TYPE,
AbortAckMsg.ROUND_SEQ: 0,
AbortAckMsg.KV_REQUEST_ID: "req-1",
}
)
loads = session.poll().loads
assert loads == [LoadResult(job_id=8, kv_request_id="req-1", success=False)]
assert session._client.has_active_loads is False
assert "req-1" not in session._client._requests
assert session.poll().loads == []
def test_finish_abort_timeout_emits_one_failure(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=9, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
session.finish_request("req-1")
_client_load(session, "req-1").aborted_at = (
time.monotonic() - _ABORT_ACK_TIMEOUT_S - 1.0
)
loads = session.poll().loads
assert loads == [LoadResult(job_id=9, kv_request_id="req-1", success=False)]
assert session._client.has_active_loads is False
assert "req-1" not in session._client._requests
assert session.poll().loads == []
def test_late_transfer_done_after_abort_ack_is_ignored(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=10, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
session.finish_request("req-1")
conn.enqueue(
{
TYPE_KEY: AbortAckMsg.TYPE,
AbortAckMsg.ROUND_SEQ: 0,
AbortAckMsg.KV_REQUEST_ID: "req-1",
}
)
assert session.poll().loads == [
LoadResult(job_id=10, kv_request_id="req-1", success=False)
]
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: True,
}
)
assert session.poll().loads == []
assert session._client.has_active_loads is False
def test_active_loads_work_list_tracks_in_flight(self):
"""collect_results / has_active_loads use the _active_loads work-list,
armed when a fetch is issued and discarded exactly when its load
clears — a probe-only request never enters it, and completion empties
it while the entry may briefly linger for GC."""
session, conn, _ = _make_session()
_activate(session, conn)
client = session._client
# A probe-only request has no in-flight load: not in _active_loads.
session.register_lookup("req-probe", b"hp")
assert client._active_loads == set()
assert client.has_active_loads is False
# Issuing a fetch arms the work-list.
session.request_blocks(
job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
assert client._active_loads == {"req-1"}
assert client.has_active_loads is True
# Completion clears the load and discards it from the work-list.
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: True,
}
)
session.poll()
assert client._active_loads == set()
assert client.has_active_loads is False
def test_load_timeout_sends_abort(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0]
)
_client_load(session, "req-1").submitted_at = time.monotonic() - 60.0
session.poll()
abort = conn._sent[-1]
assert abort[TYPE_KEY] == AbortFetchMsg.TYPE
def test_load_abort_ack_timeout_surfaces_failure(self):
"""After load timeout sends AbortFetch, if no AbortAck arrives within
_ABORT_ACK_TIMEOUT_S the request is surfaced as failed and removed
from _requests — the engine cannot wait forever on a peer that won't
ack.
"""
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=7, kv_request_id="req-7", keys=[b"k"], block_ids=[0]
)
# 1) Trip the load timeout to send AbortFetch and stamp aborted_at.
_client_load(session, "req-7").submitted_at = (
time.monotonic() - _LOAD_TIMEOUT_S - 1.0
)
loads = session.poll().loads
assert loads == []
assert any(
m.get(TYPE_KEY) == AbortFetchMsg.TYPE
and m[AbortFetchMsg.KV_REQUEST_ID] == "req-7"
for m in conn._sent
)
assert _client_load(session, "req-7").aborted_at is not None
# 2) Now backdate aborted_at past the abort-ack timeout. No ack ever
# arrived from the peer.
_client_load(session, "req-7").aborted_at = (
time.monotonic() - _ABORT_ACK_TIMEOUT_S - 1.0
)
loads = session.poll().loads
assert loads == [LoadResult(job_id=7, kv_request_id="req-7", success=False)]
assert "req-7" not in session._client._requests
def test_load_abort_ack_clears_request(self):
"""After load timeout sends AbortFetch, an arriving AbortAckMsg from
the peer surfaces the failure cleanly and removes the request from
_requests — covers the on_abort_ack arrival path."""
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
job_id=8, kv_request_id="req-8", keys=[b"k"], block_ids=[0]
)
_client_load(session, "req-8").submitted_at = (
time.monotonic() - _LOAD_TIMEOUT_S - 1.0
)
# First poll: AbortFetch goes out.
session.poll()
assert _client_load(session, "req-8").aborted_at is not None
# Peer acks the abort.
conn.enqueue(
{
TYPE_KEY: AbortAckMsg.TYPE,
AbortAckMsg.ROUND_SEQ: 0,
AbortAckMsg.KV_REQUEST_ID: "req-8",
}
)
loads = session.poll().loads
assert loads == [LoadResult(job_id=8, kv_request_id="req-8", success=False)]
assert "req-8" not in session._client._requests
# ---------------------------------------------------------------------------
# Symmetric-P2P lookup flow (do_p2p_fetch)
# ---------------------------------------------------------------------------
class TestLookupFlow:
"""Consumer-side state machine for do_p2p_fetch lookups."""
def test_lookup_deadline_starts_after_handshake(self, lookup_clock):
session, conn, _ = _make_session()
session.register_lookup("req-1", b"hA")
session.flush_pending_lookups()
# Complete the handshake before its independent timeout expires.
lookup_clock.now = _LOOKUP_TIMEOUT_S - 1
session.poll()
assert session.alive
_activate(session, conn)
session.flush_pending_lookups()
lookups = [m for m in conn._sent if m[TYPE_KEY] == LookupMsg.TYPE]
assert len(lookups) == 1
lookup_clock.now += _LOOKUP_TIMEOUT_S - 1
session.poll()
assert session.alive
lookup_clock.now += 1
session.poll()
assert not session.alive
assert session.close().failed_req_ids == ["req-1"]
def test_partial_response_rearms_lookup_deadline(self, lookup_clock):
session, conn, _ = _make_session()
_activate(session, conn)
session.register_lookup("req-1", b"hA")
session.register_lookup("req-1", b"hB")
session.flush_pending_lookups()
lookup_clock.now = _LOOKUP_TIMEOUT_S - 1
conn.enqueue(
{
TYPE_KEY: LookupRespMsg.TYPE,
LookupRespMsg.KV_REQUEST_ID: "req-1",
LookupRespMsg.KEYS: [b"hA"],
LookupRespMsg.HITS: [True],
}
)
session.poll()
assert session.register_lookup("req-1", b"hA") is True
assert session.register_lookup("req-1", b"hB") is None
session.register_lookup("req-1", b"hC")
session.flush_pending_lookups()
lookup_clock.now += 1
session.poll()
assert session.alive
lookup_clock.now += _LOOKUP_TIMEOUT_S - 1
session.poll()
assert not session.alive
assert session.close().failed_req_ids == ["req-1"]
def test_lookup_progress_keeps_overlapping_batches_alive(self, lookup_clock):
"""Prompt batch replies keep a long-running lookup phase alive."""
session, conn, _ = _make_session()
_activate(session, conn)
previous_key = b"key-0"
session.register_lookup("req-1", previous_key)
session.flush_pending_lookups()
for step in range(1, int(2 * _LOOKUP_TIMEOUT_S) + 1):
lookup_clock.now = float(step)
next_key = f"key-{step}".encode()
hit = step % 2 == 0
session.register_lookup("req-1", next_key)
session.flush_pending_lookups()
conn.enqueue(
{
TYPE_KEY: LookupRespMsg.TYPE,
LookupRespMsg.KV_REQUEST_ID: "req-1",
LookupRespMsg.KEYS: [previous_key],
LookupRespMsg.HITS: [hit],
}
)
session.poll()
assert session.alive
assert session.register_lookup("req-1", previous_key) is hit
previous_key = next_key
def test_buffered_lookup_response_precedes_expiry(self, lookup_clock):
session, conn, _ = _make_session()
_activate(session, conn)
session.register_lookup("req-1", b"hA")
session.flush_pending_lookups()
lookup_clock.now = _LOOKUP_TIMEOUT_S - 1
conn.enqueue(
{
TYPE_KEY: LookupRespMsg.TYPE,
LookupRespMsg.KV_REQUEST_ID: "req-1",
LookupRespMsg.KEYS: [b"hA"],
LookupRespMsg.HITS: [True],
}
)
lookup_clock.now = _LOOKUP_TIMEOUT_S
session.poll()
assert session.alive
assert session.register_lookup("req-1", b"hA") is True
def test_duplicate_responses_and_new_keys_do_not_rearm(self, lookup_clock):
session, conn, _ = _make_session()
_activate(session, conn)
session.register_lookup("req-1", b"hA")
session.register_lookup("req-1", b"hB")
session.flush_pending_lookups()
response = {
TYPE_KEY: LookupRespMsg.TYPE,
LookupRespMsg.KV_REQUEST_ID: "req-1",
LookupRespMsg.KEYS: [b"hA", b"unknown"],
LookupRespMsg.HITS: [True, False],
}
conn.enqueue(response)
session.poll()
lookup_clock.now = _LOOKUP_TIMEOUT_S - 1
conn.enqueue(response)
session.register_lookup("req-1", b"hC")
session.flush_pending_lookups()
session.poll()
assert session.alive
lookup_clock.now = _LOOKUP_TIMEOUT_S
session.poll()
assert not session.alive
@pytest.mark.parametrize("partial_fetch", [False, True])
def test_lookup_expiry_preserves_completed_transfers(
self, lookup_clock, partial_fetch
):
session, conn, transport = _make_session()
_activate(session, conn)
session.register_lookup("stalled", b"key")
session.flush_pending_lookups()
lookup_clock.now = _LOOKUP_TIMEOUT_S - 1
session.request_blocks(1, "load", [b"key"], [0])
session.add_stored_blocks("store", [b"key"], [1], 2)
if partial_fetch:
session.add_stored_blocks("store", [b"unused"], [2], 3)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.KV_REQUEST_ID: "store",
FetchMsg.KEYS: [b"key"],
FetchMsg.BLOCK_INDEXES: [0],
FetchMsg.ROUND_SEQ: 0,
}
)
session.poll()
transport._poll_done = list(transport._transfers)
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.KV_REQUEST_ID: "load",
TransferDoneMsg.SUCCESS: True,
TransferDoneMsg.ROUND_SEQ: 0,
}
)
lookup_clock.now = _LOOKUP_TIMEOUT_S
result = session.poll()
assert not session.alive
assert result.loads == [LoadResult(1, "load", True)]
expected_stores = [StoreResult(2, True)]
if partial_fetch:
expected_stores.append(StoreResult(3, True))
assert result.stores == expected_stores
closed = session.close()
assert closed.failed_req_ids == ["stalled"]
assert closed.failed_jobs == []
assert closed.failed_stores == []
@pytest.mark.parametrize("action", ["resolve", "finish", "fetch"])
def test_lookup_deadline_cleared_before_next_batch(self, lookup_clock, action):
session, conn, _ = _make_session()
_activate(session, conn)
session.register_lookup("req-1", b"hA")
session.register_lookup("req-1", b"hB")
session.flush_pending_lookups()
lookup_clock.now = _LOOKUP_TIMEOUT_S - 1
conn.enqueue(
{
TYPE_KEY: LookupRespMsg.TYPE,
LookupRespMsg.KV_REQUEST_ID: "req-1",
LookupRespMsg.KEYS: [b"hA"],
LookupRespMsg.HITS: [True],
}
)
session.poll()
if action == "resolve":
conn.enqueue(
{
TYPE_KEY: LookupRespMsg.TYPE,
LookupRespMsg.KV_REQUEST_ID: "req-1",
LookupRespMsg.KEYS: [b"hB"],
LookupRespMsg.HITS: [False],
}
)
session.poll()
elif action == "finish":
session.finish_request("req-1")
else:
session.request_blocks(1, "req-1", [b"hA"], [0])
lookup_clock.now += 2
session.poll()
assert session.alive
session.register_lookup("req-1", b"hC")
session.flush_pending_lookups()
lookup_clock.now += _LOOKUP_TIMEOUT_S - 1
session.poll()
assert session.alive
lookup_clock.now += 1
session.poll()
assert not session.alive
def test_aggregate_flush_resolve_round_trip(self):
"""register_lookup → flush sends one LookupMsg → response
resolves entries → register_lookup returns the cached bool on
every call, and repeat probes never re-issue a LookupMsg."""
session, conn, _ = _make_session()
_activate(session, conn)
# Aggregate two keys for the same kv_request_id; both return None.
assert session.register_lookup("req-1", b"hA") is None
assert session.register_lookup("req-1", b"hB") is None
# Flush sends one LookupMsg with both keys.
sent_before = len(conn._sent)
session.flush_pending_lookups()
new = conn._sent[sent_before:]
assert len(new) == 1
msg = new[0]
assert msg[TYPE_KEY] == LookupMsg.TYPE
assert msg[LookupMsg.KV_REQUEST_ID] == "req-1"
assert sorted(msg[LookupMsg.KEYS]) == [b"hA", b"hB"]
# Idempotent re-flush sends nothing — the entries are now in-flight.
sent_before = len(conn._sent)
session.flush_pending_lookups()
assert conn._sent[sent_before:] == []
# While in-flight, register_lookup keeps returning None.
assert session.register_lookup("req-1", b"hA") is None
# Peer answers: hA hit, hB miss.
conn.enqueue(
{
TYPE_KEY: LookupRespMsg.TYPE,
LookupRespMsg.KV_REQUEST_ID: "req-1",
LookupRespMsg.KEYS: [b"hA", b"hB"],
LookupRespMsg.HITS: [True, False],
}
)
session.poll()
# register_lookup returns the resolved bool.
assert session.register_lookup("req-1", b"hA") is True
assert session.register_lookup("req-1", b"hB") is False
# The entry is cached, not popped: repeat probes keep returning the
# same result and never re-queue the key, so a flush sends nothing.
assert session.register_lookup("req-1", b"hA") is True
assert session.register_lookup("req-1", b"hB") is False
sent_before = len(conn._sent)
session.flush_pending_lookups()
assert [
m for m in conn._sent[sent_before:] if m[TYPE_KEY] == LookupMsg.TYPE
] == []
def test_request_blocks_clears_probe_cache(self):
"""A resolved HIT probe is popped when its fetch is issued, so a
re-scheduled request re-probes instead of trusting the stale True
(the served block is unpinned and may have been evicted)."""
session, conn, _ = _make_session()
_activate(session, conn)
# Probe hA, flush, and let the peer resolve it to a HIT.
assert session.register_lookup("req-1", b"hA") is None
session.flush_pending_lookups()
conn.enqueue(
{
TYPE_KEY: LookupRespMsg.TYPE,
LookupRespMsg.KV_REQUEST_ID: "req-1",
LookupRespMsg.KEYS: [b"hA"],
LookupRespMsg.HITS: [True],
}
)
session.poll()
assert session.register_lookup("req-1", b"hA") is True
# Fetch consumes the probe.
session.request_blocks(
job_id=1, kv_request_id="req-1", keys=[b"hA"], block_ids=[0]
)
assert b"hA" not in session._client._requests["req-1"].probes
# Re-scheduled probe of the same key is treated as brand-new: it
# returns None and re-queues, so a flush emits a fresh LookupMsg.
assert session.register_lookup("req-1", b"hA") is None
sent_before = len(conn._sent)
session.flush_pending_lookups()
fresh = [m for m in conn._sent[sent_before:] if m[TYPE_KEY] == LookupMsg.TYPE]
assert len(fresh) == 1
assert fresh[0][LookupMsg.KEYS] == [b"hA"]
def test_flush_uses_work_list_not_full_scan(self):
"""Flush drains a work-list rather than scanning every live request.
A request with no newly-registered keys is not revisited: after a
flush the work-list is empty, an idle re-flush sends nothing, and a
subsequent register re-arms exactly the one affected id — even while
an unrelated request stays live in ``_requests``.
"""
session, conn, _ = _make_session()
_activate(session, conn)
client = session._client
# Two requests register keys; both are queued for flush.
session.register_lookup("req-A", b"hA")
session.register_lookup("req-B", b"hB")
assert client._flush_pending == {"req-A", "req-B"}
# Flush drains the work-list even though both requests stay live.
session.flush_pending_lookups()
assert client._flush_pending == set()
assert set(client._requests) == {"req-A", "req-B"}
# An idle re-flush visits nothing and sends no LookupMsg.
sent_before = len(conn._sent)
session.flush_pending_lookups()
assert [
m for m in conn._sent[sent_before:] if m[TYPE_KEY] == LookupMsg.TYPE
] == []
# A new register re-arms only that id.
session.register_lookup("req-A", b"hA2")
assert client._flush_pending == {"req-A"}
def test_separate_lookup_msg_per_kv_request_id(self):
"""Hashes for different kv_request_ids flush as separate LookupMsgs."""
session, conn, _ = _make_session()
_activate(session, conn)
session.register_lookup("req-A", b"h1")
session.register_lookup("req-B", b"h2")
session.register_lookup("req-A", b"h3")
sent_before = len(conn._sent)
session.flush_pending_lookups()
sent = [m for m in conn._sent[sent_before:] if m[TYPE_KEY] == LookupMsg.TYPE]
assert len(sent) == 2
by_req = {m[LookupMsg.KV_REQUEST_ID]: m[LookupMsg.KEYS] for m in sent}
assert sorted(by_req["req-A"]) == [b"h1", b"h3"]
assert by_req["req-B"] == [b"h2"]
def test_multiple_lookup_msgs_across_steps(self):
"""A request's block set may be discovered across scheduler steps:
each step that registers new keys flushes its own LookupMsg for
the same kv_request_id, carrying only the newly-probed keys."""
session, conn, _ = _make_session()
_activate(session, conn)
# Step 1: probe hA, hB.
session.register_lookup("req-1", b"hA")
session.register_lookup("req-1", b"hB")
sent_before = len(conn._sent)
session.flush_pending_lookups()
first = [m for m in conn._sent[sent_before:] if m[TYPE_KEY] == LookupMsg.TYPE]
assert len(first) == 1
assert first[0][LookupMsg.KV_REQUEST_ID] == "req-1"
assert sorted(first[0][LookupMsg.KEYS]) == [b"hA", b"hB"]
# Step 2: a new key is discovered for the same request. The
# in-flight keys from step 1 are not re-sent; a second LookupMsg
# goes out carrying only the newly-probed key.
assert session.register_lookup("req-1", b"hA") is None # in-flight no-op
session.register_lookup("req-1", b"hC")
sent_before = len(conn._sent)
session.flush_pending_lookups()
second = [m for m in conn._sent[sent_before:] if m[TYPE_KEY] == LookupMsg.TYPE]
assert len(second) == 1
assert second[0][LookupMsg.KV_REQUEST_ID] == "req-1"
assert second[0][LookupMsg.KEYS] == [b"hC"]
def test_split_response_resolves_across_messages(self):
"""Producer may answer one LookupMsg's keys across multiple
LookupRespMsgs — pairs are self-describing so each lands."""
session, conn, _ = _make_session()
_activate(session, conn)
session.register_lookup("req-1", b"hA")
session.register_lookup("req-1", b"hB")
session.flush_pending_lookups()
# Two responses, each carrying one of the two keys.
conn.enqueue(
{
TYPE_KEY: LookupRespMsg.TYPE,
LookupRespMsg.KV_REQUEST_ID: "req-1",
LookupRespMsg.KEYS: [b"hA"],
LookupRespMsg.HITS: [True],
}
)
conn.enqueue(
{
TYPE_KEY: LookupRespMsg.TYPE,
LookupRespMsg.KV_REQUEST_ID: "req-1",
LookupRespMsg.KEYS: [b"hB"],
LookupRespMsg.HITS: [False],
}
)
session.poll()
assert session.register_lookup("req-1", b"hA") is True
assert session.register_lookup("req-1", b"hB") is False
def test_finish_request_cancels_pending_lookups(self):
"""finish_request drops every pending lookup for the kv_request_id."""
session, conn, _ = _make_session()
_activate(session, conn)
session.register_lookup("req-1", b"hA")
session.register_lookup("req-1", b"hB")
session.register_lookup("req-2", b"hC")
session.finish_request("req-1")
# req-1 entries gone, req-2 untouched.
assert "req-1" not in session._client._requests
assert b"hC" in session._client._requests["req-2"].probes
def test_finish_after_flushed_lookup_sends_empty_fetch(self):
"""LookupMsg flushed but no FetchMsg sent (all-miss case) →
finish_request emits an empty FetchMsg so the peer can drop its
lookup state and call parent.on_request_finished."""
session, conn, _ = _make_session()
_activate(session, conn)
session.register_lookup("req-1", b"hA")
session.flush_pending_lookups()
sent_before = len(conn._sent)
session.finish_request("req-1")
fetches = [m for m in conn._sent[sent_before:] if m[TYPE_KEY] == FetchMsg.TYPE]
assert len(fetches) == 1
assert fetches[0][FetchMsg.KV_REQUEST_ID] == "req-1"
assert fetches[0][FetchMsg.KEYS] == []
assert fetches[0][FetchMsg.BLOCK_INDEXES] == []
def test_finish_without_flushed_lookup_sends_no_fetch(self):
"""No LookupMsg was ever sent → finish_request must not emit an
empty FetchMsg (the peer has no state to release)."""
session, conn, _ = _make_session()
_activate(session, conn)
# Register but never flush.
session.register_lookup("req-1", b"hA")
sent_before = len(conn._sent)
session.finish_request("req-1")
fetches = [m for m in conn._sent[sent_before:] if m[TYPE_KEY] == FetchMsg.TYPE]
assert fetches == []
def test_finish_after_real_fetch_sends_no_second_fetch(self):
"""A real FetchMsg was already sent for the id → finish_request
must not emit a second (empty) FetchMsg."""
session, conn, _ = _make_session()
_activate(session, conn)
session.register_lookup("req-1", b"hA")
session.flush_pending_lookups()
# Resolve the probe to a HIT before fetching, as the manager only
# loads confirmed hits (an unresolved probe yields RETRY).
conn.enqueue(
{
TYPE_KEY: LookupRespMsg.TYPE,
LookupRespMsg.KV_REQUEST_ID: "req-1",
LookupRespMsg.KEYS: [b"hA"],
LookupRespMsg.HITS: [True],
}
)
session.poll()
session.request_blocks(
job_id=1, kv_request_id="req-1", keys=[b"hA"], block_ids=[7]
)
sent_before = len(conn._sent)
session.finish_request("req-1")
fetches = [m for m in conn._sent[sent_before:] if m[TYPE_KEY] == FetchMsg.TYPE]
assert fetches == []
def test_server_lookup_deferred_until_serve_then_all_misses(self):
"""``poll()`` only enqueues an inbound LookupMsg — no response is
sent until ``serve_external_requests``. With an all-miss parent
the aggregated LookupRespMsg carries the same keys and
``hits=[False, ...]``."""
session, conn, _ = _make_session()
_activate(session, conn)
sent_before = len(conn._sent)
conn.enqueue(
{
TYPE_KEY: LookupMsg.TYPE,
LookupMsg.ROUND_SEQ: 0,
LookupMsg.KV_REQUEST_ID: "req-1",
LookupMsg.KEYS: [b"hX", b"hY", b"hZ"],
}
)
session.poll()
# Dispatch alone must not answer — the parent handle is only valid
# during serve_external_requests.
assert [
m for m in conn._sent[sent_before:] if m[TYPE_KEY] == LookupRespMsg.TYPE
] == []
_serve(session, FakeParent())
resps = [
m for m in conn._sent[sent_before:] if m[TYPE_KEY] == LookupRespMsg.TYPE
]
assert len(resps) == 1
resp = resps[0]
assert resp[LookupRespMsg.KV_REQUEST_ID] == "req-1"
assert resp[LookupRespMsg.KEYS] == [b"hX", b"hY", b"hZ"]
assert resp[LookupRespMsg.HITS] == [False, False, False]
# ---------------------------------------------------------------------------
# Server-side handling of inbound LookupMsg (ParentManager-driven)
#
# poll() only enqueues the LookupMsg; serve_external_requests(parent)
# resolves it. Tests follow the poll() → _serve() pattern.
# ---------------------------------------------------------------------------
def _send_lookup(conn: FakeConnection, kv_request_id: str, keys: list[bytes]):
conn.enqueue(
{
TYPE_KEY: LookupMsg.TYPE,
LookupMsg.ROUND_SEQ: 0,
LookupMsg.KV_REQUEST_ID: kv_request_id,
LookupMsg.KEYS: list(keys),
}
)
def _lookup_resps(conn: FakeConnection, since: int = 0) -> list[dict]:
return [m for m in conn._sent[since:] if m[TYPE_KEY] == LookupRespMsg.TYPE]
class TestServerLookupHandling:
def test_immediate_hits_create_one_store_job(self):
"""All-HIT batch: one create_store_job call with all keys, one
LookupRespMsg with hits=[True]*N, on_request_finished fires at the
end of serve, and `available` is populated for the eventual fetch."""
cb = FakeParent(stored={b"hA": 1, b"hB": 2, b"hC": 3})
session, conn, _ = _make_session()
_activate(session, conn)
sent_before = len(conn._sent)
_send_lookup(conn, "req-1", [b"hA", b"hB", b"hC"])
session.poll()
_serve(session, cb)
resps = _lookup_resps(conn, sent_before)
assert len(resps) == 1
assert resps[0][LookupRespMsg.KEYS] == [b"hA", b"hB", b"hC"]
assert resps[0][LookupRespMsg.HITS] == [True, True, True]
kinds = [c[0] for c in cb.calls]
assert kinds.count("create_store_job") == 1
cs = next(c for c in cb.calls if c[0] == "create_store_job")
assert cs[1] == (b"hA", b"hB", b"hC")
assert cb.calls[-1][0] == "on_request_finished"
# Hits are pinned in outbound state for the upcoming FetchMsg match.
assert set(_srv_outbound(session, "req-1").available) == {
b"hA",
b"hB",
b"hC",
}
def test_all_misses_no_store_job_finish_fires(self):
"""All-MISS batch: no create_store_job call; one LookupRespMsg
with hits=[False]*N; on_request_finished fires at end of serve."""
cb = FakeParent()
session, conn, _ = _make_session()
_activate(session, conn)
sent_before = len(conn._sent)
_send_lookup(conn, "req-1", [b"hA", b"hB"])
session.poll()
_serve(session, cb)
resps = _lookup_resps(conn, sent_before)
assert len(resps) == 1
assert resps[0][LookupRespMsg.HITS] == [False, False]
assert all(c[0] != "create_store_job" for c in cb.calls)
assert cb.calls[-1][0] == "on_request_finished"
def test_mixed_hit_miss_pending_defers_response_until_aggregate(self):
"""HIT/MISS resolutions do not go out on first sight when any
key is still HIT_PENDING / RETRY. The lookup parks until every
key has settled (or the deadline fires), then one
LookupRespMsg carries all keys in wire order."""
cb = FakeParent(
stored={b"hA": 1},
pending={b"hB"},
retry={b"hD"},
)
session, conn, _ = _make_session()
_activate(session, conn)
sent_before = len(conn._sent)
_send_lookup(conn, "req-1", [b"hA", b"hB", b"hC", b"hD"])
session.poll()
_serve(session, cb)
# No LookupRespMsg yet — hB and hD are still pending.
assert _lookup_resps(conn, sent_before) == []
# HIT is still pinned immediately so the eventual FetchMsg matches.
cs_calls = [c for c in cb.calls if c[0] == "create_store_job"]
assert len(cs_calls) == 1
assert cs_calls[0][1] == (b"hA",)
# Lookup is parked; on_request_finished not yet called.
assert all(c[0] != "on_request_finished" for c in cb.calls)
assert len(_srv_lookups(session)) == 1
def test_pending_resolves_then_aggregate_response_fires(self):
"""A HIT_PENDING key that becomes HIT on a later poll releases
the deferred aggregate response: one LookupRespMsg carrying
both keys in wire order, and one create_store_job call per
HIT (the second HIT is pinned when it resolves, not when the
response goes out)."""
cb = FakeParent(stored={b"hA": 1}, pending={b"hB"})
session, conn, _ = _make_session()
_activate(session, conn)
sent_before = len(conn._sent)
_send_lookup(conn, "req-1", [b"hA", b"hB"])
session.poll()
_serve(session, cb)
# No response yet — hB still pending.
assert _lookup_resps(conn, sent_before) == []
# Promote hB.
cb.pending.discard(b"hB")
cb.stored[b"hB"] = 2
# Drive resolver via a second serve_external_requests.
_serve(session, cb)
resps = _lookup_resps(conn, sent_before)
assert len(resps) == 1
assert resps[0][LookupRespMsg.KEYS] == [b"hA", b"hB"]
assert resps[0][LookupRespMsg.HITS] == [True, True]
cs_calls = [c for c in cb.calls if c[0] == "create_store_job"]
assert len(cs_calls) == 2
assert cs_calls[0][1] == (b"hA",)
assert cs_calls[1][1] == (b"hB",)
# on_request_finished fires once after the aggregate resolve.
assert sum(1 for c in cb.calls if c[0] == "on_request_finished") == 1
assert b"hA" in _srv_outbound(session, "req-1").available
assert b"hB" in _srv_outbound(session, "req-1").available
def test_pending_timeout_replies_miss_no_store_job(self):
"""A HIT_PENDING key that stays pending past the batch
``deadline`` is force-MISS and never pinned; the deferred
aggregate response fires with hits=[False]."""
cb = FakeParent(pending={b"hA"})
session, conn, _ = _make_session()
_activate(session, conn)
sent_before = len(conn._sent)
_send_lookup(conn, "req-1", [b"hA"])
session.poll()
_serve(session, cb)
# Initial serve: nothing immediate, lookup parked, no LookupRespMsg.
assert _lookup_resps(conn, sent_before) == []
# Forge the deadline into the past to trigger the timeout branch.
lookup = _srv_lookups(session)[0]
lookup.deadline = time.monotonic() - 0.1
_serve(session, cb)
resps = _lookup_resps(conn, sent_before)
assert len(resps) == 1
assert resps[0][LookupRespMsg.KEYS] == [b"hA"]
assert resps[0][LookupRespMsg.HITS] == [False]
assert all(c[0] != "create_store_job" for c in cb.calls)
assert sum(1 for c in cb.calls if c[0] == "on_request_finished") == 1
def test_finish_request_called_per_lookup_msg_not_per_kv_request_id(self):
"""Two LookupMsgs for the same kv_request_id get distinct ctxs
and two on_request_finished calls (one per batch)."""
cb = FakeParent(stored={b"hA": 1, b"hB": 2})
session, conn, _ = _make_session()
_activate(session, conn)
_send_lookup(conn, "req-1", [b"hA"])
session.poll()
_serve(session, cb)
_send_lookup(conn, "req-1", [b"hB"])
session.poll()
_serve(session, cb)
finish_calls = [c for c in cb.calls if c[0] == "on_request_finished"]
assert len(finish_calls) == 2
# Distinct synthetic req_ids
assert finish_calls[0][1] != finish_calls[1][1]
# Both namespaced under the same kv_request_id
assert ":req-1:" in finish_calls[0][1]
assert ":req-1:" in finish_calls[1][1]
def test_close_returns_open_batch_ctxs_as_failed_serves(self):
"""Tearing the session down with a parked batch returns the
synthetic ctx as a failed serve (no parent handle at teardown) so
the manager can release the TieringManager's state on its next
serve."""
cb = FakeParent(pending={b"hA"})
session, conn, _ = _make_session()
_activate(session, conn)
_send_lookup(conn, "req-1", [b"hA"])
session.poll()
_serve(session, cb)
assert len(_srv_lookups(session)) == 1
assert all(c[0] != "on_request_finished" for c in cb.calls)
result = session.close()
assert len(result.failed_serves) == 1
assert ":req-1:" in result.failed_serves[0].req_id
# close() itself must not call the parent.
assert all(c[0] != "on_request_finished" for c in cb.calls)
def test_wire_finish_drops_pending_batches_for_kv_request_id(self):
"""``ServerRole.finish(kv_request_id)`` drops every parked batch
whose kv_request_id matches and queues its ctx for the next
serve's on_request_finished."""
cb = FakeParent(pending={b"hA", b"hB"})
session, conn, _ = _make_session()
_activate(session, conn)
_send_lookup(conn, "req-1", [b"hA"])
session.poll()
_serve(session, cb)
_send_lookup(conn, "req-2", [b"hB"])
session.poll()
_serve(session, cb)
assert len(_srv_lookups(session)) == 2
session._server.finish("req-1")
# req-1 batch dropped from parked lookups; its ctx queued for release.
remaining_kv_request_ids = {b.kv_request_id for b in _srv_lookups(session)}
assert remaining_kv_request_ids == {"req-2"}
queued = session._server._finished_lookup_ctxs
assert len(queued) == 1
assert ":req-1:" in queued[0].req_id
# The next serve fires on_request_finished exactly once for req-1.
_serve(session, cb)
finish_calls = [c for c in cb.calls if c[0] == "on_request_finished"]
assert len(finish_calls) == 1
assert ":req-1:" in finish_calls[0][1]
def test_incoming_fetch_drops_pending_lookups_for_kv_request_id(self):
"""A peer FetchMsg terminates the lookup phase for its id: parked
lookups with matching kv_request_id are dropped and their
ctx queued for on_request_finished; other kv_request_ids untouched."""
cb = FakeParent(pending={b"hA", b"hB"})
session, conn, _ = _make_session()
_activate(session, conn)
_send_lookup(conn, "req-1", [b"hA"])
session.poll()
_serve(session, cb)
_send_lookup(conn, "req-2", [b"hB"])
session.poll()
_serve(session, cb)
assert len(_srv_lookups(session)) == 2
# Empty FetchMsg: peer signals "lookup phase done" without asking
# for any blocks (the all-miss case).
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [],
FetchMsg.BLOCK_INDEXES: [],
}
)
session.poll()
remaining_kv_request_ids = {lu.kv_request_id for lu in _srv_lookups(session)}
assert remaining_kv_request_ids == {"req-2"}
# Dispatch queues the ctx but does not call the parent yet.
queued = session._server._finished_lookup_ctxs
assert len(queued) == 1
assert ":req-1:" in queued[0].req_id
# The next serve fires on_request_finished exactly once for req-1.
_serve(session, cb)
finish_calls = [c for c in cb.calls if c[0] == "on_request_finished"]
assert len(finish_calls) == 1
assert ":req-1:" in finish_calls[0][1]
def test_lookup_then_fetch_round_trip_emits_store_result(self):
"""End-to-end: lookup pins primary slots → fetch matches them →
NIXL transfer completes → StoreResult surfaces with the
create_store_job's job_id (the engine releases the pin)."""
cb = FakeParent(stored={b"hA": 7, b"hB": 8})
session, conn, transport = _make_session()
_activate(session, conn)
_send_lookup(conn, "req-1", [b"hA", b"hB"])
session.poll()
_serve(session, cb)
cs = next(c for c in cb.calls if c[0] == "create_store_job")
# FakeParent issues monotonic job_ids starting at 1000.
expected_job_id = 1000
# Consumer issues FetchMsg on the resolved hits.
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"hA", b"hB"],
FetchMsg.BLOCK_INDEXES: [20, 21],
}
)
session.poll()
# NIXL write_blocks called with our pinned local block_ids.
assert len(transport._transfers) == 1
_, (_peer, local, remote) = next(iter(transport._transfers.items()))
assert local == [7, 8]
assert remote == [20, 21]
# Drive the transport completion.
transport._poll_done.append(0)
result = session.poll()
store_results = [s for s in result.stores if s.success]
assert any(s.job_id == expected_job_id for s in store_results)
# Sanity: kv mention in synthetic ctx.
assert cs[2].startswith("p2p:")
# ---------------------------------------------------------------------------
# Server-role flows
# ---------------------------------------------------------------------------
class TestServerFlows:
def test_store_then_fetch_matches(self):
"""Blocks stored before fetch demand are matched on demand arrival."""
session, conn, transport = _make_session()
_activate(session, conn)
session.add_stored_blocks("req-1", [b"k1", b"k2"], [0, 1], job_id=1)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [10, 11],
}
)
session.poll()
assert len(transport._transfers) == 1
_, (peer, local, remote) = next(iter(transport._transfers.items()))
assert local == [0, 1]
assert remote == [10, 11]
def test_fetch_then_store_matches(self):
"""Fetch demand registered before store; store fulfills it."""
session, conn, transport = _make_session()
_activate(session, conn)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
}
)
session.poll()
assert len(transport._transfers) == 0
session.add_stored_blocks("req-1", [b"k1"], [3], job_id=1)
assert len(transport._transfers) == 1
def test_transfer_completion_emits_store_result_and_done(self):
"""Completed transfer reports StoreResult and sends TransferDoneMsg."""
session, conn, transport = _make_session()
_activate(session, conn)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
}
)
session.poll()
tid = next(iter(transport._transfers))
transport._poll_done.append(tid)
stores = session.poll().stores
assert StoreResult(job_id=1, success=True) in stores
assert any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
def test_abort_fetch_replies_with_ack(self):
session, conn, _ = _make_session()
_activate(session, conn)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
session.poll()
ack = next(m for m in conn._sent if m[TYPE_KEY] == AbortAckMsg.TYPE)
assert ack[AbortAckMsg.KV_REQUEST_ID] == "req-1"
assert _srv_abort_started(session, "req-1") is None
def test_abort_fetch_defers_ack_when_cancel_pending(self):
"""If cancel(mode='wait') reports still-inflight tids, the ack is
deferred and the abort is parked (abort_started_at set)."""
session, conn, transport = _make_session()
_activate(session, conn)
# Seed an inflight transfer for req-1 that the transport pretends
# cannot be canceled yet.
tid = 42
session._server._inflight_add(
tid,
_InflightXfer(
kv_request_id="req-1",
block_count=1,
job_ids={1},
round=_OutboundRequestState(inflight=1),
),
)
transport._cancel_still_inflight.add(tid)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
session.poll()
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
assert _srv_abort_started(session, "req-1") is not None
# First attempt happens inside _on_abort_fetch; the per-tick
# drain runs again at the end of poll() — both are wait-mode.
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert tid in session._server._inflight # still tracked
def test_abort_fetch_acks_after_drain(self):
"""Once the transport reports the tid as DONE the parked abort
completes and AbortAckMsg is sent."""
session, conn, transport = _make_session()
_activate(session, conn)
tid = 42
session._server._inflight_add(
tid,
_InflightXfer(
kv_request_id="req-1",
block_count=1,
job_ids={1},
round=_OutboundRequestState(inflight=1),
),
)
transport._cancel_still_inflight.add(tid)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
session.poll()
assert _srv_abort_started(session, "req-1") is not None
# Backend finishes draining: transport.poll() will return tid as
# DONE, and the next cancel(mode='wait') call sees it's gone.
transport._cancel_still_inflight.discard(tid)
transport._poll_done.append(tid)
session.poll()
ack = next(m for m in conn._sent if m[TYPE_KEY] == AbortAckMsg.TYPE)
assert ack[AbortAckMsg.KV_REQUEST_ID] == "req-1"
assert _srv_abort_started(session, "req-1") is None
assert tid not in session._server._inflight
def test_abort_fetch_never_force_cancels_after_timeout(self):
"""Past the drain deadline a transfer that is still running is not
force-released and no ack is sent: releasing the handle or removing
the peer does not stop a NIXL WRITE, so the ack would let the peer
reuse blocks that are still being written."""
session, conn, transport = _make_session()
_activate(session, conn)
tid = 42
session._server._inflight_add(
tid,
_InflightXfer(
kv_request_id="req-1",
block_count=1,
job_ids={1},
round=_OutboundRequestState(inflight=1),
),
)
transport._cancel_still_inflight.add(tid)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
session.poll()
assert _srv_abort_started(session, "req-1") is not None
peers = dict(transport._remote_peers)
# Backdate past the drain deadline.
session._server._pending_aborts[("req-1", 0)] = (
time.monotonic() - _CANCEL_DRAIN_TIMEOUT_S - 1.0
)
transport._cancel_calls.clear()
for _ in range(3):
session.poll()
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
assert tid in session._server._inflight
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert transport._remote_peers == peers
# Once NIXL reports the transfer terminal, the ack follows.
transport._cancel_still_inflight.discard(tid)
transport._poll_done.append(tid)
session.poll()
acks = [m for m in conn._sent if m[TYPE_KEY] == AbortAckMsg.TYPE]
assert len(acks) == 1
assert _srv_abort_started(session, "req-1") is None
assert tid not in session._server._inflight
def test_abort_fetch_idempotent_while_draining(self):
"""Receiving AbortFetchMsg twice for the same kv_request_id
keeps a single pending entry and produces a single ack."""
session, conn, transport = _make_session()
_activate(session, conn)
tid = 42
session._server._inflight_add(
tid,
_InflightXfer(
kv_request_id="req-1",
block_count=1,
job_ids={1},
round=_OutboundRequestState(inflight=1),
),
)
transport._cancel_still_inflight.add(tid)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
session.poll()
first_started_at = _srv_abort_started(session, "req-1")
# Second AbortFetchMsg for the same kv_request_id while still
# draining must not reset the deadline.
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.ROUND_SEQ: 0,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
}
)
session.poll()
assert _srv_abort_started(session, "req-1") == first_started_at
# Now let the drain succeed and confirm exactly one ack ever.
transport._cancel_still_inflight.discard(tid)
transport._poll_done.append(tid)
session.poll()
acks = [m for m in conn._sent if m[TYPE_KEY] == AbortAckMsg.TYPE]
assert len(acks) == 1
assert acks[0][AbortAckMsg.KV_REQUEST_ID] == "req-1"
def test_store_timeout(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
# Backdate.
session._server._store_jobs[1] = time.monotonic() - 60.0
stores = session.poll().stores
assert StoreResult(job_id=1, success=False) in stores
@pytest.mark.parametrize("cancel_pending", [False, True])
def test_store_timeout_cancels_stuck_transfer_after_confirmation(
self, cancel_pending, monkeypatch
):
# Retry on every poll; pacing is covered by the *_paced tests.
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
}
)
session.poll()
tid = next(iter(transport._transfers))
session._server._store_jobs[1] = time.monotonic() - 60.0
if cancel_pending:
transport._cancel_still_inflight.add(tid)
for _ in range(3):
assert session.poll().stores == []
assert 1 in session._server._store_jobs
assert tid in session._server._inflight
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert transport._cancel_calls
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert tid not in session._server._inflight
assert tid not in transport._transfers
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
assert len(terminal) == 1
assert terminal[0][TransferDoneMsg.SUCCESS] is False
assert session.poll().stores == []
@pytest.mark.parametrize("success", [True, False])
def test_store_timeout_keeps_pins_until_cancel_or_late_terminal(self, success):
session, conn, transport = _make_session()
_activate(session, conn)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
session.add_stored_blocks("req-2", [b"k2"], [1], job_id=2)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
}
)
session.poll()
tid = next(iter(transport._transfers))
transport._cancel_still_inflight.add(tid)
for jid in (1, 2):
session._server._store_jobs[jid] = time.monotonic() - 60.0
# A StoreResult releases the primary slot. Only the job still
# waiting for a fetch may time out while the transfer is active.
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
assert session.poll().stores == []
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
completed = transport._poll_done if success else transport._poll_failed
completed.append(tid)
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
assert len(terminal) == 1
assert terminal[0][TransferDoneMsg.SUCCESS] is False
assert session.poll().stores == []
@pytest.mark.parametrize("success", [True, False, None])
def test_store_timeout_waits_for_other_transfers_in_round(
self, success, monkeypatch
):
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [5, 6],
}
)
session.poll()
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
session.add_stored_blocks("req-1", [b"k2"], [1], job_id=2)
first, second = list(transport._transfers)
session._server._store_jobs[1] = time.monotonic() - 60.0
transport._cancel_still_inflight.add(second)
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert first not in session._server._inflight
assert second in session._server._inflight
assert 2 in session._server._store_jobs
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
# Additional supply must not restart the timed-out round.
session.add_stored_blocks("req-1", [b"k3"], [2], job_id=3)
assert session.poll().stores == [StoreResult(job_id=3, success=False)]
assert set(session._server._inflight) == {second}
if success is None:
transport._cancel_still_inflight.clear()
else:
completed = transport._poll_done if success else transport._poll_failed
completed.append(second)
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
assert len(terminal) == 1
assert terminal[0][TransferDoneMsg.SUCCESS] is False
assert session.poll().stores == []
def test_failed_transfer_keeps_sibling_pins_until_terminal(self, monkeypatch):
"""A failed transfer fails its round, but a sibling transfer that may
still be writing is not force-cancelled: its pin and the round's
TransferDoneMsg wait until it is terminal."""
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [5, 6],
}
)
session.poll()
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
session.add_stored_blocks("req-1", [b"k2"], [1], job_id=2)
first, second = list(transport._transfers)
transport._cancel_still_inflight.add(second)
transport._poll_failed.append(first)
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
for _ in range(3):
assert session.poll().stores == []
assert second in session._server._inflight
assert 2 in session._server._store_jobs
assert transport._cancel_calls
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
assert len(terminal) == 1
assert terminal[0][TransferDoneMsg.SUCCESS] is False
assert session.poll().stores == []
def test_abort_during_store_timeout_keeps_unconfirmed_pins(self, monkeypatch):
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
}
)
session.poll()
tid = next(iter(transport._transfers))
transport._cancel_still_inflight.add(tid)
session._server._store_jobs[1] = time.monotonic() - 60.0
assert session.poll().stores == []
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
AbortFetchMsg.ROUND_SEQ: 0,
}
)
assert session.poll().stores == []
session._server._pending_aborts[("req-1", 0)] = time.monotonic() - 60.0
assert session.poll().stores == []
assert 1 in session._server._store_jobs
assert tid in session._server._inflight
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
assert session.poll().stores == []
@staticmethod
def _fake_clock(monkeypatch) -> list[float]:
clock = [1000.0]
monkeypatch.setattr(
server_mod, "time", SimpleNamespace(monotonic=lambda: clock[0])
)
return clock
@staticmethod
def _submit_unconfirmed_transfer(session, conn, transport) -> int:
"""Start one transfer for job 1 whose wait-cancel never confirms."""
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
}
)
session.poll()
tid = next(iter(transport._transfers))
transport._cancel_still_inflight.add(tid)
return tid
@staticmethod
def _wait_cancels(transport) -> int:
return sum(1 for _, mode in transport._cancel_calls if mode == "wait")
def test_store_timeout_cancel_retry_is_paced(self, monkeypatch):
clock = self._fake_clock(monkeypatch)
session, conn, transport = _make_session()
_activate(session, conn)
tid = self._submit_unconfirmed_transfer(session, conn, transport)
session._server._store_jobs[1] = clock[0] - 60.0
assert session.poll().stores == []
assert self._wait_cancels(transport) == 1
start = clock[0]
for dt in (0.0, 0.05, 0.099):
clock[0] = start + dt
for _ in range(5):
assert session.poll().stores == []
assert self._wait_cancels(transport) == 1
clock[0] = start + server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
assert session.poll().stores == []
assert self._wait_cancels(transport) == 2
transport._cancel_still_inflight.clear()
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert self._wait_cancels(transport) == 3
assert tid not in session._server._inflight
def test_abort_drain_cancel_retry_is_paced(self, monkeypatch):
clock = self._fake_clock(monkeypatch)
session, conn, transport = _make_session()
_activate(session, conn)
self._submit_unconfirmed_transfer(session, conn, transport)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
AbortFetchMsg.ROUND_SEQ: 0,
}
)
# The first drain attempt is immediate.
session.poll()
assert self._wait_cancels(transport) == 1
for _ in range(5):
session.poll()
assert self._wait_cancels(transport) == 1
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
session.poll()
assert self._wait_cancels(transport) == 2
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
session.poll()
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
def test_unconfirmed_store_cancel_never_acks_or_tears_down(self, monkeypatch):
"""A cancellation that is never confirmed must not release the pin,
send AbortAck, force-cancel, or remove the peer, however long it
stays pending. AbortAck follows the release once it is confirmed."""
clock = self._fake_clock(monkeypatch)
session, conn, transport = _make_session()
_activate(session, conn)
peers = dict(transport._remote_peers)
tid = self._submit_unconfirmed_transfer(session, conn, transport)
session._server._store_jobs[1] = clock[0] - 60.0
assert session.poll().stores == []
abort = {
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
AbortFetchMsg.ROUND_SEQ: 0,
}
started_at = clock[0]
conn.enqueue(dict(abort))
assert session.poll().stores == []
# A duplicate abort neither resets the drain start nor adds a cancel.
conn.enqueue(dict(abort))
assert session.poll().stores == []
assert session._server._pending_aborts[("req-1", 0)] == started_at
# The store sweep and the abort drain share the round's pacing.
assert self._wait_cancels(transport) == 1
ticks = 50
for step in range(1, ticks + 1):
clock[0] = started_at + step * 10.0
assert session.poll().stores == []
assert clock[0] - started_at > _CANCEL_DRAIN_TIMEOUT_S
assert self._wait_cancels(transport) == 1 + ticks
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert 1 in session._server._store_jobs
assert tid in session._server._inflight
assert transport._remote_peers == peers
assert not any(
m[TYPE_KEY] in (AbortAckMsg.TYPE, TransferDoneMsg.TYPE) for m in conn._sent
)
transport._cancel_still_inflight.clear()
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert 1 not in session._server._store_jobs
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
assert session.poll().stores == []
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
def test_abort_keeps_pins_only_of_jobs_being_written(self, monkeypatch):
"""An abort fails the round's jobs at once, except those an inflight
transfer still reads; their pins and the ack wait for the transfer."""
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
tid = self._submit_unconfirmed_transfer(session, conn, transport)
# Job 2 supplies a block nobody demanded: no transfer reads it.
session.add_stored_blocks("req-1", [b"k2"], [1], job_id=2)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
AbortFetchMsg.ROUND_SEQ: 0,
}
)
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
for _ in range(3):
assert session.poll().stores == []
assert 1 in session._server._store_jobs
assert tid in session._server._inflight
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
assert session.poll().stores == []
@pytest.mark.parametrize("success", [True, False])
def test_abort_acks_after_late_terminal_without_transfer_done(
self, success, monkeypatch
):
"""A transfer that finishes on its own after the abort releases its
pin and lets the parked abort ack; the aborted round sends no
TransferDoneMsg."""
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
tid = self._submit_unconfirmed_transfer(session, conn, transport)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
AbortFetchMsg.ROUND_SEQ: 0,
}
)
assert session.poll().stores == []
assert 1 in session._server._store_jobs
completed = transport._poll_done if success else transport._poll_failed
completed.append(tid)
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
assert session.poll().stores == []
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
def test_fetch_after_symmetric_supply_timed_out_reads_nothing(self):
"""A lookup pin whose store job timed out has been reported failed,
so the engine unpinned the primary slot. A late FetchMsg must not
read that slot: the round fails fast instead."""
parent = FakeParent(stored={b"hA": 7})
session, conn, transport = _make_session()
_activate(session, conn)
_send_lookup(conn, "req-1", [b"hA"])
session.poll()
_serve(session, parent)
(job_id,) = session._server._store_jobs
session._server._store_jobs[job_id] = time.monotonic() - 60.0
assert StoreResult(job_id=job_id, success=False) in session.poll().stores
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"hA"],
FetchMsg.BLOCK_INDEXES: [20],
}
)
session.poll()
assert transport._transfers == {}
dones = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
assert dones and dones[-1][TransferDoneMsg.SUCCESS] is False
# ---------------------------------------------------------------------------
# finish_request server-role early-fail flow
# ---------------------------------------------------------------------------
class TestFinishRequestServerSide:
def _last_transfer_done(self, conn: FakeConnection) -> dict | None:
for msg in reversed(conn._sent):
if msg[TYPE_KEY] == TransferDoneMsg.TYPE:
return msg
return None
def test_no_inflight_unmatched_demand_sends_failure(self):
"""finish_request with unmatched demand and no inflight ->
immediate TransferDoneMsg(success=False); _outbound cleared."""
session, conn, _ = _make_session()
_activate(session, conn)
# Decoder demanded a block we never stored.
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
}
)
session.poll()
assert _srv_outbound(session, "req-1") is not None
session.finish_request("req-1")
msg = self._last_transfer_done(conn)
assert msg is not None
assert msg[TransferDoneMsg.KV_REQUEST_ID] == "req-1"
assert msg[TransferDoneMsg.SUCCESS] is False
assert _srv_outbound(session, "req-1") is None
def test_with_inflight_defers_then_fires_on_last_transfer(self):
"""finish_request with inflight defers; last transfer fires the
early-fail message and clears _outbound."""
session, conn, transport = _make_session()
_activate(session, conn)
# Demand 2 blocks; we store 1 (kicks one inflight transfer).
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [10, 11],
}
)
session.poll()
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
assert len(transport._transfers) == 1
# finish_request while inflight: no early-fail yet.
before = len(conn._sent)
session.finish_request("req-1")
assert len(conn._sent) == before
assert _srv_outbound(session, "req-1") is not None
assert _srv_outbound(session, "req-1").finishing
# Last inflight settles -> early-fail fires.
tid = next(iter(transport._transfers))
transport._poll_done.append(tid)
session.poll()
msg = self._last_transfer_done(conn)
assert msg is not None
assert msg[TransferDoneMsg.KV_REQUEST_ID] == "req-1"
assert msg[TransferDoneMsg.SUCCESS] is False
assert _srv_outbound(session, "req-1") is None
def test_full_demand_satisfied_still_sends_success(self):
"""finish_request must not override a fully-satisfied transfer:
when remaining hits 0, success=True still fires."""
session, conn, transport = _make_session()
_activate(session, conn)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [10],
}
)
session.poll()
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
# Mark finishing (e.g., on_request_finished racing with the last
# store) — but all demand is satisfied.
session.finish_request("req-1")
tid = next(iter(transport._transfers))
transport._poll_done.append(tid)
session.poll()
msg = next(m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE)
assert msg[TransferDoneMsg.SUCCESS] is True
def test_prefiller_first_finish_before_fetch(self):
"""Prefiller-first: finish_request runs before the decoder's
fetch arrives. State is held until fetch, then
finalized — success=True if all demand was matched against
available blocks, else success=False."""
# Case A: all demand satisfied by available blocks.
session, conn, transport = _make_session()
_activate(session, conn)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
# finish_request first — no demand received yet -> defer.
session.finish_request("req-1")
assert _srv_outbound(session, "req-1") is not None
# Fetch arrives now: demand fully satisfied by available.
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [10],
}
)
session.poll()
# The transfer was inflight; finalize it.
tid = next(iter(transport._transfers))
transport._poll_done.append(tid)
session.poll()
msg = next(m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE)
assert msg[TransferDoneMsg.SUCCESS] is True
# Case B: demand exceeds available -> early-fail fires from fetch.
session, conn, transport = _make_session()
_activate(session, conn)
session.add_stored_blocks("req-2", [b"k1"], [0], job_id=2)
session.finish_request("req-2")
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-2",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [10, 11],
}
)
session.poll()
# Inflight for k1 still in flight; nothing yet for the early-fail
# — the same code path will fire from _collect_store_results.
tid = next(iter(transport._transfers))
transport._poll_done.append(tid)
session.poll()
msg = next(m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE)
assert msg[TransferDoneMsg.SUCCESS] is False
assert _srv_outbound(session, "req-2") is None
def test_unknown_request_is_noop(self):
session, conn, _ = _make_session()
_activate(session, conn)
before = len(conn._sent)
session.finish_request("never-existed")
assert len(conn._sent) == before
def test_finish_request_no_inflight_emits_store_failure(self):
"""finish_request with a stored-but-unmatched job and no inflight ->
TransferDoneMsg(success=False) AND deferred
StoreResult(success=False) for the submit_store'd job, instead of
the 30s _STORE_TIMEOUT_S path."""
session, conn, _ = _make_session()
_activate(session, conn)
# Decoder demanded b"demand"; we never stored it.
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"demand"],
FetchMsg.BLOCK_INDEXES: [5],
}
)
session.poll()
# We did submit_store a different block — goes to available, never
# matches demand. Without the shortcut, job 42 sits in _store_jobs
# for _STORE_TIMEOUT_S.
session.add_stored_blocks("req-1", [b"unrelated"], [0], job_id=42)
assert _srv_outbound(session, "req-1").pending_job_ids == {42}
session.finish_request("req-1")
# Peer notified immediately with success=False (remaining > 0).
msg = self._last_transfer_done(conn)
assert msg is not None
assert msg[TransferDoneMsg.SUCCESS] is False
assert _srv_outbound(session, "req-1") is None
# Local store job surfaces on the next poll, success=False.
stores = session.poll().stores
assert StoreResult(job_id=42, success=False) in stores
assert 42 not in session._server._store_jobs
def test_finish_request_remaining_zero_emits_success_via_inflight(self):
"""Deferred-via-inflight path: finish_request with inflight, last
transfer drains remaining to 0 -> TransferDoneMsg(success=True)
AND StoreResult(success=True)."""
session, conn, transport = _make_session()
_activate(session, conn)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [10],
}
)
session.poll()
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=7)
# finish_request races with the inflight transfer.
session.finish_request("req-1")
assert _srv_outbound(session, "req-1") is not None # deferred
# Last inflight completes -> _finalize_outbound(success=True) fires.
tid = next(iter(transport._transfers))
transport._poll_done.append(tid)
stores = session.poll().stores
msg = self._last_transfer_done(conn)
assert msg is not None
assert msg[TransferDoneMsg.SUCCESS] is True
assert StoreResult(job_id=7, success=True) in stores
assert _srv_outbound(session, "req-1") is None
def test_write_blocks_failure_finalizes_with_failure(self):
"""write_blocks returning None must not leave the request hanging.
The matched blocks are gone from req.demanded but no inflight
will satisfy them, so remaining > 0 forever. Setting finishing
and calling _finalize_outbound(success=False) immediately (no
other inflight) tells the peer + emits StoreResult(success=False)
without waiting on _STORE_TIMEOUT_S or _LOAD_TIMEOUT_S.
"""
session, conn, transport = _make_session()
_activate(session, conn)
# Decoder demands b"k1".
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [10],
}
)
session.poll()
# Force write_blocks to fail on the next call.
transport.write_blocks = lambda *a, **kw: None # type: ignore[method-assign]
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=42)
# Outbound was finalized immediately (no other inflight).
assert _srv_outbound(session, "req-1") is None
# Peer notified with success=False.
msg = next(m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE)
assert msg[TransferDoneMsg.KV_REQUEST_ID] == "req-1"
assert msg[TransferDoneMsg.SUCCESS] is False
# Local store job surfaces on the next poll.
stores = session.poll().stores
assert StoreResult(job_id=42, success=False) in stores
assert 42 not in session._server._store_jobs
def test_partial_match_completes_in_two_rounds(self):
"""Peer demand for [k1, k2, k3]; first round only k1 is available,
second round adds k2 and k3. Each round transfers what's matched
and the request finalizes with success once remaining hits zero.
"""
session, conn, transport = _make_session()
_activate(session, conn)
# Peer fetches three blocks.
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2", b"k3"],
FetchMsg.BLOCK_INDEXES: [10, 11, 12],
}
)
session.poll()
# Demand registered, no matches yet.
assert session._server._inflight == {}
outbound = _srv_outbound(session, "req-1")
assert outbound.remaining == 3
assert set(outbound.demanded.keys()) == {b"k1", b"k2", b"k3"}
# Round 1: only k1 is stored locally.
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=100)
# One inflight transfer for k1.
assert len(session._server._inflight) == 1
tid_1 = next(iter(session._server._inflight))
assert transport._transfers[tid_1][1] == [0]
assert transport._transfers[tid_1][2] == [10]
# k2 and k3 still demanded.
assert set(outbound.demanded.keys()) == {b"k2", b"k3"}
# Transfer 1 completes.
transport._poll_done.append(tid_1)
stores = session.poll().stores
assert StoreResult(job_id=100, success=True) in stores
assert session._server._inflight == {}
assert outbound.remaining == 2
# Not yet finalized — still 2 blocks demanded.
assert _srv_outbound(session, "req-1") is not None
# Round 2: k2 and k3 arrive together.
session.add_stored_blocks("req-1", [b"k2", b"k3"], [1, 2], job_id=200)
assert len(session._server._inflight) == 1
tid_2 = next(iter(session._server._inflight))
assert tid_2 != tid_1
assert sorted(transport._transfers[tid_2][1]) == [1, 2]
# Transfer 2 completes — request now fully satisfied.
transport._poll_done.append(tid_2)
stores = session.poll().stores
assert StoreResult(job_id=200, success=True) in stores
# _finalize_outbound fired — request gone, peer notified with success.
assert _srv_outbound(session, "req-1") is None
done = next(m for m in conn._sent if m.get(TYPE_KEY) == TransferDoneMsg.TYPE)
assert done[TransferDoneMsg.KV_REQUEST_ID] == "req-1"
assert done[TransferDoneMsg.SUCCESS] is True
def test_write_blocks_failure_finalizes_after_last_inflight_completes(self):
"""write_blocks returns None on a SECOND match while a first transfer
is still inflight. The request should NOT finalize until the inflight
completes, then the elif branch in collect_results
(``finishing and not _has_inflight_for(...)``) finalizes it as failure.
"""
session, conn, transport = _make_session()
_activate(session, conn)
# Peer demands two blocks.
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [10, 11],
}
)
session.poll()
# Round 1: k1 transfers cleanly.
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=100)
assert len(session._server._inflight) == 1
tid_1 = next(iter(session._server._inflight))
outbound = _srv_outbound(session, "req-1")
assert outbound.remaining == 2 # decrement happens on completion
assert outbound.finishing is False
# Round 2: write_blocks fails for k2 while transfer_1 is still inflight.
transport.write_blocks = lambda *a, **kw: None # type: ignore[method-assign]
session.add_stored_blocks("req-1", [b"k2"], [1], job_id=200)
# No new transfer was registered.
assert list(session._server._inflight.keys()) == [tid_1]
# Marked finishing, but NOT finalized yet (transfer_1 still inflight).
assert outbound.finishing is True
assert _srv_outbound(session, "req-1") is not None
done_msgs = [m for m in conn._sent if m.get(TYPE_KEY) == TransferDoneMsg.TYPE]
assert done_msgs == []
# Transfer 1 completes — now ``_has_inflight_for("req-1")`` is False
# and the elif branch in collect_results fires _finalize(success=False).
# Both the k1 success and the deferred k2 failure surface in this poll.
transport._poll_done.append(tid_1)
assert session.poll().stores == [
StoreResult(job_id=100, success=True),
StoreResult(job_id=200, success=False),
]
# Outbound state cleaned up; peer notified with success=False.
assert _srv_outbound(session, "req-1") is None
done = next(m for m in conn._sent if m.get(TYPE_KEY) == TransferDoneMsg.TYPE)
assert done[TransferDoneMsg.KV_REQUEST_ID] == "req-1"
assert done[TransferDoneMsg.SUCCESS] is False
assert session.poll().stores == []
# ---------------------------------------------------------------------------
# Bidirectional — the case the unification is meant to fix
# ---------------------------------------------------------------------------
class TestBidirectional:
def test_session_handles_both_roles_concurrently(self):
"""Single session simultaneously serves a fetch and completes a load.
This is the regression test for the unification: with the old
split design the inbound FetchMsg would be dispatched to a
client-only session and dropped (or a server-only session would
miss the TransferDoneMsg). One unified session handles both.
"""
session, conn, transport = _make_session()
_activate(session, conn)
# Server role: the peer fetches a block from us.
session.add_stored_blocks("req-srv", [b"served"], [0], job_id=100)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-srv",
FetchMsg.KEYS: [b"served"],
FetchMsg.BLOCK_INDEXES: [7],
}
)
# Client role: we ask the peer for a different block.
session.request_blocks(
job_id=200, kv_request_id="req-cli", keys=[b"loaded"], block_ids=[3]
)
# Both flows progress in the same poll.
session.poll()
# Server side: write_blocks was submitted.
assert len(transport._transfers) == 1
# Client side: the lookup was sent.
assert any(
m[TYPE_KEY] == FetchMsg.TYPE and m[FetchMsg.KV_REQUEST_ID] == "req-cli"
for m in conn._sent
)
# Peer now signals: server-side transfer completes AND a
# TransferDoneMsg arrives for our client-side request, all in
# one batch on the same connection.
tid = next(iter(transport._transfers))
transport._poll_done.append(tid)
conn.enqueue(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-cli",
TransferDoneMsg.SUCCESS: True,
}
)
result_ = session.poll()
loads = result_.loads
stores = result_.stores
assert LoadResult(job_id=200, kv_request_id="req-cli", success=True) in loads
assert StoreResult(job_id=100, success=True) in stores
# ---------------------------------------------------------------------------
# Pending sessions (no connection yet)
# ---------------------------------------------------------------------------
class TestPendingSession:
def test_pending_session_buffers_stored_blocks(self):
"""Pending session accepts add_stored_blocks but cannot send."""
transport = FakeDataTransport()
session = P2PSession(
peer_id="peer:8000",
local_id="local:9000",
transport=transport,
local_block_len=4096,
local_hash_seed=_DEFAULT_HASH_SEED,
conn=None,
)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
result_ = session.poll()
loads = result_.loads
stores = result_.stores
assert loads == []
assert stores == []
assert not session.connected
assert session.alive
def test_attach_connection_sends_connect(self):
"""attach_connection triggers our ConnectMsg send."""
transport = FakeDataTransport()
session = P2PSession(
peer_id="peer:8000",
local_id="local:9000",
transport=transport,
local_block_len=4096,
local_hash_seed=_DEFAULT_HASH_SEED,
conn=None,
)
conn = FakeConnection(peer_id="peer:8000")
session.attach_connection(conn)
assert conn._sent
assert conn._sent[0][TYPE_KEY] == ConnectMsg.TYPE
def test_attach_connection_twice_raises(self):
"""attach_connection on an already-connected session raises."""
session, conn, _ = _make_session()
with pytest.raises(ValueError, match="already connected"):
session.attach_connection(FakeConnection())
def test_pending_close_returns_pending_stores(self):
"""Closing a pending session reports buffered stores as failed."""
transport = FakeDataTransport()
session = P2PSession(
peer_id="peer:8000",
local_id="local:9000",
transport=transport,
local_block_len=4096,
local_hash_seed=_DEFAULT_HASH_SEED,
conn=None,
)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
session.add_stored_blocks("req-2", [b"k2"], [1], job_id=2)
result = session.close()
assert result.failed_jobs == []
assert result.failed_req_ids == []
assert set(result.failed_stores) == {1, 2}
assert result.failed_serves == []
# ---------------------------------------------------------------------------
# Disconnect
# ---------------------------------------------------------------------------
class TestDisconnect:
def test_disconnect_marks_session_dead(self):
session, conn, _ = _make_session()
_activate(session, conn)
conn.enqueue({TYPE_KEY: DisconnectMsg.TYPE})
session.poll()
assert not session.alive
def test_close_returns_pending_loads_and_stores(self):
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(1, "req-1", [b"k"], [0])
session.request_blocks(2, "req-2", [b"k"], [0])
session.add_stored_blocks("req-srv", [b"k"], [0], job_id=10)
result = session.close()
assert set(result.failed_jobs) == {1, 2}
assert set(result.failed_req_ids) == {"req-1", "req-2"}
assert set(result.failed_stores) == {10}
assert result.failed_serves == []
def test_send_failure_marks_connection_dead(self):
"""A raising send must mark the connection dead, not silently drop
the message — otherwise the session lingers alive, is never reaped,
and in-flight lookups/loads toward the dead peer hang forever."""
session, conn, _ = _make_session()
_activate(session, conn)
assert session.alive
conn.fail_send = True
# request_blocks flushes a FetchMsg synchronously via _do_send.
session.request_blocks(1, "req-1", [b"k"], [0])
assert not session.alive
def test_close_surfaces_inflight_lookups(self):
"""close() reports kv_request_ids whose symmetric-P2P probe is still
unresolved; resolved probes are not reported (their answer is in)."""
session, conn, _ = _make_session()
_activate(session, conn)
session.register_lookup("req-hit", b"hA")
session.register_lookup("req-inflight", b"hB")
session.flush_pending_lookups()
# Only req-hit is answered; req-inflight stays in flight.
conn.enqueue(
{
TYPE_KEY: LookupRespMsg.TYPE,
LookupRespMsg.KV_REQUEST_ID: "req-hit",
LookupRespMsg.KEYS: [b"hA"],
LookupRespMsg.HITS: [True],
}
)
session.poll()
result = session.close()
assert result.failed_jobs == []
assert result.failed_req_ids == ["req-inflight"]
# ---------------------------------------------------------------------------
# Adversarial / malformed messages
# ---------------------------------------------------------------------------
class TestAdversarial:
def test_unknown_message_type_logged(self):
session, conn, _ = _make_session()
_activate(session, conn)
conn.enqueue({TYPE_KEY: "evil_command"})
result_ = session.poll()
loads = result_.loads
stores = result_.stores
assert loads == []
assert stores == []
def test_empty_message(self):
session, conn, _ = _make_session()
_activate(session, conn)
conn.enqueue({})
result_ = session.poll()
loads = result_.loads
stores = result_.stores
assert loads == []
assert stores == []
def test_non_dict_message(self):
session, conn, _ = _make_session()
_activate(session, conn)
conn._inbox.append(42)
result_ = session.poll()
loads = result_.loads
stores = result_.stores
assert loads == []
assert stores == []
def test_fetch_mismatched_lengths(self):
session, conn, transport = _make_session()
_activate(session, conn)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-bad",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [1],
}
)
session.poll()
assert len(transport._transfers) == 0
# Protocol violation: session disconnects immediately so the peer
# can't keep wedging us with malformed traffic.
assert not session.alive
assert any(m[TYPE_KEY] == DisconnectMsg.TYPE for m in conn._sent)
def test_transfer_done_missing_kv_request_id(self):
session, conn, _ = _make_session()
_activate(session, conn)
conn.enqueue({TYPE_KEY: TransferDoneMsg.TYPE, TransferDoneMsg.SUCCESS: True})
loads = session.poll().loads
assert loads == []
assert not session.alive
assert any(m[TYPE_KEY] == DisconnectMsg.TYPE for m in conn._sent)
def test_duplicate_connect_ack(self):
session, conn, _ = _make_session()
_activate(session, conn)
conn.enqueue({TYPE_KEY: ConnectAckMsg.TYPE, ConnectAckMsg.PEER_ID: "peer:8000"})
session.poll()
assert session.ready
class TestDispatchErrorHandling:
"""Errors raised by message handlers split into two classes:
- Protocol-contract violations from the peer (ValueError) → disconnect
on the first occurrence; retrying won't help and may corrupt state.
- Anything else is treated as an internal bug: log loudly, count, and
only disconnect once errors arrive in a tight burst. A successful
dispatch in between resets the counter.
"""
def test_value_error_disconnects_on_first_occurrence(self):
"""A FetchMsg that fails validate() raises ValueError and must
terminate the session immediately, with a DisconnectMsg sent."""
session, conn, _ = _make_session()
_activate(session, conn)
# length mismatch → FetchMsg.validate raises ValueError
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-bad",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [1],
}
)
session.poll()
assert not session.alive
assert any(m[TYPE_KEY] == DisconnectMsg.TYPE for m in conn._sent)
def test_transfer_done_missing_field_disconnects(self):
"""Same contract for a malformed TransferDoneMsg from the peer."""
session, conn, _ = _make_session()
_activate(session, conn)
conn.enqueue({TYPE_KEY: TransferDoneMsg.TYPE, TransferDoneMsg.SUCCESS: True})
session.poll()
assert not session.alive
def test_internal_error_does_not_disconnect_once(self):
"""A non-ValueError raised by a handler is treated as an internal
bug: counter increments, session stays alive on a single hit."""
session, conn, _ = _make_session()
_activate(session, conn)
def _boom(*args, **kwargs):
raise RuntimeError("simulated internal bug")
session._server.on_fetch = _boom
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [0],
}
)
session.poll()
assert session.alive
assert session._dispatch_error_count == 1
def test_internal_error_threshold_disconnects(self):
"""Once consecutive non-protocol errors hit the threshold, the
session tears down via _protocol_error."""
session, conn, _ = _make_session()
_activate(session, conn)
def _boom(*args, **kwargs):
raise RuntimeError("simulated internal bug")
session._server.on_fetch = _boom
for _ in range(_MAX_CONSECUTIVE_DISPATCH_ERRORS):
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [0],
}
)
session.poll()
assert not session.alive
assert any(m[TYPE_KEY] == DisconnectMsg.TYPE for m in conn._sent)
def test_internal_error_counter_resets_on_success(self):
"""A successful dispatch between errors prevents the threshold
from being reached."""
session, conn, _ = _make_session()
_activate(session, conn)
original_on_fetch = session._server.on_fetch
def _boom(*args, **kwargs):
raise RuntimeError("simulated internal bug")
# Alternate (boom, success) (_MAX-1) times: counter rises to 1
# then resets to 0 each cycle, never reaching the threshold.
for _ in range(_MAX_CONSECUTIVE_DISPATCH_ERRORS - 1):
session._server.on_fetch = _boom
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [0],
}
)
session.poll()
session._server.on_fetch = original_on_fetch
# A benign no-op message (unknown type) dispatches cleanly
# and resets the consecutive-error counter.
conn.enqueue({TYPE_KEY: "unknown_for_test"})
session.poll()
assert session.alive
assert session._dispatch_error_count == 0
class TestInflightPerReqInvariant:
"""Per-request `inflight_tids` is the O(1) replacement for the
previous O(N) scan in `_has_inflight_for`. These tests check that
every mutation site keeps the set in sync with `_inflight` and
that the lookup is correct under high fan-out.
"""
def test_invariant_holds_through_lifecycle(self):
"""Run a full submit→complete sequence for two concurrent
kv_request_ids and assert the counter matches `_inflight` at
every observable step, including the empty-after-finish case."""
session, conn, transport = _make_session()
_activate(session, conn)
def _invariant_holds() -> bool:
return _srv_total_inflight(session) == len(session._server._inflight)
assert _invariant_holds()
# Two requests, two blocks each, all dispatched in one batch.
session.add_stored_blocks("req-A", [b"a1", b"a2"], [0, 1], job_id=10)
session.add_stored_blocks("req-B", [b"b1", b"b2"], [2, 3], job_id=11)
for kv_id, keys, indexes in (
("req-A", [b"a1", b"a2"], [100, 101]),
("req-B", [b"b1", b"b2"], [102, 103]),
):
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: kv_id,
FetchMsg.KEYS: keys,
FetchMsg.BLOCK_INDEXES: indexes,
}
)
session.poll()
assert _invariant_holds()
assert session._server._has_inflight_for("req-A")
assert session._server._has_inflight_for("req-B")
assert not session._server._has_inflight_for("req-C")
# Complete req-A's transfer first; req-B should still be inflight.
a_tids = [
tid
for tid, x in session._server._inflight.items()
if x.kv_request_id == "req-A"
]
for tid in a_tids:
transport._poll_done.append(tid)
session.poll()
assert _invariant_holds()
assert not session._server._has_inflight_for("req-A")
assert _srv_inflight_count(session, "req-A") == 0 # entry drained
assert session._server._has_inflight_for("req-B")
# Complete req-B; counter must drain to empty.
b_tids = [
tid
for tid, x in session._server._inflight.items()
if x.kv_request_id == "req-B"
]
for tid in b_tids:
transport._poll_done.append(tid)
session.poll()
assert _invariant_holds()
assert session._server._inflight == {}
assert _srv_total_inflight(session) == 0
def test_has_inflight_for_correct_with_many_requests(self):
"""Populate many inflight xfers across many ids; lookup must
match the actual presence in `_inflight` for both hits and
misses. The whole point of the counter is that this lookup is
constant-time, but we assert correctness, not timing."""
session, _, _ = _make_session()
for kv_id_idx in range(100):
kv_id = f"req-{kv_id_idx}"
for j in range(10):
tid = kv_id_idx * 10 + j
session._server._inflight_add(
tid,
_InflightXfer(
kv_request_id=kv_id,
block_count=1,
job_ids={tid},
round=_OutboundRequestState(inflight=1),
),
)
assert _srv_total_inflight(session) == len(session._server._inflight)
assert session._server._has_inflight_for("req-0")
assert session._server._has_inflight_for("req-99")
assert not session._server._has_inflight_for("req-missing")
# Drain all entries for req-50 and confirm the entry disappears.
tids_50 = [
tid
for tid, x in session._server._inflight.items()
if x.kv_request_id == "req-50"
]
for tid in tids_50:
session._server._inflight_pop(tid)
assert _srv_inflight_count(session, "req-50") == 0
assert not session._server._has_inflight_for("req-50")
# Other ids unaffected.
assert session._server._has_inflight_for("req-49")
# ---------------------------------------------------------------------------
# Protocol validation tests (unchanged from the prior file)
# ---------------------------------------------------------------------------
class TestConnectMsgValidation:
def _valid_msg(self) -> dict:
return {
TYPE_KEY: ConnectMsg.TYPE,
ConnectMsg.PEER_ID: "peer:1",
ConnectMsg.AGENT_METADATA: b"meta",
ConnectMsg.BASE_ADDR: 0x1000,
ConnectMsg.NUM_BLOCKS: 8,
ConnectMsg.BLOCK_LEN: 4096,
ConnectMsg.HASH_SEED: "0",
}
def test_valid_message_passes(self):
ConnectMsg.validate(self._valid_msg())
def test_missing_peer_id(self):
msg = self._valid_msg()
del msg[ConnectMsg.PEER_ID]
with pytest.raises(ValueError, match="peer_id"):
ConnectMsg.validate(msg)
def test_missing_agent_metadata(self):
msg = self._valid_msg()
del msg[ConnectMsg.AGENT_METADATA]
with pytest.raises(ValueError, match="agent_metadata"):
ConnectMsg.validate(msg)
def test_agent_metadata_wrong_type(self):
msg = self._valid_msg()
msg[ConnectMsg.AGENT_METADATA] = "not bytes"
with pytest.raises(ValueError, match="agent_metadata"):
ConnectMsg.validate(msg)
def test_base_addr_negative(self):
msg = self._valid_msg()
msg[ConnectMsg.BASE_ADDR] = -1
with pytest.raises(ValueError, match="base_addr"):
ConnectMsg.validate(msg)
def test_num_blocks_zero(self):
msg = self._valid_msg()
msg[ConnectMsg.NUM_BLOCKS] = 0
with pytest.raises(ValueError, match="num_blocks"):
ConnectMsg.validate(msg)
def test_block_len_zero(self):
msg = self._valid_msg()
msg[ConnectMsg.BLOCK_LEN] = 0
with pytest.raises(ValueError, match="block_len"):
ConnectMsg.validate(msg)
def test_missing_hash_seed(self):
msg = self._valid_msg()
del msg[ConnectMsg.HASH_SEED]
with pytest.raises(ValueError, match="hash_seed"):
ConnectMsg.validate(msg)
def test_hash_seed_wrong_type(self):
msg = self._valid_msg()
msg[ConnectMsg.HASH_SEED] = 12345 # int, not str
with pytest.raises(ValueError, match="hash_seed"):
ConnectMsg.validate(msg)
class TestFetchMsgValidation:
def _valid_msg(self) -> dict:
return {
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [0, 1],
}
def test_valid_message_passes(self):
FetchMsg.validate(self._valid_msg())
def test_length_mismatch(self):
msg = self._valid_msg()
msg[FetchMsg.BLOCK_INDEXES] = [0]
with pytest.raises(ValueError, match="length mismatch"):
FetchMsg.validate(msg)
def test_negative_index(self):
msg = self._valid_msg()
msg[FetchMsg.BLOCK_INDEXES] = [0, -1]
with pytest.raises(ValueError, match="invalid index"):
FetchMsg.validate(msg)
class TestTransferDoneMsgValidation:
def test_valid_message_passes(self):
msg = {
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: True,
}
TransferDoneMsg.validate(msg)
def test_success_wrong_type(self):
msg = {
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.ROUND_SEQ: 0,
TransferDoneMsg.KV_REQUEST_ID: "req-1",
TransferDoneMsg.SUCCESS: 1,
}
with pytest.raises(ValueError, match="success"):
TransferDoneMsg.validate(msg)
# ---------------------------------------------------------------------------
# Tests for handshake deadline
# ---------------------------------------------------------------------------
class TestHandshakeDeadline:
"""A connected-but-not-ready session is marked dead after the
handshake timeout expires."""
def test_marks_dead_after_timeout(self, monkeypatch):
monkeypatch.setenv("VLLM_P2P_HANDSHAKE_TIMEOUT_S", "1")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_HANDSHAKE_TIMEOUT_S", None)
transport = FakeDataTransport()
conn = FakeConnection(peer_id="stale:9999")
session = P2PSession(
peer_id="stale:9999",
local_id="local:1",
transport=transport,
local_block_len=4096,
local_hash_seed=_DEFAULT_HASH_SEED,
conn=conn,
)
assert session.connected
assert not session.ready
assert session.alive
session._connected_at = time.monotonic() - 5
session.poll()
assert not session.alive
def test_no_timeout_when_ready(self, monkeypatch):
monkeypatch.setenv("VLLM_P2P_HANDSHAKE_TIMEOUT_S", "1")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_HANDSHAKE_TIMEOUT_S", None)
transport = FakeDataTransport()
conn = FakeConnection(peer_id="good:1234")
session = P2PSession(
peer_id="good:1234",
local_id="local:1",
transport=transport,
local_block_len=4096,
local_hash_seed=_DEFAULT_HASH_SEED,
conn=conn,
)
# Complete the handshake by feeding ConnectMsg + ConnectAckMsg.
conn._inbox = [
{
TYPE_KEY: ConnectMsg.TYPE,
ConnectMsg.PEER_ID: "good:1234",
ConnectMsg.AGENT_METADATA: b"meta",
ConnectMsg.BASE_ADDR: 0x1000,
ConnectMsg.NUM_BLOCKS: 16,
ConnectMsg.BLOCK_LEN: 4096,
ConnectMsg.CONFIG_FINGERPRINT: "",
ConnectMsg.HASH_SEED: _DEFAULT_HASH_SEED,
},
{TYPE_KEY: ConnectAckMsg.TYPE, ConnectAckMsg.PEER_ID: "good:1234"},
]
session.poll()
assert session.ready
session._connected_at = time.monotonic() - 100
session.poll()
assert session.alive
def test_buffered_ack_at_deadline_completes_handshake(self, monkeypatch):
"""A ConnectAck already in the inbox at the deadline must complete
the handshake instead of being discarded by mark_dead."""
monkeypatch.setenv("VLLM_P2P_HANDSHAKE_TIMEOUT_S", "1")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_HANDSHAKE_TIMEOUT_S", None)
transport = FakeDataTransport()
conn = FakeConnection(peer_id="late-ack:1")
session = P2PSession(
peer_id="late-ack:1",
local_id="local:1",
transport=transport,
local_block_len=4096,
local_hash_seed=_DEFAULT_HASH_SEED,
conn=conn,
)
session._connected_at = time.monotonic() - 5
conn._inbox = [
{
TYPE_KEY: ConnectMsg.TYPE,
ConnectMsg.PEER_ID: "late-ack:1",
ConnectMsg.AGENT_METADATA: b"meta",
ConnectMsg.BASE_ADDR: 0x1000,
ConnectMsg.NUM_BLOCKS: 16,
ConnectMsg.BLOCK_LEN: 4096,
ConnectMsg.CONFIG_FINGERPRINT: "",
ConnectMsg.HASH_SEED: _DEFAULT_HASH_SEED,
},
{TYPE_KEY: ConnectAckMsg.TYPE, ConnectAckMsg.PEER_ID: "late-ack:1"},
]
session.poll()
assert session.ready
assert session.alive
def test_activity_timestamp_updated_on_recv(self):
transport = FakeDataTransport()
conn = FakeConnection(peer_id="peer:1")
session = P2PSession(
peer_id="peer:1",
local_id="local:1",
transport=transport,
local_block_len=4096,
local_hash_seed=_DEFAULT_HASH_SEED,
conn=conn,
)
before = session._last_activity_at
conn._inbox = [
{
TYPE_KEY: ConnectMsg.TYPE,
ConnectMsg.PEER_ID: "peer:1",
ConnectMsg.AGENT_METADATA: b"meta",
ConnectMsg.BASE_ADDR: 0x1000,
ConnectMsg.NUM_BLOCKS: 16,
ConnectMsg.BLOCK_LEN: 4096,
ConnectMsg.CONFIG_FINGERPRINT: "",
ConnectMsg.HASH_SEED: _DEFAULT_HASH_SEED,
},
]
session.poll()
assert session._last_activity_at >= before