[Bugfix][KV Offload][P2P] Keep store pins and AbortAck until NIXL transfers stop (#59790)

Signed-off-by: apex-mochen <2756823972@qq.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
dingdangmao
2026-10-14 18:31:31 +03:00
committed by GitHub
co-authored by Codex Claude Opus 5.5 Kevin H. Luu
parent 5d88c034f6
commit e2aabc8b15
6 changed files with 695 additions and 108 deletions
@@ -14,6 +14,15 @@ import pytest
from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult
from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport
class nixlRemoteDisconnectError(Exception): # noqa: N801
"""Stand-in for NIXL's binding exception of the same name."""
def _remote_disconnect() -> Exception:
return nixlRemoteDisconnectError("NIXL_ERR_REMOTE_DISCONNECT")
# ---------------------------------------------------------------------------
# DataTransport base class tests
# ---------------------------------------------------------------------------
@@ -286,25 +295,101 @@ class TestNixlTransportWithMockedAgent:
assert transport.cancel([999, 1000]) == []
assert transport.cancel([999, 1000], mode="wait") == []
def test_cancel_wait_release_succeeds(self):
"""wait-mode cancel that succeeds pops the entry and returns []."""
@pytest.mark.parametrize("state", ["DONE", "ERR"])
def test_cancel_wait_release_succeeds(self, state):
"""wait-mode cancel of a stopped transfer pops it and returns []."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
assert tid in transport._inflight
transport._agent.check_xfer_state.return_value = state
result = transport.cancel([tid], mode="wait")
assert result == []
assert tid not in transport._inflight
transport._agent.release_xfer_handle.assert_called_once()
@pytest.mark.parametrize("state", ["PROC", "PEND"])
def test_cancel_wait_keeps_running_transfer(self, state):
"""Releasing a running NIXL WRITE does not stop it, so wait-mode
cancel must not release it or report it as cancelled."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.return_value = state
assert transport.cancel([tid], mode="wait") == [tid]
assert tid in transport._inflight
transport._agent.release_xfer_handle.assert_not_called()
def test_cancel_wait_state_check_raises(self):
"""A failed state check is not proof the transfer stopped."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.side_effect = RuntimeError("NOT_FOUND")
assert transport.cancel([tid], mode="wait") == [tid]
assert tid in transport._inflight
transport._agent.release_xfer_handle.assert_not_called()
def test_cancel_wait_remote_disconnect_is_terminal(self):
"""A dead peer can no longer be written to: wait-mode cancel drops
the transfer even if its handle can no longer be released."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.side_effect = _remote_disconnect()
transport._agent.release_xfer_handle.side_effect = RuntimeError("gone")
assert transport.cancel([tid], mode="wait") == []
assert tid not in transport._inflight
assert transport.poll() == PollResult(done=(), failed=())
@pytest.mark.parametrize(
"exc",
[
_remote_disconnect(),
# Recognized by status code even if the binding class is renamed.
RuntimeError("getXferStatus: NIXL_ERR_REMOTE_DISCONNECT"),
],
)
def test_poll_reports_remote_disconnect_as_failed(self, exc):
"""check_xfer_state raises instead of returning ERR once the peer
disconnected; poll must surface that as a failure, not keep the
transfer inflight forever."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.side_effect = exc
result = transport.poll()
assert result.done == ()
assert tid in result.failed
assert tid not in transport._inflight
def test_poll_keeps_transfer_on_other_state_check_errors(self):
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.side_effect = RuntimeError("NOT_FOUND")
assert transport.poll() == PollResult(done=(), failed=())
assert tid in transport._inflight
def test_cancel_wait_release_raises(self):
"""wait-mode cancel keeps the entry and returns the tid on raise."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.return_value = "DONE"
transport._agent.release_xfer_handle.side_effect = RuntimeError(
"NIXL_ERR_REPOST_ACTIVE"
)
+370 -38
View File
@@ -15,10 +15,12 @@ from __future__ import annotations
import time
from collections.abc import Callable, Sequence
from concurrent.futures import Future
from types import SimpleNamespace
import numpy as np
import pytest
import vllm.v1.kv_offload.tiering.p2p.session.server as server_mod
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadKey,
@@ -1632,9 +1634,11 @@ class TestServerFlows:
assert _srv_abort_started(session, "req-1") is None
assert tid not in session._server._inflight
def test_abort_fetch_force_cancels_after_timeout(self):
"""If wait-mode never drains, the deadline forces immediate
cancel and an ack is still sent."""
def test_abort_fetch_never_force_cancels_after_timeout(self):
"""Past the drain deadline a transfer that is still running is not
force-released and no ack is sent: releasing the handle or removing
the peer does not stop a NIXL WRITE, so the ack would let the peer
reuse blocks that are still being written."""
session, conn, transport = _make_session()
_activate(session, conn)
tid = 42
@@ -1658,21 +1662,28 @@ class TestServerFlows:
)
session.poll()
assert _srv_abort_started(session, "req-1") is not None
peers = dict(transport._remote_peers)
# Backdate past the drain deadline.
session._server._pending_aborts[("req-1", 0)] = (
time.monotonic() - _CANCEL_DRAIN_TIMEOUT_S - 1.0
)
# Even if the transport still claims it can't cancel, the
# session must force-pop and ack.
transport._cancel_calls.clear()
session.poll()
for _ in range(3):
session.poll()
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
assert tid in session._server._inflight
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert transport._remote_peers == peers
ack = next(m for m in conn._sent if m[TYPE_KEY] == AbortAckMsg.TYPE)
assert ack[AbortAckMsg.KV_REQUEST_ID] == "req-1"
# Once NIXL reports the transfer terminal, the ack follows.
transport._cancel_still_inflight.discard(tid)
transport._poll_done.append(tid)
session.poll()
acks = [m for m in conn._sent if m[TYPE_KEY] == AbortAckMsg.TYPE]
assert len(acks) == 1
assert _srv_abort_started(session, "req-1") is None
assert tid not in session._server._inflight
assert ([tid], "immediate") in transport._cancel_calls
def test_abort_fetch_idempotent_while_draining(self):
"""Receiving AbortFetchMsg twice for the same kv_request_id
@@ -1731,10 +1742,12 @@ class TestServerFlows:
stores = session.poll().stores
assert StoreResult(job_id=1, success=False) in stores
def test_store_timeout_then_late_completion_no_duplicate(self):
"""A job timed out by _timeout_pending_store_jobs must not also
emit a contradictory StoreResult(success=True) when the transport
later reports the same transfer as done."""
@pytest.mark.parametrize("cancel_pending", [False, True])
def test_store_timeout_cancels_stuck_transfer_after_confirmation(
self, cancel_pending, monkeypatch
):
# Retry on every poll; pacing is covered by the *_paced tests.
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
@@ -1749,25 +1762,143 @@ class TestServerFlows:
)
session.poll()
tid = next(iter(transport._transfers))
# Backdate the store job so the next poll times it out.
session._server._store_jobs[1] = time.monotonic() - 60.0
stores = session.poll().stores
assert StoreResult(job_id=1, success=False) in stores
assert StoreResult(job_id=1, success=True) not in stores
if cancel_pending:
transport._cancel_still_inflight.add(tid)
for _ in range(3):
assert session.poll().stores == []
assert 1 in session._server._store_jobs
assert tid in session._server._inflight
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert transport._cancel_calls
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert tid not in session._server._inflight
assert tid not in transport._transfers
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
assert len(terminal) == 1
assert terminal[0][TransferDoneMsg.SUCCESS] is False
assert session.poll().stores == []
# Transport later reports the same transfer as done — must not
# emit a second (contradictory) StoreResult for job_id=1.
transport._poll_done.append(tid)
stores = session.poll().stores
assert all(s.job_id != 1 for s in stores), (
f"unexpected duplicate StoreResult after timeout: {stores}"
@pytest.mark.parametrize("success", [True, False])
def test_store_timeout_keeps_pins_until_cancel_or_late_terminal(self, success):
session, conn, transport = _make_session()
_activate(session, conn)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
session.add_stored_blocks("req-2", [b"k2"], [1], job_id=2)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
}
)
session.poll()
tid = next(iter(transport._transfers))
transport._cancel_still_inflight.add(tid)
def test_store_timeout_then_late_failure_no_duplicate(self):
"""Symmetric guard: a timed-out job must not also emit a second
StoreResult(success=False) when the transport later reports the
same transfer as failed."""
for jid in (1, 2):
session._server._store_jobs[jid] = time.monotonic() - 60.0
# A StoreResult releases the primary slot. Only the job still
# waiting for a fetch may time out while the transfer is active.
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
assert session.poll().stores == []
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
completed = transport._poll_done if success else transport._poll_failed
completed.append(tid)
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
assert len(terminal) == 1
assert terminal[0][TransferDoneMsg.SUCCESS] is False
assert session.poll().stores == []
@pytest.mark.parametrize("success", [True, False, None])
def test_store_timeout_waits_for_other_transfers_in_round(
self, success, monkeypatch
):
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [5, 6],
}
)
session.poll()
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
session.add_stored_blocks("req-1", [b"k2"], [1], job_id=2)
first, second = list(transport._transfers)
session._server._store_jobs[1] = time.monotonic() - 60.0
transport._cancel_still_inflight.add(second)
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert first not in session._server._inflight
assert second in session._server._inflight
assert 2 in session._server._store_jobs
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
# Additional supply must not restart the timed-out round.
session.add_stored_blocks("req-1", [b"k3"], [2], job_id=3)
assert session.poll().stores == [StoreResult(job_id=3, success=False)]
assert set(session._server._inflight) == {second}
if success is None:
transport._cancel_still_inflight.clear()
else:
completed = transport._poll_done if success else transport._poll_failed
completed.append(second)
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
assert len(terminal) == 1
assert terminal[0][TransferDoneMsg.SUCCESS] is False
assert session.poll().stores == []
def test_failed_transfer_keeps_sibling_pins_until_terminal(self, monkeypatch):
"""A failed transfer fails its round, but a sibling transfer that may
still be writing is not force-cancelled: its pin and the round's
TransferDoneMsg wait until it is terminal."""
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [5, 6],
}
)
session.poll()
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
session.add_stored_blocks("req-1", [b"k2"], [1], job_id=2)
first, second = list(transport._transfers)
transport._cancel_still_inflight.add(second)
transport._poll_failed.append(first)
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
for _ in range(3):
assert session.poll().stores == []
assert second in session._server._inflight
assert 2 in session._server._store_jobs
assert transport._cancel_calls
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
assert len(terminal) == 1
assert terminal[0][TransferDoneMsg.SUCCESS] is False
assert session.poll().stores == []
def test_abort_during_store_timeout_keeps_unconfirmed_pins(self, monkeypatch):
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
@@ -1782,18 +1913,219 @@ class TestServerFlows:
)
session.poll()
tid = next(iter(transport._transfers))
transport._cancel_still_inflight.add(tid)
session._server._store_jobs[1] = time.monotonic() - 60.0
stores = session.poll().stores
assert [s for s in stores if s.job_id == 1] == [
StoreResult(job_id=1, success=False)
]
transport._poll_failed.append(tid)
stores = session.poll().stores
assert all(s.job_id != 1 for s in stores), (
f"unexpected duplicate StoreResult after timeout: {stores}"
assert session.poll().stores == []
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
AbortFetchMsg.ROUND_SEQ: 0,
}
)
assert session.poll().stores == []
session._server._pending_aborts[("req-1", 0)] = time.monotonic() - 60.0
assert session.poll().stores == []
assert 1 in session._server._store_jobs
assert tid in session._server._inflight
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
assert session.poll().stores == []
@staticmethod
def _fake_clock(monkeypatch) -> list[float]:
clock = [1000.0]
monkeypatch.setattr(
server_mod, "time", SimpleNamespace(monotonic=lambda: clock[0])
)
return clock
@staticmethod
def _submit_unconfirmed_transfer(session, conn, transport) -> int:
"""Start one transfer for job 1 whose wait-cancel never confirms."""
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
}
)
session.poll()
tid = next(iter(transport._transfers))
transport._cancel_still_inflight.add(tid)
return tid
@staticmethod
def _wait_cancels(transport) -> int:
return sum(1 for _, mode in transport._cancel_calls if mode == "wait")
def test_store_timeout_cancel_retry_is_paced(self, monkeypatch):
clock = self._fake_clock(monkeypatch)
session, conn, transport = _make_session()
_activate(session, conn)
tid = self._submit_unconfirmed_transfer(session, conn, transport)
session._server._store_jobs[1] = clock[0] - 60.0
assert session.poll().stores == []
assert self._wait_cancels(transport) == 1
start = clock[0]
for dt in (0.0, 0.05, 0.099):
clock[0] = start + dt
for _ in range(5):
assert session.poll().stores == []
assert self._wait_cancels(transport) == 1
clock[0] = start + server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
assert session.poll().stores == []
assert self._wait_cancels(transport) == 2
transport._cancel_still_inflight.clear()
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert self._wait_cancels(transport) == 3
assert tid not in session._server._inflight
def test_abort_drain_cancel_retry_is_paced(self, monkeypatch):
clock = self._fake_clock(monkeypatch)
session, conn, transport = _make_session()
_activate(session, conn)
self._submit_unconfirmed_transfer(session, conn, transport)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
AbortFetchMsg.ROUND_SEQ: 0,
}
)
# The first drain attempt is immediate.
session.poll()
assert self._wait_cancels(transport) == 1
for _ in range(5):
session.poll()
assert self._wait_cancels(transport) == 1
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
session.poll()
assert self._wait_cancels(transport) == 2
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
session.poll()
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
def test_unconfirmed_store_cancel_never_acks_or_tears_down(self, monkeypatch):
"""A cancellation that is never confirmed must not release the pin,
send AbortAck, force-cancel, or remove the peer, however long it
stays pending. AbortAck follows the release once it is confirmed."""
clock = self._fake_clock(monkeypatch)
session, conn, transport = _make_session()
_activate(session, conn)
peers = dict(transport._remote_peers)
tid = self._submit_unconfirmed_transfer(session, conn, transport)
session._server._store_jobs[1] = clock[0] - 60.0
assert session.poll().stores == []
abort = {
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
AbortFetchMsg.ROUND_SEQ: 0,
}
started_at = clock[0]
conn.enqueue(dict(abort))
assert session.poll().stores == []
# A duplicate abort neither resets the drain start nor adds a cancel.
conn.enqueue(dict(abort))
assert session.poll().stores == []
assert session._server._pending_aborts[("req-1", 0)] == started_at
# The store sweep and the abort drain share the round's pacing.
assert self._wait_cancels(transport) == 1
ticks = 50
for step in range(1, ticks + 1):
clock[0] = started_at + step * 10.0
assert session.poll().stores == []
assert clock[0] - started_at > _CANCEL_DRAIN_TIMEOUT_S
assert self._wait_cancels(transport) == 1 + ticks
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert 1 in session._server._store_jobs
assert tid in session._server._inflight
assert transport._remote_peers == peers
assert not any(
m[TYPE_KEY] in (AbortAckMsg.TYPE, TransferDoneMsg.TYPE) for m in conn._sent
)
transport._cancel_still_inflight.clear()
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert 1 not in session._server._store_jobs
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
assert session.poll().stores == []
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
def test_abort_keeps_pins_only_of_jobs_being_written(self, monkeypatch):
"""An abort fails the round's jobs at once, except those an inflight
transfer still reads; their pins and the ack wait for the transfer."""
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
tid = self._submit_unconfirmed_transfer(session, conn, transport)
# Job 2 supplies a block nobody demanded: no transfer reads it.
session.add_stored_blocks("req-1", [b"k2"], [1], job_id=2)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
AbortFetchMsg.ROUND_SEQ: 0,
}
)
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
for _ in range(3):
assert session.poll().stores == []
assert 1 in session._server._store_jobs
assert tid in session._server._inflight
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
assert session.poll().stores == []
@pytest.mark.parametrize("success", [True, False])
def test_abort_acks_after_late_terminal_without_transfer_done(
self, success, monkeypatch
):
"""A transfer that finishes on its own after the abort releases its
pin and lets the parked abort ack; the aborted round sends no
TransferDoneMsg."""
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
tid = self._submit_unconfirmed_transfer(session, conn, transport)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
AbortFetchMsg.ROUND_SEQ: 0,
}
)
assert session.poll().stores == []
assert 1 in session._server._store_jobs
completed = transport._poll_done if success else transport._poll_failed
completed.append(tid)
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
assert session.poll().stores == []
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
def test_fetch_after_symmetric_supply_timed_out_reads_nothing(self):
"""A lookup pin whose store job timed out has been reported failed,