[Bugfix][KV Offload][P2P] Keep store pins and AbortAck until NIXL transfers stop (#59790)
Signed-off-by: apex-mochen <2756823972@qq.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Codex
Claude Opus 5.5
Kevin H. Luu
parent
5d88c034f6
commit
e2aabc8b15
@@ -14,6 +14,15 @@ import pytest
|
||||
from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult
|
||||
from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport
|
||||
|
||||
|
||||
class nixlRemoteDisconnectError(Exception): # noqa: N801
|
||||
"""Stand-in for NIXL's binding exception of the same name."""
|
||||
|
||||
|
||||
def _remote_disconnect() -> Exception:
|
||||
return nixlRemoteDisconnectError("NIXL_ERR_REMOTE_DISCONNECT")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DataTransport base class tests
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -286,25 +295,101 @@ class TestNixlTransportWithMockedAgent:
|
||||
assert transport.cancel([999, 1000]) == []
|
||||
assert transport.cancel([999, 1000], mode="wait") == []
|
||||
|
||||
def test_cancel_wait_release_succeeds(self):
|
||||
"""wait-mode cancel that succeeds pops the entry and returns []."""
|
||||
@pytest.mark.parametrize("state", ["DONE", "ERR"])
|
||||
def test_cancel_wait_release_succeeds(self, state):
|
||||
"""wait-mode cancel of a stopped transfer pops it and returns []."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
assert tid in transport._inflight
|
||||
|
||||
transport._agent.check_xfer_state.return_value = state
|
||||
result = transport.cancel([tid], mode="wait")
|
||||
assert result == []
|
||||
assert tid not in transport._inflight
|
||||
transport._agent.release_xfer_handle.assert_called_once()
|
||||
|
||||
@pytest.mark.parametrize("state", ["PROC", "PEND"])
|
||||
def test_cancel_wait_keeps_running_transfer(self, state):
|
||||
"""Releasing a running NIXL WRITE does not stop it, so wait-mode
|
||||
cancel must not release it or report it as cancelled."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
transport._agent.check_xfer_state.return_value = state
|
||||
|
||||
assert transport.cancel([tid], mode="wait") == [tid]
|
||||
assert tid in transport._inflight
|
||||
transport._agent.release_xfer_handle.assert_not_called()
|
||||
|
||||
def test_cancel_wait_state_check_raises(self):
|
||||
"""A failed state check is not proof the transfer stopped."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
transport._agent.check_xfer_state.side_effect = RuntimeError("NOT_FOUND")
|
||||
|
||||
assert transport.cancel([tid], mode="wait") == [tid]
|
||||
assert tid in transport._inflight
|
||||
transport._agent.release_xfer_handle.assert_not_called()
|
||||
|
||||
def test_cancel_wait_remote_disconnect_is_terminal(self):
|
||||
"""A dead peer can no longer be written to: wait-mode cancel drops
|
||||
the transfer even if its handle can no longer be released."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
transport._agent.check_xfer_state.side_effect = _remote_disconnect()
|
||||
transport._agent.release_xfer_handle.side_effect = RuntimeError("gone")
|
||||
|
||||
assert transport.cancel([tid], mode="wait") == []
|
||||
assert tid not in transport._inflight
|
||||
assert transport.poll() == PollResult(done=(), failed=())
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"exc",
|
||||
[
|
||||
_remote_disconnect(),
|
||||
# Recognized by status code even if the binding class is renamed.
|
||||
RuntimeError("getXferStatus: NIXL_ERR_REMOTE_DISCONNECT"),
|
||||
],
|
||||
)
|
||||
def test_poll_reports_remote_disconnect_as_failed(self, exc):
|
||||
"""check_xfer_state raises instead of returning ERR once the peer
|
||||
disconnected; poll must surface that as a failure, not keep the
|
||||
transfer inflight forever."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
transport._agent.check_xfer_state.side_effect = exc
|
||||
|
||||
result = transport.poll()
|
||||
assert result.done == ()
|
||||
assert tid in result.failed
|
||||
assert tid not in transport._inflight
|
||||
|
||||
def test_poll_keeps_transfer_on_other_state_check_errors(self):
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
transport._agent.check_xfer_state.side_effect = RuntimeError("NOT_FOUND")
|
||||
|
||||
assert transport.poll() == PollResult(done=(), failed=())
|
||||
assert tid in transport._inflight
|
||||
|
||||
def test_cancel_wait_release_raises(self):
|
||||
"""wait-mode cancel keeps the entry and returns the tid on raise."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
transport._agent.check_xfer_state.return_value = "DONE"
|
||||
transport._agent.release_xfer_handle.side_effect = RuntimeError(
|
||||
"NIXL_ERR_REPOST_ACTIVE"
|
||||
)
|
||||
|
||||
@@ -15,10 +15,12 @@ from __future__ import annotations
|
||||
import time
|
||||
from collections.abc import Callable, Sequence
|
||||
from concurrent.futures import Future
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import vllm.v1.kv_offload.tiering.p2p.session.server as server_mod
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadKey,
|
||||
@@ -1632,9 +1634,11 @@ class TestServerFlows:
|
||||
assert _srv_abort_started(session, "req-1") is None
|
||||
assert tid not in session._server._inflight
|
||||
|
||||
def test_abort_fetch_force_cancels_after_timeout(self):
|
||||
"""If wait-mode never drains, the deadline forces immediate
|
||||
cancel and an ack is still sent."""
|
||||
def test_abort_fetch_never_force_cancels_after_timeout(self):
|
||||
"""Past the drain deadline a transfer that is still running is not
|
||||
force-released and no ack is sent: releasing the handle or removing
|
||||
the peer does not stop a NIXL WRITE, so the ack would let the peer
|
||||
reuse blocks that are still being written."""
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
tid = 42
|
||||
@@ -1658,21 +1662,28 @@ class TestServerFlows:
|
||||
)
|
||||
session.poll()
|
||||
assert _srv_abort_started(session, "req-1") is not None
|
||||
peers = dict(transport._remote_peers)
|
||||
# Backdate past the drain deadline.
|
||||
session._server._pending_aborts[("req-1", 0)] = (
|
||||
time.monotonic() - _CANCEL_DRAIN_TIMEOUT_S - 1.0
|
||||
)
|
||||
# Even if the transport still claims it can't cancel, the
|
||||
# session must force-pop and ack.
|
||||
transport._cancel_calls.clear()
|
||||
|
||||
session.poll()
|
||||
for _ in range(3):
|
||||
session.poll()
|
||||
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
|
||||
assert tid in session._server._inflight
|
||||
assert all(mode == "wait" for _, mode in transport._cancel_calls)
|
||||
assert transport._remote_peers == peers
|
||||
|
||||
ack = next(m for m in conn._sent if m[TYPE_KEY] == AbortAckMsg.TYPE)
|
||||
assert ack[AbortAckMsg.KV_REQUEST_ID] == "req-1"
|
||||
# Once NIXL reports the transfer terminal, the ack follows.
|
||||
transport._cancel_still_inflight.discard(tid)
|
||||
transport._poll_done.append(tid)
|
||||
session.poll()
|
||||
acks = [m for m in conn._sent if m[TYPE_KEY] == AbortAckMsg.TYPE]
|
||||
assert len(acks) == 1
|
||||
assert _srv_abort_started(session, "req-1") is None
|
||||
assert tid not in session._server._inflight
|
||||
assert ([tid], "immediate") in transport._cancel_calls
|
||||
|
||||
def test_abort_fetch_idempotent_while_draining(self):
|
||||
"""Receiving AbortFetchMsg twice for the same kv_request_id
|
||||
@@ -1731,10 +1742,12 @@ class TestServerFlows:
|
||||
stores = session.poll().stores
|
||||
assert StoreResult(job_id=1, success=False) in stores
|
||||
|
||||
def test_store_timeout_then_late_completion_no_duplicate(self):
|
||||
"""A job timed out by _timeout_pending_store_jobs must not also
|
||||
emit a contradictory StoreResult(success=True) when the transport
|
||||
later reports the same transfer as done."""
|
||||
@pytest.mark.parametrize("cancel_pending", [False, True])
|
||||
def test_store_timeout_cancels_stuck_transfer_after_confirmation(
|
||||
self, cancel_pending, monkeypatch
|
||||
):
|
||||
# Retry on every poll; pacing is covered by the *_paced tests.
|
||||
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
|
||||
@@ -1749,25 +1762,143 @@ class TestServerFlows:
|
||||
)
|
||||
session.poll()
|
||||
tid = next(iter(transport._transfers))
|
||||
|
||||
# Backdate the store job so the next poll times it out.
|
||||
session._server._store_jobs[1] = time.monotonic() - 60.0
|
||||
stores = session.poll().stores
|
||||
assert StoreResult(job_id=1, success=False) in stores
|
||||
assert StoreResult(job_id=1, success=True) not in stores
|
||||
if cancel_pending:
|
||||
transport._cancel_still_inflight.add(tid)
|
||||
for _ in range(3):
|
||||
assert session.poll().stores == []
|
||||
assert 1 in session._server._store_jobs
|
||||
assert tid in session._server._inflight
|
||||
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
|
||||
transport._cancel_still_inflight.clear()
|
||||
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
|
||||
assert transport._cancel_calls
|
||||
assert all(mode == "wait" for _, mode in transport._cancel_calls)
|
||||
assert tid not in session._server._inflight
|
||||
assert tid not in transport._transfers
|
||||
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
|
||||
assert len(terminal) == 1
|
||||
assert terminal[0][TransferDoneMsg.SUCCESS] is False
|
||||
assert session.poll().stores == []
|
||||
|
||||
# Transport later reports the same transfer as done — must not
|
||||
# emit a second (contradictory) StoreResult for job_id=1.
|
||||
transport._poll_done.append(tid)
|
||||
stores = session.poll().stores
|
||||
assert all(s.job_id != 1 for s in stores), (
|
||||
f"unexpected duplicate StoreResult after timeout: {stores}"
|
||||
@pytest.mark.parametrize("success", [True, False])
|
||||
def test_store_timeout_keeps_pins_until_cancel_or_late_terminal(self, success):
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
|
||||
session.add_stored_blocks("req-2", [b"k2"], [1], job_id=2)
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1"],
|
||||
FetchMsg.BLOCK_INDEXES: [5],
|
||||
}
|
||||
)
|
||||
session.poll()
|
||||
tid = next(iter(transport._transfers))
|
||||
transport._cancel_still_inflight.add(tid)
|
||||
|
||||
def test_store_timeout_then_late_failure_no_duplicate(self):
|
||||
"""Symmetric guard: a timed-out job must not also emit a second
|
||||
StoreResult(success=False) when the transport later reports the
|
||||
same transfer as failed."""
|
||||
for jid in (1, 2):
|
||||
session._server._store_jobs[jid] = time.monotonic() - 60.0
|
||||
# A StoreResult releases the primary slot. Only the job still
|
||||
# waiting for a fetch may time out while the transfer is active.
|
||||
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
|
||||
assert session.poll().stores == []
|
||||
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
|
||||
|
||||
completed = transport._poll_done if success else transport._poll_failed
|
||||
completed.append(tid)
|
||||
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
|
||||
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
|
||||
assert len(terminal) == 1
|
||||
assert terminal[0][TransferDoneMsg.SUCCESS] is False
|
||||
assert session.poll().stores == []
|
||||
|
||||
@pytest.mark.parametrize("success", [True, False, None])
|
||||
def test_store_timeout_waits_for_other_transfers_in_round(
|
||||
self, success, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1", b"k2"],
|
||||
FetchMsg.BLOCK_INDEXES: [5, 6],
|
||||
}
|
||||
)
|
||||
session.poll()
|
||||
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
|
||||
session.add_stored_blocks("req-1", [b"k2"], [1], job_id=2)
|
||||
first, second = list(transport._transfers)
|
||||
session._server._store_jobs[1] = time.monotonic() - 60.0
|
||||
transport._cancel_still_inflight.add(second)
|
||||
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
|
||||
assert first not in session._server._inflight
|
||||
assert second in session._server._inflight
|
||||
assert 2 in session._server._store_jobs
|
||||
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
|
||||
# Additional supply must not restart the timed-out round.
|
||||
session.add_stored_blocks("req-1", [b"k3"], [2], job_id=3)
|
||||
assert session.poll().stores == [StoreResult(job_id=3, success=False)]
|
||||
assert set(session._server._inflight) == {second}
|
||||
if success is None:
|
||||
transport._cancel_still_inflight.clear()
|
||||
else:
|
||||
completed = transport._poll_done if success else transport._poll_failed
|
||||
completed.append(second)
|
||||
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
|
||||
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
|
||||
assert len(terminal) == 1
|
||||
assert terminal[0][TransferDoneMsg.SUCCESS] is False
|
||||
assert session.poll().stores == []
|
||||
|
||||
def test_failed_transfer_keeps_sibling_pins_until_terminal(self, monkeypatch):
|
||||
"""A failed transfer fails its round, but a sibling transfer that may
|
||||
still be writing is not force-cancelled: its pin and the round's
|
||||
TransferDoneMsg wait until it is terminal."""
|
||||
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1", b"k2"],
|
||||
FetchMsg.BLOCK_INDEXES: [5, 6],
|
||||
}
|
||||
)
|
||||
session.poll()
|
||||
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
|
||||
session.add_stored_blocks("req-1", [b"k2"], [1], job_id=2)
|
||||
first, second = list(transport._transfers)
|
||||
transport._cancel_still_inflight.add(second)
|
||||
|
||||
transport._poll_failed.append(first)
|
||||
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
|
||||
for _ in range(3):
|
||||
assert session.poll().stores == []
|
||||
assert second in session._server._inflight
|
||||
assert 2 in session._server._store_jobs
|
||||
assert transport._cancel_calls
|
||||
assert all(mode == "wait" for _, mode in transport._cancel_calls)
|
||||
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
|
||||
|
||||
transport._cancel_still_inflight.clear()
|
||||
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
|
||||
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
|
||||
assert len(terminal) == 1
|
||||
assert terminal[0][TransferDoneMsg.SUCCESS] is False
|
||||
assert session.poll().stores == []
|
||||
|
||||
def test_abort_during_store_timeout_keeps_unconfirmed_pins(self, monkeypatch):
|
||||
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
|
||||
@@ -1782,18 +1913,219 @@ class TestServerFlows:
|
||||
)
|
||||
session.poll()
|
||||
tid = next(iter(transport._transfers))
|
||||
|
||||
transport._cancel_still_inflight.add(tid)
|
||||
session._server._store_jobs[1] = time.monotonic() - 60.0
|
||||
stores = session.poll().stores
|
||||
assert [s for s in stores if s.job_id == 1] == [
|
||||
StoreResult(job_id=1, success=False)
|
||||
]
|
||||
|
||||
transport._poll_failed.append(tid)
|
||||
stores = session.poll().stores
|
||||
assert all(s.job_id != 1 for s in stores), (
|
||||
f"unexpected duplicate StoreResult after timeout: {stores}"
|
||||
assert session.poll().stores == []
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.KV_REQUEST_ID: "req-1",
|
||||
AbortFetchMsg.ROUND_SEQ: 0,
|
||||
}
|
||||
)
|
||||
assert session.poll().stores == []
|
||||
session._server._pending_aborts[("req-1", 0)] = time.monotonic() - 60.0
|
||||
assert session.poll().stores == []
|
||||
assert 1 in session._server._store_jobs
|
||||
assert tid in session._server._inflight
|
||||
assert all(mode == "wait" for _, mode in transport._cancel_calls)
|
||||
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
|
||||
transport._cancel_still_inflight.clear()
|
||||
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
|
||||
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
|
||||
assert session.poll().stores == []
|
||||
|
||||
@staticmethod
|
||||
def _fake_clock(monkeypatch) -> list[float]:
|
||||
clock = [1000.0]
|
||||
monkeypatch.setattr(
|
||||
server_mod, "time", SimpleNamespace(monotonic=lambda: clock[0])
|
||||
)
|
||||
return clock
|
||||
|
||||
@staticmethod
|
||||
def _submit_unconfirmed_transfer(session, conn, transport) -> int:
|
||||
"""Start one transfer for job 1 whose wait-cancel never confirms."""
|
||||
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.ROUND_SEQ: 0,
|
||||
FetchMsg.KV_REQUEST_ID: "req-1",
|
||||
FetchMsg.KEYS: [b"k1"],
|
||||
FetchMsg.BLOCK_INDEXES: [5],
|
||||
}
|
||||
)
|
||||
session.poll()
|
||||
tid = next(iter(transport._transfers))
|
||||
transport._cancel_still_inflight.add(tid)
|
||||
return tid
|
||||
|
||||
@staticmethod
|
||||
def _wait_cancels(transport) -> int:
|
||||
return sum(1 for _, mode in transport._cancel_calls if mode == "wait")
|
||||
|
||||
def test_store_timeout_cancel_retry_is_paced(self, monkeypatch):
|
||||
clock = self._fake_clock(monkeypatch)
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
tid = self._submit_unconfirmed_transfer(session, conn, transport)
|
||||
session._server._store_jobs[1] = clock[0] - 60.0
|
||||
|
||||
assert session.poll().stores == []
|
||||
assert self._wait_cancels(transport) == 1
|
||||
start = clock[0]
|
||||
for dt in (0.0, 0.05, 0.099):
|
||||
clock[0] = start + dt
|
||||
for _ in range(5):
|
||||
assert session.poll().stores == []
|
||||
assert self._wait_cancels(transport) == 1
|
||||
|
||||
clock[0] = start + server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
|
||||
assert session.poll().stores == []
|
||||
assert self._wait_cancels(transport) == 2
|
||||
|
||||
transport._cancel_still_inflight.clear()
|
||||
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
|
||||
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
|
||||
assert self._wait_cancels(transport) == 3
|
||||
assert tid not in session._server._inflight
|
||||
|
||||
def test_abort_drain_cancel_retry_is_paced(self, monkeypatch):
|
||||
clock = self._fake_clock(monkeypatch)
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
self._submit_unconfirmed_transfer(session, conn, transport)
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.KV_REQUEST_ID: "req-1",
|
||||
AbortFetchMsg.ROUND_SEQ: 0,
|
||||
}
|
||||
)
|
||||
# The first drain attempt is immediate.
|
||||
session.poll()
|
||||
assert self._wait_cancels(transport) == 1
|
||||
for _ in range(5):
|
||||
session.poll()
|
||||
assert self._wait_cancels(transport) == 1
|
||||
|
||||
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
|
||||
session.poll()
|
||||
assert self._wait_cancels(transport) == 2
|
||||
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
|
||||
|
||||
transport._cancel_still_inflight.clear()
|
||||
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
|
||||
session.poll()
|
||||
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
|
||||
|
||||
def test_unconfirmed_store_cancel_never_acks_or_tears_down(self, monkeypatch):
|
||||
"""A cancellation that is never confirmed must not release the pin,
|
||||
send AbortAck, force-cancel, or remove the peer, however long it
|
||||
stays pending. AbortAck follows the release once it is confirmed."""
|
||||
clock = self._fake_clock(monkeypatch)
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
peers = dict(transport._remote_peers)
|
||||
tid = self._submit_unconfirmed_transfer(session, conn, transport)
|
||||
session._server._store_jobs[1] = clock[0] - 60.0
|
||||
assert session.poll().stores == []
|
||||
|
||||
abort = {
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.KV_REQUEST_ID: "req-1",
|
||||
AbortFetchMsg.ROUND_SEQ: 0,
|
||||
}
|
||||
started_at = clock[0]
|
||||
conn.enqueue(dict(abort))
|
||||
assert session.poll().stores == []
|
||||
# A duplicate abort neither resets the drain start nor adds a cancel.
|
||||
conn.enqueue(dict(abort))
|
||||
assert session.poll().stores == []
|
||||
assert session._server._pending_aborts[("req-1", 0)] == started_at
|
||||
# The store sweep and the abort drain share the round's pacing.
|
||||
assert self._wait_cancels(transport) == 1
|
||||
|
||||
ticks = 50
|
||||
for step in range(1, ticks + 1):
|
||||
clock[0] = started_at + step * 10.0
|
||||
assert session.poll().stores == []
|
||||
assert clock[0] - started_at > _CANCEL_DRAIN_TIMEOUT_S
|
||||
assert self._wait_cancels(transport) == 1 + ticks
|
||||
assert all(mode == "wait" for _, mode in transport._cancel_calls)
|
||||
assert 1 in session._server._store_jobs
|
||||
assert tid in session._server._inflight
|
||||
assert transport._remote_peers == peers
|
||||
assert not any(
|
||||
m[TYPE_KEY] in (AbortAckMsg.TYPE, TransferDoneMsg.TYPE) for m in conn._sent
|
||||
)
|
||||
|
||||
transport._cancel_still_inflight.clear()
|
||||
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
|
||||
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
|
||||
assert 1 not in session._server._store_jobs
|
||||
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
|
||||
assert session.poll().stores == []
|
||||
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
|
||||
|
||||
def test_abort_keeps_pins_only_of_jobs_being_written(self, monkeypatch):
|
||||
"""An abort fails the round's jobs at once, except those an inflight
|
||||
transfer still reads; their pins and the ack wait for the transfer."""
|
||||
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
tid = self._submit_unconfirmed_transfer(session, conn, transport)
|
||||
# Job 2 supplies a block nobody demanded: no transfer reads it.
|
||||
session.add_stored_blocks("req-1", [b"k2"], [1], job_id=2)
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.KV_REQUEST_ID: "req-1",
|
||||
AbortFetchMsg.ROUND_SEQ: 0,
|
||||
}
|
||||
)
|
||||
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
|
||||
for _ in range(3):
|
||||
assert session.poll().stores == []
|
||||
assert 1 in session._server._store_jobs
|
||||
assert tid in session._server._inflight
|
||||
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
|
||||
|
||||
transport._cancel_still_inflight.clear()
|
||||
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
|
||||
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
|
||||
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
|
||||
assert session.poll().stores == []
|
||||
|
||||
@pytest.mark.parametrize("success", [True, False])
|
||||
def test_abort_acks_after_late_terminal_without_transfer_done(
|
||||
self, success, monkeypatch
|
||||
):
|
||||
"""A transfer that finishes on its own after the abort releases its
|
||||
pin and lets the parked abort ack; the aborted round sends no
|
||||
TransferDoneMsg."""
|
||||
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
tid = self._submit_unconfirmed_transfer(session, conn, transport)
|
||||
conn.enqueue(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.KV_REQUEST_ID: "req-1",
|
||||
AbortFetchMsg.ROUND_SEQ: 0,
|
||||
}
|
||||
)
|
||||
assert session.poll().stores == []
|
||||
assert 1 in session._server._store_jobs
|
||||
|
||||
completed = transport._poll_done if success else transport._poll_failed
|
||||
completed.append(tid)
|
||||
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
|
||||
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
|
||||
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
|
||||
assert session.poll().stores == []
|
||||
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
|
||||
|
||||
def test_fetch_after_symmetric_supply_timed_out_reads_nothing(self):
|
||||
"""A lookup pin whose store job timed out has been reported failed,
|
||||
|
||||
Reference in New Issue
Block a user