[Bugfix][KV Offload][P2P] Keep store pins and AbortAck until NIXL transfers stop (#59790)

Signed-off-by: apex-mochen <2756823972@qq.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
dingdangmao
2026-10-14 18:31:31 +03:00
committed by GitHub
co-authored by Codex Claude Opus 5.5 Kevin H. Luu
parent 5d88c034f6
commit e2aabc8b15
6 changed files with 695 additions and 108 deletions
@@ -14,6 +14,15 @@ import pytest
from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult
from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport
class nixlRemoteDisconnectError(Exception): # noqa: N801
"""Stand-in for NIXL's binding exception of the same name."""
def _remote_disconnect() -> Exception:
return nixlRemoteDisconnectError("NIXL_ERR_REMOTE_DISCONNECT")
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# DataTransport base class tests # DataTransport base class tests
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -286,25 +295,101 @@ class TestNixlTransportWithMockedAgent:
assert transport.cancel([999, 1000]) == [] assert transport.cancel([999, 1000]) == []
assert transport.cancel([999, 1000], mode="wait") == [] assert transport.cancel([999, 1000], mode="wait") == []
def test_cancel_wait_release_succeeds(self): @pytest.mark.parametrize("state", ["DONE", "ERR"])
"""wait-mode cancel that succeeds pops the entry and returns [].""" def test_cancel_wait_release_succeeds(self, state):
"""wait-mode cancel of a stopped transfer pops it and returns []."""
transport = self._make_transport() transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1]) tid = transport.write_blocks("peer:1", [0], [1])
assert tid in transport._inflight assert tid in transport._inflight
transport._agent.check_xfer_state.return_value = state
result = transport.cancel([tid], mode="wait") result = transport.cancel([tid], mode="wait")
assert result == [] assert result == []
assert tid not in transport._inflight assert tid not in transport._inflight
transport._agent.release_xfer_handle.assert_called_once() transport._agent.release_xfer_handle.assert_called_once()
@pytest.mark.parametrize("state", ["PROC", "PEND"])
def test_cancel_wait_keeps_running_transfer(self, state):
"""Releasing a running NIXL WRITE does not stop it, so wait-mode
cancel must not release it or report it as cancelled."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.return_value = state
assert transport.cancel([tid], mode="wait") == [tid]
assert tid in transport._inflight
transport._agent.release_xfer_handle.assert_not_called()
def test_cancel_wait_state_check_raises(self):
"""A failed state check is not proof the transfer stopped."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.side_effect = RuntimeError("NOT_FOUND")
assert transport.cancel([tid], mode="wait") == [tid]
assert tid in transport._inflight
transport._agent.release_xfer_handle.assert_not_called()
def test_cancel_wait_remote_disconnect_is_terminal(self):
"""A dead peer can no longer be written to: wait-mode cancel drops
the transfer even if its handle can no longer be released."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.side_effect = _remote_disconnect()
transport._agent.release_xfer_handle.side_effect = RuntimeError("gone")
assert transport.cancel([tid], mode="wait") == []
assert tid not in transport._inflight
assert transport.poll() == PollResult(done=(), failed=())
@pytest.mark.parametrize(
"exc",
[
_remote_disconnect(),
# Recognized by status code even if the binding class is renamed.
RuntimeError("getXferStatus: NIXL_ERR_REMOTE_DISCONNECT"),
],
)
def test_poll_reports_remote_disconnect_as_failed(self, exc):
"""check_xfer_state raises instead of returning ERR once the peer
disconnected; poll must surface that as a failure, not keep the
transfer inflight forever."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.side_effect = exc
result = transport.poll()
assert result.done == ()
assert tid in result.failed
assert tid not in transport._inflight
def test_poll_keeps_transfer_on_other_state_check_errors(self):
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.side_effect = RuntimeError("NOT_FOUND")
assert transport.poll() == PollResult(done=(), failed=())
assert tid in transport._inflight
def test_cancel_wait_release_raises(self): def test_cancel_wait_release_raises(self):
"""wait-mode cancel keeps the entry and returns the tid on raise.""" """wait-mode cancel keeps the entry and returns the tid on raise."""
transport = self._make_transport() transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1]) tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.return_value = "DONE"
transport._agent.release_xfer_handle.side_effect = RuntimeError( transport._agent.release_xfer_handle.side_effect = RuntimeError(
"NIXL_ERR_REPOST_ACTIVE" "NIXL_ERR_REPOST_ACTIVE"
) )
+370 -38
View File
@@ -15,10 +15,12 @@ from __future__ import annotations
import time import time
from collections.abc import Callable, Sequence from collections.abc import Callable, Sequence
from concurrent.futures import Future from concurrent.futures import Future
from types import SimpleNamespace
import numpy as np import numpy as np
import pytest import pytest
import vllm.v1.kv_offload.tiering.p2p.session.server as server_mod
from vllm.v1.kv_offload.base import ( from vllm.v1.kv_offload.base import (
LookupResult, LookupResult,
OffloadKey, OffloadKey,
@@ -1632,9 +1634,11 @@ class TestServerFlows:
assert _srv_abort_started(session, "req-1") is None assert _srv_abort_started(session, "req-1") is None
assert tid not in session._server._inflight assert tid not in session._server._inflight
def test_abort_fetch_force_cancels_after_timeout(self): def test_abort_fetch_never_force_cancels_after_timeout(self):
"""If wait-mode never drains, the deadline forces immediate """Past the drain deadline a transfer that is still running is not
cancel and an ack is still sent.""" force-released and no ack is sent: releasing the handle or removing
the peer does not stop a NIXL WRITE, so the ack would let the peer
reuse blocks that are still being written."""
session, conn, transport = _make_session() session, conn, transport = _make_session()
_activate(session, conn) _activate(session, conn)
tid = 42 tid = 42
@@ -1658,21 +1662,28 @@ class TestServerFlows:
) )
session.poll() session.poll()
assert _srv_abort_started(session, "req-1") is not None assert _srv_abort_started(session, "req-1") is not None
peers = dict(transport._remote_peers)
# Backdate past the drain deadline. # Backdate past the drain deadline.
session._server._pending_aborts[("req-1", 0)] = ( session._server._pending_aborts[("req-1", 0)] = (
time.monotonic() - _CANCEL_DRAIN_TIMEOUT_S - 1.0 time.monotonic() - _CANCEL_DRAIN_TIMEOUT_S - 1.0
) )
# Even if the transport still claims it can't cancel, the
# session must force-pop and ack.
transport._cancel_calls.clear() transport._cancel_calls.clear()
session.poll() for _ in range(3):
session.poll()
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
assert tid in session._server._inflight
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert transport._remote_peers == peers
ack = next(m for m in conn._sent if m[TYPE_KEY] == AbortAckMsg.TYPE) # Once NIXL reports the transfer terminal, the ack follows.
assert ack[AbortAckMsg.KV_REQUEST_ID] == "req-1" transport._cancel_still_inflight.discard(tid)
transport._poll_done.append(tid)
session.poll()
acks = [m for m in conn._sent if m[TYPE_KEY] == AbortAckMsg.TYPE]
assert len(acks) == 1
assert _srv_abort_started(session, "req-1") is None assert _srv_abort_started(session, "req-1") is None
assert tid not in session._server._inflight assert tid not in session._server._inflight
assert ([tid], "immediate") in transport._cancel_calls
def test_abort_fetch_idempotent_while_draining(self): def test_abort_fetch_idempotent_while_draining(self):
"""Receiving AbortFetchMsg twice for the same kv_request_id """Receiving AbortFetchMsg twice for the same kv_request_id
@@ -1731,10 +1742,12 @@ class TestServerFlows:
stores = session.poll().stores stores = session.poll().stores
assert StoreResult(job_id=1, success=False) in stores assert StoreResult(job_id=1, success=False) in stores
def test_store_timeout_then_late_completion_no_duplicate(self): @pytest.mark.parametrize("cancel_pending", [False, True])
"""A job timed out by _timeout_pending_store_jobs must not also def test_store_timeout_cancels_stuck_transfer_after_confirmation(
emit a contradictory StoreResult(success=True) when the transport self, cancel_pending, monkeypatch
later reports the same transfer as done.""" ):
# Retry on every poll; pacing is covered by the *_paced tests.
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session() session, conn, transport = _make_session()
_activate(session, conn) _activate(session, conn)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1) session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
@@ -1749,25 +1762,143 @@ class TestServerFlows:
) )
session.poll() session.poll()
tid = next(iter(transport._transfers)) tid = next(iter(transport._transfers))
# Backdate the store job so the next poll times it out.
session._server._store_jobs[1] = time.monotonic() - 60.0 session._server._store_jobs[1] = time.monotonic() - 60.0
stores = session.poll().stores if cancel_pending:
assert StoreResult(job_id=1, success=False) in stores transport._cancel_still_inflight.add(tid)
assert StoreResult(job_id=1, success=True) not in stores for _ in range(3):
assert session.poll().stores == []
assert 1 in session._server._store_jobs
assert tid in session._server._inflight
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert transport._cancel_calls
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert tid not in session._server._inflight
assert tid not in transport._transfers
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
assert len(terminal) == 1
assert terminal[0][TransferDoneMsg.SUCCESS] is False
assert session.poll().stores == []
# Transport later reports the same transfer as done — must not @pytest.mark.parametrize("success", [True, False])
# emit a second (contradictory) StoreResult for job_id=1. def test_store_timeout_keeps_pins_until_cancel_or_late_terminal(self, success):
transport._poll_done.append(tid) session, conn, transport = _make_session()
stores = session.poll().stores _activate(session, conn)
assert all(s.job_id != 1 for s in stores), ( session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
f"unexpected duplicate StoreResult after timeout: {stores}" session.add_stored_blocks("req-2", [b"k2"], [1], job_id=2)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
}
) )
session.poll()
tid = next(iter(transport._transfers))
transport._cancel_still_inflight.add(tid)
def test_store_timeout_then_late_failure_no_duplicate(self): for jid in (1, 2):
"""Symmetric guard: a timed-out job must not also emit a second session._server._store_jobs[jid] = time.monotonic() - 60.0
StoreResult(success=False) when the transport later reports the # A StoreResult releases the primary slot. Only the job still
same transfer as failed.""" # waiting for a fetch may time out while the transfer is active.
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
assert session.poll().stores == []
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
completed = transport._poll_done if success else transport._poll_failed
completed.append(tid)
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
assert len(terminal) == 1
assert terminal[0][TransferDoneMsg.SUCCESS] is False
assert session.poll().stores == []
@pytest.mark.parametrize("success", [True, False, None])
def test_store_timeout_waits_for_other_transfers_in_round(
self, success, monkeypatch
):
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [5, 6],
}
)
session.poll()
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
session.add_stored_blocks("req-1", [b"k2"], [1], job_id=2)
first, second = list(transport._transfers)
session._server._store_jobs[1] = time.monotonic() - 60.0
transport._cancel_still_inflight.add(second)
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert first not in session._server._inflight
assert second in session._server._inflight
assert 2 in session._server._store_jobs
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
# Additional supply must not restart the timed-out round.
session.add_stored_blocks("req-1", [b"k3"], [2], job_id=3)
assert session.poll().stores == [StoreResult(job_id=3, success=False)]
assert set(session._server._inflight) == {second}
if success is None:
transport._cancel_still_inflight.clear()
else:
completed = transport._poll_done if success else transport._poll_failed
completed.append(second)
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
assert len(terminal) == 1
assert terminal[0][TransferDoneMsg.SUCCESS] is False
assert session.poll().stores == []
def test_failed_transfer_keeps_sibling_pins_until_terminal(self, monkeypatch):
"""A failed transfer fails its round, but a sibling transfer that may
still be writing is not force-cancelled: its pin and the round's
TransferDoneMsg wait until it is terminal."""
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1", b"k2"],
FetchMsg.BLOCK_INDEXES: [5, 6],
}
)
session.poll()
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
session.add_stored_blocks("req-1", [b"k2"], [1], job_id=2)
first, second = list(transport._transfers)
transport._cancel_still_inflight.add(second)
transport._poll_failed.append(first)
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
for _ in range(3):
assert session.poll().stores == []
assert second in session._server._inflight
assert 2 in session._server._store_jobs
assert transport._cancel_calls
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
terminal = [m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE]
assert len(terminal) == 1
assert terminal[0][TransferDoneMsg.SUCCESS] is False
assert session.poll().stores == []
def test_abort_during_store_timeout_keeps_unconfirmed_pins(self, monkeypatch):
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session() session, conn, transport = _make_session()
_activate(session, conn) _activate(session, conn)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1) session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
@@ -1782,18 +1913,219 @@ class TestServerFlows:
) )
session.poll() session.poll()
tid = next(iter(transport._transfers)) tid = next(iter(transport._transfers))
transport._cancel_still_inflight.add(tid)
session._server._store_jobs[1] = time.monotonic() - 60.0 session._server._store_jobs[1] = time.monotonic() - 60.0
stores = session.poll().stores assert session.poll().stores == []
assert [s for s in stores if s.job_id == 1] == [ conn.enqueue(
StoreResult(job_id=1, success=False) {
] TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
transport._poll_failed.append(tid) AbortFetchMsg.ROUND_SEQ: 0,
stores = session.poll().stores }
assert all(s.job_id != 1 for s in stores), (
f"unexpected duplicate StoreResult after timeout: {stores}"
) )
assert session.poll().stores == []
session._server._pending_aborts[("req-1", 0)] = time.monotonic() - 60.0
assert session.poll().stores == []
assert 1 in session._server._store_jobs
assert tid in session._server._inflight
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
assert session.poll().stores == []
@staticmethod
def _fake_clock(monkeypatch) -> list[float]:
clock = [1000.0]
monkeypatch.setattr(
server_mod, "time", SimpleNamespace(monotonic=lambda: clock[0])
)
return clock
@staticmethod
def _submit_unconfirmed_transfer(session, conn, transport) -> int:
"""Start one transfer for job 1 whose wait-cancel never confirms."""
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
conn.enqueue(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.ROUND_SEQ: 0,
FetchMsg.KV_REQUEST_ID: "req-1",
FetchMsg.KEYS: [b"k1"],
FetchMsg.BLOCK_INDEXES: [5],
}
)
session.poll()
tid = next(iter(transport._transfers))
transport._cancel_still_inflight.add(tid)
return tid
@staticmethod
def _wait_cancels(transport) -> int:
return sum(1 for _, mode in transport._cancel_calls if mode == "wait")
def test_store_timeout_cancel_retry_is_paced(self, monkeypatch):
clock = self._fake_clock(monkeypatch)
session, conn, transport = _make_session()
_activate(session, conn)
tid = self._submit_unconfirmed_transfer(session, conn, transport)
session._server._store_jobs[1] = clock[0] - 60.0
assert session.poll().stores == []
assert self._wait_cancels(transport) == 1
start = clock[0]
for dt in (0.0, 0.05, 0.099):
clock[0] = start + dt
for _ in range(5):
assert session.poll().stores == []
assert self._wait_cancels(transport) == 1
clock[0] = start + server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
assert session.poll().stores == []
assert self._wait_cancels(transport) == 2
transport._cancel_still_inflight.clear()
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert self._wait_cancels(transport) == 3
assert tid not in session._server._inflight
def test_abort_drain_cancel_retry_is_paced(self, monkeypatch):
clock = self._fake_clock(monkeypatch)
session, conn, transport = _make_session()
_activate(session, conn)
self._submit_unconfirmed_transfer(session, conn, transport)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
AbortFetchMsg.ROUND_SEQ: 0,
}
)
# The first drain attempt is immediate.
session.poll()
assert self._wait_cancels(transport) == 1
for _ in range(5):
session.poll()
assert self._wait_cancels(transport) == 1
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
session.poll()
assert self._wait_cancels(transport) == 2
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
session.poll()
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
def test_unconfirmed_store_cancel_never_acks_or_tears_down(self, monkeypatch):
"""A cancellation that is never confirmed must not release the pin,
send AbortAck, force-cancel, or remove the peer, however long it
stays pending. AbortAck follows the release once it is confirmed."""
clock = self._fake_clock(monkeypatch)
session, conn, transport = _make_session()
_activate(session, conn)
peers = dict(transport._remote_peers)
tid = self._submit_unconfirmed_transfer(session, conn, transport)
session._server._store_jobs[1] = clock[0] - 60.0
assert session.poll().stores == []
abort = {
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
AbortFetchMsg.ROUND_SEQ: 0,
}
started_at = clock[0]
conn.enqueue(dict(abort))
assert session.poll().stores == []
# A duplicate abort neither resets the drain start nor adds a cancel.
conn.enqueue(dict(abort))
assert session.poll().stores == []
assert session._server._pending_aborts[("req-1", 0)] == started_at
# The store sweep and the abort drain share the round's pacing.
assert self._wait_cancels(transport) == 1
ticks = 50
for step in range(1, ticks + 1):
clock[0] = started_at + step * 10.0
assert session.poll().stores == []
assert clock[0] - started_at > _CANCEL_DRAIN_TIMEOUT_S
assert self._wait_cancels(transport) == 1 + ticks
assert all(mode == "wait" for _, mode in transport._cancel_calls)
assert 1 in session._server._store_jobs
assert tid in session._server._inflight
assert transport._remote_peers == peers
assert not any(
m[TYPE_KEY] in (AbortAckMsg.TYPE, TransferDoneMsg.TYPE) for m in conn._sent
)
transport._cancel_still_inflight.clear()
clock[0] += server_mod._CANCEL_RETRY_INTERVAL_S + 1e-3
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert 1 not in session._server._store_jobs
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
assert session.poll().stores == []
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
def test_abort_keeps_pins_only_of_jobs_being_written(self, monkeypatch):
"""An abort fails the round's jobs at once, except those an inflight
transfer still reads; their pins and the ack wait for the transfer."""
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
tid = self._submit_unconfirmed_transfer(session, conn, transport)
# Job 2 supplies a block nobody demanded: no transfer reads it.
session.add_stored_blocks("req-1", [b"k2"], [1], job_id=2)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
AbortFetchMsg.ROUND_SEQ: 0,
}
)
assert session.poll().stores == [StoreResult(job_id=2, success=False)]
for _ in range(3):
assert session.poll().stores == []
assert 1 in session._server._store_jobs
assert tid in session._server._inflight
assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent)
transport._cancel_still_inflight.clear()
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
assert session.poll().stores == []
@pytest.mark.parametrize("success", [True, False])
def test_abort_acks_after_late_terminal_without_transfer_done(
self, success, monkeypatch
):
"""A transfer that finishes on its own after the abort releases its
pin and lets the parked abort ack; the aborted round sends no
TransferDoneMsg."""
monkeypatch.setattr(server_mod, "_CANCEL_RETRY_INTERVAL_S", 0.0)
session, conn, transport = _make_session()
_activate(session, conn)
tid = self._submit_unconfirmed_transfer(session, conn, transport)
conn.enqueue(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: "req-1",
AbortFetchMsg.ROUND_SEQ: 0,
}
)
assert session.poll().stores == []
assert 1 in session._server._store_jobs
completed = transport._poll_done if success else transport._poll_failed
completed.append(tid)
assert session.poll().stores == [StoreResult(job_id=1, success=False)]
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
assert not any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent)
assert session.poll().stores == []
assert sum(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) == 1
def test_fetch_after_symmetric_supply_timed_out_reads_nothing(self): def test_fetch_after_symmetric_supply_timed_out_reads_nothing(self):
"""A lookup pin whose store job timed out has been reported failed, """A lookup pin whose store job timed out has been reported failed,
+6 -6
View File
@@ -296,12 +296,12 @@ class DataTransport(ABC):
return []. Matches the legacy fire-and-forget return []. Matches the legacy fire-and-forget
behavior — the caller does not wait for the behavior — the caller does not wait for the
underlying transfer to drain. underlying transfer to drain.
"wait": attempt to release each handle. If the release "wait": release only transfers that have stopped. A
cannot complete because the transfer is still transfer still PROC/PEND (or whose release fails)
PROC/PEND, the entry stays in the inflight set and stays in the inflight set and its id is included in
its id is included in the returned list. The the returned list. The caller is expected to keep
caller is expected to keep calling poll() until calling poll() until every returned id surfaces in
every returned id surfaces in done/failed. done/failed.
Returns: Returns:
For mode="wait", the subset of *transfer_ids* still For mode="wait", the subset of *transfer_ids* still
+48 -5
View File
@@ -29,6 +29,22 @@ logger = init_logger(__name__)
_EMPTY_POLL_RESULT: PollResult = PollResult(done=(), failed=()) _EMPTY_POLL_RESULT: PollResult = PollResult(done=(), failed=())
def _is_remote_disconnect(exc: Exception) -> bool:
"""True if a NIXL state check failed because the peer disconnected.
NIXL raises ``nixlRemoteDisconnectError`` (NIXL_ERR_REMOTE_DISCONNECT)
from ``check_xfer_state`` once the target endpoint is gone, e.g. after
the peer process died, instead of reporting an ``"ERR"`` state. Matched
by name and status code so the bindings need not be imported here; an
unrecognized error keeps the transfer inflight, the conservative side.
"""
return type(
exc
).__name__ == "nixlRemoteDisconnectError" or "NIXL_ERR_REMOTE_DISCONNECT" in str(
exc
)
class _Inflight(NamedTuple): class _Inflight(NamedTuple):
"""A submitted-but-not-yet-drained transfer. """A submitted-but-not-yet-drained transfer.
@@ -325,7 +341,9 @@ class NixlTransport(DataTransport):
transfer_id, transfer_id,
exc, exc,
) )
continue if not _is_remote_disconnect(exc):
continue
state = "ERR"
if state == "DONE": if state == "DONE":
if done_ids is None: if done_ids is None:
done_ids = [] done_ids = []
@@ -357,10 +375,14 @@ class NixlTransport(DataTransport):
) -> list[int]: ) -> list[int]:
"""Cancel inflight transfers by their IDs. """Cancel inflight transfers by their IDs.
See ``DataTransport.cancel`` for the contract. In "wait" mode, See ``DataTransport.cancel`` for the contract. In "wait" mode, a
transfers whose ``release_xfer_handle`` raises (NIXL could not transfer is released only once NIXL reports it terminal. Releasing
complete the abort because the backend is still draining) stay a PROC/PEND handle does not stop the WRITE: the UCX backend frees
in ``self._inflight`` so a later ``poll()`` will observe them. the request without cancelling RMA, and the bytes keep landing. So
PROC/PEND transfers, and those whose state check or
``release_xfer_handle`` raises, stay in ``self._inflight`` for a
later ``poll()`` to observe. A state check that fails because the
peer disconnected counts as terminal.
""" """
if mode == "immediate": if mode == "immediate":
handles = [ handles = [
@@ -376,6 +398,27 @@ class NixlTransport(DataTransport):
entry = self._inflight.get(tid) entry = self._inflight.get(tid)
if entry is None: if entry is None:
continue continue
try:
state = self._agent.check_xfer_state(entry.handle)
except Exception as exc:
logger.debug(
"NixlTransport %s: cancel state check failed for "
"transfer_id=%d: %s",
self._agent_name,
tid,
exc,
)
if not _is_remote_disconnect(exc):
still_inflight.append(tid)
continue
# The handle may no longer be releasable; it is terminal
# either way, so drop it instead of retrying forever.
self._release_handles([entry.handle])
del self._inflight[tid]
continue
if state in ("PROC", "PEND"):
still_inflight.append(tid)
continue
try: try:
self._agent.release_xfer_handle(entry.handle) self._agent.release_xfer_handle(entry.handle)
except Exception as exc: except Exception as exc:
+5 -4
View File
@@ -1018,11 +1018,12 @@ class P2PSecondaryTierManager(SecondaryTierManager):
def _drain_inflight_for_shutdown(self) -> None: def _drain_inflight_for_shutdown(self) -> None:
"""Best-effort drain of inflight transfers before closing _data. """Best-effort drain of inflight transfers before closing _data.
Mirrors session._drain_abort but as a single bounded loop. Collects Collects inflight transfer_ids from each session, repeatedly calls
inflight transfer_ids from each session, repeatedly calls
_data.cancel(..., mode="wait") and _data.poll() so handles can _data.cancel(..., mode="wait") and _data.poll() so handles can
surface as done/failed, and falls back to mode="immediate" once surface as done/failed. Unlike session._drain_abort, which never
_SHUTDOWN_DRAIN_TIMEOUT_S elapses so a wedged peer can't hang us. force-releases, it falls back to mode="immediate" once
_SHUTDOWN_DRAIN_TIMEOUT_S elapses: the engine is exiting, and a
wedged peer must not hang shutdown.
""" """
ids = [tid for s in self._sessions.values() for tid in s._server._inflight] ids = [tid for s in self._sessions.values() for tid in s._server._inflight]
if not ids: if not ids:
+179 -53
View File
@@ -39,6 +39,10 @@ logger = init_logger(__name__)
_STORE_TIMEOUT_S = 30.0 _STORE_TIMEOUT_S = 30.0
_CANCEL_DRAIN_TIMEOUT_S = 10.0 _CANCEL_DRAIN_TIMEOUT_S = 10.0
# Minimum spacing between wait-mode cancel attempts for one round. Each
# attempt calls into NIXL on the scheduler thread, so an unconfirmed
# cancellation is retried at this rate instead of on every poll tick.
_CANCEL_RETRY_INTERVAL_S = 0.1
# Cap on time the server holds a HIT_PENDING / RETRY key from an inbound # Cap on time the server holds a HIT_PENDING / RETRY key from an inbound
# LookupMsg before falling back to MISS. Long enough that an in-flight # LookupMsg before falling back to MISS. Long enough that an in-flight
# primary write or a just-started promotion typically completes; short # primary write or a just-started promotion typically completes; short
@@ -85,6 +89,13 @@ class _OutboundRequestState:
remaining: int = 0 # blocks that need to be transferred to client remaining: int = 0 # blocks that need to be transferred to client
finishing: bool = False # Signal finish request ASAP finishing: bool = False # Signal finish request ASAP
inflight: int = 0 # transfers submitted for this round, not yet polled inflight: int = 0 # transfers submitted for this round, not yet polled
# The round was cancelled (a store job timed out, or the peer aborted
# it) while transfers may still be writing to the peer. Its jobs fail,
# but a job keeps its pin until every transfer carrying it is terminal:
# NIXL cannot stop a WRITE that is already in flight.
cancelled: bool = False
# Last wait-mode cancel attempt for this round's transfers (monotonic).
last_cancel_at: float | None = None
# Job IDs that submit_store'd blocks for this round and have not # Job IDs that submit_store'd blocks for this round and have not
# yet emitted a StoreResult. The terminal-finalize helper drains # yet emitted a StoreResult. The terminal-finalize helper drains
# this set; poll-done and poll-failed discard entries as their # this set; poll-done and poll-failed discard entries as their
@@ -272,6 +283,9 @@ class ServerRole:
# Parked aborts awaiting drain, keyed by (kv_request_id, round) # Parked aborts awaiting drain, keyed by (kv_request_id, round)
# with the abort start time. # with the abort start time.
self._pending_aborts: dict[tuple[str, int], float] = {} self._pending_aborts: dict[tuple[str, int], float] = {}
# Parked aborts already warned about for exceeding
# _CANCEL_DRAIN_TIMEOUT_S.
self._slow_aborts: set[tuple[str, int]] = set()
# ------------------------------------------------------------------ # ------------------------------------------------------------------
# State helpers # State helpers
@@ -322,6 +336,10 @@ class ServerRole:
rnd = st.outbound.get(round_seq) rnd = st.outbound.get(round_seq)
if rnd is None: if rnd is None:
rnd = st.outbound[round_seq] = _OutboundRequestState() rnd = st.outbound[round_seq] = _OutboundRequestState()
if rnd.cancelled:
self._store_jobs.pop(job_id)
self._pending_store_results.append(StoreResult(job_id, success=False))
return
if from_lookup: if from_lookup:
rnd.lookup_supplied = True rnd.lookup_supplied = True
result = rnd.add_stored_blocks(keys, block_ids, job_id) result = rnd.add_stored_blocks(keys, block_ids, job_id)
@@ -765,7 +783,9 @@ class ServerRole:
tid, tid,
) )
continue continue
results.extend(self._settle_xfer_jobs(xfer, success=True)) results.extend(
self._settle_xfer_jobs(xfer, success=not xfer.round.cancelled)
)
rnd = xfer.round rnd = xfer.round
st = self._requests.get(xfer.kv_request_id) st = self._requests.get(xfer.kv_request_id)
if st is not None and st.outbound.get(xfer.round_key) is rnd: if st is not None and st.outbound.get(xfer.round_key) is rnd:
@@ -773,7 +793,10 @@ class ServerRole:
assert rnd.remaining >= 0, ( assert rnd.remaining >= 0, (
f"remaining went negative for kv_request_id={xfer.kv_request_id}" f"remaining went negative for kv_request_id={xfer.kv_request_id}"
) )
if rnd.remaining == 0: if rnd.cancelled:
if rnd.inflight == 0:
self._finalize_cancelled(xfer.kv_request_id, xfer.round_key)
elif rnd.remaining == 0:
self._finalize_outbound( self._finalize_outbound(
xfer.kv_request_id, xfer.round_key, success=True xfer.kv_request_id, xfer.round_key, success=True
) )
@@ -783,7 +806,6 @@ class ServerRole:
) )
self._maybe_prune(xfer.kv_request_id) self._maybe_prune(xfer.kv_request_id)
failed_rounds: list[tuple[str, _OutboundRequestState]] | None = None
for tid in poll_result.failed: for tid in poll_result.failed:
xfer = self._inflight_pop(tid) xfer = self._inflight_pop(tid)
if xfer is None: if xfer is None:
@@ -796,38 +818,21 @@ class ServerRole:
tid, tid,
) )
continue continue
results.extend(self._settle_xfer_jobs(xfer, success=False))
rnd = xfer.round rnd = xfer.round
st = self._requests.get(xfer.kv_request_id) st = self._requests.get(xfer.kv_request_id)
if st is not None and st.outbound.get(xfer.round_key) is rnd: in_round = st is not None and st.outbound.get(xfer.round_key) is rnd
del st.outbound[xfer.round_key] if in_round and not rnd.cancelled:
if failed_rounds is None: # One failed transfer fails the round, but its other
failed_rounds = [] # transfers may still be writing. Cancel them like a
failed_rounds.append((xfer.kv_request_id, rnd)) # timed-out round: their pins and the TransferDoneMsg wait
self._send( # until they are terminal.
{ rnd.cancelled = True
TYPE_KEY: TransferDoneMsg.TYPE, results.extend(self._fail_unwritten_round_jobs(rnd))
TransferDoneMsg.KV_REQUEST_ID: xfer.kv_request_id, results.extend(self._settle_xfer_jobs(xfer, success=False))
TransferDoneMsg.SUCCESS: False, if in_round and rnd.inflight == 0:
TransferDoneMsg.ROUND_SEQ: xfer.round_key, self._finalize_cancelled(xfer.kv_request_id, xfer.round_key)
}
)
self._maybe_prune(xfer.kv_request_id) self._maybe_prune(xfer.kv_request_id)
# Cancel each failed round's other inflight and fail its
# remaining store jobs — nothing else will settle them.
if failed_rounds:
for kv_request_id, rnd in failed_rounds:
ids_to_cancel = [
tid for tid, x in self._inflight.items() if x.round is rnd
]
for tid in ids_to_cancel:
self._inflight_pop(tid)
if ids_to_cancel:
self._transport.cancel(ids_to_cancel)
results.extend(self._fail_round_jobs(rnd))
self._maybe_prune(kv_request_id)
return results return results
def collect_idle_timeouts(self) -> list[StoreResult]: def collect_idle_timeouts(self) -> list[StoreResult]:
@@ -870,6 +875,7 @@ class ServerRole:
self._requests.clear() self._requests.clear()
self._serve_pending.clear() self._serve_pending.clear()
self._pending_aborts.clear() self._pending_aborts.clear()
self._slow_aborts.clear()
self._finished_lookup_ctxs.clear() self._finished_lookup_ctxs.clear()
return failed_stores, failed_serves return failed_stores, failed_serves
@@ -918,7 +924,10 @@ class ServerRole:
result. result.
""" """
results: list[StoreResult] = [] results: list[StoreResult] = []
still_pinned = self._read_job_ids() if xfer.round.cancelled else set()
for job_id in xfer.job_ids: for job_id in xfer.job_ids:
if job_id in still_pinned:
continue
if self._store_jobs.pop(job_id, None) is None: if self._store_jobs.pop(job_id, None) is None:
continue continue
results.append(StoreResult(job_id=job_id, success=success)) results.append(StoreResult(job_id=job_id, success=success))
@@ -939,6 +948,48 @@ class ServerRole:
rnd.pending_job_ids.clear() rnd.pending_job_ids.clear()
return results return results
def _finalize_cancelled(self, kv_request_id: str, round_key: int) -> None:
"""Finalize a cancelled round once its last transfer is terminal.
An aborted round gets no TransferDoneMsg: its parked abort answers
the peer with AbortAckMsg on the next drain.
"""
self._finalize_outbound(
kv_request_id,
round_key,
success=False,
send_done=(kv_request_id, round_key) not in self._pending_aborts,
)
def _read_job_ids(self, rnd: _OutboundRequestState | None = None) -> set[int]:
"""Jobs whose blocks an inflight transfer (of ``rnd``, if given)
still reads; their pins must be held."""
return {
jid
for xfer in self._inflight.values()
if rnd is None or xfer.round is rnd
for jid in xfer.job_ids
}
def _fail_unwritten_round_jobs(
self, rnd: _OutboundRequestState
) -> list[StoreResult]:
"""Fail a cancelled round's jobs that no inflight transfer carries.
Jobs that an inflight transfer carries keep their pins; they settle
when that transfer turns terminal (``_settle_xfer_jobs``).
"""
written = self._read_job_ids(rnd)
results: list[StoreResult] = []
for job_id in list(rnd.pending_job_ids):
if job_id in written:
continue
rnd.pending_job_ids.discard(job_id)
if self._store_jobs.pop(job_id, None) is None:
continue
results.append(StoreResult(job_id=job_id, success=False))
return results
def _finalize_outbound( def _finalize_outbound(
self, self,
kv_request_id: str, kv_request_id: str,
@@ -953,7 +1004,12 @@ class ServerRole:
fetch). Other rounds of the id are untouched. fetch). Other rounds of the id are untouched.
""" """
st = self._requests[kv_request_id] st = self._requests[kv_request_id]
req = st.outbound.pop(round_key) req = st.outbound[round_key]
if req.cancelled and req.inflight:
return
del st.outbound[round_key]
if req.cancelled:
success = False
if success is None: if success is None:
success = req.demand_received and req.remaining == 0 success = req.demand_received and req.remaining == 0
settled = self._fail_round_jobs(req) if not success else None settled = self._fail_round_jobs(req) if not success else None
@@ -992,41 +1048,56 @@ class ServerRole:
def _drain_abort(self, kv_request_id: str, round_seq: int) -> None: def _drain_abort(self, kv_request_id: str, round_seq: int) -> None:
"""One drain attempt for a pending abort. """One drain attempt for a pending abort.
Detaches the aborted round, then asks the transport to cancel its Cancels the aborted round, then asks the transport to cancel its
inflight transfers in ``mode="wait"``. Sends ``AbortAckMsg`` once inflight transfers in ``mode="wait"``. Sends ``AbortAckMsg`` only
nothing remains inflight, or after ``_CANCEL_DRAIN_TIMEOUT_S`` once none of them can still write to the peer: the ack lets the
falls back to ``mode="immediate"`` and acks anyway. peer reuse the destination blocks. A transfer that NIXL does not
report terminal is never force-released, however long the drain
takes: neither releasing its handle nor removing the remote agent
stops a WRITE that is already in flight. Past
``_CANCEL_DRAIN_TIMEOUT_S`` the drain only logs a warning.
""" """
st = self._requests[kv_request_id] st = self._requests[kv_request_id]
rnd = st.outbound.pop(round_seq, None) rnd = st.outbound.get(round_seq)
if rnd is not None: if rnd is not None and not rnd.cancelled:
# Its transfers are being cancelled; fail its jobs now # Fail the round's jobs now instead of leaking them to the
# instead of leaking them to the store timeout. # store timeout, but keep the pins of those still being written.
self._pending_store_results.extend(self._fail_round_jobs(rnd)) rnd.cancelled = True
self._pending_store_results.extend(self._fail_unwritten_round_jobs(rnd))
ids = [ ids = [
tid tid
for tid, x in self._inflight.items() for tid, x in self._inflight.items()
if x.kv_request_id == kv_request_id and x.round_key == round_seq if x.kv_request_id == kv_request_id and x.round_key == round_seq
] ]
if not ids: if not ids:
if rnd is not None and st.outbound.get(round_seq) is rnd:
self._finalize_outbound(
kv_request_id, round_seq, success=False, send_done=False
)
self._finalize_abort(kv_request_id, round_seq) self._finalize_abort(kv_request_id, round_seq)
return return
started_at = self._pending_aborts[(kv_request_id, round_seq)] now = time.monotonic()
if time.monotonic() - started_at >= _CANCEL_DRAIN_TIMEOUT_S: key = (kv_request_id, round_seq)
for tid in ids: if (
self._inflight_pop(tid) now - self._pending_aborts[key] >= _CANCEL_DRAIN_TIMEOUT_S
self._transport.cancel(ids, mode="immediate") and key not in self._slow_aborts
):
self._slow_aborts.add(key)
logger.warning( logger.warning(
"P2PSession %s: cancel drain timed out for kv_request_id=%s," "P2PSession %s: abort of kv_request_id=%s round=%s still "
" force-canceled %d transfers", "waiting after %.0fs for %d transfers to stop; holding their "
"pins and the AbortAck until NIXL reports them terminal",
self._peer_id, self._peer_id,
kv_request_id, kv_request_id,
round_seq,
_CANCEL_DRAIN_TIMEOUT_S,
len(ids), len(ids),
) )
self._finalize_abort(kv_request_id, round_seq)
return
# All of the aborted round's transfers share one round state.
if not self._cancel_due(self._inflight[ids[0]].round, now):
return
still = self._transport.cancel(ids, mode="wait") still = self._transport.cancel(ids, mode="wait")
# Tids the transport successfully released are gone from its # Tids the transport successfully released are gone from its
# _inflight; mirror that in session bookkeeping so they don't # _inflight; mirror that in session bookkeeping so they don't
@@ -1037,10 +1108,30 @@ class ServerRole:
if tid not in still_set: if tid not in still_set:
self._inflight_pop(tid) self._inflight_pop(tid)
if not still: if not still:
if rnd is not None and st.outbound.get(round_seq) is rnd:
self._finalize_outbound(
kv_request_id, round_seq, success=False, send_done=False
)
self._finalize_abort(kv_request_id, round_seq) self._finalize_abort(kv_request_id, round_seq)
@staticmethod
def _cancel_due(rnd: _OutboundRequestState, now: float) -> bool:
"""Rate-limit wait-mode cancels of ``rnd``'s transfers.
Shared by the store-timeout sweep and the abort drain, so a round
gets at most one attempt per ``_CANCEL_RETRY_INTERVAL_S``.
"""
if (
rnd.last_cancel_at is not None
and now - rnd.last_cancel_at < _CANCEL_RETRY_INTERVAL_S
):
return False
rnd.last_cancel_at = now
return True
def _finalize_abort(self, kv_request_id: str, round_seq: int) -> None: def _finalize_abort(self, kv_request_id: str, round_seq: int) -> None:
self._pending_aborts.pop((kv_request_id, round_seq), None) self._pending_aborts.pop((kv_request_id, round_seq), None)
self._slow_aborts.discard((kv_request_id, round_seq))
self._send( self._send(
{ {
TYPE_KEY: AbortAckMsg.TYPE, TYPE_KEY: AbortAckMsg.TYPE,
@@ -1121,7 +1212,8 @@ class ServerRole:
def _timeout_pending_store_jobs(self) -> list[StoreResult]: def _timeout_pending_store_jobs(self) -> list[StoreResult]:
if not self._store_jobs: if not self._store_jobs:
return [] return []
deadline = time.monotonic() - _STORE_TIMEOUT_S now = time.monotonic()
deadline = now - _STORE_TIMEOUT_S
timed_out: list[int] | None = None timed_out: list[int] | None = None
for jid, submitted_at in self._store_jobs.items(): for jid, submitted_at in self._store_jobs.items():
if submitted_at <= deadline: if submitted_at <= deadline:
@@ -1129,10 +1221,44 @@ class ServerRole:
timed_out = [] timed_out = []
timed_out.append(jid) timed_out.append(jid)
if timed_out is None: if timed_out is None:
return [] if not any(xfer.round.cancelled for xfer in self._inflight.values()):
return []
timed_out = []
results: list[StoreResult] = [] results: list[StoreResult] = []
expired = set(timed_out)
rounds = {
(xfer.kv_request_id, xfer.round_key): xfer.round
for xfer in self._inflight.values()
if xfer.round.cancelled or expired.intersection(xfer.job_ids)
}
for (kv_request_id, round_key), rnd in rounds.items():
rnd.cancelled = True
if not self._cancel_due(rnd, now):
continue
tids = [tid for tid, xfer in self._inflight.items() if xfer.round is rnd]
still = set(self._transport.cancel(tids, mode="wait"))
for tid in tids:
if tid in still:
continue
xfer = self._inflight_pop(tid)
assert xfer is not None
results.extend(self._settle_xfer_jobs(xfer, success=False))
st = self._requests.get(kv_request_id)
if (
rnd.inflight == 0
and st is not None
and st.outbound.get(round_key) is rnd
):
self._finalize_cancelled(kv_request_id, round_key)
# Cancellation may still be draining. Keep every affected pin until
# the transport confirms that no transfer reads its primary slot.
inflight_job_ids = self._read_job_ids()
for jid in timed_out: for jid in timed_out:
del self._store_jobs[jid] if jid in inflight_job_ids:
continue
if self._store_jobs.pop(jid, None) is None:
continue
results.append(StoreResult(job_id=jid, success=False)) results.append(StoreResult(job_id=jid, success=False))
logger.warning("P2PSession %s: store job %d timed out", self._peer_id, jid) logger.warning("P2PSession %s: store job %d timed out", self._peer_id, jid)
# A failed store job's primary slots are unpinned by the engine, so its # A failed store job's primary slots are unpinned by the engine, so its