[Bugfix][KV Connector] Reap expired NIXL leases behind a heartbeated head (#58292)

Signed-off-by: GokayAI <60583610+gokay-ai@users.noreply.github.com>
Co-authored-by: GokayAI <gokay-ai@users.noreply.github.com>
This commit is contained in:
GokayAI
2026-09-25 17:23:44 +00:00
committed by GitHub
co-authored by GokayAI
parent c88f4824ff
commit c89b5b4176
2 changed files with 117 additions and 18 deletions
@@ -30,6 +30,10 @@ def _worker_stub():
w = object.__new__(NixlConnectorWorker)
w._reqs_to_send = {}
w._lease_extension = 20
w._reqs_to_process = set()
w.consumer_notification_counts_by_req = {}
w.expected_consumer_notifications_by_req = {}
w.xfer_stats = MagicMock()
return w
@@ -163,3 +167,87 @@ def test_handle_heartbeat():
assert w._reqs_to_send["req-b"] >= far_future
# req-unknown: not added.
assert "req-unknown" not in w._reqs_to_send
def test_reaper_reclaims_expired_leases_behind_heartbeated_head(monkeypatch):
"""A heartbeated head entry must not strand expired leases behind it.
Mirrors the stdlib repro in https://github.com/vllm-project/vllm/issues/58222:
A is inserted first and kept alive via heartbeats; B and C expire and
must still be reaped. The old early-exit scan returned nothing here.
"""
w = _worker_stub()
w._reqs_to_process.update(("A", "B", "C"))
# Insertion order is A, B, C. Heartbeats will keep A at the head.
w._reqs_to_send = {"A": 30.0, "B": 32.0, "C": 34.0}
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker
now = {"t": 0.0}
monkeypatch.setattr(base_worker.time, "perf_counter", lambda: now["t"])
for t in (20.0, 40.0, 60.0, 80.0):
now["t"] = t
w._handle_heartbeat("A")
# A was renewed in place: still the dict head, expiry 100.
assert next(iter(w._reqs_to_send)) == "A"
assert w._reqs_to_send["A"] == pytest.approx(100.0)
done: set[str] = set()
now["t"] = 50.0
w._reap_expired_send_leases(done)
assert done == {"B", "C"}
assert list(w._reqs_to_send) == ["A"]
assert w._reqs_to_process == {"A"}
assert w.xfer_stats.record_kv_expired_req.call_count == 2
# now == expires is expired (now >= expires): A is due at 100.
done_eq: set[str] = set()
now["t"] = 100.0
w._reap_expired_send_leases(done_eq)
assert done_eq == {"A"}
assert not w._reqs_to_send
assert not w._reqs_to_process
assert w.xfer_stats.record_kv_expired_req.call_count == 3
def test_reaper_reclaims_when_now_equals_expiry(monkeypatch):
"""The reaper must reclaim on the now == expires boundary."""
w = _worker_stub()
w._reqs_to_process.add("eq")
w._reqs_to_send = {"eq": 50.0}
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker
monkeypatch.setattr(base_worker.time, "perf_counter", lambda: 50.0)
done: set[str] = set()
w._reap_expired_send_leases(done)
assert done == {"eq"}
assert not w._reqs_to_send
assert not w._reqs_to_process
w.xfer_stats.record_kv_expired_req.assert_called_once()
def test_reaper_reclaims_shorter_lease_behind_later_deadline(monkeypatch):
"""A later-inserted longer TTL must not block an earlier-expiring entry.
Delete+reinsert on heartbeat cannot restore expiry order in this case
(issue #58222), so the reaper must scan the whole map.
"""
w = _worker_stub()
w._reqs_to_process.update(("X", "Y"))
w._reqs_to_send = {"X": 55.0, "Y": 46.0}
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker
monkeypatch.setattr(base_worker.time, "perf_counter", lambda: 50.0)
done: set[str] = set()
w._reap_expired_send_leases(done)
assert done == {"Y"}
assert list(w._reqs_to_send) == ["X"]
assert w._reqs_to_process == {"X"}
@@ -2951,24 +2951,7 @@ class NixlBaseConnectorWorker:
self._sync_device_after_direct_recv(direct_device_recving)
# Handle timeout to avoid stranding blocks on remote.
now = time.perf_counter()
while self._reqs_to_send:
req_id, expires = next(iter(self._reqs_to_send.items()))
# Sorted dict, oldest requests are put first so we can exit early.
if now < expires:
break
count = self.consumer_notification_counts_by_req.pop(req_id, 0)
self.expected_consumer_notifications_by_req.pop(req_id, None)
self.xfer_stats.record_kv_expired_req()
logger.warning(
"Releasing expired KV blocks for request %s which were "
"retrieved by %d remote worker(s) before lease expired.",
req_id,
count,
)
self._reqs_to_process.remove(req_id)
del self._reqs_to_send[req_id]
done_sending.add(req_id)
self._reap_expired_send_leases(done_sending)
return KVConnectorTransferResults(
finished_sending=done_sending,
@@ -2996,6 +2979,34 @@ class NixlBaseConnectorWorker:
"""
raise NotImplementedError
def _reap_expired_send_leases(self, done_sending: set[str]) -> None:
"""Reclaim expired send-side KV leases into ``done_sending``.
``_reqs_to_send`` is not ordered by expiry: heartbeats update the
deadline in place, and mixed TTLs share the map, so a live head
entry can sit in front of already-expired ones. Scan every entry
rather than stopping at the first still-live request.
"""
if not self._reqs_to_send:
return
now = time.perf_counter()
expired = [
req_id for req_id, expires in self._reqs_to_send.items() if now >= expires
]
for req_id in expired:
count = self.consumer_notification_counts_by_req.pop(req_id, 0)
self.expected_consumer_notifications_by_req.pop(req_id, None)
self.xfer_stats.record_kv_expired_req()
logger.warning(
"Releasing expired KV blocks for request %s which were "
"retrieved by %d remote worker(s) before lease expired.",
req_id,
count,
)
self._reqs_to_process.remove(req_id)
del self._reqs_to_send[req_id]
done_sending.add(req_id)
def _handle_heartbeat(self, payload: str) -> None:
"""Extend leases for requests referenced in a heartbeat.