From c89b5b4176066d689a5fe40b0bea8b425ac0c65b Mon Sep 17 00:00:00 2001 From: GokayAI <60583610+gokay-ai@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:23:44 +0300 Subject: [PATCH] [Bugfix][KV Connector] Reap expired NIXL leases behind a heartbeated head (#58292) Signed-off-by: GokayAI <60583610+gokay-ai@users.noreply.github.com> Co-authored-by: GokayAI --- .../kv_connector/unit/test_nixl_heartbeat.py | 88 +++++++++++++++++++ .../kv_connector/v1/nixl/base_worker.py | 47 ++++++---- 2 files changed, 117 insertions(+), 18 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_nixl_heartbeat.py b/tests/v1/kv_connector/unit/test_nixl_heartbeat.py index 345e10621f..f2d95daa48 100644 --- a/tests/v1/kv_connector/unit/test_nixl_heartbeat.py +++ b/tests/v1/kv_connector/unit/test_nixl_heartbeat.py @@ -30,6 +30,10 @@ def _worker_stub(): w = object.__new__(NixlConnectorWorker) w._reqs_to_send = {} w._lease_extension = 20 + w._reqs_to_process = set() + w.consumer_notification_counts_by_req = {} + w.expected_consumer_notifications_by_req = {} + w.xfer_stats = MagicMock() return w @@ -163,3 +167,87 @@ def test_handle_heartbeat(): assert w._reqs_to_send["req-b"] >= far_future # req-unknown: not added. assert "req-unknown" not in w._reqs_to_send + + +def test_reaper_reclaims_expired_leases_behind_heartbeated_head(monkeypatch): + """A heartbeated head entry must not strand expired leases behind it. + + Mirrors the stdlib repro in https://github.com/vllm-project/vllm/issues/58222: + A is inserted first and kept alive via heartbeats; B and C expire and + must still be reaped. The old early-exit scan returned nothing here. + """ + w = _worker_stub() + w._reqs_to_process.update(("A", "B", "C")) + # Insertion order is A, B, C. Heartbeats will keep A at the head. + w._reqs_to_send = {"A": 30.0, "B": 32.0, "C": 34.0} + + from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker + + now = {"t": 0.0} + monkeypatch.setattr(base_worker.time, "perf_counter", lambda: now["t"]) + for t in (20.0, 40.0, 60.0, 80.0): + now["t"] = t + w._handle_heartbeat("A") + + # A was renewed in place: still the dict head, expiry 100. + assert next(iter(w._reqs_to_send)) == "A" + assert w._reqs_to_send["A"] == pytest.approx(100.0) + + done: set[str] = set() + now["t"] = 50.0 + w._reap_expired_send_leases(done) + + assert done == {"B", "C"} + assert list(w._reqs_to_send) == ["A"] + assert w._reqs_to_process == {"A"} + assert w.xfer_stats.record_kv_expired_req.call_count == 2 + + # now == expires is expired (now >= expires): A is due at 100. + done_eq: set[str] = set() + now["t"] = 100.0 + w._reap_expired_send_leases(done_eq) + assert done_eq == {"A"} + assert not w._reqs_to_send + assert not w._reqs_to_process + assert w.xfer_stats.record_kv_expired_req.call_count == 3 + + +def test_reaper_reclaims_when_now_equals_expiry(monkeypatch): + """The reaper must reclaim on the now == expires boundary.""" + w = _worker_stub() + w._reqs_to_process.add("eq") + w._reqs_to_send = {"eq": 50.0} + + from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker + + monkeypatch.setattr(base_worker.time, "perf_counter", lambda: 50.0) + + done: set[str] = set() + w._reap_expired_send_leases(done) + + assert done == {"eq"} + assert not w._reqs_to_send + assert not w._reqs_to_process + w.xfer_stats.record_kv_expired_req.assert_called_once() + + +def test_reaper_reclaims_shorter_lease_behind_later_deadline(monkeypatch): + """A later-inserted longer TTL must not block an earlier-expiring entry. + + Delete+reinsert on heartbeat cannot restore expiry order in this case + (issue #58222), so the reaper must scan the whole map. + """ + w = _worker_stub() + w._reqs_to_process.update(("X", "Y")) + w._reqs_to_send = {"X": 55.0, "Y": 46.0} + + from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker + + monkeypatch.setattr(base_worker.time, "perf_counter", lambda: 50.0) + + done: set[str] = set() + w._reap_expired_send_leases(done) + + assert done == {"Y"} + assert list(w._reqs_to_send) == ["X"] + assert w._reqs_to_process == {"X"} diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index b0257de2e5..b630dedfc3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -2951,24 +2951,7 @@ class NixlBaseConnectorWorker: self._sync_device_after_direct_recv(direct_device_recving) # Handle timeout to avoid stranding blocks on remote. - now = time.perf_counter() - while self._reqs_to_send: - req_id, expires = next(iter(self._reqs_to_send.items())) - # Sorted dict, oldest requests are put first so we can exit early. - if now < expires: - break - count = self.consumer_notification_counts_by_req.pop(req_id, 0) - self.expected_consumer_notifications_by_req.pop(req_id, None) - self.xfer_stats.record_kv_expired_req() - logger.warning( - "Releasing expired KV blocks for request %s which were " - "retrieved by %d remote worker(s) before lease expired.", - req_id, - count, - ) - self._reqs_to_process.remove(req_id) - del self._reqs_to_send[req_id] - done_sending.add(req_id) + self._reap_expired_send_leases(done_sending) return KVConnectorTransferResults( finished_sending=done_sending, @@ -2996,6 +2979,34 @@ class NixlBaseConnectorWorker: """ raise NotImplementedError + def _reap_expired_send_leases(self, done_sending: set[str]) -> None: + """Reclaim expired send-side KV leases into ``done_sending``. + + ``_reqs_to_send`` is not ordered by expiry: heartbeats update the + deadline in place, and mixed TTLs share the map, so a live head + entry can sit in front of already-expired ones. Scan every entry + rather than stopping at the first still-live request. + """ + if not self._reqs_to_send: + return + now = time.perf_counter() + expired = [ + req_id for req_id, expires in self._reqs_to_send.items() if now >= expires + ] + for req_id in expired: + count = self.consumer_notification_counts_by_req.pop(req_id, 0) + self.expected_consumer_notifications_by_req.pop(req_id, None) + self.xfer_stats.record_kv_expired_req() + logger.warning( + "Releasing expired KV blocks for request %s which were " + "retrieved by %d remote worker(s) before lease expired.", + req_id, + count, + ) + self._reqs_to_process.remove(req_id) + del self._reqs_to_send[req_id] + done_sending.add(req_id) + def _handle_heartbeat(self, payload: str) -> None: """Extend leases for requests referenced in a heartbeat.