[Bugfix][KV Connector] Reap expired NIXL leases behind a heartbeated head (#58292)

Signed-off-by: GokayAI <60583610+gokay-ai@users.noreply.github.com>
Co-authored-by: GokayAI <gokay-ai@users.noreply.github.com>
This commit is contained in:
GokayAI
2026-09-25 17:23:44 +00:00
committed by GitHub
co-authored by GokayAI
parent c88f4824ff
commit c89b5b4176
2 changed files with 117 additions and 18 deletions
@@ -2951,24 +2951,7 @@ class NixlBaseConnectorWorker:
self._sync_device_after_direct_recv(direct_device_recving)
# Handle timeout to avoid stranding blocks on remote.
now = time.perf_counter()
while self._reqs_to_send:
req_id, expires = next(iter(self._reqs_to_send.items()))
# Sorted dict, oldest requests are put first so we can exit early.
if now < expires:
break
count = self.consumer_notification_counts_by_req.pop(req_id, 0)
self.expected_consumer_notifications_by_req.pop(req_id, None)
self.xfer_stats.record_kv_expired_req()
logger.warning(
"Releasing expired KV blocks for request %s which were "
"retrieved by %d remote worker(s) before lease expired.",
req_id,
count,
)
self._reqs_to_process.remove(req_id)
del self._reqs_to_send[req_id]
done_sending.add(req_id)
self._reap_expired_send_leases(done_sending)
return KVConnectorTransferResults(
finished_sending=done_sending,
@@ -2996,6 +2979,34 @@ class NixlBaseConnectorWorker:
"""
raise NotImplementedError
def _reap_expired_send_leases(self, done_sending: set[str]) -> None:
"""Reclaim expired send-side KV leases into ``done_sending``.
``_reqs_to_send`` is not ordered by expiry: heartbeats update the
deadline in place, and mixed TTLs share the map, so a live head
entry can sit in front of already-expired ones. Scan every entry
rather than stopping at the first still-live request.
"""
if not self._reqs_to_send:
return
now = time.perf_counter()
expired = [
req_id for req_id, expires in self._reqs_to_send.items() if now >= expires
]
for req_id in expired:
count = self.consumer_notification_counts_by_req.pop(req_id, 0)
self.expected_consumer_notifications_by_req.pop(req_id, None)
self.xfer_stats.record_kv_expired_req()
logger.warning(
"Releasing expired KV blocks for request %s which were "
"retrieved by %d remote worker(s) before lease expired.",
req_id,
count,
)
self._reqs_to_process.remove(req_id)
del self._reqs_to_send[req_id]
done_sending.add(req_id)
def _handle_heartbeat(self, payload: str) -> None:
"""Extend leases for requests referenced in a heartbeat.