[Bugfix][KV Connector] Reap expired NIXL leases behind a heartbeated head (#58292)
Signed-off-by: GokayAI <60583610+gokay-ai@users.noreply.github.com> Co-authored-by: GokayAI <gokay-ai@users.noreply.github.com>
This commit is contained in:
@@ -2951,24 +2951,7 @@ class NixlBaseConnectorWorker:
|
||||
self._sync_device_after_direct_recv(direct_device_recving)
|
||||
|
||||
# Handle timeout to avoid stranding blocks on remote.
|
||||
now = time.perf_counter()
|
||||
while self._reqs_to_send:
|
||||
req_id, expires = next(iter(self._reqs_to_send.items()))
|
||||
# Sorted dict, oldest requests are put first so we can exit early.
|
||||
if now < expires:
|
||||
break
|
||||
count = self.consumer_notification_counts_by_req.pop(req_id, 0)
|
||||
self.expected_consumer_notifications_by_req.pop(req_id, None)
|
||||
self.xfer_stats.record_kv_expired_req()
|
||||
logger.warning(
|
||||
"Releasing expired KV blocks for request %s which were "
|
||||
"retrieved by %d remote worker(s) before lease expired.",
|
||||
req_id,
|
||||
count,
|
||||
)
|
||||
self._reqs_to_process.remove(req_id)
|
||||
del self._reqs_to_send[req_id]
|
||||
done_sending.add(req_id)
|
||||
self._reap_expired_send_leases(done_sending)
|
||||
|
||||
return KVConnectorTransferResults(
|
||||
finished_sending=done_sending,
|
||||
@@ -2996,6 +2979,34 @@ class NixlBaseConnectorWorker:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def _reap_expired_send_leases(self, done_sending: set[str]) -> None:
|
||||
"""Reclaim expired send-side KV leases into ``done_sending``.
|
||||
|
||||
``_reqs_to_send`` is not ordered by expiry: heartbeats update the
|
||||
deadline in place, and mixed TTLs share the map, so a live head
|
||||
entry can sit in front of already-expired ones. Scan every entry
|
||||
rather than stopping at the first still-live request.
|
||||
"""
|
||||
if not self._reqs_to_send:
|
||||
return
|
||||
now = time.perf_counter()
|
||||
expired = [
|
||||
req_id for req_id, expires in self._reqs_to_send.items() if now >= expires
|
||||
]
|
||||
for req_id in expired:
|
||||
count = self.consumer_notification_counts_by_req.pop(req_id, 0)
|
||||
self.expected_consumer_notifications_by_req.pop(req_id, None)
|
||||
self.xfer_stats.record_kv_expired_req()
|
||||
logger.warning(
|
||||
"Releasing expired KV blocks for request %s which were "
|
||||
"retrieved by %d remote worker(s) before lease expired.",
|
||||
req_id,
|
||||
count,
|
||||
)
|
||||
self._reqs_to_process.remove(req_id)
|
||||
del self._reqs_to_send[req_id]
|
||||
done_sending.add(req_id)
|
||||
|
||||
def _handle_heartbeat(self, payload: str) -> None:
|
||||
"""Extend leases for requests referenced in a heartbeat.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user