[PD][Bugfix] Rebase KV lease deadlines onto worker clock (#50326)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: esmeetu <jasonailu87@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Nick Hill
2026-07-30 01:25:42 +00:00
committed by GitHub
co-authored by esmeetu Yifan Qiao
parent 437e0b7f8e
commit 4e582c5b54
5 changed files with 76 additions and 0 deletions
@@ -453,6 +453,10 @@ class NixlBaseConnectorScheduler:
self._build_save_meta(meta, scheduler_output)
meta.reqs_to_send = self._reqs_need_send
# Clock reference for reqs_to_send: deadlines above are in this
# process's perf_counter domain; workers (possibly on other nodes,
# where perf_counter has a different epoch) rebase against this.
meta.scheduler_clock = time.perf_counter()
meta.reqs_in_batch = self._reqs_in_batch
meta.reqs_not_processed = self._reqs_not_processed
@@ -180,6 +180,12 @@ class NixlConnectorMetadata(KVConnectorMetadata):
self.reqs_to_recv: dict[ReqId, ReqMeta] = {}
self.reqs_to_save: dict[ReqId, ReqMeta] = {}
self.reqs_to_send: dict[ReqId, float] = {}
# The scheduler process's time.perf_counter() when this metadata was
# built. reqs_to_send deadlines are stamped with the scheduler's
# clock, which is NOT comparable across processes (perf_counter is
# process/boot-local): workers must rebase the remaining TTL onto
# their own clock via this reference. 0.0 = unset (legacy metadata).
self.scheduler_clock: float = 0.0
self.reqs_in_batch: set[ReqId] = set()
self.reqs_not_processed: set[ReqId] = set()
# Heartbeat data grouped by remote engine, sent by D worker to P.
@@ -92,8 +92,20 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
assert req_id not in self._reqs_to_send
# Add to requests that are waiting to be read and track expiration.
# Deadlines are stamped with the scheduler process's perf_counter,
# which is not comparable to ours when the worker runs in another
# process on another node (perf_counter epochs differ by boot time).
# Rebase the remaining TTL onto our clock; broadcast latency only
# lengthens the lease, which is the safe direction. A cross-node
# epoch gap larger than the TTL otherwise expires the lease on
# arrival and the blocks are freed before D reads them.
now_local = time.perf_counter()
for req_id, expiration_time in metadata.reqs_to_send.items():
if req_id in self._reqs_to_process:
if metadata.scheduler_clock:
expiration_time = now_local + (
expiration_time - metadata.scheduler_clock
)
self._reqs_to_send[req_id] = expiration_time
# Send heartbeats to P-side engines to keep KV blocks alive while
@@ -191,8 +191,15 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
for req_id in metadata.reqs_not_processed:
self._reqs_to_process.discard(req_id)
assert req_id not in self._reqs_to_send
# Rebase scheduler-clock deadlines onto this worker's clock — see the
# equivalent block in pull_worker.start_load_kv for the rationale.
now_local = time.perf_counter()
for req_id, expiration_time in metadata.reqs_to_send.items():
if req_id in self._reqs_to_process:
if metadata.scheduler_clock:
expiration_time = now_local + (
expiration_time - metadata.scheduler_clock
)
self._reqs_to_send[req_id] = expiration_time
# Heartbeats still leave from the main thread (base worker behaviour).