[KV Connector] Handle Mooncake finish after preemption (#43281)

Signed-off-by: Zhewen Li <zhewenli@inferact.ai>
Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
This commit is contained in:
Zhewen Li
2026-05-25 01:58:38 -07:00
committed by GitHub
co-authored by Zhewen Li
parent 5c1aec3dc0
commit 873758c13a
3 changed files with 71 additions and 3 deletions
@@ -173,6 +173,13 @@ class RequestTracker:
# request it includes previously-generated tokens, which are re-prefilled.
prefill_end_tokens: int = 0
def reset(self) -> None:
self.token_len = 0
self.allocated_block_ids = ()
self.num_saved_tokens = 0
self.token_ids = None
self.prefill_end_tokens = 0
def update(
self,
new_block_ids: tuple[list[int], ...] | list[int],
@@ -171,7 +171,9 @@ class MooncakeStoreScheduler:
preempted_ids = scheduler_output.preempted_req_ids or set()
self._preempted_req_ids.update(preempted_ids)
for req_id in preempted_ids:
self._request_trackers.pop(req_id, None)
self.load_specs.pop(req_id, None)
if request_tracker := self._request_trackers.get(req_id):
request_tracker.reset()
self._unfinished_requests.pop(req_id, None)
meta = MooncakeStoreConnectorMetadata(
@@ -355,8 +357,10 @@ class MooncakeStoreScheduler:
if self.kv_role == "kv_consumer":
return False, None
tracker = self._request_trackers.get(request.request_id)
assert tracker is not None
if tracker.num_saved_tokens <= 0:
# Missing tracker can happen when the request is aborted before the
# connector observes the normal finished lifecycle or is preempted
# before finishing.
if tracker is None or tracker.num_saved_tokens <= 0:
return False, None
total_blocks = sum(len(g) for g in block_ids)
delay_free_blocks = total_blocks > 0