[KV Connector] Handle Mooncake finish after preemption (#43281)

Signed-off-by: Zhewen Li <zhewenli@inferact.ai>
Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
This commit is contained in:
Zhewen Li
2026-05-25 01:58:38 -07:00
committed by GitHub
co-authored by Zhewen Li
parent 5c1aec3dc0
commit 873758c13a
3 changed files with 71 additions and 3 deletions
@@ -43,6 +43,21 @@ def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
)
def _make_preemption_scheduler_output():
return SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids={"req-0"},
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=[],
new_block_ids=[],
num_computed_tokens=[],
),
num_scheduled_tokens={},
scheduled_spec_decode_tokens={},
)
def _add_unfinished_request(
scheduler: MooncakeStoreScheduler,
*,
@@ -112,6 +127,48 @@ def test_cached_request_without_spec_decode_keeps_current_step_save_overlap():
assert tracker.num_saved_tokens == 48
def test_preemption_resets_tracker_before_request_finished():
scheduler = _make_bare_scheduler()
_add_unfinished_request(
scheduler,
token_ids=list(range(44)),
block_hashes=[b"h0", b"h1"],
prefill_end_tokens=48,
)
scheduler.build_connector_meta(_make_preemption_scheduler_output())
tracker = scheduler._request_trackers["req-0"]
assert tracker.token_len == 0
assert tracker.allocated_block_ids == ()
assert tracker.num_saved_tokens == 0
assert tracker.token_ids is None
assert tracker.prefill_end_tokens == 0
request = SimpleNamespace(request_id="req-0")
assert scheduler.request_finished(request, ([0, 1],)) == (False, None)
def test_preemption_clears_stale_load_state():
scheduler = _make_bare_scheduler()
_make_pending_load_unfinished_request(
scheduler,
num_tokens=48,
block_hashes=[b"h0", b"h1", b"h2"],
block_ids=([10, 11, 12],),
)
scheduler.load_specs["req-0"] = LoadSpec(
vllm_cached_tokens=0,
kvpool_cached_tokens=48,
can_load=True,
)
meta = scheduler.build_connector_meta(_make_preemption_scheduler_output())
assert meta.requests == []
assert "req-0" not in scheduler.load_specs
assert "req-0" not in scheduler._unfinished_requests
def _make_pending_load_unfinished_request(
scheduler: MooncakeStoreScheduler,
*,