[Bugfix][MooncakeStore] track resumed requests via scheduler's resumed_req_ids (#46595)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Yifan Qiao
2026-06-24 23:50:56 +00:00
committed by GitHub
parent fc7fc421e9
commit b69816043a
3 changed files with 77 additions and 11 deletions
@@ -19,7 +19,6 @@ def _make_bare_scheduler() -> MooncakeStoreScheduler:
scheduler.lookup_async = False
scheduler._block_size = 16
scheduler.load_specs = {}
scheduler._preempted_req_ids = set()
scheduler._unfinished_request_ids = {"req-0"}
scheduler._unfinished_requests = {}
scheduler._request_trackers = {}
@@ -35,6 +34,7 @@ def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
req_ids=["req-0"],
new_block_ids=[([2],)],
num_computed_tokens=[44],
resumed_req_ids=set(),
),
num_scheduled_tokens={"req-0": 4},
scheduled_spec_decode_tokens=(
@@ -52,6 +52,7 @@ def _make_preemption_scheduler_output():
req_ids=[],
new_block_ids=[],
num_computed_tokens=[],
resumed_req_ids=set(),
),
num_scheduled_tokens={},
scheduled_spec_decode_tokens={},
@@ -195,6 +196,7 @@ def _make_pending_load_scheduler_output() -> SimpleNamespace:
req_ids=[],
new_block_ids=[],
num_computed_tokens=[],
resumed_req_ids=set(),
),
num_scheduled_tokens={},
scheduled_spec_decode_tokens={},
@@ -253,14 +255,17 @@ def _make_resumed_unfinished_request(
def _make_resumed_scheduler_output(*, num_scheduled_tokens: int) -> SimpleNamespace:
# A resumed-from-preemption step: the scheduler lists the request in
# resumed_req_ids and sends the FULL block table (replace semantics).
return SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=["req-0"],
new_block_ids=[([2],)],
new_block_ids=[([0, 1, 2],)],
num_computed_tokens=[0],
resumed_req_ids={"req-0"},
),
num_scheduled_tokens={"req-0": num_scheduled_tokens},
scheduled_spec_decode_tokens={},
@@ -273,7 +278,6 @@ def test_resumed_from_preemption_with_load_skips_save():
# passes load_spec.can_load=True. Skip save in this step; subsequent
# cached_reqs steps will save new tokens normally.
scheduler = _make_bare_scheduler()
scheduler._preempted_req_ids = {"req-0"}
_make_resumed_unfinished_request(
scheduler,
token_ids=list(range(48)),
@@ -303,7 +307,6 @@ def test_resumed_from_preemption_with_load_skips_save():
def test_resumed_from_preemption_without_load_still_saves():
# No load_spec → behavior is unchanged: save proceeds.
scheduler = _make_bare_scheduler()
scheduler._preempted_req_ids = {"req-0"}
_make_resumed_unfinished_request(
scheduler,
token_ids=list(range(48)),
@@ -324,6 +327,71 @@ def test_resumed_from_preemption_without_load_still_saves():
assert tracker.num_saved_tokens == 48
def test_running_request_not_in_resumed_req_ids_appends_blocks():
"""Regression: the replace-vs-append choice must follow the scheduler's
cached_reqs.resumed_req_ids, NOT connector-local preemption history.
A running request that is not resumed this step carries a *delta*
new_block_ids and must be APPENDED to the tracker's existing blocks.
Treating it as resumed would replace allocated_block_ids with just the
delta while token_len stays at the full computed length, so the store
path's block_ids[start // block_size] runs off the end (the
"list index out of range" / token_len >> len(block_ids) bug).
"""
scheduler = _make_bare_scheduler()
_add_unfinished_request(
scheduler,
token_ids=list(range(48)),
block_hashes=[b"h0", b"h1", b"h2"],
prefill_end_tokens=48,
)
out = _make_scheduler_output(scheduled_spec_tokens=None)
assert "req-0" not in out.scheduled_cached_reqs.resumed_req_ids
meta = scheduler.build_connector_meta(out)
tracker = scheduler._request_trackers["req-0"]
# Delta [2] appended to existing [0, 1] (decode path), not replaced by [2].
assert tracker.allocated_block_ids == ([0, 1, 2],)
# token_len stays covered by the block table: no store-path under-count.
blocks_held = sum(len(g) for g in tracker.allocated_block_ids)
assert tracker.token_len // scheduler._block_size <= blocks_held
assert len(meta.requests) == 1
assert meta.requests[0].token_len_chunk == 48
def test_resumed_request_in_resumed_req_ids_replaces_blocks():
"""A request the scheduler marks resumed gets the FULL block table in
new_block_ids and must REPLACE the tracker's blocks (not append), even if
a stale tracker from before preemption is still present."""
scheduler = _make_bare_scheduler()
_make_resumed_unfinished_request(
scheduler,
token_ids=list(range(48)),
block_hashes=[b"h0", b"h1", b"h2"],
num_computed_tokens=0,
)
# Stale pre-preemption tracker that must be overwritten, not appended to.
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=99,
allocated_block_ids=([7, 8, 9],),
num_saved_tokens=0,
)
scheduler.build_connector_meta(
_make_resumed_scheduler_output(num_scheduled_tokens=48)
)
tracker = scheduler._request_trackers["req-0"]
# Replaced with the full table from new_block_ids, not appended to [7,8,9].
assert tracker.allocated_block_ids == ([0, 1, 2],)
assert tracker.token_len == 48
blocks_held = sum(len(g) for g in tracker.allocated_block_ids)
assert tracker.token_len // scheduler._block_size <= blocks_held
# Focused tests for ReqMeta.from_request_tracker — the centralized guard that
# enforces "a ReqMeta never carries both a save and a load".
@@ -67,7 +67,6 @@ class MooncakeStoreScheduler:
# Per-request state
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
self._preempted_req_ids: set[str] = set() # preempted requests
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
self._unfinished_request_ids: set[str] = set()
@@ -175,10 +174,8 @@ class MooncakeStoreScheduler:
self._request_trackers.pop(finished_req_id, None)
self._unfinished_requests.pop(finished_req_id, None)
self._unfinished_request_ids.discard(finished_req_id)
self._preempted_req_ids.discard(finished_req_id)
preempted_ids = scheduler_output.preempted_req_ids or set()
self._preempted_req_ids.update(preempted_ids)
for req_id in preempted_ids:
self.load_specs.pop(req_id, None)
if request_tracker := self._request_trackers.get(req_id):
@@ -243,13 +240,12 @@ class MooncakeStoreScheduler:
continue
req_meta = None
if req_id in self._preempted_req_ids:
if req_id in cached_reqs.resumed_req_ids:
# Resumed after preemption
if isinstance(new_block_ids, tuple):
new_block_ids = tuple(b.copy() for b in new_block_ids)
else:
new_block_ids = (new_block_ids.copy(),)
self._preempted_req_ids.discard(req_id)
load_spec = self.load_specs.pop(req_id, None)
request_tuple = self._unfinished_requests.get(req_id)
request_real = request_tuple[0] # type: ignore[index]
@@ -389,6 +389,7 @@ class KVTransferThread(threading.Thread):
def run(self):
self.ready_event.set()
while True:
request_data = None
try:
request_data = self.request_queue.get()
if request_data is None:
@@ -396,8 +397,9 @@ class KVTransferThread(threading.Thread):
self.request_queue.task_done()
continue
self._handle_request(request_data)
except Exception as e:
logger.error("Error in %s: %s", self.name, e)
except Exception:
req_id = getattr(request_data, "req_id", "<unknown>")
logger.exception("Error in %s (req=%s)", self.name, req_id)
def _handle_request(self, req_meta: Any):
pass