[Bugfix][MooncakeStore] track resumed requests via scheduler's resumed_req_ids (#46595)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
@@ -19,7 +19,6 @@ def _make_bare_scheduler() -> MooncakeStoreScheduler:
|
||||
scheduler.lookup_async = False
|
||||
scheduler._block_size = 16
|
||||
scheduler.load_specs = {}
|
||||
scheduler._preempted_req_ids = set()
|
||||
scheduler._unfinished_request_ids = {"req-0"}
|
||||
scheduler._unfinished_requests = {}
|
||||
scheduler._request_trackers = {}
|
||||
@@ -35,6 +34,7 @@ def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
|
||||
req_ids=["req-0"],
|
||||
new_block_ids=[([2],)],
|
||||
num_computed_tokens=[44],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={"req-0": 4},
|
||||
scheduled_spec_decode_tokens=(
|
||||
@@ -52,6 +52,7 @@ def _make_preemption_scheduler_output():
|
||||
req_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
@@ -195,6 +196,7 @@ def _make_pending_load_scheduler_output() -> SimpleNamespace:
|
||||
req_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
@@ -253,14 +255,17 @@ def _make_resumed_unfinished_request(
|
||||
|
||||
|
||||
def _make_resumed_scheduler_output(*, num_scheduled_tokens: int) -> SimpleNamespace:
|
||||
# A resumed-from-preemption step: the scheduler lists the request in
|
||||
# resumed_req_ids and sends the FULL block table (replace semantics).
|
||||
return SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=["req-0"],
|
||||
new_block_ids=[([2],)],
|
||||
new_block_ids=[([0, 1, 2],)],
|
||||
num_computed_tokens=[0],
|
||||
resumed_req_ids={"req-0"},
|
||||
),
|
||||
num_scheduled_tokens={"req-0": num_scheduled_tokens},
|
||||
scheduled_spec_decode_tokens={},
|
||||
@@ -273,7 +278,6 @@ def test_resumed_from_preemption_with_load_skips_save():
|
||||
# passes load_spec.can_load=True. Skip save in this step; subsequent
|
||||
# cached_reqs steps will save new tokens normally.
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler._preempted_req_ids = {"req-0"}
|
||||
_make_resumed_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=list(range(48)),
|
||||
@@ -303,7 +307,6 @@ def test_resumed_from_preemption_with_load_skips_save():
|
||||
def test_resumed_from_preemption_without_load_still_saves():
|
||||
# No load_spec → behavior is unchanged: save proceeds.
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler._preempted_req_ids = {"req-0"}
|
||||
_make_resumed_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=list(range(48)),
|
||||
@@ -324,6 +327,71 @@ def test_resumed_from_preemption_without_load_still_saves():
|
||||
assert tracker.num_saved_tokens == 48
|
||||
|
||||
|
||||
def test_running_request_not_in_resumed_req_ids_appends_blocks():
|
||||
"""Regression: the replace-vs-append choice must follow the scheduler's
|
||||
cached_reqs.resumed_req_ids, NOT connector-local preemption history.
|
||||
|
||||
A running request that is not resumed this step carries a *delta*
|
||||
new_block_ids and must be APPENDED to the tracker's existing blocks.
|
||||
Treating it as resumed would replace allocated_block_ids with just the
|
||||
delta while token_len stays at the full computed length, so the store
|
||||
path's block_ids[start // block_size] runs off the end (the
|
||||
"list index out of range" / token_len >> len(block_ids) bug).
|
||||
"""
|
||||
scheduler = _make_bare_scheduler()
|
||||
_add_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=list(range(48)),
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
prefill_end_tokens=48,
|
||||
)
|
||||
|
||||
out = _make_scheduler_output(scheduled_spec_tokens=None)
|
||||
assert "req-0" not in out.scheduled_cached_reqs.resumed_req_ids
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
# Delta [2] appended to existing [0, 1] (decode path), not replaced by [2].
|
||||
assert tracker.allocated_block_ids == ([0, 1, 2],)
|
||||
# token_len stays covered by the block table: no store-path under-count.
|
||||
blocks_held = sum(len(g) for g in tracker.allocated_block_ids)
|
||||
assert tracker.token_len // scheduler._block_size <= blocks_held
|
||||
assert len(meta.requests) == 1
|
||||
assert meta.requests[0].token_len_chunk == 48
|
||||
|
||||
|
||||
def test_resumed_request_in_resumed_req_ids_replaces_blocks():
|
||||
"""A request the scheduler marks resumed gets the FULL block table in
|
||||
new_block_ids and must REPLACE the tracker's blocks (not append), even if
|
||||
a stale tracker from before preemption is still present."""
|
||||
scheduler = _make_bare_scheduler()
|
||||
_make_resumed_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=list(range(48)),
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
num_computed_tokens=0,
|
||||
)
|
||||
# Stale pre-preemption tracker that must be overwritten, not appended to.
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=99,
|
||||
allocated_block_ids=([7, 8, 9],),
|
||||
num_saved_tokens=0,
|
||||
)
|
||||
|
||||
scheduler.build_connector_meta(
|
||||
_make_resumed_scheduler_output(num_scheduled_tokens=48)
|
||||
)
|
||||
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
# Replaced with the full table from new_block_ids, not appended to [7,8,9].
|
||||
assert tracker.allocated_block_ids == ([0, 1, 2],)
|
||||
assert tracker.token_len == 48
|
||||
blocks_held = sum(len(g) for g in tracker.allocated_block_ids)
|
||||
assert tracker.token_len // scheduler._block_size <= blocks_held
|
||||
|
||||
|
||||
# Focused tests for ReqMeta.from_request_tracker — the centralized guard that
|
||||
# enforces "a ReqMeta never carries both a save and a load".
|
||||
|
||||
|
||||
@@ -67,7 +67,6 @@ class MooncakeStoreScheduler:
|
||||
# Per-request state
|
||||
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
|
||||
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
|
||||
self._preempted_req_ids: set[str] = set() # preempted requests
|
||||
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
|
||||
self._unfinished_request_ids: set[str] = set()
|
||||
|
||||
@@ -175,10 +174,8 @@ class MooncakeStoreScheduler:
|
||||
self._request_trackers.pop(finished_req_id, None)
|
||||
self._unfinished_requests.pop(finished_req_id, None)
|
||||
self._unfinished_request_ids.discard(finished_req_id)
|
||||
self._preempted_req_ids.discard(finished_req_id)
|
||||
|
||||
preempted_ids = scheduler_output.preempted_req_ids or set()
|
||||
self._preempted_req_ids.update(preempted_ids)
|
||||
for req_id in preempted_ids:
|
||||
self.load_specs.pop(req_id, None)
|
||||
if request_tracker := self._request_trackers.get(req_id):
|
||||
@@ -243,13 +240,12 @@ class MooncakeStoreScheduler:
|
||||
continue
|
||||
|
||||
req_meta = None
|
||||
if req_id in self._preempted_req_ids:
|
||||
if req_id in cached_reqs.resumed_req_ids:
|
||||
# Resumed after preemption
|
||||
if isinstance(new_block_ids, tuple):
|
||||
new_block_ids = tuple(b.copy() for b in new_block_ids)
|
||||
else:
|
||||
new_block_ids = (new_block_ids.copy(),)
|
||||
self._preempted_req_ids.discard(req_id)
|
||||
load_spec = self.load_specs.pop(req_id, None)
|
||||
request_tuple = self._unfinished_requests.get(req_id)
|
||||
request_real = request_tuple[0] # type: ignore[index]
|
||||
|
||||
@@ -389,6 +389,7 @@ class KVTransferThread(threading.Thread):
|
||||
def run(self):
|
||||
self.ready_event.set()
|
||||
while True:
|
||||
request_data = None
|
||||
try:
|
||||
request_data = self.request_queue.get()
|
||||
if request_data is None:
|
||||
@@ -396,8 +397,9 @@ class KVTransferThread(threading.Thread):
|
||||
self.request_queue.task_done()
|
||||
continue
|
||||
self._handle_request(request_data)
|
||||
except Exception as e:
|
||||
logger.error("Error in %s: %s", self.name, e)
|
||||
except Exception:
|
||||
req_id = getattr(request_data, "req_id", "<unknown>")
|
||||
logger.exception("Error in %s (req=%s)", self.name, req_id)
|
||||
|
||||
def _handle_request(self, req_meta: Any):
|
||||
pass
|
||||
|
||||
Reference in New Issue
Block a user