[Bugfix][MooncakeStore] track resumed requests via scheduler's resumed_req_ids (#46595)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
@@ -19,7 +19,6 @@ def _make_bare_scheduler() -> MooncakeStoreScheduler:
|
||||
scheduler.lookup_async = False
|
||||
scheduler._block_size = 16
|
||||
scheduler.load_specs = {}
|
||||
scheduler._preempted_req_ids = set()
|
||||
scheduler._unfinished_request_ids = {"req-0"}
|
||||
scheduler._unfinished_requests = {}
|
||||
scheduler._request_trackers = {}
|
||||
@@ -35,6 +34,7 @@ def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
|
||||
req_ids=["req-0"],
|
||||
new_block_ids=[([2],)],
|
||||
num_computed_tokens=[44],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={"req-0": 4},
|
||||
scheduled_spec_decode_tokens=(
|
||||
@@ -52,6 +52,7 @@ def _make_preemption_scheduler_output():
|
||||
req_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
@@ -195,6 +196,7 @@ def _make_pending_load_scheduler_output() -> SimpleNamespace:
|
||||
req_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
@@ -253,14 +255,17 @@ def _make_resumed_unfinished_request(
|
||||
|
||||
|
||||
def _make_resumed_scheduler_output(*, num_scheduled_tokens: int) -> SimpleNamespace:
|
||||
# A resumed-from-preemption step: the scheduler lists the request in
|
||||
# resumed_req_ids and sends the FULL block table (replace semantics).
|
||||
return SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=["req-0"],
|
||||
new_block_ids=[([2],)],
|
||||
new_block_ids=[([0, 1, 2],)],
|
||||
num_computed_tokens=[0],
|
||||
resumed_req_ids={"req-0"},
|
||||
),
|
||||
num_scheduled_tokens={"req-0": num_scheduled_tokens},
|
||||
scheduled_spec_decode_tokens={},
|
||||
@@ -273,7 +278,6 @@ def test_resumed_from_preemption_with_load_skips_save():
|
||||
# passes load_spec.can_load=True. Skip save in this step; subsequent
|
||||
# cached_reqs steps will save new tokens normally.
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler._preempted_req_ids = {"req-0"}
|
||||
_make_resumed_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=list(range(48)),
|
||||
@@ -303,7 +307,6 @@ def test_resumed_from_preemption_with_load_skips_save():
|
||||
def test_resumed_from_preemption_without_load_still_saves():
|
||||
# No load_spec → behavior is unchanged: save proceeds.
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler._preempted_req_ids = {"req-0"}
|
||||
_make_resumed_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=list(range(48)),
|
||||
@@ -324,6 +327,71 @@ def test_resumed_from_preemption_without_load_still_saves():
|
||||
assert tracker.num_saved_tokens == 48
|
||||
|
||||
|
||||
def test_running_request_not_in_resumed_req_ids_appends_blocks():
|
||||
"""Regression: the replace-vs-append choice must follow the scheduler's
|
||||
cached_reqs.resumed_req_ids, NOT connector-local preemption history.
|
||||
|
||||
A running request that is not resumed this step carries a *delta*
|
||||
new_block_ids and must be APPENDED to the tracker's existing blocks.
|
||||
Treating it as resumed would replace allocated_block_ids with just the
|
||||
delta while token_len stays at the full computed length, so the store
|
||||
path's block_ids[start // block_size] runs off the end (the
|
||||
"list index out of range" / token_len >> len(block_ids) bug).
|
||||
"""
|
||||
scheduler = _make_bare_scheduler()
|
||||
_add_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=list(range(48)),
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
prefill_end_tokens=48,
|
||||
)
|
||||
|
||||
out = _make_scheduler_output(scheduled_spec_tokens=None)
|
||||
assert "req-0" not in out.scheduled_cached_reqs.resumed_req_ids
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
# Delta [2] appended to existing [0, 1] (decode path), not replaced by [2].
|
||||
assert tracker.allocated_block_ids == ([0, 1, 2],)
|
||||
# token_len stays covered by the block table: no store-path under-count.
|
||||
blocks_held = sum(len(g) for g in tracker.allocated_block_ids)
|
||||
assert tracker.token_len // scheduler._block_size <= blocks_held
|
||||
assert len(meta.requests) == 1
|
||||
assert meta.requests[0].token_len_chunk == 48
|
||||
|
||||
|
||||
def test_resumed_request_in_resumed_req_ids_replaces_blocks():
|
||||
"""A request the scheduler marks resumed gets the FULL block table in
|
||||
new_block_ids and must REPLACE the tracker's blocks (not append), even if
|
||||
a stale tracker from before preemption is still present."""
|
||||
scheduler = _make_bare_scheduler()
|
||||
_make_resumed_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=list(range(48)),
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
num_computed_tokens=0,
|
||||
)
|
||||
# Stale pre-preemption tracker that must be overwritten, not appended to.
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=99,
|
||||
allocated_block_ids=([7, 8, 9],),
|
||||
num_saved_tokens=0,
|
||||
)
|
||||
|
||||
scheduler.build_connector_meta(
|
||||
_make_resumed_scheduler_output(num_scheduled_tokens=48)
|
||||
)
|
||||
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
# Replaced with the full table from new_block_ids, not appended to [7,8,9].
|
||||
assert tracker.allocated_block_ids == ([0, 1, 2],)
|
||||
assert tracker.token_len == 48
|
||||
blocks_held = sum(len(g) for g in tracker.allocated_block_ids)
|
||||
assert tracker.token_len // scheduler._block_size <= blocks_held
|
||||
|
||||
|
||||
# Focused tests for ReqMeta.from_request_tracker — the centralized guard that
|
||||
# enforces "a ReqMeta never carries both a save and a load".
|
||||
|
||||
|
||||
Reference in New Issue
Block a user