diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py index 8ef1277bb3..7e29196298 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py @@ -19,7 +19,6 @@ def _make_bare_scheduler() -> MooncakeStoreScheduler: scheduler.lookup_async = False scheduler._block_size = 16 scheduler.load_specs = {} - scheduler._preempted_req_ids = set() scheduler._unfinished_request_ids = {"req-0"} scheduler._unfinished_requests = {} scheduler._request_trackers = {} @@ -35,6 +34,7 @@ def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None): req_ids=["req-0"], new_block_ids=[([2],)], num_computed_tokens=[44], + resumed_req_ids=set(), ), num_scheduled_tokens={"req-0": 4}, scheduled_spec_decode_tokens=( @@ -52,6 +52,7 @@ def _make_preemption_scheduler_output(): req_ids=[], new_block_ids=[], num_computed_tokens=[], + resumed_req_ids=set(), ), num_scheduled_tokens={}, scheduled_spec_decode_tokens={}, @@ -195,6 +196,7 @@ def _make_pending_load_scheduler_output() -> SimpleNamespace: req_ids=[], new_block_ids=[], num_computed_tokens=[], + resumed_req_ids=set(), ), num_scheduled_tokens={}, scheduled_spec_decode_tokens={}, @@ -253,14 +255,17 @@ def _make_resumed_unfinished_request( def _make_resumed_scheduler_output(*, num_scheduled_tokens: int) -> SimpleNamespace: + # A resumed-from-preemption step: the scheduler lists the request in + # resumed_req_ids and sends the FULL block table (replace semantics). return SimpleNamespace( finished_req_ids=set(), preempted_req_ids=set(), scheduled_new_reqs=[], scheduled_cached_reqs=SimpleNamespace( req_ids=["req-0"], - new_block_ids=[([2],)], + new_block_ids=[([0, 1, 2],)], num_computed_tokens=[0], + resumed_req_ids={"req-0"}, ), num_scheduled_tokens={"req-0": num_scheduled_tokens}, scheduled_spec_decode_tokens={}, @@ -273,7 +278,6 @@ def test_resumed_from_preemption_with_load_skips_save(): # passes load_spec.can_load=True. Skip save in this step; subsequent # cached_reqs steps will save new tokens normally. scheduler = _make_bare_scheduler() - scheduler._preempted_req_ids = {"req-0"} _make_resumed_unfinished_request( scheduler, token_ids=list(range(48)), @@ -303,7 +307,6 @@ def test_resumed_from_preemption_with_load_skips_save(): def test_resumed_from_preemption_without_load_still_saves(): # No load_spec → behavior is unchanged: save proceeds. scheduler = _make_bare_scheduler() - scheduler._preempted_req_ids = {"req-0"} _make_resumed_unfinished_request( scheduler, token_ids=list(range(48)), @@ -324,6 +327,71 @@ def test_resumed_from_preemption_without_load_still_saves(): assert tracker.num_saved_tokens == 48 +def test_running_request_not_in_resumed_req_ids_appends_blocks(): + """Regression: the replace-vs-append choice must follow the scheduler's + cached_reqs.resumed_req_ids, NOT connector-local preemption history. + + A running request that is not resumed this step carries a *delta* + new_block_ids and must be APPENDED to the tracker's existing blocks. + Treating it as resumed would replace allocated_block_ids with just the + delta while token_len stays at the full computed length, so the store + path's block_ids[start // block_size] runs off the end (the + "list index out of range" / token_len >> len(block_ids) bug). + """ + scheduler = _make_bare_scheduler() + _add_unfinished_request( + scheduler, + token_ids=list(range(48)), + block_hashes=[b"h0", b"h1", b"h2"], + prefill_end_tokens=48, + ) + + out = _make_scheduler_output(scheduled_spec_tokens=None) + assert "req-0" not in out.scheduled_cached_reqs.resumed_req_ids + + meta = scheduler.build_connector_meta(out) + + tracker = scheduler._request_trackers["req-0"] + # Delta [2] appended to existing [0, 1] (decode path), not replaced by [2]. + assert tracker.allocated_block_ids == ([0, 1, 2],) + # token_len stays covered by the block table: no store-path under-count. + blocks_held = sum(len(g) for g in tracker.allocated_block_ids) + assert tracker.token_len // scheduler._block_size <= blocks_held + assert len(meta.requests) == 1 + assert meta.requests[0].token_len_chunk == 48 + + +def test_resumed_request_in_resumed_req_ids_replaces_blocks(): + """A request the scheduler marks resumed gets the FULL block table in + new_block_ids and must REPLACE the tracker's blocks (not append), even if + a stale tracker from before preemption is still present.""" + scheduler = _make_bare_scheduler() + _make_resumed_unfinished_request( + scheduler, + token_ids=list(range(48)), + block_hashes=[b"h0", b"h1", b"h2"], + num_computed_tokens=0, + ) + # Stale pre-preemption tracker that must be overwritten, not appended to. + scheduler._request_trackers["req-0"] = RequestTracker( + req_id="req-0", + token_len=99, + allocated_block_ids=([7, 8, 9],), + num_saved_tokens=0, + ) + + scheduler.build_connector_meta( + _make_resumed_scheduler_output(num_scheduled_tokens=48) + ) + + tracker = scheduler._request_trackers["req-0"] + # Replaced with the full table from new_block_ids, not appended to [7,8,9]. + assert tracker.allocated_block_ids == ([0, 1, 2],) + assert tracker.token_len == 48 + blocks_held = sum(len(g) for g in tracker.allocated_block_ids) + assert tracker.token_len // scheduler._block_size <= blocks_held + + # Focused tests for ReqMeta.from_request_tracker — the centralized guard that # enforces "a ReqMeta never carries both a save and a load". diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py index 620fa2f5ba..58dfd5e428 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py @@ -67,7 +67,6 @@ class MooncakeStoreScheduler: # Per-request state self.load_specs: dict[str, LoadSpec] = {} # to be loaded self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests - self._preempted_req_ids: set[str] = set() # preempted requests self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {} self._unfinished_request_ids: set[str] = set() @@ -175,10 +174,8 @@ class MooncakeStoreScheduler: self._request_trackers.pop(finished_req_id, None) self._unfinished_requests.pop(finished_req_id, None) self._unfinished_request_ids.discard(finished_req_id) - self._preempted_req_ids.discard(finished_req_id) preempted_ids = scheduler_output.preempted_req_ids or set() - self._preempted_req_ids.update(preempted_ids) for req_id in preempted_ids: self.load_specs.pop(req_id, None) if request_tracker := self._request_trackers.get(req_id): @@ -243,13 +240,12 @@ class MooncakeStoreScheduler: continue req_meta = None - if req_id in self._preempted_req_ids: + if req_id in cached_reqs.resumed_req_ids: # Resumed after preemption if isinstance(new_block_ids, tuple): new_block_ids = tuple(b.copy() for b in new_block_ids) else: new_block_ids = (new_block_ids.copy(),) - self._preempted_req_ids.discard(req_id) load_spec = self.load_specs.pop(req_id, None) request_tuple = self._unfinished_requests.get(req_id) request_real = request_tuple[0] # type: ignore[index] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 127b8e4d4b..a6a75adf81 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -389,6 +389,7 @@ class KVTransferThread(threading.Thread): def run(self): self.ready_event.set() while True: + request_data = None try: request_data = self.request_queue.get() if request_data is None: @@ -396,8 +397,9 @@ class KVTransferThread(threading.Thread): self.request_queue.task_done() continue self._handle_request(request_data) - except Exception as e: - logger.error("Error in %s: %s", self.name, e) + except Exception: + req_id = getattr(request_data, "req_id", "") + logger.exception("Error in %s (req=%s)", self.name, req_id) def _handle_request(self, req_meta: Any): pass