diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index 5cc7065c1a..20c230a4c2 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -1305,3 +1305,79 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool): (1, 7), ), ) + + +@pytest.mark.parametrize("async_scheduling", [True, False]) +def test_stale_sliding_window_block_after_prepare_store_failure( + request_runner, async_scheduling: bool +): + """Regression test: when prepare_store fails (returns None), offloading is + delayed. Meanwhile, sliding window blocks get freed and reallocated to the + same request. On retry, the stale block_id must be detected and skipped. + + Without the fix, the stale block_id would either: + - Cause a KeyError in _remove_pending_job (duplicate in + _block_id_to_pending_jobs) + - Silently offload wrong data under a wrong key + """ + block_size = 4 + # sliding_window = 8 -> window of 2 blocks + sliding_window = 8 + # Use a tight GPU block budget so freed sliding window blocks are + # immediately reused by the same request's new allocations. + num_gpu_blocks = 4 + + kv_cache_groups = [ + KVCacheGroupSpec( + ["layer0"], + SlidingWindowSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=sliding_window, + ), + ), + ] + + runner = request_runner( + block_size=block_size, + num_gpu_blocks=num_gpu_blocks, + async_scheduling=async_scheduling, + kv_cache_groups=kv_cache_groups, + ) + + # Request with 3 blocks of prompt. Window = 2 blocks, so block 0 is + # outside the window but won't be freed until the next allocate_slots. + runner.new_request(token_ids=[0] * block_size * 3) + + # First step: prepare_store FAILS -> offloading delayed. + # next_stored_block_idx stays at 0, block_ids[0] still holds the + # original block_id for position 0. + runner.manager.prepare_store.side_effect = lambda keys, req_context: None + runner.run(decoded_tokens=[0]) + runner.manager.prepare_store.assert_called() + + # Second step: decode more tokens -> block 3 allocated. + # allocate_slots calls remove_skipped_blocks which frees block 0 + # (it's now outside the sliding window). With num_gpu_blocks=4, + # the freed block is immediately reused for the new allocation. + # prepare_store still fails so offloading is still delayed. + runner.manager.prepare_store.side_effect = lambda keys, req_context: None + runner.run(decoded_tokens=[0] * block_size) + + # Now prepare_store succeeds. + # Without the fix, the request would try to offload the stale block_id + # at position 0 (now reused at position 3), causing a duplicate in + # sliding_window_block_ids and eventually a KeyError. + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(keys) + ) + # block_ids=[0, ?, 3, 1]: positions 0 and 1 are zeroed (stale blocks that + # were freed by the sliding window and reallocated). Only blocks at + # positions 2 and 3 (request offsets 2, 3) are stored. + runner.run( + decoded_tokens=[EOS_TOKEN_ID], + expected_stored=(2, 3), + expected_flushed=(2, 3) if not async_scheduling else (), + ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 6560bcf9de..e9c814bb72 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -286,6 +286,8 @@ class OffloadingConnectorScheduler: self._req_status: dict[ReqId, RequestOffloadState] = {} self._current_batch_load_jobs: dict[int, TransferJob] = {} self._current_batch_jobs_to_flush: set[int] = set() + # GPU block IDs allocated in the current engine step + self._current_batch_allocated_block_ids: set[int] = set() # if GPU prefix caching is enabled, # track loaded blocks to avoid redundant loads self._blocks_being_loaded: set[OffloadKey] | None = ( @@ -589,6 +591,10 @@ class OffloadingConnectorScheduler: req_status.group_states, blocks.blocks, ): + self._current_batch_allocated_block_ids.update( + block.block_id for block in group_blocks if block.block_id != 0 + ) + gpu_block_size = group_config.gpu_block_size offloaded_block_size = group_config.offloaded_block_size offload_keys = group_state.offload_keys @@ -638,17 +644,6 @@ class OffloadingConnectorScheduler: if req_status.offloading_context.policy == OffloadPolicy.BLOCK_LEVEL: group_state.next_stored_block_idx = num_blocks - # Fence dst blocks against finished-request pending stores. - if ( - self._block_id_to_pending_jobs - and not self._block_id_to_pending_jobs.keys().isdisjoint(dst_block_ids) - ): - self._current_batch_jobs_to_flush.update( - jid - for bid in dst_block_ids - for jid in self._block_id_to_pending_jobs.get(bid, ()) - ) - src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context) dst_spec = GPULoadStoreSpec( dst_block_ids, group_sizes=group_sizes, block_indices=block_indices @@ -672,37 +667,68 @@ class OffloadingConnectorScheduler: if self._blocks_being_loaded is not None: self._blocks_being_loaded.update(keys_to_load) - def _build_store_jobs( - self, - scheduler_output: SchedulerOutput, - ) -> dict[int, TransferJob]: - block_size_factor = self.config.block_size_factor - store_jobs: dict[int, TransferJob] = {} - # iterate over both new and cached requests + def _update_req_states(self, scheduler_output: SchedulerOutput) -> None: + """ + Update request states from the Scheduler's output. + """ + + # new_block_ids_end[req_id][i] = end of pre-existing block_ids for + # the i-th sliding window group (before this step's extend). + # Used to detect sliding window blocks that got re-allocated. + new_block_ids_end: dict[str, tuple[int, ...]] = {} + for req_id, new_block_id_groups, preempted in yield_req_data(scheduler_output): req_status = self._req_status[req_id] req_status.update_offload_keys() - req = req_status.req if preempted: for group_state in req_status.group_states: group_state.block_ids.clear() if new_block_id_groups: + if self._sliding_window_groups: + new_block_ids_end[req_id] = tuple( + len(req_status.group_states[grp_idx].block_ids) + for grp_idx in self._sliding_window_groups + ) req_status.update_block_id_groups(new_block_id_groups) - # Fence new blocks against in-flight stores. - if self._block_id_to_pending_jobs: - new_blocks_flat = [ - bid for new_blocks in new_block_id_groups for bid in new_blocks - ] - if not self._block_id_to_pending_jobs.keys().isdisjoint( - new_blocks_flat - ): - self._current_batch_jobs_to_flush.update( - jid - for bid in new_blocks_flat - for jid in self._block_id_to_pending_jobs.get(bid, ()) - ) + for new_blocks in new_block_id_groups: + for bid in new_blocks: + if bid != 0: + self._current_batch_allocated_block_ids.add(bid) + + # Zero out stale block_ids in sliding window groups' pending-store + # positions. Only sliding window groups can have stale entries (blocks + # freed by remove_skipped_blocks then reallocated). Only positions in + # [next_stored_block_idx * bsf, end) need checking where end is the + # pre-extend length: earlier positions were already offloaded, later + # ones are fresh allocations from this step. + if self._sliding_window_groups and self._current_batch_allocated_block_ids: + block_size_factor = self.config.block_size_factor + for req_id, req_status in self._req_status.items(): + ends = new_block_ids_end.get(req_id) + for i, grp_idx in enumerate(self._sliding_window_groups): + group_state = req_status.group_states[grp_idx] + start = group_state.next_stored_block_idx * block_size_factor + end = ends[i] if ends is not None else len(group_state.block_ids) + for j in range(start, end): + if ( + group_state.block_ids[j] + in self._current_batch_allocated_block_ids + ): + group_state.block_ids[j] = 0 + + def _build_store_jobs( + self, + scheduler_output: SchedulerOutput, + ) -> dict[int, TransferJob]: + block_size_factor = self.config.block_size_factor + store_jobs: dict[int, TransferJob] = {} + for req_id in scheduler_output.num_scheduled_tokens: + req_status = self._req_status.get(req_id) + if req_status is None: + continue + req = req_status.req num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id] num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens @@ -735,11 +761,8 @@ class OffloadingConnectorScheduler: # For each block to offload, take the last corresponding GPU block. # e.g. if block size factor is 3 and GPU block IDs are # 1 5 6 7 2 4 9 3 8 then we'll take blocks 6 4 8. - # We will use these GPU blocks to determine if the block needs - # offloading, or (if the GPU block ID is 0) this block should - # be skipped due to sliding window attention / SSM. - # We know that if a block is skipped, then all the previous blocks - # are skipped as well. This is why we take the last of each block. + # A block_id of 0 means either a sliding window / SSM skip + # or a stale entry that was zeroed out — skip it either way. offload_block_ids = group_state.block_ids[ start_block_idx * block_size_factor + block_size_factor @@ -814,10 +837,8 @@ class OffloadingConnectorScheduler: for i in range(block_size_factor): block_id = block_ids[gpu_block_idx + i] if block_id == 0: - # skipped blocks cannot appear after non-skipped blocks - assert start_gpu_block_idx is None continue - elif start_gpu_block_idx is None: + if start_gpu_block_idx is None: start_gpu_block_idx = gpu_block_idx + i src_block_ids.append(block_id) num_group_blocks += 1 @@ -875,6 +896,9 @@ class OffloadingConnectorScheduler: def build_connector_meta( self, scheduler_output: SchedulerOutput ) -> KVConnectorMetadata: + self._update_req_states(scheduler_output) + + # Flush jobs for preempted requests. for req_id in scheduler_output.preempted_req_ids or (): req_status = self._req_status.get(req_id) if req_status is None or not req_status.transfer_jobs: @@ -883,6 +907,20 @@ class OffloadingConnectorScheduler: assert self._jobs[any_jid].is_store self._current_batch_jobs_to_flush.update(req_status.transfer_jobs) + # Flush jobs that contain re-allocated blocks. + if ( + self._block_id_to_pending_jobs + and not self._block_id_to_pending_jobs.keys().isdisjoint( + self._current_batch_allocated_block_ids + ) + ): + self._current_batch_jobs_to_flush.update( + jid + for bid in self._current_batch_allocated_block_ids + if bid in self._block_id_to_pending_jobs + for jid in self._block_id_to_pending_jobs[bid] + ) + # If all tracked requests are finished, flush all pending jobs # (both store and load) - there might not be a future scheduler # step to trigger their completion. @@ -898,6 +936,7 @@ class OffloadingConnectorScheduler: ) self._current_batch_load_jobs = {} self._current_batch_jobs_to_flush = set() + self._current_batch_allocated_block_ids = set() return meta def update_connector_output(self, connector_output: KVConnectorOutput): @@ -1013,6 +1052,7 @@ class OffloadingConnectorScheduler: # reset_cache cannot be called in the middle of a schedule step assert not self._current_batch_load_jobs assert not self._current_batch_jobs_to_flush + assert not self._current_batch_allocated_block_ids # Flush all in-flight jobs self._current_batch_jobs_to_flush.update(self._jobs.keys())