diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index 017be3bbd5..bc5190db00 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -64,6 +64,47 @@ def test_scheduler_reports_allocation_failure(request_runner): assert reduced[_ConnectorMetricName.ALLOCATION_FAILURE] == 1 +@pytest.mark.parametrize("async_scheduling", [True, False]) +@pytest.mark.parametrize("prompt_offset", [-1, -2]) +def test_last_block_offloaded_at_request_finish( + request_runner, async_scheduling: bool, prompt_offset: int +): + """EOS fills the last block at request finish — verify req_status is kept alive. + + prompt = block_size + prompt_offset tokens → not a full block at schedule time, + so _build_store_jobs creates no store job. After EOS, request_finished + keeps req_status alive so _build_store_jobs can process it on the next step. + + prompt_offset=-1: EOS fills the block → store job created on next step. + prompt_offset=-2: block remains partial → no store job, cleanup in + _build_store_jobs deletes req_status. + """ + block_size = 4 + runner = request_runner( + block_size=block_size, + num_gpu_blocks=10, + async_scheduling=async_scheduling, + ) + # prompt = block_size + prompt_offset tokens + runner.new_request(token_ids=[0] * (block_size + prompt_offset)) + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(list(keys)) + ) + + # Run with one step (EOS) + runner.run( + decoded_tokens=[EOS_TOKEN_ID], + ) + + cs = runner.connector_scheduler + # Verify req_status is kept alive for _build_store_jobs to process + # regardless of whether there are storable blocks + assert "0" in cs._req_status, ( + "req_status was deleted but should be kept alive " + "for _build_store_jobs to process finished_req_ids." + ) + + def test_scheduler_reports_lookup_sync_delay(request_runner): runner = request_runner( block_size=4, @@ -560,11 +601,19 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo # Group 1 (sliding window, window=2): only the last 2 blocks # are within the window → loads blocks 1,2 expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)), + # The deferred store from the previous request's last block + # completes during this step, and its blocks are flushed because + # they were reallocated to the new request. + # Only block 1 (sliding window group) is stored — block 0's + # deferred store is flushed because it was reallocated. + expected_stored=((0, 1),), + expected_flushed=((0, 1),), ) - # one touch in get_num_new_matched_tokens x 2 groups + # 4 touch calls: 2 from get_num_new_matched_tokens (2 groups) + # + 2 from _get_reqs_to_store (2 groups) touch_calls = runner.manager.touch.call_args_list - assert len(touch_calls) == 2 + assert len(touch_calls) == 4 # full attention group touched all 3 blocks assert len(touch_calls[0].args[0]) == 3 # sliding window group touched just the last 2 blocks diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 3e0d09a10f..5e98c1266e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -3,7 +3,7 @@ import time from collections.abc import Iterable, Sequence from dataclasses import dataclass, field -from itertools import islice +from itertools import chain, islice from typing import Any, NamedTuple from vllm.config import VllmConfig @@ -444,6 +444,24 @@ class OffloadingConnectorScheduler: if not pending: del self._block_id_to_pending_jobs[bid] + def _calc_num_offloadable_tokens( + self, req_status: RequestOffloadState, num_computed_tokens: int + ) -> int: + num = min(num_computed_tokens, req_status.req.num_tokens) + max_offload_tokens = req_status.max_offload_tokens + if max_offload_tokens is not None: + num = min(num, max_offload_tokens) + if self.config.offload_prompt_only: + num = min(num, req_status.req.num_prompt_tokens) + return num + + def _maybe_cleanup_finished_req( + self, req_id: str, req_status: RequestOffloadState + ) -> None: + """Clean up req_status if finished and no in-flight jobs.""" + if req_status.req.is_finished() and not req_status.transfer_jobs: + del self._req_status[req_id] + def _maximal_prefix_lookup( self, keys: Iterable[OffloadKey], req_context: ReqContext ) -> int | None: @@ -911,28 +929,24 @@ class OffloadingConnectorScheduler: ) -> dict[int, TransferJob]: blocks_per_chunk = self.config.blocks_per_chunk store_jobs: dict[int, TransferJob] = {} - for req_id in scheduler_output.num_scheduled_tokens: + for req_id in chain( + scheduler_output.num_scheduled_tokens, + scheduler_output.finished_req_ids or (), + ): req_status = self._req_status.get(req_id) if req_status is None: continue req = req_status.req - num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id] - num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens - # with async scheduling, some tokens may be missing - num_offloadable_tokens = min(num_tokens_after_batch, req.num_tokens) - max_offload_tokens = req_status.max_offload_tokens - if max_offload_tokens is not None: - num_offloadable_tokens = min(num_offloadable_tokens, max_offload_tokens) + if req.is_finished(): + num_tokens_after_batch = req.num_tokens + else: + num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id] + num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens - # Skip decode-phase chunks: clamp to the prompt length so only - # prefill chunks become eligible for store. next_stored_chunk_idx - # never advances past this boundary, so decode chunks are never - # queued in this or any later step. - if self.config.offload_prompt_only: - num_offloadable_tokens = min( - num_offloadable_tokens, req.num_prompt_tokens - ) + num_offloadable_tokens = self._calc_num_offloadable_tokens( + req_status, num_tokens_after_batch + ) # Filter out chunks skipped due to sliding window attention / SSM # or unreachable by the load path's alignment constraints. @@ -983,6 +997,7 @@ class OffloadingConnectorScheduler: if not new_offload_keys: req_status.advance_stored_idx(num_offloadable_tokens) + self._maybe_cleanup_finished_req(req_id, req_status) continue store_output = self.manager.prepare_store( @@ -993,10 +1008,12 @@ class OffloadingConnectorScheduler: _ConnectorMetricName.ALLOCATION_FAILURE ) logger.warning("Request %s: cannot store chunks", req_id) + self._maybe_cleanup_finished_req(req_id, req_status) continue if not store_output.keys_to_store: req_status.advance_stored_idx(num_offloadable_tokens) + self._maybe_cleanup_finished_req(req_id, req_status) continue self._touch(req_status) @@ -1093,6 +1110,13 @@ class OffloadingConnectorScheduler: job_id, ) + if req.is_finished(): + # Register non-sliding-window blocks for flush detection. + for bid in non_sliding_window_block_ids: + self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id) + if bid in self._current_batch_allocated_block_ids: + self._current_batch_jobs_to_flush.add(job_id) + return store_jobs def build_connector_meta( @@ -1255,8 +1279,6 @@ class OffloadingConnectorScheduler: Optional KVTransferParams to be included in the request outputs returned by the engine. """ - # TODO(orozery): possibly kickoff offload for last block - # which may have been deferred due to async scheduling req_status = self._req_status.get(request.request_id) if req_status is None: @@ -1269,20 +1291,20 @@ class OffloadingConnectorScheduler: self.manager.on_request_finished(req_status.req_context) self._maybe_observe_lookup_async_delay(req_status) - if not req_status.transfer_jobs: - # No in-flight jobs: no later complete_store()/complete_load() calls - # need this request's state. - del self._req_status[request.request_id] - return False, None - # In-flight jobs remain after the request stopped. Their completion may - # still call manager.complete_store()/complete_load(), so keep req_status. - # Pending stores outlive the request's block ownership; register them so - # future reuse of those blocks triggers a flush. + # Update offload keys with final block hash so _build_store_jobs can + # create store jobs for the last block(s) on the next schedule step. + req_status.update_offload_keys() + + # Keep req_status alive: _build_store_jobs will process finished_req_ids + # on the next step and handle cleanup after creating store jobs. + # Register non_sliding_window_block_ids so future block reuse triggers + # a flush via _block_id_to_pending_jobs. for job_id in req_status.transfer_jobs: job_status = self._jobs[job_id] for bid in job_status.non_sliding_window_block_ids or (): self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id) + return False, None def take_events(self) -> Iterable[KVCacheEvent]: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py index e82e1e45c0..045e513c3f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py @@ -280,7 +280,21 @@ class OffloadingConnectorWorker: def handle_preemptions(self, kv_connector_metadata: OffloadingConnectorMetadata): assert self.worker is not None + + # Pop jobs_to_flush from store_jobs into _unsubmitted_store_jobs + # so the existing submission loop below submits them before wait(). + if kv_connector_metadata.jobs_to_flush: + for job_id in kv_connector_metadata.jobs_to_flush: + entry = kv_connector_metadata.store_jobs.pop(job_id, None) + if entry is not None: + assert isinstance(entry.src_spec, GPULoadStoreSpec) + self._unsubmitted_store_jobs.append( + (job_id, entry.src_spec, entry.dst_spec) + ) + + # Submit deferred stores from previous step (and jobs_to_flush above). for job_id, src_spec, dst_spec in self._unsubmitted_store_jobs: + assert isinstance(src_spec, GPULoadStoreSpec) success = self.worker.submit_store(job_id, src_spec, dst_spec) assert success self._unsubmitted_store_jobs.clear()