diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index 973fcc63e3..1e12a7adde 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -243,6 +243,77 @@ def test_request_preemption(request_runner, async_scheduling: bool): assert runner.connector_scheduler._block_id_to_pending_jobs == {} +@pytest.mark.parametrize("async_scheduling", [True, False]) +def test_no_offload_call_after_on_request_finished( + request_runner, async_scheduling: bool +): + """on_request_finished is not issued before a per-request offload + call. + + A request can finish while its GPU->primary store is still in flight; the + later worker completion then drives complete_store. The scheduler defers + on_request_finished until the request is finished AND has no in-flight + transfer jobs, so complete_store is observed BEFORE on_request_finished, + and it is called exactly once. + """ + block_size = 4 + block_size_factor = 3 + offloaded_block_size = block_size * block_size_factor + + runner = request_runner( + block_size=block_size, + num_gpu_blocks=100, + async_scheduling=async_scheduling, + block_size_factor=block_size_factor, + ) + + # Record the order of per-request connector calls on the (mocked) manager. + # The external list survives manager.reset_mock() between run() calls. + calls: list[tuple[str, str]] = [] + runner.manager.on_request_finished.side_effect = lambda req_context: calls.append( + ("on_request_finished", req_context.req_id) + ) + runner.manager.complete_store.side_effect = ( + lambda keys, req_context, *args, **kwargs: calls.append( + ("complete_store", req_context.req_id) + ) + ) + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(keys) + ) + + # Decode a couple of blocks, keeping every transfer in flight + # (complete_transfers=False) so no store completes while the request runs. + runner.new_request(token_ids=[0] * offloaded_block_size * 2) + runner.run(decoded_tokens=[0], complete_transfers=False) + runner.run( + decoded_tokens=[0] * (2 * offloaded_block_size), + complete_transfers=False, + ) + + # Finish the request, completing its pending stores. on_request_finished is + # deferred until the stores drain, so it lands after the last complete_store. + # 4 offloaded blocks are stored (2 prompt + 2 decode) -> 4 * block_size_factor + # GPU blocks. + runner.run( + decoded_tokens=[EOS_TOKEN_ID], + expected_stored=tuple(range(4 * block_size_factor)), + ) + + req_id = str(runner.req_id) + + # on_request_finished is issued exactly once. + assert calls.count(("on_request_finished", req_id)) == 1, calls + + finished_idx = calls.index(("on_request_finished", req_id)) + store_indices = [i for i, c in enumerate(calls) if c == ("complete_store", req_id)] + + # All of the request's complete_store calls must precede its single + # on_request_finished. + assert store_indices, calls + assert max(store_indices) < finished_idx, calls + + @pytest.mark.parametrize("async_scheduling", [True, False]) def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling: bool): block_size = 4 @@ -1149,6 +1220,64 @@ def test_reset_cache(request_runner, async_scheduling: bool): assert group_state.next_stored_block_idx == 0 +@pytest.mark.parametrize("async_scheduling", [True, False]) +def test_reset_cache_finalizes_finished_request_with_pending_store( + request_runner, async_scheduling: bool +): + """reset_cache must finalize a finished request whose in-flight stores it + discards: call on_request_finished and drop its _req_status entry. + + Otherwise the deferred hook (which waits for the now-discarded jobs to + complete) never fires and the entry leaks. + """ + block_size = 4 + block_size_factor = 3 + offloaded_block_size = block_size * block_size_factor + + runner = request_runner( + block_size=block_size, + num_gpu_blocks=100, + async_scheduling=async_scheduling, + block_size_factor=block_size_factor, + ) + + finalized: list[str] = [] + runner.manager.on_request_finished.side_effect = ( + lambda req_context: finalized.append(req_context.req_id) + ) + runner.manager.prepare_store.side_effect = ( + lambda keys, req_context: generate_store_output(keys) + ) + + # Decode a couple of blocks and keep every transfer in flight, so the + # request has pending store jobs. + runner.new_request(token_ids=[0] * offloaded_block_size * 2) + runner.run(decoded_tokens=[0], complete_transfers=False) + runner.run( + decoded_tokens=[0] * (2 * offloaded_block_size), + complete_transfers=False, + ) + + cs = runner.connector_scheduler + req_id = str(runner.req_id) + req_status = cs._req_status[req_id] + assert req_status.transfer_jobs, "expected an in-flight store before finish" + assert any(job.is_store for job in cs._jobs.values()) + + # Finish the request while its store is still in flight. request_finished + # takes the defer branch (pending jobs), so on_request_finished is NOT + # called yet and the entry stays tracked. + req_status.req.status = RequestStatus.FINISHED_STOPPED + cs.request_finished(req_status.req) + assert finalized == [] + assert req_id in cs._req_status + + # reset_cache discards the in-flight store; it must finalize the request. + cs.reset_cache() + assert finalized == [req_id] + assert req_id not in cs._req_status + + @pytest.mark.parametrize("async_scheduling", [True, False]) def test_swa_alignment_skip(request_runner, async_scheduling: bool): """SWA blocks unreachable by the load path are skipped during store. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 9c3cb7e5a5..21be16e486 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -1115,6 +1115,11 @@ class OffloadingConnectorScheduler: del self._jobs[job_id] req_status.transfer_jobs.remove(job_id) if not req_status.transfer_jobs and req_status.req.is_finished(): + # Deferred from request_finished: the request's last in-flight + # job is now done, so fire the finalize hook here, after the + # final complete_store/complete_load above (and any submit_store + # the complete_store cascade issued). + self.manager.on_request_finished(req_status.req_context) del self._req_status[job_status.req_id] def get_stats(self) -> OffloadingConnectorStats | None: @@ -1148,18 +1153,23 @@ class OffloadingConnectorScheduler: # which may have been deferred due to async scheduling req_status = self._req_status.get(request.request_id) - req_context = ( - req_status.req_context if req_status else _create_req_context(request) - ) - self.manager.on_request_finished(req_context) - if req_status is None: + # Untracked request (offloading never started): no in-flight jobs, + # nothing was deferred, so finalize immediately. + self.manager.on_request_finished(_create_req_context(request)) return False, None + if not req_status.transfer_jobs: + # No in-flight jobs: all per-request calls are done, finalize now. + self.manager.on_request_finished(req_status.req_context) del self._req_status[request.request_id] return False, None - # Pending stores will outlive the request's block ownership. - # Register them so future block reuse triggers a flush. + + # In-flight jobs remain, so defer on_request_finished to + # update_connector_output, which fires it once the last job completes + # (after the final complete_store and any cascade submit_store it + # issues). These pending stores outlive the request's block ownership; + # register them so future reuse of those blocks triggers a flush. for job_id in req_status.transfer_jobs: job_status = self._jobs[job_id] for bid in job_status.non_sliding_window_block_ids or (): @@ -1198,6 +1208,16 @@ class OffloadingConnectorScheduler: # Flush all in-flight jobs self._current_batch_jobs_to_flush.update(self._jobs.keys()) + # A finished request may still be tracked here with in-flight jobs that + # this reset discards, so its deferred on_request_finished() would never + # fire (completions are skipped as stale) and its _req_status entry would + # leak. Finalize such requests now, before resetting the manager. + # list() snapshots because we delete while iterating. + for req_id, status in list(self._req_status.items()): + if status.req.is_finished(): + self.manager.on_request_finished(status.req_context) + del self._req_status[req_id] + # Reset offloading manager cache self.manager.reset_cache() diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index 15781bbc8a..2d27c14fe8 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -266,6 +266,17 @@ class OffloadingManager(ABC): """ Called when a request has finished. + By the time this is called, all per-request offload calls for this + request (prepare_store/complete_store, prepare_load/complete_load, + touch, lookup) have already been issued, and none will follow. The + scheduler defers this call until the request is finished and has no + in-flight transfer jobs. + + Note this signals only that no further calls will be made; it does NOT + imply the data has been persisted. Asynchronous transfers already + submitted for this request (e.g. CPU->secondary cascades) may still be + in flight. This is the right place to release per-request bookkeeping. + Args: req_context: per-request context. """ diff --git a/vllm/v1/kv_offload/tiering/base.py b/vllm/v1/kv_offload/tiering/base.py index f9fbdf9495..87481603f5 100644 --- a/vllm/v1/kv_offload/tiering/base.py +++ b/vllm/v1/kv_offload/tiering/base.py @@ -188,6 +188,13 @@ class SecondaryTierManager(ABC): """ Called when a request has finished. + By the time this is called, all per-request calls for this request + (submit_store, submit_load, touch) have already been issued, and none + will follow. Note this does NOT imply the tier's transfers have + completed: jobs already submitted may still be in flight and will + report via get_finished_jobs(). This is the right place to release + per-request bookkeeping. + Args: req_context: per-request context. """