diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index eb3b0f4175..2a1b599661 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -371,18 +371,20 @@ def test_scheduler_reports_allocation_failure(request_runner): @pytest.mark.parametrize("async_scheduling", [True, False]) @pytest.mark.parametrize("prompt_offset", [-1, -2]) -def test_last_block_offloaded_at_request_finish( +def test_final_sampled_token_does_not_complete_an_offloaded_block( request_runner, async_scheduling: bool, prompt_offset: int ): - """EOS fills the last block at request finish - verify the final block is stored. + """A block whose last slot holds the final sampled token is not stored. - prompt = block_size + prompt_offset tokens → not a full block at schedule time, - so _build_store_jobs creates no store job. After EOS, request_finished - keeps req_status alive so _build_store_jobs can process it on the next step. + That token came out of the previous position's forward pass; its own KV + slot is never written, and under spec decode it holds the first rejected + draft's KV. Storing the block would publish it under a content hash that + the GPU prefix cache itself declines to commit, so a later request with + the same tokens would load the unwritten slot. - prompt_offset=-1: EOS fills the block → store job created on next step. - prompt_offset=-2: block remains partial → no store job, cleanup in - _build_store_jobs deletes req_status. + prompt = 2 * block_size + prompt_offset tokens. Block 0 is filled by + prefill and stored either way. At prompt_offset=-1 the EOS fills block 1, + which the finishing step must still decline; at -2 block 1 stays partial. """ block_size = 4 runner = request_runner( @@ -390,18 +392,11 @@ def test_last_block_offloaded_at_request_finish( num_gpu_blocks=10, async_scheduling=async_scheduling, ) - # prompt = block_size + prompt_offset tokens - runner.new_request(token_ids=[0] * (block_size + prompt_offset)) + runner.new_request(token_ids=[0] * (2 * block_size + prompt_offset)) runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output(list(keys)) ) - - if prompt_offset == -1: - # EOS fills the block, so a store job is created for block 0. - runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_stored=(0,)) - else: - # Block remains partial, so no store job is created. - runner.run(decoded_tokens=[EOS_TOKEN_ID]) + runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_stored=(0,)) cs = runner.connector_scheduler # After the full run completes, req_status is cleaned up. @@ -933,10 +928,13 @@ def test_on_request_finished_not_deferred_until_store_completion( def test_on_request_finished_fires_after_final_block_store( request_runner, async_scheduling: bool ): - """on_request_finished fires after the final-block prepare_store at EOS. + """on_request_finished fires after the final prepare_store of the request. - When EOS fills a partial block, request_finished() keeps req_status alive - so _build_store_jobs can create a store job for it on the next step. + request_finished() keeps req_status alive so _build_store_jobs can retry a + block on the next step; the retry must be issued before the manager is told + the request is over. Every attempt made before the request finishes is + declined here, so the store that succeeds is the one from the finishing + step and the ordering has something to order. """ block_size = 4 runner = request_runner( @@ -950,22 +948,31 @@ def test_on_request_finished_fires_after_final_block_store( ("on_request_finished", req_context.req_id) ) + runner.new_request(token_ids=[0] * block_size) + request = runner.scheduler.requests[str(runner.req_id)] + def prepare_store(keys, req_context): - calls.append(("prepare_store", req_context.req_id)) - return generate_store_output(keys) + finished = request.is_finished() + calls.append( + ("stored_after_finish" if finished else "prepare_store", req_context.req_id) + ) + return generate_store_output(keys) if finished else None runner.manager.prepare_store.side_effect = prepare_store - runner.new_request(token_ids=[0] * (block_size - 1)) runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_stored=(0,)) req_id = str(runner.req_id) assert calls.count(("on_request_finished", req_id)) == 1, calls + assert calls.count(("stored_after_finish", req_id)) == 1, calls finished_idx = calls.index(("on_request_finished", req_id)) - prepare_indices = [i for i, c in enumerate(calls) if c == ("prepare_store", req_id)] - assert prepare_indices, calls - assert finished_idx > max(prepare_indices), calls + store_indices = [ + i + for i, c in enumerate(calls) + if c in (("prepare_store", req_id), ("stored_after_finish", req_id)) + ] + assert finished_idx > max(store_indices), calls @pytest.mark.parametrize("async_scheduling", [True, False]) @@ -1156,10 +1163,9 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo touch_calls = runner.manager.touch.call_args_list assert len(touch_calls) == 6 - # EOS fills the 7th block (offset 6). The extra schedule step processes - # finished_req_ids and stores block 6 for both groups before the request's - # GPU blocks are freed. - runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_stored=(6,)) + # EOS lands in the last slot of the 7th block (offset 6). No forward pass + # writes that slot, so the finishing step declines the block. + runner.run(decoded_tokens=[EOS_TOKEN_ID]) runner.scheduler.reset_prefix_cache() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index fcc4ba9e91..03a16edc9b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -1384,7 +1384,10 @@ class OffloadingConnectorScheduler: if req.status is RequestStatus.FINISHED_ABORTED: num_tokens_after_batch = req.num_computed_tokens elif req.is_finished(): - num_tokens_after_batch = req.num_tokens + # Clamp to the GPU prefix cache's commit point. The final sampled + # token's slot is never committed (under spec decode it holds a + # rejected draft's KV), so a block ending there must not be stored. + num_tokens_after_batch = max(req.num_prompt_tokens, req.num_tokens - 1) else: num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id] num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens