From 092334c828e1114d6d8fb5828f7b0580e57e4a56 Mon Sep 17 00:00:00 2001 From: chuanzhisongshu <49055103+982945902@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:07:35 +0800 Subject: [PATCH] [Bugfix] Wait for offload keys before storing chunks (#52923) Signed-off-by: chuanzhisongshu <49055103+982945902@users.noreply.github.com> Co-authored-by: lishuo121 Co-authored-by: Or Ozeri --- .../offloading_connector/test_scheduler.py | 39 +++++++++++++++++++ .../kv_connector/v1/offloading/scheduler.py | 5 ++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index 53a3524c25..040067dc36 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -378,6 +378,45 @@ def test_abort_queued_request_does_not_build_store_job( assert queued_req_id not in runner.connector_scheduler._req_status +def test_store_jobs_wait_for_missing_offload_key(request_runner): + """Allocated chunks must wait until their block hash becomes available.""" + block_size = 4 + runner = request_runner( + block_size=block_size, + num_gpu_blocks=8, + async_scheduling=False, + ) + runner.manager.prepare_store.side_effect = lambda keys, req_context: None + runner.new_request(token_ids=[0] * (block_size * 2)) + runner.run(decoded_tokens=[1]) + + req_id = str(runner.req_id) + req_status = runner.connector_scheduler._req_status[req_id] + req = req_status.req + group_state = req_status.group_states[0] + assert len(group_state.block_ids) >= 2 + assert len(group_state.offload_keys) == 2 + + missing_hash = req.block_hashes.pop() + group_state.offload_keys.pop() + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(keys) + ) + scheduler_output = SimpleNamespace( + num_scheduled_tokens={req_id: 0}, finished_req_ids=set() + ) + + jobs = runner.connector_scheduler._build_store_jobs(scheduler_output) + assert len(jobs) == 1 + assert group_state.next_stored_chunk_idx == 1 + + req.block_hashes.append(missing_hash) + req_status.update_offload_keys() + jobs = runner.connector_scheduler._build_store_jobs(scheduler_output) + assert len(jobs) == 1 + assert group_state.next_stored_chunk_idx == 2 + + def test_scheduler_reports_lookup_sync_delay(request_runner): runner = request_runner( block_size=4, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 402bd55dc5..435823eab6 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -385,7 +385,7 @@ class RequestOffloadState: group_state: RequestGroupState, num_offloadable_tokens: int, ) -> int: - """Number of allocated leading offloaded chunks eligible for store. + """Number of allocated and keyed leading chunks eligible for store. For eagle/MTP groups the volatile trailing chunk of the offloadable range is excluded while decoding: the draft-layer KV of the last @@ -405,7 +405,8 @@ class RequestOffloadState: num_allocated_chunks = ( len(group_state.block_ids) // self.config.blocks_per_chunk ) - return min(num_chunks, num_allocated_chunks) + num_keyed_chunks = len(group_state.offload_keys) + return min(num_chunks, num_allocated_chunks, num_keyed_chunks) def advance_stored_idx(self, num_offloadable_tokens: int) -> None: # max(): at the prefill->decode transition of a chunk-aligned prompt,