diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index 89b77ae43f..035fb6b005 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -258,6 +258,9 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources(): jobs = scheduler._build_partial_tail_store_jobs(output) assert len(jobs) == 1 + offered_keys = scheduler.manager.prepare_store.call_args.args[0] + req_context = scheduler._req_status["req"].req_context + assert all(req_context.get_offload_key_position(key) == 28 for key in offered_keys) [job_id] = jobs src_spec = jobs[job_id].src_spec assert isinstance(src_spec, GPULoadStoreSpec) @@ -380,6 +383,13 @@ def test_normal_store_excludes_align_mode_mamba_sources(): req_status.group_states[0].block_ids[:] = [11] req_status.group_states[1].block_ids[:] = [99] req_status.update_offload_keys() + for group_config, group_state in zip( + scheduler.config.kv_group_configs, req_status.group_states + ): + assert ( + req_status.req_context.get_offload_key_position(group_state.offload_keys[0]) + == group_config.tokens_per_chunk + ) scheduler.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output(keys) ) @@ -921,7 +931,6 @@ def test_offloading_connector(request_runner, async_scheduling: bool): runner.run(decoded_tokens=[0] * (tokens_per_chunk + 1)) # 1 more block (+ token for kicking off offloading) - # now check touch was called with all 6 blocks runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output(keys) ) @@ -929,9 +938,6 @@ def test_offloading_connector(request_runner, async_scheduling: bool): decoded_tokens=[0] * (tokens_per_chunk + 1), expected_stored=(15, 16, 17), ) - runner.manager.touch.assert_called() - block_hashes1 = list(runner.manager.touch.call_args.args[0]) - assert len(block_hashes1) == 6 # terminate request runner.run(decoded_tokens=[EOS_TOKEN_ID]) @@ -939,13 +945,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool): # create a new request differing only on the last token runner.new_request(token_ids=[0] * (tokens_per_chunk * 6 - 1) + [1]) runner.run(decoded_tokens=[0]) - runner.manager.touch.assert_called() - block_hashes2 = list(runner.manager.touch.call_args.args[0]) - assert len(block_hashes2) == 6 - - # verify hashes are the same, except for the last block - assert block_hashes1[:5] == block_hashes2[:5] - assert block_hashes1[5] != block_hashes2[5] + runner.manager.touch.assert_not_called() # terminate request runner.run( @@ -1353,14 +1353,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo generate_store_output(keys) ) runner.run(decoded_tokens=[0]) - # _touch called from get_num_new_matched_tokens (2 groups) and - # _get_reqs_to_store (2 groups) → 4 touch calls total. - touch_calls = runner.manager.touch.call_args_list - assert len(touch_calls) == 4 - assert len(touch_calls[0].args[0]) == 3 - assert len(touch_calls[1].args[0]) == 3 - assert len(touch_calls[2].args[0]) == 3 - assert len(touch_calls[3].args[0]) == 3 + runner.manager.touch.assert_not_called() # store 3 more block runner.manager.prepare_store.side_effect = lambda keys, req_context: ( @@ -1371,9 +1364,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo expected_stored=(0, 1, 2, 3, 4, 5), ) - # touch called from _get_reqs_to_store * 3 blocks, once for each group - touch_calls = runner.manager.touch.call_args_list - assert len(touch_calls) == 6 + runner.manager.touch.assert_not_called() # EOS lands in the last slot of the 7th block (offset 6). No forward pass # writes that slot, so the finishing step declines the block. @@ -1392,13 +1383,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)), ) - # 2 touch calls from get_num_new_matched_tokens (2 groups) - touch_calls = runner.manager.touch.call_args_list - assert len(touch_calls) == 2 - # full attention group touched all 3 blocks - assert len(touch_calls[0].args[0]) == 3 - # sliding window group touched just the last 2 blocks - assert len(touch_calls[1].args[0]) == 2 + runner.manager.touch.assert_not_called() # 3 blocks are hit on GPU [0, 1, 2] # 1 block loaded [3,] @@ -1464,15 +1449,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool generate_store_output(keys) ) runner.run(decoded_tokens=[0]) - # _touch called from get_num_new_matched_tokens (2 groups) and - # _get_reqs_to_store (2 groups) → 4 touch calls total. - # Group 0 has 2 offload keys, group 1 has 1. - touch_calls = runner.manager.touch.call_args_list - assert len(touch_calls) == 4 - assert len(touch_calls[0].args[0]) == 2 - assert len(touch_calls[1].args[0]) == 1 - assert len(touch_calls[2].args[0]) == 2 - assert len(touch_calls[3].args[0]) == 1 + runner.manager.touch.assert_not_called() # Get to 31 tokens # No further blocks offloaded @@ -1482,11 +1459,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool # Group 0 blocks: [0, 1], ending_token_offset = 24 # Group 1 blocks: [0, 1], ending_token_offset = 32 runner.run(decoded_tokens=[0]) - # _get_reqs_to_store touch: only group 1 has a new block to store - touch_calls = runner.manager.touch.call_args_list - assert len(touch_calls) == 2 - assert len(touch_calls[0].args[0]) == 2 - assert len(touch_calls[1].args[0]) == 2 + runner.manager.touch.assert_not_called() # Get to 35 tokens # No further blocks offloaded @@ -1496,11 +1469,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool # Group 0 blocks: [0, 1, 2], ending_token_offset = 36 # Group 1 blocks: [0, 1], ending_token_offset = 32 runner.run(decoded_tokens=[0]) - # _get_reqs_to_store touch: only group 0 has a new block to store - touch_calls = runner.manager.touch.call_args_list - assert len(touch_calls) == 2 - assert len(touch_calls[0].args[0]) == 3 - assert len(touch_calls[1].args[0]) == 2 + runner.manager.touch.assert_not_called() # Get to 47 tokens # No further blocks offloaded @@ -1510,11 +1479,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool # Group 0 blocks: [0, 1, 2, 3], ending_token_offset = 4 # Group 1 blocks: [0, 1, 2], ending_token_offset = 48 runner.run(decoded_tokens=[0]) - # _get_reqs_to_store touch: both groups have a new block, each with 1 key - touch_calls = runner.manager.touch.call_args_list - assert len(touch_calls) == 2 - assert len(touch_calls[0].args[0]) == 4 - assert len(touch_calls[1].args[0]) == 3 + runner.manager.touch.assert_not_called() runner.run(decoded_tokens=[0], expected_stored=((0, 3), (1, 2))) diff --git a/tests/v1/kv_offload/cpu/test_manager.py b/tests/v1/kv_offload/cpu/test_manager.py index a79f480ff5..4fce65fb2a 100644 --- a/tests/v1/kv_offload/cpu/test_manager.py +++ b/tests/v1/kv_offload/cpu/test_manager.py @@ -23,6 +23,7 @@ from vllm.v1.kv_offload.cpu.common import ( ) from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy +from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy def make_req_context( @@ -430,31 +431,34 @@ def test_cpu_manager(): assert cpu_manager.lookup(to_key(0), _EMPTY_REQ_CTX) is LookupResult.MISS # prepare load [2, 3] - prepare_load_output = cpu_manager.prepare_load(to_keys([2, 3]), _EMPTY_REQ_CTX) + load_ctx = make_req_context("load-2-3") + prepare_load_output = cpu_manager.prepare_load(to_keys([2, 3]), load_ctx) verify_load_output(prepare_load_output, [1, 2]) # prepare store with no space ([2, 3] is being loaded) assert cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX) is None # complete load [2, 3]. Load changes the eviction list, making 2, 3 recent. - cpu_manager.complete_load(to_keys([2, 3]), _EMPTY_REQ_CTX) + cpu_manager.complete_load(to_keys([2, 3]), load_ctx) + cpu_manager.on_request_finished(load_ctx) - # prepare store [6, 7, 8] -> evicts [4, 5, 2] (oldest) + # prepare store [6, 7, 8] -> evicts [4, 5, 3] (oldest). Within + # the accessed prefix [2, 3], the tail is less valuable than the head. prepare_store_output = cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX) verify_store_output( prepare_store_output, ExpectedPrepareStoreOutput( keys_to_store=[6, 7, 8], - store_chunk_ids=[1, 0, 3], - evicted_keys=[4, 5, 2], + store_chunk_ids=[2, 0, 3], + evicted_keys=[4, 5, 3], ), ) # complete store [6, 7, 8] cpu_manager.complete_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX) - # touch [3, 6, 7] (move to end of LRU order) - cpu_manager.touch(to_keys([3, 6, 7]), _EMPTY_REQ_CTX) + # touch [2, 6, 7] (move to end of LRU order) + cpu_manager.touch(to_keys([2, 6, 7]), _EMPTY_REQ_CTX) # prepare store [7, 9] -> evicts [8] (oldest following previous touch) prepare_store_output = cpu_manager.prepare_store(to_keys([9]), _EMPTY_REQ_CTX) @@ -477,7 +481,7 @@ def test_cpu_manager(): verify_events( cpu_manager.take_events(), expected_stores=({3, 4, 5}, {6, 7, 8}), - expected_evictions=({4, 5, 2}, {8}), + expected_evictions=({4, 5, 3}, {8}), ) @@ -518,6 +522,65 @@ def test_prepare_load_preserves_key_order(): manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant +def test_lru_batch_eviction_failure_is_atomic(): + manager = make_cpu_manager(num_chunks=4, cache_policy="lru") + policy = manager._policy + assert isinstance(policy, LRUCachePolicy) + keys = to_keys([1, 2, 3, 4]) + assert manager.prepare_store(keys, _EMPTY_REQ_CTX) is not None + manager.complete_store(keys, _EMPTY_REQ_CTX) + + protected = set(keys[1:]) + assert policy.evict(2, protected) is None + assert all(manager.lookup(key, _EMPTY_REQ_CTX) is LookupResult.HIT for key in keys) + + evicted = policy.evict(1, protected) + assert evicted is not None + assert [key for key, _ in evicted] == [keys[0]] + + +def test_lru_repeated_pin_cycles_compact_lazy_heap_entries(): + manager = make_cpu_manager(num_chunks=1, cache_policy="lru") + policy = manager._policy + assert isinstance(policy, LRUCachePolicy) + key = to_key(1) + assert manager.prepare_store([key], _EMPTY_REQ_CTX) is not None + manager.complete_store([key], _EMPTY_REQ_CTX) + + for _ in range(128): + manager.prepare_load([key], _EMPTY_REQ_CTX) + manager.complete_load([key], _EMPTY_REQ_CTX) + + assert len(policy._heap) < 64 + evicted = policy.evict(1, set()) + assert evicted is not None + assert [evicted_key for evicted_key, _ in evicted] == [key] + + +def test_reset_discards_stale_request_access_classification(): + manager = make_cpu_manager(num_chunks=1, cache_policy="arc") + policy = manager._policy + assert isinstance(policy, ARCCachePolicy) + key = to_key(1) + resumed_ctx = make_req_context("resumed") + + assert manager.prepare_store([key], resumed_ctx) is not None + manager.complete_store([key], resumed_ctx) + manager.reset_cache() + + replacement_ctx = make_req_context("replacement") + assert manager.prepare_store([key], replacement_ctx) is not None + manager.complete_store([key], replacement_ctx) + manager.on_request_finished(replacement_ctx) + assert key in policy.t1 + + manager.prepare_load([key], resumed_ctx) + manager.complete_load([key], resumed_ctx) + manager.on_request_finished(resumed_ctx) + + assert key in policy.t2 + + class TestARCPolicy: """Unit tests for CPUOffloadingManager with ARC eviction policy.""" @@ -1042,6 +1105,48 @@ def test_filter_reused_manager_oversized_offer_makes_progress(): assert stored_keys == set(keys) +def test_store_threshold_does_not_hide_ready_reuse(): + manager = make_cpu_manager( + num_chunks=2, + cache_policy="lru", + store_threshold=2, + max_tracker_size=2, + ) + head, tail, noise_1, noise_2, replacement = to_keys(list(range(5))) + + seed_ctx = make_req_context("seed") + output = manager.prepare_store([head, tail], seed_ctx) + assert output is not None + assert not output.keys_to_store + output = manager.prepare_store([head, tail], seed_ctx) + assert output is not None + assert output.keys_to_store == [head, tail] + manager.complete_store([head, tail], seed_ctx) + manager.on_request_finished(seed_ctx) + + # Age both resident keys out of the admission tracker without storing the + # noise keys. Reusing the tail recreates its counter at one, below the + # threshold, but must still refresh its cache recency. + output = manager.prepare_store([noise_1, noise_2], make_req_context("noise")) + assert output is not None + assert not output.keys_to_store + reuse_ctx = make_req_context("reuse") + skipped_before_reuse = manager.stores_skipped_in_current_batch + output = manager.prepare_store([tail], reuse_ctx) + assert output is not None + assert not output.keys_to_store + assert manager.stores_skipped_in_current_batch == skipped_before_reuse + manager.on_request_finished(reuse_ctx) + + replacement_ctx = make_req_context("replacement") + output = manager.prepare_store([replacement], replacement_ctx) + assert output is not None + assert not output.keys_to_store + output = manager.prepare_store([replacement], replacement_ctx) + assert output is not None + assert output.evicted_keys == [head] + + def test_evictable_cache_chunk_count(): """ Verifies _num_evictable_cache_chunks is maintained correctly through the @@ -1156,3 +1261,318 @@ def test_touch_forwards_req_context_to_policy(monkeypatch): assert len(received) == 1 assert received[0][0] == keys assert received[0][1] is ctx + + +@pytest.mark.parametrize("finish_before_completion", [False, True]) +def test_request_finish_orders_lru_prefix_independent_of_store_completion( + finish_before_completion: bool, +): + """A transfer's completion order must not make the prefix head LRU.""" + manager = make_cpu_manager(num_chunks=4, cache_policy="lru") + prefix = to_keys([1, 2, 3, 4]) + ctx = make_req_context("prefix") + + output = manager.prepare_store(prefix, ctx) + assert output is not None + if finish_before_completion: + manager.on_request_finished(ctx) + manager.complete_store(set(prefix), ctx) + if not finish_before_completion: + manager.on_request_finished(ctx) + + output = manager.prepare_store(to_keys([5]), make_req_context("evict")) + assert output is not None + assert output.evicted_keys == to_keys([4]) + + +@pytest.mark.parametrize("finish_before_completion", [False, True]) +def test_request_finish_orders_lru_prefix_independent_of_load_completion( + finish_before_completion: bool, +): + manager = make_cpu_manager(num_chunks=4, cache_policy="lru") + prefix = to_keys([1, 2, 3, 4]) + + seed_ctx = make_req_context("seed") + output = manager.prepare_store(prefix, seed_ctx) + assert output is not None + manager.complete_store(prefix, seed_ctx) + + reuse_ctx = make_req_context("reuse") + manager.prepare_load(prefix, reuse_ctx) + if finish_before_completion: + manager.on_request_finished(reuse_ctx) + manager.complete_load(set(prefix), reuse_ctx) + if not finish_before_completion: + manager.on_request_finished(reuse_ctx) + + output = manager.prepare_store(to_keys([5]), make_req_context("evict")) + assert output is not None + assert output.evicted_keys == to_keys([4]) + + +def test_late_old_completion_does_not_override_newer_request_recency(): + manager = make_cpu_manager(num_chunks=4, cache_policy="lru") + old_keys = to_keys([1, 2]) + new_keys = to_keys([3, 4]) + + old_ctx = make_req_context("old") + assert manager.prepare_store(old_keys, old_ctx) is not None + manager.on_request_finished(old_ctx) + + new_ctx = make_req_context("new") + assert manager.prepare_store(new_keys, new_ctx) is not None + manager.complete_store(new_keys, new_ctx) + manager.on_request_finished(new_ctx) + + # Physical completion is late, but the logical access is still older. + manager.complete_store(old_keys, old_ctx) + + output = manager.prepare_store(to_keys([5]), make_req_context("evict")) + assert output is not None + assert output.evicted_keys == [old_keys[-1]] + + +def test_arc_counts_reuse_once_and_keeps_insertions_in_t1(): + manager = make_cpu_manager(num_chunks=4, cache_policy="arc") + policy = manager._policy + assert isinstance(policy, ARCCachePolicy) + prefix = to_keys([1, 2, 3, 4]) + + seed_ctx = make_req_context("seed") + assert manager.prepare_store(prefix, seed_ctx) is not None + manager.complete_store(prefix, seed_ctx) + # Repeated store offers from the same request must not become ARC hits. + assert manager.prepare_store(prefix, seed_ctx) is not None + # Nor should an internal read used to cascade the new chunks to a tier. + manager.prepare_load(prefix, seed_ctx) + manager.complete_load(prefix, seed_ctx) + manager.on_request_finished(seed_ctx) + + assert list(policy.t1) == list(reversed(prefix)) + assert not policy.t2 + + reuse_ctx = make_req_context("reuse") + manager.prepare_load(prefix, reuse_ctx) + manager.complete_load(prefix, reuse_ctx) + # A duplicate internal pin, as used by tiering cascades, is deduplicated. + manager.prepare_load(prefix, reuse_ctx) + manager.complete_load(prefix, reuse_ctx) + manager.on_request_finished(reuse_ctx) + + assert not policy.t1 + assert list(policy.t2) == list(reversed(prefix)) + + +def test_arc_reuse_wins_if_same_request_later_reinserts_key(): + manager = make_cpu_manager(num_chunks=1, cache_policy="arc") + policy = manager._policy + assert isinstance(policy, ARCCachePolicy) + key_1, key_2 = to_keys([1, 2]) + + seed_ctx = make_req_context("seed") + assert manager.prepare_store([key_1], seed_ctx) is not None + manager.complete_store([key_1], seed_ctx) + manager.on_request_finished(seed_ctx) + + reuse_ctx = make_req_context("reuse-and-reinsert") + manager.prepare_load([key_1], reuse_ctx) + manager.complete_load([key_1], reuse_ctx) + + other_ctx = make_req_context("evict-reused-key") + assert manager.prepare_store([key_2], other_ctx) is not None + manager.complete_store([key_2], other_ctx) + manager.on_request_finished(other_ctx) + + assert manager.prepare_store([key_1], reuse_ctx) is not None + manager.complete_store([key_1], reuse_ctx) + manager.on_request_finished(reuse_ctx) + + assert key_1 not in policy.t1 + assert list(policy.t2) == [key_1] + + +def test_request_finish_forwards_grouped_deduplicated_accesses(monkeypatch): + manager = make_cpu_manager(num_chunks=5) + existing = make_offload_key(b"existing", 0) + group_0_new = make_offload_key(b"group-0-new", 0) + group_1_head = make_offload_key(b"group-1-head", 1) + group_1_tail = make_offload_key(b"group-1-tail", 1) + + seed_ctx = make_req_context("seed") + assert manager.prepare_store([existing], seed_ctx) is not None + manager.complete_store([existing], seed_ctx) + + ctx = make_req_context("grouped") + # Group 1 is observed first, but policy finalization follows group index. + offered = [group_1_head, existing, group_0_new] + assert manager.prepare_store(offered, ctx) is not None + # Later offers extend a group's prefix and may repeat earlier keys. + assert ( + manager.prepare_store([group_0_new, group_1_head, group_1_tail], ctx) + is not None + ) + + received = [] + + def on_request_finished(key_groups, insertion_only_keys, reused_keys, req_context): + received.append((key_groups, insertion_only_keys, reused_keys, req_context)) + + monkeypatch.setattr(manager._policy, "on_request_finished", on_request_finished) + manager.on_request_finished(ctx) + manager.on_request_finished(ctx) + + assert received == [ + ( + ((existing, group_0_new), (group_1_head, group_1_tail)), + {group_0_new, group_1_head, group_1_tail}, + {existing}, + ctx, + ) + ] + + +def test_failed_store_records_access_once_and_uses_store_miss_hook(monkeypatch): + manager = make_cpu_manager(num_chunks=1) + resident, candidate = to_keys([1, 2]) + + seed_ctx = make_req_context("seed") + assert manager.prepare_store([resident], seed_ctx) is not None + manager.complete_store([resident], seed_ctx) + + pin_ctx = make_req_context("pin") + manager.prepare_load([resident], pin_ctx) + + record_calls = 0 + original_record = manager._record_request_cache_access + + def record_access(*args, **kwargs): + nonlocal record_calls + record_calls += 1 + return original_record(*args, **kwargs) + + store_miss_calls = [] + + def on_store_miss(keys, req_context): + store_miss_calls.append((list(keys), req_context)) + + finalized = [] + + def on_request_finished(key_groups, insertion_only_keys, reused_keys, req_context): + finalized.append((key_groups, insertion_only_keys, reused_keys, req_context)) + + monkeypatch.setattr(manager, "_record_request_cache_access", record_access) + monkeypatch.setattr(manager._policy, "on_store_miss", on_store_miss) + monkeypatch.setattr(manager._policy, "on_request_finished", on_request_finished) + + ctx = make_req_context("failed-store") + assert manager.prepare_store([candidate, resident], ctx) is None + assert record_calls == 1 + assert store_miss_calls == [([candidate], ctx)] + + manager.on_request_finished(ctx) + assert finalized == [ + (((candidate, resident),), set(), {resident}, ctx), + ] + + manager.complete_load([resident], pin_ctx) + + +def test_arc_ghost_hit_adapts_once_per_request_before_insertion(): + manager = make_cpu_manager(num_chunks=2, cache_policy="arc") + policy = manager._policy + assert isinstance(policy, ARCCachePolicy) + keys = to_keys([1, 2, 3]) + + seed_ctx = make_req_context("seed") + assert manager.prepare_store(keys[:2], seed_ctx) is not None + manager.complete_store(keys[:2], seed_ctx) + manager.on_request_finished(seed_ctx) + + evict_ctx = make_req_context("create-ghost") + output = manager.prepare_store([keys[2]], evict_ctx) + assert output is not None + [ghost_key] = output.evicted_keys + manager.complete_store([keys[2]], evict_ctx) + manager.on_request_finished(evict_ctx) + assert ghost_key in policy.b1 + + resident_keys = [key for key in keys if key != ghost_key] + pin_ctx = make_req_context("pin-residents") + manager.prepare_load(resident_keys, pin_ctx) + + ghost_ctx = make_req_context("ghost-hit") + target_before = policy.target_t1_size + assert manager.prepare_store([ghost_key], ghost_ctx) is None + target_after_first_offer = policy.target_t1_size + assert target_after_first_offer > target_before + assert manager.prepare_store([ghost_key], ghost_ctx) is None + assert policy.target_t1_size == target_after_first_offer + + manager.complete_load(resident_keys, pin_ctx) + output = manager.prepare_store([ghost_key], ghost_ctx) + assert output is not None + assert policy.target_t1_size == target_after_first_offer + manager.complete_store([ghost_key], ghost_ctx) + manager.on_request_finished(ghost_ctx) + assert ghost_key in policy.t1 + assert ghost_key not in policy.t2 + + +def test_arc_pending_chunk_is_not_a_frequency_hit(): + manager = make_cpu_manager(num_chunks=2, cache_policy="arc") + policy = manager._policy + assert isinstance(policy, ARCCachePolicy) + key = to_key(1) + + writer_ctx = make_req_context("writer") + assert manager.prepare_store([key], writer_ctx) is not None + + observer_ctx = make_req_context("pending-observer") + output = manager.prepare_store([key], observer_ctx) + assert output is not None and not output.keys_to_store + manager.on_request_finished(observer_ctx) + + chunk = policy.get(key) + assert chunk is not None and not chunk.is_ready + assert key in policy.t1 + assert key not in policy.t2 + + +def test_request_key_positions_override_store_observation_order(): + manager = make_cpu_manager(num_chunks=2, cache_policy="lru") + head, tail = to_keys([1, 2]) + ctx = make_req_context("out-of-order-store") + ctx.set_offload_key_position(head, 16) + ctx.set_offload_key_position(tail, 32) + + # A backfill or partial-tail path may offer a later key first. + assert manager.prepare_store([tail], ctx) is not None + assert manager.prepare_store([head], ctx) is not None + manager.on_request_finished(ctx) + manager.complete_store({head, tail}, ctx) + + output = manager.prepare_store([to_key(3)], make_req_context("evict-tail")) + assert output is not None + assert output.evicted_keys == [tail] + + +def test_request_key_positions_order_tails_across_kv_groups(): + manager = make_cpu_manager(num_chunks=4, cache_policy="lru") + group_0_head = make_offload_key(b"group-0-head", 0) + group_0_tail = make_offload_key(b"group-0-tail", 0) + group_1_head = make_offload_key(b"group-1-head", 1) + group_1_tail = make_offload_key(b"group-1-tail", 1) + keys = [group_0_head, group_0_tail, group_1_head, group_1_tail] + ctx = make_req_context("hybrid-groups") + for key in (group_0_head, group_1_head): + ctx.set_offload_key_position(key, 16) + for key in (group_0_tail, group_1_tail): + ctx.set_offload_key_position(key, 32) + + assert manager.prepare_store(keys, ctx) is not None + manager.complete_store(keys, ctx) + manager.on_request_finished(ctx) + + output = manager.prepare_store(to_keys([10, 11]), make_req_context("evict")) + assert output is not None + assert set(output.evicted_keys) == {group_0_tail, group_1_tail} diff --git a/tests/v1/kv_offload/tiering/test_tiering_offloading.py b/tests/v1/kv_offload/tiering/test_tiering_offloading.py index f737685621..771153ec47 100644 --- a/tests/v1/kv_offload/tiering/test_tiering_offloading.py +++ b/tests/v1/kv_offload/tiering/test_tiering_offloading.py @@ -40,6 +40,7 @@ from vllm.v1.kv_offload.base import ( TierMatcher, make_offload_key, ) +from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy from vllm.v1.kv_offload.tiering.base import ( JobResult, SecondaryTierManager, @@ -753,14 +754,64 @@ class TestTieringOffloadingManager: # Touch chunks self.manager.touch(chunks, _CTX) - # Verify touch was called on primary tier (check LRU order) - primary_keys = list(self.primary_tier._policy.evictable_chunks.keys()) - assert primary_keys[-3:] == list(reversed(chunks)) + # Verify touch was called on the primary tier (head is most recent). + policy = self.primary_tier._policy + assert isinstance(policy, LRUCachePolicy) + ranks = policy._ranks + assert ranks[chunks[2]] < ranks[chunks[1]] < ranks[chunks[0]] # Verify touch was propagated to all secondary tiers self.secondary_tier1.touch.assert_called_once_with(chunks, _CTX) self.secondary_tier2.touch.assert_called_once_with(chunks, _CTX) + def test_request_order_survives_late_cascade_completion(self, manager_setup): + """Cascade completion must not replace request order with I/O order.""" + chunks = to_keys(range(5)) + self._start_request() + output = self.manager.prepare_store(chunks, _CTX) + assert output is not None + self.manager.complete_store(chunks, _CTX, success=True) + + # Finalize while both secondary-tier cascades still pin primary chunks. + self.manager.on_request_finished(_CTX) + self._simulate_on_schedule_end() + self._simulate_on_schedule_end() + + evict_ctx = ReqContext(req_id="evict") + self._start_request(evict_ctx) + output = self.manager.prepare_store(to_keys([5]), evict_ctx) + assert output is not None + assert output.evicted_keys == [chunks[-1]] + + def test_late_old_store_submission_does_not_override_newer_request_order( + self, manager_setup + ): + """Cascade submission after finish is not a new primary access.""" + old_blocks = to_keys(range(2)) + old_ctx = ReqContext(req_id="old") + self._start_request(old_ctx) + assert self.manager.prepare_store(old_blocks, old_ctx) is not None + self.manager.on_request_finished(old_ctx) + + new_blocks = to_keys(range(2, 5)) + new_ctx = ReqContext(req_id="new") + self._start_request(new_ctx) + assert self.manager.prepare_store(new_blocks, new_ctx) is not None + self.manager.complete_store(new_blocks, new_ctx, success=True) + self.manager.on_request_finished(new_ctx) + + # The old GPU->primary write lands after the newer request finished. + # Its cascade pins must not turn it into the more recent request. + self.manager.complete_store(old_blocks, old_ctx, success=True) + self._simulate_on_schedule_end() + self._simulate_on_schedule_end() + + evict_ctx = ReqContext(req_id="evict") + self._start_request(evict_ctx) + output = self.manager.prepare_store(to_keys([5]), evict_ctx) + assert output is not None + assert output.evicted_keys == [old_blocks[-1]] + def test_failed_store_no_cascade(self, manager_setup): """Test that failed GPU→primary store doesn't cascade.""" chunks = to_keys(range(3)) @@ -873,10 +924,10 @@ class TestTieringOffloadingManager: job_metadata = self.secondary_tier1.submit_store.call_args.args[0] assert job_metadata.req_context is ctx - def test_on_request_finished_delays_secondary_until_store_submitted( + def test_on_request_finished_delays_only_secondary_until_store_submitted( self, manager_setup ): - """Manager hook is eager; secondary hooks wait for cascade submission.""" + """Primary order commits immediately; secondary cleanup waits.""" chunks = to_keys(range(2)) ctx = ReqContext(req_id="req_delayed_secondary") calls: list[tuple[str, str]] = [] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 5cc66e4bf0..b253a23998 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -427,8 +427,11 @@ class RequestOffloadState: None, group_config.hashes_per_chunk, ): - group_state.offload_keys.append( - make_offload_key(req_block_hash, group_config.group_idx) + key = make_offload_key(req_block_hash, group_config.group_idx) + group_state.offload_keys.append(key) + self.req_context.set_offload_key_position( + key, + len(group_state.offload_keys) * group_config.tokens_per_chunk, ) def update_block_id_groups( @@ -742,37 +745,6 @@ class OffloadingConnectorScheduler: ) return None if defer_lookup or pending_in_window else consecutive_hits - def _touch(self, req_status: RequestOffloadState): - for group_config, group_state in zip( - self.config.kv_group_configs, req_status.group_states - ): - if group_config.sliding_window_size_in_chunks is None: - self.manager.touch(group_state.offload_keys, req_status.req_context) - else: - # Keep only chunks needed to hit the original request, plus - # decoded chunks. - chunks_to_skip = max( - 0, - group_state.num_hit_chunks - - group_config.sliding_window_size_in_chunks, - ) - self.manager.touch( - group_state.offload_keys[chunks_to_skip:], - req_status.req_context, - ) - if req_status.partial_tail_boundary is not None: - self.manager.touch( - tuple( - self._make_boundary_key( - req_status.req, - group.group_idx, - req_status.partial_tail_boundary, - ) - for group in self.config.kv_group_configs - ), - req_status.req_context, - ) - def _lookup_complete_chunks( self, req_status: RequestOffloadState, @@ -977,10 +949,16 @@ class OffloadingConnectorScheduler: return num_hit_tokens def _make_boundary_key( - self, request: Request, group_idx: int, boundary_tokens: int + self, + request: Request, + group_idx: int, + boundary_tokens: int, + req_context: ReqContext, ) -> OffloadKey: hash_idx = boundary_tokens // self.config.tokens_per_hash - 1 - return make_offload_key(request.block_hashes[hash_idx], group_idx) + key = make_offload_key(request.block_hashes[hash_idx], group_idx) + req_context.set_offload_key_position(key, boundary_tokens) + return key def _lookup( self, @@ -1015,7 +993,10 @@ class OffloadingConnectorScheduler: boundary_keys = [] for group_config in self.config.kv_group_configs: key = self._make_boundary_key( - req_status.req, group_config.group_idx, boundary + req_status.req, + group_config.group_idx, + boundary, + req_status.req_context, ) boundary_keys.append(key) result = self.manager.lookup(key, req_status.req_context) @@ -1108,8 +1089,6 @@ class OffloadingConnectorScheduler: self._maybe_observe_lookup_async_delay(req_status) req_status.update_num_hit_chunks(num_computed_tokens + (num_hit_tokens or 0)) - self._touch(req_status) - return num_hit_tokens, bool(num_hit_tokens) def update_state_after_alloc( @@ -1183,7 +1162,10 @@ class OffloadingConnectorScheduler: if partial_tail_boundary is not None: keys_to_load.append( self._make_boundary_key( - request, group_config.group_idx, partial_tail_boundary + request, + group_config.group_idx, + partial_tail_boundary, + req_status.req_context, ) ) @@ -1307,7 +1289,9 @@ class OffloadingConnectorScheduler: ): continue - key = self._make_boundary_key(req, group_idx, boundary) + key = self._make_boundary_key( + req, group_idx, boundary, req_status.req_context + ) store_output = self.manager.prepare_store([key], req_status.req_context) if store_output is None: self._connector_stats.increase_counter( @@ -1402,7 +1386,9 @@ class OffloadingConnectorScheduler: ): continue keys = [ - self._make_boundary_key(req, group.group_idx, boundary) + self._make_boundary_key( + req, group.group_idx, boundary, req_status.req_context + ) for group in self.config.kv_group_configs ] block_ids = [ @@ -1689,8 +1675,6 @@ class OffloadingConnectorScheduler: req_status.advance_stored_idx(num_offloadable_tokens) continue - self._touch(req_status) - keys_to_store = set(store_output.keys_to_store) group_sizes: list[int] = [] diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index 93f8e4bda4..988747e06e 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -96,6 +96,12 @@ class ReqContext: # kv_transfer_params once (in on_new_request) and read the result back # on later calls for the same request. _state: dict[type, Any] = field(default_factory=dict, repr=False, init=False) + # End-token position for each key in this request. The scheduler records + # these positions so managers can recover prefix order even when store + # calls arrive out of order (for example, SWA backfills). + _offload_key_positions: dict[OffloadKey, int] = field( + default_factory=dict, repr=False, init=False + ) def set_state(self, val: Any) -> None: self._state[type(val)] = val @@ -103,6 +109,12 @@ class ReqContext: def get_state(self, cls: type[_T]) -> _T | None: return self._state.get(cls) + def set_offload_key_position(self, key: OffloadKey, end_token: int) -> None: + self._offload_key_positions[key] = end_token + + def get_offload_key_position(self, key: OffloadKey) -> int | None: + return self._offload_key_positions.get(key) + class LookupResult(Enum): """Result of OffloadingManager.lookup().""" diff --git a/vllm/v1/kv_offload/cpu/manager.py b/vllm/v1/kv_offload/cpu/manager.py index 67b469b6a3..9d85090f2d 100644 --- a/vllm/v1/kv_offload/cpu/manager.py +++ b/vllm/v1/kv_offload/cpu/manager.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections import OrderedDict from collections.abc import Collection, Iterable +from dataclasses import dataclass, field from typing_extensions import override @@ -18,6 +19,7 @@ from vllm.v1.kv_offload.base import ( PrepareStoreOutput, ReqContext, RequestOffloadingContext, + get_offload_group_idx, ) from vllm.v1.kv_offload.cpu.common import ( CPULoadStoreSpec, @@ -27,6 +29,20 @@ from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory +@dataclass(slots=True) +class _RequestCacheAccess: + """Cache keys observed by one request, grouped in prefix order.""" + + owner: object + cache_generation: int + key_groups: dict[int, list[OffloadKey]] = field(default_factory=dict) + seen_keys: set[OffloadKey] = field(default_factory=set) + inserted_keys: set[OffloadKey] = field(default_factory=set) + reused_keys: set[OffloadKey] = field(default_factory=set) + store_miss_keys: set[OffloadKey] = field(default_factory=set) + finished: bool = False + + class CPUOffloadingManager(OffloadingManager): """ An OffloadingManager with a pluggable CachePolicy, resolved by name via @@ -66,6 +82,7 @@ class CPUOffloadingManager(OffloadingManager): self.max_tracker_size: int = max_tracker_size self.stores_skipped_in_current_batch: int = 0 self.allocation_sizes_in_current_batch: list[int] = [] + self._cache_generation = 0 # Number of chunk references. It is ordered so can evict the LRU entry in O(1). self.counts: OrderedDict[OffloadKey, int] | None = ( @@ -124,10 +141,48 @@ class CPUOffloadingManager(OffloadingManager): num_unprotected -= 1 self.counts[key] = 1 + def _get_request_cache_access(self, req_context: ReqContext) -> _RequestCacheAccess: + state = req_context.get_state(_RequestCacheAccess) + if ( + state is None + or state.owner is not self + or state.cache_generation != self._cache_generation + ): + state = _RequestCacheAccess( + owner=self, cache_generation=self._cache_generation + ) + req_context.set_state(state) + return state + + def _record_request_cache_access( + self, + keys: Iterable[OffloadKey], + req_context: ReqContext, + inserted_keys: Iterable[OffloadKey] = (), + reused_keys: Iterable[OffloadKey] = (), + ) -> None: + state = self._get_request_cache_access(req_context) + for key in keys: + if key in state.seen_keys: + continue + assert not state.finished, ( + "New cache keys observed after request finalization" + ) + group_idx = get_offload_group_idx(key) + state.key_groups.setdefault(group_idx, []).append(key) + state.seen_keys.add(key) + state.inserted_keys.update(inserted_keys) + # Re-reading a chunk inserted by this request is an internal transfer + # (for example, a tiering cascade), not a second cache access. + state.reused_keys.update( + key for key in reused_keys if key not in state.inserted_keys + ) + # --- OffloadingManager interface --- @override def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: + self._get_request_cache_access(req_context) return RequestOffloadingContext() @override @@ -144,6 +199,15 @@ class CPUOffloadingManager(OffloadingManager): self, keys: Collection[OffloadKey], req_context: ReqContext, + ) -> LoadStoreSpec: + return self._prepare_load(keys, req_context, record_access=True) + + def _prepare_load( + self, + keys: Collection[OffloadKey], + req_context: ReqContext, + *, + record_access: bool, ) -> LoadStoreSpec: chunks = [] for key in keys: @@ -156,6 +220,8 @@ class CPUOffloadingManager(OffloadingManager): assert self._num_evictable_cache_chunks >= 0 chunk.ref_cnt += 1 chunks.append(chunk) + if record_access: + self._record_request_cache_access(keys, req_context, reused_keys=keys) return self._get_load_store_spec(keys, chunks) @override @@ -181,13 +247,52 @@ class CPUOffloadingManager(OffloadingManager): keys: Collection[OffloadKey], req_context: ReqContext, ) -> PrepareStoreOutput | None: + keys = list(keys) if self.counts is not None: - num_keys = len(keys) self._record_accesses(keys) - keys = [k for k in keys if self.counts.get(k, 0) >= self.store_threshold] - self.stores_skipped_in_current_batch += num_keys - len(keys) - # filter out chunks that are already stored - keys_to_store = [k for k in keys if self._policy.get(k) is None] + # Partition the original offer before admission filtering. Ready + # resident chunks are request reuses even if their tracker entry was + # aged out; the threshold applies only to new store candidates. + keys_to_store: list[OffloadKey] = [] + ready_existing_keys: list[OffloadKey] = [] + for key in keys: + chunk = self._policy.get(key) + if chunk is None: + keys_to_store.append(key) + else: + if chunk.is_ready: + ready_existing_keys.append(key) + + if self.counts is not None: + num_store_candidates = len(keys_to_store) + keys_to_store = [ + key + for key in keys_to_store + if self.counts.get(key, 0) >= self.store_threshold + ] + self.stores_skipped_in_current_batch += num_store_candidates - len( + keys_to_store + ) + + state = self._get_request_cache_access(req_context) + new_store_misses = [ + key for key in keys_to_store if key not in state.store_miss_keys + ] + if new_store_misses: + # ARC learns from B1/B2 before insert() removes the ghost entry. + # Deduplication makes this one policy observation per request. + self._policy.on_store_miss(new_store_misses, req_context) + state.store_miss_keys.update(new_store_misses) + + recorded_keys = set(keys_to_store) + recorded_keys.update(ready_existing_keys) + # Record once before any allocation-related early return. Store + # candidates are classified as insertions only after allocation succeeds. + self._record_request_cache_access( + (key for key in keys if key in recorded_keys), + req_context, + reused_keys=ready_existing_keys, + ) if not keys_to_store: return PrepareStoreOutput( @@ -239,6 +344,7 @@ class CPUOffloadingManager(OffloadingManager): for key, chunk in zip(keys_to_store, chunks): self._policy.insert(key, chunk) self._num_write_pending_chunks += len(keys_to_store) + state.inserted_keys.update(keys_to_store) # build store specs for allocated chunks store_spec = self._get_load_store_spec(keys_to_store, chunks) @@ -284,6 +390,35 @@ class CPUOffloadingManager(OffloadingManager): ) ) + @override + def on_request_finished(self, req_context: ReqContext) -> None: + state = req_context.get_state(_RequestCacheAccess) + if ( + state is None + or state.owner is not self + or state.cache_generation != self._cache_generation + or state.finished + ): + return + state.finished = True + key_groups = [] + for group_idx in sorted(state.key_groups): + keys = state.key_groups[group_idx] + positions = { + key: position + for key in keys + if (position := req_context.get_offload_key_position(key)) is not None + } + if len(positions) == len(keys): + keys = sorted(keys, key=positions.__getitem__) + key_groups.append(tuple(keys)) + self._policy.on_request_finished( + tuple(key_groups), + state.inserted_keys - state.reused_keys, + state.reused_keys, + req_context, + ) + @override def reset_cache(self) -> None: # Clear ALL chunks unconditionally. The scheduler's _stale_job_threshold @@ -292,6 +427,7 @@ class CPUOffloadingManager(OffloadingManager): # flushes in-flight load job IDs to the workers before any new stores # can begin, preventing a cross-direction data race on reused offload chunk IDs. self._policy.clear() + self._cache_generation += 1 self._num_evictable_cache_chunks = 0 self._num_write_pending_chunks = 0 diff --git a/vllm/v1/kv_offload/cpu/policies/arc.py b/vllm/v1/kv_offload/cpu/policies/arc.py index e413bd603e..882ee02219 100644 --- a/vllm/v1/kv_offload/cpu/policies/arc.py +++ b/vllm/v1/kv_offload/cpu/policies/arc.py @@ -1,12 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections import OrderedDict -from collections.abc import Iterable, Iterator +from collections.abc import Iterable, Iterator, Sequence from typing_extensions import override from vllm.v1.kv_offload.base import OffloadKey, ReqContext -from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus +from vllm.v1.kv_offload.cpu.policies.base import ( + CachePolicy, + ChunkStatus, + order_request_keys, +) class ARCCachePolicy(CachePolicy): @@ -24,12 +28,11 @@ class ARCCachePolicy(CachePolicy): Searches T1 and T2 for chunk hashes and counts consecutive hits until a miss or non-ready chunk is encountered. - 2. Cache touch (touch) - Adaptive Learning: - For each key (in reverse order): - - If in T1: Move to T2 (promotion from recent to frequent). - - If in T2: Move to MRU position (end of queue). - - If in B1 ghost list: Increase target_t1_size. - - If in B2 ghost list: Decrease target_t1_size. + 2. Request access - Adaptive Learning: + - Ready chunks reused by a request move from T1 to T2 once, or + move to the MRU end of T2. + - B1/B2 misses adjust target_t1_size once per request before + insertion removes their ghost entries. 3. Chunk eviction (evict) - Adaptive Replacement: Determines eviction source based on adaptive target: @@ -39,7 +42,7 @@ class ARCCachePolicy(CachePolicy): 4. Chunk insertion (insert): New chunks are always inserted into T1 and removed from B1/B2 if - present. Chunks may later be promoted to T2 during touch operations. + present. A later request reuse may promote them to T2. Adaptive Behavior: The algorithm self-tunes the recency vs. frequency trade-off: @@ -71,6 +74,26 @@ class ARCCachePolicy(CachePolicy): if self.t1.pop(key, None) is None: self.t2.pop(key, None) + def _adapt_to_ghost_hit(self, key: OffloadKey) -> bool: + if key in self.b1: + delta = max(1, len(self.b2) / len(self.b1)) + self.target_t1_size = min(self.target_t1_size + delta, self.cache_capacity) + self.b1.move_to_end(key) + return True + if key in self.b2: + delta = max(1, len(self.b1) / len(self.b2)) + self.target_t1_size = max(self.target_t1_size - delta, 0) + self.b2.move_to_end(key) + return True + return False + + @override + def on_store_miss( + self, keys: Iterable[OffloadKey], req_context: ReqContext + ) -> None: + for key in reversed(list(keys)): + self._adapt_to_ghost_hit(key) + @override def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None: for key in reversed(list(keys)): @@ -86,19 +109,33 @@ class ARCCachePolicy(CachePolicy): elif key in self.t2: self.t2.move_to_end(key) - elif key in self.b1: - delta = max(1, len(self.b2) / len(self.b1)) - self.target_t1_size = min( - self.target_t1_size + delta, self.cache_capacity - ) - # move to MRU position (end) to keep it fresh in the ghost list - self.b1.move_to_end(key) + else: + self._adapt_to_ghost_hit(key) - elif key in self.b2: - delta = max(1, len(self.b1) / len(self.b2)) - self.target_t1_size = max(self.target_t1_size - delta, 0) - # move to MRU position (end) to keep it fresh in the ghost list - self.b2.move_to_end(key) + @override + def on_request_finished( + self, + key_groups: Sequence[Sequence[OffloadKey]], + insertion_only_keys: set[OffloadKey], + reused_keys: set[OffloadKey], + req_context: ReqContext, + ) -> None: + for key in reversed(order_request_keys(key_groups, req_context)): + if key in insertion_only_keys: + # A store is not a frequency hit. Preserve T1 membership + # while still restoring tail-to-head recency. + if key in self.t1: + self.t1.move_to_end(key) + elif key in self.t2: + self.t2.move_to_end(key) + continue + + # Ready chunks reused by this request count as one access. + if key in reused_keys and key in self.t1: + chunk = self.t1.pop(key) + self.t2[key] = chunk + elif key in reused_keys and key in self.t2: + self.t2.move_to_end(key) @override def clear(self) -> None: diff --git a/vllm/v1/kv_offload/cpu/policies/base.py b/vllm/v1/kv_offload/cpu/policies/base.py index 4059dd742f..3efffbd396 100644 --- a/vllm/v1/kv_offload/cpu/policies/base.py +++ b/vllm/v1/kv_offload/cpu/policies/base.py @@ -2,11 +2,26 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import ctypes from abc import ABC, abstractmethod -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from vllm.v1.kv_offload.base import OffloadKey, ReqContext +def order_request_keys( + key_groups: Sequence[Sequence[OffloadKey]], req_context: ReqContext +) -> list[OffloadKey]: + """Return one head-to-tail order across all KV cache groups.""" + keys = [key for group in key_groups for key in group] + positions = { + key: position + for key in keys + if (position := req_context.get_offload_key_position(key)) is not None + } + if len(positions) == len(keys): + keys.sort(key=positions.__getitem__) + return keys + + class ChunkStatus(ctypes.Structure): """ Offloading status for a single chunk of KV data. @@ -66,6 +81,41 @@ class CachePolicy(ABC): req_context: Per-request context for the request touching these chunks. """ + def on_store_miss( + self, keys: Iterable[OffloadKey], req_context: ReqContext + ) -> None: + """Observe store misses before their cache entries are inserted. + + The default delegates to ``touch`` for compatibility with external + policies. Policies may override this hook when a miss has distinct + semantics, such as ARC adapting to a ghost-list hit. + """ + self.touch(keys, req_context) + + def on_request_finished( + self, + key_groups: Sequence[Sequence[OffloadKey]], + insertion_only_keys: set[OffloadKey], + reused_keys: set[OffloadKey], + req_context: ReqContext, + ) -> None: + """Apply one request-scoped cache access in prefix order. + + ``key_groups`` contains keys observed for each KV cache group in + head-to-tail order. ``insertion_only_keys`` and ``reused_keys`` + distinguish chunks only created by this request from ready chunks it + actually reused, which matters for policies such as ARC where reuse + changes frequency but insertion and pending observations do not. + + The default forwards one touch per group, preserving compatibility + for experimental out-of-tree policies while moving those touches to + request finalization. Policies that distinguish insertion from reuse + can override this hook and inspect the access classifications. + """ + del insertion_only_keys, reused_keys + for keys in key_groups: + self.touch(keys, req_context) + @abstractmethod def evict( self, n: int, protected: set[OffloadKey] diff --git a/vllm/v1/kv_offload/cpu/policies/lru.py b/vllm/v1/kv_offload/cpu/policies/lru.py index c1dc631d4f..5b5f65dec7 100644 --- a/vllm/v1/kv_offload/cpu/policies/lru.py +++ b/vllm/v1/kv_offload/cpu/policies/lru.py @@ -1,28 +1,74 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections import OrderedDict -from collections.abc import Iterable +import heapq +from collections.abc import Iterable, Sequence from typing_extensions import override from vllm.v1.kv_offload.base import OffloadKey, ReqContext -from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus +from vllm.v1.kv_offload.cpu.policies.base import ( + CachePolicy, + ChunkStatus, + order_request_keys, +) class LRUCachePolicy(CachePolicy): """ - LRU Caching policy that keeps a dedicated evictable list for fast eviction. + LRU caching policy with logical recency independent of transfer pinning. + + Evictable chunks live in a lazy-invalidating min-heap. A chunk's recency + can therefore be updated while it is pinned; when it later becomes + evictable, it enters the heap with the order assigned by the request + rather than its transfer-completion time. + A use is indicated by, - First time the key is added (store). - - Load job completion - - touch + - A request-scoped access. """ def __init__(self, cache_capacity: int): super().__init__(cache_capacity) - # Chunks with ref_cnt 0 (not participating in any loads/stores) ordered in LRU - self.evictable_chunks: OrderedDict[OffloadKey, None] = OrderedDict() self.chunks: dict[OffloadKey, ChunkStatus] = {} + self._ranks: dict[OffloadKey, int] = {} + self._evictable: set[OffloadKey] = set() + self._heap: list[tuple[int, OffloadKey]] = [] + self._next_rank = 0 + + def _assign_new_rank(self, key: OffloadKey) -> bool: + if key not in self.chunks: + return False + self._next_rank += 1 + self._ranks[key] = self._next_rank + return key in self._evictable + + def _push_evictable(self, key: OffloadKey) -> None: + heapq.heappush( + self._heap, + (self._ranks[key], key), + ) + + def _is_current(self, entry: tuple[int, OffloadKey]) -> bool: + rank, key = entry + return key in self._evictable and self._ranks.get(key) == rank + + def _maybe_compact_heap(self) -> None: + if len(self._heap) < max(64, 2 * len(self._evictable)): + return + self._heap = [(self._ranks[key], key) for key in self._evictable] + heapq.heapify(self._heap) + + def _update_recency(self, keys: Iterable[OffloadKey]) -> None: + updated_evictable = [key for key in keys if self._assign_new_rank(key)] + # Rebuilding is linear and substantially cheaper than k heap pushes + # for a long prefix. Small updates retain the incremental path. + if len(updated_evictable) >= max(64, len(self._evictable) // 4): + self._heap = [(self._ranks[key], key) for key in self._evictable] + heapq.heapify(self._heap) + else: + for key in updated_evictable: + self._push_evictable(key) + self._maybe_compact_heap() @override def get(self, key: OffloadKey) -> ChunkStatus | None: @@ -31,26 +77,40 @@ class LRUCachePolicy(CachePolicy): @override def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None: self.chunks[key] = chunk + self._next_rank += 1 + self._ranks[key] = self._next_rank if chunk.ref_cnt == 0: - self.evictable_chunks[key] = None + self._evictable.add(key) + self._push_evictable(key) @override def remove(self, key: OffloadKey) -> None: del self.chunks[key] - self.evictable_chunks.pop(key, None) + self._ranks.pop(key, None) + self._evictable.discard(key) @override def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None: - for key in reversed(list(keys)): - if key in self.evictable_chunks: - self.evictable_chunks.move_to_end(key) - # active chunks are untouched as they are non-evictable now. They - # will eventually reach the end of evictable_chunks when they finish. + self._update_recency(reversed(list(keys))) + + @override + def on_request_finished( + self, + key_groups: Sequence[Sequence[OffloadKey]], + insertion_only_keys: set[OffloadKey], + reused_keys: set[OffloadKey], + req_context: ReqContext, + ) -> None: + del insertion_only_keys, reused_keys + self._update_recency(reversed(order_request_keys(key_groups, req_context))) @override def clear(self) -> None: - self.evictable_chunks.clear() self.chunks.clear() + self._ranks.clear() + self._evictable.clear() + self._heap.clear() + self._next_rank = 0 @override def evict( @@ -59,22 +119,44 @@ class LRUCachePolicy(CachePolicy): if n == 0: return [] - candidates: list[tuple[OffloadKey, ChunkStatus]] = [] - for key, _ in self.evictable_chunks.items(): + selected: list[tuple[tuple[int, OffloadKey], ChunkStatus]] = [] + selected_keys: set[OffloadKey] = set() + deferred: list[tuple[int, OffloadKey]] = [] + while self._heap and len(selected) < n: + entry = heapq.heappop(self._heap) + if not self._is_current(entry): + continue + key = entry[1] + # Re-pinning without a recency change can leave an equivalent + # lazy entry behind. Select each cache key at most once. + if key in selected_keys: + continue if key in protected: + deferred.append(entry) continue - chunk = self.chunks[key] assert chunk.ref_cnt == 0 - candidates.append((key, chunk)) - if len(candidates) == n: - break + selected.append((entry, chunk)) + selected_keys.add(key) - if len(candidates) < n: + if len(selected) < n: + for entry, _ in selected: + heapq.heappush(self._heap, entry) + for entry in deferred: + heapq.heappush(self._heap, entry) return None - for key, _ in candidates: - del self.evictable_chunks[key] + + for entry in deferred: + heapq.heappush(self._heap, entry) + + candidates: list[tuple[OffloadKey, ChunkStatus]] = [] + for entry, chunk in selected: + key = entry[1] + self._evictable.remove(key) del self.chunks[key] + del self._ranks[key] + candidates.append((key, chunk)) + self._maybe_compact_heap() return candidates @override @@ -82,9 +164,11 @@ class LRUCachePolicy(CachePolicy): # chunks can become evictable when, # store completes - i.e. ref_cnt -1 -> 0 # not in evictable list # all loads complete - i.e ref_cnt 1 -> 0 # not in evictable list - self.evictable_chunks[key] = None + self._evictable.add(key) + self._push_evictable(key) + self._maybe_compact_heap() @override def mark_non_evictable(self, key: OffloadKey) -> None: - # key must have been in the evictable list. - del self.evictable_chunks[key] + # key must have been evictable before it was pinned. + self._evictable.remove(key) diff --git a/vllm/v1/kv_offload/tiering/manager.py b/vllm/v1/kv_offload/tiering/manager.py index 0940aaeae0..f19616828d 100644 --- a/vllm/v1/kv_offload/tiering/manager.py +++ b/vllm/v1/kv_offload/tiering/manager.py @@ -110,13 +110,22 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager): # read/write is for CPU<->secondary transfers, # load/store is for CPU<->GPU transfers. # These aliases avoid calling prepare_load inside a store path. - self.prepare_read = self.prepare_load self.complete_read = self.complete_load self.prepare_write = self.prepare_store self.complete_write = self.complete_store self._kv_memoryview = mmap_region.create_kv_memoryview() + def prepare_read( + self, keys: Collection[OffloadKey], req_context: ReqContext + ) -> LoadStoreSpec: + """Pin chunks for a CPU-to-secondary transfer. + + Cascade reads are implementation details of tiering, not additional + request accesses, so they must not alter request-scoped recency. + """ + return self._prepare_load(keys, req_context, record_access=False) + def get_kv_memoryview(self) -> memoryview: """Return the memoryview over the primary tier's KV cache buffer. @@ -781,11 +790,10 @@ class TieringOffloadingManager(OffloadingManager): req_id: str, exclude_tier_idx: int | None = None, ) -> None: - """Finalize secondary tiers once no more store cascades can be submitted. + """Finalize secondary tiers once no more cascades can be submitted. - Finalization means forwarding on_request_finished() to secondary tiers. - It is delayed until pending GPU->primary stores finish, since their - complete_store() callbacks may still submit primary->secondary stores. + Their finalization is delayed until pending GPU->primary stores + finish, since those callbacks may still submit secondary stores. """ state = self._req_state[req_id] if not state.is_finished: