[Bugfix][KV Offload] Track cache recency once per request (#51787)

Signed-off-by: mindungil <alswnsrlf12@naver.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
MINJUN GIL
2026-09-16 10:47:26 +03:00
committed by GitHub
co-authored by Or Ozeri
parent ab35354c21
commit f12fe10c5c
10 changed files with 916 additions and 169 deletions
@@ -258,6 +258,9 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
jobs = scheduler._build_partial_tail_store_jobs(output) jobs = scheduler._build_partial_tail_store_jobs(output)
assert len(jobs) == 1 assert len(jobs) == 1
offered_keys = scheduler.manager.prepare_store.call_args.args[0]
req_context = scheduler._req_status["req"].req_context
assert all(req_context.get_offload_key_position(key) == 28 for key in offered_keys)
[job_id] = jobs [job_id] = jobs
src_spec = jobs[job_id].src_spec src_spec = jobs[job_id].src_spec
assert isinstance(src_spec, GPULoadStoreSpec) assert isinstance(src_spec, GPULoadStoreSpec)
@@ -380,6 +383,13 @@ def test_normal_store_excludes_align_mode_mamba_sources():
req_status.group_states[0].block_ids[:] = [11] req_status.group_states[0].block_ids[:] = [11]
req_status.group_states[1].block_ids[:] = [99] req_status.group_states[1].block_ids[:] = [99]
req_status.update_offload_keys() req_status.update_offload_keys()
for group_config, group_state in zip(
scheduler.config.kv_group_configs, req_status.group_states
):
assert (
req_status.req_context.get_offload_key_position(group_state.offload_keys[0])
== group_config.tokens_per_chunk
)
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: ( scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys) generate_store_output(keys)
) )
@@ -921,7 +931,6 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
runner.run(decoded_tokens=[0] * (tokens_per_chunk + 1)) runner.run(decoded_tokens=[0] * (tokens_per_chunk + 1))
# 1 more block (+ token for kicking off offloading) # 1 more block (+ token for kicking off offloading)
# now check touch was called with all 6 blocks
runner.manager.prepare_store.side_effect = lambda keys, req_context: ( runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys) generate_store_output(keys)
) )
@@ -929,9 +938,6 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
decoded_tokens=[0] * (tokens_per_chunk + 1), decoded_tokens=[0] * (tokens_per_chunk + 1),
expected_stored=(15, 16, 17), expected_stored=(15, 16, 17),
) )
runner.manager.touch.assert_called()
block_hashes1 = list(runner.manager.touch.call_args.args[0])
assert len(block_hashes1) == 6
# terminate request # terminate request
runner.run(decoded_tokens=[EOS_TOKEN_ID]) runner.run(decoded_tokens=[EOS_TOKEN_ID])
@@ -939,13 +945,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
# create a new request differing only on the last token # create a new request differing only on the last token
runner.new_request(token_ids=[0] * (tokens_per_chunk * 6 - 1) + [1]) runner.new_request(token_ids=[0] * (tokens_per_chunk * 6 - 1) + [1])
runner.run(decoded_tokens=[0]) runner.run(decoded_tokens=[0])
runner.manager.touch.assert_called() runner.manager.touch.assert_not_called()
block_hashes2 = list(runner.manager.touch.call_args.args[0])
assert len(block_hashes2) == 6
# verify hashes are the same, except for the last block
assert block_hashes1[:5] == block_hashes2[:5]
assert block_hashes1[5] != block_hashes2[5]
# terminate request # terminate request
runner.run( runner.run(
@@ -1353,14 +1353,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
generate_store_output(keys) generate_store_output(keys)
) )
runner.run(decoded_tokens=[0]) runner.run(decoded_tokens=[0])
# _touch called from get_num_new_matched_tokens (2 groups) and runner.manager.touch.assert_not_called()
# _get_reqs_to_store (2 groups) → 4 touch calls total.
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 4
assert len(touch_calls[0].args[0]) == 3
assert len(touch_calls[1].args[0]) == 3
assert len(touch_calls[2].args[0]) == 3
assert len(touch_calls[3].args[0]) == 3
# store 3 more block # store 3 more block
runner.manager.prepare_store.side_effect = lambda keys, req_context: ( runner.manager.prepare_store.side_effect = lambda keys, req_context: (
@@ -1371,9 +1364,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
expected_stored=(0, 1, 2, 3, 4, 5), expected_stored=(0, 1, 2, 3, 4, 5),
) )
# touch called from _get_reqs_to_store * 3 blocks, once for each group runner.manager.touch.assert_not_called()
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 6
# EOS lands in the last slot of the 7th block (offset 6). No forward pass # EOS lands in the last slot of the 7th block (offset 6). No forward pass
# writes that slot, so the finishing step declines the block. # writes that slot, so the finishing step declines the block.
@@ -1392,13 +1383,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)), expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)),
) )
# 2 touch calls from get_num_new_matched_tokens (2 groups) runner.manager.touch.assert_not_called()
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 2
# full attention group touched all 3 blocks
assert len(touch_calls[0].args[0]) == 3
# sliding window group touched just the last 2 blocks
assert len(touch_calls[1].args[0]) == 2
# 3 blocks are hit on GPU [0, 1, 2] # 3 blocks are hit on GPU [0, 1, 2]
# 1 block loaded [3,] # 1 block loaded [3,]
@@ -1464,15 +1449,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
generate_store_output(keys) generate_store_output(keys)
) )
runner.run(decoded_tokens=[0]) runner.run(decoded_tokens=[0])
# _touch called from get_num_new_matched_tokens (2 groups) and runner.manager.touch.assert_not_called()
# _get_reqs_to_store (2 groups) → 4 touch calls total.
# Group 0 has 2 offload keys, group 1 has 1.
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 4
assert len(touch_calls[0].args[0]) == 2
assert len(touch_calls[1].args[0]) == 1
assert len(touch_calls[2].args[0]) == 2
assert len(touch_calls[3].args[0]) == 1
# Get to 31 tokens # Get to 31 tokens
# No further blocks offloaded # No further blocks offloaded
@@ -1482,11 +1459,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
# Group 0 blocks: [0, 1], ending_token_offset = 24 # Group 0 blocks: [0, 1], ending_token_offset = 24
# Group 1 blocks: [0, 1], ending_token_offset = 32 # Group 1 blocks: [0, 1], ending_token_offset = 32
runner.run(decoded_tokens=[0]) runner.run(decoded_tokens=[0])
# _get_reqs_to_store touch: only group 1 has a new block to store runner.manager.touch.assert_not_called()
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 2
assert len(touch_calls[0].args[0]) == 2
assert len(touch_calls[1].args[0]) == 2
# Get to 35 tokens # Get to 35 tokens
# No further blocks offloaded # No further blocks offloaded
@@ -1496,11 +1469,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
# Group 0 blocks: [0, 1, 2], ending_token_offset = 36 # Group 0 blocks: [0, 1, 2], ending_token_offset = 36
# Group 1 blocks: [0, 1], ending_token_offset = 32 # Group 1 blocks: [0, 1], ending_token_offset = 32
runner.run(decoded_tokens=[0]) runner.run(decoded_tokens=[0])
# _get_reqs_to_store touch: only group 0 has a new block to store runner.manager.touch.assert_not_called()
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 2
assert len(touch_calls[0].args[0]) == 3
assert len(touch_calls[1].args[0]) == 2
# Get to 47 tokens # Get to 47 tokens
# No further blocks offloaded # No further blocks offloaded
@@ -1510,11 +1479,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
# Group 0 blocks: [0, 1, 2, 3], ending_token_offset = 4 # Group 0 blocks: [0, 1, 2, 3], ending_token_offset = 4
# Group 1 blocks: [0, 1, 2], ending_token_offset = 48 # Group 1 blocks: [0, 1, 2], ending_token_offset = 48
runner.run(decoded_tokens=[0]) runner.run(decoded_tokens=[0])
# _get_reqs_to_store touch: both groups have a new block, each with 1 key runner.manager.touch.assert_not_called()
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 2
assert len(touch_calls[0].args[0]) == 4
assert len(touch_calls[1].args[0]) == 3
runner.run(decoded_tokens=[0], expected_stored=((0, 3), (1, 2))) runner.run(decoded_tokens=[0], expected_stored=((0, 3), (1, 2)))
+428 -8
View File
@@ -23,6 +23,7 @@ from vllm.v1.kv_offload.cpu.common import (
) )
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
def make_req_context( def make_req_context(
@@ -430,31 +431,34 @@ def test_cpu_manager():
assert cpu_manager.lookup(to_key(0), _EMPTY_REQ_CTX) is LookupResult.MISS assert cpu_manager.lookup(to_key(0), _EMPTY_REQ_CTX) is LookupResult.MISS
# prepare load [2, 3] # prepare load [2, 3]
prepare_load_output = cpu_manager.prepare_load(to_keys([2, 3]), _EMPTY_REQ_CTX) load_ctx = make_req_context("load-2-3")
prepare_load_output = cpu_manager.prepare_load(to_keys([2, 3]), load_ctx)
verify_load_output(prepare_load_output, [1, 2]) verify_load_output(prepare_load_output, [1, 2])
# prepare store with no space ([2, 3] is being loaded) # prepare store with no space ([2, 3] is being loaded)
assert cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX) is None assert cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX) is None
# complete load [2, 3]. Load changes the eviction list, making 2, 3 recent. # complete load [2, 3]. Load changes the eviction list, making 2, 3 recent.
cpu_manager.complete_load(to_keys([2, 3]), _EMPTY_REQ_CTX) cpu_manager.complete_load(to_keys([2, 3]), load_ctx)
cpu_manager.on_request_finished(load_ctx)
# prepare store [6, 7, 8] -> evicts [4, 5, 2] (oldest) # prepare store [6, 7, 8] -> evicts [4, 5, 3] (oldest). Within
# the accessed prefix [2, 3], the tail is less valuable than the head.
prepare_store_output = cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX) prepare_store_output = cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX)
verify_store_output( verify_store_output(
prepare_store_output, prepare_store_output,
ExpectedPrepareStoreOutput( ExpectedPrepareStoreOutput(
keys_to_store=[6, 7, 8], keys_to_store=[6, 7, 8],
store_chunk_ids=[1, 0, 3], store_chunk_ids=[2, 0, 3],
evicted_keys=[4, 5, 2], evicted_keys=[4, 5, 3],
), ),
) )
# complete store [6, 7, 8] # complete store [6, 7, 8]
cpu_manager.complete_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX) cpu_manager.complete_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX)
# touch [3, 6, 7] (move to end of LRU order) # touch [2, 6, 7] (move to end of LRU order)
cpu_manager.touch(to_keys([3, 6, 7]), _EMPTY_REQ_CTX) cpu_manager.touch(to_keys([2, 6, 7]), _EMPTY_REQ_CTX)
# prepare store [7, 9] -> evicts [8] (oldest following previous touch) # prepare store [7, 9] -> evicts [8] (oldest following previous touch)
prepare_store_output = cpu_manager.prepare_store(to_keys([9]), _EMPTY_REQ_CTX) prepare_store_output = cpu_manager.prepare_store(to_keys([9]), _EMPTY_REQ_CTX)
@@ -477,7 +481,7 @@ def test_cpu_manager():
verify_events( verify_events(
cpu_manager.take_events(), cpu_manager.take_events(),
expected_stores=({3, 4, 5}, {6, 7, 8}), expected_stores=({3, 4, 5}, {6, 7, 8}),
expected_evictions=({4, 5, 2}, {8}), expected_evictions=({4, 5, 3}, {8}),
) )
@@ -518,6 +522,65 @@ def test_prepare_load_preserves_key_order():
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
def test_lru_batch_eviction_failure_is_atomic():
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
policy = manager._policy
assert isinstance(policy, LRUCachePolicy)
keys = to_keys([1, 2, 3, 4])
assert manager.prepare_store(keys, _EMPTY_REQ_CTX) is not None
manager.complete_store(keys, _EMPTY_REQ_CTX)
protected = set(keys[1:])
assert policy.evict(2, protected) is None
assert all(manager.lookup(key, _EMPTY_REQ_CTX) is LookupResult.HIT for key in keys)
evicted = policy.evict(1, protected)
assert evicted is not None
assert [key for key, _ in evicted] == [keys[0]]
def test_lru_repeated_pin_cycles_compact_lazy_heap_entries():
manager = make_cpu_manager(num_chunks=1, cache_policy="lru")
policy = manager._policy
assert isinstance(policy, LRUCachePolicy)
key = to_key(1)
assert manager.prepare_store([key], _EMPTY_REQ_CTX) is not None
manager.complete_store([key], _EMPTY_REQ_CTX)
for _ in range(128):
manager.prepare_load([key], _EMPTY_REQ_CTX)
manager.complete_load([key], _EMPTY_REQ_CTX)
assert len(policy._heap) < 64
evicted = policy.evict(1, set())
assert evicted is not None
assert [evicted_key for evicted_key, _ in evicted] == [key]
def test_reset_discards_stale_request_access_classification():
manager = make_cpu_manager(num_chunks=1, cache_policy="arc")
policy = manager._policy
assert isinstance(policy, ARCCachePolicy)
key = to_key(1)
resumed_ctx = make_req_context("resumed")
assert manager.prepare_store([key], resumed_ctx) is not None
manager.complete_store([key], resumed_ctx)
manager.reset_cache()
replacement_ctx = make_req_context("replacement")
assert manager.prepare_store([key], replacement_ctx) is not None
manager.complete_store([key], replacement_ctx)
manager.on_request_finished(replacement_ctx)
assert key in policy.t1
manager.prepare_load([key], resumed_ctx)
manager.complete_load([key], resumed_ctx)
manager.on_request_finished(resumed_ctx)
assert key in policy.t2
class TestARCPolicy: class TestARCPolicy:
"""Unit tests for CPUOffloadingManager with ARC eviction policy.""" """Unit tests for CPUOffloadingManager with ARC eviction policy."""
@@ -1042,6 +1105,48 @@ def test_filter_reused_manager_oversized_offer_makes_progress():
assert stored_keys == set(keys) assert stored_keys == set(keys)
def test_store_threshold_does_not_hide_ready_reuse():
manager = make_cpu_manager(
num_chunks=2,
cache_policy="lru",
store_threshold=2,
max_tracker_size=2,
)
head, tail, noise_1, noise_2, replacement = to_keys(list(range(5)))
seed_ctx = make_req_context("seed")
output = manager.prepare_store([head, tail], seed_ctx)
assert output is not None
assert not output.keys_to_store
output = manager.prepare_store([head, tail], seed_ctx)
assert output is not None
assert output.keys_to_store == [head, tail]
manager.complete_store([head, tail], seed_ctx)
manager.on_request_finished(seed_ctx)
# Age both resident keys out of the admission tracker without storing the
# noise keys. Reusing the tail recreates its counter at one, below the
# threshold, but must still refresh its cache recency.
output = manager.prepare_store([noise_1, noise_2], make_req_context("noise"))
assert output is not None
assert not output.keys_to_store
reuse_ctx = make_req_context("reuse")
skipped_before_reuse = manager.stores_skipped_in_current_batch
output = manager.prepare_store([tail], reuse_ctx)
assert output is not None
assert not output.keys_to_store
assert manager.stores_skipped_in_current_batch == skipped_before_reuse
manager.on_request_finished(reuse_ctx)
replacement_ctx = make_req_context("replacement")
output = manager.prepare_store([replacement], replacement_ctx)
assert output is not None
assert not output.keys_to_store
output = manager.prepare_store([replacement], replacement_ctx)
assert output is not None
assert output.evicted_keys == [head]
def test_evictable_cache_chunk_count(): def test_evictable_cache_chunk_count():
""" """
Verifies _num_evictable_cache_chunks is maintained correctly through the Verifies _num_evictable_cache_chunks is maintained correctly through the
@@ -1156,3 +1261,318 @@ def test_touch_forwards_req_context_to_policy(monkeypatch):
assert len(received) == 1 assert len(received) == 1
assert received[0][0] == keys assert received[0][0] == keys
assert received[0][1] is ctx assert received[0][1] is ctx
@pytest.mark.parametrize("finish_before_completion", [False, True])
def test_request_finish_orders_lru_prefix_independent_of_store_completion(
finish_before_completion: bool,
):
"""A transfer's completion order must not make the prefix head LRU."""
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
prefix = to_keys([1, 2, 3, 4])
ctx = make_req_context("prefix")
output = manager.prepare_store(prefix, ctx)
assert output is not None
if finish_before_completion:
manager.on_request_finished(ctx)
manager.complete_store(set(prefix), ctx)
if not finish_before_completion:
manager.on_request_finished(ctx)
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
assert output is not None
assert output.evicted_keys == to_keys([4])
@pytest.mark.parametrize("finish_before_completion", [False, True])
def test_request_finish_orders_lru_prefix_independent_of_load_completion(
finish_before_completion: bool,
):
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
prefix = to_keys([1, 2, 3, 4])
seed_ctx = make_req_context("seed")
output = manager.prepare_store(prefix, seed_ctx)
assert output is not None
manager.complete_store(prefix, seed_ctx)
reuse_ctx = make_req_context("reuse")
manager.prepare_load(prefix, reuse_ctx)
if finish_before_completion:
manager.on_request_finished(reuse_ctx)
manager.complete_load(set(prefix), reuse_ctx)
if not finish_before_completion:
manager.on_request_finished(reuse_ctx)
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
assert output is not None
assert output.evicted_keys == to_keys([4])
def test_late_old_completion_does_not_override_newer_request_recency():
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
old_keys = to_keys([1, 2])
new_keys = to_keys([3, 4])
old_ctx = make_req_context("old")
assert manager.prepare_store(old_keys, old_ctx) is not None
manager.on_request_finished(old_ctx)
new_ctx = make_req_context("new")
assert manager.prepare_store(new_keys, new_ctx) is not None
manager.complete_store(new_keys, new_ctx)
manager.on_request_finished(new_ctx)
# Physical completion is late, but the logical access is still older.
manager.complete_store(old_keys, old_ctx)
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
assert output is not None
assert output.evicted_keys == [old_keys[-1]]
def test_arc_counts_reuse_once_and_keeps_insertions_in_t1():
manager = make_cpu_manager(num_chunks=4, cache_policy="arc")
policy = manager._policy
assert isinstance(policy, ARCCachePolicy)
prefix = to_keys([1, 2, 3, 4])
seed_ctx = make_req_context("seed")
assert manager.prepare_store(prefix, seed_ctx) is not None
manager.complete_store(prefix, seed_ctx)
# Repeated store offers from the same request must not become ARC hits.
assert manager.prepare_store(prefix, seed_ctx) is not None
# Nor should an internal read used to cascade the new chunks to a tier.
manager.prepare_load(prefix, seed_ctx)
manager.complete_load(prefix, seed_ctx)
manager.on_request_finished(seed_ctx)
assert list(policy.t1) == list(reversed(prefix))
assert not policy.t2
reuse_ctx = make_req_context("reuse")
manager.prepare_load(prefix, reuse_ctx)
manager.complete_load(prefix, reuse_ctx)
# A duplicate internal pin, as used by tiering cascades, is deduplicated.
manager.prepare_load(prefix, reuse_ctx)
manager.complete_load(prefix, reuse_ctx)
manager.on_request_finished(reuse_ctx)
assert not policy.t1
assert list(policy.t2) == list(reversed(prefix))
def test_arc_reuse_wins_if_same_request_later_reinserts_key():
manager = make_cpu_manager(num_chunks=1, cache_policy="arc")
policy = manager._policy
assert isinstance(policy, ARCCachePolicy)
key_1, key_2 = to_keys([1, 2])
seed_ctx = make_req_context("seed")
assert manager.prepare_store([key_1], seed_ctx) is not None
manager.complete_store([key_1], seed_ctx)
manager.on_request_finished(seed_ctx)
reuse_ctx = make_req_context("reuse-and-reinsert")
manager.prepare_load([key_1], reuse_ctx)
manager.complete_load([key_1], reuse_ctx)
other_ctx = make_req_context("evict-reused-key")
assert manager.prepare_store([key_2], other_ctx) is not None
manager.complete_store([key_2], other_ctx)
manager.on_request_finished(other_ctx)
assert manager.prepare_store([key_1], reuse_ctx) is not None
manager.complete_store([key_1], reuse_ctx)
manager.on_request_finished(reuse_ctx)
assert key_1 not in policy.t1
assert list(policy.t2) == [key_1]
def test_request_finish_forwards_grouped_deduplicated_accesses(monkeypatch):
manager = make_cpu_manager(num_chunks=5)
existing = make_offload_key(b"existing", 0)
group_0_new = make_offload_key(b"group-0-new", 0)
group_1_head = make_offload_key(b"group-1-head", 1)
group_1_tail = make_offload_key(b"group-1-tail", 1)
seed_ctx = make_req_context("seed")
assert manager.prepare_store([existing], seed_ctx) is not None
manager.complete_store([existing], seed_ctx)
ctx = make_req_context("grouped")
# Group 1 is observed first, but policy finalization follows group index.
offered = [group_1_head, existing, group_0_new]
assert manager.prepare_store(offered, ctx) is not None
# Later offers extend a group's prefix and may repeat earlier keys.
assert (
manager.prepare_store([group_0_new, group_1_head, group_1_tail], ctx)
is not None
)
received = []
def on_request_finished(key_groups, insertion_only_keys, reused_keys, req_context):
received.append((key_groups, insertion_only_keys, reused_keys, req_context))
monkeypatch.setattr(manager._policy, "on_request_finished", on_request_finished)
manager.on_request_finished(ctx)
manager.on_request_finished(ctx)
assert received == [
(
((existing, group_0_new), (group_1_head, group_1_tail)),
{group_0_new, group_1_head, group_1_tail},
{existing},
ctx,
)
]
def test_failed_store_records_access_once_and_uses_store_miss_hook(monkeypatch):
manager = make_cpu_manager(num_chunks=1)
resident, candidate = to_keys([1, 2])
seed_ctx = make_req_context("seed")
assert manager.prepare_store([resident], seed_ctx) is not None
manager.complete_store([resident], seed_ctx)
pin_ctx = make_req_context("pin")
manager.prepare_load([resident], pin_ctx)
record_calls = 0
original_record = manager._record_request_cache_access
def record_access(*args, **kwargs):
nonlocal record_calls
record_calls += 1
return original_record(*args, **kwargs)
store_miss_calls = []
def on_store_miss(keys, req_context):
store_miss_calls.append((list(keys), req_context))
finalized = []
def on_request_finished(key_groups, insertion_only_keys, reused_keys, req_context):
finalized.append((key_groups, insertion_only_keys, reused_keys, req_context))
monkeypatch.setattr(manager, "_record_request_cache_access", record_access)
monkeypatch.setattr(manager._policy, "on_store_miss", on_store_miss)
monkeypatch.setattr(manager._policy, "on_request_finished", on_request_finished)
ctx = make_req_context("failed-store")
assert manager.prepare_store([candidate, resident], ctx) is None
assert record_calls == 1
assert store_miss_calls == [([candidate], ctx)]
manager.on_request_finished(ctx)
assert finalized == [
(((candidate, resident),), set(), {resident}, ctx),
]
manager.complete_load([resident], pin_ctx)
def test_arc_ghost_hit_adapts_once_per_request_before_insertion():
manager = make_cpu_manager(num_chunks=2, cache_policy="arc")
policy = manager._policy
assert isinstance(policy, ARCCachePolicy)
keys = to_keys([1, 2, 3])
seed_ctx = make_req_context("seed")
assert manager.prepare_store(keys[:2], seed_ctx) is not None
manager.complete_store(keys[:2], seed_ctx)
manager.on_request_finished(seed_ctx)
evict_ctx = make_req_context("create-ghost")
output = manager.prepare_store([keys[2]], evict_ctx)
assert output is not None
[ghost_key] = output.evicted_keys
manager.complete_store([keys[2]], evict_ctx)
manager.on_request_finished(evict_ctx)
assert ghost_key in policy.b1
resident_keys = [key for key in keys if key != ghost_key]
pin_ctx = make_req_context("pin-residents")
manager.prepare_load(resident_keys, pin_ctx)
ghost_ctx = make_req_context("ghost-hit")
target_before = policy.target_t1_size
assert manager.prepare_store([ghost_key], ghost_ctx) is None
target_after_first_offer = policy.target_t1_size
assert target_after_first_offer > target_before
assert manager.prepare_store([ghost_key], ghost_ctx) is None
assert policy.target_t1_size == target_after_first_offer
manager.complete_load(resident_keys, pin_ctx)
output = manager.prepare_store([ghost_key], ghost_ctx)
assert output is not None
assert policy.target_t1_size == target_after_first_offer
manager.complete_store([ghost_key], ghost_ctx)
manager.on_request_finished(ghost_ctx)
assert ghost_key in policy.t1
assert ghost_key not in policy.t2
def test_arc_pending_chunk_is_not_a_frequency_hit():
manager = make_cpu_manager(num_chunks=2, cache_policy="arc")
policy = manager._policy
assert isinstance(policy, ARCCachePolicy)
key = to_key(1)
writer_ctx = make_req_context("writer")
assert manager.prepare_store([key], writer_ctx) is not None
observer_ctx = make_req_context("pending-observer")
output = manager.prepare_store([key], observer_ctx)
assert output is not None and not output.keys_to_store
manager.on_request_finished(observer_ctx)
chunk = policy.get(key)
assert chunk is not None and not chunk.is_ready
assert key in policy.t1
assert key not in policy.t2
def test_request_key_positions_override_store_observation_order():
manager = make_cpu_manager(num_chunks=2, cache_policy="lru")
head, tail = to_keys([1, 2])
ctx = make_req_context("out-of-order-store")
ctx.set_offload_key_position(head, 16)
ctx.set_offload_key_position(tail, 32)
# A backfill or partial-tail path may offer a later key first.
assert manager.prepare_store([tail], ctx) is not None
assert manager.prepare_store([head], ctx) is not None
manager.on_request_finished(ctx)
manager.complete_store({head, tail}, ctx)
output = manager.prepare_store([to_key(3)], make_req_context("evict-tail"))
assert output is not None
assert output.evicted_keys == [tail]
def test_request_key_positions_order_tails_across_kv_groups():
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
group_0_head = make_offload_key(b"group-0-head", 0)
group_0_tail = make_offload_key(b"group-0-tail", 0)
group_1_head = make_offload_key(b"group-1-head", 1)
group_1_tail = make_offload_key(b"group-1-tail", 1)
keys = [group_0_head, group_0_tail, group_1_head, group_1_tail]
ctx = make_req_context("hybrid-groups")
for key in (group_0_head, group_1_head):
ctx.set_offload_key_position(key, 16)
for key in (group_0_tail, group_1_tail):
ctx.set_offload_key_position(key, 32)
assert manager.prepare_store(keys, ctx) is not None
manager.complete_store(keys, ctx)
manager.on_request_finished(ctx)
output = manager.prepare_store(to_keys([10, 11]), make_req_context("evict"))
assert output is not None
assert set(output.evicted_keys) == {group_0_tail, group_1_tail}
@@ -40,6 +40,7 @@ from vllm.v1.kv_offload.base import (
TierMatcher, TierMatcher,
make_offload_key, make_offload_key,
) )
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
from vllm.v1.kv_offload.tiering.base import ( from vllm.v1.kv_offload.tiering.base import (
JobResult, JobResult,
SecondaryTierManager, SecondaryTierManager,
@@ -753,14 +754,64 @@ class TestTieringOffloadingManager:
# Touch chunks # Touch chunks
self.manager.touch(chunks, _CTX) self.manager.touch(chunks, _CTX)
# Verify touch was called on primary tier (check LRU order) # Verify touch was called on the primary tier (head is most recent).
primary_keys = list(self.primary_tier._policy.evictable_chunks.keys()) policy = self.primary_tier._policy
assert primary_keys[-3:] == list(reversed(chunks)) assert isinstance(policy, LRUCachePolicy)
ranks = policy._ranks
assert ranks[chunks[2]] < ranks[chunks[1]] < ranks[chunks[0]]
# Verify touch was propagated to all secondary tiers # Verify touch was propagated to all secondary tiers
self.secondary_tier1.touch.assert_called_once_with(chunks, _CTX) self.secondary_tier1.touch.assert_called_once_with(chunks, _CTX)
self.secondary_tier2.touch.assert_called_once_with(chunks, _CTX) self.secondary_tier2.touch.assert_called_once_with(chunks, _CTX)
def test_request_order_survives_late_cascade_completion(self, manager_setup):
"""Cascade completion must not replace request order with I/O order."""
chunks = to_keys(range(5))
self._start_request()
output = self.manager.prepare_store(chunks, _CTX)
assert output is not None
self.manager.complete_store(chunks, _CTX, success=True)
# Finalize while both secondary-tier cascades still pin primary chunks.
self.manager.on_request_finished(_CTX)
self._simulate_on_schedule_end()
self._simulate_on_schedule_end()
evict_ctx = ReqContext(req_id="evict")
self._start_request(evict_ctx)
output = self.manager.prepare_store(to_keys([5]), evict_ctx)
assert output is not None
assert output.evicted_keys == [chunks[-1]]
def test_late_old_store_submission_does_not_override_newer_request_order(
self, manager_setup
):
"""Cascade submission after finish is not a new primary access."""
old_blocks = to_keys(range(2))
old_ctx = ReqContext(req_id="old")
self._start_request(old_ctx)
assert self.manager.prepare_store(old_blocks, old_ctx) is not None
self.manager.on_request_finished(old_ctx)
new_blocks = to_keys(range(2, 5))
new_ctx = ReqContext(req_id="new")
self._start_request(new_ctx)
assert self.manager.prepare_store(new_blocks, new_ctx) is not None
self.manager.complete_store(new_blocks, new_ctx, success=True)
self.manager.on_request_finished(new_ctx)
# The old GPU->primary write lands after the newer request finished.
# Its cascade pins must not turn it into the more recent request.
self.manager.complete_store(old_blocks, old_ctx, success=True)
self._simulate_on_schedule_end()
self._simulate_on_schedule_end()
evict_ctx = ReqContext(req_id="evict")
self._start_request(evict_ctx)
output = self.manager.prepare_store(to_keys([5]), evict_ctx)
assert output is not None
assert output.evicted_keys == [old_blocks[-1]]
def test_failed_store_no_cascade(self, manager_setup): def test_failed_store_no_cascade(self, manager_setup):
"""Test that failed GPU→primary store doesn't cascade.""" """Test that failed GPU→primary store doesn't cascade."""
chunks = to_keys(range(3)) chunks = to_keys(range(3))
@@ -873,10 +924,10 @@ class TestTieringOffloadingManager:
job_metadata = self.secondary_tier1.submit_store.call_args.args[0] job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
assert job_metadata.req_context is ctx assert job_metadata.req_context is ctx
def test_on_request_finished_delays_secondary_until_store_submitted( def test_on_request_finished_delays_only_secondary_until_store_submitted(
self, manager_setup self, manager_setup
): ):
"""Manager hook is eager; secondary hooks wait for cascade submission.""" """Primary order commits immediately; secondary cleanup waits."""
chunks = to_keys(range(2)) chunks = to_keys(range(2))
ctx = ReqContext(req_id="req_delayed_secondary") ctx = ReqContext(req_id="req_delayed_secondary")
calls: list[tuple[str, str]] = [] calls: list[tuple[str, str]] = []
@@ -427,8 +427,11 @@ class RequestOffloadState:
None, None,
group_config.hashes_per_chunk, group_config.hashes_per_chunk,
): ):
group_state.offload_keys.append( key = make_offload_key(req_block_hash, group_config.group_idx)
make_offload_key(req_block_hash, group_config.group_idx) group_state.offload_keys.append(key)
self.req_context.set_offload_key_position(
key,
len(group_state.offload_keys) * group_config.tokens_per_chunk,
) )
def update_block_id_groups( def update_block_id_groups(
@@ -742,37 +745,6 @@ class OffloadingConnectorScheduler:
) )
return None if defer_lookup or pending_in_window else consecutive_hits return None if defer_lookup or pending_in_window else consecutive_hits
def _touch(self, req_status: RequestOffloadState):
for group_config, group_state in zip(
self.config.kv_group_configs, req_status.group_states
):
if group_config.sliding_window_size_in_chunks is None:
self.manager.touch(group_state.offload_keys, req_status.req_context)
else:
# Keep only chunks needed to hit the original request, plus
# decoded chunks.
chunks_to_skip = max(
0,
group_state.num_hit_chunks
- group_config.sliding_window_size_in_chunks,
)
self.manager.touch(
group_state.offload_keys[chunks_to_skip:],
req_status.req_context,
)
if req_status.partial_tail_boundary is not None:
self.manager.touch(
tuple(
self._make_boundary_key(
req_status.req,
group.group_idx,
req_status.partial_tail_boundary,
)
for group in self.config.kv_group_configs
),
req_status.req_context,
)
def _lookup_complete_chunks( def _lookup_complete_chunks(
self, self,
req_status: RequestOffloadState, req_status: RequestOffloadState,
@@ -977,10 +949,16 @@ class OffloadingConnectorScheduler:
return num_hit_tokens return num_hit_tokens
def _make_boundary_key( def _make_boundary_key(
self, request: Request, group_idx: int, boundary_tokens: int self,
request: Request,
group_idx: int,
boundary_tokens: int,
req_context: ReqContext,
) -> OffloadKey: ) -> OffloadKey:
hash_idx = boundary_tokens // self.config.tokens_per_hash - 1 hash_idx = boundary_tokens // self.config.tokens_per_hash - 1
return make_offload_key(request.block_hashes[hash_idx], group_idx) key = make_offload_key(request.block_hashes[hash_idx], group_idx)
req_context.set_offload_key_position(key, boundary_tokens)
return key
def _lookup( def _lookup(
self, self,
@@ -1015,7 +993,10 @@ class OffloadingConnectorScheduler:
boundary_keys = [] boundary_keys = []
for group_config in self.config.kv_group_configs: for group_config in self.config.kv_group_configs:
key = self._make_boundary_key( key = self._make_boundary_key(
req_status.req, group_config.group_idx, boundary req_status.req,
group_config.group_idx,
boundary,
req_status.req_context,
) )
boundary_keys.append(key) boundary_keys.append(key)
result = self.manager.lookup(key, req_status.req_context) result = self.manager.lookup(key, req_status.req_context)
@@ -1108,8 +1089,6 @@ class OffloadingConnectorScheduler:
self._maybe_observe_lookup_async_delay(req_status) self._maybe_observe_lookup_async_delay(req_status)
req_status.update_num_hit_chunks(num_computed_tokens + (num_hit_tokens or 0)) req_status.update_num_hit_chunks(num_computed_tokens + (num_hit_tokens or 0))
self._touch(req_status)
return num_hit_tokens, bool(num_hit_tokens) return num_hit_tokens, bool(num_hit_tokens)
def update_state_after_alloc( def update_state_after_alloc(
@@ -1183,7 +1162,10 @@ class OffloadingConnectorScheduler:
if partial_tail_boundary is not None: if partial_tail_boundary is not None:
keys_to_load.append( keys_to_load.append(
self._make_boundary_key( self._make_boundary_key(
request, group_config.group_idx, partial_tail_boundary request,
group_config.group_idx,
partial_tail_boundary,
req_status.req_context,
) )
) )
@@ -1307,7 +1289,9 @@ class OffloadingConnectorScheduler:
): ):
continue continue
key = self._make_boundary_key(req, group_idx, boundary) key = self._make_boundary_key(
req, group_idx, boundary, req_status.req_context
)
store_output = self.manager.prepare_store([key], req_status.req_context) store_output = self.manager.prepare_store([key], req_status.req_context)
if store_output is None: if store_output is None:
self._connector_stats.increase_counter( self._connector_stats.increase_counter(
@@ -1402,7 +1386,9 @@ class OffloadingConnectorScheduler:
): ):
continue continue
keys = [ keys = [
self._make_boundary_key(req, group.group_idx, boundary) self._make_boundary_key(
req, group.group_idx, boundary, req_status.req_context
)
for group in self.config.kv_group_configs for group in self.config.kv_group_configs
] ]
block_ids = [ block_ids = [
@@ -1689,8 +1675,6 @@ class OffloadingConnectorScheduler:
req_status.advance_stored_idx(num_offloadable_tokens) req_status.advance_stored_idx(num_offloadable_tokens)
continue continue
self._touch(req_status)
keys_to_store = set(store_output.keys_to_store) keys_to_store = set(store_output.keys_to_store)
group_sizes: list[int] = [] group_sizes: list[int] = []
+12
View File
@@ -96,6 +96,12 @@ class ReqContext:
# kv_transfer_params once (in on_new_request) and read the result back # kv_transfer_params once (in on_new_request) and read the result back
# on later calls for the same request. # on later calls for the same request.
_state: dict[type, Any] = field(default_factory=dict, repr=False, init=False) _state: dict[type, Any] = field(default_factory=dict, repr=False, init=False)
# End-token position for each key in this request. The scheduler records
# these positions so managers can recover prefix order even when store
# calls arrive out of order (for example, SWA backfills).
_offload_key_positions: dict[OffloadKey, int] = field(
default_factory=dict, repr=False, init=False
)
def set_state(self, val: Any) -> None: def set_state(self, val: Any) -> None:
self._state[type(val)] = val self._state[type(val)] = val
@@ -103,6 +109,12 @@ class ReqContext:
def get_state(self, cls: type[_T]) -> _T | None: def get_state(self, cls: type[_T]) -> _T | None:
return self._state.get(cls) return self._state.get(cls)
def set_offload_key_position(self, key: OffloadKey, end_token: int) -> None:
self._offload_key_positions[key] = end_token
def get_offload_key_position(self, key: OffloadKey) -> int | None:
return self._offload_key_positions.get(key)
class LookupResult(Enum): class LookupResult(Enum):
"""Result of OffloadingManager.lookup().""" """Result of OffloadingManager.lookup()."""
+141 -5
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import OrderedDict from collections import OrderedDict
from collections.abc import Collection, Iterable from collections.abc import Collection, Iterable
from dataclasses import dataclass, field
from typing_extensions import override from typing_extensions import override
@@ -18,6 +19,7 @@ from vllm.v1.kv_offload.base import (
PrepareStoreOutput, PrepareStoreOutput,
ReqContext, ReqContext,
RequestOffloadingContext, RequestOffloadingContext,
get_offload_group_idx,
) )
from vllm.v1.kv_offload.cpu.common import ( from vllm.v1.kv_offload.cpu.common import (
CPULoadStoreSpec, CPULoadStoreSpec,
@@ -27,6 +29,20 @@ from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
@dataclass(slots=True)
class _RequestCacheAccess:
"""Cache keys observed by one request, grouped in prefix order."""
owner: object
cache_generation: int
key_groups: dict[int, list[OffloadKey]] = field(default_factory=dict)
seen_keys: set[OffloadKey] = field(default_factory=set)
inserted_keys: set[OffloadKey] = field(default_factory=set)
reused_keys: set[OffloadKey] = field(default_factory=set)
store_miss_keys: set[OffloadKey] = field(default_factory=set)
finished: bool = False
class CPUOffloadingManager(OffloadingManager): class CPUOffloadingManager(OffloadingManager):
""" """
An OffloadingManager with a pluggable CachePolicy, resolved by name via An OffloadingManager with a pluggable CachePolicy, resolved by name via
@@ -66,6 +82,7 @@ class CPUOffloadingManager(OffloadingManager):
self.max_tracker_size: int = max_tracker_size self.max_tracker_size: int = max_tracker_size
self.stores_skipped_in_current_batch: int = 0 self.stores_skipped_in_current_batch: int = 0
self.allocation_sizes_in_current_batch: list[int] = [] self.allocation_sizes_in_current_batch: list[int] = []
self._cache_generation = 0
# Number of chunk references. It is ordered so can evict the LRU entry in O(1). # Number of chunk references. It is ordered so can evict the LRU entry in O(1).
self.counts: OrderedDict[OffloadKey, int] | None = ( self.counts: OrderedDict[OffloadKey, int] | None = (
@@ -124,10 +141,48 @@ class CPUOffloadingManager(OffloadingManager):
num_unprotected -= 1 num_unprotected -= 1
self.counts[key] = 1 self.counts[key] = 1
def _get_request_cache_access(self, req_context: ReqContext) -> _RequestCacheAccess:
state = req_context.get_state(_RequestCacheAccess)
if (
state is None
or state.owner is not self
or state.cache_generation != self._cache_generation
):
state = _RequestCacheAccess(
owner=self, cache_generation=self._cache_generation
)
req_context.set_state(state)
return state
def _record_request_cache_access(
self,
keys: Iterable[OffloadKey],
req_context: ReqContext,
inserted_keys: Iterable[OffloadKey] = (),
reused_keys: Iterable[OffloadKey] = (),
) -> None:
state = self._get_request_cache_access(req_context)
for key in keys:
if key in state.seen_keys:
continue
assert not state.finished, (
"New cache keys observed after request finalization"
)
group_idx = get_offload_group_idx(key)
state.key_groups.setdefault(group_idx, []).append(key)
state.seen_keys.add(key)
state.inserted_keys.update(inserted_keys)
# Re-reading a chunk inserted by this request is an internal transfer
# (for example, a tiering cascade), not a second cache access.
state.reused_keys.update(
key for key in reused_keys if key not in state.inserted_keys
)
# --- OffloadingManager interface --- # --- OffloadingManager interface ---
@override @override
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
self._get_request_cache_access(req_context)
return RequestOffloadingContext() return RequestOffloadingContext()
@override @override
@@ -144,6 +199,15 @@ class CPUOffloadingManager(OffloadingManager):
self, self,
keys: Collection[OffloadKey], keys: Collection[OffloadKey],
req_context: ReqContext, req_context: ReqContext,
) -> LoadStoreSpec:
return self._prepare_load(keys, req_context, record_access=True)
def _prepare_load(
self,
keys: Collection[OffloadKey],
req_context: ReqContext,
*,
record_access: bool,
) -> LoadStoreSpec: ) -> LoadStoreSpec:
chunks = [] chunks = []
for key in keys: for key in keys:
@@ -156,6 +220,8 @@ class CPUOffloadingManager(OffloadingManager):
assert self._num_evictable_cache_chunks >= 0 assert self._num_evictable_cache_chunks >= 0
chunk.ref_cnt += 1 chunk.ref_cnt += 1
chunks.append(chunk) chunks.append(chunk)
if record_access:
self._record_request_cache_access(keys, req_context, reused_keys=keys)
return self._get_load_store_spec(keys, chunks) return self._get_load_store_spec(keys, chunks)
@override @override
@@ -181,13 +247,52 @@ class CPUOffloadingManager(OffloadingManager):
keys: Collection[OffloadKey], keys: Collection[OffloadKey],
req_context: ReqContext, req_context: ReqContext,
) -> PrepareStoreOutput | None: ) -> PrepareStoreOutput | None:
keys = list(keys)
if self.counts is not None: if self.counts is not None:
num_keys = len(keys)
self._record_accesses(keys) self._record_accesses(keys)
keys = [k for k in keys if self.counts.get(k, 0) >= self.store_threshold] # Partition the original offer before admission filtering. Ready
self.stores_skipped_in_current_batch += num_keys - len(keys) # resident chunks are request reuses even if their tracker entry was
# filter out chunks that are already stored # aged out; the threshold applies only to new store candidates.
keys_to_store = [k for k in keys if self._policy.get(k) is None] keys_to_store: list[OffloadKey] = []
ready_existing_keys: list[OffloadKey] = []
for key in keys:
chunk = self._policy.get(key)
if chunk is None:
keys_to_store.append(key)
else:
if chunk.is_ready:
ready_existing_keys.append(key)
if self.counts is not None:
num_store_candidates = len(keys_to_store)
keys_to_store = [
key
for key in keys_to_store
if self.counts.get(key, 0) >= self.store_threshold
]
self.stores_skipped_in_current_batch += num_store_candidates - len(
keys_to_store
)
state = self._get_request_cache_access(req_context)
new_store_misses = [
key for key in keys_to_store if key not in state.store_miss_keys
]
if new_store_misses:
# ARC learns from B1/B2 before insert() removes the ghost entry.
# Deduplication makes this one policy observation per request.
self._policy.on_store_miss(new_store_misses, req_context)
state.store_miss_keys.update(new_store_misses)
recorded_keys = set(keys_to_store)
recorded_keys.update(ready_existing_keys)
# Record once before any allocation-related early return. Store
# candidates are classified as insertions only after allocation succeeds.
self._record_request_cache_access(
(key for key in keys if key in recorded_keys),
req_context,
reused_keys=ready_existing_keys,
)
if not keys_to_store: if not keys_to_store:
return PrepareStoreOutput( return PrepareStoreOutput(
@@ -239,6 +344,7 @@ class CPUOffloadingManager(OffloadingManager):
for key, chunk in zip(keys_to_store, chunks): for key, chunk in zip(keys_to_store, chunks):
self._policy.insert(key, chunk) self._policy.insert(key, chunk)
self._num_write_pending_chunks += len(keys_to_store) self._num_write_pending_chunks += len(keys_to_store)
state.inserted_keys.update(keys_to_store)
# build store specs for allocated chunks # build store specs for allocated chunks
store_spec = self._get_load_store_spec(keys_to_store, chunks) store_spec = self._get_load_store_spec(keys_to_store, chunks)
@@ -284,6 +390,35 @@ class CPUOffloadingManager(OffloadingManager):
) )
) )
@override
def on_request_finished(self, req_context: ReqContext) -> None:
state = req_context.get_state(_RequestCacheAccess)
if (
state is None
or state.owner is not self
or state.cache_generation != self._cache_generation
or state.finished
):
return
state.finished = True
key_groups = []
for group_idx in sorted(state.key_groups):
keys = state.key_groups[group_idx]
positions = {
key: position
for key in keys
if (position := req_context.get_offload_key_position(key)) is not None
}
if len(positions) == len(keys):
keys = sorted(keys, key=positions.__getitem__)
key_groups.append(tuple(keys))
self._policy.on_request_finished(
tuple(key_groups),
state.inserted_keys - state.reused_keys,
state.reused_keys,
req_context,
)
@override @override
def reset_cache(self) -> None: def reset_cache(self) -> None:
# Clear ALL chunks unconditionally. The scheduler's _stale_job_threshold # Clear ALL chunks unconditionally. The scheduler's _stale_job_threshold
@@ -292,6 +427,7 @@ class CPUOffloadingManager(OffloadingManager):
# flushes in-flight load job IDs to the workers before any new stores # flushes in-flight load job IDs to the workers before any new stores
# can begin, preventing a cross-direction data race on reused offload chunk IDs. # can begin, preventing a cross-direction data race on reused offload chunk IDs.
self._policy.clear() self._policy.clear()
self._cache_generation += 1
self._num_evictable_cache_chunks = 0 self._num_evictable_cache_chunks = 0
self._num_write_pending_chunks = 0 self._num_write_pending_chunks = 0
+58 -21
View File
@@ -1,12 +1,16 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import OrderedDict from collections import OrderedDict
from collections.abc import Iterable, Iterator from collections.abc import Iterable, Iterator, Sequence
from typing_extensions import override from typing_extensions import override
from vllm.v1.kv_offload.base import OffloadKey, ReqContext from vllm.v1.kv_offload.base import OffloadKey, ReqContext
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus from vllm.v1.kv_offload.cpu.policies.base import (
CachePolicy,
ChunkStatus,
order_request_keys,
)
class ARCCachePolicy(CachePolicy): class ARCCachePolicy(CachePolicy):
@@ -24,12 +28,11 @@ class ARCCachePolicy(CachePolicy):
Searches T1 and T2 for chunk hashes and counts consecutive hits Searches T1 and T2 for chunk hashes and counts consecutive hits
until a miss or non-ready chunk is encountered. until a miss or non-ready chunk is encountered.
2. Cache touch (touch) - Adaptive Learning: 2. Request access - Adaptive Learning:
For each key (in reverse order): - Ready chunks reused by a request move from T1 to T2 once, or
- If in T1: Move to T2 (promotion from recent to frequent). move to the MRU end of T2.
- If in T2: Move to MRU position (end of queue). - B1/B2 misses adjust target_t1_size once per request before
- If in B1 ghost list: Increase target_t1_size. insertion removes their ghost entries.
- If in B2 ghost list: Decrease target_t1_size.
3. Chunk eviction (evict) - Adaptive Replacement: 3. Chunk eviction (evict) - Adaptive Replacement:
Determines eviction source based on adaptive target: Determines eviction source based on adaptive target:
@@ -39,7 +42,7 @@ class ARCCachePolicy(CachePolicy):
4. Chunk insertion (insert): 4. Chunk insertion (insert):
New chunks are always inserted into T1 and removed from B1/B2 if New chunks are always inserted into T1 and removed from B1/B2 if
present. Chunks may later be promoted to T2 during touch operations. present. A later request reuse may promote them to T2.
Adaptive Behavior: Adaptive Behavior:
The algorithm self-tunes the recency vs. frequency trade-off: The algorithm self-tunes the recency vs. frequency trade-off:
@@ -71,6 +74,26 @@ class ARCCachePolicy(CachePolicy):
if self.t1.pop(key, None) is None: if self.t1.pop(key, None) is None:
self.t2.pop(key, None) self.t2.pop(key, None)
def _adapt_to_ghost_hit(self, key: OffloadKey) -> bool:
if key in self.b1:
delta = max(1, len(self.b2) / len(self.b1))
self.target_t1_size = min(self.target_t1_size + delta, self.cache_capacity)
self.b1.move_to_end(key)
return True
if key in self.b2:
delta = max(1, len(self.b1) / len(self.b2))
self.target_t1_size = max(self.target_t1_size - delta, 0)
self.b2.move_to_end(key)
return True
return False
@override
def on_store_miss(
self, keys: Iterable[OffloadKey], req_context: ReqContext
) -> None:
for key in reversed(list(keys)):
self._adapt_to_ghost_hit(key)
@override @override
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None: def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
for key in reversed(list(keys)): for key in reversed(list(keys)):
@@ -86,19 +109,33 @@ class ARCCachePolicy(CachePolicy):
elif key in self.t2: elif key in self.t2:
self.t2.move_to_end(key) self.t2.move_to_end(key)
elif key in self.b1: else:
delta = max(1, len(self.b2) / len(self.b1)) self._adapt_to_ghost_hit(key)
self.target_t1_size = min(
self.target_t1_size + delta, self.cache_capacity
)
# move to MRU position (end) to keep it fresh in the ghost list
self.b1.move_to_end(key)
elif key in self.b2: @override
delta = max(1, len(self.b1) / len(self.b2)) def on_request_finished(
self.target_t1_size = max(self.target_t1_size - delta, 0) self,
# move to MRU position (end) to keep it fresh in the ghost list key_groups: Sequence[Sequence[OffloadKey]],
self.b2.move_to_end(key) insertion_only_keys: set[OffloadKey],
reused_keys: set[OffloadKey],
req_context: ReqContext,
) -> None:
for key in reversed(order_request_keys(key_groups, req_context)):
if key in insertion_only_keys:
# A store is not a frequency hit. Preserve T1 membership
# while still restoring tail-to-head recency.
if key in self.t1:
self.t1.move_to_end(key)
elif key in self.t2:
self.t2.move_to_end(key)
continue
# Ready chunks reused by this request count as one access.
if key in reused_keys and key in self.t1:
chunk = self.t1.pop(key)
self.t2[key] = chunk
elif key in reused_keys and key in self.t2:
self.t2.move_to_end(key)
@override @override
def clear(self) -> None: def clear(self) -> None:
+51 -1
View File
@@ -2,11 +2,26 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import ctypes import ctypes
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections.abc import Iterable from collections.abc import Iterable, Sequence
from vllm.v1.kv_offload.base import OffloadKey, ReqContext from vllm.v1.kv_offload.base import OffloadKey, ReqContext
def order_request_keys(
key_groups: Sequence[Sequence[OffloadKey]], req_context: ReqContext
) -> list[OffloadKey]:
"""Return one head-to-tail order across all KV cache groups."""
keys = [key for group in key_groups for key in group]
positions = {
key: position
for key in keys
if (position := req_context.get_offload_key_position(key)) is not None
}
if len(positions) == len(keys):
keys.sort(key=positions.__getitem__)
return keys
class ChunkStatus(ctypes.Structure): class ChunkStatus(ctypes.Structure):
""" """
Offloading status for a single chunk of KV data. Offloading status for a single chunk of KV data.
@@ -66,6 +81,41 @@ class CachePolicy(ABC):
req_context: Per-request context for the request touching these chunks. req_context: Per-request context for the request touching these chunks.
""" """
def on_store_miss(
self, keys: Iterable[OffloadKey], req_context: ReqContext
) -> None:
"""Observe store misses before their cache entries are inserted.
The default delegates to ``touch`` for compatibility with external
policies. Policies may override this hook when a miss has distinct
semantics, such as ARC adapting to a ghost-list hit.
"""
self.touch(keys, req_context)
def on_request_finished(
self,
key_groups: Sequence[Sequence[OffloadKey]],
insertion_only_keys: set[OffloadKey],
reused_keys: set[OffloadKey],
req_context: ReqContext,
) -> None:
"""Apply one request-scoped cache access in prefix order.
``key_groups`` contains keys observed for each KV cache group in
head-to-tail order. ``insertion_only_keys`` and ``reused_keys``
distinguish chunks only created by this request from ready chunks it
actually reused, which matters for policies such as ARC where reuse
changes frequency but insertion and pending observations do not.
The default forwards one touch per group, preserving compatibility
for experimental out-of-tree policies while moving those touches to
request finalization. Policies that distinguish insertion from reuse
can override this hook and inspect the access classifications.
"""
del insertion_only_keys, reused_keys
for keys in key_groups:
self.touch(keys, req_context)
@abstractmethod @abstractmethod
def evict( def evict(
self, n: int, protected: set[OffloadKey] self, n: int, protected: set[OffloadKey]
+112 -28
View File
@@ -1,28 +1,74 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import OrderedDict import heapq
from collections.abc import Iterable from collections.abc import Iterable, Sequence
from typing_extensions import override from typing_extensions import override
from vllm.v1.kv_offload.base import OffloadKey, ReqContext from vllm.v1.kv_offload.base import OffloadKey, ReqContext
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus from vllm.v1.kv_offload.cpu.policies.base import (
CachePolicy,
ChunkStatus,
order_request_keys,
)
class LRUCachePolicy(CachePolicy): class LRUCachePolicy(CachePolicy):
""" """
LRU Caching policy that keeps a dedicated evictable list for fast eviction. LRU caching policy with logical recency independent of transfer pinning.
Evictable chunks live in a lazy-invalidating min-heap. A chunk's recency
can therefore be updated while it is pinned; when it later becomes
evictable, it enters the heap with the order assigned by the request
rather than its transfer-completion time.
A use is indicated by, A use is indicated by,
- First time the key is added (store). - First time the key is added (store).
- Load job completion - A request-scoped access.
- touch
""" """
def __init__(self, cache_capacity: int): def __init__(self, cache_capacity: int):
super().__init__(cache_capacity) super().__init__(cache_capacity)
# Chunks with ref_cnt 0 (not participating in any loads/stores) ordered in LRU
self.evictable_chunks: OrderedDict[OffloadKey, None] = OrderedDict()
self.chunks: dict[OffloadKey, ChunkStatus] = {} self.chunks: dict[OffloadKey, ChunkStatus] = {}
self._ranks: dict[OffloadKey, int] = {}
self._evictable: set[OffloadKey] = set()
self._heap: list[tuple[int, OffloadKey]] = []
self._next_rank = 0
def _assign_new_rank(self, key: OffloadKey) -> bool:
if key not in self.chunks:
return False
self._next_rank += 1
self._ranks[key] = self._next_rank
return key in self._evictable
def _push_evictable(self, key: OffloadKey) -> None:
heapq.heappush(
self._heap,
(self._ranks[key], key),
)
def _is_current(self, entry: tuple[int, OffloadKey]) -> bool:
rank, key = entry
return key in self._evictable and self._ranks.get(key) == rank
def _maybe_compact_heap(self) -> None:
if len(self._heap) < max(64, 2 * len(self._evictable)):
return
self._heap = [(self._ranks[key], key) for key in self._evictable]
heapq.heapify(self._heap)
def _update_recency(self, keys: Iterable[OffloadKey]) -> None:
updated_evictable = [key for key in keys if self._assign_new_rank(key)]
# Rebuilding is linear and substantially cheaper than k heap pushes
# for a long prefix. Small updates retain the incremental path.
if len(updated_evictable) >= max(64, len(self._evictable) // 4):
self._heap = [(self._ranks[key], key) for key in self._evictable]
heapq.heapify(self._heap)
else:
for key in updated_evictable:
self._push_evictable(key)
self._maybe_compact_heap()
@override @override
def get(self, key: OffloadKey) -> ChunkStatus | None: def get(self, key: OffloadKey) -> ChunkStatus | None:
@@ -31,26 +77,40 @@ class LRUCachePolicy(CachePolicy):
@override @override
def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None: def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
self.chunks[key] = chunk self.chunks[key] = chunk
self._next_rank += 1
self._ranks[key] = self._next_rank
if chunk.ref_cnt == 0: if chunk.ref_cnt == 0:
self.evictable_chunks[key] = None self._evictable.add(key)
self._push_evictable(key)
@override @override
def remove(self, key: OffloadKey) -> None: def remove(self, key: OffloadKey) -> None:
del self.chunks[key] del self.chunks[key]
self.evictable_chunks.pop(key, None) self._ranks.pop(key, None)
self._evictable.discard(key)
@override @override
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None: def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
for key in reversed(list(keys)): self._update_recency(reversed(list(keys)))
if key in self.evictable_chunks:
self.evictable_chunks.move_to_end(key) @override
# active chunks are untouched as they are non-evictable now. They def on_request_finished(
# will eventually reach the end of evictable_chunks when they finish. self,
key_groups: Sequence[Sequence[OffloadKey]],
insertion_only_keys: set[OffloadKey],
reused_keys: set[OffloadKey],
req_context: ReqContext,
) -> None:
del insertion_only_keys, reused_keys
self._update_recency(reversed(order_request_keys(key_groups, req_context)))
@override @override
def clear(self) -> None: def clear(self) -> None:
self.evictable_chunks.clear()
self.chunks.clear() self.chunks.clear()
self._ranks.clear()
self._evictable.clear()
self._heap.clear()
self._next_rank = 0
@override @override
def evict( def evict(
@@ -59,22 +119,44 @@ class LRUCachePolicy(CachePolicy):
if n == 0: if n == 0:
return [] return []
candidates: list[tuple[OffloadKey, ChunkStatus]] = [] selected: list[tuple[tuple[int, OffloadKey], ChunkStatus]] = []
for key, _ in self.evictable_chunks.items(): selected_keys: set[OffloadKey] = set()
deferred: list[tuple[int, OffloadKey]] = []
while self._heap and len(selected) < n:
entry = heapq.heappop(self._heap)
if not self._is_current(entry):
continue
key = entry[1]
# Re-pinning without a recency change can leave an equivalent
# lazy entry behind. Select each cache key at most once.
if key in selected_keys:
continue
if key in protected: if key in protected:
deferred.append(entry)
continue continue
chunk = self.chunks[key] chunk = self.chunks[key]
assert chunk.ref_cnt == 0 assert chunk.ref_cnt == 0
candidates.append((key, chunk)) selected.append((entry, chunk))
if len(candidates) == n: selected_keys.add(key)
break
if len(candidates) < n: if len(selected) < n:
for entry, _ in selected:
heapq.heappush(self._heap, entry)
for entry in deferred:
heapq.heappush(self._heap, entry)
return None return None
for key, _ in candidates:
del self.evictable_chunks[key] for entry in deferred:
heapq.heappush(self._heap, entry)
candidates: list[tuple[OffloadKey, ChunkStatus]] = []
for entry, chunk in selected:
key = entry[1]
self._evictable.remove(key)
del self.chunks[key] del self.chunks[key]
del self._ranks[key]
candidates.append((key, chunk))
self._maybe_compact_heap()
return candidates return candidates
@override @override
@@ -82,9 +164,11 @@ class LRUCachePolicy(CachePolicy):
# chunks can become evictable when, # chunks can become evictable when,
# store completes - i.e. ref_cnt -1 -> 0 # not in evictable list # store completes - i.e. ref_cnt -1 -> 0 # not in evictable list
# all loads complete - i.e ref_cnt 1 -> 0 # not in evictable list # all loads complete - i.e ref_cnt 1 -> 0 # not in evictable list
self.evictable_chunks[key] = None self._evictable.add(key)
self._push_evictable(key)
self._maybe_compact_heap()
@override @override
def mark_non_evictable(self, key: OffloadKey) -> None: def mark_non_evictable(self, key: OffloadKey) -> None:
# key must have been in the evictable list. # key must have been evictable before it was pinned.
del self.evictable_chunks[key] self._evictable.remove(key)
+13 -5
View File
@@ -110,13 +110,22 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
# read/write is for CPU<->secondary transfers, # read/write is for CPU<->secondary transfers,
# load/store is for CPU<->GPU transfers. # load/store is for CPU<->GPU transfers.
# These aliases avoid calling prepare_load inside a store path. # These aliases avoid calling prepare_load inside a store path.
self.prepare_read = self.prepare_load
self.complete_read = self.complete_load self.complete_read = self.complete_load
self.prepare_write = self.prepare_store self.prepare_write = self.prepare_store
self.complete_write = self.complete_store self.complete_write = self.complete_store
self._kv_memoryview = mmap_region.create_kv_memoryview() self._kv_memoryview = mmap_region.create_kv_memoryview()
def prepare_read(
self, keys: Collection[OffloadKey], req_context: ReqContext
) -> LoadStoreSpec:
"""Pin chunks for a CPU-to-secondary transfer.
Cascade reads are implementation details of tiering, not additional
request accesses, so they must not alter request-scoped recency.
"""
return self._prepare_load(keys, req_context, record_access=False)
def get_kv_memoryview(self) -> memoryview: def get_kv_memoryview(self) -> memoryview:
"""Return the memoryview over the primary tier's KV cache buffer. """Return the memoryview over the primary tier's KV cache buffer.
@@ -781,11 +790,10 @@ class TieringOffloadingManager(OffloadingManager):
req_id: str, req_id: str,
exclude_tier_idx: int | None = None, exclude_tier_idx: int | None = None,
) -> None: ) -> None:
"""Finalize secondary tiers once no more store cascades can be submitted. """Finalize secondary tiers once no more cascades can be submitted.
Finalization means forwarding on_request_finished() to secondary tiers. Their finalization is delayed until pending GPU->primary stores
It is delayed until pending GPU->primary stores finish, since their finish, since those callbacks may still submit secondary stores.
complete_store() callbacks may still submit primary->secondary stores.
""" """
state = self._req_state[req_id] state = self._req_state[req_id]
if not state.is_finished: if not state.is_finished: