[Bugfix][KV Offload] Track cache recency once per request (#51787)
Signed-off-by: mindungil <alswnsrlf12@naver.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -258,6 +258,9 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
|
|||||||
jobs = scheduler._build_partial_tail_store_jobs(output)
|
jobs = scheduler._build_partial_tail_store_jobs(output)
|
||||||
|
|
||||||
assert len(jobs) == 1
|
assert len(jobs) == 1
|
||||||
|
offered_keys = scheduler.manager.prepare_store.call_args.args[0]
|
||||||
|
req_context = scheduler._req_status["req"].req_context
|
||||||
|
assert all(req_context.get_offload_key_position(key) == 28 for key in offered_keys)
|
||||||
[job_id] = jobs
|
[job_id] = jobs
|
||||||
src_spec = jobs[job_id].src_spec
|
src_spec = jobs[job_id].src_spec
|
||||||
assert isinstance(src_spec, GPULoadStoreSpec)
|
assert isinstance(src_spec, GPULoadStoreSpec)
|
||||||
@@ -380,6 +383,13 @@ def test_normal_store_excludes_align_mode_mamba_sources():
|
|||||||
req_status.group_states[0].block_ids[:] = [11]
|
req_status.group_states[0].block_ids[:] = [11]
|
||||||
req_status.group_states[1].block_ids[:] = [99]
|
req_status.group_states[1].block_ids[:] = [99]
|
||||||
req_status.update_offload_keys()
|
req_status.update_offload_keys()
|
||||||
|
for group_config, group_state in zip(
|
||||||
|
scheduler.config.kv_group_configs, req_status.group_states
|
||||||
|
):
|
||||||
|
assert (
|
||||||
|
req_status.req_context.get_offload_key_position(group_state.offload_keys[0])
|
||||||
|
== group_config.tokens_per_chunk
|
||||||
|
)
|
||||||
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||||
generate_store_output(keys)
|
generate_store_output(keys)
|
||||||
)
|
)
|
||||||
@@ -921,7 +931,6 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
|||||||
runner.run(decoded_tokens=[0] * (tokens_per_chunk + 1))
|
runner.run(decoded_tokens=[0] * (tokens_per_chunk + 1))
|
||||||
|
|
||||||
# 1 more block (+ token for kicking off offloading)
|
# 1 more block (+ token for kicking off offloading)
|
||||||
# now check touch was called with all 6 blocks
|
|
||||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||||
generate_store_output(keys)
|
generate_store_output(keys)
|
||||||
)
|
)
|
||||||
@@ -929,9 +938,6 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
|||||||
decoded_tokens=[0] * (tokens_per_chunk + 1),
|
decoded_tokens=[0] * (tokens_per_chunk + 1),
|
||||||
expected_stored=(15, 16, 17),
|
expected_stored=(15, 16, 17),
|
||||||
)
|
)
|
||||||
runner.manager.touch.assert_called()
|
|
||||||
block_hashes1 = list(runner.manager.touch.call_args.args[0])
|
|
||||||
assert len(block_hashes1) == 6
|
|
||||||
|
|
||||||
# terminate request
|
# terminate request
|
||||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||||
@@ -939,13 +945,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
|||||||
# create a new request differing only on the last token
|
# create a new request differing only on the last token
|
||||||
runner.new_request(token_ids=[0] * (tokens_per_chunk * 6 - 1) + [1])
|
runner.new_request(token_ids=[0] * (tokens_per_chunk * 6 - 1) + [1])
|
||||||
runner.run(decoded_tokens=[0])
|
runner.run(decoded_tokens=[0])
|
||||||
runner.manager.touch.assert_called()
|
runner.manager.touch.assert_not_called()
|
||||||
block_hashes2 = list(runner.manager.touch.call_args.args[0])
|
|
||||||
assert len(block_hashes2) == 6
|
|
||||||
|
|
||||||
# verify hashes are the same, except for the last block
|
|
||||||
assert block_hashes1[:5] == block_hashes2[:5]
|
|
||||||
assert block_hashes1[5] != block_hashes2[5]
|
|
||||||
|
|
||||||
# terminate request
|
# terminate request
|
||||||
runner.run(
|
runner.run(
|
||||||
@@ -1353,14 +1353,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
|
|||||||
generate_store_output(keys)
|
generate_store_output(keys)
|
||||||
)
|
)
|
||||||
runner.run(decoded_tokens=[0])
|
runner.run(decoded_tokens=[0])
|
||||||
# _touch called from get_num_new_matched_tokens (2 groups) and
|
runner.manager.touch.assert_not_called()
|
||||||
# _get_reqs_to_store (2 groups) → 4 touch calls total.
|
|
||||||
touch_calls = runner.manager.touch.call_args_list
|
|
||||||
assert len(touch_calls) == 4
|
|
||||||
assert len(touch_calls[0].args[0]) == 3
|
|
||||||
assert len(touch_calls[1].args[0]) == 3
|
|
||||||
assert len(touch_calls[2].args[0]) == 3
|
|
||||||
assert len(touch_calls[3].args[0]) == 3
|
|
||||||
|
|
||||||
# store 3 more block
|
# store 3 more block
|
||||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||||
@@ -1371,9 +1364,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
|
|||||||
expected_stored=(0, 1, 2, 3, 4, 5),
|
expected_stored=(0, 1, 2, 3, 4, 5),
|
||||||
)
|
)
|
||||||
|
|
||||||
# touch called from _get_reqs_to_store * 3 blocks, once for each group
|
runner.manager.touch.assert_not_called()
|
||||||
touch_calls = runner.manager.touch.call_args_list
|
|
||||||
assert len(touch_calls) == 6
|
|
||||||
|
|
||||||
# EOS lands in the last slot of the 7th block (offset 6). No forward pass
|
# EOS lands in the last slot of the 7th block (offset 6). No forward pass
|
||||||
# writes that slot, so the finishing step declines the block.
|
# writes that slot, so the finishing step declines the block.
|
||||||
@@ -1392,13 +1383,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
|
|||||||
expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)),
|
expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)),
|
||||||
)
|
)
|
||||||
|
|
||||||
# 2 touch calls from get_num_new_matched_tokens (2 groups)
|
runner.manager.touch.assert_not_called()
|
||||||
touch_calls = runner.manager.touch.call_args_list
|
|
||||||
assert len(touch_calls) == 2
|
|
||||||
# full attention group touched all 3 blocks
|
|
||||||
assert len(touch_calls[0].args[0]) == 3
|
|
||||||
# sliding window group touched just the last 2 blocks
|
|
||||||
assert len(touch_calls[1].args[0]) == 2
|
|
||||||
|
|
||||||
# 3 blocks are hit on GPU [0, 1, 2]
|
# 3 blocks are hit on GPU [0, 1, 2]
|
||||||
# 1 block loaded [3,]
|
# 1 block loaded [3,]
|
||||||
@@ -1464,15 +1449,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
|||||||
generate_store_output(keys)
|
generate_store_output(keys)
|
||||||
)
|
)
|
||||||
runner.run(decoded_tokens=[0])
|
runner.run(decoded_tokens=[0])
|
||||||
# _touch called from get_num_new_matched_tokens (2 groups) and
|
runner.manager.touch.assert_not_called()
|
||||||
# _get_reqs_to_store (2 groups) → 4 touch calls total.
|
|
||||||
# Group 0 has 2 offload keys, group 1 has 1.
|
|
||||||
touch_calls = runner.manager.touch.call_args_list
|
|
||||||
assert len(touch_calls) == 4
|
|
||||||
assert len(touch_calls[0].args[0]) == 2
|
|
||||||
assert len(touch_calls[1].args[0]) == 1
|
|
||||||
assert len(touch_calls[2].args[0]) == 2
|
|
||||||
assert len(touch_calls[3].args[0]) == 1
|
|
||||||
|
|
||||||
# Get to 31 tokens
|
# Get to 31 tokens
|
||||||
# No further blocks offloaded
|
# No further blocks offloaded
|
||||||
@@ -1482,11 +1459,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
|||||||
# Group 0 blocks: [0, 1], ending_token_offset = 24
|
# Group 0 blocks: [0, 1], ending_token_offset = 24
|
||||||
# Group 1 blocks: [0, 1], ending_token_offset = 32
|
# Group 1 blocks: [0, 1], ending_token_offset = 32
|
||||||
runner.run(decoded_tokens=[0])
|
runner.run(decoded_tokens=[0])
|
||||||
# _get_reqs_to_store touch: only group 1 has a new block to store
|
runner.manager.touch.assert_not_called()
|
||||||
touch_calls = runner.manager.touch.call_args_list
|
|
||||||
assert len(touch_calls) == 2
|
|
||||||
assert len(touch_calls[0].args[0]) == 2
|
|
||||||
assert len(touch_calls[1].args[0]) == 2
|
|
||||||
|
|
||||||
# Get to 35 tokens
|
# Get to 35 tokens
|
||||||
# No further blocks offloaded
|
# No further blocks offloaded
|
||||||
@@ -1496,11 +1469,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
|||||||
# Group 0 blocks: [0, 1, 2], ending_token_offset = 36
|
# Group 0 blocks: [0, 1, 2], ending_token_offset = 36
|
||||||
# Group 1 blocks: [0, 1], ending_token_offset = 32
|
# Group 1 blocks: [0, 1], ending_token_offset = 32
|
||||||
runner.run(decoded_tokens=[0])
|
runner.run(decoded_tokens=[0])
|
||||||
# _get_reqs_to_store touch: only group 0 has a new block to store
|
runner.manager.touch.assert_not_called()
|
||||||
touch_calls = runner.manager.touch.call_args_list
|
|
||||||
assert len(touch_calls) == 2
|
|
||||||
assert len(touch_calls[0].args[0]) == 3
|
|
||||||
assert len(touch_calls[1].args[0]) == 2
|
|
||||||
|
|
||||||
# Get to 47 tokens
|
# Get to 47 tokens
|
||||||
# No further blocks offloaded
|
# No further blocks offloaded
|
||||||
@@ -1510,11 +1479,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
|||||||
# Group 0 blocks: [0, 1, 2, 3], ending_token_offset = 4
|
# Group 0 blocks: [0, 1, 2, 3], ending_token_offset = 4
|
||||||
# Group 1 blocks: [0, 1, 2], ending_token_offset = 48
|
# Group 1 blocks: [0, 1, 2], ending_token_offset = 48
|
||||||
runner.run(decoded_tokens=[0])
|
runner.run(decoded_tokens=[0])
|
||||||
# _get_reqs_to_store touch: both groups have a new block, each with 1 key
|
runner.manager.touch.assert_not_called()
|
||||||
touch_calls = runner.manager.touch.call_args_list
|
|
||||||
assert len(touch_calls) == 2
|
|
||||||
assert len(touch_calls[0].args[0]) == 4
|
|
||||||
assert len(touch_calls[1].args[0]) == 3
|
|
||||||
|
|
||||||
runner.run(decoded_tokens=[0], expected_stored=((0, 3), (1, 2)))
|
runner.run(decoded_tokens=[0], expected_stored=((0, 3), (1, 2)))
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ from vllm.v1.kv_offload.cpu.common import (
|
|||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||||
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
||||||
|
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
|
||||||
|
|
||||||
|
|
||||||
def make_req_context(
|
def make_req_context(
|
||||||
@@ -430,31 +431,34 @@ def test_cpu_manager():
|
|||||||
assert cpu_manager.lookup(to_key(0), _EMPTY_REQ_CTX) is LookupResult.MISS
|
assert cpu_manager.lookup(to_key(0), _EMPTY_REQ_CTX) is LookupResult.MISS
|
||||||
|
|
||||||
# prepare load [2, 3]
|
# prepare load [2, 3]
|
||||||
prepare_load_output = cpu_manager.prepare_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
|
load_ctx = make_req_context("load-2-3")
|
||||||
|
prepare_load_output = cpu_manager.prepare_load(to_keys([2, 3]), load_ctx)
|
||||||
verify_load_output(prepare_load_output, [1, 2])
|
verify_load_output(prepare_load_output, [1, 2])
|
||||||
|
|
||||||
# prepare store with no space ([2, 3] is being loaded)
|
# prepare store with no space ([2, 3] is being loaded)
|
||||||
assert cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX) is None
|
assert cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX) is None
|
||||||
|
|
||||||
# complete load [2, 3]. Load changes the eviction list, making 2, 3 recent.
|
# complete load [2, 3]. Load changes the eviction list, making 2, 3 recent.
|
||||||
cpu_manager.complete_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
|
cpu_manager.complete_load(to_keys([2, 3]), load_ctx)
|
||||||
|
cpu_manager.on_request_finished(load_ctx)
|
||||||
|
|
||||||
# prepare store [6, 7, 8] -> evicts [4, 5, 2] (oldest)
|
# prepare store [6, 7, 8] -> evicts [4, 5, 3] (oldest). Within
|
||||||
|
# the accessed prefix [2, 3], the tail is less valuable than the head.
|
||||||
prepare_store_output = cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX)
|
prepare_store_output = cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX)
|
||||||
verify_store_output(
|
verify_store_output(
|
||||||
prepare_store_output,
|
prepare_store_output,
|
||||||
ExpectedPrepareStoreOutput(
|
ExpectedPrepareStoreOutput(
|
||||||
keys_to_store=[6, 7, 8],
|
keys_to_store=[6, 7, 8],
|
||||||
store_chunk_ids=[1, 0, 3],
|
store_chunk_ids=[2, 0, 3],
|
||||||
evicted_keys=[4, 5, 2],
|
evicted_keys=[4, 5, 3],
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# complete store [6, 7, 8]
|
# complete store [6, 7, 8]
|
||||||
cpu_manager.complete_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX)
|
cpu_manager.complete_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# touch [3, 6, 7] (move to end of LRU order)
|
# touch [2, 6, 7] (move to end of LRU order)
|
||||||
cpu_manager.touch(to_keys([3, 6, 7]), _EMPTY_REQ_CTX)
|
cpu_manager.touch(to_keys([2, 6, 7]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# prepare store [7, 9] -> evicts [8] (oldest following previous touch)
|
# prepare store [7, 9] -> evicts [8] (oldest following previous touch)
|
||||||
prepare_store_output = cpu_manager.prepare_store(to_keys([9]), _EMPTY_REQ_CTX)
|
prepare_store_output = cpu_manager.prepare_store(to_keys([9]), _EMPTY_REQ_CTX)
|
||||||
@@ -477,7 +481,7 @@ def test_cpu_manager():
|
|||||||
verify_events(
|
verify_events(
|
||||||
cpu_manager.take_events(),
|
cpu_manager.take_events(),
|
||||||
expected_stores=({3, 4, 5}, {6, 7, 8}),
|
expected_stores=({3, 4, 5}, {6, 7, 8}),
|
||||||
expected_evictions=({4, 5, 2}, {8}),
|
expected_evictions=({4, 5, 3}, {8}),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -518,6 +522,65 @@ def test_prepare_load_preserves_key_order():
|
|||||||
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
|
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
|
||||||
|
|
||||||
|
|
||||||
|
def test_lru_batch_eviction_failure_is_atomic():
|
||||||
|
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||||
|
policy = manager._policy
|
||||||
|
assert isinstance(policy, LRUCachePolicy)
|
||||||
|
keys = to_keys([1, 2, 3, 4])
|
||||||
|
assert manager.prepare_store(keys, _EMPTY_REQ_CTX) is not None
|
||||||
|
manager.complete_store(keys, _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
|
protected = set(keys[1:])
|
||||||
|
assert policy.evict(2, protected) is None
|
||||||
|
assert all(manager.lookup(key, _EMPTY_REQ_CTX) is LookupResult.HIT for key in keys)
|
||||||
|
|
||||||
|
evicted = policy.evict(1, protected)
|
||||||
|
assert evicted is not None
|
||||||
|
assert [key for key, _ in evicted] == [keys[0]]
|
||||||
|
|
||||||
|
|
||||||
|
def test_lru_repeated_pin_cycles_compact_lazy_heap_entries():
|
||||||
|
manager = make_cpu_manager(num_chunks=1, cache_policy="lru")
|
||||||
|
policy = manager._policy
|
||||||
|
assert isinstance(policy, LRUCachePolicy)
|
||||||
|
key = to_key(1)
|
||||||
|
assert manager.prepare_store([key], _EMPTY_REQ_CTX) is not None
|
||||||
|
manager.complete_store([key], _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
|
for _ in range(128):
|
||||||
|
manager.prepare_load([key], _EMPTY_REQ_CTX)
|
||||||
|
manager.complete_load([key], _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
|
assert len(policy._heap) < 64
|
||||||
|
evicted = policy.evict(1, set())
|
||||||
|
assert evicted is not None
|
||||||
|
assert [evicted_key for evicted_key, _ in evicted] == [key]
|
||||||
|
|
||||||
|
|
||||||
|
def test_reset_discards_stale_request_access_classification():
|
||||||
|
manager = make_cpu_manager(num_chunks=1, cache_policy="arc")
|
||||||
|
policy = manager._policy
|
||||||
|
assert isinstance(policy, ARCCachePolicy)
|
||||||
|
key = to_key(1)
|
||||||
|
resumed_ctx = make_req_context("resumed")
|
||||||
|
|
||||||
|
assert manager.prepare_store([key], resumed_ctx) is not None
|
||||||
|
manager.complete_store([key], resumed_ctx)
|
||||||
|
manager.reset_cache()
|
||||||
|
|
||||||
|
replacement_ctx = make_req_context("replacement")
|
||||||
|
assert manager.prepare_store([key], replacement_ctx) is not None
|
||||||
|
manager.complete_store([key], replacement_ctx)
|
||||||
|
manager.on_request_finished(replacement_ctx)
|
||||||
|
assert key in policy.t1
|
||||||
|
|
||||||
|
manager.prepare_load([key], resumed_ctx)
|
||||||
|
manager.complete_load([key], resumed_ctx)
|
||||||
|
manager.on_request_finished(resumed_ctx)
|
||||||
|
|
||||||
|
assert key in policy.t2
|
||||||
|
|
||||||
|
|
||||||
class TestARCPolicy:
|
class TestARCPolicy:
|
||||||
"""Unit tests for CPUOffloadingManager with ARC eviction policy."""
|
"""Unit tests for CPUOffloadingManager with ARC eviction policy."""
|
||||||
|
|
||||||
@@ -1042,6 +1105,48 @@ def test_filter_reused_manager_oversized_offer_makes_progress():
|
|||||||
assert stored_keys == set(keys)
|
assert stored_keys == set(keys)
|
||||||
|
|
||||||
|
|
||||||
|
def test_store_threshold_does_not_hide_ready_reuse():
|
||||||
|
manager = make_cpu_manager(
|
||||||
|
num_chunks=2,
|
||||||
|
cache_policy="lru",
|
||||||
|
store_threshold=2,
|
||||||
|
max_tracker_size=2,
|
||||||
|
)
|
||||||
|
head, tail, noise_1, noise_2, replacement = to_keys(list(range(5)))
|
||||||
|
|
||||||
|
seed_ctx = make_req_context("seed")
|
||||||
|
output = manager.prepare_store([head, tail], seed_ctx)
|
||||||
|
assert output is not None
|
||||||
|
assert not output.keys_to_store
|
||||||
|
output = manager.prepare_store([head, tail], seed_ctx)
|
||||||
|
assert output is not None
|
||||||
|
assert output.keys_to_store == [head, tail]
|
||||||
|
manager.complete_store([head, tail], seed_ctx)
|
||||||
|
manager.on_request_finished(seed_ctx)
|
||||||
|
|
||||||
|
# Age both resident keys out of the admission tracker without storing the
|
||||||
|
# noise keys. Reusing the tail recreates its counter at one, below the
|
||||||
|
# threshold, but must still refresh its cache recency.
|
||||||
|
output = manager.prepare_store([noise_1, noise_2], make_req_context("noise"))
|
||||||
|
assert output is not None
|
||||||
|
assert not output.keys_to_store
|
||||||
|
reuse_ctx = make_req_context("reuse")
|
||||||
|
skipped_before_reuse = manager.stores_skipped_in_current_batch
|
||||||
|
output = manager.prepare_store([tail], reuse_ctx)
|
||||||
|
assert output is not None
|
||||||
|
assert not output.keys_to_store
|
||||||
|
assert manager.stores_skipped_in_current_batch == skipped_before_reuse
|
||||||
|
manager.on_request_finished(reuse_ctx)
|
||||||
|
|
||||||
|
replacement_ctx = make_req_context("replacement")
|
||||||
|
output = manager.prepare_store([replacement], replacement_ctx)
|
||||||
|
assert output is not None
|
||||||
|
assert not output.keys_to_store
|
||||||
|
output = manager.prepare_store([replacement], replacement_ctx)
|
||||||
|
assert output is not None
|
||||||
|
assert output.evicted_keys == [head]
|
||||||
|
|
||||||
|
|
||||||
def test_evictable_cache_chunk_count():
|
def test_evictable_cache_chunk_count():
|
||||||
"""
|
"""
|
||||||
Verifies _num_evictable_cache_chunks is maintained correctly through the
|
Verifies _num_evictable_cache_chunks is maintained correctly through the
|
||||||
@@ -1156,3 +1261,318 @@ def test_touch_forwards_req_context_to_policy(monkeypatch):
|
|||||||
assert len(received) == 1
|
assert len(received) == 1
|
||||||
assert received[0][0] == keys
|
assert received[0][0] == keys
|
||||||
assert received[0][1] is ctx
|
assert received[0][1] is ctx
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("finish_before_completion", [False, True])
|
||||||
|
def test_request_finish_orders_lru_prefix_independent_of_store_completion(
|
||||||
|
finish_before_completion: bool,
|
||||||
|
):
|
||||||
|
"""A transfer's completion order must not make the prefix head LRU."""
|
||||||
|
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||||
|
prefix = to_keys([1, 2, 3, 4])
|
||||||
|
ctx = make_req_context("prefix")
|
||||||
|
|
||||||
|
output = manager.prepare_store(prefix, ctx)
|
||||||
|
assert output is not None
|
||||||
|
if finish_before_completion:
|
||||||
|
manager.on_request_finished(ctx)
|
||||||
|
manager.complete_store(set(prefix), ctx)
|
||||||
|
if not finish_before_completion:
|
||||||
|
manager.on_request_finished(ctx)
|
||||||
|
|
||||||
|
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
|
||||||
|
assert output is not None
|
||||||
|
assert output.evicted_keys == to_keys([4])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("finish_before_completion", [False, True])
|
||||||
|
def test_request_finish_orders_lru_prefix_independent_of_load_completion(
|
||||||
|
finish_before_completion: bool,
|
||||||
|
):
|
||||||
|
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||||
|
prefix = to_keys([1, 2, 3, 4])
|
||||||
|
|
||||||
|
seed_ctx = make_req_context("seed")
|
||||||
|
output = manager.prepare_store(prefix, seed_ctx)
|
||||||
|
assert output is not None
|
||||||
|
manager.complete_store(prefix, seed_ctx)
|
||||||
|
|
||||||
|
reuse_ctx = make_req_context("reuse")
|
||||||
|
manager.prepare_load(prefix, reuse_ctx)
|
||||||
|
if finish_before_completion:
|
||||||
|
manager.on_request_finished(reuse_ctx)
|
||||||
|
manager.complete_load(set(prefix), reuse_ctx)
|
||||||
|
if not finish_before_completion:
|
||||||
|
manager.on_request_finished(reuse_ctx)
|
||||||
|
|
||||||
|
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
|
||||||
|
assert output is not None
|
||||||
|
assert output.evicted_keys == to_keys([4])
|
||||||
|
|
||||||
|
|
||||||
|
def test_late_old_completion_does_not_override_newer_request_recency():
|
||||||
|
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||||
|
old_keys = to_keys([1, 2])
|
||||||
|
new_keys = to_keys([3, 4])
|
||||||
|
|
||||||
|
old_ctx = make_req_context("old")
|
||||||
|
assert manager.prepare_store(old_keys, old_ctx) is not None
|
||||||
|
manager.on_request_finished(old_ctx)
|
||||||
|
|
||||||
|
new_ctx = make_req_context("new")
|
||||||
|
assert manager.prepare_store(new_keys, new_ctx) is not None
|
||||||
|
manager.complete_store(new_keys, new_ctx)
|
||||||
|
manager.on_request_finished(new_ctx)
|
||||||
|
|
||||||
|
# Physical completion is late, but the logical access is still older.
|
||||||
|
manager.complete_store(old_keys, old_ctx)
|
||||||
|
|
||||||
|
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
|
||||||
|
assert output is not None
|
||||||
|
assert output.evicted_keys == [old_keys[-1]]
|
||||||
|
|
||||||
|
|
||||||
|
def test_arc_counts_reuse_once_and_keeps_insertions_in_t1():
|
||||||
|
manager = make_cpu_manager(num_chunks=4, cache_policy="arc")
|
||||||
|
policy = manager._policy
|
||||||
|
assert isinstance(policy, ARCCachePolicy)
|
||||||
|
prefix = to_keys([1, 2, 3, 4])
|
||||||
|
|
||||||
|
seed_ctx = make_req_context("seed")
|
||||||
|
assert manager.prepare_store(prefix, seed_ctx) is not None
|
||||||
|
manager.complete_store(prefix, seed_ctx)
|
||||||
|
# Repeated store offers from the same request must not become ARC hits.
|
||||||
|
assert manager.prepare_store(prefix, seed_ctx) is not None
|
||||||
|
# Nor should an internal read used to cascade the new chunks to a tier.
|
||||||
|
manager.prepare_load(prefix, seed_ctx)
|
||||||
|
manager.complete_load(prefix, seed_ctx)
|
||||||
|
manager.on_request_finished(seed_ctx)
|
||||||
|
|
||||||
|
assert list(policy.t1) == list(reversed(prefix))
|
||||||
|
assert not policy.t2
|
||||||
|
|
||||||
|
reuse_ctx = make_req_context("reuse")
|
||||||
|
manager.prepare_load(prefix, reuse_ctx)
|
||||||
|
manager.complete_load(prefix, reuse_ctx)
|
||||||
|
# A duplicate internal pin, as used by tiering cascades, is deduplicated.
|
||||||
|
manager.prepare_load(prefix, reuse_ctx)
|
||||||
|
manager.complete_load(prefix, reuse_ctx)
|
||||||
|
manager.on_request_finished(reuse_ctx)
|
||||||
|
|
||||||
|
assert not policy.t1
|
||||||
|
assert list(policy.t2) == list(reversed(prefix))
|
||||||
|
|
||||||
|
|
||||||
|
def test_arc_reuse_wins_if_same_request_later_reinserts_key():
|
||||||
|
manager = make_cpu_manager(num_chunks=1, cache_policy="arc")
|
||||||
|
policy = manager._policy
|
||||||
|
assert isinstance(policy, ARCCachePolicy)
|
||||||
|
key_1, key_2 = to_keys([1, 2])
|
||||||
|
|
||||||
|
seed_ctx = make_req_context("seed")
|
||||||
|
assert manager.prepare_store([key_1], seed_ctx) is not None
|
||||||
|
manager.complete_store([key_1], seed_ctx)
|
||||||
|
manager.on_request_finished(seed_ctx)
|
||||||
|
|
||||||
|
reuse_ctx = make_req_context("reuse-and-reinsert")
|
||||||
|
manager.prepare_load([key_1], reuse_ctx)
|
||||||
|
manager.complete_load([key_1], reuse_ctx)
|
||||||
|
|
||||||
|
other_ctx = make_req_context("evict-reused-key")
|
||||||
|
assert manager.prepare_store([key_2], other_ctx) is not None
|
||||||
|
manager.complete_store([key_2], other_ctx)
|
||||||
|
manager.on_request_finished(other_ctx)
|
||||||
|
|
||||||
|
assert manager.prepare_store([key_1], reuse_ctx) is not None
|
||||||
|
manager.complete_store([key_1], reuse_ctx)
|
||||||
|
manager.on_request_finished(reuse_ctx)
|
||||||
|
|
||||||
|
assert key_1 not in policy.t1
|
||||||
|
assert list(policy.t2) == [key_1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_request_finish_forwards_grouped_deduplicated_accesses(monkeypatch):
|
||||||
|
manager = make_cpu_manager(num_chunks=5)
|
||||||
|
existing = make_offload_key(b"existing", 0)
|
||||||
|
group_0_new = make_offload_key(b"group-0-new", 0)
|
||||||
|
group_1_head = make_offload_key(b"group-1-head", 1)
|
||||||
|
group_1_tail = make_offload_key(b"group-1-tail", 1)
|
||||||
|
|
||||||
|
seed_ctx = make_req_context("seed")
|
||||||
|
assert manager.prepare_store([existing], seed_ctx) is not None
|
||||||
|
manager.complete_store([existing], seed_ctx)
|
||||||
|
|
||||||
|
ctx = make_req_context("grouped")
|
||||||
|
# Group 1 is observed first, but policy finalization follows group index.
|
||||||
|
offered = [group_1_head, existing, group_0_new]
|
||||||
|
assert manager.prepare_store(offered, ctx) is not None
|
||||||
|
# Later offers extend a group's prefix and may repeat earlier keys.
|
||||||
|
assert (
|
||||||
|
manager.prepare_store([group_0_new, group_1_head, group_1_tail], ctx)
|
||||||
|
is not None
|
||||||
|
)
|
||||||
|
|
||||||
|
received = []
|
||||||
|
|
||||||
|
def on_request_finished(key_groups, insertion_only_keys, reused_keys, req_context):
|
||||||
|
received.append((key_groups, insertion_only_keys, reused_keys, req_context))
|
||||||
|
|
||||||
|
monkeypatch.setattr(manager._policy, "on_request_finished", on_request_finished)
|
||||||
|
manager.on_request_finished(ctx)
|
||||||
|
manager.on_request_finished(ctx)
|
||||||
|
|
||||||
|
assert received == [
|
||||||
|
(
|
||||||
|
((existing, group_0_new), (group_1_head, group_1_tail)),
|
||||||
|
{group_0_new, group_1_head, group_1_tail},
|
||||||
|
{existing},
|
||||||
|
ctx,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_store_records_access_once_and_uses_store_miss_hook(monkeypatch):
|
||||||
|
manager = make_cpu_manager(num_chunks=1)
|
||||||
|
resident, candidate = to_keys([1, 2])
|
||||||
|
|
||||||
|
seed_ctx = make_req_context("seed")
|
||||||
|
assert manager.prepare_store([resident], seed_ctx) is not None
|
||||||
|
manager.complete_store([resident], seed_ctx)
|
||||||
|
|
||||||
|
pin_ctx = make_req_context("pin")
|
||||||
|
manager.prepare_load([resident], pin_ctx)
|
||||||
|
|
||||||
|
record_calls = 0
|
||||||
|
original_record = manager._record_request_cache_access
|
||||||
|
|
||||||
|
def record_access(*args, **kwargs):
|
||||||
|
nonlocal record_calls
|
||||||
|
record_calls += 1
|
||||||
|
return original_record(*args, **kwargs)
|
||||||
|
|
||||||
|
store_miss_calls = []
|
||||||
|
|
||||||
|
def on_store_miss(keys, req_context):
|
||||||
|
store_miss_calls.append((list(keys), req_context))
|
||||||
|
|
||||||
|
finalized = []
|
||||||
|
|
||||||
|
def on_request_finished(key_groups, insertion_only_keys, reused_keys, req_context):
|
||||||
|
finalized.append((key_groups, insertion_only_keys, reused_keys, req_context))
|
||||||
|
|
||||||
|
monkeypatch.setattr(manager, "_record_request_cache_access", record_access)
|
||||||
|
monkeypatch.setattr(manager._policy, "on_store_miss", on_store_miss)
|
||||||
|
monkeypatch.setattr(manager._policy, "on_request_finished", on_request_finished)
|
||||||
|
|
||||||
|
ctx = make_req_context("failed-store")
|
||||||
|
assert manager.prepare_store([candidate, resident], ctx) is None
|
||||||
|
assert record_calls == 1
|
||||||
|
assert store_miss_calls == [([candidate], ctx)]
|
||||||
|
|
||||||
|
manager.on_request_finished(ctx)
|
||||||
|
assert finalized == [
|
||||||
|
(((candidate, resident),), set(), {resident}, ctx),
|
||||||
|
]
|
||||||
|
|
||||||
|
manager.complete_load([resident], pin_ctx)
|
||||||
|
|
||||||
|
|
||||||
|
def test_arc_ghost_hit_adapts_once_per_request_before_insertion():
|
||||||
|
manager = make_cpu_manager(num_chunks=2, cache_policy="arc")
|
||||||
|
policy = manager._policy
|
||||||
|
assert isinstance(policy, ARCCachePolicy)
|
||||||
|
keys = to_keys([1, 2, 3])
|
||||||
|
|
||||||
|
seed_ctx = make_req_context("seed")
|
||||||
|
assert manager.prepare_store(keys[:2], seed_ctx) is not None
|
||||||
|
manager.complete_store(keys[:2], seed_ctx)
|
||||||
|
manager.on_request_finished(seed_ctx)
|
||||||
|
|
||||||
|
evict_ctx = make_req_context("create-ghost")
|
||||||
|
output = manager.prepare_store([keys[2]], evict_ctx)
|
||||||
|
assert output is not None
|
||||||
|
[ghost_key] = output.evicted_keys
|
||||||
|
manager.complete_store([keys[2]], evict_ctx)
|
||||||
|
manager.on_request_finished(evict_ctx)
|
||||||
|
assert ghost_key in policy.b1
|
||||||
|
|
||||||
|
resident_keys = [key for key in keys if key != ghost_key]
|
||||||
|
pin_ctx = make_req_context("pin-residents")
|
||||||
|
manager.prepare_load(resident_keys, pin_ctx)
|
||||||
|
|
||||||
|
ghost_ctx = make_req_context("ghost-hit")
|
||||||
|
target_before = policy.target_t1_size
|
||||||
|
assert manager.prepare_store([ghost_key], ghost_ctx) is None
|
||||||
|
target_after_first_offer = policy.target_t1_size
|
||||||
|
assert target_after_first_offer > target_before
|
||||||
|
assert manager.prepare_store([ghost_key], ghost_ctx) is None
|
||||||
|
assert policy.target_t1_size == target_after_first_offer
|
||||||
|
|
||||||
|
manager.complete_load(resident_keys, pin_ctx)
|
||||||
|
output = manager.prepare_store([ghost_key], ghost_ctx)
|
||||||
|
assert output is not None
|
||||||
|
assert policy.target_t1_size == target_after_first_offer
|
||||||
|
manager.complete_store([ghost_key], ghost_ctx)
|
||||||
|
manager.on_request_finished(ghost_ctx)
|
||||||
|
assert ghost_key in policy.t1
|
||||||
|
assert ghost_key not in policy.t2
|
||||||
|
|
||||||
|
|
||||||
|
def test_arc_pending_chunk_is_not_a_frequency_hit():
|
||||||
|
manager = make_cpu_manager(num_chunks=2, cache_policy="arc")
|
||||||
|
policy = manager._policy
|
||||||
|
assert isinstance(policy, ARCCachePolicy)
|
||||||
|
key = to_key(1)
|
||||||
|
|
||||||
|
writer_ctx = make_req_context("writer")
|
||||||
|
assert manager.prepare_store([key], writer_ctx) is not None
|
||||||
|
|
||||||
|
observer_ctx = make_req_context("pending-observer")
|
||||||
|
output = manager.prepare_store([key], observer_ctx)
|
||||||
|
assert output is not None and not output.keys_to_store
|
||||||
|
manager.on_request_finished(observer_ctx)
|
||||||
|
|
||||||
|
chunk = policy.get(key)
|
||||||
|
assert chunk is not None and not chunk.is_ready
|
||||||
|
assert key in policy.t1
|
||||||
|
assert key not in policy.t2
|
||||||
|
|
||||||
|
|
||||||
|
def test_request_key_positions_override_store_observation_order():
|
||||||
|
manager = make_cpu_manager(num_chunks=2, cache_policy="lru")
|
||||||
|
head, tail = to_keys([1, 2])
|
||||||
|
ctx = make_req_context("out-of-order-store")
|
||||||
|
ctx.set_offload_key_position(head, 16)
|
||||||
|
ctx.set_offload_key_position(tail, 32)
|
||||||
|
|
||||||
|
# A backfill or partial-tail path may offer a later key first.
|
||||||
|
assert manager.prepare_store([tail], ctx) is not None
|
||||||
|
assert manager.prepare_store([head], ctx) is not None
|
||||||
|
manager.on_request_finished(ctx)
|
||||||
|
manager.complete_store({head, tail}, ctx)
|
||||||
|
|
||||||
|
output = manager.prepare_store([to_key(3)], make_req_context("evict-tail"))
|
||||||
|
assert output is not None
|
||||||
|
assert output.evicted_keys == [tail]
|
||||||
|
|
||||||
|
|
||||||
|
def test_request_key_positions_order_tails_across_kv_groups():
|
||||||
|
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||||
|
group_0_head = make_offload_key(b"group-0-head", 0)
|
||||||
|
group_0_tail = make_offload_key(b"group-0-tail", 0)
|
||||||
|
group_1_head = make_offload_key(b"group-1-head", 1)
|
||||||
|
group_1_tail = make_offload_key(b"group-1-tail", 1)
|
||||||
|
keys = [group_0_head, group_0_tail, group_1_head, group_1_tail]
|
||||||
|
ctx = make_req_context("hybrid-groups")
|
||||||
|
for key in (group_0_head, group_1_head):
|
||||||
|
ctx.set_offload_key_position(key, 16)
|
||||||
|
for key in (group_0_tail, group_1_tail):
|
||||||
|
ctx.set_offload_key_position(key, 32)
|
||||||
|
|
||||||
|
assert manager.prepare_store(keys, ctx) is not None
|
||||||
|
manager.complete_store(keys, ctx)
|
||||||
|
manager.on_request_finished(ctx)
|
||||||
|
|
||||||
|
output = manager.prepare_store(to_keys([10, 11]), make_req_context("evict"))
|
||||||
|
assert output is not None
|
||||||
|
assert set(output.evicted_keys) == {group_0_tail, group_1_tail}
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ from vllm.v1.kv_offload.base import (
|
|||||||
TierMatcher,
|
TierMatcher,
|
||||||
make_offload_key,
|
make_offload_key,
|
||||||
)
|
)
|
||||||
|
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
|
||||||
from vllm.v1.kv_offload.tiering.base import (
|
from vllm.v1.kv_offload.tiering.base import (
|
||||||
JobResult,
|
JobResult,
|
||||||
SecondaryTierManager,
|
SecondaryTierManager,
|
||||||
@@ -753,14 +754,64 @@ class TestTieringOffloadingManager:
|
|||||||
# Touch chunks
|
# Touch chunks
|
||||||
self.manager.touch(chunks, _CTX)
|
self.manager.touch(chunks, _CTX)
|
||||||
|
|
||||||
# Verify touch was called on primary tier (check LRU order)
|
# Verify touch was called on the primary tier (head is most recent).
|
||||||
primary_keys = list(self.primary_tier._policy.evictable_chunks.keys())
|
policy = self.primary_tier._policy
|
||||||
assert primary_keys[-3:] == list(reversed(chunks))
|
assert isinstance(policy, LRUCachePolicy)
|
||||||
|
ranks = policy._ranks
|
||||||
|
assert ranks[chunks[2]] < ranks[chunks[1]] < ranks[chunks[0]]
|
||||||
|
|
||||||
# Verify touch was propagated to all secondary tiers
|
# Verify touch was propagated to all secondary tiers
|
||||||
self.secondary_tier1.touch.assert_called_once_with(chunks, _CTX)
|
self.secondary_tier1.touch.assert_called_once_with(chunks, _CTX)
|
||||||
self.secondary_tier2.touch.assert_called_once_with(chunks, _CTX)
|
self.secondary_tier2.touch.assert_called_once_with(chunks, _CTX)
|
||||||
|
|
||||||
|
def test_request_order_survives_late_cascade_completion(self, manager_setup):
|
||||||
|
"""Cascade completion must not replace request order with I/O order."""
|
||||||
|
chunks = to_keys(range(5))
|
||||||
|
self._start_request()
|
||||||
|
output = self.manager.prepare_store(chunks, _CTX)
|
||||||
|
assert output is not None
|
||||||
|
self.manager.complete_store(chunks, _CTX, success=True)
|
||||||
|
|
||||||
|
# Finalize while both secondary-tier cascades still pin primary chunks.
|
||||||
|
self.manager.on_request_finished(_CTX)
|
||||||
|
self._simulate_on_schedule_end()
|
||||||
|
self._simulate_on_schedule_end()
|
||||||
|
|
||||||
|
evict_ctx = ReqContext(req_id="evict")
|
||||||
|
self._start_request(evict_ctx)
|
||||||
|
output = self.manager.prepare_store(to_keys([5]), evict_ctx)
|
||||||
|
assert output is not None
|
||||||
|
assert output.evicted_keys == [chunks[-1]]
|
||||||
|
|
||||||
|
def test_late_old_store_submission_does_not_override_newer_request_order(
|
||||||
|
self, manager_setup
|
||||||
|
):
|
||||||
|
"""Cascade submission after finish is not a new primary access."""
|
||||||
|
old_blocks = to_keys(range(2))
|
||||||
|
old_ctx = ReqContext(req_id="old")
|
||||||
|
self._start_request(old_ctx)
|
||||||
|
assert self.manager.prepare_store(old_blocks, old_ctx) is not None
|
||||||
|
self.manager.on_request_finished(old_ctx)
|
||||||
|
|
||||||
|
new_blocks = to_keys(range(2, 5))
|
||||||
|
new_ctx = ReqContext(req_id="new")
|
||||||
|
self._start_request(new_ctx)
|
||||||
|
assert self.manager.prepare_store(new_blocks, new_ctx) is not None
|
||||||
|
self.manager.complete_store(new_blocks, new_ctx, success=True)
|
||||||
|
self.manager.on_request_finished(new_ctx)
|
||||||
|
|
||||||
|
# The old GPU->primary write lands after the newer request finished.
|
||||||
|
# Its cascade pins must not turn it into the more recent request.
|
||||||
|
self.manager.complete_store(old_blocks, old_ctx, success=True)
|
||||||
|
self._simulate_on_schedule_end()
|
||||||
|
self._simulate_on_schedule_end()
|
||||||
|
|
||||||
|
evict_ctx = ReqContext(req_id="evict")
|
||||||
|
self._start_request(evict_ctx)
|
||||||
|
output = self.manager.prepare_store(to_keys([5]), evict_ctx)
|
||||||
|
assert output is not None
|
||||||
|
assert output.evicted_keys == [old_blocks[-1]]
|
||||||
|
|
||||||
def test_failed_store_no_cascade(self, manager_setup):
|
def test_failed_store_no_cascade(self, manager_setup):
|
||||||
"""Test that failed GPU→primary store doesn't cascade."""
|
"""Test that failed GPU→primary store doesn't cascade."""
|
||||||
chunks = to_keys(range(3))
|
chunks = to_keys(range(3))
|
||||||
@@ -873,10 +924,10 @@ class TestTieringOffloadingManager:
|
|||||||
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
||||||
assert job_metadata.req_context is ctx
|
assert job_metadata.req_context is ctx
|
||||||
|
|
||||||
def test_on_request_finished_delays_secondary_until_store_submitted(
|
def test_on_request_finished_delays_only_secondary_until_store_submitted(
|
||||||
self, manager_setup
|
self, manager_setup
|
||||||
):
|
):
|
||||||
"""Manager hook is eager; secondary hooks wait for cascade submission."""
|
"""Primary order commits immediately; secondary cleanup waits."""
|
||||||
chunks = to_keys(range(2))
|
chunks = to_keys(range(2))
|
||||||
ctx = ReqContext(req_id="req_delayed_secondary")
|
ctx = ReqContext(req_id="req_delayed_secondary")
|
||||||
calls: list[tuple[str, str]] = []
|
calls: list[tuple[str, str]] = []
|
||||||
|
|||||||
@@ -427,8 +427,11 @@ class RequestOffloadState:
|
|||||||
None,
|
None,
|
||||||
group_config.hashes_per_chunk,
|
group_config.hashes_per_chunk,
|
||||||
):
|
):
|
||||||
group_state.offload_keys.append(
|
key = make_offload_key(req_block_hash, group_config.group_idx)
|
||||||
make_offload_key(req_block_hash, group_config.group_idx)
|
group_state.offload_keys.append(key)
|
||||||
|
self.req_context.set_offload_key_position(
|
||||||
|
key,
|
||||||
|
len(group_state.offload_keys) * group_config.tokens_per_chunk,
|
||||||
)
|
)
|
||||||
|
|
||||||
def update_block_id_groups(
|
def update_block_id_groups(
|
||||||
@@ -742,37 +745,6 @@ class OffloadingConnectorScheduler:
|
|||||||
)
|
)
|
||||||
return None if defer_lookup or pending_in_window else consecutive_hits
|
return None if defer_lookup or pending_in_window else consecutive_hits
|
||||||
|
|
||||||
def _touch(self, req_status: RequestOffloadState):
|
|
||||||
for group_config, group_state in zip(
|
|
||||||
self.config.kv_group_configs, req_status.group_states
|
|
||||||
):
|
|
||||||
if group_config.sliding_window_size_in_chunks is None:
|
|
||||||
self.manager.touch(group_state.offload_keys, req_status.req_context)
|
|
||||||
else:
|
|
||||||
# Keep only chunks needed to hit the original request, plus
|
|
||||||
# decoded chunks.
|
|
||||||
chunks_to_skip = max(
|
|
||||||
0,
|
|
||||||
group_state.num_hit_chunks
|
|
||||||
- group_config.sliding_window_size_in_chunks,
|
|
||||||
)
|
|
||||||
self.manager.touch(
|
|
||||||
group_state.offload_keys[chunks_to_skip:],
|
|
||||||
req_status.req_context,
|
|
||||||
)
|
|
||||||
if req_status.partial_tail_boundary is not None:
|
|
||||||
self.manager.touch(
|
|
||||||
tuple(
|
|
||||||
self._make_boundary_key(
|
|
||||||
req_status.req,
|
|
||||||
group.group_idx,
|
|
||||||
req_status.partial_tail_boundary,
|
|
||||||
)
|
|
||||||
for group in self.config.kv_group_configs
|
|
||||||
),
|
|
||||||
req_status.req_context,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _lookup_complete_chunks(
|
def _lookup_complete_chunks(
|
||||||
self,
|
self,
|
||||||
req_status: RequestOffloadState,
|
req_status: RequestOffloadState,
|
||||||
@@ -977,10 +949,16 @@ class OffloadingConnectorScheduler:
|
|||||||
return num_hit_tokens
|
return num_hit_tokens
|
||||||
|
|
||||||
def _make_boundary_key(
|
def _make_boundary_key(
|
||||||
self, request: Request, group_idx: int, boundary_tokens: int
|
self,
|
||||||
|
request: Request,
|
||||||
|
group_idx: int,
|
||||||
|
boundary_tokens: int,
|
||||||
|
req_context: ReqContext,
|
||||||
) -> OffloadKey:
|
) -> OffloadKey:
|
||||||
hash_idx = boundary_tokens // self.config.tokens_per_hash - 1
|
hash_idx = boundary_tokens // self.config.tokens_per_hash - 1
|
||||||
return make_offload_key(request.block_hashes[hash_idx], group_idx)
|
key = make_offload_key(request.block_hashes[hash_idx], group_idx)
|
||||||
|
req_context.set_offload_key_position(key, boundary_tokens)
|
||||||
|
return key
|
||||||
|
|
||||||
def _lookup(
|
def _lookup(
|
||||||
self,
|
self,
|
||||||
@@ -1015,7 +993,10 @@ class OffloadingConnectorScheduler:
|
|||||||
boundary_keys = []
|
boundary_keys = []
|
||||||
for group_config in self.config.kv_group_configs:
|
for group_config in self.config.kv_group_configs:
|
||||||
key = self._make_boundary_key(
|
key = self._make_boundary_key(
|
||||||
req_status.req, group_config.group_idx, boundary
|
req_status.req,
|
||||||
|
group_config.group_idx,
|
||||||
|
boundary,
|
||||||
|
req_status.req_context,
|
||||||
)
|
)
|
||||||
boundary_keys.append(key)
|
boundary_keys.append(key)
|
||||||
result = self.manager.lookup(key, req_status.req_context)
|
result = self.manager.lookup(key, req_status.req_context)
|
||||||
@@ -1108,8 +1089,6 @@ class OffloadingConnectorScheduler:
|
|||||||
self._maybe_observe_lookup_async_delay(req_status)
|
self._maybe_observe_lookup_async_delay(req_status)
|
||||||
req_status.update_num_hit_chunks(num_computed_tokens + (num_hit_tokens or 0))
|
req_status.update_num_hit_chunks(num_computed_tokens + (num_hit_tokens or 0))
|
||||||
|
|
||||||
self._touch(req_status)
|
|
||||||
|
|
||||||
return num_hit_tokens, bool(num_hit_tokens)
|
return num_hit_tokens, bool(num_hit_tokens)
|
||||||
|
|
||||||
def update_state_after_alloc(
|
def update_state_after_alloc(
|
||||||
@@ -1183,7 +1162,10 @@ class OffloadingConnectorScheduler:
|
|||||||
if partial_tail_boundary is not None:
|
if partial_tail_boundary is not None:
|
||||||
keys_to_load.append(
|
keys_to_load.append(
|
||||||
self._make_boundary_key(
|
self._make_boundary_key(
|
||||||
request, group_config.group_idx, partial_tail_boundary
|
request,
|
||||||
|
group_config.group_idx,
|
||||||
|
partial_tail_boundary,
|
||||||
|
req_status.req_context,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1307,7 +1289,9 @@ class OffloadingConnectorScheduler:
|
|||||||
):
|
):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
key = self._make_boundary_key(req, group_idx, boundary)
|
key = self._make_boundary_key(
|
||||||
|
req, group_idx, boundary, req_status.req_context
|
||||||
|
)
|
||||||
store_output = self.manager.prepare_store([key], req_status.req_context)
|
store_output = self.manager.prepare_store([key], req_status.req_context)
|
||||||
if store_output is None:
|
if store_output is None:
|
||||||
self._connector_stats.increase_counter(
|
self._connector_stats.increase_counter(
|
||||||
@@ -1402,7 +1386,9 @@ class OffloadingConnectorScheduler:
|
|||||||
):
|
):
|
||||||
continue
|
continue
|
||||||
keys = [
|
keys = [
|
||||||
self._make_boundary_key(req, group.group_idx, boundary)
|
self._make_boundary_key(
|
||||||
|
req, group.group_idx, boundary, req_status.req_context
|
||||||
|
)
|
||||||
for group in self.config.kv_group_configs
|
for group in self.config.kv_group_configs
|
||||||
]
|
]
|
||||||
block_ids = [
|
block_ids = [
|
||||||
@@ -1689,8 +1675,6 @@ class OffloadingConnectorScheduler:
|
|||||||
req_status.advance_stored_idx(num_offloadable_tokens)
|
req_status.advance_stored_idx(num_offloadable_tokens)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
self._touch(req_status)
|
|
||||||
|
|
||||||
keys_to_store = set(store_output.keys_to_store)
|
keys_to_store = set(store_output.keys_to_store)
|
||||||
|
|
||||||
group_sizes: list[int] = []
|
group_sizes: list[int] = []
|
||||||
|
|||||||
@@ -96,6 +96,12 @@ class ReqContext:
|
|||||||
# kv_transfer_params once (in on_new_request) and read the result back
|
# kv_transfer_params once (in on_new_request) and read the result back
|
||||||
# on later calls for the same request.
|
# on later calls for the same request.
|
||||||
_state: dict[type, Any] = field(default_factory=dict, repr=False, init=False)
|
_state: dict[type, Any] = field(default_factory=dict, repr=False, init=False)
|
||||||
|
# End-token position for each key in this request. The scheduler records
|
||||||
|
# these positions so managers can recover prefix order even when store
|
||||||
|
# calls arrive out of order (for example, SWA backfills).
|
||||||
|
_offload_key_positions: dict[OffloadKey, int] = field(
|
||||||
|
default_factory=dict, repr=False, init=False
|
||||||
|
)
|
||||||
|
|
||||||
def set_state(self, val: Any) -> None:
|
def set_state(self, val: Any) -> None:
|
||||||
self._state[type(val)] = val
|
self._state[type(val)] = val
|
||||||
@@ -103,6 +109,12 @@ class ReqContext:
|
|||||||
def get_state(self, cls: type[_T]) -> _T | None:
|
def get_state(self, cls: type[_T]) -> _T | None:
|
||||||
return self._state.get(cls)
|
return self._state.get(cls)
|
||||||
|
|
||||||
|
def set_offload_key_position(self, key: OffloadKey, end_token: int) -> None:
|
||||||
|
self._offload_key_positions[key] = end_token
|
||||||
|
|
||||||
|
def get_offload_key_position(self, key: OffloadKey) -> int | None:
|
||||||
|
return self._offload_key_positions.get(key)
|
||||||
|
|
||||||
|
|
||||||
class LookupResult(Enum):
|
class LookupResult(Enum):
|
||||||
"""Result of OffloadingManager.lookup()."""
|
"""Result of OffloadingManager.lookup()."""
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
from collections.abc import Collection, Iterable
|
from collections.abc import Collection, Iterable
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
from typing_extensions import override
|
from typing_extensions import override
|
||||||
|
|
||||||
@@ -18,6 +19,7 @@ from vllm.v1.kv_offload.base import (
|
|||||||
PrepareStoreOutput,
|
PrepareStoreOutput,
|
||||||
ReqContext,
|
ReqContext,
|
||||||
RequestOffloadingContext,
|
RequestOffloadingContext,
|
||||||
|
get_offload_group_idx,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.cpu.common import (
|
from vllm.v1.kv_offload.cpu.common import (
|
||||||
CPULoadStoreSpec,
|
CPULoadStoreSpec,
|
||||||
@@ -27,6 +29,20 @@ from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
|
|||||||
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
|
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class _RequestCacheAccess:
|
||||||
|
"""Cache keys observed by one request, grouped in prefix order."""
|
||||||
|
|
||||||
|
owner: object
|
||||||
|
cache_generation: int
|
||||||
|
key_groups: dict[int, list[OffloadKey]] = field(default_factory=dict)
|
||||||
|
seen_keys: set[OffloadKey] = field(default_factory=set)
|
||||||
|
inserted_keys: set[OffloadKey] = field(default_factory=set)
|
||||||
|
reused_keys: set[OffloadKey] = field(default_factory=set)
|
||||||
|
store_miss_keys: set[OffloadKey] = field(default_factory=set)
|
||||||
|
finished: bool = False
|
||||||
|
|
||||||
|
|
||||||
class CPUOffloadingManager(OffloadingManager):
|
class CPUOffloadingManager(OffloadingManager):
|
||||||
"""
|
"""
|
||||||
An OffloadingManager with a pluggable CachePolicy, resolved by name via
|
An OffloadingManager with a pluggable CachePolicy, resolved by name via
|
||||||
@@ -66,6 +82,7 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
self.max_tracker_size: int = max_tracker_size
|
self.max_tracker_size: int = max_tracker_size
|
||||||
self.stores_skipped_in_current_batch: int = 0
|
self.stores_skipped_in_current_batch: int = 0
|
||||||
self.allocation_sizes_in_current_batch: list[int] = []
|
self.allocation_sizes_in_current_batch: list[int] = []
|
||||||
|
self._cache_generation = 0
|
||||||
|
|
||||||
# Number of chunk references. It is ordered so can evict the LRU entry in O(1).
|
# Number of chunk references. It is ordered so can evict the LRU entry in O(1).
|
||||||
self.counts: OrderedDict[OffloadKey, int] | None = (
|
self.counts: OrderedDict[OffloadKey, int] | None = (
|
||||||
@@ -124,10 +141,48 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
num_unprotected -= 1
|
num_unprotected -= 1
|
||||||
self.counts[key] = 1
|
self.counts[key] = 1
|
||||||
|
|
||||||
|
def _get_request_cache_access(self, req_context: ReqContext) -> _RequestCacheAccess:
|
||||||
|
state = req_context.get_state(_RequestCacheAccess)
|
||||||
|
if (
|
||||||
|
state is None
|
||||||
|
or state.owner is not self
|
||||||
|
or state.cache_generation != self._cache_generation
|
||||||
|
):
|
||||||
|
state = _RequestCacheAccess(
|
||||||
|
owner=self, cache_generation=self._cache_generation
|
||||||
|
)
|
||||||
|
req_context.set_state(state)
|
||||||
|
return state
|
||||||
|
|
||||||
|
def _record_request_cache_access(
|
||||||
|
self,
|
||||||
|
keys: Iterable[OffloadKey],
|
||||||
|
req_context: ReqContext,
|
||||||
|
inserted_keys: Iterable[OffloadKey] = (),
|
||||||
|
reused_keys: Iterable[OffloadKey] = (),
|
||||||
|
) -> None:
|
||||||
|
state = self._get_request_cache_access(req_context)
|
||||||
|
for key in keys:
|
||||||
|
if key in state.seen_keys:
|
||||||
|
continue
|
||||||
|
assert not state.finished, (
|
||||||
|
"New cache keys observed after request finalization"
|
||||||
|
)
|
||||||
|
group_idx = get_offload_group_idx(key)
|
||||||
|
state.key_groups.setdefault(group_idx, []).append(key)
|
||||||
|
state.seen_keys.add(key)
|
||||||
|
state.inserted_keys.update(inserted_keys)
|
||||||
|
# Re-reading a chunk inserted by this request is an internal transfer
|
||||||
|
# (for example, a tiering cascade), not a second cache access.
|
||||||
|
state.reused_keys.update(
|
||||||
|
key for key in reused_keys if key not in state.inserted_keys
|
||||||
|
)
|
||||||
|
|
||||||
# --- OffloadingManager interface ---
|
# --- OffloadingManager interface ---
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
||||||
|
self._get_request_cache_access(req_context)
|
||||||
return RequestOffloadingContext()
|
return RequestOffloadingContext()
|
||||||
|
|
||||||
@override
|
@override
|
||||||
@@ -144,6 +199,15 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
self,
|
self,
|
||||||
keys: Collection[OffloadKey],
|
keys: Collection[OffloadKey],
|
||||||
req_context: ReqContext,
|
req_context: ReqContext,
|
||||||
|
) -> LoadStoreSpec:
|
||||||
|
return self._prepare_load(keys, req_context, record_access=True)
|
||||||
|
|
||||||
|
def _prepare_load(
|
||||||
|
self,
|
||||||
|
keys: Collection[OffloadKey],
|
||||||
|
req_context: ReqContext,
|
||||||
|
*,
|
||||||
|
record_access: bool,
|
||||||
) -> LoadStoreSpec:
|
) -> LoadStoreSpec:
|
||||||
chunks = []
|
chunks = []
|
||||||
for key in keys:
|
for key in keys:
|
||||||
@@ -156,6 +220,8 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
assert self._num_evictable_cache_chunks >= 0
|
assert self._num_evictable_cache_chunks >= 0
|
||||||
chunk.ref_cnt += 1
|
chunk.ref_cnt += 1
|
||||||
chunks.append(chunk)
|
chunks.append(chunk)
|
||||||
|
if record_access:
|
||||||
|
self._record_request_cache_access(keys, req_context, reused_keys=keys)
|
||||||
return self._get_load_store_spec(keys, chunks)
|
return self._get_load_store_spec(keys, chunks)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
@@ -181,13 +247,52 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
keys: Collection[OffloadKey],
|
keys: Collection[OffloadKey],
|
||||||
req_context: ReqContext,
|
req_context: ReqContext,
|
||||||
) -> PrepareStoreOutput | None:
|
) -> PrepareStoreOutput | None:
|
||||||
|
keys = list(keys)
|
||||||
if self.counts is not None:
|
if self.counts is not None:
|
||||||
num_keys = len(keys)
|
|
||||||
self._record_accesses(keys)
|
self._record_accesses(keys)
|
||||||
keys = [k for k in keys if self.counts.get(k, 0) >= self.store_threshold]
|
# Partition the original offer before admission filtering. Ready
|
||||||
self.stores_skipped_in_current_batch += num_keys - len(keys)
|
# resident chunks are request reuses even if their tracker entry was
|
||||||
# filter out chunks that are already stored
|
# aged out; the threshold applies only to new store candidates.
|
||||||
keys_to_store = [k for k in keys if self._policy.get(k) is None]
|
keys_to_store: list[OffloadKey] = []
|
||||||
|
ready_existing_keys: list[OffloadKey] = []
|
||||||
|
for key in keys:
|
||||||
|
chunk = self._policy.get(key)
|
||||||
|
if chunk is None:
|
||||||
|
keys_to_store.append(key)
|
||||||
|
else:
|
||||||
|
if chunk.is_ready:
|
||||||
|
ready_existing_keys.append(key)
|
||||||
|
|
||||||
|
if self.counts is not None:
|
||||||
|
num_store_candidates = len(keys_to_store)
|
||||||
|
keys_to_store = [
|
||||||
|
key
|
||||||
|
for key in keys_to_store
|
||||||
|
if self.counts.get(key, 0) >= self.store_threshold
|
||||||
|
]
|
||||||
|
self.stores_skipped_in_current_batch += num_store_candidates - len(
|
||||||
|
keys_to_store
|
||||||
|
)
|
||||||
|
|
||||||
|
state = self._get_request_cache_access(req_context)
|
||||||
|
new_store_misses = [
|
||||||
|
key for key in keys_to_store if key not in state.store_miss_keys
|
||||||
|
]
|
||||||
|
if new_store_misses:
|
||||||
|
# ARC learns from B1/B2 before insert() removes the ghost entry.
|
||||||
|
# Deduplication makes this one policy observation per request.
|
||||||
|
self._policy.on_store_miss(new_store_misses, req_context)
|
||||||
|
state.store_miss_keys.update(new_store_misses)
|
||||||
|
|
||||||
|
recorded_keys = set(keys_to_store)
|
||||||
|
recorded_keys.update(ready_existing_keys)
|
||||||
|
# Record once before any allocation-related early return. Store
|
||||||
|
# candidates are classified as insertions only after allocation succeeds.
|
||||||
|
self._record_request_cache_access(
|
||||||
|
(key for key in keys if key in recorded_keys),
|
||||||
|
req_context,
|
||||||
|
reused_keys=ready_existing_keys,
|
||||||
|
)
|
||||||
|
|
||||||
if not keys_to_store:
|
if not keys_to_store:
|
||||||
return PrepareStoreOutput(
|
return PrepareStoreOutput(
|
||||||
@@ -239,6 +344,7 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
for key, chunk in zip(keys_to_store, chunks):
|
for key, chunk in zip(keys_to_store, chunks):
|
||||||
self._policy.insert(key, chunk)
|
self._policy.insert(key, chunk)
|
||||||
self._num_write_pending_chunks += len(keys_to_store)
|
self._num_write_pending_chunks += len(keys_to_store)
|
||||||
|
state.inserted_keys.update(keys_to_store)
|
||||||
|
|
||||||
# build store specs for allocated chunks
|
# build store specs for allocated chunks
|
||||||
store_spec = self._get_load_store_spec(keys_to_store, chunks)
|
store_spec = self._get_load_store_spec(keys_to_store, chunks)
|
||||||
@@ -284,6 +390,35 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@override
|
||||||
|
def on_request_finished(self, req_context: ReqContext) -> None:
|
||||||
|
state = req_context.get_state(_RequestCacheAccess)
|
||||||
|
if (
|
||||||
|
state is None
|
||||||
|
or state.owner is not self
|
||||||
|
or state.cache_generation != self._cache_generation
|
||||||
|
or state.finished
|
||||||
|
):
|
||||||
|
return
|
||||||
|
state.finished = True
|
||||||
|
key_groups = []
|
||||||
|
for group_idx in sorted(state.key_groups):
|
||||||
|
keys = state.key_groups[group_idx]
|
||||||
|
positions = {
|
||||||
|
key: position
|
||||||
|
for key in keys
|
||||||
|
if (position := req_context.get_offload_key_position(key)) is not None
|
||||||
|
}
|
||||||
|
if len(positions) == len(keys):
|
||||||
|
keys = sorted(keys, key=positions.__getitem__)
|
||||||
|
key_groups.append(tuple(keys))
|
||||||
|
self._policy.on_request_finished(
|
||||||
|
tuple(key_groups),
|
||||||
|
state.inserted_keys - state.reused_keys,
|
||||||
|
state.reused_keys,
|
||||||
|
req_context,
|
||||||
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def reset_cache(self) -> None:
|
def reset_cache(self) -> None:
|
||||||
# Clear ALL chunks unconditionally. The scheduler's _stale_job_threshold
|
# Clear ALL chunks unconditionally. The scheduler's _stale_job_threshold
|
||||||
@@ -292,6 +427,7 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
# flushes in-flight load job IDs to the workers before any new stores
|
# flushes in-flight load job IDs to the workers before any new stores
|
||||||
# can begin, preventing a cross-direction data race on reused offload chunk IDs.
|
# can begin, preventing a cross-direction data race on reused offload chunk IDs.
|
||||||
self._policy.clear()
|
self._policy.clear()
|
||||||
|
self._cache_generation += 1
|
||||||
self._num_evictable_cache_chunks = 0
|
self._num_evictable_cache_chunks = 0
|
||||||
self._num_write_pending_chunks = 0
|
self._num_write_pending_chunks = 0
|
||||||
|
|
||||||
|
|||||||
@@ -1,12 +1,16 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
from collections.abc import Iterable, Iterator
|
from collections.abc import Iterable, Iterator, Sequence
|
||||||
|
|
||||||
from typing_extensions import override
|
from typing_extensions import override
|
||||||
|
|
||||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||||
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
|
from vllm.v1.kv_offload.cpu.policies.base import (
|
||||||
|
CachePolicy,
|
||||||
|
ChunkStatus,
|
||||||
|
order_request_keys,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ARCCachePolicy(CachePolicy):
|
class ARCCachePolicy(CachePolicy):
|
||||||
@@ -24,12 +28,11 @@ class ARCCachePolicy(CachePolicy):
|
|||||||
Searches T1 and T2 for chunk hashes and counts consecutive hits
|
Searches T1 and T2 for chunk hashes and counts consecutive hits
|
||||||
until a miss or non-ready chunk is encountered.
|
until a miss or non-ready chunk is encountered.
|
||||||
|
|
||||||
2. Cache touch (touch) - Adaptive Learning:
|
2. Request access - Adaptive Learning:
|
||||||
For each key (in reverse order):
|
- Ready chunks reused by a request move from T1 to T2 once, or
|
||||||
- If in T1: Move to T2 (promotion from recent to frequent).
|
move to the MRU end of T2.
|
||||||
- If in T2: Move to MRU position (end of queue).
|
- B1/B2 misses adjust target_t1_size once per request before
|
||||||
- If in B1 ghost list: Increase target_t1_size.
|
insertion removes their ghost entries.
|
||||||
- If in B2 ghost list: Decrease target_t1_size.
|
|
||||||
|
|
||||||
3. Chunk eviction (evict) - Adaptive Replacement:
|
3. Chunk eviction (evict) - Adaptive Replacement:
|
||||||
Determines eviction source based on adaptive target:
|
Determines eviction source based on adaptive target:
|
||||||
@@ -39,7 +42,7 @@ class ARCCachePolicy(CachePolicy):
|
|||||||
|
|
||||||
4. Chunk insertion (insert):
|
4. Chunk insertion (insert):
|
||||||
New chunks are always inserted into T1 and removed from B1/B2 if
|
New chunks are always inserted into T1 and removed from B1/B2 if
|
||||||
present. Chunks may later be promoted to T2 during touch operations.
|
present. A later request reuse may promote them to T2.
|
||||||
|
|
||||||
Adaptive Behavior:
|
Adaptive Behavior:
|
||||||
The algorithm self-tunes the recency vs. frequency trade-off:
|
The algorithm self-tunes the recency vs. frequency trade-off:
|
||||||
@@ -71,6 +74,26 @@ class ARCCachePolicy(CachePolicy):
|
|||||||
if self.t1.pop(key, None) is None:
|
if self.t1.pop(key, None) is None:
|
||||||
self.t2.pop(key, None)
|
self.t2.pop(key, None)
|
||||||
|
|
||||||
|
def _adapt_to_ghost_hit(self, key: OffloadKey) -> bool:
|
||||||
|
if key in self.b1:
|
||||||
|
delta = max(1, len(self.b2) / len(self.b1))
|
||||||
|
self.target_t1_size = min(self.target_t1_size + delta, self.cache_capacity)
|
||||||
|
self.b1.move_to_end(key)
|
||||||
|
return True
|
||||||
|
if key in self.b2:
|
||||||
|
delta = max(1, len(self.b1) / len(self.b2))
|
||||||
|
self.target_t1_size = max(self.target_t1_size - delta, 0)
|
||||||
|
self.b2.move_to_end(key)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
@override
|
||||||
|
def on_store_miss(
|
||||||
|
self, keys: Iterable[OffloadKey], req_context: ReqContext
|
||||||
|
) -> None:
|
||||||
|
for key in reversed(list(keys)):
|
||||||
|
self._adapt_to_ghost_hit(key)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
||||||
for key in reversed(list(keys)):
|
for key in reversed(list(keys)):
|
||||||
@@ -86,19 +109,33 @@ class ARCCachePolicy(CachePolicy):
|
|||||||
elif key in self.t2:
|
elif key in self.t2:
|
||||||
self.t2.move_to_end(key)
|
self.t2.move_to_end(key)
|
||||||
|
|
||||||
elif key in self.b1:
|
else:
|
||||||
delta = max(1, len(self.b2) / len(self.b1))
|
self._adapt_to_ghost_hit(key)
|
||||||
self.target_t1_size = min(
|
|
||||||
self.target_t1_size + delta, self.cache_capacity
|
|
||||||
)
|
|
||||||
# move to MRU position (end) to keep it fresh in the ghost list
|
|
||||||
self.b1.move_to_end(key)
|
|
||||||
|
|
||||||
elif key in self.b2:
|
@override
|
||||||
delta = max(1, len(self.b1) / len(self.b2))
|
def on_request_finished(
|
||||||
self.target_t1_size = max(self.target_t1_size - delta, 0)
|
self,
|
||||||
# move to MRU position (end) to keep it fresh in the ghost list
|
key_groups: Sequence[Sequence[OffloadKey]],
|
||||||
self.b2.move_to_end(key)
|
insertion_only_keys: set[OffloadKey],
|
||||||
|
reused_keys: set[OffloadKey],
|
||||||
|
req_context: ReqContext,
|
||||||
|
) -> None:
|
||||||
|
for key in reversed(order_request_keys(key_groups, req_context)):
|
||||||
|
if key in insertion_only_keys:
|
||||||
|
# A store is not a frequency hit. Preserve T1 membership
|
||||||
|
# while still restoring tail-to-head recency.
|
||||||
|
if key in self.t1:
|
||||||
|
self.t1.move_to_end(key)
|
||||||
|
elif key in self.t2:
|
||||||
|
self.t2.move_to_end(key)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Ready chunks reused by this request count as one access.
|
||||||
|
if key in reused_keys and key in self.t1:
|
||||||
|
chunk = self.t1.pop(key)
|
||||||
|
self.t2[key] = chunk
|
||||||
|
elif key in reused_keys and key in self.t2:
|
||||||
|
self.t2.move_to_end(key)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def clear(self) -> None:
|
def clear(self) -> None:
|
||||||
|
|||||||
@@ -2,11 +2,26 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
import ctypes
|
import ctypes
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable, Sequence
|
||||||
|
|
||||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||||
|
|
||||||
|
|
||||||
|
def order_request_keys(
|
||||||
|
key_groups: Sequence[Sequence[OffloadKey]], req_context: ReqContext
|
||||||
|
) -> list[OffloadKey]:
|
||||||
|
"""Return one head-to-tail order across all KV cache groups."""
|
||||||
|
keys = [key for group in key_groups for key in group]
|
||||||
|
positions = {
|
||||||
|
key: position
|
||||||
|
for key in keys
|
||||||
|
if (position := req_context.get_offload_key_position(key)) is not None
|
||||||
|
}
|
||||||
|
if len(positions) == len(keys):
|
||||||
|
keys.sort(key=positions.__getitem__)
|
||||||
|
return keys
|
||||||
|
|
||||||
|
|
||||||
class ChunkStatus(ctypes.Structure):
|
class ChunkStatus(ctypes.Structure):
|
||||||
"""
|
"""
|
||||||
Offloading status for a single chunk of KV data.
|
Offloading status for a single chunk of KV data.
|
||||||
@@ -66,6 +81,41 @@ class CachePolicy(ABC):
|
|||||||
req_context: Per-request context for the request touching these chunks.
|
req_context: Per-request context for the request touching these chunks.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
def on_store_miss(
|
||||||
|
self, keys: Iterable[OffloadKey], req_context: ReqContext
|
||||||
|
) -> None:
|
||||||
|
"""Observe store misses before their cache entries are inserted.
|
||||||
|
|
||||||
|
The default delegates to ``touch`` for compatibility with external
|
||||||
|
policies. Policies may override this hook when a miss has distinct
|
||||||
|
semantics, such as ARC adapting to a ghost-list hit.
|
||||||
|
"""
|
||||||
|
self.touch(keys, req_context)
|
||||||
|
|
||||||
|
def on_request_finished(
|
||||||
|
self,
|
||||||
|
key_groups: Sequence[Sequence[OffloadKey]],
|
||||||
|
insertion_only_keys: set[OffloadKey],
|
||||||
|
reused_keys: set[OffloadKey],
|
||||||
|
req_context: ReqContext,
|
||||||
|
) -> None:
|
||||||
|
"""Apply one request-scoped cache access in prefix order.
|
||||||
|
|
||||||
|
``key_groups`` contains keys observed for each KV cache group in
|
||||||
|
head-to-tail order. ``insertion_only_keys`` and ``reused_keys``
|
||||||
|
distinguish chunks only created by this request from ready chunks it
|
||||||
|
actually reused, which matters for policies such as ARC where reuse
|
||||||
|
changes frequency but insertion and pending observations do not.
|
||||||
|
|
||||||
|
The default forwards one touch per group, preserving compatibility
|
||||||
|
for experimental out-of-tree policies while moving those touches to
|
||||||
|
request finalization. Policies that distinguish insertion from reuse
|
||||||
|
can override this hook and inspect the access classifications.
|
||||||
|
"""
|
||||||
|
del insertion_only_keys, reused_keys
|
||||||
|
for keys in key_groups:
|
||||||
|
self.touch(keys, req_context)
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def evict(
|
def evict(
|
||||||
self, n: int, protected: set[OffloadKey]
|
self, n: int, protected: set[OffloadKey]
|
||||||
|
|||||||
@@ -1,28 +1,74 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from collections import OrderedDict
|
import heapq
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable, Sequence
|
||||||
|
|
||||||
from typing_extensions import override
|
from typing_extensions import override
|
||||||
|
|
||||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||||
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
|
from vllm.v1.kv_offload.cpu.policies.base import (
|
||||||
|
CachePolicy,
|
||||||
|
ChunkStatus,
|
||||||
|
order_request_keys,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class LRUCachePolicy(CachePolicy):
|
class LRUCachePolicy(CachePolicy):
|
||||||
"""
|
"""
|
||||||
LRU Caching policy that keeps a dedicated evictable list for fast eviction.
|
LRU caching policy with logical recency independent of transfer pinning.
|
||||||
|
|
||||||
|
Evictable chunks live in a lazy-invalidating min-heap. A chunk's recency
|
||||||
|
can therefore be updated while it is pinned; when it later becomes
|
||||||
|
evictable, it enters the heap with the order assigned by the request
|
||||||
|
rather than its transfer-completion time.
|
||||||
|
|
||||||
A use is indicated by,
|
A use is indicated by,
|
||||||
- First time the key is added (store).
|
- First time the key is added (store).
|
||||||
- Load job completion
|
- A request-scoped access.
|
||||||
- touch
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, cache_capacity: int):
|
def __init__(self, cache_capacity: int):
|
||||||
super().__init__(cache_capacity)
|
super().__init__(cache_capacity)
|
||||||
# Chunks with ref_cnt 0 (not participating in any loads/stores) ordered in LRU
|
|
||||||
self.evictable_chunks: OrderedDict[OffloadKey, None] = OrderedDict()
|
|
||||||
self.chunks: dict[OffloadKey, ChunkStatus] = {}
|
self.chunks: dict[OffloadKey, ChunkStatus] = {}
|
||||||
|
self._ranks: dict[OffloadKey, int] = {}
|
||||||
|
self._evictable: set[OffloadKey] = set()
|
||||||
|
self._heap: list[tuple[int, OffloadKey]] = []
|
||||||
|
self._next_rank = 0
|
||||||
|
|
||||||
|
def _assign_new_rank(self, key: OffloadKey) -> bool:
|
||||||
|
if key not in self.chunks:
|
||||||
|
return False
|
||||||
|
self._next_rank += 1
|
||||||
|
self._ranks[key] = self._next_rank
|
||||||
|
return key in self._evictable
|
||||||
|
|
||||||
|
def _push_evictable(self, key: OffloadKey) -> None:
|
||||||
|
heapq.heappush(
|
||||||
|
self._heap,
|
||||||
|
(self._ranks[key], key),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _is_current(self, entry: tuple[int, OffloadKey]) -> bool:
|
||||||
|
rank, key = entry
|
||||||
|
return key in self._evictable and self._ranks.get(key) == rank
|
||||||
|
|
||||||
|
def _maybe_compact_heap(self) -> None:
|
||||||
|
if len(self._heap) < max(64, 2 * len(self._evictable)):
|
||||||
|
return
|
||||||
|
self._heap = [(self._ranks[key], key) for key in self._evictable]
|
||||||
|
heapq.heapify(self._heap)
|
||||||
|
|
||||||
|
def _update_recency(self, keys: Iterable[OffloadKey]) -> None:
|
||||||
|
updated_evictable = [key for key in keys if self._assign_new_rank(key)]
|
||||||
|
# Rebuilding is linear and substantially cheaper than k heap pushes
|
||||||
|
# for a long prefix. Small updates retain the incremental path.
|
||||||
|
if len(updated_evictable) >= max(64, len(self._evictable) // 4):
|
||||||
|
self._heap = [(self._ranks[key], key) for key in self._evictable]
|
||||||
|
heapq.heapify(self._heap)
|
||||||
|
else:
|
||||||
|
for key in updated_evictable:
|
||||||
|
self._push_evictable(key)
|
||||||
|
self._maybe_compact_heap()
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def get(self, key: OffloadKey) -> ChunkStatus | None:
|
def get(self, key: OffloadKey) -> ChunkStatus | None:
|
||||||
@@ -31,26 +77,40 @@ class LRUCachePolicy(CachePolicy):
|
|||||||
@override
|
@override
|
||||||
def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
|
def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
|
||||||
self.chunks[key] = chunk
|
self.chunks[key] = chunk
|
||||||
|
self._next_rank += 1
|
||||||
|
self._ranks[key] = self._next_rank
|
||||||
if chunk.ref_cnt == 0:
|
if chunk.ref_cnt == 0:
|
||||||
self.evictable_chunks[key] = None
|
self._evictable.add(key)
|
||||||
|
self._push_evictable(key)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def remove(self, key: OffloadKey) -> None:
|
def remove(self, key: OffloadKey) -> None:
|
||||||
del self.chunks[key]
|
del self.chunks[key]
|
||||||
self.evictable_chunks.pop(key, None)
|
self._ranks.pop(key, None)
|
||||||
|
self._evictable.discard(key)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
||||||
for key in reversed(list(keys)):
|
self._update_recency(reversed(list(keys)))
|
||||||
if key in self.evictable_chunks:
|
|
||||||
self.evictable_chunks.move_to_end(key)
|
@override
|
||||||
# active chunks are untouched as they are non-evictable now. They
|
def on_request_finished(
|
||||||
# will eventually reach the end of evictable_chunks when they finish.
|
self,
|
||||||
|
key_groups: Sequence[Sequence[OffloadKey]],
|
||||||
|
insertion_only_keys: set[OffloadKey],
|
||||||
|
reused_keys: set[OffloadKey],
|
||||||
|
req_context: ReqContext,
|
||||||
|
) -> None:
|
||||||
|
del insertion_only_keys, reused_keys
|
||||||
|
self._update_recency(reversed(order_request_keys(key_groups, req_context)))
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def clear(self) -> None:
|
def clear(self) -> None:
|
||||||
self.evictable_chunks.clear()
|
|
||||||
self.chunks.clear()
|
self.chunks.clear()
|
||||||
|
self._ranks.clear()
|
||||||
|
self._evictable.clear()
|
||||||
|
self._heap.clear()
|
||||||
|
self._next_rank = 0
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def evict(
|
def evict(
|
||||||
@@ -59,22 +119,44 @@ class LRUCachePolicy(CachePolicy):
|
|||||||
if n == 0:
|
if n == 0:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
candidates: list[tuple[OffloadKey, ChunkStatus]] = []
|
selected: list[tuple[tuple[int, OffloadKey], ChunkStatus]] = []
|
||||||
for key, _ in self.evictable_chunks.items():
|
selected_keys: set[OffloadKey] = set()
|
||||||
|
deferred: list[tuple[int, OffloadKey]] = []
|
||||||
|
while self._heap and len(selected) < n:
|
||||||
|
entry = heapq.heappop(self._heap)
|
||||||
|
if not self._is_current(entry):
|
||||||
|
continue
|
||||||
|
key = entry[1]
|
||||||
|
# Re-pinning without a recency change can leave an equivalent
|
||||||
|
# lazy entry behind. Select each cache key at most once.
|
||||||
|
if key in selected_keys:
|
||||||
|
continue
|
||||||
if key in protected:
|
if key in protected:
|
||||||
|
deferred.append(entry)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
chunk = self.chunks[key]
|
chunk = self.chunks[key]
|
||||||
assert chunk.ref_cnt == 0
|
assert chunk.ref_cnt == 0
|
||||||
candidates.append((key, chunk))
|
selected.append((entry, chunk))
|
||||||
if len(candidates) == n:
|
selected_keys.add(key)
|
||||||
break
|
|
||||||
|
|
||||||
if len(candidates) < n:
|
if len(selected) < n:
|
||||||
|
for entry, _ in selected:
|
||||||
|
heapq.heappush(self._heap, entry)
|
||||||
|
for entry in deferred:
|
||||||
|
heapq.heappush(self._heap, entry)
|
||||||
return None
|
return None
|
||||||
for key, _ in candidates:
|
|
||||||
del self.evictable_chunks[key]
|
for entry in deferred:
|
||||||
|
heapq.heappush(self._heap, entry)
|
||||||
|
|
||||||
|
candidates: list[tuple[OffloadKey, ChunkStatus]] = []
|
||||||
|
for entry, chunk in selected:
|
||||||
|
key = entry[1]
|
||||||
|
self._evictable.remove(key)
|
||||||
del self.chunks[key]
|
del self.chunks[key]
|
||||||
|
del self._ranks[key]
|
||||||
|
candidates.append((key, chunk))
|
||||||
|
self._maybe_compact_heap()
|
||||||
return candidates
|
return candidates
|
||||||
|
|
||||||
@override
|
@override
|
||||||
@@ -82,9 +164,11 @@ class LRUCachePolicy(CachePolicy):
|
|||||||
# chunks can become evictable when,
|
# chunks can become evictable when,
|
||||||
# store completes - i.e. ref_cnt -1 -> 0 # not in evictable list
|
# store completes - i.e. ref_cnt -1 -> 0 # not in evictable list
|
||||||
# all loads complete - i.e ref_cnt 1 -> 0 # not in evictable list
|
# all loads complete - i.e ref_cnt 1 -> 0 # not in evictable list
|
||||||
self.evictable_chunks[key] = None
|
self._evictable.add(key)
|
||||||
|
self._push_evictable(key)
|
||||||
|
self._maybe_compact_heap()
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def mark_non_evictable(self, key: OffloadKey) -> None:
|
def mark_non_evictable(self, key: OffloadKey) -> None:
|
||||||
# key must have been in the evictable list.
|
# key must have been evictable before it was pinned.
|
||||||
del self.evictable_chunks[key]
|
self._evictable.remove(key)
|
||||||
|
|||||||
@@ -110,13 +110,22 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
|
|||||||
# read/write is for CPU<->secondary transfers,
|
# read/write is for CPU<->secondary transfers,
|
||||||
# load/store is for CPU<->GPU transfers.
|
# load/store is for CPU<->GPU transfers.
|
||||||
# These aliases avoid calling prepare_load inside a store path.
|
# These aliases avoid calling prepare_load inside a store path.
|
||||||
self.prepare_read = self.prepare_load
|
|
||||||
self.complete_read = self.complete_load
|
self.complete_read = self.complete_load
|
||||||
self.prepare_write = self.prepare_store
|
self.prepare_write = self.prepare_store
|
||||||
self.complete_write = self.complete_store
|
self.complete_write = self.complete_store
|
||||||
|
|
||||||
self._kv_memoryview = mmap_region.create_kv_memoryview()
|
self._kv_memoryview = mmap_region.create_kv_memoryview()
|
||||||
|
|
||||||
|
def prepare_read(
|
||||||
|
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||||
|
) -> LoadStoreSpec:
|
||||||
|
"""Pin chunks for a CPU-to-secondary transfer.
|
||||||
|
|
||||||
|
Cascade reads are implementation details of tiering, not additional
|
||||||
|
request accesses, so they must not alter request-scoped recency.
|
||||||
|
"""
|
||||||
|
return self._prepare_load(keys, req_context, record_access=False)
|
||||||
|
|
||||||
def get_kv_memoryview(self) -> memoryview:
|
def get_kv_memoryview(self) -> memoryview:
|
||||||
"""Return the memoryview over the primary tier's KV cache buffer.
|
"""Return the memoryview over the primary tier's KV cache buffer.
|
||||||
|
|
||||||
@@ -781,11 +790,10 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
req_id: str,
|
req_id: str,
|
||||||
exclude_tier_idx: int | None = None,
|
exclude_tier_idx: int | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Finalize secondary tiers once no more store cascades can be submitted.
|
"""Finalize secondary tiers once no more cascades can be submitted.
|
||||||
|
|
||||||
Finalization means forwarding on_request_finished() to secondary tiers.
|
Their finalization is delayed until pending GPU->primary stores
|
||||||
It is delayed until pending GPU->primary stores finish, since their
|
finish, since those callbacks may still submit secondary stores.
|
||||||
complete_store() callbacks may still submit primary->secondary stores.
|
|
||||||
"""
|
"""
|
||||||
state = self._req_state[req_id]
|
state = self._req_state[req_id]
|
||||||
if not state.is_finished:
|
if not state.is_finished:
|
||||||
|
|||||||
Reference in New Issue
Block a user