[Bugfix][KV Offload] Track cache recency once per request (#51787)
Signed-off-by: mindungil <alswnsrlf12@naver.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -258,6 +258,9 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
|
||||
jobs = scheduler._build_partial_tail_store_jobs(output)
|
||||
|
||||
assert len(jobs) == 1
|
||||
offered_keys = scheduler.manager.prepare_store.call_args.args[0]
|
||||
req_context = scheduler._req_status["req"].req_context
|
||||
assert all(req_context.get_offload_key_position(key) == 28 for key in offered_keys)
|
||||
[job_id] = jobs
|
||||
src_spec = jobs[job_id].src_spec
|
||||
assert isinstance(src_spec, GPULoadStoreSpec)
|
||||
@@ -380,6 +383,13 @@ def test_normal_store_excludes_align_mode_mamba_sources():
|
||||
req_status.group_states[0].block_ids[:] = [11]
|
||||
req_status.group_states[1].block_ids[:] = [99]
|
||||
req_status.update_offload_keys()
|
||||
for group_config, group_state in zip(
|
||||
scheduler.config.kv_group_configs, req_status.group_states
|
||||
):
|
||||
assert (
|
||||
req_status.req_context.get_offload_key_position(group_state.offload_keys[0])
|
||||
== group_config.tokens_per_chunk
|
||||
)
|
||||
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -921,7 +931,6 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
runner.run(decoded_tokens=[0] * (tokens_per_chunk + 1))
|
||||
|
||||
# 1 more block (+ token for kicking off offloading)
|
||||
# now check touch was called with all 6 blocks
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -929,9 +938,6 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
decoded_tokens=[0] * (tokens_per_chunk + 1),
|
||||
expected_stored=(15, 16, 17),
|
||||
)
|
||||
runner.manager.touch.assert_called()
|
||||
block_hashes1 = list(runner.manager.touch.call_args.args[0])
|
||||
assert len(block_hashes1) == 6
|
||||
|
||||
# terminate request
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
@@ -939,13 +945,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
# create a new request differing only on the last token
|
||||
runner.new_request(token_ids=[0] * (tokens_per_chunk * 6 - 1) + [1])
|
||||
runner.run(decoded_tokens=[0])
|
||||
runner.manager.touch.assert_called()
|
||||
block_hashes2 = list(runner.manager.touch.call_args.args[0])
|
||||
assert len(block_hashes2) == 6
|
||||
|
||||
# verify hashes are the same, except for the last block
|
||||
assert block_hashes1[:5] == block_hashes2[:5]
|
||||
assert block_hashes1[5] != block_hashes2[5]
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# terminate request
|
||||
runner.run(
|
||||
@@ -1353,14 +1353,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(decoded_tokens=[0])
|
||||
# _touch called from get_num_new_matched_tokens (2 groups) and
|
||||
# _get_reqs_to_store (2 groups) → 4 touch calls total.
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 4
|
||||
assert len(touch_calls[0].args[0]) == 3
|
||||
assert len(touch_calls[1].args[0]) == 3
|
||||
assert len(touch_calls[2].args[0]) == 3
|
||||
assert len(touch_calls[3].args[0]) == 3
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# store 3 more block
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
@@ -1371,9 +1364,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
|
||||
expected_stored=(0, 1, 2, 3, 4, 5),
|
||||
)
|
||||
|
||||
# touch called from _get_reqs_to_store * 3 blocks, once for each group
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 6
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# EOS lands in the last slot of the 7th block (offset 6). No forward pass
|
||||
# writes that slot, so the finishing step declines the block.
|
||||
@@ -1392,13 +1383,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
|
||||
expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)),
|
||||
)
|
||||
|
||||
# 2 touch calls from get_num_new_matched_tokens (2 groups)
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 2
|
||||
# full attention group touched all 3 blocks
|
||||
assert len(touch_calls[0].args[0]) == 3
|
||||
# sliding window group touched just the last 2 blocks
|
||||
assert len(touch_calls[1].args[0]) == 2
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# 3 blocks are hit on GPU [0, 1, 2]
|
||||
# 1 block loaded [3,]
|
||||
@@ -1464,15 +1449,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(decoded_tokens=[0])
|
||||
# _touch called from get_num_new_matched_tokens (2 groups) and
|
||||
# _get_reqs_to_store (2 groups) → 4 touch calls total.
|
||||
# Group 0 has 2 offload keys, group 1 has 1.
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 4
|
||||
assert len(touch_calls[0].args[0]) == 2
|
||||
assert len(touch_calls[1].args[0]) == 1
|
||||
assert len(touch_calls[2].args[0]) == 2
|
||||
assert len(touch_calls[3].args[0]) == 1
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# Get to 31 tokens
|
||||
# No further blocks offloaded
|
||||
@@ -1482,11 +1459,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
||||
# Group 0 blocks: [0, 1], ending_token_offset = 24
|
||||
# Group 1 blocks: [0, 1], ending_token_offset = 32
|
||||
runner.run(decoded_tokens=[0])
|
||||
# _get_reqs_to_store touch: only group 1 has a new block to store
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 2
|
||||
assert len(touch_calls[0].args[0]) == 2
|
||||
assert len(touch_calls[1].args[0]) == 2
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# Get to 35 tokens
|
||||
# No further blocks offloaded
|
||||
@@ -1496,11 +1469,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
||||
# Group 0 blocks: [0, 1, 2], ending_token_offset = 36
|
||||
# Group 1 blocks: [0, 1], ending_token_offset = 32
|
||||
runner.run(decoded_tokens=[0])
|
||||
# _get_reqs_to_store touch: only group 0 has a new block to store
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 2
|
||||
assert len(touch_calls[0].args[0]) == 3
|
||||
assert len(touch_calls[1].args[0]) == 2
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# Get to 47 tokens
|
||||
# No further blocks offloaded
|
||||
@@ -1510,11 +1479,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
||||
# Group 0 blocks: [0, 1, 2, 3], ending_token_offset = 4
|
||||
# Group 1 blocks: [0, 1, 2], ending_token_offset = 48
|
||||
runner.run(decoded_tokens=[0])
|
||||
# _get_reqs_to_store touch: both groups have a new block, each with 1 key
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 2
|
||||
assert len(touch_calls[0].args[0]) == 4
|
||||
assert len(touch_calls[1].args[0]) == 3
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
runner.run(decoded_tokens=[0], expected_stored=((0, 3), (1, 2)))
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.v1.kv_offload.cpu.common import (
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
||||
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
|
||||
|
||||
|
||||
def make_req_context(
|
||||
@@ -430,31 +431,34 @@ def test_cpu_manager():
|
||||
assert cpu_manager.lookup(to_key(0), _EMPTY_REQ_CTX) is LookupResult.MISS
|
||||
|
||||
# prepare load [2, 3]
|
||||
prepare_load_output = cpu_manager.prepare_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
|
||||
load_ctx = make_req_context("load-2-3")
|
||||
prepare_load_output = cpu_manager.prepare_load(to_keys([2, 3]), load_ctx)
|
||||
verify_load_output(prepare_load_output, [1, 2])
|
||||
|
||||
# prepare store with no space ([2, 3] is being loaded)
|
||||
assert cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX) is None
|
||||
|
||||
# complete load [2, 3]. Load changes the eviction list, making 2, 3 recent.
|
||||
cpu_manager.complete_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_load(to_keys([2, 3]), load_ctx)
|
||||
cpu_manager.on_request_finished(load_ctx)
|
||||
|
||||
# prepare store [6, 7, 8] -> evicts [4, 5, 2] (oldest)
|
||||
# prepare store [6, 7, 8] -> evicts [4, 5, 3] (oldest). Within
|
||||
# the accessed prefix [2, 3], the tail is less valuable than the head.
|
||||
prepare_store_output = cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX)
|
||||
verify_store_output(
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[6, 7, 8],
|
||||
store_chunk_ids=[1, 0, 3],
|
||||
evicted_keys=[4, 5, 2],
|
||||
store_chunk_ids=[2, 0, 3],
|
||||
evicted_keys=[4, 5, 3],
|
||||
),
|
||||
)
|
||||
|
||||
# complete store [6, 7, 8]
|
||||
cpu_manager.complete_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX)
|
||||
|
||||
# touch [3, 6, 7] (move to end of LRU order)
|
||||
cpu_manager.touch(to_keys([3, 6, 7]), _EMPTY_REQ_CTX)
|
||||
# touch [2, 6, 7] (move to end of LRU order)
|
||||
cpu_manager.touch(to_keys([2, 6, 7]), _EMPTY_REQ_CTX)
|
||||
|
||||
# prepare store [7, 9] -> evicts [8] (oldest following previous touch)
|
||||
prepare_store_output = cpu_manager.prepare_store(to_keys([9]), _EMPTY_REQ_CTX)
|
||||
@@ -477,7 +481,7 @@ def test_cpu_manager():
|
||||
verify_events(
|
||||
cpu_manager.take_events(),
|
||||
expected_stores=({3, 4, 5}, {6, 7, 8}),
|
||||
expected_evictions=({4, 5, 2}, {8}),
|
||||
expected_evictions=({4, 5, 3}, {8}),
|
||||
)
|
||||
|
||||
|
||||
@@ -518,6 +522,65 @@ def test_prepare_load_preserves_key_order():
|
||||
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
|
||||
|
||||
|
||||
def test_lru_batch_eviction_failure_is_atomic():
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
policy = manager._policy
|
||||
assert isinstance(policy, LRUCachePolicy)
|
||||
keys = to_keys([1, 2, 3, 4])
|
||||
assert manager.prepare_store(keys, _EMPTY_REQ_CTX) is not None
|
||||
manager.complete_store(keys, _EMPTY_REQ_CTX)
|
||||
|
||||
protected = set(keys[1:])
|
||||
assert policy.evict(2, protected) is None
|
||||
assert all(manager.lookup(key, _EMPTY_REQ_CTX) is LookupResult.HIT for key in keys)
|
||||
|
||||
evicted = policy.evict(1, protected)
|
||||
assert evicted is not None
|
||||
assert [key for key, _ in evicted] == [keys[0]]
|
||||
|
||||
|
||||
def test_lru_repeated_pin_cycles_compact_lazy_heap_entries():
|
||||
manager = make_cpu_manager(num_chunks=1, cache_policy="lru")
|
||||
policy = manager._policy
|
||||
assert isinstance(policy, LRUCachePolicy)
|
||||
key = to_key(1)
|
||||
assert manager.prepare_store([key], _EMPTY_REQ_CTX) is not None
|
||||
manager.complete_store([key], _EMPTY_REQ_CTX)
|
||||
|
||||
for _ in range(128):
|
||||
manager.prepare_load([key], _EMPTY_REQ_CTX)
|
||||
manager.complete_load([key], _EMPTY_REQ_CTX)
|
||||
|
||||
assert len(policy._heap) < 64
|
||||
evicted = policy.evict(1, set())
|
||||
assert evicted is not None
|
||||
assert [evicted_key for evicted_key, _ in evicted] == [key]
|
||||
|
||||
|
||||
def test_reset_discards_stale_request_access_classification():
|
||||
manager = make_cpu_manager(num_chunks=1, cache_policy="arc")
|
||||
policy = manager._policy
|
||||
assert isinstance(policy, ARCCachePolicy)
|
||||
key = to_key(1)
|
||||
resumed_ctx = make_req_context("resumed")
|
||||
|
||||
assert manager.prepare_store([key], resumed_ctx) is not None
|
||||
manager.complete_store([key], resumed_ctx)
|
||||
manager.reset_cache()
|
||||
|
||||
replacement_ctx = make_req_context("replacement")
|
||||
assert manager.prepare_store([key], replacement_ctx) is not None
|
||||
manager.complete_store([key], replacement_ctx)
|
||||
manager.on_request_finished(replacement_ctx)
|
||||
assert key in policy.t1
|
||||
|
||||
manager.prepare_load([key], resumed_ctx)
|
||||
manager.complete_load([key], resumed_ctx)
|
||||
manager.on_request_finished(resumed_ctx)
|
||||
|
||||
assert key in policy.t2
|
||||
|
||||
|
||||
class TestARCPolicy:
|
||||
"""Unit tests for CPUOffloadingManager with ARC eviction policy."""
|
||||
|
||||
@@ -1042,6 +1105,48 @@ def test_filter_reused_manager_oversized_offer_makes_progress():
|
||||
assert stored_keys == set(keys)
|
||||
|
||||
|
||||
def test_store_threshold_does_not_hide_ready_reuse():
|
||||
manager = make_cpu_manager(
|
||||
num_chunks=2,
|
||||
cache_policy="lru",
|
||||
store_threshold=2,
|
||||
max_tracker_size=2,
|
||||
)
|
||||
head, tail, noise_1, noise_2, replacement = to_keys(list(range(5)))
|
||||
|
||||
seed_ctx = make_req_context("seed")
|
||||
output = manager.prepare_store([head, tail], seed_ctx)
|
||||
assert output is not None
|
||||
assert not output.keys_to_store
|
||||
output = manager.prepare_store([head, tail], seed_ctx)
|
||||
assert output is not None
|
||||
assert output.keys_to_store == [head, tail]
|
||||
manager.complete_store([head, tail], seed_ctx)
|
||||
manager.on_request_finished(seed_ctx)
|
||||
|
||||
# Age both resident keys out of the admission tracker without storing the
|
||||
# noise keys. Reusing the tail recreates its counter at one, below the
|
||||
# threshold, but must still refresh its cache recency.
|
||||
output = manager.prepare_store([noise_1, noise_2], make_req_context("noise"))
|
||||
assert output is not None
|
||||
assert not output.keys_to_store
|
||||
reuse_ctx = make_req_context("reuse")
|
||||
skipped_before_reuse = manager.stores_skipped_in_current_batch
|
||||
output = manager.prepare_store([tail], reuse_ctx)
|
||||
assert output is not None
|
||||
assert not output.keys_to_store
|
||||
assert manager.stores_skipped_in_current_batch == skipped_before_reuse
|
||||
manager.on_request_finished(reuse_ctx)
|
||||
|
||||
replacement_ctx = make_req_context("replacement")
|
||||
output = manager.prepare_store([replacement], replacement_ctx)
|
||||
assert output is not None
|
||||
assert not output.keys_to_store
|
||||
output = manager.prepare_store([replacement], replacement_ctx)
|
||||
assert output is not None
|
||||
assert output.evicted_keys == [head]
|
||||
|
||||
|
||||
def test_evictable_cache_chunk_count():
|
||||
"""
|
||||
Verifies _num_evictable_cache_chunks is maintained correctly through the
|
||||
@@ -1156,3 +1261,318 @@ def test_touch_forwards_req_context_to_policy(monkeypatch):
|
||||
assert len(received) == 1
|
||||
assert received[0][0] == keys
|
||||
assert received[0][1] is ctx
|
||||
|
||||
|
||||
@pytest.mark.parametrize("finish_before_completion", [False, True])
|
||||
def test_request_finish_orders_lru_prefix_independent_of_store_completion(
|
||||
finish_before_completion: bool,
|
||||
):
|
||||
"""A transfer's completion order must not make the prefix head LRU."""
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
prefix = to_keys([1, 2, 3, 4])
|
||||
ctx = make_req_context("prefix")
|
||||
|
||||
output = manager.prepare_store(prefix, ctx)
|
||||
assert output is not None
|
||||
if finish_before_completion:
|
||||
manager.on_request_finished(ctx)
|
||||
manager.complete_store(set(prefix), ctx)
|
||||
if not finish_before_completion:
|
||||
manager.on_request_finished(ctx)
|
||||
|
||||
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
|
||||
assert output is not None
|
||||
assert output.evicted_keys == to_keys([4])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("finish_before_completion", [False, True])
|
||||
def test_request_finish_orders_lru_prefix_independent_of_load_completion(
|
||||
finish_before_completion: bool,
|
||||
):
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
prefix = to_keys([1, 2, 3, 4])
|
||||
|
||||
seed_ctx = make_req_context("seed")
|
||||
output = manager.prepare_store(prefix, seed_ctx)
|
||||
assert output is not None
|
||||
manager.complete_store(prefix, seed_ctx)
|
||||
|
||||
reuse_ctx = make_req_context("reuse")
|
||||
manager.prepare_load(prefix, reuse_ctx)
|
||||
if finish_before_completion:
|
||||
manager.on_request_finished(reuse_ctx)
|
||||
manager.complete_load(set(prefix), reuse_ctx)
|
||||
if not finish_before_completion:
|
||||
manager.on_request_finished(reuse_ctx)
|
||||
|
||||
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
|
||||
assert output is not None
|
||||
assert output.evicted_keys == to_keys([4])
|
||||
|
||||
|
||||
def test_late_old_completion_does_not_override_newer_request_recency():
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
old_keys = to_keys([1, 2])
|
||||
new_keys = to_keys([3, 4])
|
||||
|
||||
old_ctx = make_req_context("old")
|
||||
assert manager.prepare_store(old_keys, old_ctx) is not None
|
||||
manager.on_request_finished(old_ctx)
|
||||
|
||||
new_ctx = make_req_context("new")
|
||||
assert manager.prepare_store(new_keys, new_ctx) is not None
|
||||
manager.complete_store(new_keys, new_ctx)
|
||||
manager.on_request_finished(new_ctx)
|
||||
|
||||
# Physical completion is late, but the logical access is still older.
|
||||
manager.complete_store(old_keys, old_ctx)
|
||||
|
||||
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
|
||||
assert output is not None
|
||||
assert output.evicted_keys == [old_keys[-1]]
|
||||
|
||||
|
||||
def test_arc_counts_reuse_once_and_keeps_insertions_in_t1():
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="arc")
|
||||
policy = manager._policy
|
||||
assert isinstance(policy, ARCCachePolicy)
|
||||
prefix = to_keys([1, 2, 3, 4])
|
||||
|
||||
seed_ctx = make_req_context("seed")
|
||||
assert manager.prepare_store(prefix, seed_ctx) is not None
|
||||
manager.complete_store(prefix, seed_ctx)
|
||||
# Repeated store offers from the same request must not become ARC hits.
|
||||
assert manager.prepare_store(prefix, seed_ctx) is not None
|
||||
# Nor should an internal read used to cascade the new chunks to a tier.
|
||||
manager.prepare_load(prefix, seed_ctx)
|
||||
manager.complete_load(prefix, seed_ctx)
|
||||
manager.on_request_finished(seed_ctx)
|
||||
|
||||
assert list(policy.t1) == list(reversed(prefix))
|
||||
assert not policy.t2
|
||||
|
||||
reuse_ctx = make_req_context("reuse")
|
||||
manager.prepare_load(prefix, reuse_ctx)
|
||||
manager.complete_load(prefix, reuse_ctx)
|
||||
# A duplicate internal pin, as used by tiering cascades, is deduplicated.
|
||||
manager.prepare_load(prefix, reuse_ctx)
|
||||
manager.complete_load(prefix, reuse_ctx)
|
||||
manager.on_request_finished(reuse_ctx)
|
||||
|
||||
assert not policy.t1
|
||||
assert list(policy.t2) == list(reversed(prefix))
|
||||
|
||||
|
||||
def test_arc_reuse_wins_if_same_request_later_reinserts_key():
|
||||
manager = make_cpu_manager(num_chunks=1, cache_policy="arc")
|
||||
policy = manager._policy
|
||||
assert isinstance(policy, ARCCachePolicy)
|
||||
key_1, key_2 = to_keys([1, 2])
|
||||
|
||||
seed_ctx = make_req_context("seed")
|
||||
assert manager.prepare_store([key_1], seed_ctx) is not None
|
||||
manager.complete_store([key_1], seed_ctx)
|
||||
manager.on_request_finished(seed_ctx)
|
||||
|
||||
reuse_ctx = make_req_context("reuse-and-reinsert")
|
||||
manager.prepare_load([key_1], reuse_ctx)
|
||||
manager.complete_load([key_1], reuse_ctx)
|
||||
|
||||
other_ctx = make_req_context("evict-reused-key")
|
||||
assert manager.prepare_store([key_2], other_ctx) is not None
|
||||
manager.complete_store([key_2], other_ctx)
|
||||
manager.on_request_finished(other_ctx)
|
||||
|
||||
assert manager.prepare_store([key_1], reuse_ctx) is not None
|
||||
manager.complete_store([key_1], reuse_ctx)
|
||||
manager.on_request_finished(reuse_ctx)
|
||||
|
||||
assert key_1 not in policy.t1
|
||||
assert list(policy.t2) == [key_1]
|
||||
|
||||
|
||||
def test_request_finish_forwards_grouped_deduplicated_accesses(monkeypatch):
|
||||
manager = make_cpu_manager(num_chunks=5)
|
||||
existing = make_offload_key(b"existing", 0)
|
||||
group_0_new = make_offload_key(b"group-0-new", 0)
|
||||
group_1_head = make_offload_key(b"group-1-head", 1)
|
||||
group_1_tail = make_offload_key(b"group-1-tail", 1)
|
||||
|
||||
seed_ctx = make_req_context("seed")
|
||||
assert manager.prepare_store([existing], seed_ctx) is not None
|
||||
manager.complete_store([existing], seed_ctx)
|
||||
|
||||
ctx = make_req_context("grouped")
|
||||
# Group 1 is observed first, but policy finalization follows group index.
|
||||
offered = [group_1_head, existing, group_0_new]
|
||||
assert manager.prepare_store(offered, ctx) is not None
|
||||
# Later offers extend a group's prefix and may repeat earlier keys.
|
||||
assert (
|
||||
manager.prepare_store([group_0_new, group_1_head, group_1_tail], ctx)
|
||||
is not None
|
||||
)
|
||||
|
||||
received = []
|
||||
|
||||
def on_request_finished(key_groups, insertion_only_keys, reused_keys, req_context):
|
||||
received.append((key_groups, insertion_only_keys, reused_keys, req_context))
|
||||
|
||||
monkeypatch.setattr(manager._policy, "on_request_finished", on_request_finished)
|
||||
manager.on_request_finished(ctx)
|
||||
manager.on_request_finished(ctx)
|
||||
|
||||
assert received == [
|
||||
(
|
||||
((existing, group_0_new), (group_1_head, group_1_tail)),
|
||||
{group_0_new, group_1_head, group_1_tail},
|
||||
{existing},
|
||||
ctx,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_failed_store_records_access_once_and_uses_store_miss_hook(monkeypatch):
|
||||
manager = make_cpu_manager(num_chunks=1)
|
||||
resident, candidate = to_keys([1, 2])
|
||||
|
||||
seed_ctx = make_req_context("seed")
|
||||
assert manager.prepare_store([resident], seed_ctx) is not None
|
||||
manager.complete_store([resident], seed_ctx)
|
||||
|
||||
pin_ctx = make_req_context("pin")
|
||||
manager.prepare_load([resident], pin_ctx)
|
||||
|
||||
record_calls = 0
|
||||
original_record = manager._record_request_cache_access
|
||||
|
||||
def record_access(*args, **kwargs):
|
||||
nonlocal record_calls
|
||||
record_calls += 1
|
||||
return original_record(*args, **kwargs)
|
||||
|
||||
store_miss_calls = []
|
||||
|
||||
def on_store_miss(keys, req_context):
|
||||
store_miss_calls.append((list(keys), req_context))
|
||||
|
||||
finalized = []
|
||||
|
||||
def on_request_finished(key_groups, insertion_only_keys, reused_keys, req_context):
|
||||
finalized.append((key_groups, insertion_only_keys, reused_keys, req_context))
|
||||
|
||||
monkeypatch.setattr(manager, "_record_request_cache_access", record_access)
|
||||
monkeypatch.setattr(manager._policy, "on_store_miss", on_store_miss)
|
||||
monkeypatch.setattr(manager._policy, "on_request_finished", on_request_finished)
|
||||
|
||||
ctx = make_req_context("failed-store")
|
||||
assert manager.prepare_store([candidate, resident], ctx) is None
|
||||
assert record_calls == 1
|
||||
assert store_miss_calls == [([candidate], ctx)]
|
||||
|
||||
manager.on_request_finished(ctx)
|
||||
assert finalized == [
|
||||
(((candidate, resident),), set(), {resident}, ctx),
|
||||
]
|
||||
|
||||
manager.complete_load([resident], pin_ctx)
|
||||
|
||||
|
||||
def test_arc_ghost_hit_adapts_once_per_request_before_insertion():
|
||||
manager = make_cpu_manager(num_chunks=2, cache_policy="arc")
|
||||
policy = manager._policy
|
||||
assert isinstance(policy, ARCCachePolicy)
|
||||
keys = to_keys([1, 2, 3])
|
||||
|
||||
seed_ctx = make_req_context("seed")
|
||||
assert manager.prepare_store(keys[:2], seed_ctx) is not None
|
||||
manager.complete_store(keys[:2], seed_ctx)
|
||||
manager.on_request_finished(seed_ctx)
|
||||
|
||||
evict_ctx = make_req_context("create-ghost")
|
||||
output = manager.prepare_store([keys[2]], evict_ctx)
|
||||
assert output is not None
|
||||
[ghost_key] = output.evicted_keys
|
||||
manager.complete_store([keys[2]], evict_ctx)
|
||||
manager.on_request_finished(evict_ctx)
|
||||
assert ghost_key in policy.b1
|
||||
|
||||
resident_keys = [key for key in keys if key != ghost_key]
|
||||
pin_ctx = make_req_context("pin-residents")
|
||||
manager.prepare_load(resident_keys, pin_ctx)
|
||||
|
||||
ghost_ctx = make_req_context("ghost-hit")
|
||||
target_before = policy.target_t1_size
|
||||
assert manager.prepare_store([ghost_key], ghost_ctx) is None
|
||||
target_after_first_offer = policy.target_t1_size
|
||||
assert target_after_first_offer > target_before
|
||||
assert manager.prepare_store([ghost_key], ghost_ctx) is None
|
||||
assert policy.target_t1_size == target_after_first_offer
|
||||
|
||||
manager.complete_load(resident_keys, pin_ctx)
|
||||
output = manager.prepare_store([ghost_key], ghost_ctx)
|
||||
assert output is not None
|
||||
assert policy.target_t1_size == target_after_first_offer
|
||||
manager.complete_store([ghost_key], ghost_ctx)
|
||||
manager.on_request_finished(ghost_ctx)
|
||||
assert ghost_key in policy.t1
|
||||
assert ghost_key not in policy.t2
|
||||
|
||||
|
||||
def test_arc_pending_chunk_is_not_a_frequency_hit():
|
||||
manager = make_cpu_manager(num_chunks=2, cache_policy="arc")
|
||||
policy = manager._policy
|
||||
assert isinstance(policy, ARCCachePolicy)
|
||||
key = to_key(1)
|
||||
|
||||
writer_ctx = make_req_context("writer")
|
||||
assert manager.prepare_store([key], writer_ctx) is not None
|
||||
|
||||
observer_ctx = make_req_context("pending-observer")
|
||||
output = manager.prepare_store([key], observer_ctx)
|
||||
assert output is not None and not output.keys_to_store
|
||||
manager.on_request_finished(observer_ctx)
|
||||
|
||||
chunk = policy.get(key)
|
||||
assert chunk is not None and not chunk.is_ready
|
||||
assert key in policy.t1
|
||||
assert key not in policy.t2
|
||||
|
||||
|
||||
def test_request_key_positions_override_store_observation_order():
|
||||
manager = make_cpu_manager(num_chunks=2, cache_policy="lru")
|
||||
head, tail = to_keys([1, 2])
|
||||
ctx = make_req_context("out-of-order-store")
|
||||
ctx.set_offload_key_position(head, 16)
|
||||
ctx.set_offload_key_position(tail, 32)
|
||||
|
||||
# A backfill or partial-tail path may offer a later key first.
|
||||
assert manager.prepare_store([tail], ctx) is not None
|
||||
assert manager.prepare_store([head], ctx) is not None
|
||||
manager.on_request_finished(ctx)
|
||||
manager.complete_store({head, tail}, ctx)
|
||||
|
||||
output = manager.prepare_store([to_key(3)], make_req_context("evict-tail"))
|
||||
assert output is not None
|
||||
assert output.evicted_keys == [tail]
|
||||
|
||||
|
||||
def test_request_key_positions_order_tails_across_kv_groups():
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
group_0_head = make_offload_key(b"group-0-head", 0)
|
||||
group_0_tail = make_offload_key(b"group-0-tail", 0)
|
||||
group_1_head = make_offload_key(b"group-1-head", 1)
|
||||
group_1_tail = make_offload_key(b"group-1-tail", 1)
|
||||
keys = [group_0_head, group_0_tail, group_1_head, group_1_tail]
|
||||
ctx = make_req_context("hybrid-groups")
|
||||
for key in (group_0_head, group_1_head):
|
||||
ctx.set_offload_key_position(key, 16)
|
||||
for key in (group_0_tail, group_1_tail):
|
||||
ctx.set_offload_key_position(key, 32)
|
||||
|
||||
assert manager.prepare_store(keys, ctx) is not None
|
||||
manager.complete_store(keys, ctx)
|
||||
manager.on_request_finished(ctx)
|
||||
|
||||
output = manager.prepare_store(to_keys([10, 11]), make_req_context("evict"))
|
||||
assert output is not None
|
||||
assert set(output.evicted_keys) == {group_0_tail, group_1_tail}
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.v1.kv_offload.base import (
|
||||
TierMatcher,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobResult,
|
||||
SecondaryTierManager,
|
||||
@@ -753,14 +754,64 @@ class TestTieringOffloadingManager:
|
||||
# Touch chunks
|
||||
self.manager.touch(chunks, _CTX)
|
||||
|
||||
# Verify touch was called on primary tier (check LRU order)
|
||||
primary_keys = list(self.primary_tier._policy.evictable_chunks.keys())
|
||||
assert primary_keys[-3:] == list(reversed(chunks))
|
||||
# Verify touch was called on the primary tier (head is most recent).
|
||||
policy = self.primary_tier._policy
|
||||
assert isinstance(policy, LRUCachePolicy)
|
||||
ranks = policy._ranks
|
||||
assert ranks[chunks[2]] < ranks[chunks[1]] < ranks[chunks[0]]
|
||||
|
||||
# Verify touch was propagated to all secondary tiers
|
||||
self.secondary_tier1.touch.assert_called_once_with(chunks, _CTX)
|
||||
self.secondary_tier2.touch.assert_called_once_with(chunks, _CTX)
|
||||
|
||||
def test_request_order_survives_late_cascade_completion(self, manager_setup):
|
||||
"""Cascade completion must not replace request order with I/O order."""
|
||||
chunks = to_keys(range(5))
|
||||
self._start_request()
|
||||
output = self.manager.prepare_store(chunks, _CTX)
|
||||
assert output is not None
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
|
||||
# Finalize while both secondary-tier cascades still pin primary chunks.
|
||||
self.manager.on_request_finished(_CTX)
|
||||
self._simulate_on_schedule_end()
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
evict_ctx = ReqContext(req_id="evict")
|
||||
self._start_request(evict_ctx)
|
||||
output = self.manager.prepare_store(to_keys([5]), evict_ctx)
|
||||
assert output is not None
|
||||
assert output.evicted_keys == [chunks[-1]]
|
||||
|
||||
def test_late_old_store_submission_does_not_override_newer_request_order(
|
||||
self, manager_setup
|
||||
):
|
||||
"""Cascade submission after finish is not a new primary access."""
|
||||
old_blocks = to_keys(range(2))
|
||||
old_ctx = ReqContext(req_id="old")
|
||||
self._start_request(old_ctx)
|
||||
assert self.manager.prepare_store(old_blocks, old_ctx) is not None
|
||||
self.manager.on_request_finished(old_ctx)
|
||||
|
||||
new_blocks = to_keys(range(2, 5))
|
||||
new_ctx = ReqContext(req_id="new")
|
||||
self._start_request(new_ctx)
|
||||
assert self.manager.prepare_store(new_blocks, new_ctx) is not None
|
||||
self.manager.complete_store(new_blocks, new_ctx, success=True)
|
||||
self.manager.on_request_finished(new_ctx)
|
||||
|
||||
# The old GPU->primary write lands after the newer request finished.
|
||||
# Its cascade pins must not turn it into the more recent request.
|
||||
self.manager.complete_store(old_blocks, old_ctx, success=True)
|
||||
self._simulate_on_schedule_end()
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
evict_ctx = ReqContext(req_id="evict")
|
||||
self._start_request(evict_ctx)
|
||||
output = self.manager.prepare_store(to_keys([5]), evict_ctx)
|
||||
assert output is not None
|
||||
assert output.evicted_keys == [old_blocks[-1]]
|
||||
|
||||
def test_failed_store_no_cascade(self, manager_setup):
|
||||
"""Test that failed GPU→primary store doesn't cascade."""
|
||||
chunks = to_keys(range(3))
|
||||
@@ -873,10 +924,10 @@ class TestTieringOffloadingManager:
|
||||
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
||||
assert job_metadata.req_context is ctx
|
||||
|
||||
def test_on_request_finished_delays_secondary_until_store_submitted(
|
||||
def test_on_request_finished_delays_only_secondary_until_store_submitted(
|
||||
self, manager_setup
|
||||
):
|
||||
"""Manager hook is eager; secondary hooks wait for cascade submission."""
|
||||
"""Primary order commits immediately; secondary cleanup waits."""
|
||||
chunks = to_keys(range(2))
|
||||
ctx = ReqContext(req_id="req_delayed_secondary")
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
Reference in New Issue
Block a user