[Core] Preserve Marconi caching with selective hybrid cache retention (#47782)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-07-13 21:24:20 +01:00
committed by GitHub
parent 7dc447dda7
commit 8ac8375270
9 changed files with 501 additions and 235 deletions
@@ -110,7 +110,7 @@ def test_hybrid_mamba_align_partial_hash_hit():
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
blocks = manager.allocate_slots(req0, 6, num_computed, computed_blocks)
assert blocks is not None
@@ -125,7 +125,7 @@ def test_hybrid_mamba_align_partial_hash_hit():
assert partial_mamba_block[0].block_hash_num_tokens == 6
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req1)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 6
assert [len(group) for group in computed_blocks.blocks] == [3, 2]
@@ -185,7 +185,7 @@ def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue():
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -258,7 +258,7 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -279,7 +279,7 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
manager.new_step_starts()
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 4, 4], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req1)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 6
# The later request hits the moved (private-copy) entry, not the source.
assert computed_blocks.get_block_ids()[1][1] == moved_block_id
@@ -338,7 +338,7 @@ def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
manager.new_step_starts()
@@ -351,13 +351,13 @@ def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
# A request hitting the moved entry in the SAME step must be deferred.
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 4, 4], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req1)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 6
assert manager.allocate_slots(req1, 2, num_computed, computed_blocks) is None
# Next step the moved entry is consumable.
manager.new_step_starts()
computed_blocks, num_computed = manager.get_computed_blocks(req1)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 6
assert manager.allocate_slots(req1, 2, num_computed, computed_blocks) is not None
@@ -397,7 +397,7 @@ def test_hybrid_full_attention_partial_hash_hit_uses_cow():
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
manager.free(req0)
@@ -410,7 +410,7 @@ def test_hybrid_full_attention_partial_hash_hit_uses_cow():
assert partial_full_block is not None
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req1)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 6
assert [len(group) for group in computed_blocks.blocks] == [2, 2]
@@ -471,7 +471,7 @@ def test_hybrid_partial_hit_cow_target_starts_uncached():
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
manager.free(req0)
@@ -488,7 +488,7 @@ def test_hybrid_partial_hit_cow_target_starts_uncached():
assert partial_mamba_block is not None
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req1)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 6
new_blocks = manager.allocate_slots(
@@ -586,7 +586,7 @@ def test_hybrid_partial_hash_truncates_full_attention_hit_length():
block_size=block_size,
)
computed_blocks, num_computed = manager.get_computed_blocks(req)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
assert num_computed == 6
assert [len(group) for group in computed_blocks.blocks] == [2, 2]
@@ -627,7 +627,7 @@ def test_cow_retained_blocks_returned_for_release():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
# The owner's move queues a copy and retains both endpoints.
@@ -797,7 +797,7 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
# The owner prefills in scheduler-split style: stop at the block boundary
# (4), then at the prompt's last hash boundary (6, partial entries).
req0 = make_request("0", [7] * 6, hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
req0.num_computed_tokens = 4
manager.new_step_starts()
@@ -808,7 +808,7 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
# A longer request with eagle: full attention drops the partial tail, so
# the joint hit must fall back to the block boundary the FA blocks cover.
req1 = make_request("1", [7] * 6 + [9] * 2, hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req1)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 4
assert all(
len(group) * block_size >= num_computed for group in computed_blocks.blocks
+330 -86
View File
@@ -244,7 +244,7 @@ def test_prefill(hash_fn):
unique_token_ids = [3] * 7
all_token_ids = common_token_ids + unique_token_ids
req0 = make_request("0", all_token_ids, block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert len(req0.block_hashes) == 3
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
@@ -274,7 +274,7 @@ def test_prefill(hash_fn):
# Incomplete 1 block (5 tokens)
unique_token_ids = [3] * 5
req1 = make_request("1", common_token_ids + unique_token_ids, block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert len(req1.block_hashes) == 3
assert computed_blocks.get_block_ids() == ([1, 2, 3],)
assert num_computed_tokens == 3 * 16
@@ -307,7 +307,7 @@ def test_prefill(hash_fn):
# Incomplete 1 block (6 tokens)
unique_token_ids = [3] * 6
req2 = make_request("2", common_token_ids + unique_token_ids, block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
assert len(req2.block_hashes) == 3
assert computed_blocks.get_block_ids() == ([1, 2, 3],)
assert num_computed_tokens == 3 * 16
@@ -327,7 +327,7 @@ def test_prefill(hash_fn):
# Cache miss and eviction.
req3 = make_request("3", [99] * (16 * 10), block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req3)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req3)
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
@@ -369,7 +369,7 @@ def test_prefill_hybrid_model():
unique_token_ids = [3] * 7
all_token_ids = common_token_ids + unique_token_ids
req0 = make_request("0", all_token_ids, block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert len(req0.block_hashes) == 3
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
@@ -405,7 +405,7 @@ def test_prefill_hybrid_model():
unique_token_ids = [3] * 5
all_token_ids = common_token_ids + unique_token_ids
req1 = make_request("1", common_token_ids + unique_token_ids, block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert len(req1.block_hashes) == 3
assert computed_blocks.get_block_ids() == ([1, 2, 3], [0, 6, 7], [0, 10, 11])
assert num_computed_tokens == 3 * 16
@@ -539,7 +539,7 @@ def test_prefill_hybrid_model_eagle():
unique_token_ids = [6] * 7
all_token_ids = common_token_ids + unique_token_ids
req0 = make_request("0", all_token_ids, block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert len(req0.block_hashes) == len(all_token_ids) // block_size
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
@@ -576,7 +576,7 @@ def test_prefill_hybrid_model_eagle():
unique_token_ids = [6] * 5
all_token_ids = common_token_ids + unique_token_ids
req1 = make_request("1", all_token_ids, block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert len(req1.block_hashes) == num_full_blocks
assert computed_blocks.get_block_ids() == (
[1, 2, 3, 4, 5],
@@ -712,7 +712,7 @@ def _test_partial_request_hit(
req = make_request(request_id, prompt_token_ids, block_size, sha256)
for hash_with_group_id in hash_to_evict:
manager.block_pool.cached_block_hash_to_block._cache.pop(hash_with_group_id)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req)
assert len(req.block_hashes) == num_full_blocks
assert num_computed_tokens == expect_hit_length * block_size
for block_per_group in computed_blocks.blocks:
@@ -873,7 +873,7 @@ def test_prefill_hybrid_model_combinations(spec_types: list[str]):
# First request: no cache hit initially
req0 = make_request("0", all_token_ids, block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert len(req0.block_hashes) == 3
assert not computed_blocks.blocks[0] # No cache hit initially
@@ -890,7 +890,7 @@ def test_prefill_hybrid_model_combinations(spec_types: list[str]):
# Second request: should hit cached blocks for common prefix
req1 = make_request("1", common_token_ids + [4] * 5, block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
# Should hit cached blocks for all groups
assert num_computed_tokens == 3 * block_size
@@ -950,7 +950,7 @@ def test_prefill_hybrid_model_combinations_eagle(
# First request: no cache hit initially
req0 = make_request("0", all_token_ids, block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert len(req0.block_hashes) == num_full_blocks
assert not computed_blocks.blocks[0] # No cache hit initially
@@ -966,7 +966,7 @@ def test_prefill_hybrid_model_combinations_eagle(
# Second request: should hit cached blocks for common prefix
all_token_ids = common_token_ids + [6] * 5
req1 = make_request("1", all_token_ids, block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
# Should hit cached blocks for all groups
assert num_computed_tokens == expect_hit_length * block_size
@@ -1019,7 +1019,7 @@ def test_prefill_hybrid_model_mamba_align():
# First request: allocate_slots should not crash with the assertion error
# in MambaManager.cache_blocks() when null blocks are present.
req0 = make_request("0", all_token_ids, block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert num_computed_tokens == 0
blocks = manager.allocate_slots(req0, 55, num_computed_tokens, computed_blocks)
@@ -1048,28 +1048,33 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
# Request: 3 blocks
prefix = [i for i in range(3) for _ in range(block_size)]
req_0 = make_request("0", prefix, block_size, hash_fn)
computed_blocks, num_computed = manager.get_computed_blocks(req_0)
num_uncached_common = manager.coordinator.num_uncached_common_prefix_tokens
computed_blocks, num_computed, shared_prefix_boundary = manager.get_computed_blocks(
req_0
)
assert num_computed == 0 # nothing cached yet
assert num_uncached_common == 0
assert shared_prefix_boundary == 0
manager.allocate_slots(req_0, 3 * block_size, 0, computed_blocks)
# Request: 3 blocks (shared with above) + 7 different tokens
req_1 = make_request("1", prefix + [100] * 7, block_size, hash_fn)
computed_blocks, num_computed = manager.get_computed_blocks(req_1)
num_uncached_common = manager.coordinator.num_uncached_common_prefix_tokens
computed_blocks, num_computed, shared_prefix_boundary = manager.get_computed_blocks(
req_1
)
assert num_computed == 3 * block_size # we should observe a 3-block cache hit
assert num_uncached_common == 0
assert shared_prefix_boundary == 0
manager.allocate_slots(req_1, 7, 3 * block_size, computed_blocks)
# Request: 3 blocks, but only 2 blocks shared (replace the last token in 3rd block):
req_2 = make_request("2", prefix[:-1] + [101], block_size, hash_fn)
computed_blocks, num_computed = manager.get_computed_blocks(req_2)
num_uncached_common = manager.coordinator.num_uncached_common_prefix_tokens
computed_blocks, num_computed, shared_prefix_boundary = manager.get_computed_blocks(
req_2
)
assert num_computed == 0 # mamba_align doesn't cache intermediate blocks
assert num_uncached_common == 2 * block_size # heuristic detects a shared prefix
# boundary == junction: 2 shared blocks (num_computed 0 + 2 uncached blocks).
assert shared_prefix_boundary == 2 * block_size
# Next, validate scheduler logic for num_uncached_common_prefix_tokens > 0
# Next, validate the align-mode Marconi split. It takes the shared-prefix
# junction (absolute) and stops the chunk there (req_2 has num_computed 0).
# Create minimal mock with just the needed attributes
mock = SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size),
@@ -1077,11 +1082,11 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
hash_block_size=block_size,
mamba_partial_cache_hit=False,
)
req_2.shared_prefix_boundary = shared_prefix_boundary
num_new_tokens_adjusted = Scheduler._mamba_block_aligned_split(
self=mock,
request=req_2,
num_new_tokens=3 * block_size,
num_uncached_common_prefix_tokens=num_uncached_common,
)
assert num_new_tokens_adjusted == 2 * block_size # adjust to the common prefix
@@ -1118,7 +1123,7 @@ def test_hybrid_model_mamba_align_with_dynamic_draft_tokens():
all_token_ids = [i for i in range(3) for _ in range(block_size)] + [3] * 7
req0 = make_request("0", all_token_ids, block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
req0, len(all_token_ids), num_computed_tokens, computed_blocks
@@ -1170,7 +1175,7 @@ def test_prefill_plp():
unique_token_ids = [3] * 7
all_token_ids = common_token_ids + unique_token_ids
req0 = make_request("0", all_token_ids, block_size, hash_fn, prompt_logprobs=5)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert len(req0.block_hashes) == 3
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
@@ -1202,7 +1207,7 @@ def test_prefill_plp():
# Incomplete 1 block (5 tokens)
unique_token_ids = [3] * 5
req1 = make_request("1", common_token_ids + unique_token_ids, block_size, hash_fn)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert len(req1.block_hashes) == 3
assert computed_blocks.get_block_ids() == ([1, 2, 3],)
assert num_computed_tokens == 3 * 16
@@ -1236,7 +1241,7 @@ def test_prefill_plp():
req2 = make_request(
"2", common_token_ids + unique_token_ids, block_size, hash_fn, prompt_logprobs=5
)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
assert len(req2.block_hashes) == 3
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
@@ -1273,7 +1278,7 @@ def test_decode():
# Incomplete 1 block (7 tokens)
unique_token_ids = [3] * 7
req0 = make_request("0", common_token_ids + unique_token_ids, block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
@@ -1331,7 +1336,7 @@ def test_evict():
last_token_id = 5 * 16 + 7
req0 = make_request("0", list(range(last_token_id)), block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
@@ -1344,7 +1349,7 @@ def test_evict():
req1 = make_request(
"1", list(range(last_token_id, last_token_id + 3 * 16)), block_size, sha256
)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
@@ -1369,7 +1374,7 @@ def test_evict():
# Touch the first 2 blocks.
req2 = make_request("2", list(range(2 * 16 + 3)), block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
assert computed_blocks.get_block_ids() == ([1, 2],)
assert num_computed_tokens == 2 * 16
blocks = manager.allocate_slots(
@@ -1395,7 +1400,7 @@ def test_hash_block_correct_reuse():
# Allocate 1 block and cache it.
num_tokens = block_size * 1
req = make_request("0", list(range(num_tokens)), block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req)
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
@@ -1409,7 +1414,7 @@ def test_hash_block_correct_reuse():
# Allocate a new block that's not full, make sure hash info on the
# block is cleared.
req = make_request("1", list(range(num_tokens - 1)), block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req)
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
@@ -1436,7 +1441,7 @@ def test_computed_blocks_not_evicted():
# Allocate a block and cache it.
num_tokens = block_size * 1
req0 = make_request("0", list(range(num_tokens)), block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
@@ -1449,7 +1454,7 @@ def test_computed_blocks_not_evicted():
req1 = make_request(
"1", list(range(num_tokens, num_tokens * 2)), block_size, sha256
)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
@@ -1465,7 +1470,7 @@ def test_computed_blocks_not_evicted():
# Now if we have a cache hit on the first block, we should evict the second
# cached block rather than the first one.
req2 = make_request("2", list(range(num_tokens * 2)), block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
assert len(computed_blocks.blocks[0]) == 1
assert computed_blocks.blocks[0][0].block_id == 1
assert num_computed_tokens == block_size
@@ -1496,7 +1501,7 @@ def test_basic_prefix_caching_disabled():
"1", list(range(10)), block_size, sha256
) # 2 blocks and some more
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
@@ -1509,7 +1514,7 @@ def test_basic_prefix_caching_disabled():
# No caching.
req2 = make_request("2", list(range(16)), block_size, sha256) # shared prefix
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
@@ -1519,7 +1524,7 @@ def test_basic_prefix_caching_disabled():
# New requests should not have any blocks.
req3 = make_request("3", list(range(4)), block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req3)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req3)
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
@@ -1701,7 +1706,7 @@ def test_mm_prefix_caching():
mm_positions=mm_positions,
mm_hashes=mm_hashes,
)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
# Completed block should have hashes
assert not computed_blocks.blocks[0]
@@ -1766,7 +1771,7 @@ def test_mm_prefix_caching():
mm_positions=mm_positions,
mm_hashes=mm_hashes,
)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert len(computed_blocks.blocks[0]) == 3
assert num_computed_tokens == 3 * 16
@@ -1788,7 +1793,7 @@ def test_cache_key_salting():
common_token_ids = [i for i in range(3) for _ in range(block_size)]
token_ids = common_token_ids + [3] * 11
req0 = make_request("0", token_ids, block_size, sha256, cache_salt="salt1")
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
# Completed block should have hashes
assert not computed_blocks.blocks[0]
@@ -1827,7 +1832,7 @@ def test_cache_key_salting():
# Test cache hit with a new request that has the same salt.
token_ids = common_token_ids + [4] * 11
req1 = make_request("1", token_ids, block_size, sha256, cache_salt="salt1")
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
# Should match only a prefix of 3 blocks.
assert len(computed_blocks.blocks[0]) == 3
assert num_computed_tokens == 3 * block_size
@@ -1835,7 +1840,7 @@ def test_cache_key_salting():
# Test cache miss with same content but different salt.
token_ids = common_token_ids + [4] * 11
req2 = make_request("2", token_ids, block_size, sha256, cache_salt="salt2")
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
assert len(computed_blocks.blocks[0]) == 0
assert num_computed_tokens == 0
block_hashes = req2.block_hashes
@@ -1869,7 +1874,7 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
# | Common-0 | Common-1 | Common-2 | ... |
common_token_ids = [i for i in range(3) for _ in range(16)]
req0 = make_request("0", common_token_ids, block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
manager.allocate_slots(
@@ -1881,7 +1886,7 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
# | Common-0 | Common-1 | Common-2 | Req1-3 | Req1-4 | Req1-5 | ... |
req1 = make_request("1", common_token_ids * 2, block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert computed_blocks.blocks[0] == block_part0
assert num_computed_tokens == 3 * 16
manager.allocate_slots(
@@ -1899,7 +1904,7 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
# | Common-0 | Common-1 | Common-2 | Req1-3 (F) | Req1-4 (F) |
# | Req1-5(F)| Req2-0 | Req2-1 | ... |
req2 = make_request("2", [7] * block_size * 2, block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
manager.allocate_slots(
@@ -1914,7 +1919,7 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
# In this case, the ref_cnt of the computed blocks should not be changed.
assert manager.block_pool.free_block_queue.num_free_blocks == 5
req3 = make_request("3", common_token_ids * 3, block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req3)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req3)
assert computed_blocks.blocks[0] == block_part1
assert num_computed_tokens == 6 * 16
# Req3 cannot be allocated.
@@ -1949,7 +1954,7 @@ def test_reset_prefix_cache():
unique_token_ids = [4] * 7
all_token_ids = full_block_token_ids + unique_token_ids
req1 = make_request("1", all_token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req1)
computed_blocks, _, _ = manager.get_computed_blocks(req1)
assert len(req1.block_hashes) == 3
assert len(computed_blocks.blocks[0]) == 3
blocks = manager.allocate_slots(
@@ -1984,7 +1989,7 @@ def test_prefix_cache_stats_disabled():
# Call all functions that check whether log_stats is disabled.
req = make_request("0", list(range(16)), block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req)
assert not computed_blocks.blocks[0]
assert num_computed_tokens == 0
manager.allocate_slots(
@@ -2591,7 +2596,7 @@ def test_eagle_enabled_removes_last_block():
req = make_request("divisible_request", token_ids, block_size, sha256)
# Prime the cache
computed_blocks, _ = manager.get_computed_blocks(req)
computed_blocks, _, _ = manager.get_computed_blocks(req)
manager.allocate_slots(
req, len(token_ids), len(computed_blocks.blocks[0]) * 16, computed_blocks
)
@@ -2599,7 +2604,7 @@ def test_eagle_enabled_removes_last_block():
# New request with same tokens + Eagle enabled
req_eagle = make_request("eagle_divisible", token_ids, block_size, sha256)
computed_blocks, num_tokens = manager.get_computed_blocks(req_eagle)
computed_blocks, num_tokens, _ = manager.get_computed_blocks(req_eagle)
# Should retain 1 block:
# 1. Original 3 blocks → pop last hash → 2 matched blocks
@@ -2623,7 +2628,7 @@ def test_eagle_with_partial_blocks():
req = make_request("partial_block_test", token_ids, block_size, sha256)
# Prime the cache
computed_blocks, _ = manager.get_computed_blocks(req)
computed_blocks, _, _ = manager.get_computed_blocks(req)
manager.allocate_slots(
req, len(token_ids), len(computed_blocks.blocks[0]) * 16, computed_blocks
)
@@ -2631,7 +2636,7 @@ def test_eagle_with_partial_blocks():
# New request with Eagle enabled
req_eagle = make_request("partial_eagle", token_ids, block_size, sha256)
computed_blocks, num_tokens = manager.get_computed_blocks(req_eagle)
computed_blocks, num_tokens, _ = manager.get_computed_blocks(req_eagle)
# Original match: 2 full blocks → Eagle removes 1 → 1 remaining
assert len(computed_blocks.blocks[0]) == 1
assert num_tokens == 1 * block_size
@@ -2664,7 +2669,7 @@ def test_eagle_with_sliding_window():
req = make_request("partial_block_test", token_ids, block_size, sha256)
# Prime the cache
computed_blocks, _ = manager.get_computed_blocks(req)
computed_blocks, _, _ = manager.get_computed_blocks(req)
manager.allocate_slots(
req, len(token_ids), len(computed_blocks.blocks[0]) * 16, computed_blocks
)
@@ -2675,7 +2680,7 @@ def test_eagle_with_sliding_window():
# New request with Eagle enabled
req_eagle = make_request("partial_eagle", token_ids, block_size, sha256)
computed_blocks, num_tokens = manager.get_computed_blocks(req_eagle)
computed_blocks, num_tokens, _ = manager.get_computed_blocks(req_eagle)
# Original match: 2 full blocks → Eagle removes 1 → 1 remaining
assert len(computed_blocks.blocks[0]) == 1
assert num_tokens == 1 * block_size
@@ -2695,7 +2700,7 @@ def test_eagle_with_sliding_window():
req_after_evict = make_request(
"partial_eagle_after_evict", token_ids, block_size, sha256
)
computed_blocks, num_tokens = manager.get_computed_blocks(req_after_evict)
computed_blocks, num_tokens, _ = manager.get_computed_blocks(req_after_evict)
# Cache miss. The only hit prefix is [NULL_BLOCK, BLOCK_2] if eagle is
# not considered. But after dropping the last matched block due to eagle,
# there will be no matched prefix.
@@ -2754,7 +2759,7 @@ def test_eagle_swa_alignment_caches_extra_block():
# Prime the cache with a long prompt (16 swa blocks = 4 aligned segments).
token_ids = [i for i in range(16) for _ in range(block_size)]
req0 = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req0)
computed_blocks, _, _ = manager.get_computed_blocks(req0)
blocks = manager.allocate_slots(
req0,
len(token_ids),
@@ -2768,7 +2773,7 @@ def test_eagle_swa_alignment_caches_extra_block():
# Without the fix, ``num_computed_tokens`` is 0; with the fix, it lands at
# an alignment boundary (multiple of 32 tokens, minus the EAGLE drop).
req1 = make_request("1", token_ids, block_size, sha256)
_, num_computed_tokens = manager.get_computed_blocks(req1)
_, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert num_computed_tokens > 0, (
"EAGLE + SWA with sliding_window <= alignment failed to find any "
"cache hit; the +1 block past each segment boundary must be cached."
@@ -2823,7 +2828,7 @@ def test_eagle_swa_boundary_caches_post_boundary_block():
token_ids = [i for i in range(5) for _ in range(block_size)]
req0 = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req0)
computed_blocks, _, _ = manager.get_computed_blocks(req0)
blocks = manager.allocate_slots(
req0,
len(token_ids),
@@ -2838,7 +2843,7 @@ def test_eagle_swa_boundary_caches_post_boundary_block():
manager.free(req0)
req1 = make_request("1", token_ids + [999], block_size, sha256)
_, num_computed_tokens = manager.get_computed_blocks(req1)
_, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert num_computed_tokens == 4 * block_size
@@ -2879,7 +2884,7 @@ def test_eagle_grouped_swa_siblings_use_same_cache_mask():
token_ids = [i for i in range(9) for _ in range(block_size)]
req0 = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req0)
computed_blocks, _, _ = manager.get_computed_blocks(req0)
blocks = manager.allocate_slots(
req0,
len(token_ids),
@@ -2894,7 +2899,7 @@ def test_eagle_grouped_swa_siblings_use_same_cache_mask():
manager.free(req0)
req1 = make_request("1", token_ids + [999], block_size, sha256)
_, num_computed_tokens = manager.get_computed_blocks(req1)
_, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert num_computed_tokens == 8 * block_size
@@ -2938,7 +2943,7 @@ def test_different_block_size():
common_token_ids = [i for i in range(10) for _ in range(block_size)]
req0 = make_request("0", common_token_ids, block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert not computed_blocks.blocks[0]
assert not computed_blocks.blocks[1]
assert num_computed_tokens == 0
@@ -2947,13 +2952,13 @@ def test_different_block_size():
)
assert blocks.get_block_ids() == ([1, 2, 3, 4], [5, 6, 7, 8, 9, 10, 11])
req1 = make_request("1", common_token_ids[: 7 * block_size + 1], block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert len(computed_blocks.blocks[0]) == 3
assert len(computed_blocks.blocks[1]) == 6
assert num_computed_tokens == 6 * 16
req2 = make_request("2", common_token_ids[: 6 * block_size + 1], block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
assert len(computed_blocks.blocks[0]) == 3
assert len(computed_blocks.blocks[1]) == 6
assert num_computed_tokens == 6 * 16
@@ -2967,7 +2972,7 @@ def test_different_block_size():
manager.block_pool.cached_block_hash_to_block.pop(
make_block_hash_with_group_id(req1.block_hashes[5], 1), 10
)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert len(computed_blocks.blocks[0]) == 2
assert len(computed_blocks.blocks[1]) == 4
assert num_computed_tokens == 4 * 16
@@ -3020,7 +3025,7 @@ def test_hybrid_cache_blocks_swa_tail_window_only():
# 8 hash-blocks of 8 tokens (64 tokens, two lcm-aligned segments).
token_ids = [i for i in range(8) for _ in range(block_size)]
req = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req)
computed_blocks, _, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req,
8 * block_size,
@@ -3089,7 +3094,7 @@ def test_hybrid_cache_blocks_clamped_to_lcm():
# to 96 tokens — SWA caches 6 hashes, full-attn caches 3.
token_ids = [i for i in range(7) for _ in range(block_size)]
req = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req)
computed_blocks, _, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req,
7 * block_size,
@@ -3154,7 +3159,7 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch):
# 96-token replay boundary, and the 128-token interval boundary.
token_ids = [i for i in range(16) for _ in range(block_size)]
req = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req)
computed_blocks, _, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req,
len(token_ids),
@@ -3296,13 +3301,13 @@ def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch):
token_ids = fill_request("fill_0", 0)
replay_req = make_request("replay", token_ids[:1800], hash_block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(replay_req)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(replay_req)
assert num_computed_tokens == 1024
assert [len(blocks) for blocks in computed_blocks.blocks] == [4, 16, 128, 256]
fill_request("fill_1", 100_000)
replay_req = make_request("replay_again", token_ids[:1800], hash_block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(replay_req)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(replay_req)
assert num_computed_tokens == 1024
assert [len(blocks) for blocks in computed_blocks.blocks] == [4, 16, 128, 256]
@@ -3345,7 +3350,7 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc
token_ids = [i for i in range(16) for _ in range(block_size)]
req0 = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req0)
computed_blocks, _, _ = manager.get_computed_blocks(req0)
blocks = manager.allocate_slots(
req0,
len(token_ids),
@@ -3369,14 +3374,14 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc
assert retained_swa_block[0].ref_cnt == 0
req1 = make_request("1", token_ids, block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
# Full prompt hits intentionally recompute the final block for logits, so
# the longest usable hit is the previous LCM boundary: 96 tokens.
assert num_computed_tokens == 12 * block_size
assert len(computed_blocks.blocks[1]) == 12
shorter_req = make_request("2", token_ids[: 12 * block_size], block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(shorter_req)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(shorter_req)
assert num_computed_tokens == 0
assert len(computed_blocks.blocks[1]) == 0
@@ -3429,7 +3434,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch):
# 104 tokens, and that tail is two 8-token blocks wide: hashes 11 and 12.
token_ids = [i for i in range(15) for _ in range(block_size)] + [15] * 7
req0 = make_request("0", token_ids, block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
req0,
@@ -3451,7 +3456,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch):
manager.free(req0)
req1 = make_request("1", token_ids, block_size, sha256)
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert num_computed_tokens == 12 * block_size
assert [len(blocks) for blocks in computed_blocks.blocks] == [3, 12]
@@ -3858,7 +3863,7 @@ def test_swa_free_split_keeps_cached_tail_ahead_of_scratch(monkeypatch):
token_ids = [i for i in range(16) for _ in range(block_size)]
req = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req)
computed_blocks, _, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req,
len(token_ids),
@@ -3941,7 +3946,7 @@ def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch):
token_ids = [i for i in range(16) for _ in range(block_size)]
req = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req)
computed_blocks, _, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req,
len(token_ids),
@@ -3965,7 +3970,7 @@ def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch):
# A replay of the same prompt hits the latest replayable boundary (240).
replay = make_request("1", token_ids, block_size, sha256)
_, num_computed = manager.get_computed_blocks(replay)
_, num_computed, _ = manager.get_computed_blocks(replay)
assert num_computed == 240
@@ -3977,7 +3982,7 @@ def test_pure_swa_retention_latest_only(monkeypatch):
token_ids = [i for i in range(16) for _ in range(block_size)]
req = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req)
computed_blocks, _, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req,
len(token_ids),
@@ -3997,7 +4002,7 @@ def test_pure_swa_retention_latest_only(monkeypatch):
assert cached == {14}
replay = make_request("1", token_ids, block_size, sha256)
_, num_computed = manager.get_computed_blocks(replay)
_, num_computed, _ = manager.get_computed_blocks(replay)
assert num_computed == 240
@@ -4010,7 +4015,7 @@ def test_pure_swa_retention_dense_default_caches_all(monkeypatch):
token_ids = [i for i in range(16) for _ in range(block_size)]
req = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req)
computed_blocks, _, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req,
len(token_ids),
@@ -4052,7 +4057,7 @@ def test_mamba_reachable_block_mask_sparsifies_retention():
kv_cache_spec=spec,
use_eagle=False,
retention_interval=retention_interval,
num_prompt_tokens=num_prompt_tokens,
reachable_boundaries=(num_prompt_tokens - 1,),
)
return None if m is None else {i for i, v in enumerate(m) if v}
@@ -4065,3 +4070,242 @@ def test_mamba_reachable_block_mask_sparsifies_retention():
assert retained(64) == {3, 7, 11, 14, 15}
# interval 0 -> only the latest replay boundary (block 14).
assert retained(0) == {14}
def test_mamba_reachable_block_mask_pins_shared_prefix():
"""A Marconi-detected shared prefix (``shared_prefix_boundary``) lands before
``num_prompt`` so the replay-boundary rule alone would drop it. The mask must
pin the single state block ending on that boundary so sparse retention does
not defeat cross-request shared-prefix reuse."""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
block_size = 16
spec = MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
def retained(retention_interval, shared_prefix_boundary, end_block=16):
boundaries = [255] # replay boundary (num_prompt 256 - 1)
if shared_prefix_boundary:
boundaries.append(shared_prefix_boundary)
m = MambaManager.reachable_block_mask(
start_block=0,
end_block=end_block,
alignment_tokens=block_size,
kv_cache_spec=spec,
use_eagle=False,
retention_interval=retention_interval,
reachable_boundaries=boundaries,
)
return None if m is None else {i for i, v in enumerate(m) if v}
# interval 0 keeps only replay boundary 14; the shared prefix at token 96
# (state block 96//16 - 1 = 5) is now pinned too.
assert retained(0, 96) == {5, 14}
# A non-aligned boundary floors to the enclosing aligned boundary.
assert retained(0, 100) == {5, 14}
# Coexists with segment tails (interval 64 -> {3,7,11,15} + replay 14).
assert retained(64, 96) == {3, 5, 7, 11, 14, 15}
# Dense default ignores the hint (nothing to sparsify).
assert retained(None, 96) is None
# Out-of-range boundary is a no-op (only replay 14 remains).
assert retained(0, 16 * block_size * 2) == {14}
# No boundary given -> unchanged replay-only behavior.
assert retained(0, 0) == {14}
assert retained(0, None) == {14}
def test_mamba_shared_prefix_survives_zero_retention(monkeypatch):
"""Manager-level check of the full wiring: a pinned shared-prefix boundary
(``Request.shared_prefix_boundary``, set by the scheduler on Marconi-style
detection) keeps its Mamba state block cached under
``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``, which otherwise retains only the
end-of-prompt replay boundary. Without this, a shared prefix (junction
before ``num_prompt``) would be recomputed by every sharing request."""
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")
block_size = 16
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
token_ids = [i for i in range(16) for _ in range(block_size)]
def cached_mamba_blocks(shared_prefix_boundary):
# Fresh manager per scenario so cached blocks don't leak between runs.
manager = make_kv_cache_manager(
_make_hybrid_kv_cache_config(block_size, 100, ["full", "mamba"]),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
)
req = make_request("r", token_ids, block_size, sha256)
req.shared_prefix_boundary = shared_prefix_boundary
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req, len(token_ids), num_computed, computed_blocks
)
assert blocks is not None
pool = manager.block_pool
return {
i
for i in range(16)
if pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
is not None
}
# Without a pinned boundary, retention=0 keeps only the replay boundary (14).
assert cached_mamba_blocks(0) == {14}
# Pinning the shared prefix at token 96 (state block 5) retains it too, so a
# later request sharing that prefix can hit the Mamba state.
assert cached_mamba_blocks(96) == {5, 14}
def test_mamba_shared_prefix_reuse_under_zero_retention(monkeypatch):
"""Full cross-request Marconi flow: a partial shared prefix cached by the
detecting request must stay reusable by a later request under
``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without the pin the junction is
masked out and the later request misses; with it (and under dense) the reuse
is preserved."""
block_size = 16
def last_req_hit(retention, pin):
if retention is None:
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
else:
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention))
manager = make_kv_cache_manager(
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba_align"]),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
)
shared = [7 for _ in range(2 * block_size)] # 2-block shared prefix
def distinct(v):
return [v for _ in range(2 * block_size)]
# req0 primes the shared prefix's (dense) attention cache; align-mode
# Mamba keeps only its own tail, not the shared-prefix junction.
req0 = make_request("0", shared + distinct(50), block_size, sha256)
cb, nc, _ = manager.get_computed_blocks(req0)
manager.allocate_slots(req0, len(req0.all_token_ids), nc, cb)
# req1 detects the shared prefix (attn hit, Mamba lag) and, via the
# Marconi chunk, caches the junction's Mamba state.
req1 = make_request("1", shared + distinct(60), block_size, sha256)
cb, nc, boundary = manager.get_computed_blocks(req1)
assert boundary == 2 * block_size # junction detected at 2 shared blocks
if pin:
req1.shared_prefix_boundary = boundary
# Simulate the Marconi chunk: schedule up to the junction (boundary - nc).
manager.allocate_slots(req1, boundary - nc, nc, cb)
# req2 shares the same prefix: does it reuse the cached junction?
req2 = make_request("2", shared + distinct(70), block_size, sha256)
_, nc2, _ = manager.get_computed_blocks(req2)
return nc2
# Dense retains the junction -> reuse works (baseline ceiling).
assert last_req_hit(retention=None, pin=False) == 2 * block_size
# retention=0 without the pin masks the junction out -> reuse lost (the bug).
assert last_req_hit(retention=0, pin=False) == 0
# retention=0 with the pin keeps the junction -> reuse restored.
assert last_req_hit(retention=0, pin=True) == 2 * block_size
def test_swa_reachable_block_mask_pins_shared_prefix():
"""SWA analog of the Mamba pin: the shared-prefix junction must keep the
``need``-block sliding-window tail ending on that boundary (not a single
block), so a windowed hit can land there under sparse retention."""
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
block_size = 16
def retained(retention, boundary, window, end_block=16):
spec = SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=window,
)
boundaries = [255] # replay boundary (num_prompt 256 - 1)
if boundary:
boundaries.append(boundary)
m = SlidingWindowManager.reachable_block_mask(
start_block=0,
end_block=end_block,
alignment_tokens=block_size,
kv_cache_spec=spec,
use_eagle=False,
retention_interval=retention,
reachable_boundaries=boundaries,
)
return None if m is None else {i for i, v in enumerate(m) if v}
# window == block_size -> need = cdiv(15, 16) = 1: single-block tail (like
# Mamba). Junction at token 96 -> block 5; replay boundary -> block 14.
assert retained(0, 96, block_size) == {5, 14}
# window == 3 * block_size -> need = cdiv(47, 16) = 3: the junction keeps a
# 3-block WINDOW {3,4,5} (the SWA distinction), plus replay window {12,13,14}.
assert retained(0, 96, 3 * block_size) == {3, 4, 5, 12, 13, 14}
# Coexists with segment tails (interval 64, need=1): {3,7,11,15} + replay 14
# + junction 5.
assert retained(64, 96, block_size) == {3, 5, 7, 11, 14, 15}
# Dense (all blocks reachable) ignores the hint.
assert retained(None, 96, block_size) is None
# No boundary -> unchanged replay-only behavior.
assert retained(0, 0, block_size) == {14}
def test_swa_shared_prefix_reuse_under_zero_retention(monkeypatch):
"""SWA cross-request analog: a partial shared prefix's sliding-window tail
must stay reusable under ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without
the pin the junction window is masked out and a later request misses; with
it (and under dense) reuse is preserved."""
block_size = 16
def last_req_hit(retention, pin):
if retention is None:
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
else:
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention))
manager = make_kv_cache_manager(
_make_hybrid_kv_cache_config(block_size, 200, ["full", "sliding_window"]),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
)
shared = [7 for _ in range(4 * block_size)] # 4-block shared prefix
def distinct(v):
return [v for _ in range(2 * block_size)]
# req0 primes the (dense) full-attention cache; SWA under RET=0 keeps
# only its own replay window, not the shared-prefix boundary.
req0 = make_request("0", shared + distinct(50), block_size, sha256)
cb, nc, _ = manager.get_computed_blocks(req0)
manager.allocate_slots(req0, len(req0.all_token_ids), nc, cb)
# req1 detects the shared prefix (full-attn hit, SWA lag). No chunk for
# SWA -- the pin fires during normal prefill.
req1 = make_request("1", shared + distinct(60), block_size, sha256)
cb, nc, boundary = manager.get_computed_blocks(req1)
if retention == 0:
assert boundary == 4 * block_size # junction detected when SWA lags
if pin and boundary:
req1.shared_prefix_boundary = boundary
manager.allocate_slots(req1, len(req1.all_token_ids), nc, cb)
# req2 shares the same prefix: does its SWA window reuse the junction?
req2 = make_request("2", shared + distinct(70), block_size, sha256)
_, nc2, _ = manager.get_computed_blocks(req2)
return nc2
# Dense keeps every SWA tail -> reuse works (ceiling).
assert last_req_hit(retention=None, pin=False) == 4 * block_size
# retention=0 without the pin drops the junction window -> reuse lost (bug).
assert last_req_hit(retention=0, pin=False) == 0
# retention=0 with the pin keeps the junction window -> reuse restored.
assert last_req_hit(retention=0, pin=True) == 4 * block_size
@@ -235,6 +235,9 @@ class MooncakeStoreCoordinator:
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
assert manager_cls is not None
use_eagle = g_idx in self.eagle_group_ids
reachable_boundaries = (
() if num_prompt_tokens is None else (num_prompt_tokens - 1,)
)
mask = manager_cls.reachable_block_mask(
start_block=start_chunk,
end_block=end_chunk,
@@ -242,7 +245,7 @@ class MooncakeStoreCoordinator:
kv_cache_spec=spec,
use_eagle=use_eagle,
retention_interval=retention_interval,
num_prompt_tokens=num_prompt_tokens,
reachable_boundaries=reachable_boundaries,
)
if mask is not None:
assert len(mask) == end_chunk - start_chunk
+20 -12
View File
@@ -370,7 +370,10 @@ class KVCacheCoordinator(ABC):
self,
block_hashes: list[BlockHash],
max_cache_hit_length: int,
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
) -> tuple[tuple[list[KVCacheBlock], ...], int, int]:
"""Returns the per-group hit blocks, the hit length, and the number of
``num_uncached_common_prefix_tokens`` (a shared prefix that a
sparse-retention group has not cached yet; 0 unless hybrid)."""
pass
def new_step_starts(self) -> None:
@@ -422,11 +425,11 @@ class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator):
self,
block_hashes: list[BlockHash],
max_cache_hit_length: int,
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
) -> tuple[tuple[list[KVCacheBlock], ...], int, int]:
blocks: tuple[list[KVCacheBlock], ...] = tuple(
[] for _ in range(self.num_single_type_manager)
)
return blocks, 0
return blocks, 0, 0
class UnitaryKVCacheCoordinator(KVCacheCoordinator):
@@ -486,7 +489,7 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator):
self,
block_hashes: list[BlockHash],
max_cache_hit_length: int,
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
) -> tuple[tuple[list[KVCacheBlock], ...], int, int]:
hit_blocks, hit_length = self.single_type_managers[0].find_longest_cache_hit(
block_hashes=block_hashes,
max_length=max_cache_hit_length,
@@ -498,7 +501,8 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator):
dcp_world_size=self.dcp_world_size,
pcp_world_size=self.pcp_world_size,
)
return hit_blocks, hit_length
# Single group: nothing "uncached common" -- no other group to lag it.
return hit_blocks, hit_length, 0
class SpecGroup(NamedTuple):
@@ -675,7 +679,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
self,
block_hashes: list[BlockHash],
max_cache_hit_length: int,
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
) -> tuple[tuple[list[KVCacheBlock], ...], int, int]:
"""
Find the longest cache hit using an iterative fixed-point algorithm.
@@ -691,7 +695,9 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
Returns:
A tuple containing:
- A tuple of the cache hit blocks for each single type manager.
- The number of tokens of the longest cache hit.
- The number of tokens of the reconciled (combined) cache hit.
- ``num_uncached_common_prefix_tokens``: a shared prefix that a
sparse-retention group has not cached yet (0 unless hybrid).
"""
num_groups = len(self.kv_cache_config.kv_cache_groups)
@@ -794,12 +800,14 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
del blks[num_blocks:]
hit_length_by_group[group_id] = hit_length
# Uncached shared prefix detection: If any attn. group cached a longer prefix
# than the current prefix, it is an uncached common prefix across requests:
self.num_uncached_common_prefix_tokens = longest_hit_length - hit_length
return tuple(
# Uncached shared prefix detection: if any attn. group cached a longer
# prefix than the reconciled hit, it is an uncached common prefix across
# requests that a sparse-retention group hasn't cached yet.
num_uncached_common_prefix_tokens = longest_hit_length - hit_length
cache_hit_blocks = tuple(
blocks if blocks is not None else [] for blocks in hit_blocks_by_group
), hit_length
)
return cache_hit_blocks, hit_length, num_uncached_common_prefix_tokens
def find_longest_cache_hit_per_group(
self,
+18 -4
View File
@@ -204,7 +204,7 @@ class KVCacheManager:
self.prefix_cache_stats = PrefixCacheStats()
return stats
def get_computed_blocks(self, request: Request) -> tuple[KVCacheBlocks, int]:
def get_computed_blocks(self, request: Request) -> tuple[KVCacheBlocks, int, int]:
"""Get the computed (cached) blocks for the request.
Note that the computed blocks must be full.
@@ -215,13 +215,18 @@ class KVCacheManager:
A tuple containing:
- A list of blocks that are computed for the request.
- The number of computed tokens.
- ``shared_prefix_boundary``: the block-aligned token position of
a shared prefix that a sparse-retention group (Mamba / sliding
window) has not cached yet (Marconi-style APC), or 0 if none.
Pinned so ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL`` does not drop
the junction and defeat cross-request reuse.
"""
# We skip finding the prefix cache hit when prefix caching is
# disabled or the request is marked as skipping kv cache read
# (which happens when the request requires prompt logprobs
# or calls a pooling model with all pooling).
if not self.enable_caching or request.skip_reading_prefix_cache:
return self.empty_kv_cache_blocks, 0
return self.empty_kv_cache_blocks, 0, 0
# NOTE: When all tokens hit the cache, we must recompute the last token
# to obtain logits. Thus, set max_cache_hit_length to prompt_length - 1.
@@ -230,7 +235,7 @@ class KVCacheManager:
# num_computed_tokens to be block-size aligned. Removing this limitation
# could slightly improve performance in the future.
max_cache_hit_length = request.num_tokens - 1
computed_blocks, num_new_computed_tokens = (
computed_blocks, num_new_computed_tokens, num_uncached = (
self.coordinator.find_longest_cache_hit(
request.block_hashes, max_cache_hit_length
)
@@ -256,6 +261,14 @@ class KVCacheManager:
group_idx,
)
# The junction to pin is where the lagging sparse-retention group stops
# (``num_new_computed_tokens``) plus the uncached shared prefix -- i.e.
# the longest single-group hit. Sub-block gaps are left to the mask,
# which floors to the alignment boundary (a no-op there).
shared_prefix_boundary = (
num_new_computed_tokens + num_uncached if num_uncached else 0
)
if self.log_stats:
assert self.prefix_cache_stats is not None
self.prefix_cache_stats.record(
@@ -264,7 +277,8 @@ class KVCacheManager:
preempted=request.num_preemptions > 0,
)
return self.create_kv_cache_blocks(computed_blocks), num_new_computed_tokens
blocks = self.create_kv_cache_blocks(computed_blocks)
return blocks, num_new_computed_tokens, shared_prefix_boundary
def allocate_slots(
self,
+70 -88
View File
@@ -350,85 +350,69 @@ class Scheduler(SchedulerInterface):
num_new_tokens: int,
num_new_local_computed_tokens: int = 0,
num_external_computed_tokens: int = 0,
num_uncached_common_prefix_tokens: int = 0,
) -> int:
num_computed_tokens = (
"""Clip a prefill chunk so it ends where Mamba state must be cached.
In "align" cache mode the SSM state is only materialized at chunk
ends, so chunk ends are steered onto cacheable positions: block
boundaries by default, plus mandatory early stops (the prompt's
partial-tail hash boundary, a detected shared-prefix junction).
"""
start = (
request.num_computed_tokens
+ num_new_local_computed_tokens
+ num_external_computed_tokens
)
# Perform block-aligned splitting at prefill phase, including:
# * non-resumed requests: num_computed_tokens < num_prompt_tokens + 0
# * resumed requests: num_computed_tokens < (
# num_prompt_tokens + num_output_tokens
# )
# NOTE: Use `request.num_tokens - 1` to bypass normal decoding.
if num_computed_tokens < max(request.num_prompt_tokens, request.num_tokens - 1):
# To enable block-aligned caching of the Mamba state, `num_new_tokens`
# must be a multiple of `block_size`.
# As an exception, if `num_new_tokens` is less than `block_size`, the
# state is simply not cached, requiring no special handling.
# Additionally, when Eagle mode is enabled, FullAttn prunes the last
# matching block. To prevent this from causing a Mamba cache miss, the
# last chunk must be not smaller than `block_size`.
block_size = self.cache_config.block_size
last_cache_position = request.num_tokens - request.num_tokens % block_size
# eagle prune
if self.use_eagle:
last_cache_position = max(last_cache_position - block_size, 0)
num_computed_tokens_after_sched = num_computed_tokens + num_new_tokens
next_boundary = (num_computed_tokens // block_size + 1) * block_size
if (
num_computed_tokens % block_size != 0
and next_boundary <= last_cache_position
and num_computed_tokens_after_sched > next_boundary
):
# Resumed mid-block (partial hash hit): stop the first chunk
# at the next block boundary so chunk ends re-align to blocks
# and the boundary state is materialized before running past.
num_new_tokens = next_boundary - num_computed_tokens
elif num_computed_tokens_after_sched < last_cache_position:
# Align the chunk END (not its length) to block_size;
# identical to flooring the length when the start is
# block-aligned. May yield 0 (insufficient budget to reach
# the next boundary); the caller then skips the request.
aligned_end = num_computed_tokens_after_sched // block_size * block_size
num_new_tokens = max(aligned_end - num_computed_tokens, 0)
elif (
num_computed_tokens
< last_cache_position
< num_computed_tokens_after_sched
):
# force to cache the last chunk
num_new_tokens = last_cache_position - num_computed_tokens
elif self.mamba_partial_cache_hit:
# Prefill of the final partial block: stop once at the
# prompt's last hash boundary so the mamba partial tail entry
# can be registered.
tail_boundary = (
request.num_prompt_tokens
// self.hash_block_size
* self.hash_block_size
)
if (
num_computed_tokens
< tail_boundary
< num_computed_tokens_after_sched
and tail_boundary < request.num_prompt_tokens
and tail_boundary > last_cache_position
):
num_new_tokens = tail_boundary - num_computed_tokens
# Split only during prefill: `request.num_tokens - 1` extends this to
# resumed requests replaying their output tokens.
if start >= max(request.num_prompt_tokens, request.num_tokens - 1):
return num_new_tokens
# Marconi cache admission optimization:
# cache common prefixes by scheduling num_new_tokens = common prefix length
if (
num_uncached_common_prefix_tokens >= block_size
and num_new_tokens > num_uncached_common_prefix_tokens
):
num_new_tokens = num_uncached_common_prefix_tokens
# keep alignment to block_size
num_new_tokens = num_new_tokens // block_size * block_size
return num_new_tokens
block_size = self.cache_config.block_size
# The last block-aligned position whose state can be cached. With
# Eagle, FullAttn prunes the last matching block, so back off one
# block to avoid a Mamba cache miss.
last_cache_position = request.num_tokens - request.num_tokens % block_size
if self.use_eagle:
last_cache_position = max(last_cache_position - block_size, 0)
end = start + num_new_tokens
# Until `last_cache_position`, chunk ends must land on block
# boundaries. May yield an empty chunk (budget cannot reach the next
# boundary); the caller then skips the request.
if end < last_cache_position:
end = end // block_size * block_size
next_block_boundary = (start // block_size + 1) * block_size
tail_boundary = (
request.num_prompt_tokens // self.hash_block_size * self.hash_block_size
if self.mamba_partial_cache_hit
else 0
)
stops = (
# Resumed mid-block (fine-grained partial hash hit): re-align to
# the block grid before running on, so the crossed boundary's
# state is materialized (unless it is past the cacheable range).
next_block_boundary
if start % block_size != 0 and next_block_boundary <= last_cache_position
else 0,
# Never run past the last cacheable block boundary mid-chunk.
last_cache_position,
# Fine-grained hits: the prompt's partial-tail entry can only be
# registered by a chunk ending exactly at its last hash boundary.
tail_boundary
if last_cache_position < tail_boundary < request.num_prompt_tokens
else 0,
# Marconi shared-prefix junction, block-floored (a sub-block
# junction's state is not separately cacheable): cache its state
# so sibling requests sharing the prefix can reuse it.
start + (request.shared_prefix_boundary - start) // block_size * block_size
if start < request.shared_prefix_boundary < end
else 0,
)
# Stop at the earliest mandatory position strictly inside the chunk.
end = min((s for s in stops if start < s < end), default=end)
return max(end - start, 0)
def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput:
self.current_step += 1
@@ -718,7 +702,6 @@ class Scheduler(SchedulerInterface):
num_external_computed_tokens = 0
load_kv_async = False
connector_prefix_cache_queries, connector_prefix_cache_hits = 0, 0
num_uncached_common_prefix_tokens = 0
# Get already-cached tokens.
if request.num_computed_tokens == 0:
@@ -733,8 +716,7 @@ class Scheduler(SchedulerInterface):
):
computed, per_group_hits = (
self.kv_cache_manager.coordinator.find_longest_cache_hit_per_group(
request.block_hashes,
request.num_tokens - 1,
request.block_hashes, request.num_tokens - 1
)
)
new_computed_blocks = (
@@ -750,6 +732,9 @@ class Scheduler(SchedulerInterface):
# the last block) is transferred unconditionally by
# _apply_prefix_caching in nixl/worker.py.
num_new_local_computed_tokens = max(per_group_hits)
# The per-group lookup does not detect an uncached shared
# prefix, so there is no junction to pin in this path.
request.shared_prefix_boundary = 0
if self.kv_cache_manager.log_stats:
assert self.kv_cache_manager.prefix_cache_stats is not None
self.kv_cache_manager.prefix_cache_stats.record(
@@ -758,17 +743,15 @@ class Scheduler(SchedulerInterface):
preempted=request.num_preemptions > 0,
)
else:
new_computed_blocks, num_new_local_computed_tokens = (
self.kv_cache_manager.get_computed_blocks(request)
)
# In case of hybrid models, obtain hint for Marconi-style APC logic
if self.has_mamba_layers:
num_uncached_common_prefix_tokens = getattr(
self.kv_cache_manager.coordinator,
"num_uncached_common_prefix_tokens",
0,
)
(
new_computed_blocks,
num_new_local_computed_tokens,
# Junction to pin (Marconi-style APC) so its
# sparse-retention state (Mamba block / sliding-window
# tail) survives retention and serves a later hit; 0
# if no uncached shared prefix was detected.
request.shared_prefix_boundary,
) = self.kv_cache_manager.get_computed_blocks(request)
# Get externally-cached tokens if using a KVConnector.
if self.connector is not None:
@@ -906,7 +889,6 @@ class Scheduler(SchedulerInterface):
num_new_tokens,
num_new_local_computed_tokens,
num_external_computed_tokens,
num_uncached_common_prefix_tokens,
)
if num_new_tokens == 0:
break
+37 -27
View File
@@ -419,6 +419,13 @@ class SingleTypeKVCacheManager(ABC):
if num_cached_blocks >= num_full_blocks:
return
# Token boundaries whose reachable tail must be retained under sparse
# retention: the replay boundary (``num_prompt - 1``, capped by
# ``get_computed_blocks``) and any detected shared-prefix junction.
reachable_boundaries = [request.num_prompt_tokens - 1]
if request.shared_prefix_boundary:
reachable_boundaries.append(request.shared_prefix_boundary)
block_mask = self.reachable_block_mask(
start_block=num_cached_blocks,
end_block=num_full_blocks,
@@ -426,7 +433,7 @@ class SingleTypeKVCacheManager(ABC):
kv_cache_spec=self.kv_cache_spec,
use_eagle=self.use_eagle,
retention_interval=retention_interval,
num_prompt_tokens=request.num_prompt_tokens,
reachable_boundaries=reachable_boundaries,
)
self.block_pool.cache_full_blocks(
request=request,
@@ -449,14 +456,15 @@ class SingleTypeKVCacheManager(ABC):
kv_cache_spec: KVCacheSpec,
use_eagle: bool,
retention_interval: int | None = None,
num_prompt_tokens: int | None = None,
reachable_boundaries: Sequence[int] = (),
) -> list[bool] | None:
"""Per-block mask for ``cache_full_blocks``. ``None`` means cache
every (non-null) block — the default for full attention.
Subclasses with sparse hit semantics (SWA) override this to skip
blocks that can never serve a hit at any alignment-aligned prefix
length.
Subclasses with sparse hit semantics (SWA / Mamba) override this to skip
blocks that can never serve a hit at any alignment-aligned prefix length.
``reachable_boundaries`` are token positions whose reachable tail must be
retained; the base (dense) policy ignores them.
"""
return None
@@ -962,7 +970,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
kv_cache_spec: KVCacheSpec,
use_eagle: bool,
retention_interval: int | None = None,
num_prompt_tokens: int | None = None,
reachable_boundaries: Sequence[int] = (),
) -> list[bool] | None:
assert isinstance(kv_cache_spec, SlidingWindowSpec)
if alignment_tokens is None:
@@ -1002,18 +1010,16 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
if i >= shift and (i - shift) % per_segment >= per_segment - need:
mask[i - start_block] = True
# (2) Replay-boundary tail. ``get_computed_blocks`` caps hits at
# ``num_prompt - 1`` (to recompute the last token's logits), so an exact
# prompt replay can only land on the latest *fine*-aligned boundary.
# Sparse retention would otherwise skip it, so keep its tail explicitly.
if retention_interval is not None and num_prompt_tokens is not None:
latest = (num_prompt_tokens - 1) // alignment_tokens * alignment_tokens
prompt_end_block = latest // block_size + shift
for i in range(
max(start_block, prompt_end_block - need),
min(end_block, prompt_end_block),
):
mask[i - start_block] = True
# (2) Reachable-boundary tails: the replay boundary (``num_prompt - 1``,
# capped by ``get_computed_blocks``) and any shared-prefix junction. Both
# land before segments would cover them under sparse retention, so keep
# the ``need``-block tail ending on each boundary explicitly.
if retention_interval is not None:
for boundary_tokens in reachable_boundaries:
aligned = boundary_tokens // alignment_tokens * alignment_tokens
end = aligned // block_size + shift
for j in range(max(start_block, end - need), min(end_block, end)):
mask[j - start_block] = True
return mask
@@ -1321,15 +1327,19 @@ class MambaManager(SingleTypeKVCacheManager):
kv_cache_spec: KVCacheSpec,
use_eagle: bool,
retention_interval: int | None = None,
num_prompt_tokens: int | None = None,
reachable_boundaries: Sequence[int] = (),
) -> list[bool] | None:
"""Sparse Mamba state-snapshot retention.
``retention_interval``:
``None`` -> dense (cache every block; default, unchanged behavior)
``0`` -> keep only the latest replay boundary
``0`` -> keep only the ``reachable_boundaries`` states
``> 0`` -> keep one state per ``retention_interval``-sized segment
``reachable_boundaries`` are proven reuse points (the replay boundary and
any cross-request shared-prefix junction, Marconi-style APC); their
boundary state is always kept so sparse retention does not defeat reuse.
"""
if retention_interval is None or alignment_tokens is None:
# Dense caching (default) or no alignment constraint imposed.
@@ -1354,13 +1364,13 @@ class MambaManager(SingleTypeKVCacheManager):
for i in range(first_boundary - start_block, len(mask), per_segment):
mask[i] = True
# (2) Replay boundary. ``get_computed_blocks`` caps hits at
# ``num_prompt - 1``, so an exact prompt replay lands on the latest
# fine-aligned boundary. Sparse retention would otherwise skip its
# state, so keep it explicitly.
if num_prompt_tokens is not None:
latest = (num_prompt_tokens - 1) // alignment_tokens * alignment_tokens
boundary_block = latest // block_size - 1
# (2) Reachable-boundary states: the replay boundary (``num_prompt - 1``,
# capped by ``get_computed_blocks``) and any shared-prefix junction, both
# of which segments would otherwise skip under sparse retention. A Mamba
# hit needs exactly the single state block ending on the boundary.
for boundary_tokens in reachable_boundaries:
aligned = boundary_tokens // alignment_tokens * alignment_tokens
boundary_block = aligned // block_size - 1
if start_block <= boundary_block < end_block:
mask[boundary_block - start_block] = True
+5
View File
@@ -182,6 +182,11 @@ class Request:
# True if this request is scheduled as a non-final prefill chunk.
self.is_prefill_chunk = False
# Block-aligned token position of a proven shared prefix worth pinning
# in the (sparse) prefix cache; 0 means none. Set at admission for
# hybrid/Mamba models when a shared prefix is detected (Marconi-style).
self.shared_prefix_boundary = 0
# The number of NaNs in logits. A value greater than 0
# indicates that the output is corrupted
self.num_nans_in_logits = 0
+1 -1
View File
@@ -262,7 +262,7 @@ class SimpleCPUOffloadScheduler:
max_hit_len = request.num_tokens - 1 - num_computed_tokens
if max_hit_len <= 0:
return 0, False
cpu_hit_blocks, hit_length = self.cpu_coordinator.find_longest_cache_hit(
cpu_hit_blocks, hit_length, _ = self.cpu_coordinator.find_longest_cache_hit(
remaining_hashes, max_hit_len
)