[Core] Preserve Marconi caching with selective hybrid cache retention (#47782)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -110,7 +110,7 @@ def test_hybrid_mamba_align_partial_hash_hit():
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
blocks = manager.allocate_slots(req0, 6, num_computed, computed_blocks)
|
||||
assert blocks is not None
|
||||
@@ -125,7 +125,7 @@ def test_hybrid_mamba_align_partial_hash_hit():
|
||||
assert partial_mamba_block[0].block_hash_num_tokens == 6
|
||||
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
assert [len(group) for group in computed_blocks.blocks] == [3, 2]
|
||||
|
||||
@@ -185,7 +185,7 @@ def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue():
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
|
||||
@@ -258,7 +258,7 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
|
||||
@@ -279,7 +279,7 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
|
||||
manager.new_step_starts()
|
||||
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 4, 4], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
# The later request hits the moved (private-copy) entry, not the source.
|
||||
assert computed_blocks.get_block_ids()[1][1] == moved_block_id
|
||||
@@ -338,7 +338,7 @@ def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
manager.new_step_starts()
|
||||
@@ -351,13 +351,13 @@ def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
|
||||
|
||||
# A request hitting the moved entry in the SAME step must be deferred.
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 4, 4], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
assert manager.allocate_slots(req1, 2, num_computed, computed_blocks) is None
|
||||
|
||||
# Next step the moved entry is consumable.
|
||||
manager.new_step_starts()
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
assert manager.allocate_slots(req1, 2, num_computed, computed_blocks) is not None
|
||||
|
||||
@@ -397,7 +397,7 @@ def test_hybrid_full_attention_partial_hash_hit_uses_cow():
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
manager.free(req0)
|
||||
@@ -410,7 +410,7 @@ def test_hybrid_full_attention_partial_hash_hit_uses_cow():
|
||||
assert partial_full_block is not None
|
||||
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
assert [len(group) for group in computed_blocks.blocks] == [2, 2]
|
||||
|
||||
@@ -471,7 +471,7 @@ def test_hybrid_partial_hit_cow_target_starts_uncached():
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
manager.free(req0)
|
||||
@@ -488,7 +488,7 @@ def test_hybrid_partial_hit_cow_target_starts_uncached():
|
||||
assert partial_mamba_block is not None
|
||||
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
|
||||
new_blocks = manager.allocate_slots(
|
||||
@@ -586,7 +586,7 @@ def test_hybrid_partial_hash_truncates_full_attention_hit_length():
|
||||
block_size=block_size,
|
||||
)
|
||||
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
|
||||
assert num_computed == 6
|
||||
assert [len(group) for group in computed_blocks.blocks] == [2, 2]
|
||||
|
||||
@@ -627,7 +627,7 @@ def test_cow_retained_blocks_returned_for_release():
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
|
||||
# The owner's move queues a copy and retains both endpoints.
|
||||
@@ -797,7 +797,7 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
|
||||
# The owner prefills in scheduler-split style: stop at the block boundary
|
||||
# (4), then at the prompt's last hash boundary (6, partial entries).
|
||||
req0 = make_request("0", [7] * 6, hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
|
||||
req0.num_computed_tokens = 4
|
||||
manager.new_step_starts()
|
||||
@@ -808,7 +808,7 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
|
||||
# A longer request with eagle: full attention drops the partial tail, so
|
||||
# the joint hit must fall back to the block boundary the FA blocks cover.
|
||||
req1 = make_request("1", [7] * 6 + [9] * 2, hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 4
|
||||
assert all(
|
||||
len(group) * block_size >= num_computed for group in computed_blocks.blocks
|
||||
|
||||
@@ -244,7 +244,7 @@ def test_prefill(hash_fn):
|
||||
unique_token_ids = [3] * 7
|
||||
all_token_ids = common_token_ids + unique_token_ids
|
||||
req0 = make_request("0", all_token_ids, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert len(req0.block_hashes) == 3
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
@@ -274,7 +274,7 @@ def test_prefill(hash_fn):
|
||||
# Incomplete 1 block (5 tokens)
|
||||
unique_token_ids = [3] * 5
|
||||
req1 = make_request("1", common_token_ids + unique_token_ids, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert len(req1.block_hashes) == 3
|
||||
assert computed_blocks.get_block_ids() == ([1, 2, 3],)
|
||||
assert num_computed_tokens == 3 * 16
|
||||
@@ -307,7 +307,7 @@ def test_prefill(hash_fn):
|
||||
# Incomplete 1 block (6 tokens)
|
||||
unique_token_ids = [3] * 6
|
||||
req2 = make_request("2", common_token_ids + unique_token_ids, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
|
||||
assert len(req2.block_hashes) == 3
|
||||
assert computed_blocks.get_block_ids() == ([1, 2, 3],)
|
||||
assert num_computed_tokens == 3 * 16
|
||||
@@ -327,7 +327,7 @@ def test_prefill(hash_fn):
|
||||
|
||||
# Cache miss and eviction.
|
||||
req3 = make_request("3", [99] * (16 * 10), block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req3)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req3)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
@@ -369,7 +369,7 @@ def test_prefill_hybrid_model():
|
||||
unique_token_ids = [3] * 7
|
||||
all_token_ids = common_token_ids + unique_token_ids
|
||||
req0 = make_request("0", all_token_ids, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert len(req0.block_hashes) == 3
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
@@ -405,7 +405,7 @@ def test_prefill_hybrid_model():
|
||||
unique_token_ids = [3] * 5
|
||||
all_token_ids = common_token_ids + unique_token_ids
|
||||
req1 = make_request("1", common_token_ids + unique_token_ids, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert len(req1.block_hashes) == 3
|
||||
assert computed_blocks.get_block_ids() == ([1, 2, 3], [0, 6, 7], [0, 10, 11])
|
||||
assert num_computed_tokens == 3 * 16
|
||||
@@ -539,7 +539,7 @@ def test_prefill_hybrid_model_eagle():
|
||||
unique_token_ids = [6] * 7
|
||||
all_token_ids = common_token_ids + unique_token_ids
|
||||
req0 = make_request("0", all_token_ids, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert len(req0.block_hashes) == len(all_token_ids) // block_size
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
@@ -576,7 +576,7 @@ def test_prefill_hybrid_model_eagle():
|
||||
unique_token_ids = [6] * 5
|
||||
all_token_ids = common_token_ids + unique_token_ids
|
||||
req1 = make_request("1", all_token_ids, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert len(req1.block_hashes) == num_full_blocks
|
||||
assert computed_blocks.get_block_ids() == (
|
||||
[1, 2, 3, 4, 5],
|
||||
@@ -712,7 +712,7 @@ def _test_partial_request_hit(
|
||||
req = make_request(request_id, prompt_token_ids, block_size, sha256)
|
||||
for hash_with_group_id in hash_to_evict:
|
||||
manager.block_pool.cached_block_hash_to_block._cache.pop(hash_with_group_id)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req)
|
||||
assert len(req.block_hashes) == num_full_blocks
|
||||
assert num_computed_tokens == expect_hit_length * block_size
|
||||
for block_per_group in computed_blocks.blocks:
|
||||
@@ -873,7 +873,7 @@ def test_prefill_hybrid_model_combinations(spec_types: list[str]):
|
||||
|
||||
# First request: no cache hit initially
|
||||
req0 = make_request("0", all_token_ids, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
|
||||
assert len(req0.block_hashes) == 3
|
||||
assert not computed_blocks.blocks[0] # No cache hit initially
|
||||
@@ -890,7 +890,7 @@ def test_prefill_hybrid_model_combinations(spec_types: list[str]):
|
||||
|
||||
# Second request: should hit cached blocks for common prefix
|
||||
req1 = make_request("1", common_token_ids + [4] * 5, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
|
||||
# Should hit cached blocks for all groups
|
||||
assert num_computed_tokens == 3 * block_size
|
||||
@@ -950,7 +950,7 @@ def test_prefill_hybrid_model_combinations_eagle(
|
||||
|
||||
# First request: no cache hit initially
|
||||
req0 = make_request("0", all_token_ids, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
|
||||
assert len(req0.block_hashes) == num_full_blocks
|
||||
assert not computed_blocks.blocks[0] # No cache hit initially
|
||||
@@ -966,7 +966,7 @@ def test_prefill_hybrid_model_combinations_eagle(
|
||||
# Second request: should hit cached blocks for common prefix
|
||||
all_token_ids = common_token_ids + [6] * 5
|
||||
req1 = make_request("1", all_token_ids, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
|
||||
# Should hit cached blocks for all groups
|
||||
assert num_computed_tokens == expect_hit_length * block_size
|
||||
@@ -1019,7 +1019,7 @@ def test_prefill_hybrid_model_mamba_align():
|
||||
# First request: allocate_slots should not crash with the assertion error
|
||||
# in MambaManager.cache_blocks() when null blocks are present.
|
||||
req0 = make_request("0", all_token_ids, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed_tokens == 0
|
||||
|
||||
blocks = manager.allocate_slots(req0, 55, num_computed_tokens, computed_blocks)
|
||||
@@ -1048,28 +1048,33 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
|
||||
# Request: 3 blocks
|
||||
prefix = [i for i in range(3) for _ in range(block_size)]
|
||||
req_0 = make_request("0", prefix, block_size, hash_fn)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req_0)
|
||||
num_uncached_common = manager.coordinator.num_uncached_common_prefix_tokens
|
||||
computed_blocks, num_computed, shared_prefix_boundary = manager.get_computed_blocks(
|
||||
req_0
|
||||
)
|
||||
assert num_computed == 0 # nothing cached yet
|
||||
assert num_uncached_common == 0
|
||||
assert shared_prefix_boundary == 0
|
||||
manager.allocate_slots(req_0, 3 * block_size, 0, computed_blocks)
|
||||
|
||||
# Request: 3 blocks (shared with above) + 7 different tokens
|
||||
req_1 = make_request("1", prefix + [100] * 7, block_size, hash_fn)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req_1)
|
||||
num_uncached_common = manager.coordinator.num_uncached_common_prefix_tokens
|
||||
computed_blocks, num_computed, shared_prefix_boundary = manager.get_computed_blocks(
|
||||
req_1
|
||||
)
|
||||
assert num_computed == 3 * block_size # we should observe a 3-block cache hit
|
||||
assert num_uncached_common == 0
|
||||
assert shared_prefix_boundary == 0
|
||||
manager.allocate_slots(req_1, 7, 3 * block_size, computed_blocks)
|
||||
|
||||
# Request: 3 blocks, but only 2 blocks shared (replace the last token in 3rd block):
|
||||
req_2 = make_request("2", prefix[:-1] + [101], block_size, hash_fn)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req_2)
|
||||
num_uncached_common = manager.coordinator.num_uncached_common_prefix_tokens
|
||||
computed_blocks, num_computed, shared_prefix_boundary = manager.get_computed_blocks(
|
||||
req_2
|
||||
)
|
||||
assert num_computed == 0 # mamba_align doesn't cache intermediate blocks
|
||||
assert num_uncached_common == 2 * block_size # heuristic detects a shared prefix
|
||||
# boundary == junction: 2 shared blocks (num_computed 0 + 2 uncached blocks).
|
||||
assert shared_prefix_boundary == 2 * block_size
|
||||
|
||||
# Next, validate scheduler logic for num_uncached_common_prefix_tokens > 0
|
||||
# Next, validate the align-mode Marconi split. It takes the shared-prefix
|
||||
# junction (absolute) and stops the chunk there (req_2 has num_computed 0).
|
||||
# Create minimal mock with just the needed attributes
|
||||
mock = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
@@ -1077,11 +1082,11 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
|
||||
hash_block_size=block_size,
|
||||
mamba_partial_cache_hit=False,
|
||||
)
|
||||
req_2.shared_prefix_boundary = shared_prefix_boundary
|
||||
num_new_tokens_adjusted = Scheduler._mamba_block_aligned_split(
|
||||
self=mock,
|
||||
request=req_2,
|
||||
num_new_tokens=3 * block_size,
|
||||
num_uncached_common_prefix_tokens=num_uncached_common,
|
||||
)
|
||||
assert num_new_tokens_adjusted == 2 * block_size # adjust to the common prefix
|
||||
|
||||
@@ -1118,7 +1123,7 @@ def test_hybrid_model_mamba_align_with_dynamic_draft_tokens():
|
||||
|
||||
all_token_ids = [i for i in range(3) for _ in range(block_size)] + [3] * 7
|
||||
req0 = make_request("0", all_token_ids, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
req0, len(all_token_ids), num_computed_tokens, computed_blocks
|
||||
@@ -1170,7 +1175,7 @@ def test_prefill_plp():
|
||||
unique_token_ids = [3] * 7
|
||||
all_token_ids = common_token_ids + unique_token_ids
|
||||
req0 = make_request("0", all_token_ids, block_size, hash_fn, prompt_logprobs=5)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert len(req0.block_hashes) == 3
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
@@ -1202,7 +1207,7 @@ def test_prefill_plp():
|
||||
# Incomplete 1 block (5 tokens)
|
||||
unique_token_ids = [3] * 5
|
||||
req1 = make_request("1", common_token_ids + unique_token_ids, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert len(req1.block_hashes) == 3
|
||||
assert computed_blocks.get_block_ids() == ([1, 2, 3],)
|
||||
assert num_computed_tokens == 3 * 16
|
||||
@@ -1236,7 +1241,7 @@ def test_prefill_plp():
|
||||
req2 = make_request(
|
||||
"2", common_token_ids + unique_token_ids, block_size, hash_fn, prompt_logprobs=5
|
||||
)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
|
||||
assert len(req2.block_hashes) == 3
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
@@ -1273,7 +1278,7 @@ def test_decode():
|
||||
# Incomplete 1 block (7 tokens)
|
||||
unique_token_ids = [3] * 7
|
||||
req0 = make_request("0", common_token_ids + unique_token_ids, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
@@ -1331,7 +1336,7 @@ def test_evict():
|
||||
|
||||
last_token_id = 5 * 16 + 7
|
||||
req0 = make_request("0", list(range(last_token_id)), block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
@@ -1344,7 +1349,7 @@ def test_evict():
|
||||
req1 = make_request(
|
||||
"1", list(range(last_token_id, last_token_id + 3 * 16)), block_size, sha256
|
||||
)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
@@ -1369,7 +1374,7 @@ def test_evict():
|
||||
|
||||
# Touch the first 2 blocks.
|
||||
req2 = make_request("2", list(range(2 * 16 + 3)), block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
|
||||
assert computed_blocks.get_block_ids() == ([1, 2],)
|
||||
assert num_computed_tokens == 2 * 16
|
||||
blocks = manager.allocate_slots(
|
||||
@@ -1395,7 +1400,7 @@ def test_hash_block_correct_reuse():
|
||||
# Allocate 1 block and cache it.
|
||||
num_tokens = block_size * 1
|
||||
req = make_request("0", list(range(num_tokens)), block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
@@ -1409,7 +1414,7 @@ def test_hash_block_correct_reuse():
|
||||
# Allocate a new block that's not full, make sure hash info on the
|
||||
# block is cleared.
|
||||
req = make_request("1", list(range(num_tokens - 1)), block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
@@ -1436,7 +1441,7 @@ def test_computed_blocks_not_evicted():
|
||||
# Allocate a block and cache it.
|
||||
num_tokens = block_size * 1
|
||||
req0 = make_request("0", list(range(num_tokens)), block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
@@ -1449,7 +1454,7 @@ def test_computed_blocks_not_evicted():
|
||||
req1 = make_request(
|
||||
"1", list(range(num_tokens, num_tokens * 2)), block_size, sha256
|
||||
)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
@@ -1465,7 +1470,7 @@ def test_computed_blocks_not_evicted():
|
||||
# Now if we have a cache hit on the first block, we should evict the second
|
||||
# cached block rather than the first one.
|
||||
req2 = make_request("2", list(range(num_tokens * 2)), block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
|
||||
assert len(computed_blocks.blocks[0]) == 1
|
||||
assert computed_blocks.blocks[0][0].block_id == 1
|
||||
assert num_computed_tokens == block_size
|
||||
@@ -1496,7 +1501,7 @@ def test_basic_prefix_caching_disabled():
|
||||
"1", list(range(10)), block_size, sha256
|
||||
) # 2 blocks and some more
|
||||
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
@@ -1509,7 +1514,7 @@ def test_basic_prefix_caching_disabled():
|
||||
|
||||
# No caching.
|
||||
req2 = make_request("2", list(range(16)), block_size, sha256) # shared prefix
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
@@ -1519,7 +1524,7 @@ def test_basic_prefix_caching_disabled():
|
||||
|
||||
# New requests should not have any blocks.
|
||||
req3 = make_request("3", list(range(4)), block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req3)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req3)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
@@ -1701,7 +1706,7 @@ def test_mm_prefix_caching():
|
||||
mm_positions=mm_positions,
|
||||
mm_hashes=mm_hashes,
|
||||
)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
|
||||
# Completed block should have hashes
|
||||
assert not computed_blocks.blocks[0]
|
||||
@@ -1766,7 +1771,7 @@ def test_mm_prefix_caching():
|
||||
mm_positions=mm_positions,
|
||||
mm_hashes=mm_hashes,
|
||||
)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert len(computed_blocks.blocks[0]) == 3
|
||||
assert num_computed_tokens == 3 * 16
|
||||
|
||||
@@ -1788,7 +1793,7 @@ def test_cache_key_salting():
|
||||
common_token_ids = [i for i in range(3) for _ in range(block_size)]
|
||||
token_ids = common_token_ids + [3] * 11
|
||||
req0 = make_request("0", token_ids, block_size, sha256, cache_salt="salt1")
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
|
||||
# Completed block should have hashes
|
||||
assert not computed_blocks.blocks[0]
|
||||
@@ -1827,7 +1832,7 @@ def test_cache_key_salting():
|
||||
# Test cache hit with a new request that has the same salt.
|
||||
token_ids = common_token_ids + [4] * 11
|
||||
req1 = make_request("1", token_ids, block_size, sha256, cache_salt="salt1")
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
# Should match only a prefix of 3 blocks.
|
||||
assert len(computed_blocks.blocks[0]) == 3
|
||||
assert num_computed_tokens == 3 * block_size
|
||||
@@ -1835,7 +1840,7 @@ def test_cache_key_salting():
|
||||
# Test cache miss with same content but different salt.
|
||||
token_ids = common_token_ids + [4] * 11
|
||||
req2 = make_request("2", token_ids, block_size, sha256, cache_salt="salt2")
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
|
||||
assert len(computed_blocks.blocks[0]) == 0
|
||||
assert num_computed_tokens == 0
|
||||
block_hashes = req2.block_hashes
|
||||
@@ -1869,7 +1874,7 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
|
||||
# | Common-0 | Common-1 | Common-2 | ... |
|
||||
common_token_ids = [i for i in range(3) for _ in range(16)]
|
||||
req0 = make_request("0", common_token_ids, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
manager.allocate_slots(
|
||||
@@ -1881,7 +1886,7 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
|
||||
|
||||
# | Common-0 | Common-1 | Common-2 | Req1-3 | Req1-4 | Req1-5 | ... |
|
||||
req1 = make_request("1", common_token_ids * 2, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert computed_blocks.blocks[0] == block_part0
|
||||
assert num_computed_tokens == 3 * 16
|
||||
manager.allocate_slots(
|
||||
@@ -1899,7 +1904,7 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
|
||||
# | Common-0 | Common-1 | Common-2 | Req1-3 (F) | Req1-4 (F) |
|
||||
# | Req1-5(F)| Req2-0 | Req2-1 | ... |
|
||||
req2 = make_request("2", [7] * block_size * 2, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
manager.allocate_slots(
|
||||
@@ -1914,7 +1919,7 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
|
||||
# In this case, the ref_cnt of the computed blocks should not be changed.
|
||||
assert manager.block_pool.free_block_queue.num_free_blocks == 5
|
||||
req3 = make_request("3", common_token_ids * 3, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req3)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req3)
|
||||
assert computed_blocks.blocks[0] == block_part1
|
||||
assert num_computed_tokens == 6 * 16
|
||||
# Req3 cannot be allocated.
|
||||
@@ -1949,7 +1954,7 @@ def test_reset_prefix_cache():
|
||||
unique_token_ids = [4] * 7
|
||||
all_token_ids = full_block_token_ids + unique_token_ids
|
||||
req1 = make_request("1", all_token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req1)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req1)
|
||||
assert len(req1.block_hashes) == 3
|
||||
assert len(computed_blocks.blocks[0]) == 3
|
||||
blocks = manager.allocate_slots(
|
||||
@@ -1984,7 +1989,7 @@ def test_prefix_cache_stats_disabled():
|
||||
|
||||
# Call all functions that check whether log_stats is disabled.
|
||||
req = make_request("0", list(range(16)), block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert num_computed_tokens == 0
|
||||
manager.allocate_slots(
|
||||
@@ -2591,7 +2596,7 @@ def test_eagle_enabled_removes_last_block():
|
||||
req = make_request("divisible_request", token_ids, block_size, sha256)
|
||||
|
||||
# Prime the cache
|
||||
computed_blocks, _ = manager.get_computed_blocks(req)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req)
|
||||
manager.allocate_slots(
|
||||
req, len(token_ids), len(computed_blocks.blocks[0]) * 16, computed_blocks
|
||||
)
|
||||
@@ -2599,7 +2604,7 @@ def test_eagle_enabled_removes_last_block():
|
||||
|
||||
# New request with same tokens + Eagle enabled
|
||||
req_eagle = make_request("eagle_divisible", token_ids, block_size, sha256)
|
||||
computed_blocks, num_tokens = manager.get_computed_blocks(req_eagle)
|
||||
computed_blocks, num_tokens, _ = manager.get_computed_blocks(req_eagle)
|
||||
|
||||
# Should retain 1 block:
|
||||
# 1. Original 3 blocks → pop last hash → 2 matched blocks
|
||||
@@ -2623,7 +2628,7 @@ def test_eagle_with_partial_blocks():
|
||||
req = make_request("partial_block_test", token_ids, block_size, sha256)
|
||||
|
||||
# Prime the cache
|
||||
computed_blocks, _ = manager.get_computed_blocks(req)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req)
|
||||
manager.allocate_slots(
|
||||
req, len(token_ids), len(computed_blocks.blocks[0]) * 16, computed_blocks
|
||||
)
|
||||
@@ -2631,7 +2636,7 @@ def test_eagle_with_partial_blocks():
|
||||
|
||||
# New request with Eagle enabled
|
||||
req_eagle = make_request("partial_eagle", token_ids, block_size, sha256)
|
||||
computed_blocks, num_tokens = manager.get_computed_blocks(req_eagle)
|
||||
computed_blocks, num_tokens, _ = manager.get_computed_blocks(req_eagle)
|
||||
# Original match: 2 full blocks → Eagle removes 1 → 1 remaining
|
||||
assert len(computed_blocks.blocks[0]) == 1
|
||||
assert num_tokens == 1 * block_size
|
||||
@@ -2664,7 +2669,7 @@ def test_eagle_with_sliding_window():
|
||||
req = make_request("partial_block_test", token_ids, block_size, sha256)
|
||||
|
||||
# Prime the cache
|
||||
computed_blocks, _ = manager.get_computed_blocks(req)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req)
|
||||
manager.allocate_slots(
|
||||
req, len(token_ids), len(computed_blocks.blocks[0]) * 16, computed_blocks
|
||||
)
|
||||
@@ -2675,7 +2680,7 @@ def test_eagle_with_sliding_window():
|
||||
|
||||
# New request with Eagle enabled
|
||||
req_eagle = make_request("partial_eagle", token_ids, block_size, sha256)
|
||||
computed_blocks, num_tokens = manager.get_computed_blocks(req_eagle)
|
||||
computed_blocks, num_tokens, _ = manager.get_computed_blocks(req_eagle)
|
||||
# Original match: 2 full blocks → Eagle removes 1 → 1 remaining
|
||||
assert len(computed_blocks.blocks[0]) == 1
|
||||
assert num_tokens == 1 * block_size
|
||||
@@ -2695,7 +2700,7 @@ def test_eagle_with_sliding_window():
|
||||
req_after_evict = make_request(
|
||||
"partial_eagle_after_evict", token_ids, block_size, sha256
|
||||
)
|
||||
computed_blocks, num_tokens = manager.get_computed_blocks(req_after_evict)
|
||||
computed_blocks, num_tokens, _ = manager.get_computed_blocks(req_after_evict)
|
||||
# Cache miss. The only hit prefix is [NULL_BLOCK, BLOCK_2] if eagle is
|
||||
# not considered. But after dropping the last matched block due to eagle,
|
||||
# there will be no matched prefix.
|
||||
@@ -2754,7 +2759,7 @@ def test_eagle_swa_alignment_caches_extra_block():
|
||||
# Prime the cache with a long prompt (16 swa blocks = 4 aligned segments).
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
req0 = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req0)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req0)
|
||||
blocks = manager.allocate_slots(
|
||||
req0,
|
||||
len(token_ids),
|
||||
@@ -2768,7 +2773,7 @@ def test_eagle_swa_alignment_caches_extra_block():
|
||||
# Without the fix, ``num_computed_tokens`` is 0; with the fix, it lands at
|
||||
# an alignment boundary (multiple of 32 tokens, minus the EAGLE drop).
|
||||
req1 = make_request("1", token_ids, block_size, sha256)
|
||||
_, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
_, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed_tokens > 0, (
|
||||
"EAGLE + SWA with sliding_window <= alignment failed to find any "
|
||||
"cache hit; the +1 block past each segment boundary must be cached."
|
||||
@@ -2823,7 +2828,7 @@ def test_eagle_swa_boundary_caches_post_boundary_block():
|
||||
|
||||
token_ids = [i for i in range(5) for _ in range(block_size)]
|
||||
req0 = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req0)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req0)
|
||||
blocks = manager.allocate_slots(
|
||||
req0,
|
||||
len(token_ids),
|
||||
@@ -2838,7 +2843,7 @@ def test_eagle_swa_boundary_caches_post_boundary_block():
|
||||
manager.free(req0)
|
||||
|
||||
req1 = make_request("1", token_ids + [999], block_size, sha256)
|
||||
_, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
_, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed_tokens == 4 * block_size
|
||||
|
||||
|
||||
@@ -2879,7 +2884,7 @@ def test_eagle_grouped_swa_siblings_use_same_cache_mask():
|
||||
|
||||
token_ids = [i for i in range(9) for _ in range(block_size)]
|
||||
req0 = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req0)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req0)
|
||||
blocks = manager.allocate_slots(
|
||||
req0,
|
||||
len(token_ids),
|
||||
@@ -2894,7 +2899,7 @@ def test_eagle_grouped_swa_siblings_use_same_cache_mask():
|
||||
manager.free(req0)
|
||||
|
||||
req1 = make_request("1", token_ids + [999], block_size, sha256)
|
||||
_, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
_, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed_tokens == 8 * block_size
|
||||
|
||||
|
||||
@@ -2938,7 +2943,7 @@ def test_different_block_size():
|
||||
common_token_ids = [i for i in range(10) for _ in range(block_size)]
|
||||
|
||||
req0 = make_request("0", common_token_ids, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert not computed_blocks.blocks[0]
|
||||
assert not computed_blocks.blocks[1]
|
||||
assert num_computed_tokens == 0
|
||||
@@ -2947,13 +2952,13 @@ def test_different_block_size():
|
||||
)
|
||||
assert blocks.get_block_ids() == ([1, 2, 3, 4], [5, 6, 7, 8, 9, 10, 11])
|
||||
req1 = make_request("1", common_token_ids[: 7 * block_size + 1], block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert len(computed_blocks.blocks[0]) == 3
|
||||
assert len(computed_blocks.blocks[1]) == 6
|
||||
assert num_computed_tokens == 6 * 16
|
||||
|
||||
req2 = make_request("2", common_token_ids[: 6 * block_size + 1], block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2)
|
||||
assert len(computed_blocks.blocks[0]) == 3
|
||||
assert len(computed_blocks.blocks[1]) == 6
|
||||
assert num_computed_tokens == 6 * 16
|
||||
@@ -2967,7 +2972,7 @@ def test_different_block_size():
|
||||
manager.block_pool.cached_block_hash_to_block.pop(
|
||||
make_block_hash_with_group_id(req1.block_hashes[5], 1), 10
|
||||
)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert len(computed_blocks.blocks[0]) == 2
|
||||
assert len(computed_blocks.blocks[1]) == 4
|
||||
assert num_computed_tokens == 4 * 16
|
||||
@@ -3020,7 +3025,7 @@ def test_hybrid_cache_blocks_swa_tail_window_only():
|
||||
# 8 hash-blocks of 8 tokens (64 tokens, two lcm-aligned segments).
|
||||
token_ids = [i for i in range(8) for _ in range(block_size)]
|
||||
req = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req)
|
||||
blocks = manager.allocate_slots(
|
||||
req,
|
||||
8 * block_size,
|
||||
@@ -3089,7 +3094,7 @@ def test_hybrid_cache_blocks_clamped_to_lcm():
|
||||
# to 96 tokens — SWA caches 6 hashes, full-attn caches 3.
|
||||
token_ids = [i for i in range(7) for _ in range(block_size)]
|
||||
req = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req)
|
||||
blocks = manager.allocate_slots(
|
||||
req,
|
||||
7 * block_size,
|
||||
@@ -3154,7 +3159,7 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch):
|
||||
# 96-token replay boundary, and the 128-token interval boundary.
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
req = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req)
|
||||
blocks = manager.allocate_slots(
|
||||
req,
|
||||
len(token_ids),
|
||||
@@ -3296,13 +3301,13 @@ def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch):
|
||||
|
||||
token_ids = fill_request("fill_0", 0)
|
||||
replay_req = make_request("replay", token_ids[:1800], hash_block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(replay_req)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(replay_req)
|
||||
assert num_computed_tokens == 1024
|
||||
assert [len(blocks) for blocks in computed_blocks.blocks] == [4, 16, 128, 256]
|
||||
|
||||
fill_request("fill_1", 100_000)
|
||||
replay_req = make_request("replay_again", token_ids[:1800], hash_block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(replay_req)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(replay_req)
|
||||
assert num_computed_tokens == 1024
|
||||
assert [len(blocks) for blocks in computed_blocks.blocks] == [4, 16, 128, 256]
|
||||
|
||||
@@ -3345,7 +3350,7 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc
|
||||
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
req0 = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req0)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req0)
|
||||
blocks = manager.allocate_slots(
|
||||
req0,
|
||||
len(token_ids),
|
||||
@@ -3369,14 +3374,14 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc
|
||||
assert retained_swa_block[0].ref_cnt == 0
|
||||
|
||||
req1 = make_request("1", token_ids, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
# Full prompt hits intentionally recompute the final block for logits, so
|
||||
# the longest usable hit is the previous LCM boundary: 96 tokens.
|
||||
assert num_computed_tokens == 12 * block_size
|
||||
assert len(computed_blocks.blocks[1]) == 12
|
||||
|
||||
shorter_req = make_request("2", token_ids[: 12 * block_size], block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(shorter_req)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(shorter_req)
|
||||
assert num_computed_tokens == 0
|
||||
assert len(computed_blocks.blocks[1]) == 0
|
||||
|
||||
@@ -3429,7 +3434,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch):
|
||||
# 104 tokens, and that tail is two 8-token blocks wide: hashes 11 and 12.
|
||||
token_ids = [i for i in range(15) for _ in range(block_size)] + [15] * 7
|
||||
req0 = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
req0,
|
||||
@@ -3451,7 +3456,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch):
|
||||
manager.free(req0)
|
||||
|
||||
req1 = make_request("1", token_ids, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed_tokens == 12 * block_size
|
||||
assert [len(blocks) for blocks in computed_blocks.blocks] == [3, 12]
|
||||
|
||||
@@ -3858,7 +3863,7 @@ def test_swa_free_split_keeps_cached_tail_ahead_of_scratch(monkeypatch):
|
||||
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
req = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req)
|
||||
blocks = manager.allocate_slots(
|
||||
req,
|
||||
len(token_ids),
|
||||
@@ -3941,7 +3946,7 @@ def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch):
|
||||
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
req = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req)
|
||||
blocks = manager.allocate_slots(
|
||||
req,
|
||||
len(token_ids),
|
||||
@@ -3965,7 +3970,7 @@ def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch):
|
||||
|
||||
# A replay of the same prompt hits the latest replayable boundary (240).
|
||||
replay = make_request("1", token_ids, block_size, sha256)
|
||||
_, num_computed = manager.get_computed_blocks(replay)
|
||||
_, num_computed, _ = manager.get_computed_blocks(replay)
|
||||
assert num_computed == 240
|
||||
|
||||
|
||||
@@ -3977,7 +3982,7 @@ def test_pure_swa_retention_latest_only(monkeypatch):
|
||||
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
req = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req)
|
||||
blocks = manager.allocate_slots(
|
||||
req,
|
||||
len(token_ids),
|
||||
@@ -3997,7 +4002,7 @@ def test_pure_swa_retention_latest_only(monkeypatch):
|
||||
assert cached == {14}
|
||||
|
||||
replay = make_request("1", token_ids, block_size, sha256)
|
||||
_, num_computed = manager.get_computed_blocks(replay)
|
||||
_, num_computed, _ = manager.get_computed_blocks(replay)
|
||||
assert num_computed == 240
|
||||
|
||||
|
||||
@@ -4010,7 +4015,7 @@ def test_pure_swa_retention_dense_default_caches_all(monkeypatch):
|
||||
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
req = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req)
|
||||
computed_blocks, _, _ = manager.get_computed_blocks(req)
|
||||
blocks = manager.allocate_slots(
|
||||
req,
|
||||
len(token_ids),
|
||||
@@ -4052,7 +4057,7 @@ def test_mamba_reachable_block_mask_sparsifies_retention():
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=False,
|
||||
retention_interval=retention_interval,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
reachable_boundaries=(num_prompt_tokens - 1,),
|
||||
)
|
||||
return None if m is None else {i for i, v in enumerate(m) if v}
|
||||
|
||||
@@ -4065,3 +4070,242 @@ def test_mamba_reachable_block_mask_sparsifies_retention():
|
||||
assert retained(64) == {3, 7, 11, 14, 15}
|
||||
# interval 0 -> only the latest replay boundary (block 14).
|
||||
assert retained(0) == {14}
|
||||
|
||||
|
||||
def test_mamba_reachable_block_mask_pins_shared_prefix():
|
||||
"""A Marconi-detected shared prefix (``shared_prefix_boundary``) lands before
|
||||
``num_prompt`` so the replay-boundary rule alone would drop it. The mask must
|
||||
pin the single state block ending on that boundary so sparse retention does
|
||||
not defeat cross-request shared-prefix reuse."""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
|
||||
block_size = 16
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
|
||||
def retained(retention_interval, shared_prefix_boundary, end_block=16):
|
||||
boundaries = [255] # replay boundary (num_prompt 256 - 1)
|
||||
if shared_prefix_boundary:
|
||||
boundaries.append(shared_prefix_boundary)
|
||||
m = MambaManager.reachable_block_mask(
|
||||
start_block=0,
|
||||
end_block=end_block,
|
||||
alignment_tokens=block_size,
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=False,
|
||||
retention_interval=retention_interval,
|
||||
reachable_boundaries=boundaries,
|
||||
)
|
||||
return None if m is None else {i for i, v in enumerate(m) if v}
|
||||
|
||||
# interval 0 keeps only replay boundary 14; the shared prefix at token 96
|
||||
# (state block 96//16 - 1 = 5) is now pinned too.
|
||||
assert retained(0, 96) == {5, 14}
|
||||
# A non-aligned boundary floors to the enclosing aligned boundary.
|
||||
assert retained(0, 100) == {5, 14}
|
||||
# Coexists with segment tails (interval 64 -> {3,7,11,15} + replay 14).
|
||||
assert retained(64, 96) == {3, 5, 7, 11, 14, 15}
|
||||
# Dense default ignores the hint (nothing to sparsify).
|
||||
assert retained(None, 96) is None
|
||||
# Out-of-range boundary is a no-op (only replay 14 remains).
|
||||
assert retained(0, 16 * block_size * 2) == {14}
|
||||
# No boundary given -> unchanged replay-only behavior.
|
||||
assert retained(0, 0) == {14}
|
||||
assert retained(0, None) == {14}
|
||||
|
||||
|
||||
def test_mamba_shared_prefix_survives_zero_retention(monkeypatch):
|
||||
"""Manager-level check of the full wiring: a pinned shared-prefix boundary
|
||||
(``Request.shared_prefix_boundary``, set by the scheduler on Marconi-style
|
||||
detection) keeps its Mamba state block cached under
|
||||
``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``, which otherwise retains only the
|
||||
end-of-prompt replay boundary. Without this, a shared prefix (junction
|
||||
before ``num_prompt``) would be recomputed by every sharing request."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")
|
||||
block_size = 16
|
||||
|
||||
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
|
||||
def cached_mamba_blocks(shared_prefix_boundary):
|
||||
# Fresh manager per scenario so cached blocks don't leak between runs.
|
||||
manager = make_kv_cache_manager(
|
||||
_make_hybrid_kv_cache_config(block_size, 100, ["full", "mamba"]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
req = make_request("r", token_ids, block_size, sha256)
|
||||
req.shared_prefix_boundary = shared_prefix_boundary
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
|
||||
blocks = manager.allocate_slots(
|
||||
req, len(token_ids), num_computed, computed_blocks
|
||||
)
|
||||
assert blocks is not None
|
||||
pool = manager.block_pool
|
||||
return {
|
||||
i
|
||||
for i in range(16)
|
||||
if pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
|
||||
is not None
|
||||
}
|
||||
|
||||
# Without a pinned boundary, retention=0 keeps only the replay boundary (14).
|
||||
assert cached_mamba_blocks(0) == {14}
|
||||
# Pinning the shared prefix at token 96 (state block 5) retains it too, so a
|
||||
# later request sharing that prefix can hit the Mamba state.
|
||||
assert cached_mamba_blocks(96) == {5, 14}
|
||||
|
||||
|
||||
def test_mamba_shared_prefix_reuse_under_zero_retention(monkeypatch):
|
||||
"""Full cross-request Marconi flow: a partial shared prefix cached by the
|
||||
detecting request must stay reusable by a later request under
|
||||
``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without the pin the junction is
|
||||
masked out and the later request misses; with it (and under dense) the reuse
|
||||
is preserved."""
|
||||
block_size = 16
|
||||
|
||||
def last_req_hit(retention, pin):
|
||||
if retention is None:
|
||||
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention))
|
||||
manager = make_kv_cache_manager(
|
||||
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba_align"]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
shared = [7 for _ in range(2 * block_size)] # 2-block shared prefix
|
||||
|
||||
def distinct(v):
|
||||
return [v for _ in range(2 * block_size)]
|
||||
|
||||
# req0 primes the shared prefix's (dense) attention cache; align-mode
|
||||
# Mamba keeps only its own tail, not the shared-prefix junction.
|
||||
req0 = make_request("0", shared + distinct(50), block_size, sha256)
|
||||
cb, nc, _ = manager.get_computed_blocks(req0)
|
||||
manager.allocate_slots(req0, len(req0.all_token_ids), nc, cb)
|
||||
|
||||
# req1 detects the shared prefix (attn hit, Mamba lag) and, via the
|
||||
# Marconi chunk, caches the junction's Mamba state.
|
||||
req1 = make_request("1", shared + distinct(60), block_size, sha256)
|
||||
cb, nc, boundary = manager.get_computed_blocks(req1)
|
||||
assert boundary == 2 * block_size # junction detected at 2 shared blocks
|
||||
if pin:
|
||||
req1.shared_prefix_boundary = boundary
|
||||
# Simulate the Marconi chunk: schedule up to the junction (boundary - nc).
|
||||
manager.allocate_slots(req1, boundary - nc, nc, cb)
|
||||
|
||||
# req2 shares the same prefix: does it reuse the cached junction?
|
||||
req2 = make_request("2", shared + distinct(70), block_size, sha256)
|
||||
_, nc2, _ = manager.get_computed_blocks(req2)
|
||||
return nc2
|
||||
|
||||
# Dense retains the junction -> reuse works (baseline ceiling).
|
||||
assert last_req_hit(retention=None, pin=False) == 2 * block_size
|
||||
# retention=0 without the pin masks the junction out -> reuse lost (the bug).
|
||||
assert last_req_hit(retention=0, pin=False) == 0
|
||||
# retention=0 with the pin keeps the junction -> reuse restored.
|
||||
assert last_req_hit(retention=0, pin=True) == 2 * block_size
|
||||
|
||||
|
||||
def test_swa_reachable_block_mask_pins_shared_prefix():
|
||||
"""SWA analog of the Mamba pin: the shared-prefix junction must keep the
|
||||
``need``-block sliding-window tail ending on that boundary (not a single
|
||||
block), so a windowed hit can land there under sparse retention."""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
|
||||
|
||||
block_size = 16
|
||||
|
||||
def retained(retention, boundary, window, end_block=16):
|
||||
spec = SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=window,
|
||||
)
|
||||
boundaries = [255] # replay boundary (num_prompt 256 - 1)
|
||||
if boundary:
|
||||
boundaries.append(boundary)
|
||||
m = SlidingWindowManager.reachable_block_mask(
|
||||
start_block=0,
|
||||
end_block=end_block,
|
||||
alignment_tokens=block_size,
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=False,
|
||||
retention_interval=retention,
|
||||
reachable_boundaries=boundaries,
|
||||
)
|
||||
return None if m is None else {i for i, v in enumerate(m) if v}
|
||||
|
||||
# window == block_size -> need = cdiv(15, 16) = 1: single-block tail (like
|
||||
# Mamba). Junction at token 96 -> block 5; replay boundary -> block 14.
|
||||
assert retained(0, 96, block_size) == {5, 14}
|
||||
# window == 3 * block_size -> need = cdiv(47, 16) = 3: the junction keeps a
|
||||
# 3-block WINDOW {3,4,5} (the SWA distinction), plus replay window {12,13,14}.
|
||||
assert retained(0, 96, 3 * block_size) == {3, 4, 5, 12, 13, 14}
|
||||
# Coexists with segment tails (interval 64, need=1): {3,7,11,15} + replay 14
|
||||
# + junction 5.
|
||||
assert retained(64, 96, block_size) == {3, 5, 7, 11, 14, 15}
|
||||
# Dense (all blocks reachable) ignores the hint.
|
||||
assert retained(None, 96, block_size) is None
|
||||
# No boundary -> unchanged replay-only behavior.
|
||||
assert retained(0, 0, block_size) == {14}
|
||||
|
||||
|
||||
def test_swa_shared_prefix_reuse_under_zero_retention(monkeypatch):
|
||||
"""SWA cross-request analog: a partial shared prefix's sliding-window tail
|
||||
must stay reusable under ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without
|
||||
the pin the junction window is masked out and a later request misses; with
|
||||
it (and under dense) reuse is preserved."""
|
||||
block_size = 16
|
||||
|
||||
def last_req_hit(retention, pin):
|
||||
if retention is None:
|
||||
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention))
|
||||
manager = make_kv_cache_manager(
|
||||
_make_hybrid_kv_cache_config(block_size, 200, ["full", "sliding_window"]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
shared = [7 for _ in range(4 * block_size)] # 4-block shared prefix
|
||||
|
||||
def distinct(v):
|
||||
return [v for _ in range(2 * block_size)]
|
||||
|
||||
# req0 primes the (dense) full-attention cache; SWA under RET=0 keeps
|
||||
# only its own replay window, not the shared-prefix boundary.
|
||||
req0 = make_request("0", shared + distinct(50), block_size, sha256)
|
||||
cb, nc, _ = manager.get_computed_blocks(req0)
|
||||
manager.allocate_slots(req0, len(req0.all_token_ids), nc, cb)
|
||||
|
||||
# req1 detects the shared prefix (full-attn hit, SWA lag). No chunk for
|
||||
# SWA -- the pin fires during normal prefill.
|
||||
req1 = make_request("1", shared + distinct(60), block_size, sha256)
|
||||
cb, nc, boundary = manager.get_computed_blocks(req1)
|
||||
if retention == 0:
|
||||
assert boundary == 4 * block_size # junction detected when SWA lags
|
||||
if pin and boundary:
|
||||
req1.shared_prefix_boundary = boundary
|
||||
manager.allocate_slots(req1, len(req1.all_token_ids), nc, cb)
|
||||
|
||||
# req2 shares the same prefix: does its SWA window reuse the junction?
|
||||
req2 = make_request("2", shared + distinct(70), block_size, sha256)
|
||||
_, nc2, _ = manager.get_computed_blocks(req2)
|
||||
return nc2
|
||||
|
||||
# Dense keeps every SWA tail -> reuse works (ceiling).
|
||||
assert last_req_hit(retention=None, pin=False) == 4 * block_size
|
||||
# retention=0 without the pin drops the junction window -> reuse lost (bug).
|
||||
assert last_req_hit(retention=0, pin=False) == 0
|
||||
# retention=0 with the pin keeps the junction window -> reuse restored.
|
||||
assert last_req_hit(retention=0, pin=True) == 4 * block_size
|
||||
|
||||
@@ -235,6 +235,9 @@ class MooncakeStoreCoordinator:
|
||||
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
|
||||
assert manager_cls is not None
|
||||
use_eagle = g_idx in self.eagle_group_ids
|
||||
reachable_boundaries = (
|
||||
() if num_prompt_tokens is None else (num_prompt_tokens - 1,)
|
||||
)
|
||||
mask = manager_cls.reachable_block_mask(
|
||||
start_block=start_chunk,
|
||||
end_block=end_chunk,
|
||||
@@ -242,7 +245,7 @@ class MooncakeStoreCoordinator:
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=use_eagle,
|
||||
retention_interval=retention_interval,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
reachable_boundaries=reachable_boundaries,
|
||||
)
|
||||
if mask is not None:
|
||||
assert len(mask) == end_chunk - start_chunk
|
||||
|
||||
@@ -370,7 +370,10 @@ class KVCacheCoordinator(ABC):
|
||||
self,
|
||||
block_hashes: list[BlockHash],
|
||||
max_cache_hit_length: int,
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int, int]:
|
||||
"""Returns the per-group hit blocks, the hit length, and the number of
|
||||
``num_uncached_common_prefix_tokens`` (a shared prefix that a
|
||||
sparse-retention group has not cached yet; 0 unless hybrid)."""
|
||||
pass
|
||||
|
||||
def new_step_starts(self) -> None:
|
||||
@@ -422,11 +425,11 @@ class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator):
|
||||
self,
|
||||
block_hashes: list[BlockHash],
|
||||
max_cache_hit_length: int,
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int, int]:
|
||||
blocks: tuple[list[KVCacheBlock], ...] = tuple(
|
||||
[] for _ in range(self.num_single_type_manager)
|
||||
)
|
||||
return blocks, 0
|
||||
return blocks, 0, 0
|
||||
|
||||
|
||||
class UnitaryKVCacheCoordinator(KVCacheCoordinator):
|
||||
@@ -486,7 +489,7 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator):
|
||||
self,
|
||||
block_hashes: list[BlockHash],
|
||||
max_cache_hit_length: int,
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int, int]:
|
||||
hit_blocks, hit_length = self.single_type_managers[0].find_longest_cache_hit(
|
||||
block_hashes=block_hashes,
|
||||
max_length=max_cache_hit_length,
|
||||
@@ -498,7 +501,8 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator):
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
pcp_world_size=self.pcp_world_size,
|
||||
)
|
||||
return hit_blocks, hit_length
|
||||
# Single group: nothing "uncached common" -- no other group to lag it.
|
||||
return hit_blocks, hit_length, 0
|
||||
|
||||
|
||||
class SpecGroup(NamedTuple):
|
||||
@@ -675,7 +679,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
self,
|
||||
block_hashes: list[BlockHash],
|
||||
max_cache_hit_length: int,
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int, int]:
|
||||
"""
|
||||
Find the longest cache hit using an iterative fixed-point algorithm.
|
||||
|
||||
@@ -691,7 +695,9 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
Returns:
|
||||
A tuple containing:
|
||||
- A tuple of the cache hit blocks for each single type manager.
|
||||
- The number of tokens of the longest cache hit.
|
||||
- The number of tokens of the reconciled (combined) cache hit.
|
||||
- ``num_uncached_common_prefix_tokens``: a shared prefix that a
|
||||
sparse-retention group has not cached yet (0 unless hybrid).
|
||||
"""
|
||||
|
||||
num_groups = len(self.kv_cache_config.kv_cache_groups)
|
||||
@@ -794,12 +800,14 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
del blks[num_blocks:]
|
||||
hit_length_by_group[group_id] = hit_length
|
||||
|
||||
# Uncached shared prefix detection: If any attn. group cached a longer prefix
|
||||
# than the current prefix, it is an uncached common prefix across requests:
|
||||
self.num_uncached_common_prefix_tokens = longest_hit_length - hit_length
|
||||
return tuple(
|
||||
# Uncached shared prefix detection: if any attn. group cached a longer
|
||||
# prefix than the reconciled hit, it is an uncached common prefix across
|
||||
# requests that a sparse-retention group hasn't cached yet.
|
||||
num_uncached_common_prefix_tokens = longest_hit_length - hit_length
|
||||
cache_hit_blocks = tuple(
|
||||
blocks if blocks is not None else [] for blocks in hit_blocks_by_group
|
||||
), hit_length
|
||||
)
|
||||
return cache_hit_blocks, hit_length, num_uncached_common_prefix_tokens
|
||||
|
||||
def find_longest_cache_hit_per_group(
|
||||
self,
|
||||
|
||||
@@ -204,7 +204,7 @@ class KVCacheManager:
|
||||
self.prefix_cache_stats = PrefixCacheStats()
|
||||
return stats
|
||||
|
||||
def get_computed_blocks(self, request: Request) -> tuple[KVCacheBlocks, int]:
|
||||
def get_computed_blocks(self, request: Request) -> tuple[KVCacheBlocks, int, int]:
|
||||
"""Get the computed (cached) blocks for the request.
|
||||
Note that the computed blocks must be full.
|
||||
|
||||
@@ -215,13 +215,18 @@ class KVCacheManager:
|
||||
A tuple containing:
|
||||
- A list of blocks that are computed for the request.
|
||||
- The number of computed tokens.
|
||||
- ``shared_prefix_boundary``: the block-aligned token position of
|
||||
a shared prefix that a sparse-retention group (Mamba / sliding
|
||||
window) has not cached yet (Marconi-style APC), or 0 if none.
|
||||
Pinned so ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL`` does not drop
|
||||
the junction and defeat cross-request reuse.
|
||||
"""
|
||||
# We skip finding the prefix cache hit when prefix caching is
|
||||
# disabled or the request is marked as skipping kv cache read
|
||||
# (which happens when the request requires prompt logprobs
|
||||
# or calls a pooling model with all pooling).
|
||||
if not self.enable_caching or request.skip_reading_prefix_cache:
|
||||
return self.empty_kv_cache_blocks, 0
|
||||
return self.empty_kv_cache_blocks, 0, 0
|
||||
|
||||
# NOTE: When all tokens hit the cache, we must recompute the last token
|
||||
# to obtain logits. Thus, set max_cache_hit_length to prompt_length - 1.
|
||||
@@ -230,7 +235,7 @@ class KVCacheManager:
|
||||
# num_computed_tokens to be block-size aligned. Removing this limitation
|
||||
# could slightly improve performance in the future.
|
||||
max_cache_hit_length = request.num_tokens - 1
|
||||
computed_blocks, num_new_computed_tokens = (
|
||||
computed_blocks, num_new_computed_tokens, num_uncached = (
|
||||
self.coordinator.find_longest_cache_hit(
|
||||
request.block_hashes, max_cache_hit_length
|
||||
)
|
||||
@@ -256,6 +261,14 @@ class KVCacheManager:
|
||||
group_idx,
|
||||
)
|
||||
|
||||
# The junction to pin is where the lagging sparse-retention group stops
|
||||
# (``num_new_computed_tokens``) plus the uncached shared prefix -- i.e.
|
||||
# the longest single-group hit. Sub-block gaps are left to the mask,
|
||||
# which floors to the alignment boundary (a no-op there).
|
||||
shared_prefix_boundary = (
|
||||
num_new_computed_tokens + num_uncached if num_uncached else 0
|
||||
)
|
||||
|
||||
if self.log_stats:
|
||||
assert self.prefix_cache_stats is not None
|
||||
self.prefix_cache_stats.record(
|
||||
@@ -264,7 +277,8 @@ class KVCacheManager:
|
||||
preempted=request.num_preemptions > 0,
|
||||
)
|
||||
|
||||
return self.create_kv_cache_blocks(computed_blocks), num_new_computed_tokens
|
||||
blocks = self.create_kv_cache_blocks(computed_blocks)
|
||||
return blocks, num_new_computed_tokens, shared_prefix_boundary
|
||||
|
||||
def allocate_slots(
|
||||
self,
|
||||
|
||||
@@ -350,85 +350,69 @@ class Scheduler(SchedulerInterface):
|
||||
num_new_tokens: int,
|
||||
num_new_local_computed_tokens: int = 0,
|
||||
num_external_computed_tokens: int = 0,
|
||||
num_uncached_common_prefix_tokens: int = 0,
|
||||
) -> int:
|
||||
num_computed_tokens = (
|
||||
"""Clip a prefill chunk so it ends where Mamba state must be cached.
|
||||
|
||||
In "align" cache mode the SSM state is only materialized at chunk
|
||||
ends, so chunk ends are steered onto cacheable positions: block
|
||||
boundaries by default, plus mandatory early stops (the prompt's
|
||||
partial-tail hash boundary, a detected shared-prefix junction).
|
||||
"""
|
||||
start = (
|
||||
request.num_computed_tokens
|
||||
+ num_new_local_computed_tokens
|
||||
+ num_external_computed_tokens
|
||||
)
|
||||
# Perform block-aligned splitting at prefill phase, including:
|
||||
# * non-resumed requests: num_computed_tokens < num_prompt_tokens + 0
|
||||
# * resumed requests: num_computed_tokens < (
|
||||
# num_prompt_tokens + num_output_tokens
|
||||
# )
|
||||
# NOTE: Use `request.num_tokens - 1` to bypass normal decoding.
|
||||
if num_computed_tokens < max(request.num_prompt_tokens, request.num_tokens - 1):
|
||||
# To enable block-aligned caching of the Mamba state, `num_new_tokens`
|
||||
# must be a multiple of `block_size`.
|
||||
# As an exception, if `num_new_tokens` is less than `block_size`, the
|
||||
# state is simply not cached, requiring no special handling.
|
||||
# Additionally, when Eagle mode is enabled, FullAttn prunes the last
|
||||
# matching block. To prevent this from causing a Mamba cache miss, the
|
||||
# last chunk must be not smaller than `block_size`.
|
||||
block_size = self.cache_config.block_size
|
||||
last_cache_position = request.num_tokens - request.num_tokens % block_size
|
||||
# eagle prune
|
||||
if self.use_eagle:
|
||||
last_cache_position = max(last_cache_position - block_size, 0)
|
||||
num_computed_tokens_after_sched = num_computed_tokens + num_new_tokens
|
||||
next_boundary = (num_computed_tokens // block_size + 1) * block_size
|
||||
if (
|
||||
num_computed_tokens % block_size != 0
|
||||
and next_boundary <= last_cache_position
|
||||
and num_computed_tokens_after_sched > next_boundary
|
||||
):
|
||||
# Resumed mid-block (partial hash hit): stop the first chunk
|
||||
# at the next block boundary so chunk ends re-align to blocks
|
||||
# and the boundary state is materialized before running past.
|
||||
num_new_tokens = next_boundary - num_computed_tokens
|
||||
elif num_computed_tokens_after_sched < last_cache_position:
|
||||
# Align the chunk END (not its length) to block_size;
|
||||
# identical to flooring the length when the start is
|
||||
# block-aligned. May yield 0 (insufficient budget to reach
|
||||
# the next boundary); the caller then skips the request.
|
||||
aligned_end = num_computed_tokens_after_sched // block_size * block_size
|
||||
num_new_tokens = max(aligned_end - num_computed_tokens, 0)
|
||||
elif (
|
||||
num_computed_tokens
|
||||
< last_cache_position
|
||||
< num_computed_tokens_after_sched
|
||||
):
|
||||
# force to cache the last chunk
|
||||
num_new_tokens = last_cache_position - num_computed_tokens
|
||||
elif self.mamba_partial_cache_hit:
|
||||
# Prefill of the final partial block: stop once at the
|
||||
# prompt's last hash boundary so the mamba partial tail entry
|
||||
# can be registered.
|
||||
tail_boundary = (
|
||||
request.num_prompt_tokens
|
||||
// self.hash_block_size
|
||||
* self.hash_block_size
|
||||
)
|
||||
if (
|
||||
num_computed_tokens
|
||||
< tail_boundary
|
||||
< num_computed_tokens_after_sched
|
||||
and tail_boundary < request.num_prompt_tokens
|
||||
and tail_boundary > last_cache_position
|
||||
):
|
||||
num_new_tokens = tail_boundary - num_computed_tokens
|
||||
# Split only during prefill: `request.num_tokens - 1` extends this to
|
||||
# resumed requests replaying their output tokens.
|
||||
if start >= max(request.num_prompt_tokens, request.num_tokens - 1):
|
||||
return num_new_tokens
|
||||
|
||||
# Marconi cache admission optimization:
|
||||
# cache common prefixes by scheduling num_new_tokens = common prefix length
|
||||
if (
|
||||
num_uncached_common_prefix_tokens >= block_size
|
||||
and num_new_tokens > num_uncached_common_prefix_tokens
|
||||
):
|
||||
num_new_tokens = num_uncached_common_prefix_tokens
|
||||
# keep alignment to block_size
|
||||
num_new_tokens = num_new_tokens // block_size * block_size
|
||||
return num_new_tokens
|
||||
block_size = self.cache_config.block_size
|
||||
# The last block-aligned position whose state can be cached. With
|
||||
# Eagle, FullAttn prunes the last matching block, so back off one
|
||||
# block to avoid a Mamba cache miss.
|
||||
last_cache_position = request.num_tokens - request.num_tokens % block_size
|
||||
if self.use_eagle:
|
||||
last_cache_position = max(last_cache_position - block_size, 0)
|
||||
|
||||
end = start + num_new_tokens
|
||||
# Until `last_cache_position`, chunk ends must land on block
|
||||
# boundaries. May yield an empty chunk (budget cannot reach the next
|
||||
# boundary); the caller then skips the request.
|
||||
if end < last_cache_position:
|
||||
end = end // block_size * block_size
|
||||
|
||||
next_block_boundary = (start // block_size + 1) * block_size
|
||||
tail_boundary = (
|
||||
request.num_prompt_tokens // self.hash_block_size * self.hash_block_size
|
||||
if self.mamba_partial_cache_hit
|
||||
else 0
|
||||
)
|
||||
stops = (
|
||||
# Resumed mid-block (fine-grained partial hash hit): re-align to
|
||||
# the block grid before running on, so the crossed boundary's
|
||||
# state is materialized (unless it is past the cacheable range).
|
||||
next_block_boundary
|
||||
if start % block_size != 0 and next_block_boundary <= last_cache_position
|
||||
else 0,
|
||||
# Never run past the last cacheable block boundary mid-chunk.
|
||||
last_cache_position,
|
||||
# Fine-grained hits: the prompt's partial-tail entry can only be
|
||||
# registered by a chunk ending exactly at its last hash boundary.
|
||||
tail_boundary
|
||||
if last_cache_position < tail_boundary < request.num_prompt_tokens
|
||||
else 0,
|
||||
# Marconi shared-prefix junction, block-floored (a sub-block
|
||||
# junction's state is not separately cacheable): cache its state
|
||||
# so sibling requests sharing the prefix can reuse it.
|
||||
start + (request.shared_prefix_boundary - start) // block_size * block_size
|
||||
if start < request.shared_prefix_boundary < end
|
||||
else 0,
|
||||
)
|
||||
# Stop at the earliest mandatory position strictly inside the chunk.
|
||||
end = min((s for s in stops if start < s < end), default=end)
|
||||
return max(end - start, 0)
|
||||
|
||||
def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput:
|
||||
self.current_step += 1
|
||||
@@ -718,7 +702,6 @@ class Scheduler(SchedulerInterface):
|
||||
num_external_computed_tokens = 0
|
||||
load_kv_async = False
|
||||
connector_prefix_cache_queries, connector_prefix_cache_hits = 0, 0
|
||||
num_uncached_common_prefix_tokens = 0
|
||||
|
||||
# Get already-cached tokens.
|
||||
if request.num_computed_tokens == 0:
|
||||
@@ -733,8 +716,7 @@ class Scheduler(SchedulerInterface):
|
||||
):
|
||||
computed, per_group_hits = (
|
||||
self.kv_cache_manager.coordinator.find_longest_cache_hit_per_group(
|
||||
request.block_hashes,
|
||||
request.num_tokens - 1,
|
||||
request.block_hashes, request.num_tokens - 1
|
||||
)
|
||||
)
|
||||
new_computed_blocks = (
|
||||
@@ -750,6 +732,9 @@ class Scheduler(SchedulerInterface):
|
||||
# the last block) is transferred unconditionally by
|
||||
# _apply_prefix_caching in nixl/worker.py.
|
||||
num_new_local_computed_tokens = max(per_group_hits)
|
||||
# The per-group lookup does not detect an uncached shared
|
||||
# prefix, so there is no junction to pin in this path.
|
||||
request.shared_prefix_boundary = 0
|
||||
if self.kv_cache_manager.log_stats:
|
||||
assert self.kv_cache_manager.prefix_cache_stats is not None
|
||||
self.kv_cache_manager.prefix_cache_stats.record(
|
||||
@@ -758,17 +743,15 @@ class Scheduler(SchedulerInterface):
|
||||
preempted=request.num_preemptions > 0,
|
||||
)
|
||||
else:
|
||||
new_computed_blocks, num_new_local_computed_tokens = (
|
||||
self.kv_cache_manager.get_computed_blocks(request)
|
||||
)
|
||||
|
||||
# In case of hybrid models, obtain hint for Marconi-style APC logic
|
||||
if self.has_mamba_layers:
|
||||
num_uncached_common_prefix_tokens = getattr(
|
||||
self.kv_cache_manager.coordinator,
|
||||
"num_uncached_common_prefix_tokens",
|
||||
0,
|
||||
)
|
||||
(
|
||||
new_computed_blocks,
|
||||
num_new_local_computed_tokens,
|
||||
# Junction to pin (Marconi-style APC) so its
|
||||
# sparse-retention state (Mamba block / sliding-window
|
||||
# tail) survives retention and serves a later hit; 0
|
||||
# if no uncached shared prefix was detected.
|
||||
request.shared_prefix_boundary,
|
||||
) = self.kv_cache_manager.get_computed_blocks(request)
|
||||
|
||||
# Get externally-cached tokens if using a KVConnector.
|
||||
if self.connector is not None:
|
||||
@@ -906,7 +889,6 @@ class Scheduler(SchedulerInterface):
|
||||
num_new_tokens,
|
||||
num_new_local_computed_tokens,
|
||||
num_external_computed_tokens,
|
||||
num_uncached_common_prefix_tokens,
|
||||
)
|
||||
if num_new_tokens == 0:
|
||||
break
|
||||
|
||||
@@ -419,6 +419,13 @@ class SingleTypeKVCacheManager(ABC):
|
||||
if num_cached_blocks >= num_full_blocks:
|
||||
return
|
||||
|
||||
# Token boundaries whose reachable tail must be retained under sparse
|
||||
# retention: the replay boundary (``num_prompt - 1``, capped by
|
||||
# ``get_computed_blocks``) and any detected shared-prefix junction.
|
||||
reachable_boundaries = [request.num_prompt_tokens - 1]
|
||||
if request.shared_prefix_boundary:
|
||||
reachable_boundaries.append(request.shared_prefix_boundary)
|
||||
|
||||
block_mask = self.reachable_block_mask(
|
||||
start_block=num_cached_blocks,
|
||||
end_block=num_full_blocks,
|
||||
@@ -426,7 +433,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
kv_cache_spec=self.kv_cache_spec,
|
||||
use_eagle=self.use_eagle,
|
||||
retention_interval=retention_interval,
|
||||
num_prompt_tokens=request.num_prompt_tokens,
|
||||
reachable_boundaries=reachable_boundaries,
|
||||
)
|
||||
self.block_pool.cache_full_blocks(
|
||||
request=request,
|
||||
@@ -449,14 +456,15 @@ class SingleTypeKVCacheManager(ABC):
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
use_eagle: bool,
|
||||
retention_interval: int | None = None,
|
||||
num_prompt_tokens: int | None = None,
|
||||
reachable_boundaries: Sequence[int] = (),
|
||||
) -> list[bool] | None:
|
||||
"""Per-block mask for ``cache_full_blocks``. ``None`` means cache
|
||||
every (non-null) block — the default for full attention.
|
||||
|
||||
Subclasses with sparse hit semantics (SWA) override this to skip
|
||||
blocks that can never serve a hit at any alignment-aligned prefix
|
||||
length.
|
||||
Subclasses with sparse hit semantics (SWA / Mamba) override this to skip
|
||||
blocks that can never serve a hit at any alignment-aligned prefix length.
|
||||
``reachable_boundaries`` are token positions whose reachable tail must be
|
||||
retained; the base (dense) policy ignores them.
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -962,7 +970,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
use_eagle: bool,
|
||||
retention_interval: int | None = None,
|
||||
num_prompt_tokens: int | None = None,
|
||||
reachable_boundaries: Sequence[int] = (),
|
||||
) -> list[bool] | None:
|
||||
assert isinstance(kv_cache_spec, SlidingWindowSpec)
|
||||
if alignment_tokens is None:
|
||||
@@ -1002,18 +1010,16 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
if i >= shift and (i - shift) % per_segment >= per_segment - need:
|
||||
mask[i - start_block] = True
|
||||
|
||||
# (2) Replay-boundary tail. ``get_computed_blocks`` caps hits at
|
||||
# ``num_prompt - 1`` (to recompute the last token's logits), so an exact
|
||||
# prompt replay can only land on the latest *fine*-aligned boundary.
|
||||
# Sparse retention would otherwise skip it, so keep its tail explicitly.
|
||||
if retention_interval is not None and num_prompt_tokens is not None:
|
||||
latest = (num_prompt_tokens - 1) // alignment_tokens * alignment_tokens
|
||||
prompt_end_block = latest // block_size + shift
|
||||
for i in range(
|
||||
max(start_block, prompt_end_block - need),
|
||||
min(end_block, prompt_end_block),
|
||||
):
|
||||
mask[i - start_block] = True
|
||||
# (2) Reachable-boundary tails: the replay boundary (``num_prompt - 1``,
|
||||
# capped by ``get_computed_blocks``) and any shared-prefix junction. Both
|
||||
# land before segments would cover them under sparse retention, so keep
|
||||
# the ``need``-block tail ending on each boundary explicitly.
|
||||
if retention_interval is not None:
|
||||
for boundary_tokens in reachable_boundaries:
|
||||
aligned = boundary_tokens // alignment_tokens * alignment_tokens
|
||||
end = aligned // block_size + shift
|
||||
for j in range(max(start_block, end - need), min(end_block, end)):
|
||||
mask[j - start_block] = True
|
||||
|
||||
return mask
|
||||
|
||||
@@ -1321,15 +1327,19 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
use_eagle: bool,
|
||||
retention_interval: int | None = None,
|
||||
num_prompt_tokens: int | None = None,
|
||||
reachable_boundaries: Sequence[int] = (),
|
||||
) -> list[bool] | None:
|
||||
"""Sparse Mamba state-snapshot retention.
|
||||
|
||||
``retention_interval``:
|
||||
|
||||
``None`` -> dense (cache every block; default, unchanged behavior)
|
||||
``0`` -> keep only the latest replay boundary
|
||||
``0`` -> keep only the ``reachable_boundaries`` states
|
||||
``> 0`` -> keep one state per ``retention_interval``-sized segment
|
||||
|
||||
``reachable_boundaries`` are proven reuse points (the replay boundary and
|
||||
any cross-request shared-prefix junction, Marconi-style APC); their
|
||||
boundary state is always kept so sparse retention does not defeat reuse.
|
||||
"""
|
||||
if retention_interval is None or alignment_tokens is None:
|
||||
# Dense caching (default) or no alignment constraint imposed.
|
||||
@@ -1354,13 +1364,13 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
for i in range(first_boundary - start_block, len(mask), per_segment):
|
||||
mask[i] = True
|
||||
|
||||
# (2) Replay boundary. ``get_computed_blocks`` caps hits at
|
||||
# ``num_prompt - 1``, so an exact prompt replay lands on the latest
|
||||
# fine-aligned boundary. Sparse retention would otherwise skip its
|
||||
# state, so keep it explicitly.
|
||||
if num_prompt_tokens is not None:
|
||||
latest = (num_prompt_tokens - 1) // alignment_tokens * alignment_tokens
|
||||
boundary_block = latest // block_size - 1
|
||||
# (2) Reachable-boundary states: the replay boundary (``num_prompt - 1``,
|
||||
# capped by ``get_computed_blocks``) and any shared-prefix junction, both
|
||||
# of which segments would otherwise skip under sparse retention. A Mamba
|
||||
# hit needs exactly the single state block ending on the boundary.
|
||||
for boundary_tokens in reachable_boundaries:
|
||||
aligned = boundary_tokens // alignment_tokens * alignment_tokens
|
||||
boundary_block = aligned // block_size - 1
|
||||
if start_block <= boundary_block < end_block:
|
||||
mask[boundary_block - start_block] = True
|
||||
|
||||
|
||||
@@ -182,6 +182,11 @@ class Request:
|
||||
# True if this request is scheduled as a non-final prefill chunk.
|
||||
self.is_prefill_chunk = False
|
||||
|
||||
# Block-aligned token position of a proven shared prefix worth pinning
|
||||
# in the (sparse) prefix cache; 0 means none. Set at admission for
|
||||
# hybrid/Mamba models when a shared prefix is detected (Marconi-style).
|
||||
self.shared_prefix_boundary = 0
|
||||
|
||||
# The number of NaNs in logits. A value greater than 0
|
||||
# indicates that the output is corrupted
|
||||
self.num_nans_in_logits = 0
|
||||
|
||||
@@ -262,7 +262,7 @@ class SimpleCPUOffloadScheduler:
|
||||
max_hit_len = request.num_tokens - 1 - num_computed_tokens
|
||||
if max_hit_len <= 0:
|
||||
return 0, False
|
||||
cpu_hit_blocks, hit_length = self.cpu_coordinator.find_longest_cache_hit(
|
||||
cpu_hit_blocks, hit_length, _ = self.cpu_coordinator.find_longest_cache_hit(
|
||||
remaining_hashes, max_hit_len
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user