diff --git a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py index 51aefe63aa..a42fc41501 100644 --- a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py +++ b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py @@ -110,7 +110,7 @@ def test_hybrid_mamba_align_partial_hash_hit(): ) req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256) - computed_blocks, num_computed = manager.get_computed_blocks(req0) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req0) assert num_computed == 0 blocks = manager.allocate_slots(req0, 6, num_computed, computed_blocks) assert blocks is not None @@ -125,7 +125,7 @@ def test_hybrid_mamba_align_partial_hash_hit(): assert partial_mamba_block[0].block_hash_num_tokens == 6 req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256) - computed_blocks, num_computed = manager.get_computed_blocks(req1) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req1) assert num_computed == 6 assert [len(group) for group in computed_blocks.blocks] == [3, 2] @@ -185,7 +185,7 @@ def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue(): ) req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256) - computed_blocks, num_computed = manager.get_computed_blocks(req0) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req0) assert num_computed == 0 assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None @@ -258,7 +258,7 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit(): ) req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256) - computed_blocks, num_computed = manager.get_computed_blocks(req0) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req0) assert num_computed == 0 assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None @@ -279,7 +279,7 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit(): manager.new_step_starts() req1 = make_request("1", [0, 0, 1, 1, 2, 2, 4, 4], hash_block_size, sha256) - computed_blocks, num_computed = manager.get_computed_blocks(req1) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req1) assert num_computed == 6 # The later request hits the moved (private-copy) entry, not the source. assert computed_blocks.get_block_ids()[1][1] == moved_block_id @@ -338,7 +338,7 @@ def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit(): ) req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256) - computed_blocks, num_computed = manager.get_computed_blocks(req0) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req0) assert num_computed == 0 assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None manager.new_step_starts() @@ -351,13 +351,13 @@ def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit(): # A request hitting the moved entry in the SAME step must be deferred. req1 = make_request("1", [0, 0, 1, 1, 2, 2, 4, 4], hash_block_size, sha256) - computed_blocks, num_computed = manager.get_computed_blocks(req1) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req1) assert num_computed == 6 assert manager.allocate_slots(req1, 2, num_computed, computed_blocks) is None # Next step the moved entry is consumable. manager.new_step_starts() - computed_blocks, num_computed = manager.get_computed_blocks(req1) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req1) assert num_computed == 6 assert manager.allocate_slots(req1, 2, num_computed, computed_blocks) is not None @@ -397,7 +397,7 @@ def test_hybrid_full_attention_partial_hash_hit_uses_cow(): ) req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256) - computed_blocks, num_computed = manager.get_computed_blocks(req0) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req0) assert num_computed == 0 assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None manager.free(req0) @@ -410,7 +410,7 @@ def test_hybrid_full_attention_partial_hash_hit_uses_cow(): assert partial_full_block is not None req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256) - computed_blocks, num_computed = manager.get_computed_blocks(req1) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req1) assert num_computed == 6 assert [len(group) for group in computed_blocks.blocks] == [2, 2] @@ -471,7 +471,7 @@ def test_hybrid_partial_hit_cow_target_starts_uncached(): ) req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256) - computed_blocks, num_computed = manager.get_computed_blocks(req0) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req0) assert num_computed == 0 assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None manager.free(req0) @@ -488,7 +488,7 @@ def test_hybrid_partial_hit_cow_target_starts_uncached(): assert partial_mamba_block is not None req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256) - computed_blocks, num_computed = manager.get_computed_blocks(req1) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req1) assert num_computed == 6 new_blocks = manager.allocate_slots( @@ -586,7 +586,7 @@ def test_hybrid_partial_hash_truncates_full_attention_hit_length(): block_size=block_size, ) - computed_blocks, num_computed = manager.get_computed_blocks(req) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req) assert num_computed == 6 assert [len(group) for group in computed_blocks.blocks] == [2, 2] @@ -627,7 +627,7 @@ def test_cow_retained_blocks_returned_for_release(): hash_block_size=hash_block_size, ) req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256) - computed_blocks, num_computed = manager.get_computed_blocks(req0) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req0) assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None # The owner's move queues a copy and retains both endpoints. @@ -797,7 +797,7 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks(): # The owner prefills in scheduler-split style: stop at the block boundary # (4), then at the prompt's last hash boundary (6, partial entries). req0 = make_request("0", [7] * 6, hash_block_size, sha256) - computed_blocks, num_computed = manager.get_computed_blocks(req0) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req0) assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None req0.num_computed_tokens = 4 manager.new_step_starts() @@ -808,7 +808,7 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks(): # A longer request with eagle: full attention drops the partial tail, so # the joint hit must fall back to the block boundary the FA blocks cover. req1 = make_request("1", [7] * 6 + [9] * 2, hash_block_size, sha256) - computed_blocks, num_computed = manager.get_computed_blocks(req1) + computed_blocks, num_computed, _ = manager.get_computed_blocks(req1) assert num_computed == 4 assert all( len(group) * block_size >= num_computed for group in computed_blocks.blocks diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index d0dc8a97d6..dd89769315 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -244,7 +244,7 @@ def test_prefill(hash_fn): unique_token_ids = [3] * 7 all_token_ids = common_token_ids + unique_token_ids req0 = make_request("0", all_token_ids, block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) assert len(req0.block_hashes) == 3 assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 @@ -274,7 +274,7 @@ def test_prefill(hash_fn): # Incomplete 1 block (5 tokens) unique_token_ids = [3] * 5 req1 = make_request("1", common_token_ids + unique_token_ids, block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert len(req1.block_hashes) == 3 assert computed_blocks.get_block_ids() == ([1, 2, 3],) assert num_computed_tokens == 3 * 16 @@ -307,7 +307,7 @@ def test_prefill(hash_fn): # Incomplete 1 block (6 tokens) unique_token_ids = [3] * 6 req2 = make_request("2", common_token_ids + unique_token_ids, block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2) assert len(req2.block_hashes) == 3 assert computed_blocks.get_block_ids() == ([1, 2, 3],) assert num_computed_tokens == 3 * 16 @@ -327,7 +327,7 @@ def test_prefill(hash_fn): # Cache miss and eviction. req3 = make_request("3", [99] * (16 * 10), block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req3) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req3) assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 blocks = manager.allocate_slots( @@ -369,7 +369,7 @@ def test_prefill_hybrid_model(): unique_token_ids = [3] * 7 all_token_ids = common_token_ids + unique_token_ids req0 = make_request("0", all_token_ids, block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) assert len(req0.block_hashes) == 3 assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 @@ -405,7 +405,7 @@ def test_prefill_hybrid_model(): unique_token_ids = [3] * 5 all_token_ids = common_token_ids + unique_token_ids req1 = make_request("1", common_token_ids + unique_token_ids, block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert len(req1.block_hashes) == 3 assert computed_blocks.get_block_ids() == ([1, 2, 3], [0, 6, 7], [0, 10, 11]) assert num_computed_tokens == 3 * 16 @@ -539,7 +539,7 @@ def test_prefill_hybrid_model_eagle(): unique_token_ids = [6] * 7 all_token_ids = common_token_ids + unique_token_ids req0 = make_request("0", all_token_ids, block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) assert len(req0.block_hashes) == len(all_token_ids) // block_size assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 @@ -576,7 +576,7 @@ def test_prefill_hybrid_model_eagle(): unique_token_ids = [6] * 5 all_token_ids = common_token_ids + unique_token_ids req1 = make_request("1", all_token_ids, block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert len(req1.block_hashes) == num_full_blocks assert computed_blocks.get_block_ids() == ( [1, 2, 3, 4, 5], @@ -712,7 +712,7 @@ def _test_partial_request_hit( req = make_request(request_id, prompt_token_ids, block_size, sha256) for hash_with_group_id in hash_to_evict: manager.block_pool.cached_block_hash_to_block._cache.pop(hash_with_group_id) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req) assert len(req.block_hashes) == num_full_blocks assert num_computed_tokens == expect_hit_length * block_size for block_per_group in computed_blocks.blocks: @@ -873,7 +873,7 @@ def test_prefill_hybrid_model_combinations(spec_types: list[str]): # First request: no cache hit initially req0 = make_request("0", all_token_ids, block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) assert len(req0.block_hashes) == 3 assert not computed_blocks.blocks[0] # No cache hit initially @@ -890,7 +890,7 @@ def test_prefill_hybrid_model_combinations(spec_types: list[str]): # Second request: should hit cached blocks for common prefix req1 = make_request("1", common_token_ids + [4] * 5, block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) # Should hit cached blocks for all groups assert num_computed_tokens == 3 * block_size @@ -950,7 +950,7 @@ def test_prefill_hybrid_model_combinations_eagle( # First request: no cache hit initially req0 = make_request("0", all_token_ids, block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) assert len(req0.block_hashes) == num_full_blocks assert not computed_blocks.blocks[0] # No cache hit initially @@ -966,7 +966,7 @@ def test_prefill_hybrid_model_combinations_eagle( # Second request: should hit cached blocks for common prefix all_token_ids = common_token_ids + [6] * 5 req1 = make_request("1", all_token_ids, block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) # Should hit cached blocks for all groups assert num_computed_tokens == expect_hit_length * block_size @@ -1019,7 +1019,7 @@ def test_prefill_hybrid_model_mamba_align(): # First request: allocate_slots should not crash with the assertion error # in MambaManager.cache_blocks() when null blocks are present. req0 = make_request("0", all_token_ids, block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) assert num_computed_tokens == 0 blocks = manager.allocate_slots(req0, 55, num_computed_tokens, computed_blocks) @@ -1048,28 +1048,33 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection(): # Request: 3 blocks prefix = [i for i in range(3) for _ in range(block_size)] req_0 = make_request("0", prefix, block_size, hash_fn) - computed_blocks, num_computed = manager.get_computed_blocks(req_0) - num_uncached_common = manager.coordinator.num_uncached_common_prefix_tokens + computed_blocks, num_computed, shared_prefix_boundary = manager.get_computed_blocks( + req_0 + ) assert num_computed == 0 # nothing cached yet - assert num_uncached_common == 0 + assert shared_prefix_boundary == 0 manager.allocate_slots(req_0, 3 * block_size, 0, computed_blocks) # Request: 3 blocks (shared with above) + 7 different tokens req_1 = make_request("1", prefix + [100] * 7, block_size, hash_fn) - computed_blocks, num_computed = manager.get_computed_blocks(req_1) - num_uncached_common = manager.coordinator.num_uncached_common_prefix_tokens + computed_blocks, num_computed, shared_prefix_boundary = manager.get_computed_blocks( + req_1 + ) assert num_computed == 3 * block_size # we should observe a 3-block cache hit - assert num_uncached_common == 0 + assert shared_prefix_boundary == 0 manager.allocate_slots(req_1, 7, 3 * block_size, computed_blocks) # Request: 3 blocks, but only 2 blocks shared (replace the last token in 3rd block): req_2 = make_request("2", prefix[:-1] + [101], block_size, hash_fn) - computed_blocks, num_computed = manager.get_computed_blocks(req_2) - num_uncached_common = manager.coordinator.num_uncached_common_prefix_tokens + computed_blocks, num_computed, shared_prefix_boundary = manager.get_computed_blocks( + req_2 + ) assert num_computed == 0 # mamba_align doesn't cache intermediate blocks - assert num_uncached_common == 2 * block_size # heuristic detects a shared prefix + # boundary == junction: 2 shared blocks (num_computed 0 + 2 uncached blocks). + assert shared_prefix_boundary == 2 * block_size - # Next, validate scheduler logic for num_uncached_common_prefix_tokens > 0 + # Next, validate the align-mode Marconi split. It takes the shared-prefix + # junction (absolute) and stops the chunk there (req_2 has num_computed 0). # Create minimal mock with just the needed attributes mock = SimpleNamespace( cache_config=SimpleNamespace(block_size=block_size), @@ -1077,11 +1082,11 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection(): hash_block_size=block_size, mamba_partial_cache_hit=False, ) + req_2.shared_prefix_boundary = shared_prefix_boundary num_new_tokens_adjusted = Scheduler._mamba_block_aligned_split( self=mock, request=req_2, num_new_tokens=3 * block_size, - num_uncached_common_prefix_tokens=num_uncached_common, ) assert num_new_tokens_adjusted == 2 * block_size # adjust to the common prefix @@ -1118,7 +1123,7 @@ def test_hybrid_model_mamba_align_with_dynamic_draft_tokens(): all_token_ids = [i for i in range(3) for _ in range(block_size)] + [3] * 7 req0 = make_request("0", all_token_ids, block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) assert num_computed_tokens == 0 blocks = manager.allocate_slots( req0, len(all_token_ids), num_computed_tokens, computed_blocks @@ -1170,7 +1175,7 @@ def test_prefill_plp(): unique_token_ids = [3] * 7 all_token_ids = common_token_ids + unique_token_ids req0 = make_request("0", all_token_ids, block_size, hash_fn, prompt_logprobs=5) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) assert len(req0.block_hashes) == 3 assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 @@ -1202,7 +1207,7 @@ def test_prefill_plp(): # Incomplete 1 block (5 tokens) unique_token_ids = [3] * 5 req1 = make_request("1", common_token_ids + unique_token_ids, block_size, hash_fn) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert len(req1.block_hashes) == 3 assert computed_blocks.get_block_ids() == ([1, 2, 3],) assert num_computed_tokens == 3 * 16 @@ -1236,7 +1241,7 @@ def test_prefill_plp(): req2 = make_request( "2", common_token_ids + unique_token_ids, block_size, hash_fn, prompt_logprobs=5 ) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2) assert len(req2.block_hashes) == 3 assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 @@ -1273,7 +1278,7 @@ def test_decode(): # Incomplete 1 block (7 tokens) unique_token_ids = [3] * 7 req0 = make_request("0", common_token_ids + unique_token_ids, block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 blocks = manager.allocate_slots( @@ -1331,7 +1336,7 @@ def test_evict(): last_token_id = 5 * 16 + 7 req0 = make_request("0", list(range(last_token_id)), block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 blocks = manager.allocate_slots( @@ -1344,7 +1349,7 @@ def test_evict(): req1 = make_request( "1", list(range(last_token_id, last_token_id + 3 * 16)), block_size, sha256 ) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 blocks = manager.allocate_slots( @@ -1369,7 +1374,7 @@ def test_evict(): # Touch the first 2 blocks. req2 = make_request("2", list(range(2 * 16 + 3)), block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2) assert computed_blocks.get_block_ids() == ([1, 2],) assert num_computed_tokens == 2 * 16 blocks = manager.allocate_slots( @@ -1395,7 +1400,7 @@ def test_hash_block_correct_reuse(): # Allocate 1 block and cache it. num_tokens = block_size * 1 req = make_request("0", list(range(num_tokens)), block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req) assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 blocks = manager.allocate_slots( @@ -1409,7 +1414,7 @@ def test_hash_block_correct_reuse(): # Allocate a new block that's not full, make sure hash info on the # block is cleared. req = make_request("1", list(range(num_tokens - 1)), block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req) assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 blocks = manager.allocate_slots( @@ -1436,7 +1441,7 @@ def test_computed_blocks_not_evicted(): # Allocate a block and cache it. num_tokens = block_size * 1 req0 = make_request("0", list(range(num_tokens)), block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 blocks = manager.allocate_slots( @@ -1449,7 +1454,7 @@ def test_computed_blocks_not_evicted(): req1 = make_request( "1", list(range(num_tokens, num_tokens * 2)), block_size, sha256 ) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 blocks = manager.allocate_slots( @@ -1465,7 +1470,7 @@ def test_computed_blocks_not_evicted(): # Now if we have a cache hit on the first block, we should evict the second # cached block rather than the first one. req2 = make_request("2", list(range(num_tokens * 2)), block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2) assert len(computed_blocks.blocks[0]) == 1 assert computed_blocks.blocks[0][0].block_id == 1 assert num_computed_tokens == block_size @@ -1496,7 +1501,7 @@ def test_basic_prefix_caching_disabled(): "1", list(range(10)), block_size, sha256 ) # 2 blocks and some more - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 blocks = manager.allocate_slots( @@ -1509,7 +1514,7 @@ def test_basic_prefix_caching_disabled(): # No caching. req2 = make_request("2", list(range(16)), block_size, sha256) # shared prefix - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2) assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 blocks = manager.allocate_slots( @@ -1519,7 +1524,7 @@ def test_basic_prefix_caching_disabled(): # New requests should not have any blocks. req3 = make_request("3", list(range(4)), block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req3) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req3) assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 blocks = manager.allocate_slots( @@ -1701,7 +1706,7 @@ def test_mm_prefix_caching(): mm_positions=mm_positions, mm_hashes=mm_hashes, ) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) # Completed block should have hashes assert not computed_blocks.blocks[0] @@ -1766,7 +1771,7 @@ def test_mm_prefix_caching(): mm_positions=mm_positions, mm_hashes=mm_hashes, ) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert len(computed_blocks.blocks[0]) == 3 assert num_computed_tokens == 3 * 16 @@ -1788,7 +1793,7 @@ def test_cache_key_salting(): common_token_ids = [i for i in range(3) for _ in range(block_size)] token_ids = common_token_ids + [3] * 11 req0 = make_request("0", token_ids, block_size, sha256, cache_salt="salt1") - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) # Completed block should have hashes assert not computed_blocks.blocks[0] @@ -1827,7 +1832,7 @@ def test_cache_key_salting(): # Test cache hit with a new request that has the same salt. token_ids = common_token_ids + [4] * 11 req1 = make_request("1", token_ids, block_size, sha256, cache_salt="salt1") - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) # Should match only a prefix of 3 blocks. assert len(computed_blocks.blocks[0]) == 3 assert num_computed_tokens == 3 * block_size @@ -1835,7 +1840,7 @@ def test_cache_key_salting(): # Test cache miss with same content but different salt. token_ids = common_token_ids + [4] * 11 req2 = make_request("2", token_ids, block_size, sha256, cache_salt="salt2") - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2) assert len(computed_blocks.blocks[0]) == 0 assert num_computed_tokens == 0 block_hashes = req2.block_hashes @@ -1869,7 +1874,7 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks(): # | Common-0 | Common-1 | Common-2 | ... | common_token_ids = [i for i in range(3) for _ in range(16)] req0 = make_request("0", common_token_ids, block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 manager.allocate_slots( @@ -1881,7 +1886,7 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks(): # | Common-0 | Common-1 | Common-2 | Req1-3 | Req1-4 | Req1-5 | ... | req1 = make_request("1", common_token_ids * 2, block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert computed_blocks.blocks[0] == block_part0 assert num_computed_tokens == 3 * 16 manager.allocate_slots( @@ -1899,7 +1904,7 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks(): # | Common-0 | Common-1 | Common-2 | Req1-3 (F) | Req1-4 (F) | # | Req1-5(F)| Req2-0 | Req2-1 | ... | req2 = make_request("2", [7] * block_size * 2, block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2) assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 manager.allocate_slots( @@ -1914,7 +1919,7 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks(): # In this case, the ref_cnt of the computed blocks should not be changed. assert manager.block_pool.free_block_queue.num_free_blocks == 5 req3 = make_request("3", common_token_ids * 3, block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req3) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req3) assert computed_blocks.blocks[0] == block_part1 assert num_computed_tokens == 6 * 16 # Req3 cannot be allocated. @@ -1949,7 +1954,7 @@ def test_reset_prefix_cache(): unique_token_ids = [4] * 7 all_token_ids = full_block_token_ids + unique_token_ids req1 = make_request("1", all_token_ids, block_size, sha256) - computed_blocks, _ = manager.get_computed_blocks(req1) + computed_blocks, _, _ = manager.get_computed_blocks(req1) assert len(req1.block_hashes) == 3 assert len(computed_blocks.blocks[0]) == 3 blocks = manager.allocate_slots( @@ -1984,7 +1989,7 @@ def test_prefix_cache_stats_disabled(): # Call all functions that check whether log_stats is disabled. req = make_request("0", list(range(16)), block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req) assert not computed_blocks.blocks[0] assert num_computed_tokens == 0 manager.allocate_slots( @@ -2591,7 +2596,7 @@ def test_eagle_enabled_removes_last_block(): req = make_request("divisible_request", token_ids, block_size, sha256) # Prime the cache - computed_blocks, _ = manager.get_computed_blocks(req) + computed_blocks, _, _ = manager.get_computed_blocks(req) manager.allocate_slots( req, len(token_ids), len(computed_blocks.blocks[0]) * 16, computed_blocks ) @@ -2599,7 +2604,7 @@ def test_eagle_enabled_removes_last_block(): # New request with same tokens + Eagle enabled req_eagle = make_request("eagle_divisible", token_ids, block_size, sha256) - computed_blocks, num_tokens = manager.get_computed_blocks(req_eagle) + computed_blocks, num_tokens, _ = manager.get_computed_blocks(req_eagle) # Should retain 1 block: # 1. Original 3 blocks → pop last hash → 2 matched blocks @@ -2623,7 +2628,7 @@ def test_eagle_with_partial_blocks(): req = make_request("partial_block_test", token_ids, block_size, sha256) # Prime the cache - computed_blocks, _ = manager.get_computed_blocks(req) + computed_blocks, _, _ = manager.get_computed_blocks(req) manager.allocate_slots( req, len(token_ids), len(computed_blocks.blocks[0]) * 16, computed_blocks ) @@ -2631,7 +2636,7 @@ def test_eagle_with_partial_blocks(): # New request with Eagle enabled req_eagle = make_request("partial_eagle", token_ids, block_size, sha256) - computed_blocks, num_tokens = manager.get_computed_blocks(req_eagle) + computed_blocks, num_tokens, _ = manager.get_computed_blocks(req_eagle) # Original match: 2 full blocks → Eagle removes 1 → 1 remaining assert len(computed_blocks.blocks[0]) == 1 assert num_tokens == 1 * block_size @@ -2664,7 +2669,7 @@ def test_eagle_with_sliding_window(): req = make_request("partial_block_test", token_ids, block_size, sha256) # Prime the cache - computed_blocks, _ = manager.get_computed_blocks(req) + computed_blocks, _, _ = manager.get_computed_blocks(req) manager.allocate_slots( req, len(token_ids), len(computed_blocks.blocks[0]) * 16, computed_blocks ) @@ -2675,7 +2680,7 @@ def test_eagle_with_sliding_window(): # New request with Eagle enabled req_eagle = make_request("partial_eagle", token_ids, block_size, sha256) - computed_blocks, num_tokens = manager.get_computed_blocks(req_eagle) + computed_blocks, num_tokens, _ = manager.get_computed_blocks(req_eagle) # Original match: 2 full blocks → Eagle removes 1 → 1 remaining assert len(computed_blocks.blocks[0]) == 1 assert num_tokens == 1 * block_size @@ -2695,7 +2700,7 @@ def test_eagle_with_sliding_window(): req_after_evict = make_request( "partial_eagle_after_evict", token_ids, block_size, sha256 ) - computed_blocks, num_tokens = manager.get_computed_blocks(req_after_evict) + computed_blocks, num_tokens, _ = manager.get_computed_blocks(req_after_evict) # Cache miss. The only hit prefix is [NULL_BLOCK, BLOCK_2] if eagle is # not considered. But after dropping the last matched block due to eagle, # there will be no matched prefix. @@ -2754,7 +2759,7 @@ def test_eagle_swa_alignment_caches_extra_block(): # Prime the cache with a long prompt (16 swa blocks = 4 aligned segments). token_ids = [i for i in range(16) for _ in range(block_size)] req0 = make_request("0", token_ids, block_size, sha256) - computed_blocks, _ = manager.get_computed_blocks(req0) + computed_blocks, _, _ = manager.get_computed_blocks(req0) blocks = manager.allocate_slots( req0, len(token_ids), @@ -2768,7 +2773,7 @@ def test_eagle_swa_alignment_caches_extra_block(): # Without the fix, ``num_computed_tokens`` is 0; with the fix, it lands at # an alignment boundary (multiple of 32 tokens, minus the EAGLE drop). req1 = make_request("1", token_ids, block_size, sha256) - _, num_computed_tokens = manager.get_computed_blocks(req1) + _, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert num_computed_tokens > 0, ( "EAGLE + SWA with sliding_window <= alignment failed to find any " "cache hit; the +1 block past each segment boundary must be cached." @@ -2823,7 +2828,7 @@ def test_eagle_swa_boundary_caches_post_boundary_block(): token_ids = [i for i in range(5) for _ in range(block_size)] req0 = make_request("0", token_ids, block_size, sha256) - computed_blocks, _ = manager.get_computed_blocks(req0) + computed_blocks, _, _ = manager.get_computed_blocks(req0) blocks = manager.allocate_slots( req0, len(token_ids), @@ -2838,7 +2843,7 @@ def test_eagle_swa_boundary_caches_post_boundary_block(): manager.free(req0) req1 = make_request("1", token_ids + [999], block_size, sha256) - _, num_computed_tokens = manager.get_computed_blocks(req1) + _, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert num_computed_tokens == 4 * block_size @@ -2879,7 +2884,7 @@ def test_eagle_grouped_swa_siblings_use_same_cache_mask(): token_ids = [i for i in range(9) for _ in range(block_size)] req0 = make_request("0", token_ids, block_size, sha256) - computed_blocks, _ = manager.get_computed_blocks(req0) + computed_blocks, _, _ = manager.get_computed_blocks(req0) blocks = manager.allocate_slots( req0, len(token_ids), @@ -2894,7 +2899,7 @@ def test_eagle_grouped_swa_siblings_use_same_cache_mask(): manager.free(req0) req1 = make_request("1", token_ids + [999], block_size, sha256) - _, num_computed_tokens = manager.get_computed_blocks(req1) + _, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert num_computed_tokens == 8 * block_size @@ -2938,7 +2943,7 @@ def test_different_block_size(): common_token_ids = [i for i in range(10) for _ in range(block_size)] req0 = make_request("0", common_token_ids, block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) assert not computed_blocks.blocks[0] assert not computed_blocks.blocks[1] assert num_computed_tokens == 0 @@ -2947,13 +2952,13 @@ def test_different_block_size(): ) assert blocks.get_block_ids() == ([1, 2, 3, 4], [5, 6, 7, 8, 9, 10, 11]) req1 = make_request("1", common_token_ids[: 7 * block_size + 1], block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert len(computed_blocks.blocks[0]) == 3 assert len(computed_blocks.blocks[1]) == 6 assert num_computed_tokens == 6 * 16 req2 = make_request("2", common_token_ids[: 6 * block_size + 1], block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req2) assert len(computed_blocks.blocks[0]) == 3 assert len(computed_blocks.blocks[1]) == 6 assert num_computed_tokens == 6 * 16 @@ -2967,7 +2972,7 @@ def test_different_block_size(): manager.block_pool.cached_block_hash_to_block.pop( make_block_hash_with_group_id(req1.block_hashes[5], 1), 10 ) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert len(computed_blocks.blocks[0]) == 2 assert len(computed_blocks.blocks[1]) == 4 assert num_computed_tokens == 4 * 16 @@ -3020,7 +3025,7 @@ def test_hybrid_cache_blocks_swa_tail_window_only(): # 8 hash-blocks of 8 tokens (64 tokens, two lcm-aligned segments). token_ids = [i for i in range(8) for _ in range(block_size)] req = make_request("0", token_ids, block_size, sha256) - computed_blocks, _ = manager.get_computed_blocks(req) + computed_blocks, _, _ = manager.get_computed_blocks(req) blocks = manager.allocate_slots( req, 8 * block_size, @@ -3089,7 +3094,7 @@ def test_hybrid_cache_blocks_clamped_to_lcm(): # to 96 tokens — SWA caches 6 hashes, full-attn caches 3. token_ids = [i for i in range(7) for _ in range(block_size)] req = make_request("0", token_ids, block_size, sha256) - computed_blocks, _ = manager.get_computed_blocks(req) + computed_blocks, _, _ = manager.get_computed_blocks(req) blocks = manager.allocate_slots( req, 7 * block_size, @@ -3154,7 +3159,7 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch): # 96-token replay boundary, and the 128-token interval boundary. token_ids = [i for i in range(16) for _ in range(block_size)] req = make_request("0", token_ids, block_size, sha256) - computed_blocks, _ = manager.get_computed_blocks(req) + computed_blocks, _, _ = manager.get_computed_blocks(req) blocks = manager.allocate_slots( req, len(token_ids), @@ -3296,13 +3301,13 @@ def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch): token_ids = fill_request("fill_0", 0) replay_req = make_request("replay", token_ids[:1800], hash_block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(replay_req) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(replay_req) assert num_computed_tokens == 1024 assert [len(blocks) for blocks in computed_blocks.blocks] == [4, 16, 128, 256] fill_request("fill_1", 100_000) replay_req = make_request("replay_again", token_ids[:1800], hash_block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(replay_req) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(replay_req) assert num_computed_tokens == 1024 assert [len(blocks) for blocks in computed_blocks.blocks] == [4, 16, 128, 256] @@ -3345,7 +3350,7 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc token_ids = [i for i in range(16) for _ in range(block_size)] req0 = make_request("0", token_ids, block_size, sha256) - computed_blocks, _ = manager.get_computed_blocks(req0) + computed_blocks, _, _ = manager.get_computed_blocks(req0) blocks = manager.allocate_slots( req0, len(token_ids), @@ -3369,14 +3374,14 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc assert retained_swa_block[0].ref_cnt == 0 req1 = make_request("1", token_ids, block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) # Full prompt hits intentionally recompute the final block for logits, so # the longest usable hit is the previous LCM boundary: 96 tokens. assert num_computed_tokens == 12 * block_size assert len(computed_blocks.blocks[1]) == 12 shorter_req = make_request("2", token_ids[: 12 * block_size], block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(shorter_req) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(shorter_req) assert num_computed_tokens == 0 assert len(computed_blocks.blocks[1]) == 0 @@ -3429,7 +3434,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch): # 104 tokens, and that tail is two 8-token blocks wide: hashes 11 and 12. token_ids = [i for i in range(15) for _ in range(block_size)] + [15] * 7 req0 = make_request("0", token_ids, block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) assert num_computed_tokens == 0 blocks = manager.allocate_slots( req0, @@ -3451,7 +3456,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch): manager.free(req0) req1 = make_request("1", token_ids, block_size, sha256) - computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert num_computed_tokens == 12 * block_size assert [len(blocks) for blocks in computed_blocks.blocks] == [3, 12] @@ -3858,7 +3863,7 @@ def test_swa_free_split_keeps_cached_tail_ahead_of_scratch(monkeypatch): token_ids = [i for i in range(16) for _ in range(block_size)] req = make_request("0", token_ids, block_size, sha256) - computed_blocks, _ = manager.get_computed_blocks(req) + computed_blocks, _, _ = manager.get_computed_blocks(req) blocks = manager.allocate_slots( req, len(token_ids), @@ -3941,7 +3946,7 @@ def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch): token_ids = [i for i in range(16) for _ in range(block_size)] req = make_request("0", token_ids, block_size, sha256) - computed_blocks, _ = manager.get_computed_blocks(req) + computed_blocks, _, _ = manager.get_computed_blocks(req) blocks = manager.allocate_slots( req, len(token_ids), @@ -3965,7 +3970,7 @@ def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch): # A replay of the same prompt hits the latest replayable boundary (240). replay = make_request("1", token_ids, block_size, sha256) - _, num_computed = manager.get_computed_blocks(replay) + _, num_computed, _ = manager.get_computed_blocks(replay) assert num_computed == 240 @@ -3977,7 +3982,7 @@ def test_pure_swa_retention_latest_only(monkeypatch): token_ids = [i for i in range(16) for _ in range(block_size)] req = make_request("0", token_ids, block_size, sha256) - computed_blocks, _ = manager.get_computed_blocks(req) + computed_blocks, _, _ = manager.get_computed_blocks(req) blocks = manager.allocate_slots( req, len(token_ids), @@ -3997,7 +4002,7 @@ def test_pure_swa_retention_latest_only(monkeypatch): assert cached == {14} replay = make_request("1", token_ids, block_size, sha256) - _, num_computed = manager.get_computed_blocks(replay) + _, num_computed, _ = manager.get_computed_blocks(replay) assert num_computed == 240 @@ -4010,7 +4015,7 @@ def test_pure_swa_retention_dense_default_caches_all(monkeypatch): token_ids = [i for i in range(16) for _ in range(block_size)] req = make_request("0", token_ids, block_size, sha256) - computed_blocks, _ = manager.get_computed_blocks(req) + computed_blocks, _, _ = manager.get_computed_blocks(req) blocks = manager.allocate_slots( req, len(token_ids), @@ -4052,7 +4057,7 @@ def test_mamba_reachable_block_mask_sparsifies_retention(): kv_cache_spec=spec, use_eagle=False, retention_interval=retention_interval, - num_prompt_tokens=num_prompt_tokens, + reachable_boundaries=(num_prompt_tokens - 1,), ) return None if m is None else {i for i, v in enumerate(m) if v} @@ -4065,3 +4070,242 @@ def test_mamba_reachable_block_mask_sparsifies_retention(): assert retained(64) == {3, 7, 11, 14, 15} # interval 0 -> only the latest replay boundary (block 14). assert retained(0) == {14} + + +def test_mamba_reachable_block_mask_pins_shared_prefix(): + """A Marconi-detected shared prefix (``shared_prefix_boundary``) lands before + ``num_prompt`` so the replay-boundary rule alone would drop it. The mask must + pin the single state block ending on that boundary so sparse retention does + not defeat cross-request shared-prefix reuse.""" + from vllm.v1.core.single_type_kv_cache_manager import MambaManager + + block_size = 16 + spec = MambaSpec( + block_size=block_size, + shapes=(1, 1), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ) + + def retained(retention_interval, shared_prefix_boundary, end_block=16): + boundaries = [255] # replay boundary (num_prompt 256 - 1) + if shared_prefix_boundary: + boundaries.append(shared_prefix_boundary) + m = MambaManager.reachable_block_mask( + start_block=0, + end_block=end_block, + alignment_tokens=block_size, + kv_cache_spec=spec, + use_eagle=False, + retention_interval=retention_interval, + reachable_boundaries=boundaries, + ) + return None if m is None else {i for i, v in enumerate(m) if v} + + # interval 0 keeps only replay boundary 14; the shared prefix at token 96 + # (state block 96//16 - 1 = 5) is now pinned too. + assert retained(0, 96) == {5, 14} + # A non-aligned boundary floors to the enclosing aligned boundary. + assert retained(0, 100) == {5, 14} + # Coexists with segment tails (interval 64 -> {3,7,11,15} + replay 14). + assert retained(64, 96) == {3, 5, 7, 11, 14, 15} + # Dense default ignores the hint (nothing to sparsify). + assert retained(None, 96) is None + # Out-of-range boundary is a no-op (only replay 14 remains). + assert retained(0, 16 * block_size * 2) == {14} + # No boundary given -> unchanged replay-only behavior. + assert retained(0, 0) == {14} + assert retained(0, None) == {14} + + +def test_mamba_shared_prefix_survives_zero_retention(monkeypatch): + """Manager-level check of the full wiring: a pinned shared-prefix boundary + (``Request.shared_prefix_boundary``, set by the scheduler on Marconi-style + detection) keeps its Mamba state block cached under + ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``, which otherwise retains only the + end-of-prompt replay boundary. Without this, a shared prefix (junction + before ``num_prompt``) would be recomputed by every sharing request.""" + monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0") + block_size = 16 + + # 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14. + token_ids = [i for i in range(16) for _ in range(block_size)] + + def cached_mamba_blocks(shared_prefix_boundary): + # Fresh manager per scenario so cached blocks don't leak between runs. + manager = make_kv_cache_manager( + _make_hybrid_kv_cache_config(block_size, 100, ["full", "mamba"]), + max_model_len=8192, + enable_caching=True, + hash_block_size=block_size, + ) + req = make_request("r", token_ids, block_size, sha256) + req.shared_prefix_boundary = shared_prefix_boundary + computed_blocks, num_computed, _ = manager.get_computed_blocks(req) + blocks = manager.allocate_slots( + req, len(token_ids), num_computed, computed_blocks + ) + assert blocks is not None + pool = manager.block_pool + return { + i + for i in range(16) + if pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1]) + is not None + } + + # Without a pinned boundary, retention=0 keeps only the replay boundary (14). + assert cached_mamba_blocks(0) == {14} + # Pinning the shared prefix at token 96 (state block 5) retains it too, so a + # later request sharing that prefix can hit the Mamba state. + assert cached_mamba_blocks(96) == {5, 14} + + +def test_mamba_shared_prefix_reuse_under_zero_retention(monkeypatch): + """Full cross-request Marconi flow: a partial shared prefix cached by the + detecting request must stay reusable by a later request under + ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without the pin the junction is + masked out and the later request misses; with it (and under dense) the reuse + is preserved.""" + block_size = 16 + + def last_req_hit(retention, pin): + if retention is None: + monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False) + else: + monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention)) + manager = make_kv_cache_manager( + _make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba_align"]), + max_model_len=8192, + enable_caching=True, + hash_block_size=block_size, + ) + shared = [7 for _ in range(2 * block_size)] # 2-block shared prefix + + def distinct(v): + return [v for _ in range(2 * block_size)] + + # req0 primes the shared prefix's (dense) attention cache; align-mode + # Mamba keeps only its own tail, not the shared-prefix junction. + req0 = make_request("0", shared + distinct(50), block_size, sha256) + cb, nc, _ = manager.get_computed_blocks(req0) + manager.allocate_slots(req0, len(req0.all_token_ids), nc, cb) + + # req1 detects the shared prefix (attn hit, Mamba lag) and, via the + # Marconi chunk, caches the junction's Mamba state. + req1 = make_request("1", shared + distinct(60), block_size, sha256) + cb, nc, boundary = manager.get_computed_blocks(req1) + assert boundary == 2 * block_size # junction detected at 2 shared blocks + if pin: + req1.shared_prefix_boundary = boundary + # Simulate the Marconi chunk: schedule up to the junction (boundary - nc). + manager.allocate_slots(req1, boundary - nc, nc, cb) + + # req2 shares the same prefix: does it reuse the cached junction? + req2 = make_request("2", shared + distinct(70), block_size, sha256) + _, nc2, _ = manager.get_computed_blocks(req2) + return nc2 + + # Dense retains the junction -> reuse works (baseline ceiling). + assert last_req_hit(retention=None, pin=False) == 2 * block_size + # retention=0 without the pin masks the junction out -> reuse lost (the bug). + assert last_req_hit(retention=0, pin=False) == 0 + # retention=0 with the pin keeps the junction -> reuse restored. + assert last_req_hit(retention=0, pin=True) == 2 * block_size + + +def test_swa_reachable_block_mask_pins_shared_prefix(): + """SWA analog of the Mamba pin: the shared-prefix junction must keep the + ``need``-block sliding-window tail ending on that boundary (not a single + block), so a windowed hit can land there under sparse retention.""" + from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager + + block_size = 16 + + def retained(retention, boundary, window, end_block=16): + spec = SlidingWindowSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=window, + ) + boundaries = [255] # replay boundary (num_prompt 256 - 1) + if boundary: + boundaries.append(boundary) + m = SlidingWindowManager.reachable_block_mask( + start_block=0, + end_block=end_block, + alignment_tokens=block_size, + kv_cache_spec=spec, + use_eagle=False, + retention_interval=retention, + reachable_boundaries=boundaries, + ) + return None if m is None else {i for i, v in enumerate(m) if v} + + # window == block_size -> need = cdiv(15, 16) = 1: single-block tail (like + # Mamba). Junction at token 96 -> block 5; replay boundary -> block 14. + assert retained(0, 96, block_size) == {5, 14} + # window == 3 * block_size -> need = cdiv(47, 16) = 3: the junction keeps a + # 3-block WINDOW {3,4,5} (the SWA distinction), plus replay window {12,13,14}. + assert retained(0, 96, 3 * block_size) == {3, 4, 5, 12, 13, 14} + # Coexists with segment tails (interval 64, need=1): {3,7,11,15} + replay 14 + # + junction 5. + assert retained(64, 96, block_size) == {3, 5, 7, 11, 14, 15} + # Dense (all blocks reachable) ignores the hint. + assert retained(None, 96, block_size) is None + # No boundary -> unchanged replay-only behavior. + assert retained(0, 0, block_size) == {14} + + +def test_swa_shared_prefix_reuse_under_zero_retention(monkeypatch): + """SWA cross-request analog: a partial shared prefix's sliding-window tail + must stay reusable under ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without + the pin the junction window is masked out and a later request misses; with + it (and under dense) reuse is preserved.""" + block_size = 16 + + def last_req_hit(retention, pin): + if retention is None: + monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False) + else: + monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention)) + manager = make_kv_cache_manager( + _make_hybrid_kv_cache_config(block_size, 200, ["full", "sliding_window"]), + max_model_len=8192, + enable_caching=True, + hash_block_size=block_size, + ) + shared = [7 for _ in range(4 * block_size)] # 4-block shared prefix + + def distinct(v): + return [v for _ in range(2 * block_size)] + + # req0 primes the (dense) full-attention cache; SWA under RET=0 keeps + # only its own replay window, not the shared-prefix boundary. + req0 = make_request("0", shared + distinct(50), block_size, sha256) + cb, nc, _ = manager.get_computed_blocks(req0) + manager.allocate_slots(req0, len(req0.all_token_ids), nc, cb) + + # req1 detects the shared prefix (full-attn hit, SWA lag). No chunk for + # SWA -- the pin fires during normal prefill. + req1 = make_request("1", shared + distinct(60), block_size, sha256) + cb, nc, boundary = manager.get_computed_blocks(req1) + if retention == 0: + assert boundary == 4 * block_size # junction detected when SWA lags + if pin and boundary: + req1.shared_prefix_boundary = boundary + manager.allocate_slots(req1, len(req1.all_token_ids), nc, cb) + + # req2 shares the same prefix: does its SWA window reuse the junction? + req2 = make_request("2", shared + distinct(70), block_size, sha256) + _, nc2, _ = manager.get_computed_blocks(req2) + return nc2 + + # Dense keeps every SWA tail -> reuse works (ceiling). + assert last_req_hit(retention=None, pin=False) == 4 * block_size + # retention=0 without the pin drops the junction window -> reuse lost (bug). + assert last_req_hit(retention=0, pin=False) == 0 + # retention=0 with the pin keeps the junction window -> reuse restored. + assert last_req_hit(retention=0, pin=True) == 4 * block_size diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py index b08f451d2d..92283d3a53 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py @@ -235,6 +235,9 @@ class MooncakeStoreCoordinator: manager_cls = KVCacheSpecRegistry.get_manager_class(spec) assert manager_cls is not None use_eagle = g_idx in self.eagle_group_ids + reachable_boundaries = ( + () if num_prompt_tokens is None else (num_prompt_tokens - 1,) + ) mask = manager_cls.reachable_block_mask( start_block=start_chunk, end_block=end_chunk, @@ -242,7 +245,7 @@ class MooncakeStoreCoordinator: kv_cache_spec=spec, use_eagle=use_eagle, retention_interval=retention_interval, - num_prompt_tokens=num_prompt_tokens, + reachable_boundaries=reachable_boundaries, ) if mask is not None: assert len(mask) == end_chunk - start_chunk diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index bf88712aed..7d482192d5 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -370,7 +370,10 @@ class KVCacheCoordinator(ABC): self, block_hashes: list[BlockHash], max_cache_hit_length: int, - ) -> tuple[tuple[list[KVCacheBlock], ...], int]: + ) -> tuple[tuple[list[KVCacheBlock], ...], int, int]: + """Returns the per-group hit blocks, the hit length, and the number of + ``num_uncached_common_prefix_tokens`` (a shared prefix that a + sparse-retention group has not cached yet; 0 unless hybrid).""" pass def new_step_starts(self) -> None: @@ -422,11 +425,11 @@ class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator): self, block_hashes: list[BlockHash], max_cache_hit_length: int, - ) -> tuple[tuple[list[KVCacheBlock], ...], int]: + ) -> tuple[tuple[list[KVCacheBlock], ...], int, int]: blocks: tuple[list[KVCacheBlock], ...] = tuple( [] for _ in range(self.num_single_type_manager) ) - return blocks, 0 + return blocks, 0, 0 class UnitaryKVCacheCoordinator(KVCacheCoordinator): @@ -486,7 +489,7 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator): self, block_hashes: list[BlockHash], max_cache_hit_length: int, - ) -> tuple[tuple[list[KVCacheBlock], ...], int]: + ) -> tuple[tuple[list[KVCacheBlock], ...], int, int]: hit_blocks, hit_length = self.single_type_managers[0].find_longest_cache_hit( block_hashes=block_hashes, max_length=max_cache_hit_length, @@ -498,7 +501,8 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator): dcp_world_size=self.dcp_world_size, pcp_world_size=self.pcp_world_size, ) - return hit_blocks, hit_length + # Single group: nothing "uncached common" -- no other group to lag it. + return hit_blocks, hit_length, 0 class SpecGroup(NamedTuple): @@ -675,7 +679,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): self, block_hashes: list[BlockHash], max_cache_hit_length: int, - ) -> tuple[tuple[list[KVCacheBlock], ...], int]: + ) -> tuple[tuple[list[KVCacheBlock], ...], int, int]: """ Find the longest cache hit using an iterative fixed-point algorithm. @@ -691,7 +695,9 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): Returns: A tuple containing: - A tuple of the cache hit blocks for each single type manager. - - The number of tokens of the longest cache hit. + - The number of tokens of the reconciled (combined) cache hit. + - ``num_uncached_common_prefix_tokens``: a shared prefix that a + sparse-retention group has not cached yet (0 unless hybrid). """ num_groups = len(self.kv_cache_config.kv_cache_groups) @@ -794,12 +800,14 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): del blks[num_blocks:] hit_length_by_group[group_id] = hit_length - # Uncached shared prefix detection: If any attn. group cached a longer prefix - # than the current prefix, it is an uncached common prefix across requests: - self.num_uncached_common_prefix_tokens = longest_hit_length - hit_length - return tuple( + # Uncached shared prefix detection: if any attn. group cached a longer + # prefix than the reconciled hit, it is an uncached common prefix across + # requests that a sparse-retention group hasn't cached yet. + num_uncached_common_prefix_tokens = longest_hit_length - hit_length + cache_hit_blocks = tuple( blocks if blocks is not None else [] for blocks in hit_blocks_by_group - ), hit_length + ) + return cache_hit_blocks, hit_length, num_uncached_common_prefix_tokens def find_longest_cache_hit_per_group( self, diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 4f06b58e30..e3388a9f85 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -204,7 +204,7 @@ class KVCacheManager: self.prefix_cache_stats = PrefixCacheStats() return stats - def get_computed_blocks(self, request: Request) -> tuple[KVCacheBlocks, int]: + def get_computed_blocks(self, request: Request) -> tuple[KVCacheBlocks, int, int]: """Get the computed (cached) blocks for the request. Note that the computed blocks must be full. @@ -215,13 +215,18 @@ class KVCacheManager: A tuple containing: - A list of blocks that are computed for the request. - The number of computed tokens. + - ``shared_prefix_boundary``: the block-aligned token position of + a shared prefix that a sparse-retention group (Mamba / sliding + window) has not cached yet (Marconi-style APC), or 0 if none. + Pinned so ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL`` does not drop + the junction and defeat cross-request reuse. """ # We skip finding the prefix cache hit when prefix caching is # disabled or the request is marked as skipping kv cache read # (which happens when the request requires prompt logprobs # or calls a pooling model with all pooling). if not self.enable_caching or request.skip_reading_prefix_cache: - return self.empty_kv_cache_blocks, 0 + return self.empty_kv_cache_blocks, 0, 0 # NOTE: When all tokens hit the cache, we must recompute the last token # to obtain logits. Thus, set max_cache_hit_length to prompt_length - 1. @@ -230,7 +235,7 @@ class KVCacheManager: # num_computed_tokens to be block-size aligned. Removing this limitation # could slightly improve performance in the future. max_cache_hit_length = request.num_tokens - 1 - computed_blocks, num_new_computed_tokens = ( + computed_blocks, num_new_computed_tokens, num_uncached = ( self.coordinator.find_longest_cache_hit( request.block_hashes, max_cache_hit_length ) @@ -256,6 +261,14 @@ class KVCacheManager: group_idx, ) + # The junction to pin is where the lagging sparse-retention group stops + # (``num_new_computed_tokens``) plus the uncached shared prefix -- i.e. + # the longest single-group hit. Sub-block gaps are left to the mask, + # which floors to the alignment boundary (a no-op there). + shared_prefix_boundary = ( + num_new_computed_tokens + num_uncached if num_uncached else 0 + ) + if self.log_stats: assert self.prefix_cache_stats is not None self.prefix_cache_stats.record( @@ -264,7 +277,8 @@ class KVCacheManager: preempted=request.num_preemptions > 0, ) - return self.create_kv_cache_blocks(computed_blocks), num_new_computed_tokens + blocks = self.create_kv_cache_blocks(computed_blocks) + return blocks, num_new_computed_tokens, shared_prefix_boundary def allocate_slots( self, diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index be2918f450..371139fa72 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -350,85 +350,69 @@ class Scheduler(SchedulerInterface): num_new_tokens: int, num_new_local_computed_tokens: int = 0, num_external_computed_tokens: int = 0, - num_uncached_common_prefix_tokens: int = 0, ) -> int: - num_computed_tokens = ( + """Clip a prefill chunk so it ends where Mamba state must be cached. + + In "align" cache mode the SSM state is only materialized at chunk + ends, so chunk ends are steered onto cacheable positions: block + boundaries by default, plus mandatory early stops (the prompt's + partial-tail hash boundary, a detected shared-prefix junction). + """ + start = ( request.num_computed_tokens + num_new_local_computed_tokens + num_external_computed_tokens ) - # Perform block-aligned splitting at prefill phase, including: - # * non-resumed requests: num_computed_tokens < num_prompt_tokens + 0 - # * resumed requests: num_computed_tokens < ( - # num_prompt_tokens + num_output_tokens - # ) - # NOTE: Use `request.num_tokens - 1` to bypass normal decoding. - if num_computed_tokens < max(request.num_prompt_tokens, request.num_tokens - 1): - # To enable block-aligned caching of the Mamba state, `num_new_tokens` - # must be a multiple of `block_size`. - # As an exception, if `num_new_tokens` is less than `block_size`, the - # state is simply not cached, requiring no special handling. - # Additionally, when Eagle mode is enabled, FullAttn prunes the last - # matching block. To prevent this from causing a Mamba cache miss, the - # last chunk must be not smaller than `block_size`. - block_size = self.cache_config.block_size - last_cache_position = request.num_tokens - request.num_tokens % block_size - # eagle prune - if self.use_eagle: - last_cache_position = max(last_cache_position - block_size, 0) - num_computed_tokens_after_sched = num_computed_tokens + num_new_tokens - next_boundary = (num_computed_tokens // block_size + 1) * block_size - if ( - num_computed_tokens % block_size != 0 - and next_boundary <= last_cache_position - and num_computed_tokens_after_sched > next_boundary - ): - # Resumed mid-block (partial hash hit): stop the first chunk - # at the next block boundary so chunk ends re-align to blocks - # and the boundary state is materialized before running past. - num_new_tokens = next_boundary - num_computed_tokens - elif num_computed_tokens_after_sched < last_cache_position: - # Align the chunk END (not its length) to block_size; - # identical to flooring the length when the start is - # block-aligned. May yield 0 (insufficient budget to reach - # the next boundary); the caller then skips the request. - aligned_end = num_computed_tokens_after_sched // block_size * block_size - num_new_tokens = max(aligned_end - num_computed_tokens, 0) - elif ( - num_computed_tokens - < last_cache_position - < num_computed_tokens_after_sched - ): - # force to cache the last chunk - num_new_tokens = last_cache_position - num_computed_tokens - elif self.mamba_partial_cache_hit: - # Prefill of the final partial block: stop once at the - # prompt's last hash boundary so the mamba partial tail entry - # can be registered. - tail_boundary = ( - request.num_prompt_tokens - // self.hash_block_size - * self.hash_block_size - ) - if ( - num_computed_tokens - < tail_boundary - < num_computed_tokens_after_sched - and tail_boundary < request.num_prompt_tokens - and tail_boundary > last_cache_position - ): - num_new_tokens = tail_boundary - num_computed_tokens + # Split only during prefill: `request.num_tokens - 1` extends this to + # resumed requests replaying their output tokens. + if start >= max(request.num_prompt_tokens, request.num_tokens - 1): + return num_new_tokens - # Marconi cache admission optimization: - # cache common prefixes by scheduling num_new_tokens = common prefix length - if ( - num_uncached_common_prefix_tokens >= block_size - and num_new_tokens > num_uncached_common_prefix_tokens - ): - num_new_tokens = num_uncached_common_prefix_tokens - # keep alignment to block_size - num_new_tokens = num_new_tokens // block_size * block_size - return num_new_tokens + block_size = self.cache_config.block_size + # The last block-aligned position whose state can be cached. With + # Eagle, FullAttn prunes the last matching block, so back off one + # block to avoid a Mamba cache miss. + last_cache_position = request.num_tokens - request.num_tokens % block_size + if self.use_eagle: + last_cache_position = max(last_cache_position - block_size, 0) + + end = start + num_new_tokens + # Until `last_cache_position`, chunk ends must land on block + # boundaries. May yield an empty chunk (budget cannot reach the next + # boundary); the caller then skips the request. + if end < last_cache_position: + end = end // block_size * block_size + + next_block_boundary = (start // block_size + 1) * block_size + tail_boundary = ( + request.num_prompt_tokens // self.hash_block_size * self.hash_block_size + if self.mamba_partial_cache_hit + else 0 + ) + stops = ( + # Resumed mid-block (fine-grained partial hash hit): re-align to + # the block grid before running on, so the crossed boundary's + # state is materialized (unless it is past the cacheable range). + next_block_boundary + if start % block_size != 0 and next_block_boundary <= last_cache_position + else 0, + # Never run past the last cacheable block boundary mid-chunk. + last_cache_position, + # Fine-grained hits: the prompt's partial-tail entry can only be + # registered by a chunk ending exactly at its last hash boundary. + tail_boundary + if last_cache_position < tail_boundary < request.num_prompt_tokens + else 0, + # Marconi shared-prefix junction, block-floored (a sub-block + # junction's state is not separately cacheable): cache its state + # so sibling requests sharing the prefix can reuse it. + start + (request.shared_prefix_boundary - start) // block_size * block_size + if start < request.shared_prefix_boundary < end + else 0, + ) + # Stop at the earliest mandatory position strictly inside the chunk. + end = min((s for s in stops if start < s < end), default=end) + return max(end - start, 0) def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput: self.current_step += 1 @@ -718,7 +702,6 @@ class Scheduler(SchedulerInterface): num_external_computed_tokens = 0 load_kv_async = False connector_prefix_cache_queries, connector_prefix_cache_hits = 0, 0 - num_uncached_common_prefix_tokens = 0 # Get already-cached tokens. if request.num_computed_tokens == 0: @@ -733,8 +716,7 @@ class Scheduler(SchedulerInterface): ): computed, per_group_hits = ( self.kv_cache_manager.coordinator.find_longest_cache_hit_per_group( - request.block_hashes, - request.num_tokens - 1, + request.block_hashes, request.num_tokens - 1 ) ) new_computed_blocks = ( @@ -750,6 +732,9 @@ class Scheduler(SchedulerInterface): # the last block) is transferred unconditionally by # _apply_prefix_caching in nixl/worker.py. num_new_local_computed_tokens = max(per_group_hits) + # The per-group lookup does not detect an uncached shared + # prefix, so there is no junction to pin in this path. + request.shared_prefix_boundary = 0 if self.kv_cache_manager.log_stats: assert self.kv_cache_manager.prefix_cache_stats is not None self.kv_cache_manager.prefix_cache_stats.record( @@ -758,17 +743,15 @@ class Scheduler(SchedulerInterface): preempted=request.num_preemptions > 0, ) else: - new_computed_blocks, num_new_local_computed_tokens = ( - self.kv_cache_manager.get_computed_blocks(request) - ) - - # In case of hybrid models, obtain hint for Marconi-style APC logic - if self.has_mamba_layers: - num_uncached_common_prefix_tokens = getattr( - self.kv_cache_manager.coordinator, - "num_uncached_common_prefix_tokens", - 0, - ) + ( + new_computed_blocks, + num_new_local_computed_tokens, + # Junction to pin (Marconi-style APC) so its + # sparse-retention state (Mamba block / sliding-window + # tail) survives retention and serves a later hit; 0 + # if no uncached shared prefix was detected. + request.shared_prefix_boundary, + ) = self.kv_cache_manager.get_computed_blocks(request) # Get externally-cached tokens if using a KVConnector. if self.connector is not None: @@ -906,7 +889,6 @@ class Scheduler(SchedulerInterface): num_new_tokens, num_new_local_computed_tokens, num_external_computed_tokens, - num_uncached_common_prefix_tokens, ) if num_new_tokens == 0: break diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 21e3ad8ffd..de47ce0a2c 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -419,6 +419,13 @@ class SingleTypeKVCacheManager(ABC): if num_cached_blocks >= num_full_blocks: return + # Token boundaries whose reachable tail must be retained under sparse + # retention: the replay boundary (``num_prompt - 1``, capped by + # ``get_computed_blocks``) and any detected shared-prefix junction. + reachable_boundaries = [request.num_prompt_tokens - 1] + if request.shared_prefix_boundary: + reachable_boundaries.append(request.shared_prefix_boundary) + block_mask = self.reachable_block_mask( start_block=num_cached_blocks, end_block=num_full_blocks, @@ -426,7 +433,7 @@ class SingleTypeKVCacheManager(ABC): kv_cache_spec=self.kv_cache_spec, use_eagle=self.use_eagle, retention_interval=retention_interval, - num_prompt_tokens=request.num_prompt_tokens, + reachable_boundaries=reachable_boundaries, ) self.block_pool.cache_full_blocks( request=request, @@ -449,14 +456,15 @@ class SingleTypeKVCacheManager(ABC): kv_cache_spec: KVCacheSpec, use_eagle: bool, retention_interval: int | None = None, - num_prompt_tokens: int | None = None, + reachable_boundaries: Sequence[int] = (), ) -> list[bool] | None: """Per-block mask for ``cache_full_blocks``. ``None`` means cache every (non-null) block — the default for full attention. - Subclasses with sparse hit semantics (SWA) override this to skip - blocks that can never serve a hit at any alignment-aligned prefix - length. + Subclasses with sparse hit semantics (SWA / Mamba) override this to skip + blocks that can never serve a hit at any alignment-aligned prefix length. + ``reachable_boundaries`` are token positions whose reachable tail must be + retained; the base (dense) policy ignores them. """ return None @@ -962,7 +970,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager): kv_cache_spec: KVCacheSpec, use_eagle: bool, retention_interval: int | None = None, - num_prompt_tokens: int | None = None, + reachable_boundaries: Sequence[int] = (), ) -> list[bool] | None: assert isinstance(kv_cache_spec, SlidingWindowSpec) if alignment_tokens is None: @@ -1002,18 +1010,16 @@ class SlidingWindowManager(SingleTypeKVCacheManager): if i >= shift and (i - shift) % per_segment >= per_segment - need: mask[i - start_block] = True - # (2) Replay-boundary tail. ``get_computed_blocks`` caps hits at - # ``num_prompt - 1`` (to recompute the last token's logits), so an exact - # prompt replay can only land on the latest *fine*-aligned boundary. - # Sparse retention would otherwise skip it, so keep its tail explicitly. - if retention_interval is not None and num_prompt_tokens is not None: - latest = (num_prompt_tokens - 1) // alignment_tokens * alignment_tokens - prompt_end_block = latest // block_size + shift - for i in range( - max(start_block, prompt_end_block - need), - min(end_block, prompt_end_block), - ): - mask[i - start_block] = True + # (2) Reachable-boundary tails: the replay boundary (``num_prompt - 1``, + # capped by ``get_computed_blocks``) and any shared-prefix junction. Both + # land before segments would cover them under sparse retention, so keep + # the ``need``-block tail ending on each boundary explicitly. + if retention_interval is not None: + for boundary_tokens in reachable_boundaries: + aligned = boundary_tokens // alignment_tokens * alignment_tokens + end = aligned // block_size + shift + for j in range(max(start_block, end - need), min(end_block, end)): + mask[j - start_block] = True return mask @@ -1321,15 +1327,19 @@ class MambaManager(SingleTypeKVCacheManager): kv_cache_spec: KVCacheSpec, use_eagle: bool, retention_interval: int | None = None, - num_prompt_tokens: int | None = None, + reachable_boundaries: Sequence[int] = (), ) -> list[bool] | None: """Sparse Mamba state-snapshot retention. ``retention_interval``: ``None`` -> dense (cache every block; default, unchanged behavior) - ``0`` -> keep only the latest replay boundary + ``0`` -> keep only the ``reachable_boundaries`` states ``> 0`` -> keep one state per ``retention_interval``-sized segment + + ``reachable_boundaries`` are proven reuse points (the replay boundary and + any cross-request shared-prefix junction, Marconi-style APC); their + boundary state is always kept so sparse retention does not defeat reuse. """ if retention_interval is None or alignment_tokens is None: # Dense caching (default) or no alignment constraint imposed. @@ -1354,13 +1364,13 @@ class MambaManager(SingleTypeKVCacheManager): for i in range(first_boundary - start_block, len(mask), per_segment): mask[i] = True - # (2) Replay boundary. ``get_computed_blocks`` caps hits at - # ``num_prompt - 1``, so an exact prompt replay lands on the latest - # fine-aligned boundary. Sparse retention would otherwise skip its - # state, so keep it explicitly. - if num_prompt_tokens is not None: - latest = (num_prompt_tokens - 1) // alignment_tokens * alignment_tokens - boundary_block = latest // block_size - 1 + # (2) Reachable-boundary states: the replay boundary (``num_prompt - 1``, + # capped by ``get_computed_blocks``) and any shared-prefix junction, both + # of which segments would otherwise skip under sparse retention. A Mamba + # hit needs exactly the single state block ending on the boundary. + for boundary_tokens in reachable_boundaries: + aligned = boundary_tokens // alignment_tokens * alignment_tokens + boundary_block = aligned // block_size - 1 if start_block <= boundary_block < end_block: mask[boundary_block - start_block] = True diff --git a/vllm/v1/request.py b/vllm/v1/request.py index 00f1bdbdce..9e71591d36 100644 --- a/vllm/v1/request.py +++ b/vllm/v1/request.py @@ -182,6 +182,11 @@ class Request: # True if this request is scheduled as a non-final prefill chunk. self.is_prefill_chunk = False + # Block-aligned token position of a proven shared prefix worth pinning + # in the (sparse) prefix cache; 0 means none. Set at admission for + # hybrid/Mamba models when a shared prefix is detected (Marconi-style). + self.shared_prefix_boundary = 0 + # The number of NaNs in logits. A value greater than 0 # indicates that the output is corrupted self.num_nans_in_logits = 0 diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index f451f503d9..1d2b8a6b7e 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -262,7 +262,7 @@ class SimpleCPUOffloadScheduler: max_hit_len = request.num_tokens - 1 - num_computed_tokens if max_hit_len <= 0: return 0, False - cpu_hit_blocks, hit_length = self.cpu_coordinator.find_longest_cache_hit( + cpu_hit_blocks, hit_length, _ = self.cpu_coordinator.find_longest_cache_hit( remaining_hashes, max_hit_len )