diff --git a/tests/v1/core/test_single_type_kv_cache_manager.py b/tests/v1/core/test_single_type_kv_cache_manager.py index dcb7ba923e..415609aefb 100644 --- a/tests/v1/core/test_single_type_kv_cache_manager.py +++ b/tests/v1/core/test_single_type_kv_cache_manager.py @@ -6,10 +6,14 @@ import random import pytest import torch +from vllm.sampling_params import SamplingParams +from vllm.utils.hashing import sha256 from vllm.v1.core.block_pool import BlockPool from vllm.v1.core.kv_cache_utils import ( BlockHash, KVCacheBlock, + get_request_block_hasher, + init_none_hash, make_block_hash_with_group_id, ) from vllm.v1.core.single_type_kv_cache_manager import ( @@ -30,6 +34,7 @@ from vllm.v1.kv_cache_interface import ( RSWASpec, SlidingWindowSpec, ) +from vllm.v1.request import Request pytestmark = pytest.mark.cpu_test @@ -238,6 +243,71 @@ def test_mamba_checkpoint_admission_matches_allocation( assert admission_estimate == allocation_estimate == allocated +@pytest.mark.parametrize("num_speculative_blocks", [1, 2, 3]) +@pytest.mark.parametrize("first_chunk_blocks", [2, 4]) +@pytest.mark.parametrize("retention_interval", [0, None]) +def test_mamba_checkpoint_hash_maps_only_to_checkpoint_block( + num_speculative_blocks, first_chunk_blocks, retention_interval +): + """A checkpoint step must not leave a never-written speculative block + cached.""" + block_size = 128 + spec = MambaSpec( + block_size=block_size, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + num_speculative_blocks=num_speculative_blocks, + num_prefill_checkpoint_blocks=1, + prefill_checkpoint_alignment=16, + ) + pool = BlockPool(num_gpu_blocks=64, enable_caching=True, hash_block_size=128) + manager = MambaManager( + spec, + block_pool=pool, + enable_caching=True, + kv_cache_group_id=0, + scheduler_block_size=block_size, + ) + # MTP/EAGLE moves the checkpoint one block below the last full block. + manager.drop_eagle_checkpoint_block = True + init_none_hash(sha256) + prompt_tokens = 6 * block_size + 50 + request = Request( + request_id="r", + prompt_token_ids=list(range(prompt_tokens)), + sampling_params=SamplingParams(max_tokens=1), + pooling_params=None, + block_hasher=get_request_block_hasher(block_size, sha256), + ) + + def step(start, end): + estimate = manager.get_num_blocks_to_allocate("r", end, [], start, start, end) + free_before = pool.get_num_free_blocks() + manager.allocate_new_blocks("r", end, end) + assert free_before - pool.get_num_free_blocks() == estimate + manager.cache_blocks( + request, + end, + retention_interval=retention_interval, + replay_boundaries=[5 * block_size], + ) + request.num_computed_tokens = end + + step(0, first_chunk_blocks * block_size) + speculative_blocks = manager.req_to_blocks["r"][-num_speculative_blocks:] + manager.new_step_starts() + step(first_chunk_blocks * block_size, prompt_tokens) + + checkpoint_position, checkpoint_idx = manager._checkpoints["r"] + checkpoint_block = manager.req_to_blocks["r"][checkpoint_idx] + assert all( + b.block_hash is None or b is checkpoint_block for b in speculative_blocks + ) + block_hash = request.block_hashes[checkpoint_position // block_size - 1] + assert pool.get_cached_block(block_hash, [0]) == [checkpoint_block] + + def get_sliding_window_manager( sliding_window_spec, block_pool, diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 0ff8059562..5940a1cd0a 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -1812,7 +1812,7 @@ class MambaManager(SingleTypeKVCacheManager): if num_new_blocks > 0: blocks_allocated = request_id in self._allocated_block_reqs physical_block_cap = 1 + int(has_partial_hit) + checkpoint_block - if not blocks_allocated or checkpoint_block: + if not blocks_allocated: physical_block_cap += self.num_speculative_blocks num_new_blocks = min(num_new_blocks, physical_block_cap) @@ -1885,12 +1885,14 @@ class MambaManager(SingleTypeKVCacheManager): [self._null_block for _ in range(prev_block_len, null_end)] ) - if blocks_allocated and not checkpoint_block: - # Relocate exclusively owned speculative scratch blocks. + if blocks_allocated: + # Relocate exclusively owned scratch blocks this step leaves + # behind. In a checkpoint step, one at the checkpoint index + # stays: the worker exports the checkpoint into that column. for block_idx in range( prev_block_len - self.num_speculative_blocks, prev_block_len ): - if block_idx < num_skipped_blocks: + if block_idx < num_skipped_blocks - checkpoint_block: self._relocate_speculative_block(req_blocks, block_idx) else: break @@ -1898,7 +1900,7 @@ class MambaManager(SingleTypeKVCacheManager): if has_partial_hit: num_new_blocks = max(num_new_blocks, 0) + 1 max_new_blocks = 1 + int(has_partial_hit) + checkpoint_block - if not blocks_allocated or checkpoint_block: + if not blocks_allocated: max_new_blocks += self.num_speculative_blocks assert num_new_blocks <= max_new_blocks new_blocks = self.block_pool.get_new_blocks(num_new_blocks)