diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index a53d075fb1..702ae57456 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -253,7 +253,10 @@ class KVCacheCoordinator(ABC): # the free queue that check reads. num_blocks_to_allocate += ( manager.get_num_retained_checkpoint_blocks_to_reserve( - request_id, num_tokens_main_model, new_computed_blocks[i] + request_id, + num_tokens_main_model, + new_computed_blocks[i], + num_local_computed_tokens, ) ) return num_blocks_to_allocate diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 943fd39da1..19240e68a8 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -622,6 +622,7 @@ class SingleTypeKVCacheManager(ABC): request_id: str, num_tokens: int, new_computed_blocks: Sequence[KVCacheBlock] = (), + num_local_computed_tokens: int = 0, ) -> int: """Blocks the admission gate must add for checkpoints this request will hold outside its block table (0 for every manager but align-mode @@ -1685,7 +1686,24 @@ class MambaManager(SingleTypeKVCacheManager): and self.enable_caching and envs.VLLM_E2_MAMBA_RETAIN_CHECKPOINT ) - # {request_id: (logical block index, block)} + # {request_id: (hashed token position, block)}: the newest hashed + # state the request resumed from or produced, held with one ref of + # its own (ordered by ``block_hash_num_tokens`` rather than + # block index so a sub-block partial-tail key sorts correctly against + # full-block boundary states). + # + # With ``--prefix-match-unit`` below the block size a request + # that resumed at a partial tail (mid-block) never hashes a boundary + # state: the replay boundary lies below its resume point. Its only + # reusable GDN state is its own partial-tail checkpoint, which the + # next chunk moves into a CoW block in no block table; the scheduler + # drops the copy's retention refs once the copy has run, so that + # block sat at ref 0 in the free queue for the rest of the request, + # ahead of its attention blocks, and concurrent prefills evicted it + # (measured: 7/61 follow-ups with cached_tokens 0 at unit 64, every + # one after a partial-tail resume; none at unit 4096). The CoW paths + # therefore hold their hashed endpoint here too: the moved tail for + # a running request, and the hit source a new request copies from. self._retained_checkpoints: dict[str, tuple[int, KVCacheBlock]] = {} if self.mamba_cache_mode == "align" and self.enable_caching: # The A/B evidence line: boot logs must show which arm ran. @@ -1694,6 +1712,16 @@ class MambaManager(SingleTypeKVCacheManager): "(VLLM_E2_MAMBA_RETAIN_CHECKPOINT)", "on" if self._retain_checkpoint else "off", ) + if ( + self._retain_checkpoint + and self.block_size > self.block_pool.hash_block_size + ): + logger.info_once( + "Mamba align: each request holds its partial-tail CoW checkpoint " + "until it is freed (hash unit %d, block %d)", + self.block_pool.hash_block_size, + self.block_size, + ) def _retire_state_blocks( self, request_id: str, retired: list[tuple[int, KVCacheBlock]] @@ -1708,13 +1736,16 @@ class MambaManager(SingleTypeKVCacheManager): """ if self._retain_checkpoint: newest: tuple[int, KVCacheBlock] | None = None - for idx, block in retired: - if block.block_hash is not None and ( - newest is None or idx > newest[0] - ): - newest = (idx, block) + for _, block in retired: + if block.block_hash is None: + continue + position = block.block_hash_num_tokens + assert position is not None + if newest is None or position > newest[0]: + newest = (position, block) held = self._retained_checkpoints.get(request_id) if newest is not None and (held is None or newest[0] > held[0]): + # The retired block's table ref becomes the hold's ref. self._retained_checkpoints[request_id] = newest retired = [entry for entry in retired if entry[1] is not newest[1]] if held is not None: @@ -1723,6 +1754,43 @@ class MambaManager(SingleTypeKVCacheManager): if retired: self.block_pool.free_blocks([block for _, block in retired]) + def _hold_checkpoint(self, request_id: str, block: KVCacheBlock) -> bool: + """Hold ``block`` (a hashed state outside the request's table) as the + request's prefix checkpoint, taking one extra ref for the hold. + + Only the newest position is held: an older one is left alone (it ages + from wherever it is), a superseded one is released here. The hold's + ref is returned by ``pop_retained_checkpoints`` -> ``free_blocks``. + Returns whether the block is now held. + """ + if not self._retain_checkpoint or block.is_null or block.block_hash is None: + return False + position = block.block_hash_num_tokens + assert position is not None + held = self._retained_checkpoints.get(request_id) + if held is not None and (held[1] is block or held[0] >= position): + return False + self.block_pool.touch([block]) + self._retained_checkpoints[request_id] = (position, block) + if held is not None: + self.block_pool.free_blocks([held[1]]) + return True + + def _follow_moved_hash( + self, owner_id: str, source_block: KVCacheBlock, cow_block: KVCacheBlock + ) -> None: + """``owner_id`` just moved ``source_block``'s hashes to ``cow_block`` + and keeps writing ``source_block``. A request that resumed at that + tail while it was still the owner's live block holds the source for + those hashes; move its hold to the copy, so it pins the hashed state + and not another request's running block. + """ + for request_id, (position, block) in list(self._retained_checkpoints.items()): + if request_id != owner_id and block is source_block: + self.block_pool.touch([cow_block]) + self._retained_checkpoints[request_id] = (position, cow_block) + self.block_pool.free_blocks([source_block]) + def pop_retained_checkpoints(self, request_id: str) -> list[KVCacheBlock]: if self.mamba_cache_mode != "align": return [] @@ -1734,16 +1802,25 @@ class MambaManager(SingleTypeKVCacheManager): request_id: str, num_tokens: int, new_computed_blocks: Sequence[KVCacheBlock] = (), + num_local_computed_tokens: int = 0, ) -> int: # A request that already holds a checkpoint only ever swaps it. One # that does not will hold one once a hashed state retires, which needs # at least one state block behind the running one. - if ( - not self._retain_checkpoint - or request_id in self._retained_checkpoints - or num_tokens <= self.block_size - ): + if not self._retain_checkpoint or request_id in self._retained_checkpoints: return 0 + # A sub-block (partial-tail) hit holds the hit source only + # until the request's own tail is moved into a CoW block that is held + # in its place; at that step both are referenced, so the request's + # lifetime peak is one block above what the hit brought (whatever the + # prompt length: the tail CoW happens inside the first block too). + if self._has_partial_local_hit(new_computed_blocks, num_local_computed_tokens): + return 1 + if num_tokens <= self.block_size: + # No state block retires behind the running one, but a prompt + # whose tail is keyed inside the block still moves that tail into + # a held CoW block when the hash unit is finer than the block. + return int(self.block_pool.hash_block_size < self.block_size) # A prefix hit already brings the block that becomes the held # checkpoint -- the hit state, counted by get_num_blocks_to_allocate as # an evictable computed block (or already referenced elsewhere) -- and @@ -2195,6 +2272,7 @@ class MambaManager(SingleTypeKVCacheManager): # overwrites source_block. assert req_blocks[block_idx] is source_block self.block_pool.move_block_hashes(source_block, cow_block) + self._follow_moved_hash(request_id, source_block, cow_block) self._pending_cow_copies.append((source_block, cow_block)) source_block.ref_cnt += 1 producer_tail = self._producer_partial_tail_reqs.pop( @@ -2218,9 +2296,20 @@ class MambaManager(SingleTypeKVCacheManager): # The moved entry is only filled by this step's # copy, so defer same-step hits on it. self.cached_blocks_this_step.add(cow_block.block_hash) + # The moved tail is the request's newest + # reusable state and lives in no block table; hold it + # past the copy's retention so the request's next + # turn can resume at it (see __init__). + self._hold_checkpoint(request_id, cow_block) else: self._apply_cow(request_id, block_idx, source_block, cow_block) returned_blocks = [cow_block] + returned_blocks + # The hit source keeps the hash while the + # request advances its private copy; hold it until + # the request's own tail supersedes it, so a request + # that stops early (preempted, aborted, block-aligned + # prompt) still leaves the state it resumed from. + self._hold_checkpoint(request_id, source_block) req_blocks.extend(new_blocks) self._allocated_block_reqs.add(request_id) self._partial_hit_reqs.pop(request_id, None)