[Bugfix][Core] Retire Mamba states across null gaps (#55450)
Signed-off-by: Luca Motz <luca.motz@icloud.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Cyrus Leung
parent
a9271c750f
commit
fadfe1c7d4
@@ -1460,6 +1460,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
# Mapping from request ID to the index of the block
|
||||
# allocated in the previous step
|
||||
self.last_state_block_idx: dict[str, int] = {}
|
||||
self._num_retired_blocks: dict[str, int] = {}
|
||||
# The set of the requests that have been allocated blocks
|
||||
self._allocated_block_reqs: set[str] = set()
|
||||
# checkpoint position and reserved block index for the current
|
||||
@@ -1610,6 +1611,27 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
|
||||
return mask
|
||||
|
||||
def _remove_blocks_in_range(
|
||||
self, request_id: str, first_block: int, last_block: int
|
||||
) -> None:
|
||||
if self.mamba_cache_mode != "align":
|
||||
return super()._remove_blocks_in_range(request_id, first_block, last_block)
|
||||
blocks = self.req_to_blocks.get(request_id, [])
|
||||
first_block = max(first_block, self._num_retired_blocks.get(request_id, 0))
|
||||
last_block = min(last_block, len(blocks))
|
||||
if first_block >= last_block:
|
||||
return
|
||||
freed: list[KVCacheBlock] = []
|
||||
# Mamba prefill leaves null gaps between states awaiting retirement.
|
||||
for i in range(last_block - 1, first_block - 1, -1):
|
||||
if blocks[i].is_null:
|
||||
continue
|
||||
freed.append(blocks[i])
|
||||
blocks[i] = self._null_block
|
||||
if freed:
|
||||
self.block_pool.free_blocks(freed)
|
||||
self._num_retired_blocks[request_id] = last_block
|
||||
|
||||
def remove_skipped_blocks(
|
||||
self,
|
||||
request_id: str,
|
||||
@@ -1930,6 +1952,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
if self.mamba_cache_mode == "align":
|
||||
self._allocated_block_reqs.discard(request_id)
|
||||
self.last_state_block_idx.pop(request_id, None)
|
||||
self._num_retired_blocks.pop(request_id, None)
|
||||
self._checkpoints.pop(request_id, None)
|
||||
self._producer_partial_tail_reqs.pop(request_id, None)
|
||||
# An offer is only guaranteed to hold committed bytes until the end
|
||||
|
||||
Reference in New Issue
Block a user