[Scheduler] Don't double-count the checkpoint of a prefix-hit admission

The admission gate reserved one retained-checkpoint block per Mamba group for every request not holding one. A
follow-up turn with a prefix hit already brings that block, so it was counted twice; the gate now counts it once.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 20:13:06 +07:00
co-authored by Hinata
parent 010521f10d
commit 21fb7de7bf
2 changed files with 22 additions and 3 deletions
+1 -1
View File
@@ -241,7 +241,7 @@ class KVCacheCoordinator(ABC):
# the free queue that check reads. # the free queue that check reads.
num_blocks_to_allocate += ( num_blocks_to_allocate += (
manager.get_num_retained_checkpoint_blocks_to_reserve( manager.get_num_retained_checkpoint_blocks_to_reserve(
request_id, num_tokens_main_model request_id, num_tokens_main_model, new_computed_blocks[i]
) )
) )
return num_blocks_to_allocate return num_blocks_to_allocate
+21 -2
View File
@@ -618,7 +618,10 @@ class SingleTypeKVCacheManager(ABC):
return [] return []
def get_num_retained_checkpoint_blocks_to_reserve( def get_num_retained_checkpoint_blocks_to_reserve(
self, request_id: str, num_tokens: int self,
request_id: str,
num_tokens: int,
new_computed_blocks: Sequence[KVCacheBlock] = (),
) -> int: ) -> int:
"""Blocks the admission gate must add for checkpoints this request """Blocks the admission gate must add for checkpoints this request
will hold outside its block table (0 for every manager but align-mode will hold outside its block table (0 for every manager but align-mode
@@ -1614,7 +1617,10 @@ class MambaManager(SingleTypeKVCacheManager):
return [] if held is None else [held[1]] return [] if held is None else [held[1]]
def get_num_retained_checkpoint_blocks_to_reserve( def get_num_retained_checkpoint_blocks_to_reserve(
self, request_id: str, num_tokens: int self,
request_id: str,
num_tokens: int,
new_computed_blocks: Sequence[KVCacheBlock] = (),
) -> int: ) -> int:
# A request that already holds a checkpoint only ever swaps it. One # A request that already holds a checkpoint only ever swaps it. One
# that does not will hold one once a hashed state retires, which needs # that does not will hold one once a hashed state retires, which needs
@@ -1625,6 +1631,19 @@ class MambaManager(SingleTypeKVCacheManager):
or num_tokens <= self.block_size or num_tokens <= self.block_size
): ):
return 0 return 0
# A prefix hit already brings the block that becomes the held
# checkpoint -- the hit state, counted by get_num_blocks_to_allocate as
# an evictable computed block (or already referenced elsewhere) -- and
# the same holds for a hashed state still in the request's table.
# Reserving another one double-counts every hit admission by one block
# per group, which only delays the follow-up turns the fix is for.
if any(not blk.is_null for blk in new_computed_blocks):
return 0
if any(
not blk.is_null and blk.block_hash is not None
for blk in self.req_to_blocks.get(request_id, ())
):
return 0
return 1 return 1
@classmethod @classmethod