[Scheduler] Don't double-count the checkpoint of a prefix-hit admission

The admission gate reserved one retained-checkpoint block per Mamba group for every request not holding one. A
follow-up turn with a prefix hit already brings that block, so it was counted twice; the gate now counts it once.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 20:13:06 +07:00
co-authored by Hinata
parent 010521f10d
commit 21fb7de7bf
2 changed files with 22 additions and 3 deletions
+1 -1
View File
@@ -241,7 +241,7 @@ class KVCacheCoordinator(ABC):
# the free queue that check reads.
num_blocks_to_allocate += (
manager.get_num_retained_checkpoint_blocks_to_reserve(
request_id, num_tokens_main_model
request_id, num_tokens_main_model, new_computed_blocks[i]
)
)
return num_blocks_to_allocate
+21 -2
View File
@@ -618,7 +618,10 @@ class SingleTypeKVCacheManager(ABC):
return []
def get_num_retained_checkpoint_blocks_to_reserve(
self, request_id: str, num_tokens: int
self,
request_id: str,
num_tokens: int,
new_computed_blocks: Sequence[KVCacheBlock] = (),
) -> int:
"""Blocks the admission gate must add for checkpoints this request
will hold outside its block table (0 for every manager but align-mode
@@ -1614,7 +1617,10 @@ class MambaManager(SingleTypeKVCacheManager):
return [] if held is None else [held[1]]
def get_num_retained_checkpoint_blocks_to_reserve(
self, request_id: str, num_tokens: int
self,
request_id: str,
num_tokens: int,
new_computed_blocks: Sequence[KVCacheBlock] = (),
) -> int:
# A request that already holds a checkpoint only ever swaps it. One
# that does not will hold one once a hashed state retires, which needs
@@ -1625,6 +1631,19 @@ class MambaManager(SingleTypeKVCacheManager):
or num_tokens <= self.block_size
):
return 0
# A prefix hit already brings the block that becomes the held
# checkpoint -- the hit state, counted by get_num_blocks_to_allocate as
# an evictable computed block (or already referenced elsewhere) -- and
# the same holds for a hashed state still in the request's table.
# Reserving another one double-counts every hit admission by one block
# per group, which only delays the follow-up turns the fix is for.
if any(not blk.is_null for blk in new_computed_blocks):
return 0
if any(
not blk.is_null and blk.block_hash is not None
for blk in self.req_to_blocks.get(request_id, ())
):
return 0
return 1
@classmethod