[Scheduler] Don't double-count the checkpoint of a prefix-hit admission
The admission gate reserved one retained-checkpoint block per Mamba group for every request not holding one. A follow-up turn with a prefix hit already brings that block, so it was counted twice; the gate now counts it once. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -241,7 +241,7 @@ class KVCacheCoordinator(ABC):
|
||||
# the free queue that check reads.
|
||||
num_blocks_to_allocate += (
|
||||
manager.get_num_retained_checkpoint_blocks_to_reserve(
|
||||
request_id, num_tokens_main_model
|
||||
request_id, num_tokens_main_model, new_computed_blocks[i]
|
||||
)
|
||||
)
|
||||
return num_blocks_to_allocate
|
||||
|
||||
@@ -618,7 +618,10 @@ class SingleTypeKVCacheManager(ABC):
|
||||
return []
|
||||
|
||||
def get_num_retained_checkpoint_blocks_to_reserve(
|
||||
self, request_id: str, num_tokens: int
|
||||
self,
|
||||
request_id: str,
|
||||
num_tokens: int,
|
||||
new_computed_blocks: Sequence[KVCacheBlock] = (),
|
||||
) -> int:
|
||||
"""Blocks the admission gate must add for checkpoints this request
|
||||
will hold outside its block table (0 for every manager but align-mode
|
||||
@@ -1614,7 +1617,10 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
return [] if held is None else [held[1]]
|
||||
|
||||
def get_num_retained_checkpoint_blocks_to_reserve(
|
||||
self, request_id: str, num_tokens: int
|
||||
self,
|
||||
request_id: str,
|
||||
num_tokens: int,
|
||||
new_computed_blocks: Sequence[KVCacheBlock] = (),
|
||||
) -> int:
|
||||
# A request that already holds a checkpoint only ever swaps it. One
|
||||
# that does not will hold one once a hashed state retires, which needs
|
||||
@@ -1625,6 +1631,19 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
or num_tokens <= self.block_size
|
||||
):
|
||||
return 0
|
||||
# A prefix hit already brings the block that becomes the held
|
||||
# checkpoint -- the hit state, counted by get_num_blocks_to_allocate as
|
||||
# an evictable computed block (or already referenced elsewhere) -- and
|
||||
# the same holds for a hashed state still in the request's table.
|
||||
# Reserving another one double-counts every hit admission by one block
|
||||
# per group, which only delays the follow-up turns the fix is for.
|
||||
if any(not blk.is_null for blk in new_computed_blocks):
|
||||
return 0
|
||||
if any(
|
||||
not blk.is_null and blk.block_hash is not None
|
||||
for blk in self.req_to_blocks.get(request_id, ())
|
||||
):
|
||||
return 0
|
||||
return 1
|
||||
|
||||
@classmethod
|
||||
|
||||
Reference in New Issue
Block a user