[Scheduler] Don't double-count the checkpoint of a prefix-hit admission
The admission gate reserved one retained-checkpoint block per Mamba group for every request not holding one. A follow-up turn with a prefix hit already brings that block, so it was counted twice; the gate now counts it once. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -241,7 +241,7 @@ class KVCacheCoordinator(ABC):
|
|||||||
# the free queue that check reads.
|
# the free queue that check reads.
|
||||||
num_blocks_to_allocate += (
|
num_blocks_to_allocate += (
|
||||||
manager.get_num_retained_checkpoint_blocks_to_reserve(
|
manager.get_num_retained_checkpoint_blocks_to_reserve(
|
||||||
request_id, num_tokens_main_model
|
request_id, num_tokens_main_model, new_computed_blocks[i]
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
return num_blocks_to_allocate
|
return num_blocks_to_allocate
|
||||||
|
|||||||
@@ -618,7 +618,10 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
def get_num_retained_checkpoint_blocks_to_reserve(
|
def get_num_retained_checkpoint_blocks_to_reserve(
|
||||||
self, request_id: str, num_tokens: int
|
self,
|
||||||
|
request_id: str,
|
||||||
|
num_tokens: int,
|
||||||
|
new_computed_blocks: Sequence[KVCacheBlock] = (),
|
||||||
) -> int:
|
) -> int:
|
||||||
"""Blocks the admission gate must add for checkpoints this request
|
"""Blocks the admission gate must add for checkpoints this request
|
||||||
will hold outside its block table (0 for every manager but align-mode
|
will hold outside its block table (0 for every manager but align-mode
|
||||||
@@ -1614,7 +1617,10 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
return [] if held is None else [held[1]]
|
return [] if held is None else [held[1]]
|
||||||
|
|
||||||
def get_num_retained_checkpoint_blocks_to_reserve(
|
def get_num_retained_checkpoint_blocks_to_reserve(
|
||||||
self, request_id: str, num_tokens: int
|
self,
|
||||||
|
request_id: str,
|
||||||
|
num_tokens: int,
|
||||||
|
new_computed_blocks: Sequence[KVCacheBlock] = (),
|
||||||
) -> int:
|
) -> int:
|
||||||
# A request that already holds a checkpoint only ever swaps it. One
|
# A request that already holds a checkpoint only ever swaps it. One
|
||||||
# that does not will hold one once a hashed state retires, which needs
|
# that does not will hold one once a hashed state retires, which needs
|
||||||
@@ -1625,6 +1631,19 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
or num_tokens <= self.block_size
|
or num_tokens <= self.block_size
|
||||||
):
|
):
|
||||||
return 0
|
return 0
|
||||||
|
# A prefix hit already brings the block that becomes the held
|
||||||
|
# checkpoint -- the hit state, counted by get_num_blocks_to_allocate as
|
||||||
|
# an evictable computed block (or already referenced elsewhere) -- and
|
||||||
|
# the same holds for a hashed state still in the request's table.
|
||||||
|
# Reserving another one double-counts every hit admission by one block
|
||||||
|
# per group, which only delays the follow-up turns the fix is for.
|
||||||
|
if any(not blk.is_null for blk in new_computed_blocks):
|
||||||
|
return 0
|
||||||
|
if any(
|
||||||
|
not blk.is_null and blk.block_hash is not None
|
||||||
|
for blk in self.req_to_blocks.get(request_id, ())
|
||||||
|
):
|
||||||
|
return 0
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|||||||
Reference in New Issue
Block a user