[Perf][Core] Skip the null head of sliding-window block tables (#61724)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
4a89316efa
commit
19b0efce30
@@ -759,9 +759,11 @@ class BlockPool:
|
||||
|
||||
"""
|
||||
for block in blocks:
|
||||
if block.is_null:
|
||||
continue
|
||||
# ref_cnt=0 means this block is in the free list (i.e. eviction
|
||||
# candidate), so remove it.
|
||||
if block.ref_cnt == 0 and not block.is_null:
|
||||
if block.ref_cnt == 0:
|
||||
self.free_block_queue.remove(block)
|
||||
block.ref_cnt += 1
|
||||
if self.metrics_collector:
|
||||
@@ -785,11 +787,13 @@ class BlockPool:
|
||||
blocks_to_evict_first = []
|
||||
other_pools: dict[BlockPool, list[KVCacheBlock]] = {}
|
||||
for block in ordered_blocks:
|
||||
if block.is_null: # never pooled; its ref_cnt is not maintained
|
||||
continue
|
||||
if block.pool is not None and block.pool is not self:
|
||||
other_pools.setdefault(block.pool, []).append(block)
|
||||
continue
|
||||
block.ref_cnt -= 1
|
||||
if block.ref_cnt == 0 and not block.is_null:
|
||||
if block.ref_cnt == 0:
|
||||
if block.block_hash is None or not self.enable_caching:
|
||||
# LIFO reuse of non-cached blocks for better GPU locality.
|
||||
blocks_to_evict_first.append(block)
|
||||
|
||||
Reference in New Issue
Block a user