[Perf][Core] Skip the null head of sliding-window block tables (#61724)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Juntian Liu
2026-10-15 16:14:51 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 4a89316efa
commit 19b0efce30
6 changed files with 98 additions and 8 deletions
+6 -2
View File
@@ -759,9 +759,11 @@ class BlockPool:
"""
for block in blocks:
if block.is_null:
continue
# ref_cnt=0 means this block is in the free list (i.e. eviction
# candidate), so remove it.
if block.ref_cnt == 0 and not block.is_null:
if block.ref_cnt == 0:
self.free_block_queue.remove(block)
block.ref_cnt += 1
if self.metrics_collector:
@@ -785,11 +787,13 @@ class BlockPool:
blocks_to_evict_first = []
other_pools: dict[BlockPool, list[KVCacheBlock]] = {}
for block in ordered_blocks:
if block.is_null: # never pooled; its ref_cnt is not maintained
continue
if block.pool is not None and block.pool is not self:
other_pools.setdefault(block.pool, []).append(block)
continue
block.ref_cnt -= 1
if block.ref_cnt == 0 and not block.is_null:
if block.ref_cnt == 0:
if block.block_hash is None or not self.enable_caching:
# LIFO reuse of non-cached blocks for better GPU locality.
blocks_to_evict_first.append(block)