[Perf][Core] Skip the null head of sliding-window block tables (#61724)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Juntian Liu
2026-10-15 16:14:51 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 4a89316efa
commit 19b0efce30
6 changed files with 98 additions and 8 deletions
+6 -2
View File
@@ -759,9 +759,11 @@ class BlockPool:
"""
for block in blocks:
if block.is_null:
continue
# ref_cnt=0 means this block is in the free list (i.e. eviction
# candidate), so remove it.
if block.ref_cnt == 0 and not block.is_null:
if block.ref_cnt == 0:
self.free_block_queue.remove(block)
block.ref_cnt += 1
if self.metrics_collector:
@@ -785,11 +787,13 @@ class BlockPool:
blocks_to_evict_first = []
other_pools: dict[BlockPool, list[KVCacheBlock]] = {}
for block in ordered_blocks:
if block.is_null: # never pooled; its ref_cnt is not maintained
continue
if block.pool is not None and block.pool is not self:
other_pools.setdefault(block.pool, []).append(block)
continue
block.ref_cnt -= 1
if block.ref_cnt == 0 and not block.is_null:
if block.ref_cnt == 0:
if block.block_hash is None or not self.enable_caching:
# LIFO reuse of non-cached blocks for better GPU locality.
blocks_to_evict_first.append(block)
+2 -2
View File
@@ -14,7 +14,7 @@ from vllm.v1.core.kv_cache_coordinator import (
get_kv_cache_coordinator,
)
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy
from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy, trim_null_head
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
from vllm.v1.kv_cache_interface import (
AttentionSpec,
@@ -117,7 +117,7 @@ class KVCacheBlocks:
return [
[
block.block_id
for block in group
for block in trim_null_head(group)
if block.block_hash is None and not block.is_null
]
for group in self.blocks
+35 -1
View File
@@ -8,7 +8,15 @@ import math
import os
from collections.abc import Callable, Iterable, Iterator, Sequence
from dataclasses import dataclass, field, replace
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, overload
from typing import (
TYPE_CHECKING,
Any,
NamedTuple,
NewType,
TypeAlias,
TypeVar,
overload,
)
from vllm import envs
from vllm.config import VllmConfig
@@ -216,6 +224,32 @@ class KVCacheBlock:
)
def null_head_len(blocks: Sequence[KVCacheBlock]) -> int:
"""Number of leading null blocks, e.g. a sliding-window table's skipped prefix."""
if not blocks or not blocks[0].is_null:
return 0
lo, hi = 1, len(blocks)
while lo < hi:
mid = (lo + hi) // 2
if blocks[mid].is_null:
lo = mid + 1
else:
hi = mid
# All null entries are one shared object, so count() checks the run at C speed.
if blocks[:lo].count(blocks[0]) == lo:
return lo
return next(i for i, b in enumerate(blocks) if not b.is_null)
_BlockSeq = TypeVar("_BlockSeq", list[KVCacheBlock], Sequence[KVCacheBlock])
def trim_null_head(blocks: _BlockSeq) -> _BlockSeq:
"""``blocks`` without its leading null blocks; ``blocks`` itself if none."""
nhl = null_head_len(blocks)
return blocks[nhl:] if nhl else blocks
class KVCacheBlockCopy(NamedTuple):
src_block_id: int
dst_block_id: int
+5 -3
View File
@@ -16,6 +16,7 @@ from vllm.v1.core.kv_cache_utils import (
BlockHashWithGroupId,
KVCacheBlock,
resolve_block_hashes,
trim_null_head,
)
from vllm.v1.hisparse.block_pool import SharedEventQueueBlockPool
from vllm.v1.kv_cache_interface import (
@@ -569,14 +570,15 @@ class SingleTypeKVCacheManager(ABC):
request_id: The request ID.
Returns:
The request's blocks in allocation order.
The request's blocks in allocation order, without the leading
null blocks (e.g. a sliding-window table's skipped prefix).
"""
# Default to [] in case a request is freed (aborted) before alloc.
req_blocks = self.req_to_blocks.pop(request_id, [])
self.num_cached_block.pop(request_id, None)
self._partial_hit_reqs.pop(request_id, None)
return req_blocks
return trim_null_head(req_blocks)
def free(self, request_id: str) -> None:
"""Free the blocks for the request.
@@ -2538,7 +2540,7 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
blocks = self.get_residency_row(request_id)
self.residency_changes.pop(request_id, None)
super().pop_blocks_for_free(request_id)
return blocks
return trim_null_head(blocks)
def adopt_resident_page(
self, request_id: str, block_idx: int, block: KVCacheBlock