[1/N][Core] add partial prefix cache primitives (#45939)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Jiangyun Zhu
2026-06-21 23:43:10 -07:00
committed by GitHub
co-authored by Yifan Qiao
parent 1eb2cc961e
commit 6bc6f2d86d
5 changed files with 719 additions and 41 deletions
@@ -0,0 +1,460 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import pytest
import vllm.v1.core.kv_cache_utils as kv_cache_utils
from vllm.distributed.kv_events import BlockRemoved, BlockStored
from vllm.sampling_params import SamplingParams
from vllm.utils.hashing import sha256
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_utils import (
BlockHash,
BlockHashListWithBlockSize,
KVCacheBlock,
get_request_block_hasher,
hash_block_tokens,
init_none_hash,
)
from vllm.v1.request import Request
pytestmark = pytest.mark.cpu_test
@pytest.fixture(autouse=True)
def _auto_init_hash_fn():
init_none_hash(sha256)
def make_request(
request_id: str,
prompt_token_ids: list[int],
hash_block_size: int,
hash_fn: Callable,
) -> Request:
sampling_params = SamplingParams(max_tokens=17)
sampling_params.update_from_generation_config({}, eos_token_id=100)
return Request(
request_id=request_id,
prompt_token_ids=prompt_token_ids,
sampling_params=sampling_params,
pooling_params=None,
block_hasher=get_request_block_hasher(hash_block_size, hash_fn),
)
def boundary_hash(req: Request, hash_block_size: int, num_tokens: int) -> BlockHash:
# Every boundary at a hash_block_size multiple is just the fine-grained
# chain hash ending there.
return req.block_hashes[num_tokens // hash_block_size - 1]
def cache_full_block_and_partial_tail(
token_ids: list[int],
*,
enable_kv_cache_events: bool = False,
) -> tuple[BlockPool, Request, list[KVCacheBlock], BlockHash]:
hash_block_size = 2
block_size = 6
kv_cache_group_id = 0
req = make_request("0", token_ids, hash_block_size, sha256)
pool = BlockPool(
num_gpu_blocks=3,
enable_caching=True,
hash_block_size=hash_block_size,
enable_kv_cache_events=enable_kv_cache_events,
)
blocks = pool.get_new_blocks(2)
pool.cache_full_blocks(
request=req,
blocks=blocks,
num_cached_blocks=0,
num_full_blocks=1,
block_size=block_size,
kv_cache_group_id=kv_cache_group_id,
)
partial_hash = boundary_hash(req, hash_block_size, len(token_ids))
assert pool.cache_partial_block(
request=req,
block=blocks[1],
num_tokens=len(token_ids),
kv_cache_group_id=kv_cache_group_id,
block_size=block_size,
)
return pool, req, blocks, partial_hash
def test_boundary_hashes_reuse_fine_grained_chain():
hash_block_size = 2
block_size = 6
token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
req = make_request("0", token_ids, hash_block_size, sha256)
coarse = BlockHashListWithBlockSize(req.block_hashes, hash_block_size, block_size)
# The block_size=6 full-block hash is the fine hash at the 6-token boundary,
# not a concatenation of the three fine hashes inside the block.
assert coarse[0] == req.block_hashes[6 // hash_block_size - 1]
assert coarse[0] != BlockHash(
req.block_hashes[0] + req.block_hashes[1] + req.block_hashes[2]
)
# A partial tail at 10 tokens is the fine hash at the 10-token boundary,
# which chains over the entire prefix.
tail_hash = boundary_hash(req, hash_block_size, 10)
assert tail_hash == req.block_hashes[4]
assert tail_hash == hash_block_tokens(sha256, req.block_hashes[3], token_ids[8:10])
def test_cache_partial_block_kv_cache_events():
hash_block_size = 4
block_size = 12
kv_cache_group_id = 2
pool = BlockPool(
num_gpu_blocks=2,
enable_caching=True,
hash_block_size=hash_block_size,
enable_kv_cache_events=True,
)
req = make_request(
"req_partial_events",
prompt_token_ids=list(range(hash_block_size * 2)),
hash_block_size=hash_block_size,
hash_fn=sha256,
)
block = pool.get_new_blocks(1)[0]
partial_entry_hash = pool.cache_partial_block(
request=req,
block=block,
num_tokens=hash_block_size * 2,
kv_cache_group_id=kv_cache_group_id,
block_size=block_size,
)
events = pool.take_events()
assert len(events) == 1
stored_event = events[0]
assert isinstance(stored_event, BlockStored)
assert partial_entry_hash is not None
assert stored_event.block_hashes == [
kv_cache_utils.maybe_convert_block_hash(req.block_hashes[1])
]
assert stored_event.parent_block_hash == kv_cache_utils.maybe_convert_block_hash(
req.block_hashes[0]
)
assert stored_event.token_ids == req.all_token_ids[hash_block_size:]
assert stored_event.block_size == 4
assert stored_event.group_idx == kv_cache_group_id
duplicate_entry_hash = pool.cache_partial_block(
request=req,
block=block,
num_tokens=hash_block_size * 2,
kv_cache_group_id=kv_cache_group_id,
block_size=block_size,
)
assert duplicate_entry_hash == partial_entry_hash
assert pool.take_events() == []
pool.free_blocks([block])
pool.get_new_blocks(1)
events = pool.take_events()
assert len(events) == 1
removed_event = events[0]
assert isinstance(removed_event, BlockRemoved)
assert removed_event.block_hashes == stored_event.block_hashes
assert removed_event.group_idx == kv_cache_group_id
def test_partial_block_replacement_emits_remove_then_store_events():
hash_block_size = 2
block_size = 6
kv_cache_group_id = 0
req = make_request("0", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
pool = BlockPool(
num_gpu_blocks=3,
enable_caching=True,
hash_block_size=hash_block_size,
enable_kv_cache_events=True,
)
blocks = pool.get_new_blocks(2)
pool.cache_full_blocks(
request=req,
blocks=blocks,
num_cached_blocks=0,
num_full_blocks=1,
block_size=block_size,
kv_cache_group_id=kv_cache_group_id,
)
partial_hash_8 = boundary_hash(req, hash_block_size, 8)
assert pool.cache_partial_block(
request=req,
block=blocks[1],
num_tokens=8,
kv_cache_group_id=kv_cache_group_id,
block_size=block_size,
)
assert pool.get_cached_block(partial_hash_8, [kv_cache_group_id]) == [blocks[1]]
pool.take_events()
req.append_output_token_ids([4, 4])
partial_hash_10 = boundary_hash(req, hash_block_size, 10)
assert pool.cache_partial_block(
request=req,
block=blocks[1],
num_tokens=10,
kv_cache_group_id=kv_cache_group_id,
block_size=block_size,
)
events = pool.take_events()
assert len(events) == 2
removed_event, stored_event = events
assert isinstance(removed_event, BlockRemoved)
assert removed_event.block_hashes == [
kv_cache_utils.maybe_convert_block_hash(partial_hash_8)
]
assert removed_event.group_idx == kv_cache_group_id
assert isinstance(stored_event, BlockStored)
assert stored_event.block_hashes == [
kv_cache_utils.maybe_convert_block_hash(partial_hash_10)
]
assert stored_event.parent_block_hash == kv_cache_utils.maybe_convert_block_hash(
boundary_hash(req, hash_block_size, 8)
)
assert stored_event.token_ids == req.all_token_ids[8:10]
assert stored_event.block_size == hash_block_size
assert stored_event.group_idx == kv_cache_group_id
assert pool.get_cached_block(partial_hash_8, [kv_cache_group_id]) is None
assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) == [blocks[1]]
def test_later_request_hits_cached_partial_tail():
hash_block_size = 2
block_size = 6
kv_cache_group_id = 0
cached_token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
req = make_request("0", cached_token_ids, hash_block_size, sha256)
pool = BlockPool(
num_gpu_blocks=3,
enable_caching=True,
hash_block_size=hash_block_size,
)
blocks = pool.get_new_blocks(2)
pool.cache_full_blocks(
request=req,
blocks=blocks,
num_cached_blocks=0,
num_full_blocks=1,
block_size=block_size,
kv_cache_group_id=kv_cache_group_id,
)
partial_hash_10 = boundary_hash(req, hash_block_size, 10)
assert pool.cache_partial_block(
request=req,
block=blocks[1],
num_tokens=10,
kv_cache_group_id=kv_cache_group_id,
block_size=block_size,
)
replay = make_request("1", cached_token_ids, hash_block_size, sha256)
replay_hash_10 = boundary_hash(replay, hash_block_size, 10)
assert replay_hash_10 == partial_hash_10
assert pool.get_cached_block(replay_hash_10, [kv_cache_group_id]) == [blocks[1]]
extended = make_request("2", cached_token_ids + [10], hash_block_size, sha256)
extended_hash_10 = boundary_hash(extended, hash_block_size, 10)
assert extended_hash_10 == partial_hash_10
assert pool.get_cached_block(extended_hash_10, [kv_cache_group_id]) == [blocks[1]]
def test_cache_partial_block_uses_fine_grained_boundary_hash():
hash_block_size = 2
block_size = 6
kv_cache_group_id = 0
token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
req = make_request("0", token_ids, hash_block_size, sha256)
pool = BlockPool(
num_gpu_blocks=3,
enable_caching=True,
hash_block_size=hash_block_size,
)
blocks = pool.get_new_blocks(2)
pool.cache_full_blocks(
request=req,
blocks=blocks,
num_cached_blocks=0,
num_full_blocks=1,
block_size=block_size,
kv_cache_group_id=kv_cache_group_id,
)
partial_entry_hash = pool.cache_partial_block(
request=req,
block=blocks[1],
num_tokens=10,
kv_cache_group_id=kv_cache_group_id,
block_size=block_size,
)
# The partial entry is keyed by the fine-grained hash at the 10-token
# boundary, regardless of the owning group's block_size.
expected = boundary_hash(req, hash_block_size, 10)
assert partial_entry_hash == kv_cache_utils.make_block_hash_with_group_id(
expected, kv_cache_group_id
)
assert pool.get_cached_block(expected, [kv_cache_group_id]) == [blocks[1]]
def test_cache_partial_block_requires_hash_boundary():
hash_block_size = 2
block_size = 4
req = make_request("0", [0, 0, 1, 1], hash_block_size, sha256)
pool = BlockPool(
num_gpu_blocks=2,
enable_caching=True,
hash_block_size=hash_block_size,
)
block = pool.get_new_blocks(1)[0]
with pytest.raises(AssertionError):
pool.cache_partial_block(
request=req,
block=block,
num_tokens=3,
kv_cache_group_id=0,
block_size=block_size,
)
def test_cache_partial_block_duplicate_checks_all_blocks_for_hash():
hash_block_size = 2
block_size = 4
kv_cache_group_id = 0
req = make_request("0", [0, 0, 1, 1], hash_block_size, sha256)
pool = BlockPool(
num_gpu_blocks=4,
enable_caching=True,
hash_block_size=hash_block_size,
)
blocks = pool.get_new_blocks(2)
first_entry_hash = pool.cache_partial_block(
request=req,
block=blocks[0],
num_tokens=2,
kv_cache_group_id=kv_cache_group_id,
block_size=block_size,
)
second_entry_hash = pool.cache_partial_block(
request=req,
block=blocks[1],
num_tokens=2,
kv_cache_group_id=kv_cache_group_id,
block_size=block_size,
)
assert first_entry_hash == second_entry_hash
duplicate_entry_hash = pool.cache_partial_block(
request=req,
block=blocks[1],
num_tokens=2,
kv_cache_group_id=kv_cache_group_id,
block_size=block_size,
)
assert duplicate_entry_hash == second_entry_hash
assert pool.cached_block_hashes_by_block == {}
def test_reset_prefix_cache_clears_partial_entry_metadata():
pool, req, blocks, partial_hash_10 = cache_full_block_and_partial_tail(
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
)
full_hash = BlockHashListWithBlockSize(req.block_hashes, 2, 6)[0]
assert pool.get_cached_block(full_hash, [0]) == [blocks[0]]
assert pool.get_cached_block(partial_hash_10, [0]) == [blocks[1]]
pool.free_blocks(blocks)
assert pool.reset_prefix_cache()
assert pool.get_cached_block(full_hash, [0]) is None
assert pool.get_cached_block(partial_hash_10, [0]) is None
assert pool.cached_block_hashes_by_block == {}
def test_evict_cached_block_removes_full_hash_and_partial_entry():
pool, req, blocks, partial_hash_10 = cache_full_block_and_partial_tail(
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
)
full_hash = BlockHashListWithBlockSize(req.block_hashes, 2, 6)[0]
assert pool.get_cached_block(full_hash, [0]) == [blocks[0]]
assert pool.get_cached_block(partial_hash_10, [0]) == [blocks[1]]
pool.evict_blocks({blocks[0].block_id, blocks[1].block_id})
assert pool.get_cached_block(full_hash, [0]) is None
assert pool.get_cached_block(partial_hash_10, [0]) is None
assert pool.cached_block_hashes_by_block == {}
def test_partial_block_promotes_to_direct_full_block_hash():
hash_block_size = 2
block_size = 6
kv_cache_group_id = 0
token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
req = make_request("0", token_ids, hash_block_size, sha256)
pool = BlockPool(
num_gpu_blocks=3,
enable_caching=True,
hash_block_size=hash_block_size,
)
blocks = pool.get_new_blocks(2)
pool.cache_full_blocks(
request=req,
blocks=blocks,
num_cached_blocks=0,
num_full_blocks=1,
block_size=block_size,
kv_cache_group_id=kv_cache_group_id,
)
partial_hash_10 = boundary_hash(req, hash_block_size, 10)
assert pool.cache_partial_block(
request=req,
block=blocks[1],
num_tokens=10,
kv_cache_group_id=kv_cache_group_id,
block_size=block_size,
)
assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) == [blocks[1]]
req.append_output_token_ids([5, 5])
full_hashes = BlockHashListWithBlockSize(
req.block_hashes, hash_block_size, block_size
)
promoted_full_hash = full_hashes[1]
# The promoted full-block hash is the fine hash at the 12-token boundary,
# not a concatenation of the fine hashes inside the block.
assert promoted_full_hash == req.block_hashes[12 // hash_block_size - 1]
assert promoted_full_hash != BlockHash(
req.block_hashes[3] + req.block_hashes[4] + req.block_hashes[5]
)
pool.cache_full_blocks(
request=req,
blocks=blocks,
num_cached_blocks=1,
num_full_blocks=2,
block_size=block_size,
kv_cache_group_id=kv_cache_group_id,
)
assert pool.get_cached_block(promoted_full_hash, [kv_cache_group_id]) == [blocks[1]]
assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) is None
+1 -1
View File
@@ -225,7 +225,7 @@ def test_kv_cache_block():
# Test block hash setting and resetting
block_hash = make_block_hash_with_group_id(BlockHash(b"abc"), 0)
block.block_hash = block_hash
block.set_block_hash(block_hash)
assert block.block_hash == block_hash
block.reset_hash()
+1 -1
View File
@@ -2003,7 +2003,7 @@ def test_maybe_evict_cached_block():
assert len(pool.blocks) == len(block_hashes)
# Manually add all blocks to cached_blocks
for block, block_hash in zip(pool.blocks, block_hashes):
block.block_hash = block_hash
block.set_block_hash(block_hash)
pool.cached_block_hash_to_block.insert(block_hash, block)
block0, block1, block2, block3 = pool.blocks
+219 -23
View File
@@ -72,6 +72,20 @@ class BlockHashToBlockMap:
self._unexpected_blocks_type(blocks)
return None
def contain(self, key: BlockHashWithGroupId, block_id: int) -> bool:
"""
Checks whether the key maps to the given block ID.
"""
blocks = self._cache.get(key)
if blocks is None:
return False
if isinstance(blocks, KVCacheBlock):
return blocks.block_id == block_id
if isinstance(blocks, dict):
return block_id in blocks
self._unexpected_blocks_type(blocks)
return False
def insert(self, key: BlockHashWithGroupId, block: KVCacheBlock) -> None:
"""
Inserts the KVCacheBlock to the cache
@@ -169,6 +183,7 @@ class BlockPool:
# Cache for block lookup
self.cached_block_hash_to_block: BlockHashToBlockMap = BlockHashToBlockMap()
self.cached_block_hashes_by_block: dict[int, set[BlockHashWithGroupId]] = {}
# To represent a placeholder block with block_id=0.
# The ref_cnt of null_block is not maintained, needs special care to
@@ -245,7 +260,6 @@ class BlockPool:
if num_cached_blocks >= num_full_blocks:
return
new_full_blocks = blocks[num_cached_blocks:num_full_blocks]
assert len(request.block_hashes) >= num_full_blocks
assert block_mask is None or len(block_mask) == len(new_full_blocks)
if block_size == self.hash_block_size:
# Common case.
@@ -254,11 +268,10 @@ class BlockPool:
# block_size is a multiple of hash_block_size. This happens when
# different KV cache groups have different block sizes.
assert block_size % self.hash_block_size == 0
# Recalculate block_hashes at the granularity of block_size, using
# the original block_hashes (at the granularity of hash_block_size).
block_hashes = BlockHashListWithBlockSize(
request.block_hashes, self.hash_block_size, block_size
)
assert len(block_hashes) >= num_full_blocks
new_block_hashes = block_hashes[num_cached_blocks:]
new_hashes: list[ExternalBlockHash] | None = (
@@ -270,15 +283,27 @@ class BlockPool:
# in align mode. We skip null blocks here.
if blk.is_null or (block_mask is not None and not block_mask[i]):
continue
assert blk.block_hash is None
block_hash = new_block_hashes[i]
num_hash_tokens = (num_cached_blocks + i + 1) * block_size
# Update and added the full block to the cache.
block_hash_with_group_id = make_block_hash_with_group_id(
block_hash, kv_cache_group_id
)
blk.block_hash = block_hash_with_group_id
self.cached_block_hash_to_block.insert(block_hash_with_group_id, blk)
if blk.block_hash is not None:
# The only valid case where a "new full block" already has a
# hash is partial->full promotion of the same cache block.
assert (
blk.block_hash_num_tokens is not None
and blk.block_hash_num_tokens < num_hash_tokens
)
removed_hashes = self._remove_cached_block_hashes(blk)
self._emit_block_removed_events(removed_hashes)
self._insert_block_hash(
block_hash_with_group_id,
blk,
num_tokens=num_hash_tokens,
)
if new_hashes is not None:
new_hashes.append(maybe_convert_block_hash(block_hash))
@@ -330,6 +355,190 @@ class BlockPool:
)
)
def cache_partial_block(
self,
request: Request,
block: KVCacheBlock,
num_tokens: int,
kv_cache_group_id: int,
block_size: int,
) -> BlockHashWithGroupId | None:
"""Register a partial prefix-cache entry for an existing block.
Prefix-cache keys normally identify full cache blocks. A partial entry
makes an existing cache block reachable from a fine-grained prefix
boundary inside that block without allocating or copying a new
``KVCacheBlock``.
The partial entry is lookup metadata owned by ``block``. If ``block``
has no primary hash, the key becomes its primary hash. If the block
already has a primary hash, the partial entry is tracked in
``cached_block_hashes_by_block`` so eviction, reset, and promotion can
remove every hash key that points to the block.
Args:
request: Request whose token IDs and block hashes define the
partial entry.
block: Existing cache block to make reachable from the partial
prefix boundary.
num_tokens: Prefix length represented by the partial entry. It
must be a positive multiple of ``self.hash_block_size`` and
cannot exceed the request's computed block hashes.
kv_cache_group_id: KV cache group that owns the partial entry.
block_size: Cache block size for the owning group. The partial
entry hash itself is always the prefix-chain hash at
``num_tokens``; ``block_size`` is used to assert that the
entry is partial within the owning cache block.
Returns:
The hash key with group ID if a partial entry can be registered;
otherwise ``None`` for null blocks.
"""
if block.is_null:
return None
assert block_size > self.hash_block_size
assert block_size % self.hash_block_size == 0
assert num_tokens % block_size != 0
block_hash = self._get_partial_block_hash(request, num_tokens)
num_hash_blocks = num_tokens // self.hash_block_size
block_hash_with_group_id = make_block_hash_with_group_id(
block_hash, kv_cache_group_id
)
already_cached = block.block_hash == block_hash_with_group_id or (
self.cached_block_hash_to_block.contain(
block_hash_with_group_id, block.block_id
)
)
if (
not already_cached
and block.block_hash is not None
and block.block_hash_num_tokens is not None
and block.block_hash_num_tokens < num_hash_blocks * self.hash_block_size
):
removed_hashes = self._remove_cached_block_hashes(block)
self._emit_block_removed_events(removed_hashes)
self._insert_block_hash(
block_hash_with_group_id,
block,
num_tokens=num_hash_blocks * self.hash_block_size,
)
if self.enable_kv_cache_events and not already_cached:
parent_hash, block_start = self._get_partial_block_parent_hash_and_start(
request, num_tokens
)
parent_block_hash = (
maybe_convert_block_hash(parent_hash)
if parent_hash is not None
else None
)
block_end = num_tokens
curr_mm_idx = -1 if block_start > 0 else 0
extra_keys, _ = generate_block_hash_extra_keys(
request, block_start, block_end, curr_mm_idx
)
self.kv_event_queue.append(
BlockStored(
block_hashes=[maybe_convert_block_hash(block_hash)],
parent_block_hash=parent_block_hash,
token_ids=request.all_token_ids[block_start:block_end],
block_size=block_end - block_start,
lora_id=request.lora_request.adapter_id
if request.lora_request
else None,
medium=MEDIUM_GPU,
lora_name=request.lora_request.name
if request.lora_request
else None,
extra_keys=[extra_keys],
group_idx=kv_cache_group_id,
)
)
return block_hash_with_group_id
def _get_partial_block_hash(
self,
request: Request,
num_tokens: int,
) -> BlockHash:
assert num_tokens % self.hash_block_size == 0
num_hash_blocks = num_tokens // self.hash_block_size
assert 0 < num_hash_blocks <= len(request.block_hashes)
# Each hash_block_size hash chains over its full prefix, so the partial
# entry for any group block size is the hash at that prefix boundary.
return request.block_hashes[num_hash_blocks - 1]
def _get_partial_block_parent_hash_and_start(
self,
request: Request,
num_tokens: int,
) -> tuple[BlockHash | None, int]:
num_hash_blocks = num_tokens // self.hash_block_size
parent_hash = (
request.block_hashes[num_hash_blocks - 2] if num_hash_blocks > 1 else None
)
block_start = (num_hash_blocks - 1) * self.hash_block_size
return parent_hash, block_start
def _remove_cached_block_hashes(
self,
block: KVCacheBlock,
) -> list[BlockHashWithGroupId]:
block_hashes: list[BlockHashWithGroupId] = []
if block.block_hash is not None:
block_hashes.append(block.block_hash)
block_hashes.extend(self.cached_block_hashes_by_block.pop(block.block_id, ()))
if not block_hashes:
return []
removed_hashes: list[BlockHashWithGroupId] = []
for block_hash in block_hashes:
if (
self.cached_block_hash_to_block.pop(block_hash, block.block_id)
is not None
):
removed_hashes.append(block_hash)
block.reset_hash()
return removed_hashes
def _emit_block_removed_events(
self,
block_hashes: list[BlockHashWithGroupId],
) -> None:
if not self.enable_kv_cache_events:
return
for block_hash in block_hashes:
self.kv_event_queue.append(
BlockRemoved(
block_hashes=[maybe_convert_block_hash(get_block_hash(block_hash))],
medium=MEDIUM_GPU,
group_idx=get_group_id(block_hash),
)
)
def _insert_block_hash(
self,
block_hash_with_group_id: BlockHashWithGroupId,
block: KVCacheBlock,
num_tokens: int | None,
) -> None:
if block.block_hash == block_hash_with_group_id:
return
if self.cached_block_hash_to_block.contain(
block_hash_with_group_id, block.block_id
):
return
if block.block_hash is None:
block.set_block_hash(block_hash_with_group_id, num_tokens=num_tokens)
else:
self.cached_block_hashes_by_block.setdefault(block.block_id, set()).add(
block_hash_with_group_id
)
self.cached_block_hash_to_block.insert(block_hash_with_group_id, block)
def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]:
"""Get new blocks from the free block pool.
@@ -377,26 +586,12 @@ class BlockPool:
if self.metrics_collector:
self.metrics_collector.on_block_evicted(block)
block_hash = block.block_hash
if block_hash is None:
evicted_hashes = self._remove_cached_block_hashes(block)
if not evicted_hashes:
# The block doesn't have hash, eviction is not needed
return False
if self.cached_block_hash_to_block.pop(block_hash, block.block_id) is None:
# block not found in cached_block_hash_to_block,
# eviction is not needed
return False
block.reset_hash()
if self.enable_kv_cache_events:
self.kv_event_queue.append(
BlockRemoved(
block_hashes=[maybe_convert_block_hash(get_block_hash(block_hash))],
medium=MEDIUM_GPU,
group_idx=get_group_id(block_hash),
)
)
self._emit_block_removed_events(evicted_hashes)
return True
def touch(self, blocks: Sequence[KVCacheBlock]) -> None:
@@ -478,6 +673,7 @@ class BlockPool:
# Remove all hashes so that no new blocks will hit.
self.cached_block_hash_to_block = BlockHashToBlockMap()
self.cached_block_hashes_by_block.clear()
# Remove all hashes from all blocks.
for block in self.blocks:
+38 -16
View File
@@ -125,6 +125,9 @@ class KVCacheBlock:
# The hash key (block hash + group id) of the block, only available
# when the block is full and cached.
_block_hash: BlockHashWithGroupId | None = None
# Number of prefix tokens covered by _block_hash. For full blocks this is
# the full block boundary; partial aliases can end inside a cache block.
_block_hash_num_tokens: int | None = None
# Used to construct a doubly linked list for free blocks.
# These two attributes should only be manipulated by FreeKVCacheBlockQueue.
@@ -138,16 +141,25 @@ class KVCacheBlock:
def block_hash(self) -> BlockHashWithGroupId | None:
return self._block_hash
@block_hash.setter
def block_hash(self, block_hash: BlockHashWithGroupId):
assert self.block_hash is None, (
@property
def block_hash_num_tokens(self) -> int | None:
return self._block_hash_num_tokens
def set_block_hash(
self,
block_hash: BlockHashWithGroupId,
num_tokens: int | None = None,
) -> None:
assert self.block_hash is None and self._block_hash_num_tokens is None, (
"The block already has a hash. This should not happen."
)
self._block_hash = block_hash
self._block_hash_num_tokens = num_tokens
def reset_hash(self):
"""Reset the block hash when the block is evicted."""
self._block_hash = None
self._block_hash_num_tokens = None
def __repr__(self) -> str:
# Use block_id instead of KVCacheBlock object to avoid calling __repr__
@@ -158,6 +170,7 @@ class KVCacheBlock:
f"KVCacheBlock(block_id={self.block_id}, "
f"ref_cnt={self.ref_cnt}, "
f"_block_hash={self._block_hash!r}, "
f"_block_hash_num_tokens={self._block_hash_num_tokens}, "
f"prev_free_block={prev_block_id}, "
f"next_free_block={next_block_id})"
)
@@ -658,18 +671,24 @@ def resolve_kv_cache_block_sizes(
def get_request_block_hasher(
block_size: int,
hash_block_size: int,
caching_hash_fn: Callable[[Any], bytes],
) -> Callable[[Request], list[BlockHash]]:
"""
Returns a function which computes the list of un-computed block hashes
of a request."""
of a request.
Hashes are computed at ``hash_block_size`` granularity and chained over the
full prefix, so each hash uniquely fingerprints the prefix ending at its
boundary. Coarser group block sizes and partial-cache boundaries reuse
these hashes directly (see ``BlockHashListWithBlockSize``).
"""
def request_block_hasher(request: Request) -> list[BlockHash]:
start_token_idx = len(request.block_hashes) * block_size
start_token_idx = len(request.block_hashes) * hash_block_size
num_tokens = request.num_tokens
if start_token_idx + block_size > num_tokens:
if start_token_idx + hash_block_size > num_tokens:
# Early stop when there no new full blocks created.
return []
@@ -686,7 +705,7 @@ def get_request_block_hasher(
)
new_block_hashes: list[BlockHash] = []
while True:
end_token_idx = start_token_idx + block_size
end_token_idx = start_token_idx + hash_block_size
if end_token_idx > num_tokens:
# We only hash full blocks
break
@@ -703,7 +722,7 @@ def get_request_block_hasher(
)
new_block_hashes.append(block_hash)
start_token_idx += block_size
start_token_idx += hash_block_size
prev_block_hash_value = block_hash
return new_block_hashes
@@ -2132,11 +2151,14 @@ class BlockHashListWithBlockSize:
Currently, only scaling up by an integer factor is supported (i.e.,
`target_block_size` is a multiple of `hash_block_size`). Conversion is
performed lazily on access for efficiency, by concatenating consecutive
hashes at `hash_block_size` to form each hash at `target_block_size`.
performed lazily on access for efficiency. Each `hash_block_size` hash is
already chained over its entire prefix, so the hash at the last
`hash_block_size` boundary of a `target_block_size` block uniquely
fingerprints that block's prefix; we use it directly.
Example (`hash_block_size` = 16, `target_block_size` = 32):
concatenating two 16-size hashes yields one 32-size hash:
the second 16-size hash already covers tokens 0-31, so it is the 32-size
hash:
Block hashes with block_size 16:
| Token Range | 0-15 | 16-31 | 32-47 | 48-63 |
@@ -2146,7 +2168,7 @@ class BlockHashListWithBlockSize:
Block hashes with block_size 32:
| Token Range | 0-31 | 32-63 |
|-------------|------|-------|
| Hash | AB | CD |
| Hash | B | D |
Args:
block_hashes: Block hashes to convert, computed at `hash_block_size`.
@@ -2188,9 +2210,9 @@ class BlockHashListWithBlockSize:
yield self._get_value_at(i)
def _get_value_at(self, idx: int) -> BlockHash:
base = idx * self.scale_factor
end = base + self.scale_factor
return BlockHash(b"".join(self.block_hashes[base:end]))
# The last hash_block_size hash within the target block already chains
# over the whole prefix, so it is the target block's hash.
return self.block_hashes[(idx + 1) * self.scale_factor - 1]
BlockHashList = list[BlockHash] | BlockHashListWithBlockSize