[1/N][Core] add partial prefix cache primitives (#45939)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Yifan Qiao
parent
1eb2cc961e
commit
6bc6f2d86d
@@ -0,0 +1,460 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.v1.core.kv_cache_utils as kv_cache_utils
|
||||
from vllm.distributed.kv_events import BlockRemoved, BlockStored
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
BlockHashListWithBlockSize,
|
||||
KVCacheBlock,
|
||||
get_request_block_hasher,
|
||||
hash_block_tokens,
|
||||
init_none_hash,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _auto_init_hash_fn():
|
||||
init_none_hash(sha256)
|
||||
|
||||
|
||||
def make_request(
|
||||
request_id: str,
|
||||
prompt_token_ids: list[int],
|
||||
hash_block_size: int,
|
||||
hash_fn: Callable,
|
||||
) -> Request:
|
||||
sampling_params = SamplingParams(max_tokens=17)
|
||||
sampling_params.update_from_generation_config({}, eos_token_id=100)
|
||||
return Request(
|
||||
request_id=request_id,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
block_hasher=get_request_block_hasher(hash_block_size, hash_fn),
|
||||
)
|
||||
|
||||
|
||||
def boundary_hash(req: Request, hash_block_size: int, num_tokens: int) -> BlockHash:
|
||||
# Every boundary at a hash_block_size multiple is just the fine-grained
|
||||
# chain hash ending there.
|
||||
return req.block_hashes[num_tokens // hash_block_size - 1]
|
||||
|
||||
|
||||
def cache_full_block_and_partial_tail(
|
||||
token_ids: list[int],
|
||||
*,
|
||||
enable_kv_cache_events: bool = False,
|
||||
) -> tuple[BlockPool, Request, list[KVCacheBlock], BlockHash]:
|
||||
hash_block_size = 2
|
||||
block_size = 6
|
||||
kv_cache_group_id = 0
|
||||
req = make_request("0", token_ids, hash_block_size, sha256)
|
||||
pool = BlockPool(
|
||||
num_gpu_blocks=3,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
enable_kv_cache_events=enable_kv_cache_events,
|
||||
)
|
||||
blocks = pool.get_new_blocks(2)
|
||||
|
||||
pool.cache_full_blocks(
|
||||
request=req,
|
||||
blocks=blocks,
|
||||
num_cached_blocks=0,
|
||||
num_full_blocks=1,
|
||||
block_size=block_size,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
)
|
||||
partial_hash = boundary_hash(req, hash_block_size, len(token_ids))
|
||||
assert pool.cache_partial_block(
|
||||
request=req,
|
||||
block=blocks[1],
|
||||
num_tokens=len(token_ids),
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
block_size=block_size,
|
||||
)
|
||||
return pool, req, blocks, partial_hash
|
||||
|
||||
|
||||
def test_boundary_hashes_reuse_fine_grained_chain():
|
||||
hash_block_size = 2
|
||||
block_size = 6
|
||||
token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
|
||||
req = make_request("0", token_ids, hash_block_size, sha256)
|
||||
|
||||
coarse = BlockHashListWithBlockSize(req.block_hashes, hash_block_size, block_size)
|
||||
# The block_size=6 full-block hash is the fine hash at the 6-token boundary,
|
||||
# not a concatenation of the three fine hashes inside the block.
|
||||
assert coarse[0] == req.block_hashes[6 // hash_block_size - 1]
|
||||
assert coarse[0] != BlockHash(
|
||||
req.block_hashes[0] + req.block_hashes[1] + req.block_hashes[2]
|
||||
)
|
||||
# A partial tail at 10 tokens is the fine hash at the 10-token boundary,
|
||||
# which chains over the entire prefix.
|
||||
tail_hash = boundary_hash(req, hash_block_size, 10)
|
||||
assert tail_hash == req.block_hashes[4]
|
||||
assert tail_hash == hash_block_tokens(sha256, req.block_hashes[3], token_ids[8:10])
|
||||
|
||||
|
||||
def test_cache_partial_block_kv_cache_events():
|
||||
hash_block_size = 4
|
||||
block_size = 12
|
||||
kv_cache_group_id = 2
|
||||
|
||||
pool = BlockPool(
|
||||
num_gpu_blocks=2,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
enable_kv_cache_events=True,
|
||||
)
|
||||
req = make_request(
|
||||
"req_partial_events",
|
||||
prompt_token_ids=list(range(hash_block_size * 2)),
|
||||
hash_block_size=hash_block_size,
|
||||
hash_fn=sha256,
|
||||
)
|
||||
|
||||
block = pool.get_new_blocks(1)[0]
|
||||
partial_entry_hash = pool.cache_partial_block(
|
||||
request=req,
|
||||
block=block,
|
||||
num_tokens=hash_block_size * 2,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
block_size=block_size,
|
||||
)
|
||||
|
||||
events = pool.take_events()
|
||||
assert len(events) == 1
|
||||
stored_event = events[0]
|
||||
assert isinstance(stored_event, BlockStored)
|
||||
assert partial_entry_hash is not None
|
||||
assert stored_event.block_hashes == [
|
||||
kv_cache_utils.maybe_convert_block_hash(req.block_hashes[1])
|
||||
]
|
||||
assert stored_event.parent_block_hash == kv_cache_utils.maybe_convert_block_hash(
|
||||
req.block_hashes[0]
|
||||
)
|
||||
assert stored_event.token_ids == req.all_token_ids[hash_block_size:]
|
||||
assert stored_event.block_size == 4
|
||||
assert stored_event.group_idx == kv_cache_group_id
|
||||
|
||||
duplicate_entry_hash = pool.cache_partial_block(
|
||||
request=req,
|
||||
block=block,
|
||||
num_tokens=hash_block_size * 2,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
block_size=block_size,
|
||||
)
|
||||
assert duplicate_entry_hash == partial_entry_hash
|
||||
assert pool.take_events() == []
|
||||
|
||||
pool.free_blocks([block])
|
||||
pool.get_new_blocks(1)
|
||||
events = pool.take_events()
|
||||
assert len(events) == 1
|
||||
removed_event = events[0]
|
||||
assert isinstance(removed_event, BlockRemoved)
|
||||
assert removed_event.block_hashes == stored_event.block_hashes
|
||||
assert removed_event.group_idx == kv_cache_group_id
|
||||
|
||||
|
||||
def test_partial_block_replacement_emits_remove_then_store_events():
|
||||
hash_block_size = 2
|
||||
block_size = 6
|
||||
kv_cache_group_id = 0
|
||||
req = make_request("0", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
|
||||
pool = BlockPool(
|
||||
num_gpu_blocks=3,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
enable_kv_cache_events=True,
|
||||
)
|
||||
blocks = pool.get_new_blocks(2)
|
||||
|
||||
pool.cache_full_blocks(
|
||||
request=req,
|
||||
blocks=blocks,
|
||||
num_cached_blocks=0,
|
||||
num_full_blocks=1,
|
||||
block_size=block_size,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
)
|
||||
partial_hash_8 = boundary_hash(req, hash_block_size, 8)
|
||||
assert pool.cache_partial_block(
|
||||
request=req,
|
||||
block=blocks[1],
|
||||
num_tokens=8,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
block_size=block_size,
|
||||
)
|
||||
assert pool.get_cached_block(partial_hash_8, [kv_cache_group_id]) == [blocks[1]]
|
||||
pool.take_events()
|
||||
|
||||
req.append_output_token_ids([4, 4])
|
||||
partial_hash_10 = boundary_hash(req, hash_block_size, 10)
|
||||
assert pool.cache_partial_block(
|
||||
request=req,
|
||||
block=blocks[1],
|
||||
num_tokens=10,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
block_size=block_size,
|
||||
)
|
||||
events = pool.take_events()
|
||||
|
||||
assert len(events) == 2
|
||||
removed_event, stored_event = events
|
||||
assert isinstance(removed_event, BlockRemoved)
|
||||
assert removed_event.block_hashes == [
|
||||
kv_cache_utils.maybe_convert_block_hash(partial_hash_8)
|
||||
]
|
||||
assert removed_event.group_idx == kv_cache_group_id
|
||||
assert isinstance(stored_event, BlockStored)
|
||||
assert stored_event.block_hashes == [
|
||||
kv_cache_utils.maybe_convert_block_hash(partial_hash_10)
|
||||
]
|
||||
assert stored_event.parent_block_hash == kv_cache_utils.maybe_convert_block_hash(
|
||||
boundary_hash(req, hash_block_size, 8)
|
||||
)
|
||||
assert stored_event.token_ids == req.all_token_ids[8:10]
|
||||
assert stored_event.block_size == hash_block_size
|
||||
assert stored_event.group_idx == kv_cache_group_id
|
||||
assert pool.get_cached_block(partial_hash_8, [kv_cache_group_id]) is None
|
||||
assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) == [blocks[1]]
|
||||
|
||||
|
||||
def test_later_request_hits_cached_partial_tail():
|
||||
hash_block_size = 2
|
||||
block_size = 6
|
||||
kv_cache_group_id = 0
|
||||
cached_token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
|
||||
req = make_request("0", cached_token_ids, hash_block_size, sha256)
|
||||
pool = BlockPool(
|
||||
num_gpu_blocks=3,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
blocks = pool.get_new_blocks(2)
|
||||
|
||||
pool.cache_full_blocks(
|
||||
request=req,
|
||||
blocks=blocks,
|
||||
num_cached_blocks=0,
|
||||
num_full_blocks=1,
|
||||
block_size=block_size,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
)
|
||||
partial_hash_10 = boundary_hash(req, hash_block_size, 10)
|
||||
assert pool.cache_partial_block(
|
||||
request=req,
|
||||
block=blocks[1],
|
||||
num_tokens=10,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
block_size=block_size,
|
||||
)
|
||||
|
||||
replay = make_request("1", cached_token_ids, hash_block_size, sha256)
|
||||
replay_hash_10 = boundary_hash(replay, hash_block_size, 10)
|
||||
assert replay_hash_10 == partial_hash_10
|
||||
assert pool.get_cached_block(replay_hash_10, [kv_cache_group_id]) == [blocks[1]]
|
||||
|
||||
extended = make_request("2", cached_token_ids + [10], hash_block_size, sha256)
|
||||
extended_hash_10 = boundary_hash(extended, hash_block_size, 10)
|
||||
assert extended_hash_10 == partial_hash_10
|
||||
assert pool.get_cached_block(extended_hash_10, [kv_cache_group_id]) == [blocks[1]]
|
||||
|
||||
|
||||
def test_cache_partial_block_uses_fine_grained_boundary_hash():
|
||||
hash_block_size = 2
|
||||
block_size = 6
|
||||
kv_cache_group_id = 0
|
||||
token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
|
||||
req = make_request("0", token_ids, hash_block_size, sha256)
|
||||
pool = BlockPool(
|
||||
num_gpu_blocks=3,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
blocks = pool.get_new_blocks(2)
|
||||
|
||||
pool.cache_full_blocks(
|
||||
request=req,
|
||||
blocks=blocks,
|
||||
num_cached_blocks=0,
|
||||
num_full_blocks=1,
|
||||
block_size=block_size,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
)
|
||||
|
||||
partial_entry_hash = pool.cache_partial_block(
|
||||
request=req,
|
||||
block=blocks[1],
|
||||
num_tokens=10,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
block_size=block_size,
|
||||
)
|
||||
# The partial entry is keyed by the fine-grained hash at the 10-token
|
||||
# boundary, regardless of the owning group's block_size.
|
||||
expected = boundary_hash(req, hash_block_size, 10)
|
||||
assert partial_entry_hash == kv_cache_utils.make_block_hash_with_group_id(
|
||||
expected, kv_cache_group_id
|
||||
)
|
||||
assert pool.get_cached_block(expected, [kv_cache_group_id]) == [blocks[1]]
|
||||
|
||||
|
||||
def test_cache_partial_block_requires_hash_boundary():
|
||||
hash_block_size = 2
|
||||
block_size = 4
|
||||
req = make_request("0", [0, 0, 1, 1], hash_block_size, sha256)
|
||||
pool = BlockPool(
|
||||
num_gpu_blocks=2,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
block = pool.get_new_blocks(1)[0]
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
pool.cache_partial_block(
|
||||
request=req,
|
||||
block=block,
|
||||
num_tokens=3,
|
||||
kv_cache_group_id=0,
|
||||
block_size=block_size,
|
||||
)
|
||||
|
||||
|
||||
def test_cache_partial_block_duplicate_checks_all_blocks_for_hash():
|
||||
hash_block_size = 2
|
||||
block_size = 4
|
||||
kv_cache_group_id = 0
|
||||
req = make_request("0", [0, 0, 1, 1], hash_block_size, sha256)
|
||||
pool = BlockPool(
|
||||
num_gpu_blocks=4,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
blocks = pool.get_new_blocks(2)
|
||||
|
||||
first_entry_hash = pool.cache_partial_block(
|
||||
request=req,
|
||||
block=blocks[0],
|
||||
num_tokens=2,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
block_size=block_size,
|
||||
)
|
||||
second_entry_hash = pool.cache_partial_block(
|
||||
request=req,
|
||||
block=blocks[1],
|
||||
num_tokens=2,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
block_size=block_size,
|
||||
)
|
||||
assert first_entry_hash == second_entry_hash
|
||||
|
||||
duplicate_entry_hash = pool.cache_partial_block(
|
||||
request=req,
|
||||
block=blocks[1],
|
||||
num_tokens=2,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
block_size=block_size,
|
||||
)
|
||||
assert duplicate_entry_hash == second_entry_hash
|
||||
assert pool.cached_block_hashes_by_block == {}
|
||||
|
||||
|
||||
def test_reset_prefix_cache_clears_partial_entry_metadata():
|
||||
pool, req, blocks, partial_hash_10 = cache_full_block_and_partial_tail(
|
||||
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
|
||||
)
|
||||
full_hash = BlockHashListWithBlockSize(req.block_hashes, 2, 6)[0]
|
||||
|
||||
assert pool.get_cached_block(full_hash, [0]) == [blocks[0]]
|
||||
assert pool.get_cached_block(partial_hash_10, [0]) == [blocks[1]]
|
||||
|
||||
pool.free_blocks(blocks)
|
||||
assert pool.reset_prefix_cache()
|
||||
|
||||
assert pool.get_cached_block(full_hash, [0]) is None
|
||||
assert pool.get_cached_block(partial_hash_10, [0]) is None
|
||||
assert pool.cached_block_hashes_by_block == {}
|
||||
|
||||
|
||||
def test_evict_cached_block_removes_full_hash_and_partial_entry():
|
||||
pool, req, blocks, partial_hash_10 = cache_full_block_and_partial_tail(
|
||||
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
|
||||
)
|
||||
full_hash = BlockHashListWithBlockSize(req.block_hashes, 2, 6)[0]
|
||||
|
||||
assert pool.get_cached_block(full_hash, [0]) == [blocks[0]]
|
||||
assert pool.get_cached_block(partial_hash_10, [0]) == [blocks[1]]
|
||||
|
||||
pool.evict_blocks({blocks[0].block_id, blocks[1].block_id})
|
||||
|
||||
assert pool.get_cached_block(full_hash, [0]) is None
|
||||
assert pool.get_cached_block(partial_hash_10, [0]) is None
|
||||
assert pool.cached_block_hashes_by_block == {}
|
||||
|
||||
|
||||
def test_partial_block_promotes_to_direct_full_block_hash():
|
||||
hash_block_size = 2
|
||||
block_size = 6
|
||||
kv_cache_group_id = 0
|
||||
token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
|
||||
req = make_request("0", token_ids, hash_block_size, sha256)
|
||||
pool = BlockPool(
|
||||
num_gpu_blocks=3,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
blocks = pool.get_new_blocks(2)
|
||||
|
||||
pool.cache_full_blocks(
|
||||
request=req,
|
||||
blocks=blocks,
|
||||
num_cached_blocks=0,
|
||||
num_full_blocks=1,
|
||||
block_size=block_size,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
)
|
||||
partial_hash_10 = boundary_hash(req, hash_block_size, 10)
|
||||
assert pool.cache_partial_block(
|
||||
request=req,
|
||||
block=blocks[1],
|
||||
num_tokens=10,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
block_size=block_size,
|
||||
)
|
||||
assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) == [blocks[1]]
|
||||
|
||||
req.append_output_token_ids([5, 5])
|
||||
full_hashes = BlockHashListWithBlockSize(
|
||||
req.block_hashes, hash_block_size, block_size
|
||||
)
|
||||
promoted_full_hash = full_hashes[1]
|
||||
# The promoted full-block hash is the fine hash at the 12-token boundary,
|
||||
# not a concatenation of the fine hashes inside the block.
|
||||
assert promoted_full_hash == req.block_hashes[12 // hash_block_size - 1]
|
||||
assert promoted_full_hash != BlockHash(
|
||||
req.block_hashes[3] + req.block_hashes[4] + req.block_hashes[5]
|
||||
)
|
||||
|
||||
pool.cache_full_blocks(
|
||||
request=req,
|
||||
blocks=blocks,
|
||||
num_cached_blocks=1,
|
||||
num_full_blocks=2,
|
||||
block_size=block_size,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
)
|
||||
assert pool.get_cached_block(promoted_full_hash, [kv_cache_group_id]) == [blocks[1]]
|
||||
assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) is None
|
||||
@@ -225,7 +225,7 @@ def test_kv_cache_block():
|
||||
|
||||
# Test block hash setting and resetting
|
||||
block_hash = make_block_hash_with_group_id(BlockHash(b"abc"), 0)
|
||||
block.block_hash = block_hash
|
||||
block.set_block_hash(block_hash)
|
||||
assert block.block_hash == block_hash
|
||||
|
||||
block.reset_hash()
|
||||
|
||||
@@ -2003,7 +2003,7 @@ def test_maybe_evict_cached_block():
|
||||
assert len(pool.blocks) == len(block_hashes)
|
||||
# Manually add all blocks to cached_blocks
|
||||
for block, block_hash in zip(pool.blocks, block_hashes):
|
||||
block.block_hash = block_hash
|
||||
block.set_block_hash(block_hash)
|
||||
pool.cached_block_hash_to_block.insert(block_hash, block)
|
||||
|
||||
block0, block1, block2, block3 = pool.blocks
|
||||
|
||||
+219
-23
@@ -72,6 +72,20 @@ class BlockHashToBlockMap:
|
||||
self._unexpected_blocks_type(blocks)
|
||||
return None
|
||||
|
||||
def contain(self, key: BlockHashWithGroupId, block_id: int) -> bool:
|
||||
"""
|
||||
Checks whether the key maps to the given block ID.
|
||||
"""
|
||||
blocks = self._cache.get(key)
|
||||
if blocks is None:
|
||||
return False
|
||||
if isinstance(blocks, KVCacheBlock):
|
||||
return blocks.block_id == block_id
|
||||
if isinstance(blocks, dict):
|
||||
return block_id in blocks
|
||||
self._unexpected_blocks_type(blocks)
|
||||
return False
|
||||
|
||||
def insert(self, key: BlockHashWithGroupId, block: KVCacheBlock) -> None:
|
||||
"""
|
||||
Inserts the KVCacheBlock to the cache
|
||||
@@ -169,6 +183,7 @@ class BlockPool:
|
||||
|
||||
# Cache for block lookup
|
||||
self.cached_block_hash_to_block: BlockHashToBlockMap = BlockHashToBlockMap()
|
||||
self.cached_block_hashes_by_block: dict[int, set[BlockHashWithGroupId]] = {}
|
||||
|
||||
# To represent a placeholder block with block_id=0.
|
||||
# The ref_cnt of null_block is not maintained, needs special care to
|
||||
@@ -245,7 +260,6 @@ class BlockPool:
|
||||
if num_cached_blocks >= num_full_blocks:
|
||||
return
|
||||
new_full_blocks = blocks[num_cached_blocks:num_full_blocks]
|
||||
assert len(request.block_hashes) >= num_full_blocks
|
||||
assert block_mask is None or len(block_mask) == len(new_full_blocks)
|
||||
if block_size == self.hash_block_size:
|
||||
# Common case.
|
||||
@@ -254,11 +268,10 @@ class BlockPool:
|
||||
# block_size is a multiple of hash_block_size. This happens when
|
||||
# different KV cache groups have different block sizes.
|
||||
assert block_size % self.hash_block_size == 0
|
||||
# Recalculate block_hashes at the granularity of block_size, using
|
||||
# the original block_hashes (at the granularity of hash_block_size).
|
||||
block_hashes = BlockHashListWithBlockSize(
|
||||
request.block_hashes, self.hash_block_size, block_size
|
||||
)
|
||||
assert len(block_hashes) >= num_full_blocks
|
||||
|
||||
new_block_hashes = block_hashes[num_cached_blocks:]
|
||||
new_hashes: list[ExternalBlockHash] | None = (
|
||||
@@ -270,15 +283,27 @@ class BlockPool:
|
||||
# in align mode. We skip null blocks here.
|
||||
if blk.is_null or (block_mask is not None and not block_mask[i]):
|
||||
continue
|
||||
assert blk.block_hash is None
|
||||
block_hash = new_block_hashes[i]
|
||||
num_hash_tokens = (num_cached_blocks + i + 1) * block_size
|
||||
|
||||
# Update and added the full block to the cache.
|
||||
block_hash_with_group_id = make_block_hash_with_group_id(
|
||||
block_hash, kv_cache_group_id
|
||||
)
|
||||
blk.block_hash = block_hash_with_group_id
|
||||
self.cached_block_hash_to_block.insert(block_hash_with_group_id, blk)
|
||||
if blk.block_hash is not None:
|
||||
# The only valid case where a "new full block" already has a
|
||||
# hash is partial->full promotion of the same cache block.
|
||||
assert (
|
||||
blk.block_hash_num_tokens is not None
|
||||
and blk.block_hash_num_tokens < num_hash_tokens
|
||||
)
|
||||
removed_hashes = self._remove_cached_block_hashes(blk)
|
||||
self._emit_block_removed_events(removed_hashes)
|
||||
self._insert_block_hash(
|
||||
block_hash_with_group_id,
|
||||
blk,
|
||||
num_tokens=num_hash_tokens,
|
||||
)
|
||||
if new_hashes is not None:
|
||||
new_hashes.append(maybe_convert_block_hash(block_hash))
|
||||
|
||||
@@ -330,6 +355,190 @@ class BlockPool:
|
||||
)
|
||||
)
|
||||
|
||||
def cache_partial_block(
|
||||
self,
|
||||
request: Request,
|
||||
block: KVCacheBlock,
|
||||
num_tokens: int,
|
||||
kv_cache_group_id: int,
|
||||
block_size: int,
|
||||
) -> BlockHashWithGroupId | None:
|
||||
"""Register a partial prefix-cache entry for an existing block.
|
||||
|
||||
Prefix-cache keys normally identify full cache blocks. A partial entry
|
||||
makes an existing cache block reachable from a fine-grained prefix
|
||||
boundary inside that block without allocating or copying a new
|
||||
``KVCacheBlock``.
|
||||
|
||||
The partial entry is lookup metadata owned by ``block``. If ``block``
|
||||
has no primary hash, the key becomes its primary hash. If the block
|
||||
already has a primary hash, the partial entry is tracked in
|
||||
``cached_block_hashes_by_block`` so eviction, reset, and promotion can
|
||||
remove every hash key that points to the block.
|
||||
|
||||
Args:
|
||||
request: Request whose token IDs and block hashes define the
|
||||
partial entry.
|
||||
block: Existing cache block to make reachable from the partial
|
||||
prefix boundary.
|
||||
num_tokens: Prefix length represented by the partial entry. It
|
||||
must be a positive multiple of ``self.hash_block_size`` and
|
||||
cannot exceed the request's computed block hashes.
|
||||
kv_cache_group_id: KV cache group that owns the partial entry.
|
||||
block_size: Cache block size for the owning group. The partial
|
||||
entry hash itself is always the prefix-chain hash at
|
||||
``num_tokens``; ``block_size`` is used to assert that the
|
||||
entry is partial within the owning cache block.
|
||||
|
||||
Returns:
|
||||
The hash key with group ID if a partial entry can be registered;
|
||||
otherwise ``None`` for null blocks.
|
||||
"""
|
||||
if block.is_null:
|
||||
return None
|
||||
|
||||
assert block_size > self.hash_block_size
|
||||
assert block_size % self.hash_block_size == 0
|
||||
assert num_tokens % block_size != 0
|
||||
block_hash = self._get_partial_block_hash(request, num_tokens)
|
||||
num_hash_blocks = num_tokens // self.hash_block_size
|
||||
block_hash_with_group_id = make_block_hash_with_group_id(
|
||||
block_hash, kv_cache_group_id
|
||||
)
|
||||
already_cached = block.block_hash == block_hash_with_group_id or (
|
||||
self.cached_block_hash_to_block.contain(
|
||||
block_hash_with_group_id, block.block_id
|
||||
)
|
||||
)
|
||||
if (
|
||||
not already_cached
|
||||
and block.block_hash is not None
|
||||
and block.block_hash_num_tokens is not None
|
||||
and block.block_hash_num_tokens < num_hash_blocks * self.hash_block_size
|
||||
):
|
||||
removed_hashes = self._remove_cached_block_hashes(block)
|
||||
self._emit_block_removed_events(removed_hashes)
|
||||
self._insert_block_hash(
|
||||
block_hash_with_group_id,
|
||||
block,
|
||||
num_tokens=num_hash_blocks * self.hash_block_size,
|
||||
)
|
||||
if self.enable_kv_cache_events and not already_cached:
|
||||
parent_hash, block_start = self._get_partial_block_parent_hash_and_start(
|
||||
request, num_tokens
|
||||
)
|
||||
parent_block_hash = (
|
||||
maybe_convert_block_hash(parent_hash)
|
||||
if parent_hash is not None
|
||||
else None
|
||||
)
|
||||
block_end = num_tokens
|
||||
curr_mm_idx = -1 if block_start > 0 else 0
|
||||
extra_keys, _ = generate_block_hash_extra_keys(
|
||||
request, block_start, block_end, curr_mm_idx
|
||||
)
|
||||
self.kv_event_queue.append(
|
||||
BlockStored(
|
||||
block_hashes=[maybe_convert_block_hash(block_hash)],
|
||||
parent_block_hash=parent_block_hash,
|
||||
token_ids=request.all_token_ids[block_start:block_end],
|
||||
block_size=block_end - block_start,
|
||||
lora_id=request.lora_request.adapter_id
|
||||
if request.lora_request
|
||||
else None,
|
||||
medium=MEDIUM_GPU,
|
||||
lora_name=request.lora_request.name
|
||||
if request.lora_request
|
||||
else None,
|
||||
extra_keys=[extra_keys],
|
||||
group_idx=kv_cache_group_id,
|
||||
)
|
||||
)
|
||||
return block_hash_with_group_id
|
||||
|
||||
def _get_partial_block_hash(
|
||||
self,
|
||||
request: Request,
|
||||
num_tokens: int,
|
||||
) -> BlockHash:
|
||||
assert num_tokens % self.hash_block_size == 0
|
||||
num_hash_blocks = num_tokens // self.hash_block_size
|
||||
assert 0 < num_hash_blocks <= len(request.block_hashes)
|
||||
|
||||
# Each hash_block_size hash chains over its full prefix, so the partial
|
||||
# entry for any group block size is the hash at that prefix boundary.
|
||||
return request.block_hashes[num_hash_blocks - 1]
|
||||
|
||||
def _get_partial_block_parent_hash_and_start(
|
||||
self,
|
||||
request: Request,
|
||||
num_tokens: int,
|
||||
) -> tuple[BlockHash | None, int]:
|
||||
num_hash_blocks = num_tokens // self.hash_block_size
|
||||
parent_hash = (
|
||||
request.block_hashes[num_hash_blocks - 2] if num_hash_blocks > 1 else None
|
||||
)
|
||||
block_start = (num_hash_blocks - 1) * self.hash_block_size
|
||||
return parent_hash, block_start
|
||||
|
||||
def _remove_cached_block_hashes(
|
||||
self,
|
||||
block: KVCacheBlock,
|
||||
) -> list[BlockHashWithGroupId]:
|
||||
block_hashes: list[BlockHashWithGroupId] = []
|
||||
if block.block_hash is not None:
|
||||
block_hashes.append(block.block_hash)
|
||||
block_hashes.extend(self.cached_block_hashes_by_block.pop(block.block_id, ()))
|
||||
if not block_hashes:
|
||||
return []
|
||||
|
||||
removed_hashes: list[BlockHashWithGroupId] = []
|
||||
for block_hash in block_hashes:
|
||||
if (
|
||||
self.cached_block_hash_to_block.pop(block_hash, block.block_id)
|
||||
is not None
|
||||
):
|
||||
removed_hashes.append(block_hash)
|
||||
block.reset_hash()
|
||||
return removed_hashes
|
||||
|
||||
def _emit_block_removed_events(
|
||||
self,
|
||||
block_hashes: list[BlockHashWithGroupId],
|
||||
) -> None:
|
||||
if not self.enable_kv_cache_events:
|
||||
return
|
||||
for block_hash in block_hashes:
|
||||
self.kv_event_queue.append(
|
||||
BlockRemoved(
|
||||
block_hashes=[maybe_convert_block_hash(get_block_hash(block_hash))],
|
||||
medium=MEDIUM_GPU,
|
||||
group_idx=get_group_id(block_hash),
|
||||
)
|
||||
)
|
||||
|
||||
def _insert_block_hash(
|
||||
self,
|
||||
block_hash_with_group_id: BlockHashWithGroupId,
|
||||
block: KVCacheBlock,
|
||||
num_tokens: int | None,
|
||||
) -> None:
|
||||
if block.block_hash == block_hash_with_group_id:
|
||||
return
|
||||
|
||||
if self.cached_block_hash_to_block.contain(
|
||||
block_hash_with_group_id, block.block_id
|
||||
):
|
||||
return
|
||||
|
||||
if block.block_hash is None:
|
||||
block.set_block_hash(block_hash_with_group_id, num_tokens=num_tokens)
|
||||
else:
|
||||
self.cached_block_hashes_by_block.setdefault(block.block_id, set()).add(
|
||||
block_hash_with_group_id
|
||||
)
|
||||
self.cached_block_hash_to_block.insert(block_hash_with_group_id, block)
|
||||
|
||||
def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]:
|
||||
"""Get new blocks from the free block pool.
|
||||
|
||||
@@ -377,26 +586,12 @@ class BlockPool:
|
||||
if self.metrics_collector:
|
||||
self.metrics_collector.on_block_evicted(block)
|
||||
|
||||
block_hash = block.block_hash
|
||||
if block_hash is None:
|
||||
evicted_hashes = self._remove_cached_block_hashes(block)
|
||||
if not evicted_hashes:
|
||||
# The block doesn't have hash, eviction is not needed
|
||||
return False
|
||||
|
||||
if self.cached_block_hash_to_block.pop(block_hash, block.block_id) is None:
|
||||
# block not found in cached_block_hash_to_block,
|
||||
# eviction is not needed
|
||||
return False
|
||||
|
||||
block.reset_hash()
|
||||
|
||||
if self.enable_kv_cache_events:
|
||||
self.kv_event_queue.append(
|
||||
BlockRemoved(
|
||||
block_hashes=[maybe_convert_block_hash(get_block_hash(block_hash))],
|
||||
medium=MEDIUM_GPU,
|
||||
group_idx=get_group_id(block_hash),
|
||||
)
|
||||
)
|
||||
self._emit_block_removed_events(evicted_hashes)
|
||||
return True
|
||||
|
||||
def touch(self, blocks: Sequence[KVCacheBlock]) -> None:
|
||||
@@ -478,6 +673,7 @@ class BlockPool:
|
||||
|
||||
# Remove all hashes so that no new blocks will hit.
|
||||
self.cached_block_hash_to_block = BlockHashToBlockMap()
|
||||
self.cached_block_hashes_by_block.clear()
|
||||
|
||||
# Remove all hashes from all blocks.
|
||||
for block in self.blocks:
|
||||
|
||||
@@ -125,6 +125,9 @@ class KVCacheBlock:
|
||||
# The hash key (block hash + group id) of the block, only available
|
||||
# when the block is full and cached.
|
||||
_block_hash: BlockHashWithGroupId | None = None
|
||||
# Number of prefix tokens covered by _block_hash. For full blocks this is
|
||||
# the full block boundary; partial aliases can end inside a cache block.
|
||||
_block_hash_num_tokens: int | None = None
|
||||
|
||||
# Used to construct a doubly linked list for free blocks.
|
||||
# These two attributes should only be manipulated by FreeKVCacheBlockQueue.
|
||||
@@ -138,16 +141,25 @@ class KVCacheBlock:
|
||||
def block_hash(self) -> BlockHashWithGroupId | None:
|
||||
return self._block_hash
|
||||
|
||||
@block_hash.setter
|
||||
def block_hash(self, block_hash: BlockHashWithGroupId):
|
||||
assert self.block_hash is None, (
|
||||
@property
|
||||
def block_hash_num_tokens(self) -> int | None:
|
||||
return self._block_hash_num_tokens
|
||||
|
||||
def set_block_hash(
|
||||
self,
|
||||
block_hash: BlockHashWithGroupId,
|
||||
num_tokens: int | None = None,
|
||||
) -> None:
|
||||
assert self.block_hash is None and self._block_hash_num_tokens is None, (
|
||||
"The block already has a hash. This should not happen."
|
||||
)
|
||||
self._block_hash = block_hash
|
||||
self._block_hash_num_tokens = num_tokens
|
||||
|
||||
def reset_hash(self):
|
||||
"""Reset the block hash when the block is evicted."""
|
||||
self._block_hash = None
|
||||
self._block_hash_num_tokens = None
|
||||
|
||||
def __repr__(self) -> str:
|
||||
# Use block_id instead of KVCacheBlock object to avoid calling __repr__
|
||||
@@ -158,6 +170,7 @@ class KVCacheBlock:
|
||||
f"KVCacheBlock(block_id={self.block_id}, "
|
||||
f"ref_cnt={self.ref_cnt}, "
|
||||
f"_block_hash={self._block_hash!r}, "
|
||||
f"_block_hash_num_tokens={self._block_hash_num_tokens}, "
|
||||
f"prev_free_block={prev_block_id}, "
|
||||
f"next_free_block={next_block_id})"
|
||||
)
|
||||
@@ -658,18 +671,24 @@ def resolve_kv_cache_block_sizes(
|
||||
|
||||
|
||||
def get_request_block_hasher(
|
||||
block_size: int,
|
||||
hash_block_size: int,
|
||||
caching_hash_fn: Callable[[Any], bytes],
|
||||
) -> Callable[[Request], list[BlockHash]]:
|
||||
"""
|
||||
Returns a function which computes the list of un-computed block hashes
|
||||
of a request."""
|
||||
of a request.
|
||||
|
||||
Hashes are computed at ``hash_block_size`` granularity and chained over the
|
||||
full prefix, so each hash uniquely fingerprints the prefix ending at its
|
||||
boundary. Coarser group block sizes and partial-cache boundaries reuse
|
||||
these hashes directly (see ``BlockHashListWithBlockSize``).
|
||||
"""
|
||||
|
||||
def request_block_hasher(request: Request) -> list[BlockHash]:
|
||||
start_token_idx = len(request.block_hashes) * block_size
|
||||
start_token_idx = len(request.block_hashes) * hash_block_size
|
||||
num_tokens = request.num_tokens
|
||||
|
||||
if start_token_idx + block_size > num_tokens:
|
||||
if start_token_idx + hash_block_size > num_tokens:
|
||||
# Early stop when there no new full blocks created.
|
||||
return []
|
||||
|
||||
@@ -686,7 +705,7 @@ def get_request_block_hasher(
|
||||
)
|
||||
new_block_hashes: list[BlockHash] = []
|
||||
while True:
|
||||
end_token_idx = start_token_idx + block_size
|
||||
end_token_idx = start_token_idx + hash_block_size
|
||||
if end_token_idx > num_tokens:
|
||||
# We only hash full blocks
|
||||
break
|
||||
@@ -703,7 +722,7 @@ def get_request_block_hasher(
|
||||
)
|
||||
|
||||
new_block_hashes.append(block_hash)
|
||||
start_token_idx += block_size
|
||||
start_token_idx += hash_block_size
|
||||
prev_block_hash_value = block_hash
|
||||
|
||||
return new_block_hashes
|
||||
@@ -2132,11 +2151,14 @@ class BlockHashListWithBlockSize:
|
||||
|
||||
Currently, only scaling up by an integer factor is supported (i.e.,
|
||||
`target_block_size` is a multiple of `hash_block_size`). Conversion is
|
||||
performed lazily on access for efficiency, by concatenating consecutive
|
||||
hashes at `hash_block_size` to form each hash at `target_block_size`.
|
||||
performed lazily on access for efficiency. Each `hash_block_size` hash is
|
||||
already chained over its entire prefix, so the hash at the last
|
||||
`hash_block_size` boundary of a `target_block_size` block uniquely
|
||||
fingerprints that block's prefix; we use it directly.
|
||||
|
||||
Example (`hash_block_size` = 16, `target_block_size` = 32):
|
||||
concatenating two 16-size hashes yields one 32-size hash:
|
||||
the second 16-size hash already covers tokens 0-31, so it is the 32-size
|
||||
hash:
|
||||
|
||||
Block hashes with block_size 16:
|
||||
| Token Range | 0-15 | 16-31 | 32-47 | 48-63 |
|
||||
@@ -2146,7 +2168,7 @@ class BlockHashListWithBlockSize:
|
||||
Block hashes with block_size 32:
|
||||
| Token Range | 0-31 | 32-63 |
|
||||
|-------------|------|-------|
|
||||
| Hash | AB | CD |
|
||||
| Hash | B | D |
|
||||
|
||||
Args:
|
||||
block_hashes: Block hashes to convert, computed at `hash_block_size`.
|
||||
@@ -2188,9 +2210,9 @@ class BlockHashListWithBlockSize:
|
||||
yield self._get_value_at(i)
|
||||
|
||||
def _get_value_at(self, idx: int) -> BlockHash:
|
||||
base = idx * self.scale_factor
|
||||
end = base + self.scale_factor
|
||||
return BlockHash(b"".join(self.block_hashes[base:end]))
|
||||
# The last hash_block_size hash within the target block already chains
|
||||
# over the whole prefix, so it is the target block's hash.
|
||||
return self.block_hashes[(idx + 1) * self.scale_factor - 1]
|
||||
|
||||
|
||||
BlockHashList = list[BlockHash] | BlockHashListWithBlockSize
|
||||
|
||||
Reference in New Issue
Block a user