[Core] Use sha256 bytes instead of BlockHash to reduce GC overhead (#23673)
Signed-off-by: linzebing <linzebing1995@gmail.com>
This commit is contained in:
@@ -9,7 +9,11 @@ from vllm.distributed.kv_events import (MEDIUM_GPU, AllBlocksCleared,
|
||||
KVCacheEvent)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.kv_cache_utils import (BlockHash, BlockHashWithGroupId,
|
||||
FreeKVCacheBlockQueue, KVCacheBlock)
|
||||
ExternalBlockHash,
|
||||
FreeKVCacheBlockQueue, KVCacheBlock,
|
||||
get_block_hash,
|
||||
make_block_hash_with_group_id,
|
||||
maybe_convert_block_hash)
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -84,8 +88,10 @@ class BlockPool:
|
||||
"""
|
||||
cached_blocks = []
|
||||
for group_id in kv_cache_group_ids:
|
||||
block_hash_with_group_id = make_block_hash_with_group_id(
|
||||
block_hash, group_id)
|
||||
cached_blocks_one_group = self.cached_block_hash_to_block.get(
|
||||
BlockHashWithGroupId(block_hash, group_id))
|
||||
block_hash_with_group_id)
|
||||
if not cached_blocks_one_group:
|
||||
return None
|
||||
first_block = next(iter(cached_blocks_one_group.values()))
|
||||
@@ -124,28 +130,29 @@ class BlockPool:
|
||||
assert len(request.block_hashes) >= num_full_blocks
|
||||
new_block_hashes = request.block_hashes[num_cached_blocks:]
|
||||
|
||||
new_hashes: Optional[list[int]] = ([] if self.enable_kv_cache_events
|
||||
else None)
|
||||
new_hashes: Optional[list[ExternalBlockHash]] = (
|
||||
[] if self.enable_kv_cache_events else None)
|
||||
for i, blk in enumerate(new_full_blocks):
|
||||
assert blk.block_hash is None
|
||||
block_hash = new_block_hashes[i]
|
||||
|
||||
# Update and added the full block to the cache.
|
||||
block_hash_with_group_id = BlockHashWithGroupId(
|
||||
block_hash_with_group_id = make_block_hash_with_group_id(
|
||||
block_hash, kv_cache_group_id)
|
||||
blk.block_hash = block_hash_with_group_id
|
||||
self.cached_block_hash_to_block[block_hash_with_group_id][
|
||||
blk.block_id] = blk
|
||||
if new_hashes is not None:
|
||||
new_hashes.append(block_hash.hash_value)
|
||||
new_hashes.append(maybe_convert_block_hash(block_hash))
|
||||
|
||||
if self.enable_kv_cache_events:
|
||||
if num_cached_blocks == 0:
|
||||
parent_block_hash = None
|
||||
parent_block_hash: Optional[ExternalBlockHash] = None
|
||||
else:
|
||||
parent_block = blocks[num_cached_blocks - 1]
|
||||
assert parent_block.block_hash is not None
|
||||
parent_block_hash = parent_block.block_hash.get_hash_value()
|
||||
parent_block_hash = maybe_convert_block_hash(
|
||||
get_block_hash(parent_block.block_hash))
|
||||
|
||||
self.kv_event_queue.append(
|
||||
BlockStored(
|
||||
@@ -220,7 +227,9 @@ class BlockPool:
|
||||
# we disable hybrid kv cache manager when kv cache event is
|
||||
# enabled, so there is only one group.
|
||||
self.kv_event_queue.append(
|
||||
BlockRemoved(block_hashes=[block_hash.get_hash_value()],
|
||||
BlockRemoved(block_hashes=[
|
||||
maybe_convert_block_hash(get_block_hash(block_hash))
|
||||
],
|
||||
medium=MEDIUM_GPU))
|
||||
return True
|
||||
|
||||
|
||||
@@ -6,11 +6,12 @@ import os
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import astuple, dataclass
|
||||
from typing import Any, Callable, NamedTuple, Optional
|
||||
from typing import Any, Callable, NewType, Optional, Union
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import GiB_bytes, cdiv, sha256_cbor_64bit
|
||||
from vllm.utils import GiB_bytes, cdiv, sha256_cbor
|
||||
from vllm.v1.kv_cache_interface import (ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec, KVCacheConfig,
|
||||
KVCacheGroupSpec, KVCacheSpec,
|
||||
@@ -18,59 +19,78 @@ from vllm.v1.kv_cache_interface import (ChunkedLocalAttentionSpec,
|
||||
from vllm.v1.metrics.stats import PrefixCacheStats
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
# BlockHash represents the hash of a single KV-cache block used for
|
||||
# prefix caching. Treating it as a distinct type from ``bytes`` helps
|
||||
# catch accidental misuse when passing around raw byte strings.
|
||||
BlockHash = NewType("BlockHash", bytes)
|
||||
|
||||
# ``BlockHashWithGroupId`` combines a ``BlockHash`` with its KV cache group ID.
|
||||
# It is represented as raw bytes for compactness and efficiency. The helper
|
||||
# functions below pack/unpack the ``BlockHash`` and group id into/from the key.
|
||||
BlockHashWithGroupId = NewType("BlockHashWithGroupId", bytes)
|
||||
|
||||
# ExternalBlockHash is used for reproducible prefix-cache block hashing.
|
||||
# It's a union of ``bytes`` and ``int`` to keep backward compatibility
|
||||
# after we default block hashing to use sha256 bytes.
|
||||
ExternalBlockHash = Union[bytes, int]
|
||||
|
||||
|
||||
class BlockHash(NamedTuple):
|
||||
"""Hash value of a block (int), the token IDs in the block, and extra keys.
|
||||
We keep a tuple of token IDs and extra keys to reduce the likelihood of
|
||||
hash collisions when the hash value is the same. By using SHA256 however,
|
||||
hash collisions are practically impossible.
|
||||
def make_block_hash_with_group_id(block_hash: BlockHash,
|
||||
group_id: int) -> BlockHashWithGroupId:
|
||||
"""Pack a ``BlockHash`` and group id into a ``BlockHashWithGroupId``.
|
||||
|
||||
The group id is encoded using 4 bytes in big-endian order and appended to
|
||||
the block hash bytes. This representation avoids creating tuples while
|
||||
still allowing us to recover both components when needed.
|
||||
"""
|
||||
# Hash value of the block in an integer.
|
||||
hash_value: int
|
||||
# Token IDs in the block.
|
||||
token_ids: tuple[int, ...]
|
||||
# Extra keys for the block.
|
||||
extra_keys: Optional[Any] = None
|
||||
return BlockHashWithGroupId(block_hash +
|
||||
group_id.to_bytes(4, "big", signed=False))
|
||||
|
||||
|
||||
class BlockHashWithGroupId(NamedTuple):
|
||||
# The hash value for the contents (e.g., token_ids) of a block without group
|
||||
# ID. The value is the same for blocks representing the same tokens but for
|
||||
# different groups.
|
||||
block_hash: BlockHash
|
||||
# The KV cache group ID.
|
||||
group_id: int
|
||||
def get_block_hash(key: BlockHashWithGroupId) -> BlockHash:
|
||||
"""Extract the ``BlockHash`` from a ``BlockHashWithGroupId``."""
|
||||
return BlockHash(key[:-4])
|
||||
|
||||
def get_hash_value(self) -> int:
|
||||
return self.block_hash.hash_value
|
||||
|
||||
def get_group_id(key: BlockHashWithGroupId) -> int:
|
||||
"""Extract the group id from a ``BlockHashWithGroupId``."""
|
||||
return int.from_bytes(key[-4:], "big", signed=False)
|
||||
|
||||
|
||||
def maybe_convert_block_hash(hash_bytes: BlockHash) -> ExternalBlockHash:
|
||||
if not envs.VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES:
|
||||
return hash_bytes
|
||||
return int.from_bytes(hash_bytes, byteorder="big") & ((1 << 64) - 1)
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# The hash seed for the first block of any prefix block sequence.
|
||||
#
|
||||
# We use a random value to avoid hash collisions or PYTHONHASHSEED environment
|
||||
# variable if set such that processes can share the seed if needed.
|
||||
# This aligns with the behavior of Python's hash() function, which also uses
|
||||
# a random seed if PYTHONHASHSEED is not set.
|
||||
# variable if set such that processes can share the seed if needed. This aligns
|
||||
# with the behavior of Python's hash() function, which also uses a random seed
|
||||
# if PYTHONHASHSEED is not set.
|
||||
#
|
||||
# The function `init_none_hash` initializes this variable globally.
|
||||
NONE_HASH: int
|
||||
NONE_HASH: BlockHash
|
||||
|
||||
|
||||
def init_none_hash(hash_fn: Callable):
|
||||
def init_none_hash(hash_fn: Callable[[Any], bytes]):
|
||||
global NONE_HASH
|
||||
|
||||
hash_seed = os.getenv("PYTHONHASHSEED")
|
||||
if hash_seed is None and hash_fn is sha256_cbor_64bit:
|
||||
if hash_seed is None and hash_fn is sha256_cbor:
|
||||
logger.warning(
|
||||
"PYTHONHASHSEED is not set. This will lead to non-reproducible "
|
||||
"block-hashes when using sha256_cbor_64bit as the hash function."
|
||||
"block-hashes when using sha256_cbor as the hash function."
|
||||
"Consider setting PYTHONHASHSEED to a fixed value for "
|
||||
"reproducibility.")
|
||||
|
||||
NONE_HASH = (int.from_bytes(os.urandom(32), byteorder="big")
|
||||
if hash_seed is None else hash_fn(hash_seed))
|
||||
if hash_seed is None:
|
||||
NONE_HASH = BlockHash(os.urandom(32))
|
||||
else:
|
||||
NONE_HASH = BlockHash(hash_fn(hash_seed))
|
||||
|
||||
|
||||
class PrefixCachingMetrics:
|
||||
@@ -142,8 +162,8 @@ class KVCacheBlock:
|
||||
block_id: int
|
||||
# Reference count.
|
||||
ref_cnt: int = 0
|
||||
# The hash of the block composed of (block hash, tuple of token IDs).
|
||||
# It is only available when the block is full.
|
||||
# The hash key (block hash + group id) of the block, only available
|
||||
# when the block is full and cached.
|
||||
_block_hash: Optional[BlockHashWithGroupId] = None
|
||||
|
||||
# Used to construct a doubly linked list for free blocks.
|
||||
@@ -177,7 +197,7 @@ class KVCacheBlock:
|
||||
if self.next_free_block else None)
|
||||
return (f"KVCacheBlock(block_id={self.block_id}, "
|
||||
f"ref_cnt={self.ref_cnt}, "
|
||||
f"_block_hash={self._block_hash}, "
|
||||
f"_block_hash={self._block_hash!r}, "
|
||||
f"prev_free_block={prev_block_id}, "
|
||||
f"next_free_block={next_block_id})")
|
||||
|
||||
@@ -517,15 +537,14 @@ def generate_block_hash_extra_keys(
|
||||
|
||||
|
||||
def hash_block_tokens(
|
||||
hash_function: Callable,
|
||||
parent_block_hash: Optional[int],
|
||||
hash_function: Callable[[Any], bytes],
|
||||
parent_block_hash: Optional[BlockHash],
|
||||
curr_block_token_ids: Sequence[int],
|
||||
extra_keys: Optional[tuple[Any, ...]] = None) -> BlockHash:
|
||||
"""Computes a hash value corresponding to the contents of a block and
|
||||
the contents of the preceding block(s). The hash value is used for
|
||||
prefix caching. We use LRU cache for this function to avoid recomputing
|
||||
hash values for the same block contents.
|
||||
|
||||
Args:
|
||||
hash_function: The hash function used to compute block hash.
|
||||
parent_block_hash: The hash of the parent block. None
|
||||
@@ -533,7 +552,6 @@ def hash_block_tokens(
|
||||
curr_block_token_ids: A list of token ids in the current
|
||||
block. The current block is assumed to be full.
|
||||
extra_keys: Extra keys for the block.
|
||||
|
||||
Returns:
|
||||
The hash value of the block and the token ids in the block.
|
||||
The entire tuple is used as the hash key of the block.
|
||||
@@ -544,26 +562,16 @@ def hash_block_tokens(
|
||||
curr_block_token_ids_tuple = tuple(curr_block_token_ids)
|
||||
return BlockHash(
|
||||
hash_function(
|
||||
(parent_block_hash, curr_block_token_ids_tuple, extra_keys)),
|
||||
curr_block_token_ids_tuple, extra_keys)
|
||||
(parent_block_hash, curr_block_token_ids_tuple, extra_keys)))
|
||||
|
||||
|
||||
def get_request_block_hasher(
|
||||
block_size: int,
|
||||
caching_hash_fn: Callable[[Any],
|
||||
int]) -> Callable[[Request], list[BlockHash]]:
|
||||
caching_hash_fn: Callable[[Any], bytes],
|
||||
) -> Callable[[Request], list[BlockHash]]:
|
||||
"""
|
||||
Returns a function which computes the list of un-computed block hashes
|
||||
of a request.
|
||||
|
||||
Each request holds a list of its block hashes (request.block_hashes).
|
||||
When a request is created, it calls the below function to compute
|
||||
the hashes of all full blocks of the request's initial tokens.
|
||||
The hashes are then stored in request.block_hashes.
|
||||
Later, whenever new tokens are appended to the request, it calls
|
||||
the below function again to compute any new full blocks of tokens.
|
||||
The returned new hashes are appended to request.block_hashes.
|
||||
"""
|
||||
of a request."""
|
||||
|
||||
def request_block_hasher(request: Request) -> list[BlockHash]:
|
||||
start_token_idx = len(request.block_hashes) * block_size
|
||||
@@ -577,8 +585,8 @@ def get_request_block_hasher(
|
||||
# last mm input.
|
||||
curr_mm_idx = -1
|
||||
|
||||
prev_block_hash_value = request.block_hashes[-1].hash_value \
|
||||
if request.block_hashes else None
|
||||
prev_block_hash_value = (request.block_hashes[-1]
|
||||
if request.block_hashes else None)
|
||||
new_block_hashes: list[BlockHash] = []
|
||||
while True:
|
||||
end_token_idx = start_token_idx + block_size
|
||||
@@ -598,7 +606,7 @@ def get_request_block_hasher(
|
||||
|
||||
new_block_hashes.append(block_hash)
|
||||
start_token_idx += block_size
|
||||
prev_block_hash_value = block_hash.hash_value
|
||||
prev_block_hash_value = block_hash
|
||||
|
||||
return new_block_hashes
|
||||
|
||||
|
||||
Reference in New Issue
Block a user