[PREFIX CACHING FOLLOW UP] A bunch of fixes to block allocator performance when automatic prefix caching is disabled (#3357)

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
This commit is contained in:
ElizaWszola
2024-03-20 00:11:11 -07:00
committed by GitHub
co-authored by Zhuohan Li
parent 20478c4d3a
commit 9474e89ba4
4 changed files with 171 additions and 127 deletions
+14 -69
View File
@@ -1,5 +1,5 @@
import enum
from typing import Dict, List, Optional
from typing import Dict
from abc import ABC, abstractmethod, abstractproperty
from vllm.block import PhysicalTokenBlock
@@ -10,7 +10,6 @@ class EvictionPolicy(enum.Enum):
Evictor subclass.
"""
LRU = enum.auto()
FIFO = enum.auto()
class Evictor(ABC):
@@ -65,75 +64,23 @@ class LRUEvictor(Evictor):
return block_hash in self.free_table
# TODO: The performance of this evict function can be optimized further.
def evict(self) -> PhysicalTokenBlock:
free_blocks: List[PhysicalTokenBlock] = list(self.free_table.values())
if len(free_blocks) == 0:
raise ValueError("No usable cache memory left")
# Find lowest timestamp
lowest_timestamp = free_blocks[0].last_accessed
for block in free_blocks:
if block.last_accessed < lowest_timestamp:
lowest_timestamp = block.last_accessed
# Find all blocks with the lowest timestamp
least_recent: List[PhysicalTokenBlock] = []
for block in free_blocks:
if block.last_accessed == lowest_timestamp:
least_recent.append(block)
# Find highest prefix count per block
highest_num_hashed_tokens = 0
for block in least_recent:
if block.num_hashed_tokens > highest_num_hashed_tokens:
highest_num_hashed_tokens = block.num_hashed_tokens
evicted_block: Optional[PhysicalTokenBlock] = None
# Find the first block with the lowest timestamp
for block in least_recent:
if block.num_hashed_tokens == highest_num_hashed_tokens:
evicted_block = block
break
assert evicted_block is not None
del self.free_table[evicted_block.block_hash]
evicted_block.computed = False
return evicted_block
def add(self, block: PhysicalTokenBlock):
self.free_table[block.block_hash] = block
def remove(self, block_hash: int) -> PhysicalTokenBlock:
if block_hash not in self.free_table:
raise ValueError(
"Attempting to remove block that's not in the evictor")
block: PhysicalTokenBlock = self.free_table[block_hash]
del self.free_table[block_hash]
return block
@property
def num_blocks(self) -> int:
return len(self.free_table)
class RandomEvictor(Evictor):
"""Evicts in a first-in-first-out order"""
def __init__(self):
self.free_table: Dict[int, PhysicalTokenBlock] = {}
def __contains__(self, block_hash: int) -> bool:
return block_hash in self.free_table
def evict(self) -> PhysicalTokenBlock:
if len(self.free_table) == 0:
raise ValueError("No usable cache memory left")
evicted_block = next(iter(self.free_table.values()))
evicted_block.computed = False
free_blocks = self.free_table.values()
# Get evicted block
evicted_block: PhysicalTokenBlock = next(iter(free_blocks))
for block in free_blocks:
if (block.last_accessed < evicted_block.last_accessed
or block.last_accessed == evicted_block.last_accessed and
block.num_hashed_tokens > evicted_block.num_hashed_tokens):
evicted_block = block
del self.free_table[evicted_block.block_hash]
evicted_block.computed = False
return evicted_block
def add(self, block: PhysicalTokenBlock):
@@ -155,7 +102,5 @@ class RandomEvictor(Evictor):
def make_evictor(eviction_policy: EvictionPolicy) -> Evictor:
if eviction_policy == EvictionPolicy.LRU:
return LRUEvictor()
elif eviction_policy == EvictionPolicy.FIFO:
return RandomEvictor()
else:
raise ValueError(f"Unknown cache eviction policy: {eviction_policy}")