[PREFIX CACHING FOLLOW UP] A bunch of fixes to block allocator performance when automatic prefix caching is disabled (#3357)
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
This commit is contained in:
co-authored by
Zhuohan Li
parent
20478c4d3a
commit
9474e89ba4
+14
-69
@@ -1,5 +1,5 @@
|
||||
import enum
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Dict
|
||||
from abc import ABC, abstractmethod, abstractproperty
|
||||
|
||||
from vllm.block import PhysicalTokenBlock
|
||||
@@ -10,7 +10,6 @@ class EvictionPolicy(enum.Enum):
|
||||
Evictor subclass.
|
||||
"""
|
||||
LRU = enum.auto()
|
||||
FIFO = enum.auto()
|
||||
|
||||
|
||||
class Evictor(ABC):
|
||||
@@ -65,75 +64,23 @@ class LRUEvictor(Evictor):
|
||||
return block_hash in self.free_table
|
||||
|
||||
# TODO: The performance of this evict function can be optimized further.
|
||||
def evict(self) -> PhysicalTokenBlock:
|
||||
free_blocks: List[PhysicalTokenBlock] = list(self.free_table.values())
|
||||
if len(free_blocks) == 0:
|
||||
raise ValueError("No usable cache memory left")
|
||||
|
||||
# Find lowest timestamp
|
||||
lowest_timestamp = free_blocks[0].last_accessed
|
||||
for block in free_blocks:
|
||||
if block.last_accessed < lowest_timestamp:
|
||||
lowest_timestamp = block.last_accessed
|
||||
|
||||
# Find all blocks with the lowest timestamp
|
||||
least_recent: List[PhysicalTokenBlock] = []
|
||||
for block in free_blocks:
|
||||
if block.last_accessed == lowest_timestamp:
|
||||
least_recent.append(block)
|
||||
|
||||
# Find highest prefix count per block
|
||||
highest_num_hashed_tokens = 0
|
||||
for block in least_recent:
|
||||
if block.num_hashed_tokens > highest_num_hashed_tokens:
|
||||
highest_num_hashed_tokens = block.num_hashed_tokens
|
||||
|
||||
evicted_block: Optional[PhysicalTokenBlock] = None
|
||||
|
||||
# Find the first block with the lowest timestamp
|
||||
for block in least_recent:
|
||||
if block.num_hashed_tokens == highest_num_hashed_tokens:
|
||||
evicted_block = block
|
||||
break
|
||||
|
||||
assert evicted_block is not None
|
||||
|
||||
del self.free_table[evicted_block.block_hash]
|
||||
|
||||
evicted_block.computed = False
|
||||
return evicted_block
|
||||
|
||||
def add(self, block: PhysicalTokenBlock):
|
||||
self.free_table[block.block_hash] = block
|
||||
|
||||
def remove(self, block_hash: int) -> PhysicalTokenBlock:
|
||||
if block_hash not in self.free_table:
|
||||
raise ValueError(
|
||||
"Attempting to remove block that's not in the evictor")
|
||||
block: PhysicalTokenBlock = self.free_table[block_hash]
|
||||
del self.free_table[block_hash]
|
||||
return block
|
||||
|
||||
@property
|
||||
def num_blocks(self) -> int:
|
||||
return len(self.free_table)
|
||||
|
||||
|
||||
class RandomEvictor(Evictor):
|
||||
"""Evicts in a first-in-first-out order"""
|
||||
|
||||
def __init__(self):
|
||||
self.free_table: Dict[int, PhysicalTokenBlock] = {}
|
||||
|
||||
def __contains__(self, block_hash: int) -> bool:
|
||||
return block_hash in self.free_table
|
||||
|
||||
def evict(self) -> PhysicalTokenBlock:
|
||||
if len(self.free_table) == 0:
|
||||
raise ValueError("No usable cache memory left")
|
||||
evicted_block = next(iter(self.free_table.values()))
|
||||
evicted_block.computed = False
|
||||
free_blocks = self.free_table.values()
|
||||
|
||||
# Get evicted block
|
||||
evicted_block: PhysicalTokenBlock = next(iter(free_blocks))
|
||||
|
||||
for block in free_blocks:
|
||||
if (block.last_accessed < evicted_block.last_accessed
|
||||
or block.last_accessed == evicted_block.last_accessed and
|
||||
block.num_hashed_tokens > evicted_block.num_hashed_tokens):
|
||||
evicted_block = block
|
||||
|
||||
del self.free_table[evicted_block.block_hash]
|
||||
|
||||
evicted_block.computed = False
|
||||
return evicted_block
|
||||
|
||||
def add(self, block: PhysicalTokenBlock):
|
||||
@@ -155,7 +102,5 @@ class RandomEvictor(Evictor):
|
||||
def make_evictor(eviction_policy: EvictionPolicy) -> Evictor:
|
||||
if eviction_policy == EvictionPolicy.LRU:
|
||||
return LRUEvictor()
|
||||
elif eviction_policy == EvictionPolicy.FIFO:
|
||||
return RandomEvictor()
|
||||
else:
|
||||
raise ValueError(f"Unknown cache eviction policy: {eviction_policy}")
|
||||
|
||||
Reference in New Issue
Block a user