[Core] Sliding window for block manager v2 (#4545)
Co-authored-by: Ruth Evans <ruthevans@Ruths-MacBook-Pro.local>
This commit is contained in:
co-authored by
Ruth Evans
parent
890aa93d27
commit
d4f3985907
@@ -203,3 +203,12 @@ class DeviceAwareBlockAllocator(ABC):
|
||||
def get_common_computed_block_ids(
|
||||
self, seq_block_ids: List[List[int]]) -> List[int]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def allocate_or_get_null_block(self) -> Block:
|
||||
"""
|
||||
Null blocks are used as a placeholders for KV cache blocks that have
|
||||
been dropped due to sliding window.
|
||||
There is at most one null block per allocator.
|
||||
"""
|
||||
pass
|
||||
|
||||
Reference in New Issue
Block a user