[Core] Sliding window for block manager v2 (#4545)

Co-authored-by: Ruth Evans <ruthevans@Ruths-MacBook-Pro.local>
This commit is contained in:
Michał Moskal
2024-05-28 11:07:07 +09:00
committed by GitHub
co-authored by Ruth Evans
parent 890aa93d27
commit d4f3985907
12 changed files with 457 additions and 45 deletions
+9
View File
@@ -203,3 +203,12 @@ class DeviceAwareBlockAllocator(ABC):
def get_common_computed_block_ids(
self, seq_block_ids: List[List[int]]) -> List[int]:
pass
@abstractmethod
def allocate_or_get_null_block(self) -> Block:
"""
Null blocks are used as a placeholders for KV cache blocks that have
been dropped due to sliding window.
There is at most one null block per allocator.
"""
pass