[Core] Sliding window for block manager v2 (#4545)

Co-authored-by: Ruth Evans <ruthevans@Ruths-MacBook-Pro.local>
This commit is contained in:
Michał Moskal
2024-05-28 11:07:07 +09:00
committed by GitHub
co-authored by Ruth Evans
parent 890aa93d27
commit d4f3985907
12 changed files with 457 additions and 45 deletions
+4 -1
View File
@@ -68,8 +68,11 @@ class CacheEngine:
pin_memory = is_pin_memory_available() if device == "cpu" else False
kv_cache: List[torch.Tensor] = []
for _ in range(self.num_layers):
# null block in CpuGpuBlockAllocator requires at least that
# block to be zeroed-out.
# We zero-out everything for simplicity.
kv_cache.append(
torch.empty(kv_cache_shape,
torch.zeros(kv_cache_shape,
dtype=self.dtype,
pin_memory=pin_memory,
device=device))