[KV Offloading] Add free block iterator for CPU offload scheduling (#47849)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -407,6 +407,20 @@ class FreeKVCacheBlockQueue:
|
||||
curr_block = curr_block.next_free_block
|
||||
return ret
|
||||
|
||||
def iter_blocks_after(
|
||||
self,
|
||||
cursor: KVCacheBlock | None,
|
||||
) -> Iterator[KVCacheBlock]:
|
||||
"""Iterate free blocks in eviction order after the cursor."""
|
||||
if cursor is None:
|
||||
curr_block = self.fake_free_list_head.next_free_block
|
||||
else:
|
||||
curr_block = cursor.next_free_block
|
||||
|
||||
while curr_block is not None and curr_block is not self.fake_free_list_tail:
|
||||
yield curr_block
|
||||
curr_block = curr_block.next_free_block
|
||||
|
||||
|
||||
def need_extra_keys(request: Request) -> bool:
|
||||
"""Check whether the blocks allocated to this request need extra hash keys.
|
||||
|
||||
@@ -487,24 +487,14 @@ class SimpleCPUOffloadScheduler:
|
||||
if self._cursor is not None and self._cursor.ref_cnt > 0:
|
||||
self._cursor = None
|
||||
|
||||
# Determine start node.
|
||||
if self._cursor is None:
|
||||
node = free_queue.fake_free_list_head.next_free_block
|
||||
else:
|
||||
node = self._cursor.next_free_block
|
||||
|
||||
tail = free_queue.fake_free_list_tail
|
||||
gpu_ids: list[int] = []
|
||||
block_hashes: list[bytes] = []
|
||||
covered = 0
|
||||
last_visited = self._cursor
|
||||
|
||||
while (
|
||||
node is not None
|
||||
and node is not tail
|
||||
and covered < self._target_free
|
||||
and len(gpu_ids) < num_cpu_free
|
||||
):
|
||||
for covered, node in enumerate(free_queue.iter_blocks_after(self._cursor)):
|
||||
if covered >= self._target_free or len(gpu_ids) >= num_cpu_free:
|
||||
break
|
||||
|
||||
last_visited = node
|
||||
bhash = node.block_hash
|
||||
|
||||
@@ -516,9 +506,6 @@ class SimpleCPUOffloadScheduler:
|
||||
gpu_ids.append(node.block_id)
|
||||
block_hashes.append(bhash)
|
||||
|
||||
covered += 1
|
||||
node = node.next_free_block
|
||||
|
||||
self._cursor = last_visited
|
||||
|
||||
# Batch-allocate CPU blocks and stamp hashes.
|
||||
|
||||
Reference in New Issue
Block a user