[KV Offloading] Add free block iterator for CPU offload scheduling (#47849)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
Chauncey
2026-07-09 06:59:54 +00:00
committed by GitHub
parent b8c7c86533
commit 529af88842
2 changed files with 18 additions and 17 deletions
+14
View File
@@ -407,6 +407,20 @@ class FreeKVCacheBlockQueue:
curr_block = curr_block.next_free_block
return ret
def iter_blocks_after(
self,
cursor: KVCacheBlock | None,
) -> Iterator[KVCacheBlock]:
"""Iterate free blocks in eviction order after the cursor."""
if cursor is None:
curr_block = self.fake_free_list_head.next_free_block
else:
curr_block = cursor.next_free_block
while curr_block is not None and curr_block is not self.fake_free_list_tail:
yield curr_block
curr_block = curr_block.next_free_block
def need_extra_keys(request: Request) -> bool:
"""Check whether the blocks allocated to this request need extra hash keys.
+4 -17
View File
@@ -487,24 +487,14 @@ class SimpleCPUOffloadScheduler:
if self._cursor is not None and self._cursor.ref_cnt > 0:
self._cursor = None
# Determine start node.
if self._cursor is None:
node = free_queue.fake_free_list_head.next_free_block
else:
node = self._cursor.next_free_block
tail = free_queue.fake_free_list_tail
gpu_ids: list[int] = []
block_hashes: list[bytes] = []
covered = 0
last_visited = self._cursor
while (
node is not None
and node is not tail
and covered < self._target_free
and len(gpu_ids) < num_cpu_free
):
for covered, node in enumerate(free_queue.iter_blocks_after(self._cursor)):
if covered >= self._target_free or len(gpu_ids) >= num_cpu_free:
break
last_visited = node
bhash = node.block_hash
@@ -516,9 +506,6 @@ class SimpleCPUOffloadScheduler:
gpu_ids.append(node.block_id)
block_hashes.append(bhash)
covered += 1
node = node.next_free_block
self._cursor = last_visited
# Batch-allocate CPU blocks and stamp hashes.