[Scheduler] Preserve per-group eviction order in deferred block free (port of upstream #48532)
Repair: the port driver logged this patch as resolved but produced no commit; the -3 merge dropped the coordinator code hunk. Reapplied to HEAD. Co-Authored-By: Hinata <hinata@localhost> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -402,19 +402,24 @@ class KVCacheCoordinator(ABC):
|
||||
"""Pop the request's bookkeeping from all single-type managers and
|
||||
return its blocks without returning them to the block pool. The
|
||||
caller must eventually pass the returned blocks to
|
||||
`block_pool.free_blocks`, freeing them in reverse order (so that
|
||||
tail blocks are evicted first).
|
||||
`block_pool.free_blocks` as-is: they are already ordered so that
|
||||
tail blocks are evicted first, within each KV cache group.
|
||||
|
||||
Args:
|
||||
request_id: The request ID.
|
||||
|
||||
Returns:
|
||||
The request's blocks in allocation order.
|
||||
The request's blocks in eviction order (each group's blocks
|
||||
reversed, then concatenated group by group).
|
||||
|
||||
"""
|
||||
blocks: list[KVCacheBlock] = []
|
||||
for manager in self.single_type_managers:
|
||||
blocks.extend(manager.pop_blocks_for_free(request_id))
|
||||
# Reverse within each group so the concatenation is already in
|
||||
# eviction order, matching `free()`, which frees each group in its
|
||||
# own reversed `free_blocks()` call. Reversing the flat list
|
||||
# instead would emit whole groups in the wrong order.
|
||||
blocks.extend(reversed(manager.pop_blocks_for_free(request_id)))
|
||||
return blocks
|
||||
|
||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> list[int]:
|
||||
|
||||
@@ -651,13 +651,15 @@ class KVCacheManager:
|
||||
def pop_blocks_for_free(self, request: Request) -> list[KVCacheBlock]:
|
||||
"""Pop the request's bookkeeping and return its blocks without
|
||||
returning them to the block pool. The caller must eventually free
|
||||
them in reverse order (so that tail blocks are evicted first).
|
||||
them as-is: they are already ordered so that tail blocks are
|
||||
evicted first, within each KV cache group.
|
||||
|
||||
Args:
|
||||
request: The request to pop the blocks for.
|
||||
|
||||
Returns:
|
||||
The request's blocks in allocation order.
|
||||
The request's blocks in eviction order (each group's blocks
|
||||
reversed, then concatenated group by group).
|
||||
|
||||
"""
|
||||
return self.coordinator.pop_blocks_for_free(request.request_id)
|
||||
|
||||
@@ -2831,7 +2831,9 @@ class Scheduler(SchedulerInterface):
|
||||
if not self.defer_block_free or fence_seq <= self.processed_step_seq:
|
||||
self.kv_cache_manager.block_pool.free_blocks(blocks)
|
||||
return
|
||||
self.deferred_frees.append((fence_seq, blocks[::-1]))
|
||||
# The immediate path above frees these as-is, so enqueue them as-is:
|
||||
# the drain no longer reverses.
|
||||
self.deferred_frees.append((fence_seq, blocks))
|
||||
|
||||
def _drain_deferred_frees(self):
|
||||
"""Return deferred blocks whose fence step has completed.
|
||||
@@ -2845,8 +2847,9 @@ class Scheduler(SchedulerInterface):
|
||||
if fence > self.processed_step_seq:
|
||||
break
|
||||
_, blocks = self.deferred_frees.popleft()
|
||||
# Free in reverse order so that the tail blocks are evicted first.
|
||||
self.kv_cache_manager.block_pool.free_blocks(reversed(blocks))
|
||||
# Already in eviction order (reversed per KV cache group by
|
||||
# `pop_blocks_for_free`), so free as-is.
|
||||
self.kv_cache_manager.block_pool.free_blocks(blocks)
|
||||
|
||||
def get_num_unfinished_requests(self) -> int:
|
||||
if self._pause_state == PauseState.PAUSED_ALL:
|
||||
|
||||
Reference in New Issue
Block a user