[Bugfix] Block manager v2 with preemption and lookahead slots (#8824)
This commit is contained in:
@@ -259,25 +259,22 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
|
||||
current_swap_mapping[src_block_id] = dst_block_id
|
||||
return current_swap_mapping
|
||||
|
||||
def get_num_blocks_touched(self,
|
||||
blocks: List[Block],
|
||||
device: Device,
|
||||
num_lookahead_slots: int = 0) -> int:
|
||||
"""Returns the number of blocks that will be touched by
|
||||
def get_num_full_blocks_touched(self, blocks: List[Block],
|
||||
device: Device) -> int:
|
||||
"""Returns the number of full blocks that will be touched by
|
||||
swapping in/out the given blocks on to the 'device'.
|
||||
|
||||
Args:
|
||||
blocks: List of blocks to be swapped.
|
||||
device (Device): Device to swap the 'blocks' on.
|
||||
num_lookahead_slots (int): Number of lookahead slots used in
|
||||
speculative decoding, default to 0.
|
||||
|
||||
Returns:
|
||||
int: the number of blocks that will be touched by
|
||||
int: the number of full blocks that will be touched by
|
||||
swapping in/out the given blocks on to the 'device'.
|
||||
Non full blocks are ignored when deciding the number
|
||||
of blocks to touch.
|
||||
"""
|
||||
return self._allocators[device].get_num_blocks_touched(
|
||||
blocks, num_lookahead_slots)
|
||||
return self._allocators[device].get_num_full_blocks_touched(blocks)
|
||||
|
||||
def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
|
||||
"""Clears the copy-on-write (CoW) state and returns the mapping of
|
||||
|
||||
@@ -181,9 +181,7 @@ class BlockAllocator(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_num_blocks_touched(self,
|
||||
blocks: List[Block],
|
||||
num_lookahead_slots: int = 0) -> int:
|
||||
def get_num_full_blocks_touched(self, blocks: List[Block]) -> int:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
@@ -260,10 +258,8 @@ class DeviceAwareBlockAllocator(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_num_blocks_touched(self,
|
||||
blocks: List[Block],
|
||||
device: Device,
|
||||
num_lookahead_slots: int = 0) -> int:
|
||||
def get_num_full_blocks_touched(self, blocks: List[Block],
|
||||
device: Device) -> int:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -4,7 +4,6 @@ from typing import Deque, FrozenSet, Iterable, List, Optional, Tuple
|
||||
from vllm.core.block.common import (BlockPool, CopyOnWriteTracker, RefCounter,
|
||||
get_all_blocks_recursively)
|
||||
from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
|
||||
from vllm.utils import cdiv
|
||||
|
||||
Refcount = int
|
||||
|
||||
@@ -282,40 +281,26 @@ class NaiveBlockAllocator(BlockAllocator):
|
||||
def promote_to_immutable_block(self, block: Block) -> BlockId:
|
||||
raise NotImplementedError("There is no promotion for naive blocks")
|
||||
|
||||
def get_num_blocks_touched(self,
|
||||
blocks: List[Block],
|
||||
num_lookahead_slots: int = 0) -> int:
|
||||
"""Determine the number of blocks that will be touched by
|
||||
swapping in/out the given blocks from certain sequence
|
||||
group with the provided num_lookahead_slots.
|
||||
def get_num_full_blocks_touched(self, blocks: List[Block]) -> int:
|
||||
"""Returns the number of full blocks that will be touched by
|
||||
swapping in/out.
|
||||
|
||||
Args:
|
||||
blocks (List[Block]): The potential blocks to swap.
|
||||
num_lookahead_slots (int): number of lookahead slots (0 for swap
|
||||
out).
|
||||
|
||||
blocks: List of blocks to be swapped.
|
||||
Returns:
|
||||
int: the number of blocks that will be touched by
|
||||
swapping in/out the given blocks and num_lookahead_slots.
|
||||
int: the number of full blocks that will be touched by
|
||||
swapping in/out the given blocks. Non full blocks are ignored
|
||||
when deciding the number of blocks to touch.
|
||||
"""
|
||||
# NOTE: for naive block, we use set to eliminate common blocks among
|
||||
# seqs, also we compare the empty slots in the mutable blocks with
|
||||
# lookahead slots to get the number of unique new block that are
|
||||
# needed.
|
||||
old_block_set = set()
|
||||
new_block_count = 0
|
||||
# TODO(cade): make sure the logic is correct and clean it up.
|
||||
for block in blocks:
|
||||
if not block.is_full and num_lookahead_slots != 0:
|
||||
new_block_count += 1
|
||||
if num_lookahead_slots > block.num_empty_slots:
|
||||
new_block_count += cdiv(
|
||||
num_lookahead_slots - block.num_empty_slots,
|
||||
self._block_size)
|
||||
else:
|
||||
old_block_set.add(block.block_id)
|
||||
num_touched_blocks = new_block_count + len(old_block_set)
|
||||
return num_touched_blocks
|
||||
if block.is_full:
|
||||
old_block_set.add(block)
|
||||
return len(old_block_set)
|
||||
|
||||
def swap_out(self, blocks: List[Block]) -> None:
|
||||
for block in blocks:
|
||||
|
||||
@@ -8,7 +8,6 @@ from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
|
||||
from vllm.core.block.naive_block import (BlockPool, NaiveBlock,
|
||||
NaiveBlockAllocator)
|
||||
from vllm.core.evictor_v2 import EvictionPolicy, Evictor, make_evictor
|
||||
from vllm.utils import cdiv
|
||||
|
||||
PrefixHash = int
|
||||
|
||||
@@ -576,37 +575,27 @@ class PrefixCachingBlockAllocator(BlockAllocator):
|
||||
if ids
|
||||
])
|
||||
|
||||
def get_num_blocks_touched(self,
|
||||
blocks: List[Block],
|
||||
num_lookahead_slots: int = 0) -> int:
|
||||
"""Determine the number of blocks that will be touched by
|
||||
swapping in/out the given blocks from certain sequence
|
||||
group with the provided num_lookahead_slots.
|
||||
def get_num_full_blocks_touched(self, blocks: List[Block]) -> int:
|
||||
"""Returns the number of full blocks that will be touched by
|
||||
swapping in/out.
|
||||
|
||||
Args:
|
||||
blocks (List[Block]): The potential blocks to swap.
|
||||
num_lookahead_slots (int): number of lookahead slots (0 for
|
||||
swap out).
|
||||
|
||||
blocks: List of blocks to be swapped.
|
||||
Returns:
|
||||
int: the number of blocks that will be touched by
|
||||
swapping in/out the given blocks and num_lookahead_slots.
|
||||
int: the number of full blocks that will be touched by
|
||||
swapping in/out the given blocks. Non full blocks are ignored
|
||||
when deciding the number of blocks to touch.
|
||||
"""
|
||||
num_touched_blocks = 0
|
||||
num_touched_blocks: int = 0
|
||||
for block in blocks:
|
||||
if not block.is_full:
|
||||
# If the block has a match in the cache and the cached
|
||||
# block is not referenced, then we still count it as a
|
||||
# touched block
|
||||
if block.is_full and (not self.is_block_cached(block) or \
|
||||
(block.content_hash is not None and \
|
||||
self._cached_blocks[block.content_hash] in \
|
||||
self.evictor)):
|
||||
num_touched_blocks += 1
|
||||
if num_lookahead_slots > block.num_empty_slots:
|
||||
num_touched_blocks += cdiv(
|
||||
num_lookahead_slots - block.num_empty_slots,
|
||||
self._block_size)
|
||||
else:
|
||||
# If the block has a match in the cache and the cached block
|
||||
# is not referenced, then we still count it as a touched block
|
||||
if not self.is_block_cached(block) or \
|
||||
(block.content_hash is not None and \
|
||||
self._cached_blocks[block.content_hash] in self.evictor):
|
||||
num_touched_blocks += 1
|
||||
return num_touched_blocks
|
||||
|
||||
def swap_out(self, blocks: List[Block]) -> None:
|
||||
|
||||
Reference in New Issue
Block a user