[Bugfix] Block manager v2 with preemption and lookahead slots (#8824)
This commit is contained in:
@@ -259,25 +259,22 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
|
||||
current_swap_mapping[src_block_id] = dst_block_id
|
||||
return current_swap_mapping
|
||||
|
||||
def get_num_blocks_touched(self,
|
||||
blocks: List[Block],
|
||||
device: Device,
|
||||
num_lookahead_slots: int = 0) -> int:
|
||||
"""Returns the number of blocks that will be touched by
|
||||
def get_num_full_blocks_touched(self, blocks: List[Block],
|
||||
device: Device) -> int:
|
||||
"""Returns the number of full blocks that will be touched by
|
||||
swapping in/out the given blocks on to the 'device'.
|
||||
|
||||
Args:
|
||||
blocks: List of blocks to be swapped.
|
||||
device (Device): Device to swap the 'blocks' on.
|
||||
num_lookahead_slots (int): Number of lookahead slots used in
|
||||
speculative decoding, default to 0.
|
||||
|
||||
Returns:
|
||||
int: the number of blocks that will be touched by
|
||||
int: the number of full blocks that will be touched by
|
||||
swapping in/out the given blocks on to the 'device'.
|
||||
Non full blocks are ignored when deciding the number
|
||||
of blocks to touch.
|
||||
"""
|
||||
return self._allocators[device].get_num_blocks_touched(
|
||||
blocks, num_lookahead_slots)
|
||||
return self._allocators[device].get_num_full_blocks_touched(blocks)
|
||||
|
||||
def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
|
||||
"""Clears the copy-on-write (CoW) state and returns the mapping of
|
||||
|
||||
@@ -181,9 +181,7 @@ class BlockAllocator(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_num_blocks_touched(self,
|
||||
blocks: List[Block],
|
||||
num_lookahead_slots: int = 0) -> int:
|
||||
def get_num_full_blocks_touched(self, blocks: List[Block]) -> int:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
@@ -260,10 +258,8 @@ class DeviceAwareBlockAllocator(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_num_blocks_touched(self,
|
||||
blocks: List[Block],
|
||||
device: Device,
|
||||
num_lookahead_slots: int = 0) -> int:
|
||||
def get_num_full_blocks_touched(self, blocks: List[Block],
|
||||
device: Device) -> int:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -4,7 +4,6 @@ from typing import Deque, FrozenSet, Iterable, List, Optional, Tuple
|
||||
from vllm.core.block.common import (BlockPool, CopyOnWriteTracker, RefCounter,
|
||||
get_all_blocks_recursively)
|
||||
from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
|
||||
from vllm.utils import cdiv
|
||||
|
||||
Refcount = int
|
||||
|
||||
@@ -282,40 +281,26 @@ class NaiveBlockAllocator(BlockAllocator):
|
||||
def promote_to_immutable_block(self, block: Block) -> BlockId:
|
||||
raise NotImplementedError("There is no promotion for naive blocks")
|
||||
|
||||
def get_num_blocks_touched(self,
|
||||
blocks: List[Block],
|
||||
num_lookahead_slots: int = 0) -> int:
|
||||
"""Determine the number of blocks that will be touched by
|
||||
swapping in/out the given blocks from certain sequence
|
||||
group with the provided num_lookahead_slots.
|
||||
def get_num_full_blocks_touched(self, blocks: List[Block]) -> int:
|
||||
"""Returns the number of full blocks that will be touched by
|
||||
swapping in/out.
|
||||
|
||||
Args:
|
||||
blocks (List[Block]): The potential blocks to swap.
|
||||
num_lookahead_slots (int): number of lookahead slots (0 for swap
|
||||
out).
|
||||
|
||||
blocks: List of blocks to be swapped.
|
||||
Returns:
|
||||
int: the number of blocks that will be touched by
|
||||
swapping in/out the given blocks and num_lookahead_slots.
|
||||
int: the number of full blocks that will be touched by
|
||||
swapping in/out the given blocks. Non full blocks are ignored
|
||||
when deciding the number of blocks to touch.
|
||||
"""
|
||||
# NOTE: for naive block, we use set to eliminate common blocks among
|
||||
# seqs, also we compare the empty slots in the mutable blocks with
|
||||
# lookahead slots to get the number of unique new block that are
|
||||
# needed.
|
||||
old_block_set = set()
|
||||
new_block_count = 0
|
||||
# TODO(cade): make sure the logic is correct and clean it up.
|
||||
for block in blocks:
|
||||
if not block.is_full and num_lookahead_slots != 0:
|
||||
new_block_count += 1
|
||||
if num_lookahead_slots > block.num_empty_slots:
|
||||
new_block_count += cdiv(
|
||||
num_lookahead_slots - block.num_empty_slots,
|
||||
self._block_size)
|
||||
else:
|
||||
old_block_set.add(block.block_id)
|
||||
num_touched_blocks = new_block_count + len(old_block_set)
|
||||
return num_touched_blocks
|
||||
if block.is_full:
|
||||
old_block_set.add(block)
|
||||
return len(old_block_set)
|
||||
|
||||
def swap_out(self, blocks: List[Block]) -> None:
|
||||
for block in blocks:
|
||||
|
||||
@@ -8,7 +8,6 @@ from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
|
||||
from vllm.core.block.naive_block import (BlockPool, NaiveBlock,
|
||||
NaiveBlockAllocator)
|
||||
from vllm.core.evictor_v2 import EvictionPolicy, Evictor, make_evictor
|
||||
from vllm.utils import cdiv
|
||||
|
||||
PrefixHash = int
|
||||
|
||||
@@ -576,37 +575,27 @@ class PrefixCachingBlockAllocator(BlockAllocator):
|
||||
if ids
|
||||
])
|
||||
|
||||
def get_num_blocks_touched(self,
|
||||
blocks: List[Block],
|
||||
num_lookahead_slots: int = 0) -> int:
|
||||
"""Determine the number of blocks that will be touched by
|
||||
swapping in/out the given blocks from certain sequence
|
||||
group with the provided num_lookahead_slots.
|
||||
def get_num_full_blocks_touched(self, blocks: List[Block]) -> int:
|
||||
"""Returns the number of full blocks that will be touched by
|
||||
swapping in/out.
|
||||
|
||||
Args:
|
||||
blocks (List[Block]): The potential blocks to swap.
|
||||
num_lookahead_slots (int): number of lookahead slots (0 for
|
||||
swap out).
|
||||
|
||||
blocks: List of blocks to be swapped.
|
||||
Returns:
|
||||
int: the number of blocks that will be touched by
|
||||
swapping in/out the given blocks and num_lookahead_slots.
|
||||
int: the number of full blocks that will be touched by
|
||||
swapping in/out the given blocks. Non full blocks are ignored
|
||||
when deciding the number of blocks to touch.
|
||||
"""
|
||||
num_touched_blocks = 0
|
||||
num_touched_blocks: int = 0
|
||||
for block in blocks:
|
||||
if not block.is_full:
|
||||
# If the block has a match in the cache and the cached
|
||||
# block is not referenced, then we still count it as a
|
||||
# touched block
|
||||
if block.is_full and (not self.is_block_cached(block) or \
|
||||
(block.content_hash is not None and \
|
||||
self._cached_blocks[block.content_hash] in \
|
||||
self.evictor)):
|
||||
num_touched_blocks += 1
|
||||
if num_lookahead_slots > block.num_empty_slots:
|
||||
num_touched_blocks += cdiv(
|
||||
num_lookahead_slots - block.num_empty_slots,
|
||||
self._block_size)
|
||||
else:
|
||||
# If the block has a match in the cache and the cached block
|
||||
# is not referenced, then we still count it as a touched block
|
||||
if not self.is_block_cached(block) or \
|
||||
(block.content_hash is not None and \
|
||||
self._cached_blocks[block.content_hash] in self.evictor):
|
||||
num_touched_blocks += 1
|
||||
return num_touched_blocks
|
||||
|
||||
def swap_out(self, blocks: List[Block]) -> None:
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
"""A block manager that manages token blocks."""
|
||||
from itertools import chain
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Sequence as GenericSequence
|
||||
from typing import Tuple
|
||||
@@ -470,12 +469,31 @@ class BlockSpaceManagerV2(BlockSpaceManager):
|
||||
AllocStatus: The AllocStatus for swapping in/out the given
|
||||
sequence_group on to the 'device'.
|
||||
"""
|
||||
blocks = self._get_blocks_for_swap(seq_group, status)
|
||||
num_blocks_touched = self.block_allocator.get_num_blocks_touched(
|
||||
blocks, device, num_lookahead_slots)
|
||||
# First determine the number of blocks that will be touched by this
|
||||
# swap. Then verify if there are available blocks in the device
|
||||
# to perform the swap.
|
||||
num_blocks_touched = 0
|
||||
blocks: List[Block] = []
|
||||
for seq in seq_group.get_seqs(status=status):
|
||||
block_table = self.block_tables[seq.seq_id]
|
||||
if block_table.blocks is not None:
|
||||
# Compute the number blocks to touch for the tokens to be
|
||||
# appended. This does NOT include the full blocks that need
|
||||
# to be touched for the swap.
|
||||
num_blocks_touched += \
|
||||
block_table.get_num_blocks_touched_by_append_slots(
|
||||
block_table.get_unseen_token_ids(seq.get_token_ids()),
|
||||
num_lookahead_slots=num_lookahead_slots)
|
||||
blocks.extend(block_table.blocks)
|
||||
# Compute the number of full blocks to touch and add it to the
|
||||
# existing count of blocks to touch.
|
||||
num_blocks_touched += self.block_allocator.get_num_full_blocks_touched(
|
||||
blocks, device=device)
|
||||
|
||||
watermark_blocks = 0
|
||||
if device == Device.GPU:
|
||||
watermark_blocks = self.watermark_blocks
|
||||
|
||||
if self.block_allocator.get_num_total_blocks(
|
||||
device) < num_blocks_touched:
|
||||
return AllocStatus.NEVER
|
||||
@@ -484,23 +502,3 @@ class BlockSpaceManagerV2(BlockSpaceManager):
|
||||
return AllocStatus.OK
|
||||
else:
|
||||
return AllocStatus.LATER
|
||||
|
||||
def _get_blocks_for_swap(self, seq_group: SequenceGroup,
|
||||
status: SequenceStatus) -> List[Block]:
|
||||
"""Returns the list of blocks those are touched by the seq_group
|
||||
|
||||
Args:
|
||||
sequence_group (SequenceGroup): The sequence group to swap in.
|
||||
status (SequenceStatus): The status of sequence which is needed
|
||||
for action. RUNNING for swap out and SWAPPED for swap in
|
||||
|
||||
Returns:
|
||||
The list of blocks those are touched by the seq_group.
|
||||
"""
|
||||
blocks: Dict[int, List[Block]] = {}
|
||||
for seq in seq_group.get_seqs(status=status):
|
||||
block_table = self.block_tables[seq.seq_id]
|
||||
if block_table.blocks is not None:
|
||||
blocks[seq.seq_id] = block_table.blocks
|
||||
combined_blocks = list(chain(*blocks.values()))
|
||||
return combined_blocks
|
||||
|
||||
Reference in New Issue
Block a user