[Core][Optimization] change python dict to pytorch tensor for blocks to swap (#4659)
This commit is contained in:
@@ -473,11 +473,12 @@ class BlockSpaceManagerV1(BlockSpaceManager):
|
||||
|
||||
def swap_in(self,
|
||||
seq_group: SequenceGroup,
|
||||
num_lookahead_slots: int = 0) -> Dict[int, int]:
|
||||
num_lookahead_slots: int = 0) -> List[Tuple[int, int]]:
|
||||
assert (num_lookahead_slots == 0
|
||||
), "BlockSpaceManagerV1 does not support lookahead allocation"
|
||||
|
||||
# CPU block -> GPU block.
|
||||
# dict is efficient in lookup `if cpu_block in mapping`
|
||||
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
|
||||
for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
|
||||
new_block_table: BlockTable = []
|
||||
@@ -500,14 +501,16 @@ class BlockSpaceManagerV1(BlockSpaceManager):
|
||||
cpu_block.block_number: gpu_block.block_number
|
||||
for cpu_block, gpu_block in mapping.items()
|
||||
}
|
||||
return block_number_mapping
|
||||
# convert to list of tuples once here
|
||||
return list(block_number_mapping.items())
|
||||
|
||||
def can_swap_out(self, seq_group: SequenceGroup) -> bool:
|
||||
blocks = self._get_physical_blocks(seq_group)
|
||||
return len(blocks) <= self.cpu_allocator.get_num_free_blocks()
|
||||
|
||||
def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
|
||||
def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
|
||||
# GPU block -> CPU block.
|
||||
# dict is efficient in lookup `if gpu_block in mapping`
|
||||
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
|
||||
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
|
||||
new_block_table: BlockTable = []
|
||||
@@ -530,7 +533,8 @@ class BlockSpaceManagerV1(BlockSpaceManager):
|
||||
gpu_block.block_number: cpu_block.block_number
|
||||
for gpu_block, cpu_block in mapping.items()
|
||||
}
|
||||
return block_number_mapping
|
||||
# convert to list of tuples once here
|
||||
return list(block_number_mapping.items())
|
||||
|
||||
def _free_block_table(self, block_table: BlockTable) -> None:
|
||||
# when using a sliding window, each seq will only use up
|
||||
|
||||
@@ -243,13 +243,13 @@ class BlockSpaceManagerV2(BlockSpaceManager):
|
||||
return AllocStatus.LATER
|
||||
|
||||
def swap_in(self, seq_group: SequenceGroup,
|
||||
num_lookahead_slots: int) -> Dict[int, int]:
|
||||
num_lookahead_slots: int) -> List[Tuple[int, int]]:
|
||||
raise NotImplementedError
|
||||
|
||||
def can_swap_out(self, seq_group: SequenceGroup) -> bool:
|
||||
return False
|
||||
|
||||
def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
|
||||
def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_num_free_gpu_blocks(self) -> int:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import enum
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, List
|
||||
from typing import List
|
||||
from typing import Sequence as GenericSequence
|
||||
from typing import Tuple
|
||||
|
||||
@@ -69,7 +69,7 @@ class BlockSpaceManager(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def swap_in(self, seq_group: SequenceGroup,
|
||||
num_lookahead_slots: int) -> Dict[int, int]:
|
||||
num_lookahead_slots: int) -> List[Tuple[int, int]]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
@@ -77,7 +77,7 @@ class BlockSpaceManager(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
|
||||
def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
|
||||
+16
-16
@@ -117,10 +117,10 @@ class SchedulerOutputs:
|
||||
num_prefill_groups: int
|
||||
# Total number of batched tokens.
|
||||
num_batched_tokens: int
|
||||
# Blocks to swap in. Dict of CPU -> GPU block number.
|
||||
blocks_to_swap_in: Dict[int, int]
|
||||
# Blocks to swap out. Dict of GPU -> CPU block number.
|
||||
blocks_to_swap_out: Dict[int, int]
|
||||
# Blocks to swap in. List of CPU -> GPU block number.
|
||||
blocks_to_swap_in: List[Tuple[int, int]]
|
||||
# Blocks to swap out. List of GPU -> CPU block number.
|
||||
blocks_to_swap_out: List[Tuple[int, int]]
|
||||
# Blocks to copy. Source to dest block.
|
||||
blocks_to_copy: List[Tuple[int, int]]
|
||||
# Sequence groups that are going to be ignored.
|
||||
@@ -174,7 +174,7 @@ class SchedulerRunningOutputs:
|
||||
# Sequences that are swapped out.
|
||||
swapped_out: List[SequenceGroup]
|
||||
# The blocks to swap out.
|
||||
blocks_to_swap_out: Dict[int, int]
|
||||
blocks_to_swap_out: List[Tuple[int, int]]
|
||||
# The blocks to copy.
|
||||
blocks_to_copy: List[Tuple[int, int]]
|
||||
# The number of slots for lookahead decoding.
|
||||
@@ -187,7 +187,7 @@ class SchedulerRunningOutputs:
|
||||
prefill_seq_groups=[],
|
||||
preempted=[],
|
||||
swapped_out=[],
|
||||
blocks_to_swap_out={},
|
||||
blocks_to_swap_out=[],
|
||||
blocks_to_copy=[],
|
||||
num_lookahead_slots=0,
|
||||
)
|
||||
@@ -206,7 +206,7 @@ class SchedulerSwappedInOutputs:
|
||||
# phase. I.e., it means the prefill has been chunked.
|
||||
prefill_seq_groups: List[SequenceGroup]
|
||||
# The blocks to swap in.
|
||||
blocks_to_swap_in: Dict[int, int]
|
||||
blocks_to_swap_in: List[Tuple[int, int]]
|
||||
# The blocks to copy.
|
||||
blocks_to_copy: List[Tuple[int, int]]
|
||||
# The number of slots for lookahead decoding.
|
||||
@@ -219,7 +219,7 @@ class SchedulerSwappedInOutputs:
|
||||
return SchedulerSwappedInOutputs(
|
||||
decode_seq_groups=[],
|
||||
prefill_seq_groups=[],
|
||||
blocks_to_swap_in={},
|
||||
blocks_to_swap_in=[],
|
||||
blocks_to_copy=[],
|
||||
num_lookahead_slots=0,
|
||||
infeasible_seq_groups=[],
|
||||
@@ -392,7 +392,7 @@ class Scheduler:
|
||||
scheduling and SchedulerRunningOutputs.
|
||||
"""
|
||||
# Blocks that need to be swapped or copied before model execution.
|
||||
blocks_to_swap_out: Dict[int, int] = {}
|
||||
blocks_to_swap_out: List[Tuple[int, int]] = []
|
||||
blocks_to_copy: List[Tuple[int, int]] = []
|
||||
|
||||
decode_seq_groups: List[ScheduledSequenceGroup] = []
|
||||
@@ -509,7 +509,7 @@ class Scheduler:
|
||||
SchedulerSwappedInOutputs.
|
||||
"""
|
||||
# Blocks that need to be swapped or copied before model execution.
|
||||
blocks_to_swap_in: Dict[int, int] = {}
|
||||
blocks_to_swap_in: List[Tuple[int, int]] = []
|
||||
blocks_to_copy: List[Tuple[int, int]] = []
|
||||
decode_seq_groups: List[ScheduledSequenceGroup] = []
|
||||
prefill_seq_groups: List[ScheduledSequenceGroup] = []
|
||||
@@ -1032,7 +1032,7 @@ class Scheduler:
|
||||
def _preempt(
|
||||
self,
|
||||
seq_group: SequenceGroup,
|
||||
blocks_to_swap_out: Dict[int, int],
|
||||
blocks_to_swap_out: List[Tuple[int, int]],
|
||||
preemption_mode: Optional[PreemptionMode] = None,
|
||||
) -> PreemptionMode:
|
||||
# If preemption mode is not specified, we determine the mode as follows:
|
||||
@@ -1073,24 +1073,24 @@ class Scheduler:
|
||||
def _preempt_by_swap(
|
||||
self,
|
||||
seq_group: SequenceGroup,
|
||||
blocks_to_swap_out: Dict[int, int],
|
||||
blocks_to_swap_out: List[Tuple[int, int]],
|
||||
) -> None:
|
||||
self._swap_out(seq_group, blocks_to_swap_out)
|
||||
|
||||
def _swap_in(
|
||||
self,
|
||||
seq_group: SequenceGroup,
|
||||
blocks_to_swap_in: Dict[int, int],
|
||||
blocks_to_swap_in: List[Tuple[int, int]],
|
||||
) -> None:
|
||||
mapping = self.block_manager.swap_in(seq_group)
|
||||
blocks_to_swap_in.update(mapping)
|
||||
blocks_to_swap_in.extend(mapping)
|
||||
for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
|
||||
seq.status = SequenceStatus.RUNNING
|
||||
|
||||
def _swap_out(
|
||||
self,
|
||||
seq_group: SequenceGroup,
|
||||
blocks_to_swap_out: Dict[int, int],
|
||||
blocks_to_swap_out: List[Tuple[int, int]],
|
||||
) -> None:
|
||||
if not self.block_manager.can_swap_out(seq_group):
|
||||
# FIXME(woosuk): Abort the sequence group instead of aborting the
|
||||
@@ -1099,7 +1099,7 @@ class Scheduler:
|
||||
"Aborted due to the lack of CPU swap space. Please increase "
|
||||
"the swap space to avoid this error.")
|
||||
mapping = self.block_manager.swap_out(seq_group)
|
||||
blocks_to_swap_out.update(mapping)
|
||||
blocks_to_swap_out.extend(mapping)
|
||||
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
|
||||
seq.status = SequenceStatus.SWAPPED
|
||||
|
||||
|
||||
Reference in New Issue
Block a user