[Core][Optimization] change python dict to pytorch tensor for blocks to swap (#4659)
This commit is contained in:
@@ -39,7 +39,7 @@ class AttentionBackend(ABC):
|
||||
def swap_blocks(
|
||||
src_kv_cache: torch.Tensor,
|
||||
dst_kv_cache: torch.Tensor,
|
||||
src_to_dst: Dict[int, int],
|
||||
src_to_dst: torch.Tensor,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ XFormers backend. The duplicated code will be removed once we use flash-attn or
|
||||
flashinfer for all the attention operations.
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List, Optional, Tuple, Type
|
||||
from typing import List, Optional, Tuple, Type
|
||||
|
||||
import torch
|
||||
from flash_attn import flash_attn_varlen_func
|
||||
@@ -45,7 +45,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
def swap_blocks(
|
||||
src_kv_cache: torch.Tensor,
|
||||
dst_kv_cache: torch.Tensor,
|
||||
src_to_dst: Dict[int, int],
|
||||
src_to_dst: torch.Tensor,
|
||||
) -> None:
|
||||
PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ class FlashInferBackend(AttentionBackend):
|
||||
def swap_blocks(
|
||||
src_kv_cache: torch.Tensor,
|
||||
dst_kv_cache: torch.Tensor,
|
||||
src_to_dst: Dict[int, int],
|
||||
src_to_dst: torch.Tensor,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Attention layer ROCm GPUs."""
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List, Optional, Tuple, Type
|
||||
from typing import List, Optional, Tuple, Type
|
||||
|
||||
import torch
|
||||
|
||||
@@ -43,7 +43,7 @@ class ROCmFlashAttentionBackend(AttentionBackend):
|
||||
def swap_blocks(
|
||||
src_kv_cache: torch.Tensor,
|
||||
dst_kv_cache: torch.Tensor,
|
||||
src_to_dst: Dict[int, int],
|
||||
src_to_dst: torch.Tensor,
|
||||
) -> None:
|
||||
PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
""" Attention layer with torch scaled_dot_product_attention
|
||||
and PagedAttention."""
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List, Optional, Tuple, Type
|
||||
from typing import List, Optional, Tuple, Type
|
||||
|
||||
import torch
|
||||
from torch.nn.functional import scaled_dot_product_attention
|
||||
@@ -41,7 +41,7 @@ class TorchSDPABackend(AttentionBackend):
|
||||
def swap_blocks(
|
||||
src_kv_cache: torch.Tensor,
|
||||
dst_kv_cache: torch.Tensor,
|
||||
src_to_dst: Dict[int, int],
|
||||
src_to_dst: torch.Tensor,
|
||||
) -> None:
|
||||
PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
@@ -196,7 +196,7 @@ class PagedAttention:
|
||||
def swap_blocks(
|
||||
src_kv_cache: torch.Tensor,
|
||||
dst_kv_cache: torch.Tensor,
|
||||
src_to_dst: Dict[int, int],
|
||||
src_to_dst: torch.Tensor,
|
||||
) -> None:
|
||||
src_key_cache = src_kv_cache[0]
|
||||
dst_key_cache = dst_kv_cache[0]
|
||||
|
||||
@@ -473,11 +473,12 @@ class BlockSpaceManagerV1(BlockSpaceManager):
|
||||
|
||||
def swap_in(self,
|
||||
seq_group: SequenceGroup,
|
||||
num_lookahead_slots: int = 0) -> Dict[int, int]:
|
||||
num_lookahead_slots: int = 0) -> List[Tuple[int, int]]:
|
||||
assert (num_lookahead_slots == 0
|
||||
), "BlockSpaceManagerV1 does not support lookahead allocation"
|
||||
|
||||
# CPU block -> GPU block.
|
||||
# dict is efficient in lookup `if cpu_block in mapping`
|
||||
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
|
||||
for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
|
||||
new_block_table: BlockTable = []
|
||||
@@ -500,14 +501,16 @@ class BlockSpaceManagerV1(BlockSpaceManager):
|
||||
cpu_block.block_number: gpu_block.block_number
|
||||
for cpu_block, gpu_block in mapping.items()
|
||||
}
|
||||
return block_number_mapping
|
||||
# convert to list of tuples once here
|
||||
return list(block_number_mapping.items())
|
||||
|
||||
def can_swap_out(self, seq_group: SequenceGroup) -> bool:
|
||||
blocks = self._get_physical_blocks(seq_group)
|
||||
return len(blocks) <= self.cpu_allocator.get_num_free_blocks()
|
||||
|
||||
def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
|
||||
def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
|
||||
# GPU block -> CPU block.
|
||||
# dict is efficient in lookup `if gpu_block in mapping`
|
||||
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
|
||||
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
|
||||
new_block_table: BlockTable = []
|
||||
@@ -530,7 +533,8 @@ class BlockSpaceManagerV1(BlockSpaceManager):
|
||||
gpu_block.block_number: cpu_block.block_number
|
||||
for gpu_block, cpu_block in mapping.items()
|
||||
}
|
||||
return block_number_mapping
|
||||
# convert to list of tuples once here
|
||||
return list(block_number_mapping.items())
|
||||
|
||||
def _free_block_table(self, block_table: BlockTable) -> None:
|
||||
# when using a sliding window, each seq will only use up
|
||||
|
||||
@@ -243,13 +243,13 @@ class BlockSpaceManagerV2(BlockSpaceManager):
|
||||
return AllocStatus.LATER
|
||||
|
||||
def swap_in(self, seq_group: SequenceGroup,
|
||||
num_lookahead_slots: int) -> Dict[int, int]:
|
||||
num_lookahead_slots: int) -> List[Tuple[int, int]]:
|
||||
raise NotImplementedError
|
||||
|
||||
def can_swap_out(self, seq_group: SequenceGroup) -> bool:
|
||||
return False
|
||||
|
||||
def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
|
||||
def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_num_free_gpu_blocks(self) -> int:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import enum
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, List
|
||||
from typing import List
|
||||
from typing import Sequence as GenericSequence
|
||||
from typing import Tuple
|
||||
|
||||
@@ -69,7 +69,7 @@ class BlockSpaceManager(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def swap_in(self, seq_group: SequenceGroup,
|
||||
num_lookahead_slots: int) -> Dict[int, int]:
|
||||
num_lookahead_slots: int) -> List[Tuple[int, int]]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
@@ -77,7 +77,7 @@ class BlockSpaceManager(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
|
||||
def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
|
||||
+16
-16
@@ -117,10 +117,10 @@ class SchedulerOutputs:
|
||||
num_prefill_groups: int
|
||||
# Total number of batched tokens.
|
||||
num_batched_tokens: int
|
||||
# Blocks to swap in. Dict of CPU -> GPU block number.
|
||||
blocks_to_swap_in: Dict[int, int]
|
||||
# Blocks to swap out. Dict of GPU -> CPU block number.
|
||||
blocks_to_swap_out: Dict[int, int]
|
||||
# Blocks to swap in. List of CPU -> GPU block number.
|
||||
blocks_to_swap_in: List[Tuple[int, int]]
|
||||
# Blocks to swap out. List of GPU -> CPU block number.
|
||||
blocks_to_swap_out: List[Tuple[int, int]]
|
||||
# Blocks to copy. Source to dest block.
|
||||
blocks_to_copy: List[Tuple[int, int]]
|
||||
# Sequence groups that are going to be ignored.
|
||||
@@ -174,7 +174,7 @@ class SchedulerRunningOutputs:
|
||||
# Sequences that are swapped out.
|
||||
swapped_out: List[SequenceGroup]
|
||||
# The blocks to swap out.
|
||||
blocks_to_swap_out: Dict[int, int]
|
||||
blocks_to_swap_out: List[Tuple[int, int]]
|
||||
# The blocks to copy.
|
||||
blocks_to_copy: List[Tuple[int, int]]
|
||||
# The number of slots for lookahead decoding.
|
||||
@@ -187,7 +187,7 @@ class SchedulerRunningOutputs:
|
||||
prefill_seq_groups=[],
|
||||
preempted=[],
|
||||
swapped_out=[],
|
||||
blocks_to_swap_out={},
|
||||
blocks_to_swap_out=[],
|
||||
blocks_to_copy=[],
|
||||
num_lookahead_slots=0,
|
||||
)
|
||||
@@ -206,7 +206,7 @@ class SchedulerSwappedInOutputs:
|
||||
# phase. I.e., it means the prefill has been chunked.
|
||||
prefill_seq_groups: List[SequenceGroup]
|
||||
# The blocks to swap in.
|
||||
blocks_to_swap_in: Dict[int, int]
|
||||
blocks_to_swap_in: List[Tuple[int, int]]
|
||||
# The blocks to copy.
|
||||
blocks_to_copy: List[Tuple[int, int]]
|
||||
# The number of slots for lookahead decoding.
|
||||
@@ -219,7 +219,7 @@ class SchedulerSwappedInOutputs:
|
||||
return SchedulerSwappedInOutputs(
|
||||
decode_seq_groups=[],
|
||||
prefill_seq_groups=[],
|
||||
blocks_to_swap_in={},
|
||||
blocks_to_swap_in=[],
|
||||
blocks_to_copy=[],
|
||||
num_lookahead_slots=0,
|
||||
infeasible_seq_groups=[],
|
||||
@@ -392,7 +392,7 @@ class Scheduler:
|
||||
scheduling and SchedulerRunningOutputs.
|
||||
"""
|
||||
# Blocks that need to be swapped or copied before model execution.
|
||||
blocks_to_swap_out: Dict[int, int] = {}
|
||||
blocks_to_swap_out: List[Tuple[int, int]] = []
|
||||
blocks_to_copy: List[Tuple[int, int]] = []
|
||||
|
||||
decode_seq_groups: List[ScheduledSequenceGroup] = []
|
||||
@@ -509,7 +509,7 @@ class Scheduler:
|
||||
SchedulerSwappedInOutputs.
|
||||
"""
|
||||
# Blocks that need to be swapped or copied before model execution.
|
||||
blocks_to_swap_in: Dict[int, int] = {}
|
||||
blocks_to_swap_in: List[Tuple[int, int]] = []
|
||||
blocks_to_copy: List[Tuple[int, int]] = []
|
||||
decode_seq_groups: List[ScheduledSequenceGroup] = []
|
||||
prefill_seq_groups: List[ScheduledSequenceGroup] = []
|
||||
@@ -1032,7 +1032,7 @@ class Scheduler:
|
||||
def _preempt(
|
||||
self,
|
||||
seq_group: SequenceGroup,
|
||||
blocks_to_swap_out: Dict[int, int],
|
||||
blocks_to_swap_out: List[Tuple[int, int]],
|
||||
preemption_mode: Optional[PreemptionMode] = None,
|
||||
) -> PreemptionMode:
|
||||
# If preemption mode is not specified, we determine the mode as follows:
|
||||
@@ -1073,24 +1073,24 @@ class Scheduler:
|
||||
def _preempt_by_swap(
|
||||
self,
|
||||
seq_group: SequenceGroup,
|
||||
blocks_to_swap_out: Dict[int, int],
|
||||
blocks_to_swap_out: List[Tuple[int, int]],
|
||||
) -> None:
|
||||
self._swap_out(seq_group, blocks_to_swap_out)
|
||||
|
||||
def _swap_in(
|
||||
self,
|
||||
seq_group: SequenceGroup,
|
||||
blocks_to_swap_in: Dict[int, int],
|
||||
blocks_to_swap_in: List[Tuple[int, int]],
|
||||
) -> None:
|
||||
mapping = self.block_manager.swap_in(seq_group)
|
||||
blocks_to_swap_in.update(mapping)
|
||||
blocks_to_swap_in.extend(mapping)
|
||||
for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
|
||||
seq.status = SequenceStatus.RUNNING
|
||||
|
||||
def _swap_out(
|
||||
self,
|
||||
seq_group: SequenceGroup,
|
||||
blocks_to_swap_out: Dict[int, int],
|
||||
blocks_to_swap_out: List[Tuple[int, int]],
|
||||
) -> None:
|
||||
if not self.block_manager.can_swap_out(seq_group):
|
||||
# FIXME(woosuk): Abort the sequence group instead of aborting the
|
||||
@@ -1099,7 +1099,7 @@ class Scheduler:
|
||||
"Aborted due to the lack of CPU swap space. Please increase "
|
||||
"the swap space to avoid this error.")
|
||||
mapping = self.block_manager.swap_out(seq_group)
|
||||
blocks_to_swap_out.update(mapping)
|
||||
blocks_to_swap_out.extend(mapping)
|
||||
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
|
||||
seq.status = SequenceStatus.SWAPPED
|
||||
|
||||
|
||||
+4
-4
@@ -741,10 +741,10 @@ class ExecuteModelRequest:
|
||||
"""The model execution request."""
|
||||
# The sequence group metadata list.
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata]
|
||||
# Blocks to swap in. Dict of CPU -> GPU block number.
|
||||
blocks_to_swap_in: Dict[int, int] = field(default_factory=dict)
|
||||
# Blocks to swap out. Dict of GPU -> CPU block number.
|
||||
blocks_to_swap_out: Dict[int, int] = field(default_factory=dict)
|
||||
# Blocks to swap in. List of CPU -> GPU block number.
|
||||
blocks_to_swap_in: List[Tuple[int, int]] = field(default_factory=list)
|
||||
# Blocks to swap out. List of GPU -> CPU block number.
|
||||
blocks_to_swap_out: List[Tuple[int, int]] = field(default_factory=list)
|
||||
# Blocks to copy. Source to dest block.
|
||||
blocks_to_copy: List[Tuple[int, int]] = field(default_factory=list)
|
||||
# The number of slots for lookahead decoding.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""CacheEngine class for managing the KV cache."""
|
||||
from typing import Dict, List
|
||||
from typing import List
|
||||
|
||||
import torch
|
||||
|
||||
@@ -67,12 +67,12 @@ class CacheEngine:
|
||||
device=device))
|
||||
return kv_cache
|
||||
|
||||
def swap_in(self, src_to_dst: Dict[int, int]) -> None:
|
||||
def swap_in(self, src_to_dst: torch.Tensor) -> None:
|
||||
for i in range(self.num_layers):
|
||||
self.attn_backend.swap_blocks(self.cpu_cache[i], self.gpu_cache[i],
|
||||
src_to_dst)
|
||||
|
||||
def swap_out(self, src_to_dst: Dict[int, int]) -> None:
|
||||
def swap_out(self, src_to_dst: torch.Tensor) -> None:
|
||||
for i in range(self.num_layers):
|
||||
self.attn_backend.swap_blocks(self.gpu_cache[i], self.cpu_cache[i],
|
||||
src_to_dst)
|
||||
|
||||
+20
-7
@@ -195,15 +195,14 @@ class Worker(WorkerBase):
|
||||
|
||||
def cache_swap(
|
||||
self,
|
||||
blocks_to_swap_in: Dict[int, int],
|
||||
blocks_to_swap_out: Dict[int, int],
|
||||
blocks_to_swap_in: torch.Tensor,
|
||||
blocks_to_swap_out: torch.Tensor,
|
||||
blocks_to_copy: torch.Tensor,
|
||||
) -> None:
|
||||
# Issue cache operations.
|
||||
# TODO(woosuk): Profile swapping overhead and optimize if needed.
|
||||
if blocks_to_swap_in:
|
||||
if blocks_to_swap_in.numel() > 0:
|
||||
self.cache_engine.swap_in(blocks_to_swap_in)
|
||||
if blocks_to_swap_out:
|
||||
if blocks_to_swap_out.numel() > 0:
|
||||
self.cache_engine.swap_out(blocks_to_swap_out)
|
||||
if blocks_to_copy.numel() > 0:
|
||||
self.cache_engine.copy(blocks_to_copy)
|
||||
@@ -219,12 +218,26 @@ class Worker(WorkerBase):
|
||||
else:
|
||||
seq_group_metadata_list = execute_model_req.seq_group_metadata_list
|
||||
|
||||
blocks_to_swap_in: torch.Tensor
|
||||
blocks_to_swap_out: torch.Tensor
|
||||
blocks_to_copy: torch.Tensor
|
||||
if self.is_driver_worker:
|
||||
assert seq_group_metadata_list is not None
|
||||
assert execute_model_req is not None
|
||||
num_seq_groups = len(seq_group_metadata_list)
|
||||
blocks_to_swap_in = execute_model_req.blocks_to_swap_in
|
||||
blocks_to_swap_out = execute_model_req.blocks_to_swap_out
|
||||
# `blocks_to_swap_in` and `blocks_to_swap_out` are cpu tensors.
|
||||
# they contain parameters to launch cudamemcpyasync.
|
||||
blocks_to_swap_in = torch.tensor(
|
||||
execute_model_req.blocks_to_swap_in,
|
||||
device="cpu",
|
||||
dtype=torch.int64).view(-1, 2)
|
||||
blocks_to_swap_out = torch.tensor(
|
||||
execute_model_req.blocks_to_swap_out,
|
||||
device="cpu",
|
||||
dtype=torch.int64).view(-1, 2)
|
||||
# `blocks_to_copy` is a gpu tensor. The src and tgt of
|
||||
# blocks to copy are in the same device, and `blocks_to_copy`
|
||||
# can be used directly within cuda kernels.
|
||||
blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
|
||||
device=self.device,
|
||||
dtype=torch.int64).view(-1, 2)
|
||||
|
||||
Reference in New Issue
Block a user