[kv_offload+HMA][7/N]: Support register_kv_caches for hybrid models (#37853)
Signed-off-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections import defaultdict
|
||||
from dataclasses import replace
|
||||
|
||||
import torch
|
||||
|
||||
@@ -18,7 +19,17 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.kv_offload.spec import OffloadingSpec
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.kv_offload.spec import (
|
||||
CanonicalKVCacheRef,
|
||||
CanonicalKVCaches,
|
||||
CanonicalKVCacheTensor,
|
||||
OffloadingSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.worker.worker import (
|
||||
OffloadingWorker,
|
||||
TransferSpec,
|
||||
@@ -53,17 +64,13 @@ class OffloadingConnectorWorker:
|
||||
self._job_counter = job_id + 1
|
||||
return job_id
|
||||
|
||||
def _register_handlers(
|
||||
self,
|
||||
kv_caches: dict[str, torch.Tensor],
|
||||
attn_backends: dict[str, type[AttentionBackend]],
|
||||
):
|
||||
for src_cls, dst_cls, handler in self.spec.get_handlers(
|
||||
kv_caches, attn_backends
|
||||
):
|
||||
def _register_handlers(self, kv_caches: CanonicalKVCaches):
|
||||
for src_cls, dst_cls, handler in self.spec.get_handlers(kv_caches):
|
||||
self.worker.register_handler(src_cls, dst_cls, handler)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
def register_kv_caches(
|
||||
self, kv_caches: dict[str, torch.Tensor | list[torch.Tensor]]
|
||||
):
|
||||
layer_names = list(kv_caches.keys())
|
||||
layers = get_layers_from_vllm_config(
|
||||
self.spec.vllm_config,
|
||||
@@ -73,16 +80,221 @@ class OffloadingConnectorWorker:
|
||||
attn_backends = {
|
||||
layer_name: layers[layer_name].get_attn_backend()
|
||||
for layer_name in layer_names
|
||||
if layer_name in layers
|
||||
}
|
||||
self._register_handlers(kv_caches, attn_backends)
|
||||
|
||||
# layer_name -> list of matching KV cache tensors
|
||||
# such that each tensor starts with the num_blocks dimension.
|
||||
# FlashAttention layers which use the (2, num_blocks, ...) layout
|
||||
# will possibly map to 2 tensors, one per K and one per V.
|
||||
# All other layers will probably map to a single tensor.
|
||||
tensors_per_block: dict[str, tuple[torch.Tensor, ...]] = {}
|
||||
# layer_name -> size of (un-padded) page in bytes
|
||||
unpadded_page_size_bytes: dict[str, int] = {}
|
||||
# layer_name -> size of page in bytes
|
||||
page_size_bytes: dict[str, int] = {}
|
||||
for kv_cache_group in self.spec.kv_cache_config.kv_cache_groups:
|
||||
group_layer_names = kv_cache_group.layer_names
|
||||
group_kv_cache_spec = kv_cache_group.kv_cache_spec
|
||||
if isinstance(group_kv_cache_spec, UniformTypeKVCacheSpecs):
|
||||
per_layer_specs = group_kv_cache_spec.kv_cache_specs
|
||||
else:
|
||||
per_layer_specs = {}
|
||||
for layer_name in group_layer_names:
|
||||
layer_kv_cache_spec = per_layer_specs.get(
|
||||
layer_name, group_kv_cache_spec
|
||||
)
|
||||
if isinstance(layer_kv_cache_spec, AttentionSpec):
|
||||
layer_kv_cache = kv_caches[layer_name]
|
||||
assert isinstance(layer_kv_cache, torch.Tensor)
|
||||
assert layer_kv_cache.storage_offset() == 0
|
||||
|
||||
# get the logical dimension for num_blocks
|
||||
test_shape = attn_backends[layer_name].get_kv_cache_shape(
|
||||
num_blocks=1234,
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=256,
|
||||
)
|
||||
num_blocks_logical_dim = test_shape.index(1234)
|
||||
|
||||
# sort the logical dimensions by stride (high to low)
|
||||
# to get a physical-to-logical mapping:
|
||||
# physical_to_logical[physical_pos] = logical_dim
|
||||
logical_strides = layer_kv_cache.stride()
|
||||
physical_to_logical = sorted(
|
||||
range(len(logical_strides)),
|
||||
key=lambda idx: logical_strides[idx],
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
num_blocks_physical_dim = physical_to_logical.index(
|
||||
num_blocks_logical_dim
|
||||
)
|
||||
if num_blocks_physical_dim == 0:
|
||||
num_blocks = layer_kv_cache.shape[num_blocks_logical_dim]
|
||||
storage = layer_kv_cache.untyped_storage()
|
||||
page = layer_kv_cache_spec.page_size_bytes
|
||||
tensors_per_block[layer_name] = (
|
||||
torch.tensor(
|
||||
[],
|
||||
dtype=torch.int8,
|
||||
device=layer_kv_cache.device,
|
||||
)
|
||||
.set_(storage)
|
||||
.view(num_blocks, page),
|
||||
)
|
||||
page_size_bytes[layer_name] = (
|
||||
layer_kv_cache_spec.page_size_bytes
|
||||
)
|
||||
else:
|
||||
# Flash Attention case: (2, num_blocks, ...)
|
||||
assert test_shape[0] == 2
|
||||
assert physical_to_logical[0] == 0
|
||||
assert num_blocks_physical_dim == 1
|
||||
|
||||
# unbind the tensor to separate K and V tensors
|
||||
num_blocks = layer_kv_cache.shape[num_blocks_logical_dim]
|
||||
half_page_size = layer_kv_cache_spec.page_size_bytes // 2
|
||||
storage = layer_kv_cache.untyped_storage()
|
||||
raw = (
|
||||
torch.tensor(
|
||||
[],
|
||||
dtype=torch.int8,
|
||||
device=layer_kv_cache.device,
|
||||
)
|
||||
.set_(storage)
|
||||
.view(2, num_blocks, half_page_size)
|
||||
)
|
||||
tensors_per_block[layer_name] = tuple(raw.unbind(0))
|
||||
|
||||
page_size_bytes[layer_name] = half_page_size
|
||||
|
||||
unpadded_page_size_bytes[layer_name] = page_size_bytes[layer_name]
|
||||
|
||||
elif isinstance(layer_kv_cache_spec, MambaSpec):
|
||||
state_tensors = kv_caches[layer_name]
|
||||
assert isinstance(state_tensors, list)
|
||||
|
||||
# re-construct the raw (num_blocks, page_size) tensor
|
||||
# from the first state tensor
|
||||
assert len(state_tensors) > 0
|
||||
first_state_tensor = state_tensors[0]
|
||||
assert first_state_tensor.storage_offset() == 0
|
||||
num_blocks = first_state_tensor.shape[0]
|
||||
tensor = (
|
||||
torch.tensor(
|
||||
[],
|
||||
dtype=torch.int8,
|
||||
device=first_state_tensor.device,
|
||||
)
|
||||
.set_(first_state_tensor.untyped_storage())
|
||||
.view((num_blocks, layer_kv_cache_spec.page_size_bytes))
|
||||
)
|
||||
tensors_per_block[layer_name] = (tensor,)
|
||||
|
||||
page_size_bytes[layer_name] = layer_kv_cache_spec.page_size_bytes
|
||||
unpadded_page_size_bytes[layer_name] = replace(
|
||||
layer_kv_cache_spec, page_size_padded=None
|
||||
).page_size_bytes
|
||||
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
block_tensors: list[CanonicalKVCacheTensor] = []
|
||||
block_data_refs: dict[str, list[CanonicalKVCacheRef]] = defaultdict(list)
|
||||
for kv_cache_tensor in self.spec.kv_cache_config.kv_cache_tensors:
|
||||
tensor_layer_names = kv_cache_tensor.shared_by
|
||||
|
||||
# verify all layers in the group reference the exact same tensors
|
||||
assert len({len(tensors_per_block[n]) for n in tensor_layer_names}) == 1
|
||||
assert (
|
||||
len({tensors_per_block[n][0].data_ptr() for n in tensor_layer_names})
|
||||
== 1
|
||||
)
|
||||
assert (
|
||||
len({tensors_per_block[n][0].stride() for n in tensor_layer_names}) == 1
|
||||
)
|
||||
|
||||
# pick the first layer to represent the group
|
||||
first_layer_name = tensor_layer_names[0]
|
||||
for tensor in tensors_per_block[first_layer_name]:
|
||||
block_tensors.append(
|
||||
CanonicalKVCacheTensor(
|
||||
tensor=tensor,
|
||||
page_size_bytes=page_size_bytes[first_layer_name],
|
||||
)
|
||||
)
|
||||
|
||||
curr_tensor_idx = len(block_tensors) - 1
|
||||
for layer_name in tensor_layer_names:
|
||||
block_data_refs[layer_name].append(
|
||||
CanonicalKVCacheRef(
|
||||
tensor_idx=curr_tensor_idx,
|
||||
page_size_bytes=(unpadded_page_size_bytes[layer_name]),
|
||||
)
|
||||
)
|
||||
|
||||
group_data_refs: list[list[CanonicalKVCacheRef]] = []
|
||||
for kv_cache_group in self.spec.kv_cache_config.kv_cache_groups:
|
||||
group_refs: list[CanonicalKVCacheRef] = []
|
||||
for layer_name in kv_cache_group.layer_names:
|
||||
group_refs += block_data_refs[layer_name]
|
||||
group_data_refs.append(group_refs)
|
||||
|
||||
canonical_kv_caches = CanonicalKVCaches(
|
||||
tensors=block_tensors,
|
||||
group_data_refs=group_data_refs,
|
||||
)
|
||||
|
||||
self._register_handlers(canonical_kv_caches)
|
||||
|
||||
def register_cross_layers_kv_cache(
|
||||
self, kv_cache: torch.Tensor, attn_backend: type[AttentionBackend]
|
||||
):
|
||||
cross_layer_name = "ALL_LAYERS"
|
||||
kv_caches = {cross_layer_name: kv_cache}
|
||||
attn_backends = {cross_layer_name: attn_backend}
|
||||
self._register_handlers(kv_caches, attn_backends)
|
||||
# verify that num_blocks is at physical position 0 in the cross-layers
|
||||
# tensor layout.
|
||||
test_shape = attn_backend.get_kv_cache_shape(
|
||||
num_blocks=1234, block_size=16, num_kv_heads=1, head_size=256
|
||||
)
|
||||
num_blocks_logical_dim = test_shape.index(1234) + 1
|
||||
physical_to_logical = attn_backend.get_kv_cache_stride_order(
|
||||
include_num_layers_dimension=True
|
||||
)
|
||||
num_blocks_physical_dim = physical_to_logical.index(num_blocks_logical_dim)
|
||||
assert num_blocks_physical_dim == 0
|
||||
|
||||
kv_cache_groups = self.spec.kv_cache_config.kv_cache_groups
|
||||
assert len(kv_cache_groups) == 1
|
||||
kv_cache_spec = kv_cache_groups[0].kv_cache_spec
|
||||
num_layers = len(kv_cache_groups[0].layer_names)
|
||||
page_size_bytes = kv_cache_spec.page_size_bytes * num_layers
|
||||
|
||||
assert kv_cache.storage_offset() == 0
|
||||
storage = kv_cache.untyped_storage()
|
||||
assert len(storage) % page_size_bytes == 0
|
||||
num_blocks = len(storage) // page_size_bytes
|
||||
tensor = (
|
||||
torch.tensor(
|
||||
[],
|
||||
dtype=torch.int8,
|
||||
device=kv_cache.device,
|
||||
)
|
||||
.set_(storage)
|
||||
.view(num_blocks, page_size_bytes)
|
||||
)
|
||||
kv_cache_tensor = CanonicalKVCacheTensor(
|
||||
tensor=tensor, page_size_bytes=page_size_bytes
|
||||
)
|
||||
# in cross layers layout, there's currently only a single group
|
||||
kv_cache_data_ref = CanonicalKVCacheRef(
|
||||
tensor_idx=0, page_size_bytes=page_size_bytes
|
||||
)
|
||||
canonical_kv_caches = CanonicalKVCaches(
|
||||
tensors=[kv_cache_tensor], group_data_refs=[[kv_cache_data_ref]]
|
||||
)
|
||||
|
||||
self._register_handlers(canonical_kv_caches)
|
||||
|
||||
def handle_preemptions(self, kv_connector_metadata: OffloadingConnectorMetadata):
|
||||
for job_id, transfer_spec in self._unsubmitted_store_jobs:
|
||||
|
||||
@@ -78,9 +78,10 @@ def get_kv_cache_layout():
|
||||
return cache_layout
|
||||
|
||||
|
||||
def set_kv_cache_layout(cache_layout: KVCacheLayoutType):
|
||||
def set_kv_cache_layout(cache_layout: KVCacheLayoutType | None):
|
||||
global _KV_CACHE_LAYOUT_OVERRIDE
|
||||
_KV_CACHE_LAYOUT_OVERRIDE = cache_layout
|
||||
get_kv_cache_layout.cache_clear()
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -2,17 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterator
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_offload.abstract import LoadStoreSpec, OffloadingManager
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.mediums import CPULoadStoreSpec, GPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.reuse_manager import FilterReusedOffloadingManager
|
||||
from vllm.v1.kv_offload.spec import OffloadingSpec
|
||||
from vllm.v1.kv_offload.spec import CanonicalKVCaches, OffloadingSpec
|
||||
from vllm.v1.kv_offload.worker.cpu_gpu import CpuGpuOffloadingHandlers
|
||||
from vllm.v1.kv_offload.worker.worker import OffloadingHandler
|
||||
|
||||
@@ -90,9 +87,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
return self._manager
|
||||
|
||||
def get_handlers(
|
||||
self,
|
||||
kv_caches: dict[str, torch.Tensor],
|
||||
attn_backends: dict[str, type[AttentionBackend]],
|
||||
self, kv_caches: CanonicalKVCaches
|
||||
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
|
||||
if not self._handlers:
|
||||
if not current_platform.is_cuda_alike():
|
||||
@@ -100,15 +95,10 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
"CPU Offloading is currently only supported on CUDA-alike GPUs"
|
||||
)
|
||||
|
||||
assert len(self.gpu_block_size) == 1
|
||||
gpu_block_size = self.gpu_block_size[0]
|
||||
|
||||
self._handlers = CpuGpuOffloadingHandlers(
|
||||
attn_backends=attn_backends,
|
||||
gpu_block_size=gpu_block_size,
|
||||
cpu_block_size=gpu_block_size * self.block_size_factor,
|
||||
kv_caches=kv_caches,
|
||||
block_size_factor=self.block_size_factor,
|
||||
num_cpu_blocks=self.num_blocks,
|
||||
gpu_caches=kv_caches,
|
||||
)
|
||||
|
||||
assert self._handlers is not None
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.kv_offload.abstract import LoadStoreSpec, OffloadingManager
|
||||
from vllm.v1.kv_offload.worker.worker import OffloadingHandler
|
||||
|
||||
@@ -18,6 +18,56 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CanonicalKVCacheTensor:
|
||||
"""
|
||||
A canonicalized KV cache tensor whose first dimension is num_blocks.
|
||||
|
||||
For attention backends where the raw tensor has num_blocks at a
|
||||
non-leading physical dimension (e.g. FlashAttention's
|
||||
(2, num_blocks, ...) layout), the tensor is split so that each
|
||||
resulting CanonicalKVCacheTensor starts with (num_blocks, ...).
|
||||
"""
|
||||
|
||||
# The KV cache tensor with shape (num_blocks, ...)
|
||||
tensor: torch.Tensor
|
||||
# The (possibly padded) page size per block in bytes
|
||||
page_size_bytes: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class CanonicalKVCacheRef:
|
||||
"""
|
||||
Per-layer (or group of layers) reference to a specific (by index)
|
||||
CanonicalKVCacheTensor and records the un-padded page size used by that layer.
|
||||
"""
|
||||
|
||||
# Index into the list of CanonicalKVCacheTensor objects
|
||||
tensor_idx: int
|
||||
# The un-padded page size per block in bytes
|
||||
page_size_bytes: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class CanonicalKVCaches:
|
||||
"""
|
||||
Canonicalized block-level representation of the KV caches.
|
||||
|
||||
Composed of:
|
||||
- Unique list of KV cache data tensors,
|
||||
each with shape (num_blocks, page_size_in_bytes) and int8 dtype.
|
||||
- Per-group data references of the tensors.
|
||||
i.e. how each KV cache group maps to the tensors.
|
||||
"""
|
||||
|
||||
# Ordered list of unique block tensors, each with shape
|
||||
# (num_blocks, ...).
|
||||
tensors: list[CanonicalKVCacheTensor]
|
||||
# Per-KV-cache-group list of data references that map each layer
|
||||
# in the group to the appropriate entry in the tensors list.
|
||||
group_data_refs: list[list[CanonicalKVCacheRef]]
|
||||
|
||||
|
||||
class OffloadingSpec(ABC):
|
||||
"""Spec for an offloading connector"""
|
||||
|
||||
@@ -73,16 +123,13 @@ class OffloadingSpec(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def get_handlers(
|
||||
self,
|
||||
kv_caches: dict[str, torch.Tensor],
|
||||
attn_backends: dict[str, type[AttentionBackend]],
|
||||
self, kv_caches: CanonicalKVCaches
|
||||
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
|
||||
"""
|
||||
Get offloading handlers along with their respective src and dst types.
|
||||
|
||||
Args:
|
||||
kv_caches: A dictionary of layer_name -> gpu_kv_cache tensor.
|
||||
attn_backends: A dictionary of layer_name -> AttentionBackend.
|
||||
kv_caches: Canonicalized KV caches.
|
||||
|
||||
Yields:
|
||||
Tuples of (src_type, dst_type, offloading_handler).
|
||||
|
||||
@@ -9,8 +9,8 @@ import torch
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.kv_offload.mediums import BlockIDsLoadStoreSpec
|
||||
from vllm.v1.kv_offload.spec import CanonicalKVCacheRef, CanonicalKVCaches
|
||||
from vllm.v1.kv_offload.worker.worker import (
|
||||
OffloadingHandler,
|
||||
TransferResult,
|
||||
@@ -73,39 +73,72 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
src_tensors: list[torch.Tensor],
|
||||
dst_tensors: list[torch.Tensor],
|
||||
src_block_size_factor: int,
|
||||
dst_block_size_factor: int,
|
||||
gpu_tensors: list[torch.Tensor],
|
||||
cpu_tensors: list[torch.Tensor],
|
||||
block_size_factor: int,
|
||||
kv_cache_groups_data_refs: list[list[CanonicalKVCacheRef]],
|
||||
gpu_to_cpu: bool,
|
||||
):
|
||||
"""
|
||||
Initialize a SingleDirectionOffloadingHandler.
|
||||
|
||||
Args:
|
||||
src_tensors: list of KV cache tensors to copy from.
|
||||
dst_tensors: list of KV cache tensors to copy to.
|
||||
Order should match src_tensors.
|
||||
src_block_size_factor: The number of kernel blocks
|
||||
per KV block in a source tensor.
|
||||
dst_block_size_factor: The number of kernel blocks
|
||||
per KV block in a destination tensor.
|
||||
gpu_tensors: list of GPU KV cache tensors.
|
||||
Each of shape (num_gpu_blocks, gpu_page_size_bytes) with dtype int8.
|
||||
cpu_tensors: list of CPU KV cache tensors.
|
||||
Each of shape (num_cpu_blocks, cpu_page_size_bytes) with dtype int8.
|
||||
Order should match gpu_tensors.
|
||||
kv_cache_groups_data_refs: list of CanonicalKVCacheRef per group.
|
||||
gpu_to_cpu: if True, transfer from GPU to CPU; otherwise CPU to GPU.
|
||||
"""
|
||||
assert len(src_tensors) == len(dst_tensors)
|
||||
assert len(gpu_tensors) == len(cpu_tensors)
|
||||
assert len(gpu_tensors) > 0
|
||||
|
||||
self.src_tensors: list[torch.Tensor] = src_tensors
|
||||
self.dst_tensors: list[torch.Tensor] = dst_tensors
|
||||
min_block_size_factor = min(src_block_size_factor, dst_block_size_factor)
|
||||
self.src_block_size_factor: int = src_block_size_factor // min_block_size_factor
|
||||
self.dst_block_size_factor: int = dst_block_size_factor // min_block_size_factor
|
||||
# assert a single KV group until transfer_async supports multiple groups
|
||||
assert len(kv_cache_groups_data_refs) == 1
|
||||
|
||||
self.block_size_in_bytes = [
|
||||
tensor.element_size() * tensor.stride(0) * min_block_size_factor
|
||||
for tensor in src_tensors
|
||||
# assert input tensors are as expected
|
||||
for gpu_tensor, cpu_tensor in zip(gpu_tensors, cpu_tensors):
|
||||
assert gpu_tensor.dtype == torch.int8
|
||||
assert gpu_tensor.ndim == 2
|
||||
assert gpu_tensor.is_cuda
|
||||
assert cpu_tensor.dtype == torch.int8
|
||||
assert cpu_tensor.ndim == 2
|
||||
assert cpu_tensor.device.type == "cpu"
|
||||
_, gpu_page_size = gpu_tensor.shape
|
||||
_, cpu_page_size = cpu_tensor.shape
|
||||
assert cpu_page_size == gpu_page_size * block_size_factor
|
||||
|
||||
self.src_tensors: list[torch.Tensor] = (
|
||||
gpu_tensors if gpu_to_cpu else cpu_tensors
|
||||
)
|
||||
self.dst_tensors: list[torch.Tensor] = (
|
||||
cpu_tensors if gpu_to_cpu else gpu_tensors
|
||||
)
|
||||
self.gpu_to_cpu: bool = gpu_to_cpu
|
||||
|
||||
# GPU blocks may be smaller
|
||||
# cpu_page_size = gpu_page_size * block_size_factor.
|
||||
self.src_block_size_factor = 1 if self.gpu_to_cpu else block_size_factor
|
||||
self.dst_block_size_factor = block_size_factor if self.gpu_to_cpu else 1
|
||||
|
||||
# per-tensor block size in byte
|
||||
self.tensor_block_size_in_bytes = [
|
||||
gpu_tensor.shape[1] for gpu_tensor in gpu_tensors
|
||||
]
|
||||
self.total_block_size_in_bytes = sum(self.block_size_in_bytes)
|
||||
|
||||
assert len(src_tensors) > 0
|
||||
self.gpu_to_cpu: bool = self.src_tensors[0].is_cuda
|
||||
# per-group block size in bytes
|
||||
self.group_block_size_in_bytes = []
|
||||
for kv_cache_group_data_refs in kv_cache_groups_data_refs:
|
||||
group_block_size_in_bytes = 0
|
||||
for kv_cache_data_ref in kv_cache_group_data_refs:
|
||||
# TODO(orozery): use kv_cache_data_ref.page_size_bytes
|
||||
# once swap_blocks support it
|
||||
group_block_size_in_bytes += self.tensor_block_size_in_bytes[
|
||||
kv_cache_data_ref.tensor_idx
|
||||
]
|
||||
self.group_block_size_in_bytes.append(group_block_size_in_bytes)
|
||||
|
||||
self.transfer_type = ("GPU", "CPU") if self.gpu_to_cpu else ("CPU", "GPU")
|
||||
# job_id -> event
|
||||
self._transfer_events: dict[int, torch.Event] = {}
|
||||
@@ -167,7 +200,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
for src_tensor, dst_tensor, block_size_in_bytes in zip(
|
||||
self.src_tensors,
|
||||
self.dst_tensors,
|
||||
self.block_size_in_bytes,
|
||||
self.tensor_block_size_in_bytes,
|
||||
):
|
||||
ops.swap_blocks(
|
||||
src_tensor,
|
||||
@@ -184,7 +217,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
stream=stream,
|
||||
start_event=start_event,
|
||||
end_event=end_event,
|
||||
num_bytes=dst_sub_block_count * self.total_block_size_in_bytes,
|
||||
num_bytes=dst_sub_block_count * self.group_block_size_in_bytes[0],
|
||||
)
|
||||
)
|
||||
|
||||
@@ -223,102 +256,42 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
class CpuGpuOffloadingHandlers:
|
||||
def __init__(
|
||||
self,
|
||||
gpu_block_size: int,
|
||||
cpu_block_size: int,
|
||||
kv_caches: CanonicalKVCaches,
|
||||
block_size_factor: int,
|
||||
num_cpu_blocks: int,
|
||||
gpu_caches: dict[str, torch.Tensor],
|
||||
attn_backends: dict[str, type[AttentionBackend]],
|
||||
):
|
||||
assert gpu_caches
|
||||
assert cpu_block_size % gpu_block_size == 0
|
||||
|
||||
# find kernel block size and determine layout per each gpu tensor
|
||||
kernel_block_size: int | None = None
|
||||
# list of (gpu_tensor, split_k_and_v)
|
||||
parsed_gpu_tensors: list[tuple[torch.Tensor, bool]] = []
|
||||
for layer_name, gpu_tensor in gpu_caches.items():
|
||||
gpu_shape = gpu_tensor.shape
|
||||
attn_backend = attn_backends[layer_name]
|
||||
test_shape = attn_backend.get_kv_cache_shape(
|
||||
num_blocks=1234, block_size=16, num_kv_heads=1, head_size=256
|
||||
)
|
||||
|
||||
has_layers_dim = False
|
||||
split_k_and_v = False
|
||||
if len(gpu_shape) != len(test_shape):
|
||||
# cross-layers tensor
|
||||
# shape is (num_blocks, ...)
|
||||
assert len(gpu_shape) == len(test_shape) + 1
|
||||
has_layers_dim = True
|
||||
# prepend a dummy num_layers=80 to test_shape
|
||||
test_shape = (80,) + test_shape
|
||||
elif test_shape[0] != 1234:
|
||||
# shape should be (2, num_blocks, ...)
|
||||
assert test_shape[0] == 2
|
||||
assert test_shape[1] == 1234
|
||||
assert gpu_shape[0] == 2
|
||||
split_k_and_v = True
|
||||
|
||||
if has_layers_dim:
|
||||
# in the cross layers case, the registered kv cache tensor
|
||||
# shape matches the physical layout, whereas test_shape
|
||||
# is the logical layout.
|
||||
# To match them, we need to permute test_shape
|
||||
try:
|
||||
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order(
|
||||
include_num_layers_dimension=has_layers_dim
|
||||
)
|
||||
assert len(kv_cache_stride_order) == len(gpu_shape)
|
||||
except (AttributeError, NotImplementedError):
|
||||
kv_cache_stride_order = tuple(range(len(gpu_shape)))
|
||||
|
||||
test_shape = tuple(test_shape[i] for i in kv_cache_stride_order)
|
||||
|
||||
# find block_size (16) dimension index
|
||||
block_size_idx = test_shape.index(16)
|
||||
if kernel_block_size is not None:
|
||||
assert kernel_block_size == gpu_shape[block_size_idx]
|
||||
else:
|
||||
kernel_block_size = gpu_shape[block_size_idx]
|
||||
assert gpu_block_size % kernel_block_size == 0
|
||||
|
||||
parsed_gpu_tensors.append((gpu_tensor, split_k_and_v))
|
||||
|
||||
assert kernel_block_size is not None
|
||||
cpu_block_size_factor = cpu_block_size // kernel_block_size
|
||||
gpu_block_size_factor = gpu_block_size // kernel_block_size
|
||||
num_cpu_kernel_blocks = num_cpu_blocks * cpu_block_size_factor
|
||||
|
||||
# allocate cpu tensors
|
||||
pin_memory = is_pin_memory_available()
|
||||
logger.info("Allocating %d CPU tensors...", len(parsed_gpu_tensors))
|
||||
logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors))
|
||||
gpu_tensors: list[torch.Tensor] = []
|
||||
cpu_tensors: list[torch.Tensor] = []
|
||||
for gpu_tensor, split_k_and_v in parsed_gpu_tensors:
|
||||
cpu_shape = list(gpu_tensor.shape)
|
||||
cpu_shape[1 if split_k_and_v else 0] = num_cpu_kernel_blocks
|
||||
|
||||
logger.debug("Allocating CPU tensor of shape %r", cpu_shape)
|
||||
for kv_cache_tensor in kv_caches.tensors:
|
||||
gpu_page_size_bytes = kv_cache_tensor.page_size_bytes
|
||||
gpu_tensor = kv_cache_tensor.tensor.view(torch.int8).view(
|
||||
(-1, gpu_page_size_bytes)
|
||||
)
|
||||
cpu_page_size_bytes = gpu_page_size_bytes * block_size_factor
|
||||
cpu_tensor = torch.zeros(
|
||||
cpu_shape,
|
||||
dtype=gpu_tensor.dtype,
|
||||
(num_cpu_blocks, cpu_page_size_bytes),
|
||||
dtype=torch.int8,
|
||||
device="cpu",
|
||||
pin_memory=pin_memory,
|
||||
)
|
||||
|
||||
gpu_tensors.extend(gpu_tensor.unbind(0) if split_k_and_v else [gpu_tensor])
|
||||
cpu_tensors.extend(cpu_tensor.unbind(0) if split_k_and_v else [cpu_tensor])
|
||||
gpu_tensors.append(gpu_tensor)
|
||||
cpu_tensors.append(cpu_tensor)
|
||||
|
||||
self.gpu_to_cpu_handler = SingleDirectionOffloadingHandler(
|
||||
src_tensors=gpu_tensors,
|
||||
dst_tensors=cpu_tensors,
|
||||
src_block_size_factor=gpu_block_size_factor,
|
||||
dst_block_size_factor=cpu_block_size_factor,
|
||||
gpu_tensors=gpu_tensors,
|
||||
cpu_tensors=cpu_tensors,
|
||||
block_size_factor=block_size_factor,
|
||||
kv_cache_groups_data_refs=kv_caches.group_data_refs,
|
||||
gpu_to_cpu=True,
|
||||
)
|
||||
|
||||
self.cpu_to_gpu_handler = SingleDirectionOffloadingHandler(
|
||||
src_tensors=cpu_tensors,
|
||||
dst_tensors=gpu_tensors,
|
||||
src_block_size_factor=cpu_block_size_factor,
|
||||
dst_block_size_factor=gpu_block_size_factor,
|
||||
gpu_tensors=gpu_tensors,
|
||||
cpu_tensors=cpu_tensors,
|
||||
block_size_factor=block_size_factor,
|
||||
kv_cache_groups_data_refs=kv_caches.group_data_refs,
|
||||
gpu_to_cpu=False,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user