[KV Offload] Define clean backend configuration boundary (#48150)
Signed-off-by: Change72 <changg@nvidia.com> Signed-off-by: Chang Guo <cguo51@asu.edu> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
Claude Fable 5
Or Ozeri
OpenAI Codex
parent
8c3393f373
commit
a9531edfa6
@@ -72,7 +72,7 @@ class Transfer:
|
||||
|
||||
def compute_sub_block_ptrs(
|
||||
block_ids: np.ndarray,
|
||||
block_size_factor: int,
|
||||
blocks_per_chunk: int,
|
||||
output: np.ndarray,
|
||||
tensor: torch.Tensor,
|
||||
skip_count: int = 0,
|
||||
@@ -80,38 +80,38 @@ def compute_sub_block_ptrs(
|
||||
"""
|
||||
Compute byte pointers for sub-blocks of the given block IDs.
|
||||
|
||||
Each block in block_ids contains block_size_factor sub-blocks.
|
||||
Each block in block_ids contains blocks_per_chunk sub-blocks.
|
||||
The pointer for sub-block j of block b is:
|
||||
base_ptr + b * row_stride + j * sub_block_size
|
||||
base_ptr + b * row_stride + j * block_page_size
|
||||
|
||||
where sub_block_size = tensor.shape[1] // block_size_factor (gpu page size).
|
||||
where block_page_size = tensor.shape[1] // blocks_per_chunk (gpu page size).
|
||||
|
||||
This handles tensors where row_stride != block_size_factor * sub_block_size
|
||||
This handles tensors where row_stride != blocks_per_chunk * block_page_size
|
||||
(e.g. non-contiguous CPU tensors).
|
||||
|
||||
Args:
|
||||
block_ids: array of block IDs at the tensor's native granularity.
|
||||
block_size_factor: number of sub-blocks per block.
|
||||
blocks_per_chunk: number of sub-blocks per block.
|
||||
output: pre-allocated pointer array to write pointers into.
|
||||
tensor: the source or destination tensor.
|
||||
skip_count: sub-blocks to skip in the first block.
|
||||
"""
|
||||
assert skip_count < block_size_factor
|
||||
assert skip_count < blocks_per_chunk
|
||||
|
||||
num_sub_blocks = len(output)
|
||||
base_ptr = tensor.data_ptr()
|
||||
row_stride = tensor.stride(0)
|
||||
|
||||
if block_size_factor == 1:
|
||||
if blocks_per_chunk == 1:
|
||||
# Fast path: 1:1 mapping, no sub-block expansion needed.
|
||||
output[:] = base_ptr + block_ids.astype(np.uint64)[:num_sub_blocks] * row_stride
|
||||
return
|
||||
|
||||
# Vectorized expansion for block_size_factor > 1.
|
||||
assert tensor.shape[1] % block_size_factor == 0
|
||||
sub_block_size = tensor.shape[1] // block_size_factor
|
||||
sub_offsets = np.arange(block_size_factor, dtype=np.uint64) * sub_block_size
|
||||
# (num_blocks, 1) + (1, block_size_factor) -> (num_blocks, block_size_factor)
|
||||
# Vectorized expansion for blocks_per_chunk > 1.
|
||||
assert tensor.shape[1] % blocks_per_chunk == 0
|
||||
block_page_size = tensor.shape[1] // blocks_per_chunk
|
||||
sub_offsets = np.arange(blocks_per_chunk, dtype=np.uint64) * block_page_size
|
||||
# (num_blocks, 1) + (1, blocks_per_chunk) -> (num_blocks, blocks_per_chunk)
|
||||
all_ptrs = (
|
||||
base_ptr + block_ids.astype(np.uint64)[:, np.newaxis] * row_stride
|
||||
) + sub_offsets[np.newaxis, :]
|
||||
@@ -175,7 +175,7 @@ class SingleDirectionOffloadingHandler:
|
||||
self,
|
||||
gpu_tensors: list[torch.Tensor],
|
||||
cpu_tensors: list[torch.Tensor],
|
||||
block_size_factor: int,
|
||||
blocks_per_chunk: int,
|
||||
kv_cache_groups_data_refs: list[list[CanonicalKVCacheRef]],
|
||||
gpu_to_cpu: bool,
|
||||
mmap_region: SharedOffloadRegion | None = None,
|
||||
@@ -205,7 +205,7 @@ class SingleDirectionOffloadingHandler:
|
||||
assert cpu_tensor.device.type == "cpu"
|
||||
_, gpu_page_size = gpu_tensor.shape
|
||||
_, cpu_page_size = cpu_tensor.shape
|
||||
assert cpu_page_size == gpu_page_size * block_size_factor
|
||||
assert cpu_page_size == gpu_page_size * blocks_per_chunk
|
||||
|
||||
self.src_tensors: list[torch.Tensor] = (
|
||||
gpu_tensors if gpu_to_cpu else cpu_tensors
|
||||
@@ -220,9 +220,9 @@ class SingleDirectionOffloadingHandler:
|
||||
)
|
||||
|
||||
# GPU blocks may be smaller
|
||||
# cpu_page_size = gpu_page_size * block_size_factor.
|
||||
self.src_block_size_factor = 1 if self.gpu_to_cpu else block_size_factor
|
||||
self.dst_block_size_factor = block_size_factor if self.gpu_to_cpu else 1
|
||||
# cpu_page_size = gpu_page_size * blocks_per_chunk.
|
||||
self.src_blocks_per_chunk = 1 if self.gpu_to_cpu else blocks_per_chunk
|
||||
self.dst_blocks_per_chunk = blocks_per_chunk if self.gpu_to_cpu else 1
|
||||
|
||||
# mmap_region to clean up on shutdown (gpu_to_cpu handler owns it)
|
||||
self._mmap_region = mmap_region
|
||||
@@ -313,20 +313,16 @@ class SingleDirectionOffloadingHandler:
|
||||
if group_size == 0:
|
||||
continue
|
||||
|
||||
src_logical_blocks_to_skip = block_idx % self.src_block_size_factor
|
||||
dst_logical_blocks_to_skip = block_idx % self.dst_block_size_factor
|
||||
src_logical_blocks_to_skip = block_idx % self.src_blocks_per_chunk
|
||||
dst_logical_blocks_to_skip = block_idx % self.dst_blocks_per_chunk
|
||||
src_logical_blocks_count = group_size + src_logical_blocks_to_skip
|
||||
dst_logical_blocks_count = group_size + dst_logical_blocks_to_skip
|
||||
|
||||
dst_blocks_count = cdiv(
|
||||
dst_logical_blocks_count, self.dst_block_size_factor
|
||||
)
|
||||
dst_blocks_count = cdiv(dst_logical_blocks_count, self.dst_blocks_per_chunk)
|
||||
dst_end_offset = dst_offset + dst_blocks_count
|
||||
assert dst_end_offset <= num_dst_blocks
|
||||
|
||||
src_blocks_count = cdiv(
|
||||
src_logical_blocks_count, self.src_block_size_factor
|
||||
)
|
||||
src_blocks_count = cdiv(src_logical_blocks_count, self.src_blocks_per_chunk)
|
||||
src_end_offset = src_offset + src_blocks_count
|
||||
assert src_end_offset <= num_src_blocks
|
||||
|
||||
@@ -339,14 +335,14 @@ class SingleDirectionOffloadingHandler:
|
||||
|
||||
compute_sub_block_ptrs(
|
||||
group_src,
|
||||
self.src_block_size_factor,
|
||||
self.src_blocks_per_chunk,
|
||||
all_src[op_idx:end_idx],
|
||||
self.src_tensors[t_idx],
|
||||
skip_count=src_logical_blocks_to_skip,
|
||||
)
|
||||
compute_sub_block_ptrs(
|
||||
group_dst,
|
||||
self.dst_block_size_factor,
|
||||
self.dst_blocks_per_chunk,
|
||||
all_dst[op_idx:end_idx],
|
||||
self.dst_tensors[t_idx],
|
||||
skip_count=dst_logical_blocks_to_skip,
|
||||
@@ -476,7 +472,7 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
def __init__(
|
||||
self,
|
||||
kv_caches: CanonicalKVCaches,
|
||||
block_size_factor: int,
|
||||
blocks_per_chunk: int,
|
||||
num_cpu_blocks: int,
|
||||
mmap_region: SharedOffloadRegion | None = None,
|
||||
):
|
||||
@@ -492,7 +488,7 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
gpu_tensor = kv_cache_tensor.tensor.view(torch.int8).view(
|
||||
(-1, gpu_page_size_bytes)
|
||||
)
|
||||
cpu_page_size_bytes = gpu_page_size_bytes * block_size_factor
|
||||
cpu_page_size_bytes = gpu_page_size_bytes * blocks_per_chunk
|
||||
|
||||
if mmap_region is not None:
|
||||
cpu_tensor = mmap_region.create_next_view(cpu_page_size_bytes)
|
||||
@@ -518,7 +514,7 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
self._store_handler = SingleDirectionOffloadingHandler(
|
||||
gpu_tensors=gpu_tensors,
|
||||
cpu_tensors=cpu_tensors,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
kv_cache_groups_data_refs=kv_caches.group_data_refs,
|
||||
gpu_to_cpu=True,
|
||||
mmap_region=mmap_region,
|
||||
@@ -527,7 +523,7 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
self._load_handler = SingleDirectionOffloadingHandler(
|
||||
gpu_tensors=gpu_tensors,
|
||||
cpu_tensors=cpu_tensors,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
kv_cache_groups_data_refs=kv_caches.group_data_refs,
|
||||
gpu_to_cpu=False,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user