[KV Offload] Define clean backend configuration boundary (#48150)

Signed-off-by: Change72 <changg@nvidia.com>
Signed-off-by: Chang Guo <cguo51@asu.edu>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Chang Guo
2026-07-16 13:27:05 +03:00
committed by GitHub
co-authored by Codex Claude Fable 5 Or Ozeri OpenAI Codex
parent 8c3393f373
commit a9531edfa6
27 changed files with 1287 additions and 855 deletions
+28 -32
View File
@@ -72,7 +72,7 @@ class Transfer:
def compute_sub_block_ptrs(
block_ids: np.ndarray,
block_size_factor: int,
blocks_per_chunk: int,
output: np.ndarray,
tensor: torch.Tensor,
skip_count: int = 0,
@@ -80,38 +80,38 @@ def compute_sub_block_ptrs(
"""
Compute byte pointers for sub-blocks of the given block IDs.
Each block in block_ids contains block_size_factor sub-blocks.
Each block in block_ids contains blocks_per_chunk sub-blocks.
The pointer for sub-block j of block b is:
base_ptr + b * row_stride + j * sub_block_size
base_ptr + b * row_stride + j * block_page_size
where sub_block_size = tensor.shape[1] // block_size_factor (gpu page size).
where block_page_size = tensor.shape[1] // blocks_per_chunk (gpu page size).
This handles tensors where row_stride != block_size_factor * sub_block_size
This handles tensors where row_stride != blocks_per_chunk * block_page_size
(e.g. non-contiguous CPU tensors).
Args:
block_ids: array of block IDs at the tensor's native granularity.
block_size_factor: number of sub-blocks per block.
blocks_per_chunk: number of sub-blocks per block.
output: pre-allocated pointer array to write pointers into.
tensor: the source or destination tensor.
skip_count: sub-blocks to skip in the first block.
"""
assert skip_count < block_size_factor
assert skip_count < blocks_per_chunk
num_sub_blocks = len(output)
base_ptr = tensor.data_ptr()
row_stride = tensor.stride(0)
if block_size_factor == 1:
if blocks_per_chunk == 1:
# Fast path: 1:1 mapping, no sub-block expansion needed.
output[:] = base_ptr + block_ids.astype(np.uint64)[:num_sub_blocks] * row_stride
return
# Vectorized expansion for block_size_factor > 1.
assert tensor.shape[1] % block_size_factor == 0
sub_block_size = tensor.shape[1] // block_size_factor
sub_offsets = np.arange(block_size_factor, dtype=np.uint64) * sub_block_size
# (num_blocks, 1) + (1, block_size_factor) -> (num_blocks, block_size_factor)
# Vectorized expansion for blocks_per_chunk > 1.
assert tensor.shape[1] % blocks_per_chunk == 0
block_page_size = tensor.shape[1] // blocks_per_chunk
sub_offsets = np.arange(blocks_per_chunk, dtype=np.uint64) * block_page_size
# (num_blocks, 1) + (1, blocks_per_chunk) -> (num_blocks, blocks_per_chunk)
all_ptrs = (
base_ptr + block_ids.astype(np.uint64)[:, np.newaxis] * row_stride
) + sub_offsets[np.newaxis, :]
@@ -175,7 +175,7 @@ class SingleDirectionOffloadingHandler:
self,
gpu_tensors: list[torch.Tensor],
cpu_tensors: list[torch.Tensor],
block_size_factor: int,
blocks_per_chunk: int,
kv_cache_groups_data_refs: list[list[CanonicalKVCacheRef]],
gpu_to_cpu: bool,
mmap_region: SharedOffloadRegion | None = None,
@@ -205,7 +205,7 @@ class SingleDirectionOffloadingHandler:
assert cpu_tensor.device.type == "cpu"
_, gpu_page_size = gpu_tensor.shape
_, cpu_page_size = cpu_tensor.shape
assert cpu_page_size == gpu_page_size * block_size_factor
assert cpu_page_size == gpu_page_size * blocks_per_chunk
self.src_tensors: list[torch.Tensor] = (
gpu_tensors if gpu_to_cpu else cpu_tensors
@@ -220,9 +220,9 @@ class SingleDirectionOffloadingHandler:
)
# GPU blocks may be smaller
# cpu_page_size = gpu_page_size * block_size_factor.
self.src_block_size_factor = 1 if self.gpu_to_cpu else block_size_factor
self.dst_block_size_factor = block_size_factor if self.gpu_to_cpu else 1
# cpu_page_size = gpu_page_size * blocks_per_chunk.
self.src_blocks_per_chunk = 1 if self.gpu_to_cpu else blocks_per_chunk
self.dst_blocks_per_chunk = blocks_per_chunk if self.gpu_to_cpu else 1
# mmap_region to clean up on shutdown (gpu_to_cpu handler owns it)
self._mmap_region = mmap_region
@@ -313,20 +313,16 @@ class SingleDirectionOffloadingHandler:
if group_size == 0:
continue
src_logical_blocks_to_skip = block_idx % self.src_block_size_factor
dst_logical_blocks_to_skip = block_idx % self.dst_block_size_factor
src_logical_blocks_to_skip = block_idx % self.src_blocks_per_chunk
dst_logical_blocks_to_skip = block_idx % self.dst_blocks_per_chunk
src_logical_blocks_count = group_size + src_logical_blocks_to_skip
dst_logical_blocks_count = group_size + dst_logical_blocks_to_skip
dst_blocks_count = cdiv(
dst_logical_blocks_count, self.dst_block_size_factor
)
dst_blocks_count = cdiv(dst_logical_blocks_count, self.dst_blocks_per_chunk)
dst_end_offset = dst_offset + dst_blocks_count
assert dst_end_offset <= num_dst_blocks
src_blocks_count = cdiv(
src_logical_blocks_count, self.src_block_size_factor
)
src_blocks_count = cdiv(src_logical_blocks_count, self.src_blocks_per_chunk)
src_end_offset = src_offset + src_blocks_count
assert src_end_offset <= num_src_blocks
@@ -339,14 +335,14 @@ class SingleDirectionOffloadingHandler:
compute_sub_block_ptrs(
group_src,
self.src_block_size_factor,
self.src_blocks_per_chunk,
all_src[op_idx:end_idx],
self.src_tensors[t_idx],
skip_count=src_logical_blocks_to_skip,
)
compute_sub_block_ptrs(
group_dst,
self.dst_block_size_factor,
self.dst_blocks_per_chunk,
all_dst[op_idx:end_idx],
self.dst_tensors[t_idx],
skip_count=dst_logical_blocks_to_skip,
@@ -476,7 +472,7 @@ class CPUOffloadingWorker(OffloadingWorker):
def __init__(
self,
kv_caches: CanonicalKVCaches,
block_size_factor: int,
blocks_per_chunk: int,
num_cpu_blocks: int,
mmap_region: SharedOffloadRegion | None = None,
):
@@ -492,7 +488,7 @@ class CPUOffloadingWorker(OffloadingWorker):
gpu_tensor = kv_cache_tensor.tensor.view(torch.int8).view(
(-1, gpu_page_size_bytes)
)
cpu_page_size_bytes = gpu_page_size_bytes * block_size_factor
cpu_page_size_bytes = gpu_page_size_bytes * blocks_per_chunk
if mmap_region is not None:
cpu_tensor = mmap_region.create_next_view(cpu_page_size_bytes)
@@ -518,7 +514,7 @@ class CPUOffloadingWorker(OffloadingWorker):
self._store_handler = SingleDirectionOffloadingHandler(
gpu_tensors=gpu_tensors,
cpu_tensors=cpu_tensors,
block_size_factor=block_size_factor,
blocks_per_chunk=blocks_per_chunk,
kv_cache_groups_data_refs=kv_caches.group_data_refs,
gpu_to_cpu=True,
mmap_region=mmap_region,
@@ -527,7 +523,7 @@ class CPUOffloadingWorker(OffloadingWorker):
self._load_handler = SingleDirectionOffloadingHandler(
gpu_tensors=gpu_tensors,
cpu_tensors=cpu_tensors,
block_size_factor=block_size_factor,
blocks_per_chunk=blocks_per_chunk,
kv_cache_groups_data_refs=kv_caches.group_data_refs,
gpu_to_cpu=False,
)