[Refactor][kv_offload]: rename block→chunk (#52615)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
Ronen Schaffer
2026-09-09 17:45:49 +03:00
committed by GitHub
parent 9ffb8cea96
commit 138d137b5b
34 changed files with 933 additions and 916 deletions
+10 -10
View File
@@ -257,7 +257,7 @@ class SingleDirectionOffloadingHandler:
gpu_tensors: list of GPU KV cache tensors.
Each of shape (num_gpu_blocks, gpu_page_size_bytes) with dtype int8.
cpu_tensors: list of CPU KV cache tensors.
Each of shape (num_cpu_blocks, cpu_page_size_bytes) with dtype int8.
Each of shape (num_cpu_chunks, cpu_page_size_bytes) with dtype int8.
Order should match gpu_tensors.
layer_refs_per_group: list of CanonicalKVCacheRef per group.
gpu_to_cpu: if True, transfer from GPU to CPU; otherwise CPU to GPU.
@@ -527,20 +527,20 @@ class SingleDirectionOffloadingHandler:
# 1. GPU -> CPU
# 2. CPU -> GPU
#
# transfers are also to CPU blocks, EXCEPT MAYBE for the first and last block.
# i.e. the first and last CPU blocks in src_blocks can match against
# transfers are also to CPU chunks, EXCEPT MAYBE for the first and last chunk.
# i.e. the first and last CPU chunks in src_blocks can match against
# a smaller (byte-wise) set of GPU blocks in dst_blocks.
# In such cases, we may need to skip some gpu-sized sub-blocks,
# and start reading/writing from the middle of the first CPU block.
# and start reading/writing from the middle of the first CPU chunk.
# If we have multiple KV cache groups (when using HMA with hybrid models),
# we may have a partial first/last CPU block per each group.
# we may have a partial first/last CPU chunk per each group.
# The group_sizes parameter encodes the size of each group of blocks
# in the GPU dst_blocks.
# If group_sizes is None, we assume all blocks belong to a single group.
# The logical_offset parameter maps each group of blocks to its logical
# offset inside the request, counting in GPU blocks.
# This allows us to find the correct starting position
# in the matching first CPU block.
# in the matching first CPU chunk.
# extract group_sizes from the GPU spec
gpu_spec = src_spec if self.gpu_to_cpu else dst_spec
@@ -751,7 +751,7 @@ class CPUOffloadingWorker(OffloadingWorker):
self,
kv_caches: CanonicalKVCaches,
blocks_per_chunk: int,
num_cpu_blocks: int,
num_cpu_chunks: int,
mmap_region: SharedOffloadRegion | None = None,
canonical_layout: bool = False,
):
@@ -790,16 +790,16 @@ class CPUOffloadingWorker(OffloadingWorker):
else:
t0 = time.monotonic()
cpu_tensor = torch.zeros(
(num_cpu_blocks, cpu_page_size_bytes),
(num_cpu_chunks, cpu_page_size_bytes),
dtype=torch.int8,
device="cpu",
pin_memory=pin_memory,
)
logger.debug(
"torch.zeros pinned tensor %d×%d (%.2f GB): %.3f s",
num_cpu_blocks,
num_cpu_chunks,
cpu_page_size_bytes,
num_cpu_blocks * cpu_page_size_bytes / 1e9,
num_cpu_chunks * cpu_page_size_bytes / 1e9,
time.monotonic() - t0,
)