[KV Offload] Define clean backend configuration boundary (#48150)

Signed-off-by: Change72 <changg@nvidia.com>
Signed-off-by: Chang Guo <cguo51@asu.edu>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Chang Guo
2026-07-16 13:27:05 +03:00
committed by GitHub
co-authored by Codex Claude Fable 5 Or Ozeri OpenAI Codex
parent 8c3393f373
commit a9531edfa6
27 changed files with 1287 additions and 855 deletions
+20 -20
View File
@@ -23,7 +23,7 @@ from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
NUM_GPU_BLOCKS = [64]
NUM_CPU_BLOCKS = [256]
GPU_PAGE_SIZES = [512, 1024]
BLOCK_SIZE_FACTORS = [1, 3]
BLOCKS_PER_CHUNK_VALUES = [1, 3]
NUM_TENSORS = [4]
SEEDS = [0]
DEVICE_TYPE = current_platform.device_type
@@ -35,7 +35,7 @@ NUM_MAPPINGS_PER_GROUP = [2]
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
@pytest.mark.parametrize("block_size_factor", BLOCK_SIZE_FACTORS)
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
@pytest.mark.parametrize("num_tensors", NUM_TENSORS)
@@ -48,7 +48,7 @@ def test_transfer(
gpu_to_cpu: bool,
num_mappings: int,
gpu_page_size_bytes: int,
block_size_factor: int,
blocks_per_chunk: int,
num_gpu_blocks: int,
num_cpu_blocks: int,
num_tensors: int,
@@ -92,7 +92,7 @@ def test_transfer(
mmap_region: SharedOffloadRegion | None = None
if use_shared_memory:
cpu_page_size = round_up(
gpu_page_size_bytes * num_tensors * block_size_factor,
gpu_page_size_bytes * num_tensors * blocks_per_chunk,
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
)
mmap_region = SharedOffloadRegion(
@@ -105,25 +105,25 @@ def test_transfer(
worker = CPUOffloadingWorker(
kv_caches=kv_caches,
block_size_factor=block_size_factor,
blocks_per_chunk=blocks_per_chunk,
num_cpu_blocks=num_cpu_blocks,
mmap_region=mmap_region,
)
# select block mappings
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * block_size_factor)
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * blocks_per_chunk)
cpu_blocks = random.sample(range(num_cpu_blocks), num_mappings)
# expand cpu blocks to gpu-page granularity for uniform comparison:
# each cpu block maps to block_size_factor consecutive sub-blocks
# each cpu block maps to blocks_per_chunk consecutive sub-blocks
cpu_blocks_expanded = [
cpu_block * block_size_factor + j
cpu_block * blocks_per_chunk + j
for cpu_block in cpu_blocks
for j in range(block_size_factor)
for j in range(blocks_per_chunk)
]
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU block
blocks_to_skip = block_size_factor - 1
blocks_to_skip = blocks_per_chunk - 1
if blocks_to_skip > 0:
gpu_blocks = gpu_blocks[blocks_to_skip:]
cpu_blocks_expanded = cpu_blocks_expanded[blocks_to_skip:]
@@ -214,7 +214,7 @@ def test_transfer(
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
@pytest.mark.parametrize("num_mappings_per_group", NUM_MAPPINGS_PER_GROUP)
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
@pytest.mark.parametrize("block_size_factor", BLOCK_SIZE_FACTORS)
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
@pytest.mark.parametrize("seed", SEEDS)
@@ -225,7 +225,7 @@ def test_transfer_multi_group(
gpu_to_cpu: bool,
num_mappings_per_group: int,
gpu_page_size_bytes: int,
block_size_factor: int,
blocks_per_chunk: int,
num_gpu_blocks: int,
num_cpu_blocks: int,
seed: int,
@@ -234,7 +234,7 @@ def test_transfer_multi_group(
"""Test transfers with three KV cache groups:
- Group 0: aligned transfer with num_mappings_per_group blocks
- Group 1: zero blocks (empty group)
- Group 2: unaligned CPU->GPU transfer (logical_offset=block_size_factor-1,
- Group 2: unaligned CPU->GPU transfer (logical_offset=blocks_per_chunk-1,
causing the implementation to skip source sub-blocks) with
num_mappings_per_group blocks
"""
@@ -275,7 +275,7 @@ def test_transfer_multi_group(
worker = CPUOffloadingWorker(
kv_caches=canonical_kv_caches,
block_size_factor=block_size_factor,
blocks_per_chunk=blocks_per_chunk,
num_cpu_blocks=num_cpu_blocks,
)
@@ -283,7 +283,7 @@ def test_transfer_multi_group(
group_sizes_in_cpu_blocks = [num_mappings_per_group, 0, num_mappings_per_group]
total_cpu_blocks = sum(group_sizes_in_cpu_blocks)
total_gpu_blocks_needed = total_cpu_blocks * block_size_factor
total_gpu_blocks_needed = total_cpu_blocks * blocks_per_chunk
gpu_blocks_all = random.sample(range(num_gpu_blocks), total_gpu_blocks_needed)
cpu_blocks_all = random.sample(range(num_cpu_blocks), total_cpu_blocks)
@@ -293,7 +293,7 @@ def test_transfer_multi_group(
gpu_offset = 0
cpu_offset = 0
for size in group_sizes_in_cpu_blocks:
gpu_count = size * block_size_factor
gpu_count = size * blocks_per_chunk
gpu_blocks_per_group.append(gpu_blocks_all[gpu_offset : gpu_offset + gpu_count])
cpu_blocks_per_group.append(cpu_blocks_all[cpu_offset : cpu_offset + size])
gpu_offset += gpu_count
@@ -302,15 +302,15 @@ def test_transfer_multi_group(
# expand cpu blocks to gpu-page granularity
cpu_blocks_expanded_per_group = [
[
cpu_block * block_size_factor + j
cpu_block * blocks_per_chunk + j
for cpu_block in cpu_blocks
for j in range(block_size_factor)
for j in range(blocks_per_chunk)
]
for cpu_blocks in cpu_blocks_per_group
]
# skip sub-blocks from group 2 to test unaligned transfers.
sub_blocks_to_skip = block_size_factor - 1 # e.g. 2 when block_size_factor=3
sub_blocks_to_skip = blocks_per_chunk - 1 # e.g. 2 when blocks_per_chunk=3
if sub_blocks_to_skip > 0:
gpu_blocks_per_group[2] = gpu_blocks_per_group[2][
sub_blocks_to_skip:-sub_blocks_to_skip
@@ -347,7 +347,7 @@ def test_transfer_multi_group(
cpu_blocks_expanded_per_group, gpu_blocks_per_group
)
]
num_dst_sub_blocks = num_cpu_blocks * block_size_factor
num_dst_sub_blocks = num_cpu_blocks * blocks_per_chunk
else:
handler = worker._load_handler
src_spec = CPULoadStoreSpec(cpu_blocks)