[KV Offload] Define clean backend configuration boundary (#48150)
Signed-off-by: Change72 <changg@nvidia.com> Signed-off-by: Chang Guo <cguo51@asu.edu> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
Claude Fable 5
Or Ozeri
OpenAI Codex
parent
8c3393f373
commit
a9531edfa6
@@ -23,7 +23,7 @@ from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
NUM_GPU_BLOCKS = [64]
|
||||
NUM_CPU_BLOCKS = [256]
|
||||
GPU_PAGE_SIZES = [512, 1024]
|
||||
BLOCK_SIZE_FACTORS = [1, 3]
|
||||
BLOCKS_PER_CHUNK_VALUES = [1, 3]
|
||||
NUM_TENSORS = [4]
|
||||
SEEDS = [0]
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
@@ -35,7 +35,7 @@ NUM_MAPPINGS_PER_GROUP = [2]
|
||||
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
|
||||
@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
|
||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||
@pytest.mark.parametrize("block_size_factor", BLOCK_SIZE_FACTORS)
|
||||
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
|
||||
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_tensors", NUM_TENSORS)
|
||||
@@ -48,7 +48,7 @@ def test_transfer(
|
||||
gpu_to_cpu: bool,
|
||||
num_mappings: int,
|
||||
gpu_page_size_bytes: int,
|
||||
block_size_factor: int,
|
||||
blocks_per_chunk: int,
|
||||
num_gpu_blocks: int,
|
||||
num_cpu_blocks: int,
|
||||
num_tensors: int,
|
||||
@@ -92,7 +92,7 @@ def test_transfer(
|
||||
mmap_region: SharedOffloadRegion | None = None
|
||||
if use_shared_memory:
|
||||
cpu_page_size = round_up(
|
||||
gpu_page_size_bytes * num_tensors * block_size_factor,
|
||||
gpu_page_size_bytes * num_tensors * blocks_per_chunk,
|
||||
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
|
||||
)
|
||||
mmap_region = SharedOffloadRegion(
|
||||
@@ -105,25 +105,25 @@ def test_transfer(
|
||||
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
mmap_region=mmap_region,
|
||||
)
|
||||
|
||||
# select block mappings
|
||||
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * block_size_factor)
|
||||
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * blocks_per_chunk)
|
||||
cpu_blocks = random.sample(range(num_cpu_blocks), num_mappings)
|
||||
|
||||
# expand cpu blocks to gpu-page granularity for uniform comparison:
|
||||
# each cpu block maps to block_size_factor consecutive sub-blocks
|
||||
# each cpu block maps to blocks_per_chunk consecutive sub-blocks
|
||||
cpu_blocks_expanded = [
|
||||
cpu_block * block_size_factor + j
|
||||
cpu_block * blocks_per_chunk + j
|
||||
for cpu_block in cpu_blocks
|
||||
for j in range(block_size_factor)
|
||||
for j in range(blocks_per_chunk)
|
||||
]
|
||||
|
||||
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU block
|
||||
blocks_to_skip = block_size_factor - 1
|
||||
blocks_to_skip = blocks_per_chunk - 1
|
||||
if blocks_to_skip > 0:
|
||||
gpu_blocks = gpu_blocks[blocks_to_skip:]
|
||||
cpu_blocks_expanded = cpu_blocks_expanded[blocks_to_skip:]
|
||||
@@ -214,7 +214,7 @@ def test_transfer(
|
||||
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
|
||||
@pytest.mark.parametrize("num_mappings_per_group", NUM_MAPPINGS_PER_GROUP)
|
||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||
@pytest.mark.parametrize("block_size_factor", BLOCK_SIZE_FACTORS)
|
||||
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
|
||||
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@@ -225,7 +225,7 @@ def test_transfer_multi_group(
|
||||
gpu_to_cpu: bool,
|
||||
num_mappings_per_group: int,
|
||||
gpu_page_size_bytes: int,
|
||||
block_size_factor: int,
|
||||
blocks_per_chunk: int,
|
||||
num_gpu_blocks: int,
|
||||
num_cpu_blocks: int,
|
||||
seed: int,
|
||||
@@ -234,7 +234,7 @@ def test_transfer_multi_group(
|
||||
"""Test transfers with three KV cache groups:
|
||||
- Group 0: aligned transfer with num_mappings_per_group blocks
|
||||
- Group 1: zero blocks (empty group)
|
||||
- Group 2: unaligned CPU->GPU transfer (logical_offset=block_size_factor-1,
|
||||
- Group 2: unaligned CPU->GPU transfer (logical_offset=blocks_per_chunk-1,
|
||||
causing the implementation to skip source sub-blocks) with
|
||||
num_mappings_per_group blocks
|
||||
"""
|
||||
@@ -275,7 +275,7 @@ def test_transfer_multi_group(
|
||||
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=canonical_kv_caches,
|
||||
block_size_factor=block_size_factor,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
)
|
||||
|
||||
@@ -283,7 +283,7 @@ def test_transfer_multi_group(
|
||||
group_sizes_in_cpu_blocks = [num_mappings_per_group, 0, num_mappings_per_group]
|
||||
|
||||
total_cpu_blocks = sum(group_sizes_in_cpu_blocks)
|
||||
total_gpu_blocks_needed = total_cpu_blocks * block_size_factor
|
||||
total_gpu_blocks_needed = total_cpu_blocks * blocks_per_chunk
|
||||
gpu_blocks_all = random.sample(range(num_gpu_blocks), total_gpu_blocks_needed)
|
||||
cpu_blocks_all = random.sample(range(num_cpu_blocks), total_cpu_blocks)
|
||||
|
||||
@@ -293,7 +293,7 @@ def test_transfer_multi_group(
|
||||
gpu_offset = 0
|
||||
cpu_offset = 0
|
||||
for size in group_sizes_in_cpu_blocks:
|
||||
gpu_count = size * block_size_factor
|
||||
gpu_count = size * blocks_per_chunk
|
||||
gpu_blocks_per_group.append(gpu_blocks_all[gpu_offset : gpu_offset + gpu_count])
|
||||
cpu_blocks_per_group.append(cpu_blocks_all[cpu_offset : cpu_offset + size])
|
||||
gpu_offset += gpu_count
|
||||
@@ -302,15 +302,15 @@ def test_transfer_multi_group(
|
||||
# expand cpu blocks to gpu-page granularity
|
||||
cpu_blocks_expanded_per_group = [
|
||||
[
|
||||
cpu_block * block_size_factor + j
|
||||
cpu_block * blocks_per_chunk + j
|
||||
for cpu_block in cpu_blocks
|
||||
for j in range(block_size_factor)
|
||||
for j in range(blocks_per_chunk)
|
||||
]
|
||||
for cpu_blocks in cpu_blocks_per_group
|
||||
]
|
||||
|
||||
# skip sub-blocks from group 2 to test unaligned transfers.
|
||||
sub_blocks_to_skip = block_size_factor - 1 # e.g. 2 when block_size_factor=3
|
||||
sub_blocks_to_skip = blocks_per_chunk - 1 # e.g. 2 when blocks_per_chunk=3
|
||||
if sub_blocks_to_skip > 0:
|
||||
gpu_blocks_per_group[2] = gpu_blocks_per_group[2][
|
||||
sub_blocks_to_skip:-sub_blocks_to_skip
|
||||
@@ -347,7 +347,7 @@ def test_transfer_multi_group(
|
||||
cpu_blocks_expanded_per_group, gpu_blocks_per_group
|
||||
)
|
||||
]
|
||||
num_dst_sub_blocks = num_cpu_blocks * block_size_factor
|
||||
num_dst_sub_blocks = num_cpu_blocks * blocks_per_chunk
|
||||
else:
|
||||
handler = worker._load_handler
|
||||
src_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
|
||||
Reference in New Issue
Block a user