[Refactor][kv_offload]: rename block→chunk (#52615)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
Ronen Schaffer
2026-09-09 17:45:49 +03:00
committed by GitHub
parent 9ffb8cea96
commit 138d137b5b
34 changed files with 933 additions and 916 deletions
@@ -7,7 +7,7 @@ import pytest
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
@@ -20,10 +20,10 @@ class _DummyCachePolicy(CachePolicy):
def __init__(self, cache_capacity: int) -> None:
self.cache_capacity = cache_capacity
def get(self, key: OffloadKey) -> BlockStatus | None:
def get(self, key: OffloadKey) -> ChunkStatus | None:
return None
def insert(self, key: OffloadKey, block: BlockStatus) -> None:
def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
pass
def remove(self, key: OffloadKey) -> None:
@@ -34,7 +34,7 @@ class _DummyCachePolicy(CachePolicy):
def evict(
self, n: int, protected: set[OffloadKey]
) -> list[tuple[OffloadKey, BlockStatus]] | None:
) -> list[tuple[OffloadKey, ChunkStatus]] | None:
return None
def clear(self) -> None:
@@ -72,7 +72,7 @@ class TestCachePolicyFactory:
policy_cls = CachePolicyFactory.get_cache_policy_cls("dummy")
assert policy_cls is _DummyCachePolicy
manager = CPUOffloadingManager(num_blocks=4, cache_policy="dummy")
manager = CPUOffloadingManager(num_chunks=4, cache_policy="dummy")
assert isinstance(manager._policy, _DummyCachePolicy)
def test_unregistered_policy_raises(self):
@@ -98,7 +98,7 @@ class TestCachePolicyFactory:
"""End-to-end: CPUOffloadingManager resolves an unregistered policy
purely from cache_policy_module_path."""
manager = CPUOffloadingManager(
num_blocks=4,
num_chunks=4,
cache_policy="_DummyCachePolicy",
cache_policy_module_path="tests.v1.kv_offload.cpu.policies.test_factory",
)
@@ -232,9 +232,9 @@ def test_cross_topology_roundtrip(writer_tp: int, reader_tp: int):
def canonical_view(rank: int, world_size: int) -> torch.Tensor:
region = SharedOffloadRegion(
engine_id=engine_id,
num_blocks=num_blocks,
num_chunks=num_blocks,
rank=rank,
kv_bytes_per_block=row_stride,
kv_bytes_per_chunk=row_stride,
cpu_page_size=row_stride // world_size,
)
regions.append(region)
+46 -46
View File
@@ -26,7 +26,7 @@ from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
NUM_GPU_BLOCKS = [64]
NUM_CPU_BLOCKS = [256]
NUM_CPU_CHUNKS = [256]
GPU_PAGE_SIZES = [512, 1024]
BLOCKS_PER_CHUNK_VALUES = [1, 3]
NUM_TENSORS = [4]
@@ -225,7 +225,7 @@ def test_worker_syncs_before_cleanup_after_handler_failure(
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
@pytest.mark.parametrize("num_cpu_chunks", NUM_CPU_CHUNKS)
@pytest.mark.parametrize("num_tensors", NUM_TENSORS)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", DEVICES)
@@ -241,7 +241,7 @@ def test_transfer(
gpu_page_size_bytes: int,
blocks_per_chunk: int,
num_gpu_blocks: int,
num_cpu_blocks: int,
num_cpu_chunks: int,
num_tensors: int,
seed: int,
device: str,
@@ -288,41 +288,41 @@ def test_transfer(
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
)
simulated_world_size = 2
kv_bytes_per_block = (
kv_bytes_per_chunk = (
cpu_page_size if replicated_layout else cpu_page_size * simulated_world_size
)
mmap_region = SharedOffloadRegion(
engine_id=str(uuid.uuid4()),
num_blocks=num_cpu_blocks,
num_chunks=num_cpu_chunks,
rank=0,
kv_bytes_per_block=kv_bytes_per_block,
kv_bytes_per_chunk=kv_bytes_per_chunk,
cpu_page_size=cpu_page_size,
)
worker = CPUOffloadingWorker(
kv_caches=kv_caches,
blocks_per_chunk=blocks_per_chunk,
num_cpu_blocks=num_cpu_blocks,
num_cpu_chunks=num_cpu_chunks,
mmap_region=mmap_region,
)
# select block mappings
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * blocks_per_chunk)
cpu_blocks = random.sample(range(num_cpu_blocks), num_mappings)
cpu_chunks = random.sample(range(num_cpu_chunks), num_mappings)
# expand cpu blocks to gpu-page granularity for uniform comparison:
# each cpu block maps to blocks_per_chunk consecutive sub-blocks
cpu_blocks_expanded = [
cpu_block * blocks_per_chunk + j
for cpu_block in cpu_blocks
# expand cpu chunks to gpu-page granularity for uniform comparison:
# each cpu chunk maps to blocks_per_chunk consecutive sub-blocks
cpu_chunks_expanded = [
cpu_chunk * blocks_per_chunk + j
for cpu_chunk in cpu_chunks
for j in range(blocks_per_chunk)
]
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU block
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU chunk
blocks_to_skip = blocks_per_chunk - 1
if blocks_to_skip > 0:
gpu_blocks = gpu_blocks[blocks_to_skip:]
cpu_blocks_expanded = cpu_blocks_expanded[blocks_to_skip:]
cpu_chunks_expanded = cpu_chunks_expanded[blocks_to_skip:]
# set transfer direction
if gpu_to_cpu:
@@ -330,16 +330,16 @@ def test_transfer(
src_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
)
dst_spec = CPULoadStoreSpec(cpu_blocks)
dst_to_src = dict(zip(cpu_blocks_expanded, gpu_blocks))
dst_spec = CPULoadStoreSpec(cpu_chunks)
dst_to_src = dict(zip(cpu_chunks_expanded, gpu_blocks))
num_dst_sub_blocks = num_gpu_blocks
else:
handler = worker._load_handler
src_spec = CPULoadStoreSpec(cpu_blocks)
src_spec = CPULoadStoreSpec(cpu_chunks)
dst_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
)
dst_to_src = dict(zip(gpu_blocks, cpu_blocks_expanded))
dst_to_src = dict(zip(gpu_blocks, cpu_chunks_expanded))
num_dst_sub_blocks = num_gpu_blocks
# randomize src and dst tensors before transfer
@@ -410,7 +410,7 @@ def test_transfer(
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
@pytest.mark.parametrize("num_cpu_chunks", NUM_CPU_CHUNKS)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", DEVICES)
@torch.inference_mode()
@@ -421,7 +421,7 @@ def test_transfer_multi_group(
gpu_page_size_bytes: int,
blocks_per_chunk: int,
num_gpu_blocks: int,
num_cpu_blocks: int,
num_cpu_chunks: int,
seed: int,
device: str,
) -> None:
@@ -470,37 +470,37 @@ def test_transfer_multi_group(
worker = CPUOffloadingWorker(
kv_caches=canonical_kv_caches,
blocks_per_chunk=blocks_per_chunk,
num_cpu_blocks=num_cpu_blocks,
num_cpu_chunks=num_cpu_chunks,
)
# group 0: aligned, group 1: empty, group 2: unaligned on CPU->GPU
group_sizes_in_cpu_blocks = [num_mappings_per_group, 0, num_mappings_per_group]
group_sizes_in_cpu_chunks = [num_mappings_per_group, 0, num_mappings_per_group]
total_cpu_blocks = sum(group_sizes_in_cpu_blocks)
total_gpu_blocks_needed = total_cpu_blocks * blocks_per_chunk
total_cpu_chunks = sum(group_sizes_in_cpu_chunks)
total_gpu_blocks_needed = total_cpu_chunks * blocks_per_chunk
gpu_blocks_all = random.sample(range(num_gpu_blocks), total_gpu_blocks_needed)
cpu_blocks_all = random.sample(range(num_cpu_blocks), total_cpu_blocks)
cpu_chunks_all = random.sample(range(num_cpu_chunks), total_cpu_chunks)
# split gpu/cpu blocks per group
# split gpu blocks / cpu chunks per group
gpu_blocks_per_group: list[list[int]] = []
cpu_blocks_per_group: list[list[int]] = []
cpu_chunks_per_group: list[list[int]] = []
gpu_offset = 0
cpu_offset = 0
for size in group_sizes_in_cpu_blocks:
for size in group_sizes_in_cpu_chunks:
gpu_count = size * blocks_per_chunk
gpu_blocks_per_group.append(gpu_blocks_all[gpu_offset : gpu_offset + gpu_count])
cpu_blocks_per_group.append(cpu_blocks_all[cpu_offset : cpu_offset + size])
cpu_chunks_per_group.append(cpu_chunks_all[cpu_offset : cpu_offset + size])
gpu_offset += gpu_count
cpu_offset += size
# expand cpu blocks to gpu-page granularity
cpu_blocks_expanded_per_group = [
# expand cpu chunks to gpu-page granularity
cpu_chunks_expanded_per_group = [
[
cpu_block * blocks_per_chunk + j
for cpu_block in cpu_blocks
cpu_chunk * blocks_per_chunk + j
for cpu_chunk in cpu_chunks
for j in range(blocks_per_chunk)
]
for cpu_blocks in cpu_blocks_per_group
for cpu_chunks in cpu_chunks_per_group
]
# skip sub-blocks from group 2 to test unaligned transfers.
@@ -509,7 +509,7 @@ def test_transfer_multi_group(
gpu_blocks_per_group[2] = gpu_blocks_per_group[2][
sub_blocks_to_skip:-sub_blocks_to_skip
]
cpu_blocks_expanded_per_group[2] = cpu_blocks_expanded_per_group[2][
cpu_chunks_expanded_per_group[2] = cpu_chunks_expanded_per_group[2][
sub_blocks_to_skip:-sub_blocks_to_skip
]
@@ -520,10 +520,10 @@ def test_transfer_multi_group(
gpu_blocks.extend(gpu_blks)
group_sizes.append(len(gpu_blks))
# build flat cpu_blocks list
cpu_blocks = []
for cpu_blks in cpu_blocks_per_group:
cpu_blocks.extend(cpu_blks)
# build flat cpu_chunks list
cpu_chunks = []
for cpu_chnks in cpu_chunks_per_group:
cpu_chunks.extend(cpu_chnks)
# block_indices: only relevant for unaligned transfers
block_indices: list[int] = [0, 0, sub_blocks_to_skip]
@@ -533,18 +533,18 @@ def test_transfer_multi_group(
src_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
)
dst_spec = CPULoadStoreSpec(cpu_blocks)
dst_spec = CPULoadStoreSpec(cpu_chunks)
# per-group mapping: cpu sub-block -> gpu sub-block
dst_to_src_per_group = [
dict(zip(expanded, gpu_blks))
for expanded, gpu_blks in zip(
cpu_blocks_expanded_per_group, gpu_blocks_per_group
cpu_chunks_expanded_per_group, gpu_blocks_per_group
)
]
num_dst_sub_blocks = num_cpu_blocks * blocks_per_chunk
num_dst_sub_blocks = num_cpu_chunks * blocks_per_chunk
else:
handler = worker._load_handler
src_spec = CPULoadStoreSpec(cpu_blocks)
src_spec = CPULoadStoreSpec(cpu_chunks)
dst_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
)
@@ -552,7 +552,7 @@ def test_transfer_multi_group(
dst_to_src_per_group = [
dict(zip(gpu_blks, expanded))
for gpu_blks, expanded in zip(
gpu_blocks_per_group, cpu_blocks_expanded_per_group
gpu_blocks_per_group, cpu_chunks_expanded_per_group
)
]
num_dst_sub_blocks = num_gpu_blocks
@@ -644,7 +644,7 @@ def test_load_waits_for_pending_compute_stream_writes(default_vllm_config) -> No
],
),
blocks_per_chunk=1,
num_cpu_blocks=num_blocks,
num_cpu_chunks=num_blocks,
)
worker._load_handler.src_tensors[0].fill_(sentinel)
expected = torch.full((page_size_bytes,), sentinel, dtype=torch.int8)
+144 -144
View File
@@ -36,7 +36,7 @@ _EMPTY_REQ_CTX = make_req_context()
def make_cpu_manager(
num_blocks: int = 4,
num_chunks: int = 4,
cache_policy: str = "lru",
cache_policy_module_path: str | None = None,
enable_events: bool = False,
@@ -44,7 +44,7 @@ def make_cpu_manager(
max_tracker_size: int = 64_000,
) -> CPUOffloadingManager:
return CPUOffloadingManager(
num_blocks=num_blocks,
num_chunks=num_chunks,
cache_policy=cache_policy,
cache_policy_module_path=cache_policy_module_path,
enable_events=enable_events,
@@ -56,7 +56,7 @@ def make_cpu_manager(
@dataclass
class ExpectedPrepareStoreOutput:
keys_to_store: list[int]
store_block_ids: list[int]
store_chunk_ids: list[int]
evicted_keys: list[int]
@@ -93,9 +93,9 @@ def verify_store_output(
store_spec = prepare_store_output.store_spec
assert isinstance(store_spec, CPULoadStoreSpec)
expected_array = np.array(
expected_prepare_store_output.store_block_ids, dtype=np.int64
expected_prepare_store_output.store_chunk_ids, dtype=np.int64
)
assert np.array_equal(expected_array, store_spec.block_ids)
assert np.array_equal(expected_array, store_spec.chunk_ids)
def verify_load_output(
@@ -103,7 +103,7 @@ def verify_load_output(
):
assert isinstance(prepare_load_output, CPULoadStoreSpec)
expected_array = np.array(expected_prepare_load_output, dtype=np.int64)
assert np.array_equal(expected_array, prepare_load_output.block_ids)
assert np.array_equal(expected_array, prepare_load_output.chunk_ids)
def check_split_usage_stats(
@@ -146,7 +146,7 @@ def verify_events(
def test_cpu_eviction_removed_precedes_stored():
"""An eviction is announced before the store that reuses its capacity."""
manager = make_cpu_manager(num_blocks=2, enable_events=True)
manager = make_cpu_manager(num_chunks=2, enable_events=True)
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -164,23 +164,23 @@ def test_cpu_eviction_removed_precedes_stored():
@pytest.mark.parametrize("eviction_policy", ["lru", "arc"])
def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
def test_already_stored_chunk_not_evicted_during_prepare_store(eviction_policy):
"""
Regression test: a block that is already stored must not be evicted
by prepare_store() when it needs to make room for new blocks.
Regression test: a chunk that is already stored must not be evicted
by prepare_store() when it needs to make room for new chunks.
Applies to both lru and arc policies.
Scenario:
- Store blocks [1, 2] and complete.
- touch([1]) makes block 2 the LRU candidate.
- Store chunks [1, 2] and complete.
- touch([1]) makes chunk 2 the LRU candidate.
- prepare_store([2, 3, 4, 5]):
* block 2 is filtered out as "already stored"
* but without the fix, block 2 would be evicted as the LRU
* chunk 2 is filtered out as "already stored"
* but without the fix, chunk 2 would be evicted as the LRU
candidate to make room for [3, 4, 5]
- After complete_store([2, 3, 4, 5]), block 2 must still be present.
- After complete_store([2, 3, 4, 5]), chunk 2 must still be present.
"""
manager = make_cpu_manager(
num_blocks=4,
num_chunks=4,
cache_policy=eviction_policy,
enable_events=True,
)
@@ -189,33 +189,33 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
# touch [1] to make block 2 the LRU candidate
# touch [1] to make chunk 2 the LRU candidate
manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
# prepare_store([2, 3, 4, 5]):
# - block 2 is already stored -> filtered out of keys_to_store
# - block 2 must NOT be evicted even though it is the LRU candidate
# - block 1 (ID 0) is evicted instead; new blocks [3,4,5] get IDs 2,3,0
# - chunk 2 is already stored -> filtered out of keys_to_store
# - chunk 2 must NOT be evicted even though it is the LRU candidate
# - chunk 1 (ID 0) is evicted instead; new chunks [3,4,5] get IDs 2,3,0
prepare_store_output = manager.prepare_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
verify_store_output(
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[3, 4, 5],
store_block_ids=[2, 3, 0],
evicted_keys=[1], # block 1 evicted, not block 2
store_chunk_ids=[2, 3, 0],
evicted_keys=[1], # chunk 1 evicted, not chunk 2
),
)
# complete_store must not silently drop block 2
# complete_store must not silently drop chunk 2
manager.complete_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
# block 2 must still be present in the cache
# chunk 2 must still be present in the cache
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
def test_filter_reused_manager_reports_stores_skipped_counter():
manager = make_cpu_manager(
num_blocks=4,
num_chunks=4,
cache_policy="lru",
store_threshold=2,
)
@@ -226,7 +226,7 @@ def test_filter_reused_manager_reports_stores_skipped_counter():
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[],
store_block_ids=[],
store_chunk_ids=[],
evicted_keys=[],
),
)
@@ -247,34 +247,34 @@ def test_cpu_manager_reports_cache_usage_gauge():
] == pytest.approx(value)
# Zero-capacity manager always reports 0.0
manager = make_cpu_manager(num_blocks=0)
manager = make_cpu_manager(num_chunks=0)
check_usage_stats(manager, 0.0)
# Empty manager (4 blocks, none allocated): usage = 0.0
manager = make_cpu_manager(num_blocks=4)
# Empty manager (4 chunks, none allocated): usage = 0.0
manager = make_cpu_manager(num_chunks=4)
check_usage_stats(manager, 0.0)
# After allocating 2 of 4 blocks: usage = 0.5
# After allocating 2 of 4 chunks: usage = 0.5
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
check_usage_stats(manager, 0.5)
# After filling all 4 blocks: usage = 1.0
# After filling all 4 chunks: usage = 1.0
manager.prepare_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
check_usage_stats(manager, 1.0)
# After completing store, the blocks becomes evictable as it is not actively used
# After completing store, the chunks become evictable as not actively used
# and usage drops.
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
check_usage_stats(manager, 0.5)
# After completing store, the blocks becomes evictable as it is not actively used
# After completing store, the chunks become evictable as not actively used
# and usage drops.
manager.complete_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
check_usage_stats(manager, 0.0)
def test_cpu_manager_reports_allocation_size_histogram():
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -297,12 +297,12 @@ def test_cpu_manager_reports_allocation_size_histogram():
def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
def fail_allocate_blocks(keys):
def fail_allocate_chunks(keys):
raise RuntimeError("allocation failed")
monkeypatch.setattr(manager, "_allocate_blocks", fail_allocate_blocks)
monkeypatch.setattr(manager, "_allocate_chunks", fail_allocate_chunks)
with pytest.raises(RuntimeError, match="allocation failed"):
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
@@ -316,7 +316,7 @@ def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
def test_cpu_manager_reports_allocation_size_on_eviction_failure():
manager = make_cpu_manager(num_blocks=1, cache_policy="lru")
manager = make_cpu_manager(num_chunks=1, cache_policy="lru")
manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
manager.get_stats()
@@ -332,7 +332,7 @@ def test_cpu_manager_reports_allocation_size_on_eviction_failure():
def test_cpu_manager_reports_cache_write_and_read_usage_gauges():
manager = make_cpu_manager(num_blocks=4)
manager = make_cpu_manager(num_chunks=4)
# Store path: pins write usage until complete_store.
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -356,7 +356,7 @@ def test_cpu_manager_reports_cache_write_and_read_usage_gauges():
def test_cpu_manager_clears_write_usage_after_failed_store():
manager = make_cpu_manager(num_blocks=4)
manager = make_cpu_manager(num_chunks=4)
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
check_split_usage_stats(manager, write=0.5, read=0.0, total=0.5)
@@ -369,8 +369,8 @@ def test_cpu_manager():
"""
Tests CPUOffloadingManager with lru policy.
"""
# initialize a CPU manager with a capacity of 4 blocks
cpu_manager = make_cpu_manager(num_blocks=4, cache_policy="lru", enable_events=True)
# initialize a CPU manager with a capacity of 4 chunks
cpu_manager = make_cpu_manager(num_chunks=4, cache_policy="lru", enable_events=True)
# prepare store [1, 2]
prepare_store_output = cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -378,7 +378,7 @@ def test_cpu_manager():
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[1, 2],
store_block_ids=[0, 1],
store_chunk_ids=[0, 1],
evicted_keys=[],
),
)
@@ -407,7 +407,7 @@ def test_cpu_manager():
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[3, 4, 5],
store_block_ids=[2, 3, 0],
store_chunk_ids=[2, 3, 0],
evicted_keys=[1],
),
)
@@ -445,7 +445,7 @@ def test_cpu_manager():
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[6, 7, 8],
store_block_ids=[1, 0, 3],
store_chunk_ids=[1, 0, 3],
evicted_keys=[4, 5, 2],
),
)
@@ -462,7 +462,7 @@ def test_cpu_manager():
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[9],
store_block_ids=[3],
store_chunk_ids=[3],
evicted_keys=[8],
),
)
@@ -482,38 +482,38 @@ def test_cpu_manager():
def test_prepare_load_preserves_key_order():
"""block_ids[i] must correspond to keys[i] (co-indexed invariant)."""
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
"""chunk_ids[i] must correspond to keys[i] (co-indexed invariant)."""
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
key_a, key_b, key_c = to_key(0), to_key(1), to_key(2)
# Store all three keys and learn their block ID assignments
# Store all three keys and learn their chunk ID assignments
store_output = manager.prepare_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
assert store_output is not None
assert isinstance(store_output.store_spec, CPULoadStoreSpec)
key_to_block_id = {
k: int(bid)
for k, bid in zip(store_output.keys_to_store, store_output.store_spec.block_ids)
key_to_chunk_id = {
k: int(cid)
for k, cid in zip(store_output.keys_to_store, store_output.store_spec.chunk_ids)
}
manager.complete_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
# Forward order: [a, b, c]
spec_fwd = manager.prepare_load([key_a, key_b, key_c], _EMPTY_REQ_CTX)
assert isinstance(spec_fwd, CPULoadStoreSpec)
assert [int(x) for x in spec_fwd.block_ids] == [
key_to_block_id[key_a],
key_to_block_id[key_b],
key_to_block_id[key_c],
assert [int(x) for x in spec_fwd.chunk_ids] == [
key_to_chunk_id[key_a],
key_to_chunk_id[key_b],
key_to_chunk_id[key_c],
]
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
# Arbitrary permutation: [b, c, a]
spec_perm = manager.prepare_load([key_b, key_c, key_a], _EMPTY_REQ_CTX)
assert isinstance(spec_perm, CPULoadStoreSpec)
assert [int(x) for x in spec_perm.block_ids] == [
key_to_block_id[key_b],
key_to_block_id[key_c],
key_to_block_id[key_a],
assert [int(x) for x in spec_perm.chunk_ids] == [
key_to_chunk_id[key_b],
key_to_chunk_id[key_c],
key_to_chunk_id[key_a],
]
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
@@ -522,10 +522,10 @@ class TestARCPolicy:
"""Unit tests for CPUOffloadingManager with ARC eviction policy."""
def _make_manager(
self, num_blocks: int = 4, enable_events: bool = True
self, num_chunks: int = 4, enable_events: bool = True
) -> tuple[CPUOffloadingManager, ARCCachePolicy]:
manager = make_cpu_manager(
num_blocks=num_blocks,
num_chunks=num_chunks,
cache_policy="arc",
enable_events=enable_events,
)
@@ -548,7 +548,7 @@ class TestARCPolicy:
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[1, 2],
store_block_ids=[0, 1],
store_chunk_ids=[0, 1],
evicted_keys=[],
),
)
@@ -569,36 +569,36 @@ class TestARCPolicy:
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.MISS
# blocks should be in T1 (recent)
# chunks should be in T1 (recent)
assert len(arc_policy.t1) == 2
assert len(arc_policy.t2) == 0
def test_t1_to_t2_promotion(self):
"""
Tests that accessing a block in T1 promotes it to T2 (frequent).
Tests that accessing a chunk in T1 promotes it to T2 (frequent).
This is a key feature of ARC's adaptive behavior.
"""
cpu_manager, arc_policy = self._make_manager(enable_events=False)
# store and complete block 1
# store and complete chunk 1
cpu_manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1]), _EMPTY_REQ_CTX)
# block 1 starts in T1 (recent)
# chunk 1 starts in T1 (recent)
assert to_keys([1])[0] in arc_policy.t1
assert to_keys([1])[0] not in arc_policy.t2
# touch block 1 (simulate second access)
# touch chunk 1 (simulate second access)
cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
# block 1 should now be in T2 (frequent)
# chunk 1 should now be in T2 (frequent)
assert to_keys([1])[0] not in arc_policy.t1
assert to_keys([1])[0] in arc_policy.t2
def test_eviction_with_load(self):
"""
Tests ARC eviction behavior similar to LRU test.
Verifies that blocks being loaded (ref_cnt > 0) cannot be evicted.
Verifies that chunks being loaded (ref_cnt > 0) cannot be evicted.
"""
cpu_manager, _ = self._make_manager()
@@ -610,7 +610,7 @@ class TestARCPolicy:
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[1, 2, 3, 4],
store_block_ids=[0, 1, 2, 3],
store_chunk_ids=[0, 1, 2, 3],
evicted_keys=[],
),
)
@@ -628,36 +628,36 @@ class TestARCPolicy:
cpu_manager.complete_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
# now prepare store [5, 6, 7] should succeed
# ARC will evict blocks one at a time from T1 as needed
# ARC will evict chunks one at a time from T1 as needed
prepare_store_output = cpu_manager.prepare_store(
to_keys([5, 6, 7]), _EMPTY_REQ_CTX
)
assert prepare_store_output is not None
# Should successfully evict enough blocks to make room (at least 1)
# Should successfully evict enough chunks to make room (at least 1)
assert len(prepare_store_output.evicted_keys) >= 1
def test_adaptive_target(self):
"""
Tests ARC's adaptive target adjustment via ghost lists.
When a block in B1 (ghost list) is accessed, target_t1_size increases.
When a block in B2 is accessed, target_t1_size decreases.
When a chunk in B1 (ghost list) is accessed, target_t1_size increases.
When a chunk in B2 is accessed, target_t1_size decreases.
"""
cpu_manager, arc_policy = self._make_manager(num_blocks=2, enable_events=False)
cpu_manager, arc_policy = self._make_manager(num_chunks=2, enable_events=False)
# store blocks 1, 2 (fills cache)
# store chunks 1, 2 (fills cache)
cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
initial_target = arc_policy.target_t1_size
# store block 3, evicting block 1 (moves to B1 ghost list)
# store chunk 3, evicting chunk 1 (moves to B1 ghost list)
cpu_manager.prepare_store(to_keys([3]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([3]), _EMPTY_REQ_CTX)
# block 1 should be in B1 (ghost list)
# chunk 1 should be in B1 (ghost list)
assert to_keys([1])[0] in arc_policy.b1
# touch block 1 (cache miss, but in B1)
# touch chunk 1 (cache miss, but in B1)
# this should increase target_t1_size (favor recency)
cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
@@ -671,11 +671,11 @@ class TestARCPolicy:
"""
cpu_manager, arc_policy = self._make_manager(enable_events=False)
# store blocks 1, 2, 3, 4
# store chunks 1, 2, 3, 4
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
# promote blocks 3, 4 to T2 by touching them
# promote chunks 3, 4 to T2 by touching them
cpu_manager.touch(to_keys([3, 4]), _EMPTY_REQ_CTX)
# now: T1 = {1, 2}, T2 = {3, 4}
@@ -686,22 +686,22 @@ class TestARCPolicy:
# (when |T1| >= target, evict from T1)
arc_policy.target_t1_size = 1
# store block 5, should evict from T1 (block 1, LRU in T1)
# store chunk 5, should evict from T1 (chunk 1, LRU in T1)
output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
assert output is not None
assert to_keys([1]) == output.evicted_keys
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX)
# block 1 should be in B1 (ghost list)
# chunk 1 should be in B1 (ghost list)
assert to_keys([1])[0] in arc_policy.b1
# block 5 should be in T1
# chunk 5 should be in T1
assert to_keys([5])[0] in arc_policy.t1
def test_batch_eviction_scans_t1_and_t2_once(self):
"""ARC batch eviction must preserve order without restarting scans."""
cpu_manager, arc_policy = self._make_manager(
num_blocks=256, enable_events=False
num_chunks=256, enable_events=False
)
keys = to_keys(list(range(256)))
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
@@ -721,13 +721,13 @@ class TestARCPolicy:
t2_order = list(arc_policy.t2)
expected_t1 = [
key
for key, block in arc_policy.t1.items()
if block.ref_cnt == 0 and key not in protected
for key, chunk in arc_policy.t1.items()
if chunk.ref_cnt == 0 and key not in protected
][:num_t1_evictions]
expected_t2 = [
key
for key, block in arc_policy.t2.items()
if block.ref_cnt == 0 and key not in protected
for key, chunk in arc_policy.t2.items()
if chunk.ref_cnt == 0 and key not in protected
][:num_t2_evictions]
expected_t1_scans = t1_order.index(expected_t1[-1]) + 1
expected_t2_scans = t2_order.index(expected_t2[-1]) + 1
@@ -746,7 +746,7 @@ class TestARCPolicy:
def test_batch_eviction_falls_back_after_t1_iterator_exhausted(self):
"""An exhausted T1 scan must keep falling back to T2."""
cpu_manager, arc_policy = self._make_manager(num_blocks=8, enable_events=False)
cpu_manager, arc_policy = self._make_manager(num_chunks=8, enable_events=False)
keys = to_keys(list(range(8)))
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
@@ -764,8 +764,8 @@ class TestARCPolicy:
# the target, so each remaining selection must retry T1 then use T2.
eligible_t1 = [
key
for key, block in arc_policy.t1.items()
if block.ref_cnt == 0 and key not in protected
for key, chunk in arc_policy.t1.items()
if chunk.ref_cnt == 0 and key not in protected
]
assert eligible_t1 == t1_order[:1]
assert len(t1_order) - 1 >= int(arc_policy.target_t1_size)
@@ -839,7 +839,7 @@ class TestARCPolicy:
def test_batch_eviction_failure_is_atomic(self):
"""Finding only some candidates must not partially evict the cache."""
cpu_manager, arc_policy = self._make_manager(num_blocks=4, enable_events=False)
cpu_manager, arc_policy = self._make_manager(num_chunks=4, enable_events=False)
keys = to_keys(list(range(4)))
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
@@ -858,13 +858,13 @@ class TestARCPolicy:
Tests that ghost lists (B1, B2) don't grow unbounded.
They should be capped at cache_capacity.
"""
cpu_manager, arc_policy = self._make_manager(num_blocks=2, enable_events=False)
cpu_manager, arc_policy = self._make_manager(num_chunks=2, enable_events=False)
# fill cache with blocks 1, 2
# fill cache with chunks 1, 2
cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
# store many blocks to fill ghost lists
# store many chunks to fill ghost lists
for i in range(3, 20):
cpu_manager.prepare_store(to_keys([i]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([i]), _EMPTY_REQ_CTX)
@@ -880,7 +880,7 @@ class TestARCPolicy:
"""
cpu_manager, arc_policy = self._make_manager()
# store blocks 1, 2, 3, 4
# store chunks 1, 2, 3, 4
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
@@ -895,13 +895,13 @@ class TestARCPolicy:
assert len(arc_policy.t1) == 1
assert len(arc_policy.t2) == 3
# store block 5, should evict from T1 (block 2, only one in T1)
# store chunk 5, should evict from T1 (chunk 2, only one in T1)
prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
verify_store_output(
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[5],
store_block_ids=[1], # reuses block 2's storage
store_chunk_ids=[1], # reuses chunk 2's storage
evicted_keys=[2],
),
)
@@ -913,11 +913,11 @@ class TestARCPolicy:
"""
cpu_manager, arc_policy = self._make_manager()
# store blocks 1, 2, 3, 4
# store chunks 1, 2, 3, 4
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
# prepare store block 5 (will evict block 1)
# prepare store chunk 5 (will evict chunk 1)
prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
assert prepare_store_output is not None
assert len(prepare_store_output.evicted_keys) == 1
@@ -925,13 +925,13 @@ class TestARCPolicy:
# complete store with failure
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX, success=False)
# block 5 should not be in cache
# chunk 5 should not be in cache
assert cpu_manager.lookup(to_key(5), _EMPTY_REQ_CTX) is LookupResult.MISS
# block 5 should not be in T1 or T2
# chunk 5 should not be in T1 or T2
assert to_keys([5])[0] not in arc_policy.t1
assert to_keys([5])[0] not in arc_policy.t2
# evicted block should still be gone (in B1 ghost list)
# evicted chunk should still be gone (in B1 ghost list)
evicted_hash = prepare_store_output.evicted_keys[0]
assert evicted_hash in arc_policy.b1
@@ -954,7 +954,7 @@ class TestARCPolicy:
assert len(prepare_store_output.evicted_keys) == 1
cpu_manager.complete_store(to_keys([3, 4, 5]), _EMPTY_REQ_CTX)
# promote some blocks to T2
# promote some chunks to T2
cpu_manager.touch(to_keys([2, 3]), _EMPTY_REQ_CTX)
# T1 has {4, 5}, T2 has {2, 3}
@@ -966,7 +966,7 @@ class TestARCPolicy:
assert prepare_store_output is not None
cpu_manager.complete_store(to_keys([6]), _EMPTY_REQ_CTX)
# verify blocks 2, 3 (in T2) are still present
# verify chunks 2, 3 (in T2) are still present
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.HIT
@@ -980,7 +980,7 @@ def test_filter_reused_manager():
Tests CPUOffloadingManager reuse filtering (store_threshold=2).
"""
manager = make_cpu_manager(
num_blocks=4,
num_chunks=4,
cache_policy="lru",
enable_events=True,
store_threshold=2,
@@ -1023,7 +1023,7 @@ def test_filter_reused_manager():
def test_filter_reused_manager_oversized_offer_makes_progress():
"""An offer larger than the tracker capacity must not churn forever."""
manager = make_cpu_manager(
num_blocks=4,
num_chunks=4,
cache_policy="lru",
store_threshold=2,
max_tracker_size=3,
@@ -1042,69 +1042,69 @@ def test_filter_reused_manager_oversized_offer_makes_progress():
assert stored_keys == set(keys)
def test_evictable_cache_block_count():
def test_evictable_cache_chunk_count():
"""
Verifies _num_evictable_cache_blocks is maintained correctly through the
Verifies _num_evictable_cache_chunks is maintained correctly through the
full store/load lifecycle, eviction, failed stores, concurrent loads,
reset_cache, and the early-exit fast path in prepare_store.
"""
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
# Initially no blocks allocated.
assert manager._num_evictable_cache_blocks == 0
# Initially no chunks allocated.
assert manager._num_evictable_cache_chunks == 0
# Initial cache state [x, x, x, x]
# We get 3 blocks from the cache.
# We get 3 chunks from the cache.
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
# cache state [1', 2', 3', x] <- 1', 2', 3' are actively being used.
assert manager._num_evictable_cache_blocks == 0
assert manager._num_evictable_cache_chunks == 0
# Completing stores makes them idle.
manager.complete_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
# cache state [1, 2, 3, x] <- 1, 2, 3 blocks are idle.
assert manager._num_evictable_cache_blocks == 3
# cache state [1, 2, 3, x] <- 1, 2, 3 chunks are idle.
assert manager._num_evictable_cache_chunks == 3
# prepare_load pins a block: idle count decrements once even if the
# same block is loaded by two concurrent callers.
# prepare_load pins a chunk: idle count decrements once even if the
# same chunk is loaded by two concurrent callers.
manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX)
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
assert manager._num_evictable_cache_blocks == 2
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
assert manager._num_evictable_cache_chunks == 2
manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX) # 2nd concurrent load
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
assert manager._num_evictable_cache_blocks == 2 # no double-decrement
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
assert manager._num_evictable_cache_chunks == 2 # no double-decrement
# First complete_load does not restore idle (ref_cnt still 1).
manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX)
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
assert manager._num_evictable_cache_blocks == 2
# Second complete_load drops ref_cnt to 0 -> block becomes idle again.
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
assert manager._num_evictable_cache_chunks == 2
# Second complete_load drops ref_cnt to 0 -> chunk becomes idle again.
manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX)
# cache state [1, 2, 3, x] <- 1, 2, 3 blocks are idle.
assert manager._num_evictable_cache_blocks == 3
# cache state [1, 2, 3, x] <- 1, 2, 3 chunks are idle.
assert manager._num_evictable_cache_chunks == 3
# Eviction decrements idle count.
# Cache has 3 stored blocks and 1 free slot. Storing 3 new keys needs 2 eviction.
# Cache has 3 stored chunks and 1 free slot. Storing 3 new keys needs 2 eviction.
manager.prepare_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX)
# cache state [1, 4', 5', 6'] <- block 1 is idle
assert manager._num_evictable_cache_blocks == 1
# cache state [1, 4', 5', 6'] <- chunk 1 is idle
assert manager._num_evictable_cache_chunks == 1
# Failed store does not increment idle count (block discarded from cache).
# Failed store does not increment idle count (chunk discarded from cache).
manager.complete_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX, success=False)
# cache state [1, x, x, x] <- block 1 is idle. Other returned to cache.
assert manager._num_evictable_cache_blocks == 1
# cache state [1, x, x, x] <- chunk 1 is idle. Other returned to cache.
assert manager._num_evictable_cache_chunks == 1
# reset_cache zeroes the count unconditionally.
manager.reset_cache()
# cache state [x, x, x, x]
assert manager._num_evictable_cache_blocks == 0
assert manager._num_evictable_cache_chunks == 0
# setup 3 blocks with loads so idle count drops to 0.
# setup 3 chunks with loads so idle count drops to 0.
manager.prepare_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
manager.prepare_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
# cache state [10', 11', 12', x]
assert manager._num_evictable_cache_blocks == 0
assert manager._num_evictable_cache_chunks == 0
# prepare_store requiring eviction must return None immediately (fast exit).
# Spy on policy.evict to confirm the fast path short-circuits before calling it.
@@ -1120,24 +1120,24 @@ def test_evictable_cache_block_count():
# cache state [10', 11', 12', x] <- cannot evict anything
assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is None
assert not evict_called, (
"_num_evictable_cache_blocks==0 should short-circuit before evict()"
"_num_evictable_cache_chunks==0 should short-circuit before evict()"
)
# After releasing the loads, eviction becomes possible again.
manager.complete_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
# cache state [10, 11, 12, x] <- 10, 11, 12 are idle
assert manager._num_evictable_cache_blocks == 3
assert manager._num_evictable_cache_chunks == 3
assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is not None
# cache state [10, 11, 14', 15'] <- 10, 11 are idle
assert manager._num_evictable_cache_blocks == 2
assert manager._num_evictable_cache_chunks == 2
manager.complete_store(to_keys([14, 15]), _EMPTY_REQ_CTX)
# cache state [10, 11, 14, 15] <- all blocks idle
assert manager._num_evictable_cache_blocks == 4
# cache state [10, 11, 14, 15] <- all chunks idle
assert manager._num_evictable_cache_chunks == 4
def test_touch_forwards_req_context_to_policy(monkeypatch):
"""Regression: CPUOffloadingManager.touch forwards ReqContext to policy."""
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
received = []
def spy_touch(keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
@@ -38,7 +38,7 @@ def _set_spawn_method(monkeypatch):
def _make_region(
engine_id: str,
num_blocks: int = 4,
num_chunks: int = 4,
cpu_page_size: int = PAGE_SIZE,
num_workers: int = 1,
rank: int = 0,
@@ -47,9 +47,9 @@ def _make_region(
assert cpu_page_size % PAGE_SIZE == 0
return SharedOffloadRegion(
engine_id=engine_id,
num_blocks=num_blocks,
num_chunks=num_chunks,
rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size,
kv_bytes_per_chunk=num_workers * cpu_page_size,
cpu_page_size=cpu_page_size,
barrier=barrier,
)
@@ -99,16 +99,16 @@ def _region(engine_id: str, **kwargs):
def _multi_region(
engine_id: str,
num_workers: int,
num_blocks: int = 4,
num_chunks: int = 4,
cpu_page_size: int = PAGE_SIZE,
):
"""Context manager: create one SharedOffloadRegion per rank, clean up on exit."""
regions = [
SharedOffloadRegion(
engine_id=engine_id,
num_blocks=num_blocks,
num_chunks=num_chunks,
rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size,
kv_bytes_per_chunk=num_workers * cpu_page_size,
cpu_page_size=cpu_page_size,
)
for rank in range(num_workers)
@@ -124,7 +124,7 @@ def _multi_region(
def _race_construct(
engine_id: str,
num_workers: int,
num_blocks: int = 4,
num_chunks: int = 4,
cpu_page_size: int = PAGE_SIZE,
) -> tuple[list[SharedOffloadRegion], list[Exception]]:
"""Spawn num_workers threads that all race to construct SharedOffloadRegion."""
@@ -137,9 +137,9 @@ def _race_construct(
try:
regions[rank] = SharedOffloadRegion(
engine_id=engine_id,
num_blocks=num_blocks,
num_chunks=num_chunks,
rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size,
kv_bytes_per_chunk=num_workers * cpu_page_size,
cpu_page_size=cpu_page_size,
)
except Exception as e:
@@ -156,7 +156,7 @@ def _race_construct(
def _mp_race_construct_and_write(
engine_id: str,
num_blocks: int,
num_chunks: int,
rank: int,
num_workers: int,
cpu_page_size: int,
@@ -170,9 +170,9 @@ def _mp_race_construct_and_write(
try:
region = SharedOffloadRegion(
engine_id=engine_id,
num_blocks=num_blocks,
num_chunks=num_chunks,
rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size,
kv_bytes_per_chunk=num_workers * cpu_page_size,
cpu_page_size=cpu_page_size,
)
t = region.create_next_worker_view(cpu_page_size)
@@ -198,9 +198,9 @@ def _mp_barrier_construct_and_hold(
try:
region = SharedOffloadRegion(
engine_id=engine_id,
num_blocks=2,
num_chunks=2,
rank=rank,
kv_bytes_per_block=num_workers * PAGE_SIZE,
kv_bytes_per_chunk=num_workers * PAGE_SIZE,
cpu_page_size=PAGE_SIZE,
barrier=lambda: barrier.wait(30),
)
@@ -233,9 +233,9 @@ def iid():
def test_create_next_worker_view_shape_and_stride(iid):
"""Returned tensor must have shape (num_blocks, tensor_page_size) and
"""Returned tensor must have shape (num_chunks, tensor_page_size) and
stride (row_stride, 1) where row_stride = cpu_page_size * num_workers."""
with _region(iid, num_blocks=4, cpu_page_size=2 * PAGE_SIZE) as r:
with _region(iid, num_chunks=4, cpu_page_size=2 * PAGE_SIZE) as r:
t = r.create_next_worker_view(PAGE_SIZE)
assert t.shape == (4, PAGE_SIZE)
# num_workers=1 → row_stride = cpu_page_size
@@ -253,7 +253,7 @@ def test_create_next_worker_view_storage_offset_rank0(iid):
def test_create_next_worker_view_storage_offset_rank1(iid):
"""rank=1 worker's first tensor must start cpu_page_size bytes into the mmap."""
with _multi_region(iid, num_workers=2, num_blocks=4) as (r0, r1):
with _multi_region(iid, num_workers=2, num_chunks=4) as (r0, r1):
t1 = r1.create_next_worker_view(PAGE_SIZE)
assert t1.data_ptr() == r1._base.data_ptr() + PAGE_SIZE
del t1
@@ -261,7 +261,7 @@ def test_create_next_worker_view_storage_offset_rank1(iid):
def test_create_next_worker_view_row_stride_with_multiple_workers(iid):
"""With num_workers=4, row_stride must be 4 * cpu_page_size."""
with _region(iid, num_blocks=2, num_workers=4) as r:
with _region(iid, num_chunks=2, num_workers=4) as r:
t = r.create_next_worker_view(PAGE_SIZE)
assert t.stride(0) == 4 * PAGE_SIZE
del t
@@ -332,12 +332,12 @@ def test_create_next_worker_view_overflow_does_not_mutate_cursor(iid):
def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
"""Writes into a create_next_worker_view view must appear at the correct
raw mmap offset"""
with _region(iid, num_blocks=4) as r:
with _region(iid, num_chunks=4) as r:
t = r.create_next_worker_view(PAGE_SIZE)
t[2, :] = 42 # write to block row 2
t[2, :] = 42 # write to chunk row 2
raw = memoryview(r.mmap_obj)
# num_workers=1 → row_stride = PAGE_SIZE; block 2 starts at byte 2*PAGE_SIZE
# num_workers=1 → row_stride = PAGE_SIZE; chunk 2 starts at byte 2*PAGE_SIZE
chunk = bytes(raw[2 * PAGE_SIZE : 3 * PAGE_SIZE])
assert all(b == 42 for b in chunk)
del raw, t
@@ -345,7 +345,7 @@ def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
def test_create_next_worker_view_multi_tensor_layout(iid):
"""Two tensors from the same worker land at consecutive byte offsets per row."""
with _region(iid, num_blocks=2, cpu_page_size=2 * PAGE_SIZE) as r:
with _region(iid, num_chunks=2, cpu_page_size=2 * PAGE_SIZE) as r:
ta = r.create_next_worker_view(PAGE_SIZE)
tb = r.create_next_worker_view(PAGE_SIZE)
@@ -353,8 +353,8 @@ def test_create_next_worker_view_multi_tensor_layout(iid):
tb[:, :] = 2
raw = memoryview(r.mmap_obj)
for blk in range(2):
row_offset = blk * 2 * PAGE_SIZE # num_workers=1
for chunk in range(2):
row_offset = chunk * 2 * PAGE_SIZE # num_workers=1
assert all(b == 1 for b in raw[row_offset : row_offset + PAGE_SIZE])
assert all(
b == 2 for b in raw[row_offset + PAGE_SIZE : row_offset + 2 * PAGE_SIZE]
@@ -366,7 +366,7 @@ def test_create_next_worker_view_multiprocess_slots(iid):
"""Each worker process calls create_next_worker_view and writes distinct data;
the parent verifies each slot lands at the correct interleaved offset."""
num_workers = 2
num_blocks = 4
num_chunks = 4
ctx = get_mp_context()
done_queue = ctx.Queue()
@@ -375,9 +375,9 @@ def test_create_next_worker_view_multiprocess_slots(iid):
# Parent is rank 0 (creator); child is rank 1 (joiner).
region = SharedOffloadRegion(
engine_id=iid,
num_blocks=num_blocks,
num_chunks=num_chunks,
rank=0,
kv_bytes_per_block=num_workers * PAGE_SIZE,
kv_bytes_per_chunk=num_workers * PAGE_SIZE,
cpu_page_size=PAGE_SIZE,
)
try:
@@ -385,7 +385,7 @@ def test_create_next_worker_view_multiprocess_slots(iid):
target=_mp_race_construct_and_write,
args=(
iid,
num_blocks,
num_chunks,
1,
num_workers,
PAGE_SIZE,
@@ -403,12 +403,12 @@ def test_create_next_worker_view_multiprocess_slots(iid):
assert result["error"] is None, result["error"]
raw = memoryview(region.mmap_obj)
for blk in range(num_blocks):
row_start = blk * num_workers * PAGE_SIZE
for chunk in range(num_chunks):
row_start = chunk * num_workers * PAGE_SIZE
w0 = bytes(raw[row_start : row_start + PAGE_SIZE])
w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE])
assert all(b == 11 for b in w0), f"block {blk}: rank0 slot wrong"
assert all(b == 22 for b in w1), f"block {blk}: rank1 slot wrong"
assert all(b == 11 for b in w0), f"chunk {chunk}: rank0 slot wrong"
assert all(b == 22 for b in w1), f"chunk {chunk}: rank1 slot wrong"
del raw, t0 # release before finally triggers cleanup
cleanup_queue.put(True)
@@ -422,8 +422,8 @@ def test_create_next_worker_view_multiprocess_slots(iid):
def test_create_next_worker_view_worker_isolation(iid):
"""Writes by worker 0 must not affect worker 1's slot and vice versa."""
num_workers = 2
num_blocks = 4
with _multi_region(iid, num_workers=num_workers, num_blocks=num_blocks) as regions:
num_chunks = 4
with _multi_region(iid, num_workers=num_workers, num_chunks=num_chunks) as regions:
t0 = regions[0].create_next_worker_view(PAGE_SIZE)
t1 = regions[1].create_next_worker_view(PAGE_SIZE)
@@ -431,12 +431,12 @@ def test_create_next_worker_view_worker_isolation(iid):
t1[:, :] = 22
raw = memoryview(regions[0].mmap_obj)
for blk in range(num_blocks):
row_start = blk * num_workers * PAGE_SIZE
for chunk in range(num_chunks):
row_start = chunk * num_workers * PAGE_SIZE
w0 = bytes(raw[row_start : row_start + PAGE_SIZE])
w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE])
assert all(b == 11 for b in w0), f"block {blk}: worker0 slot corrupted"
assert all(b == 22 for b in w1), f"block {blk}: worker1 slot corrupted"
assert all(b == 11 for b in w0), f"chunk {chunk}: worker0 slot corrupted"
assert all(b == 22 for b in w1), f"chunk {chunk}: worker1 slot corrupted"
del raw, t0, t1 # release before finally triggers cleanup
@@ -466,7 +466,7 @@ def test_file_exists_after_construction(iid):
def test_file_has_correct_size(iid):
"""The mmap file size on disk must equal total_size_bytes."""
with _region(iid, num_blocks=4) as r:
with _region(iid, num_chunks=4) as r:
assert os.path.getsize(r.mmap_path) == 4 * PAGE_SIZE
@@ -491,11 +491,11 @@ def test_madvise_success_selects_madvise_population(iid, monkeypatch):
monkeypatch.setattr(sor, "_madvise_populate_write", _spy_madvise)
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
num_blocks = 3
num_chunks = 3
num_workers = 2
with _region(iid, num_blocks=num_blocks, num_workers=num_workers, rank=1):
# 1 probe (PAGESIZE) + N populate calls (one per block per worker column).
expected_populate = num_blocks # ranked path: one call per block
with _region(iid, num_chunks=num_chunks, num_workers=num_workers, rank=1):
# 1 probe (PAGESIZE) + N populate calls (one per chunk per worker column).
expected_populate = num_chunks # ranked path: one call per chunk
mmap_id = madvise_calls[0][2]
assert madvise_calls == [
(0, mmap.PAGESIZE, mmap_id),
@@ -510,7 +510,7 @@ def test_madvise_success_selects_madvise_population(iid, monkeypatch):
def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
"""An EINVAL probe must select fallback for every ranked block."""
"""An EINVAL probe must select fallback for every ranked chunk."""
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
fallback_calls: list[tuple[int, int]] = []
@@ -526,7 +526,7 @@ def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
with _region(iid, num_blocks=3, num_workers=2, rank=1):
with _region(iid, num_chunks=3, num_workers=2, rank=1):
assert fallback_calls == [
(mmap.PAGESIZE, mmap.PAGESIZE),
(3 * mmap.PAGESIZE, mmap.PAGESIZE),
@@ -551,7 +551,7 @@ def test_madvise_einval_selects_fallback_for_unranked_region(iid, monkeypatch):
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
with _region(iid, num_blocks=3, num_workers=2, rank=None):
with _region(iid, num_chunks=3, num_workers=2, rank=None):
assert fallback_calls == [(0, 6 * mmap.PAGESIZE)]
@@ -651,7 +651,7 @@ def test_multiprocess_race_construct_and_write(iid):
"""N processes race to construct the same SharedOffloadRegion, each writes
fill_value = rank+1 into their slot; parent verifies interleaved layout."""
num_workers = 4
num_blocks = 3
num_chunks = 3
ctx = get_mp_context()
done_queue = ctx.Queue()
@@ -662,7 +662,7 @@ def test_multiprocess_race_construct_and_write(iid):
target=_mp_race_construct_and_write,
args=(
iid,
num_blocks,
num_chunks,
rank,
num_workers,
PAGE_SIZE,
@@ -689,13 +689,13 @@ def test_multiprocess_race_construct_and_write(iid):
with open(mmap_path, "rb") as f:
raw = f.read()
for blk in range(num_blocks):
for chunk in range(num_chunks):
for w in range(num_workers):
slot_start = (blk * num_workers + w) * PAGE_SIZE
slot_start = (chunk * num_workers + w) * PAGE_SIZE
slot = raw[slot_start : slot_start + PAGE_SIZE]
expected = w + 1 # fill_value = rank + 1
assert all(b == expected for b in slot), (
f"block {blk}, worker {w}: expected {expected} but got wrong bytes"
f"chunk {chunk}, worker {w}: expected {expected} but got wrong bytes"
)
# Unblock all workers to clean up.
@@ -839,9 +839,9 @@ def test_insufficient_space_raises_clear_error(monkeypatch):
with pytest.raises(RuntimeError, match="Insufficient space"):
SharedOffloadRegion(
engine_id=engine_id,
num_blocks=4,
num_chunks=4,
rank=0,
kv_bytes_per_block=PAGE_SIZE,
kv_bytes_per_chunk=PAGE_SIZE,
cpu_page_size=PAGE_SIZE,
)
@@ -870,9 +870,9 @@ def test_ftruncate_failure_cleans_up_creator(monkeypatch):
with pytest.raises(OSError, match="ftruncate failed"):
SharedOffloadRegion(
engine_id=engine_id,
num_blocks=4,
num_chunks=4,
rank=0,
kv_bytes_per_block=PAGE_SIZE,
kv_bytes_per_chunk=PAGE_SIZE,
cpu_page_size=PAGE_SIZE,
)