[Refactor][kv_offload]: rename block→chunk (#52615)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
Ronen Schaffer
2026-09-09 17:45:49 +03:00
committed by GitHub
parent 9ffb8cea96
commit 138d137b5b
34 changed files with 933 additions and 916 deletions
@@ -704,7 +704,7 @@ def test_prom_metrics_registers_tiering_metrics_from_spec():
)
metric = prom_metrics._offloading_metric_defs[
TieringOffloadingMetrics.BLOCK_QUERIES
TieringOffloadingMetrics.CHUNK_QUERIES
]
assert metric.kwargs["labelnames"] == ["model_name", "engine", "tier"]
@@ -96,7 +96,7 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
groups.append(
KVCacheGroupSpec([f"layer{i}"], kv_spec, is_eagle_group=eagle and i == 1)
)
manager = CPUOffloadingManager(num_blocks=100)
manager = CPUOffloadingManager(num_chunks=100)
spec = SimpleNamespace(
tokens_per_block=(256, 64, 8),
tokens_per_hash=8,
@@ -2888,7 +2888,7 @@ class TestEagle:
req=req,
req_context=ReqContext(req_id="test-req"),
offloading_context=RequestOffloadingContext(
policy=OffloadPolicy.BLOCK_LEVEL
policy=OffloadPolicy.CHUNK_LEVEL
),
num_locally_computed_tokens=num_computed_tokens,
)
@@ -7,7 +7,7 @@ import pytest
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
@@ -20,10 +20,10 @@ class _DummyCachePolicy(CachePolicy):
def __init__(self, cache_capacity: int) -> None:
self.cache_capacity = cache_capacity
def get(self, key: OffloadKey) -> BlockStatus | None:
def get(self, key: OffloadKey) -> ChunkStatus | None:
return None
def insert(self, key: OffloadKey, block: BlockStatus) -> None:
def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
pass
def remove(self, key: OffloadKey) -> None:
@@ -34,7 +34,7 @@ class _DummyCachePolicy(CachePolicy):
def evict(
self, n: int, protected: set[OffloadKey]
) -> list[tuple[OffloadKey, BlockStatus]] | None:
) -> list[tuple[OffloadKey, ChunkStatus]] | None:
return None
def clear(self) -> None:
@@ -72,7 +72,7 @@ class TestCachePolicyFactory:
policy_cls = CachePolicyFactory.get_cache_policy_cls("dummy")
assert policy_cls is _DummyCachePolicy
manager = CPUOffloadingManager(num_blocks=4, cache_policy="dummy")
manager = CPUOffloadingManager(num_chunks=4, cache_policy="dummy")
assert isinstance(manager._policy, _DummyCachePolicy)
def test_unregistered_policy_raises(self):
@@ -98,7 +98,7 @@ class TestCachePolicyFactory:
"""End-to-end: CPUOffloadingManager resolves an unregistered policy
purely from cache_policy_module_path."""
manager = CPUOffloadingManager(
num_blocks=4,
num_chunks=4,
cache_policy="_DummyCachePolicy",
cache_policy_module_path="tests.v1.kv_offload.cpu.policies.test_factory",
)
@@ -232,9 +232,9 @@ def test_cross_topology_roundtrip(writer_tp: int, reader_tp: int):
def canonical_view(rank: int, world_size: int) -> torch.Tensor:
region = SharedOffloadRegion(
engine_id=engine_id,
num_blocks=num_blocks,
num_chunks=num_blocks,
rank=rank,
kv_bytes_per_block=row_stride,
kv_bytes_per_chunk=row_stride,
cpu_page_size=row_stride // world_size,
)
regions.append(region)
+46 -46
View File
@@ -26,7 +26,7 @@ from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
NUM_GPU_BLOCKS = [64]
NUM_CPU_BLOCKS = [256]
NUM_CPU_CHUNKS = [256]
GPU_PAGE_SIZES = [512, 1024]
BLOCKS_PER_CHUNK_VALUES = [1, 3]
NUM_TENSORS = [4]
@@ -225,7 +225,7 @@ def test_worker_syncs_before_cleanup_after_handler_failure(
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
@pytest.mark.parametrize("num_cpu_chunks", NUM_CPU_CHUNKS)
@pytest.mark.parametrize("num_tensors", NUM_TENSORS)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", DEVICES)
@@ -241,7 +241,7 @@ def test_transfer(
gpu_page_size_bytes: int,
blocks_per_chunk: int,
num_gpu_blocks: int,
num_cpu_blocks: int,
num_cpu_chunks: int,
num_tensors: int,
seed: int,
device: str,
@@ -288,41 +288,41 @@ def test_transfer(
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
)
simulated_world_size = 2
kv_bytes_per_block = (
kv_bytes_per_chunk = (
cpu_page_size if replicated_layout else cpu_page_size * simulated_world_size
)
mmap_region = SharedOffloadRegion(
engine_id=str(uuid.uuid4()),
num_blocks=num_cpu_blocks,
num_chunks=num_cpu_chunks,
rank=0,
kv_bytes_per_block=kv_bytes_per_block,
kv_bytes_per_chunk=kv_bytes_per_chunk,
cpu_page_size=cpu_page_size,
)
worker = CPUOffloadingWorker(
kv_caches=kv_caches,
blocks_per_chunk=blocks_per_chunk,
num_cpu_blocks=num_cpu_blocks,
num_cpu_chunks=num_cpu_chunks,
mmap_region=mmap_region,
)
# select block mappings
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * blocks_per_chunk)
cpu_blocks = random.sample(range(num_cpu_blocks), num_mappings)
cpu_chunks = random.sample(range(num_cpu_chunks), num_mappings)
# expand cpu blocks to gpu-page granularity for uniform comparison:
# each cpu block maps to blocks_per_chunk consecutive sub-blocks
cpu_blocks_expanded = [
cpu_block * blocks_per_chunk + j
for cpu_block in cpu_blocks
# expand cpu chunks to gpu-page granularity for uniform comparison:
# each cpu chunk maps to blocks_per_chunk consecutive sub-blocks
cpu_chunks_expanded = [
cpu_chunk * blocks_per_chunk + j
for cpu_chunk in cpu_chunks
for j in range(blocks_per_chunk)
]
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU block
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU chunk
blocks_to_skip = blocks_per_chunk - 1
if blocks_to_skip > 0:
gpu_blocks = gpu_blocks[blocks_to_skip:]
cpu_blocks_expanded = cpu_blocks_expanded[blocks_to_skip:]
cpu_chunks_expanded = cpu_chunks_expanded[blocks_to_skip:]
# set transfer direction
if gpu_to_cpu:
@@ -330,16 +330,16 @@ def test_transfer(
src_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
)
dst_spec = CPULoadStoreSpec(cpu_blocks)
dst_to_src = dict(zip(cpu_blocks_expanded, gpu_blocks))
dst_spec = CPULoadStoreSpec(cpu_chunks)
dst_to_src = dict(zip(cpu_chunks_expanded, gpu_blocks))
num_dst_sub_blocks = num_gpu_blocks
else:
handler = worker._load_handler
src_spec = CPULoadStoreSpec(cpu_blocks)
src_spec = CPULoadStoreSpec(cpu_chunks)
dst_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
)
dst_to_src = dict(zip(gpu_blocks, cpu_blocks_expanded))
dst_to_src = dict(zip(gpu_blocks, cpu_chunks_expanded))
num_dst_sub_blocks = num_gpu_blocks
# randomize src and dst tensors before transfer
@@ -410,7 +410,7 @@ def test_transfer(
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
@pytest.mark.parametrize("num_cpu_chunks", NUM_CPU_CHUNKS)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", DEVICES)
@torch.inference_mode()
@@ -421,7 +421,7 @@ def test_transfer_multi_group(
gpu_page_size_bytes: int,
blocks_per_chunk: int,
num_gpu_blocks: int,
num_cpu_blocks: int,
num_cpu_chunks: int,
seed: int,
device: str,
) -> None:
@@ -470,37 +470,37 @@ def test_transfer_multi_group(
worker = CPUOffloadingWorker(
kv_caches=canonical_kv_caches,
blocks_per_chunk=blocks_per_chunk,
num_cpu_blocks=num_cpu_blocks,
num_cpu_chunks=num_cpu_chunks,
)
# group 0: aligned, group 1: empty, group 2: unaligned on CPU->GPU
group_sizes_in_cpu_blocks = [num_mappings_per_group, 0, num_mappings_per_group]
group_sizes_in_cpu_chunks = [num_mappings_per_group, 0, num_mappings_per_group]
total_cpu_blocks = sum(group_sizes_in_cpu_blocks)
total_gpu_blocks_needed = total_cpu_blocks * blocks_per_chunk
total_cpu_chunks = sum(group_sizes_in_cpu_chunks)
total_gpu_blocks_needed = total_cpu_chunks * blocks_per_chunk
gpu_blocks_all = random.sample(range(num_gpu_blocks), total_gpu_blocks_needed)
cpu_blocks_all = random.sample(range(num_cpu_blocks), total_cpu_blocks)
cpu_chunks_all = random.sample(range(num_cpu_chunks), total_cpu_chunks)
# split gpu/cpu blocks per group
# split gpu blocks / cpu chunks per group
gpu_blocks_per_group: list[list[int]] = []
cpu_blocks_per_group: list[list[int]] = []
cpu_chunks_per_group: list[list[int]] = []
gpu_offset = 0
cpu_offset = 0
for size in group_sizes_in_cpu_blocks:
for size in group_sizes_in_cpu_chunks:
gpu_count = size * blocks_per_chunk
gpu_blocks_per_group.append(gpu_blocks_all[gpu_offset : gpu_offset + gpu_count])
cpu_blocks_per_group.append(cpu_blocks_all[cpu_offset : cpu_offset + size])
cpu_chunks_per_group.append(cpu_chunks_all[cpu_offset : cpu_offset + size])
gpu_offset += gpu_count
cpu_offset += size
# expand cpu blocks to gpu-page granularity
cpu_blocks_expanded_per_group = [
# expand cpu chunks to gpu-page granularity
cpu_chunks_expanded_per_group = [
[
cpu_block * blocks_per_chunk + j
for cpu_block in cpu_blocks
cpu_chunk * blocks_per_chunk + j
for cpu_chunk in cpu_chunks
for j in range(blocks_per_chunk)
]
for cpu_blocks in cpu_blocks_per_group
for cpu_chunks in cpu_chunks_per_group
]
# skip sub-blocks from group 2 to test unaligned transfers.
@@ -509,7 +509,7 @@ def test_transfer_multi_group(
gpu_blocks_per_group[2] = gpu_blocks_per_group[2][
sub_blocks_to_skip:-sub_blocks_to_skip
]
cpu_blocks_expanded_per_group[2] = cpu_blocks_expanded_per_group[2][
cpu_chunks_expanded_per_group[2] = cpu_chunks_expanded_per_group[2][
sub_blocks_to_skip:-sub_blocks_to_skip
]
@@ -520,10 +520,10 @@ def test_transfer_multi_group(
gpu_blocks.extend(gpu_blks)
group_sizes.append(len(gpu_blks))
# build flat cpu_blocks list
cpu_blocks = []
for cpu_blks in cpu_blocks_per_group:
cpu_blocks.extend(cpu_blks)
# build flat cpu_chunks list
cpu_chunks = []
for cpu_chnks in cpu_chunks_per_group:
cpu_chunks.extend(cpu_chnks)
# block_indices: only relevant for unaligned transfers
block_indices: list[int] = [0, 0, sub_blocks_to_skip]
@@ -533,18 +533,18 @@ def test_transfer_multi_group(
src_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
)
dst_spec = CPULoadStoreSpec(cpu_blocks)
dst_spec = CPULoadStoreSpec(cpu_chunks)
# per-group mapping: cpu sub-block -> gpu sub-block
dst_to_src_per_group = [
dict(zip(expanded, gpu_blks))
for expanded, gpu_blks in zip(
cpu_blocks_expanded_per_group, gpu_blocks_per_group
cpu_chunks_expanded_per_group, gpu_blocks_per_group
)
]
num_dst_sub_blocks = num_cpu_blocks * blocks_per_chunk
num_dst_sub_blocks = num_cpu_chunks * blocks_per_chunk
else:
handler = worker._load_handler
src_spec = CPULoadStoreSpec(cpu_blocks)
src_spec = CPULoadStoreSpec(cpu_chunks)
dst_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
)
@@ -552,7 +552,7 @@ def test_transfer_multi_group(
dst_to_src_per_group = [
dict(zip(gpu_blks, expanded))
for gpu_blks, expanded in zip(
gpu_blocks_per_group, cpu_blocks_expanded_per_group
gpu_blocks_per_group, cpu_chunks_expanded_per_group
)
]
num_dst_sub_blocks = num_gpu_blocks
@@ -644,7 +644,7 @@ def test_load_waits_for_pending_compute_stream_writes(default_vllm_config) -> No
],
),
blocks_per_chunk=1,
num_cpu_blocks=num_blocks,
num_cpu_chunks=num_blocks,
)
worker._load_handler.src_tensors[0].fill_(sentinel)
expected = torch.full((page_size_bytes,), sentinel, dtype=torch.int8)
+144 -144
View File
@@ -36,7 +36,7 @@ _EMPTY_REQ_CTX = make_req_context()
def make_cpu_manager(
num_blocks: int = 4,
num_chunks: int = 4,
cache_policy: str = "lru",
cache_policy_module_path: str | None = None,
enable_events: bool = False,
@@ -44,7 +44,7 @@ def make_cpu_manager(
max_tracker_size: int = 64_000,
) -> CPUOffloadingManager:
return CPUOffloadingManager(
num_blocks=num_blocks,
num_chunks=num_chunks,
cache_policy=cache_policy,
cache_policy_module_path=cache_policy_module_path,
enable_events=enable_events,
@@ -56,7 +56,7 @@ def make_cpu_manager(
@dataclass
class ExpectedPrepareStoreOutput:
keys_to_store: list[int]
store_block_ids: list[int]
store_chunk_ids: list[int]
evicted_keys: list[int]
@@ -93,9 +93,9 @@ def verify_store_output(
store_spec = prepare_store_output.store_spec
assert isinstance(store_spec, CPULoadStoreSpec)
expected_array = np.array(
expected_prepare_store_output.store_block_ids, dtype=np.int64
expected_prepare_store_output.store_chunk_ids, dtype=np.int64
)
assert np.array_equal(expected_array, store_spec.block_ids)
assert np.array_equal(expected_array, store_spec.chunk_ids)
def verify_load_output(
@@ -103,7 +103,7 @@ def verify_load_output(
):
assert isinstance(prepare_load_output, CPULoadStoreSpec)
expected_array = np.array(expected_prepare_load_output, dtype=np.int64)
assert np.array_equal(expected_array, prepare_load_output.block_ids)
assert np.array_equal(expected_array, prepare_load_output.chunk_ids)
def check_split_usage_stats(
@@ -146,7 +146,7 @@ def verify_events(
def test_cpu_eviction_removed_precedes_stored():
"""An eviction is announced before the store that reuses its capacity."""
manager = make_cpu_manager(num_blocks=2, enable_events=True)
manager = make_cpu_manager(num_chunks=2, enable_events=True)
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -164,23 +164,23 @@ def test_cpu_eviction_removed_precedes_stored():
@pytest.mark.parametrize("eviction_policy", ["lru", "arc"])
def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
def test_already_stored_chunk_not_evicted_during_prepare_store(eviction_policy):
"""
Regression test: a block that is already stored must not be evicted
by prepare_store() when it needs to make room for new blocks.
Regression test: a chunk that is already stored must not be evicted
by prepare_store() when it needs to make room for new chunks.
Applies to both lru and arc policies.
Scenario:
- Store blocks [1, 2] and complete.
- touch([1]) makes block 2 the LRU candidate.
- Store chunks [1, 2] and complete.
- touch([1]) makes chunk 2 the LRU candidate.
- prepare_store([2, 3, 4, 5]):
* block 2 is filtered out as "already stored"
* but without the fix, block 2 would be evicted as the LRU
* chunk 2 is filtered out as "already stored"
* but without the fix, chunk 2 would be evicted as the LRU
candidate to make room for [3, 4, 5]
- After complete_store([2, 3, 4, 5]), block 2 must still be present.
- After complete_store([2, 3, 4, 5]), chunk 2 must still be present.
"""
manager = make_cpu_manager(
num_blocks=4,
num_chunks=4,
cache_policy=eviction_policy,
enable_events=True,
)
@@ -189,33 +189,33 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
# touch [1] to make block 2 the LRU candidate
# touch [1] to make chunk 2 the LRU candidate
manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
# prepare_store([2, 3, 4, 5]):
# - block 2 is already stored -> filtered out of keys_to_store
# - block 2 must NOT be evicted even though it is the LRU candidate
# - block 1 (ID 0) is evicted instead; new blocks [3,4,5] get IDs 2,3,0
# - chunk 2 is already stored -> filtered out of keys_to_store
# - chunk 2 must NOT be evicted even though it is the LRU candidate
# - chunk 1 (ID 0) is evicted instead; new chunks [3,4,5] get IDs 2,3,0
prepare_store_output = manager.prepare_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
verify_store_output(
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[3, 4, 5],
store_block_ids=[2, 3, 0],
evicted_keys=[1], # block 1 evicted, not block 2
store_chunk_ids=[2, 3, 0],
evicted_keys=[1], # chunk 1 evicted, not chunk 2
),
)
# complete_store must not silently drop block 2
# complete_store must not silently drop chunk 2
manager.complete_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
# block 2 must still be present in the cache
# chunk 2 must still be present in the cache
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
def test_filter_reused_manager_reports_stores_skipped_counter():
manager = make_cpu_manager(
num_blocks=4,
num_chunks=4,
cache_policy="lru",
store_threshold=2,
)
@@ -226,7 +226,7 @@ def test_filter_reused_manager_reports_stores_skipped_counter():
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[],
store_block_ids=[],
store_chunk_ids=[],
evicted_keys=[],
),
)
@@ -247,34 +247,34 @@ def test_cpu_manager_reports_cache_usage_gauge():
] == pytest.approx(value)
# Zero-capacity manager always reports 0.0
manager = make_cpu_manager(num_blocks=0)
manager = make_cpu_manager(num_chunks=0)
check_usage_stats(manager, 0.0)
# Empty manager (4 blocks, none allocated): usage = 0.0
manager = make_cpu_manager(num_blocks=4)
# Empty manager (4 chunks, none allocated): usage = 0.0
manager = make_cpu_manager(num_chunks=4)
check_usage_stats(manager, 0.0)
# After allocating 2 of 4 blocks: usage = 0.5
# After allocating 2 of 4 chunks: usage = 0.5
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
check_usage_stats(manager, 0.5)
# After filling all 4 blocks: usage = 1.0
# After filling all 4 chunks: usage = 1.0
manager.prepare_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
check_usage_stats(manager, 1.0)
# After completing store, the blocks becomes evictable as it is not actively used
# After completing store, the chunks become evictable as not actively used
# and usage drops.
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
check_usage_stats(manager, 0.5)
# After completing store, the blocks becomes evictable as it is not actively used
# After completing store, the chunks become evictable as not actively used
# and usage drops.
manager.complete_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
check_usage_stats(manager, 0.0)
def test_cpu_manager_reports_allocation_size_histogram():
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -297,12 +297,12 @@ def test_cpu_manager_reports_allocation_size_histogram():
def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
def fail_allocate_blocks(keys):
def fail_allocate_chunks(keys):
raise RuntimeError("allocation failed")
monkeypatch.setattr(manager, "_allocate_blocks", fail_allocate_blocks)
monkeypatch.setattr(manager, "_allocate_chunks", fail_allocate_chunks)
with pytest.raises(RuntimeError, match="allocation failed"):
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
@@ -316,7 +316,7 @@ def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
def test_cpu_manager_reports_allocation_size_on_eviction_failure():
manager = make_cpu_manager(num_blocks=1, cache_policy="lru")
manager = make_cpu_manager(num_chunks=1, cache_policy="lru")
manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
manager.get_stats()
@@ -332,7 +332,7 @@ def test_cpu_manager_reports_allocation_size_on_eviction_failure():
def test_cpu_manager_reports_cache_write_and_read_usage_gauges():
manager = make_cpu_manager(num_blocks=4)
manager = make_cpu_manager(num_chunks=4)
# Store path: pins write usage until complete_store.
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -356,7 +356,7 @@ def test_cpu_manager_reports_cache_write_and_read_usage_gauges():
def test_cpu_manager_clears_write_usage_after_failed_store():
manager = make_cpu_manager(num_blocks=4)
manager = make_cpu_manager(num_chunks=4)
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
check_split_usage_stats(manager, write=0.5, read=0.0, total=0.5)
@@ -369,8 +369,8 @@ def test_cpu_manager():
"""
Tests CPUOffloadingManager with lru policy.
"""
# initialize a CPU manager with a capacity of 4 blocks
cpu_manager = make_cpu_manager(num_blocks=4, cache_policy="lru", enable_events=True)
# initialize a CPU manager with a capacity of 4 chunks
cpu_manager = make_cpu_manager(num_chunks=4, cache_policy="lru", enable_events=True)
# prepare store [1, 2]
prepare_store_output = cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -378,7 +378,7 @@ def test_cpu_manager():
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[1, 2],
store_block_ids=[0, 1],
store_chunk_ids=[0, 1],
evicted_keys=[],
),
)
@@ -407,7 +407,7 @@ def test_cpu_manager():
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[3, 4, 5],
store_block_ids=[2, 3, 0],
store_chunk_ids=[2, 3, 0],
evicted_keys=[1],
),
)
@@ -445,7 +445,7 @@ def test_cpu_manager():
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[6, 7, 8],
store_block_ids=[1, 0, 3],
store_chunk_ids=[1, 0, 3],
evicted_keys=[4, 5, 2],
),
)
@@ -462,7 +462,7 @@ def test_cpu_manager():
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[9],
store_block_ids=[3],
store_chunk_ids=[3],
evicted_keys=[8],
),
)
@@ -482,38 +482,38 @@ def test_cpu_manager():
def test_prepare_load_preserves_key_order():
"""block_ids[i] must correspond to keys[i] (co-indexed invariant)."""
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
"""chunk_ids[i] must correspond to keys[i] (co-indexed invariant)."""
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
key_a, key_b, key_c = to_key(0), to_key(1), to_key(2)
# Store all three keys and learn their block ID assignments
# Store all three keys and learn their chunk ID assignments
store_output = manager.prepare_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
assert store_output is not None
assert isinstance(store_output.store_spec, CPULoadStoreSpec)
key_to_block_id = {
k: int(bid)
for k, bid in zip(store_output.keys_to_store, store_output.store_spec.block_ids)
key_to_chunk_id = {
k: int(cid)
for k, cid in zip(store_output.keys_to_store, store_output.store_spec.chunk_ids)
}
manager.complete_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
# Forward order: [a, b, c]
spec_fwd = manager.prepare_load([key_a, key_b, key_c], _EMPTY_REQ_CTX)
assert isinstance(spec_fwd, CPULoadStoreSpec)
assert [int(x) for x in spec_fwd.block_ids] == [
key_to_block_id[key_a],
key_to_block_id[key_b],
key_to_block_id[key_c],
assert [int(x) for x in spec_fwd.chunk_ids] == [
key_to_chunk_id[key_a],
key_to_chunk_id[key_b],
key_to_chunk_id[key_c],
]
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
# Arbitrary permutation: [b, c, a]
spec_perm = manager.prepare_load([key_b, key_c, key_a], _EMPTY_REQ_CTX)
assert isinstance(spec_perm, CPULoadStoreSpec)
assert [int(x) for x in spec_perm.block_ids] == [
key_to_block_id[key_b],
key_to_block_id[key_c],
key_to_block_id[key_a],
assert [int(x) for x in spec_perm.chunk_ids] == [
key_to_chunk_id[key_b],
key_to_chunk_id[key_c],
key_to_chunk_id[key_a],
]
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
@@ -522,10 +522,10 @@ class TestARCPolicy:
"""Unit tests for CPUOffloadingManager with ARC eviction policy."""
def _make_manager(
self, num_blocks: int = 4, enable_events: bool = True
self, num_chunks: int = 4, enable_events: bool = True
) -> tuple[CPUOffloadingManager, ARCCachePolicy]:
manager = make_cpu_manager(
num_blocks=num_blocks,
num_chunks=num_chunks,
cache_policy="arc",
enable_events=enable_events,
)
@@ -548,7 +548,7 @@ class TestARCPolicy:
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[1, 2],
store_block_ids=[0, 1],
store_chunk_ids=[0, 1],
evicted_keys=[],
),
)
@@ -569,36 +569,36 @@ class TestARCPolicy:
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.MISS
# blocks should be in T1 (recent)
# chunks should be in T1 (recent)
assert len(arc_policy.t1) == 2
assert len(arc_policy.t2) == 0
def test_t1_to_t2_promotion(self):
"""
Tests that accessing a block in T1 promotes it to T2 (frequent).
Tests that accessing a chunk in T1 promotes it to T2 (frequent).
This is a key feature of ARC's adaptive behavior.
"""
cpu_manager, arc_policy = self._make_manager(enable_events=False)
# store and complete block 1
# store and complete chunk 1
cpu_manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1]), _EMPTY_REQ_CTX)
# block 1 starts in T1 (recent)
# chunk 1 starts in T1 (recent)
assert to_keys([1])[0] in arc_policy.t1
assert to_keys([1])[0] not in arc_policy.t2
# touch block 1 (simulate second access)
# touch chunk 1 (simulate second access)
cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
# block 1 should now be in T2 (frequent)
# chunk 1 should now be in T2 (frequent)
assert to_keys([1])[0] not in arc_policy.t1
assert to_keys([1])[0] in arc_policy.t2
def test_eviction_with_load(self):
"""
Tests ARC eviction behavior similar to LRU test.
Verifies that blocks being loaded (ref_cnt > 0) cannot be evicted.
Verifies that chunks being loaded (ref_cnt > 0) cannot be evicted.
"""
cpu_manager, _ = self._make_manager()
@@ -610,7 +610,7 @@ class TestARCPolicy:
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[1, 2, 3, 4],
store_block_ids=[0, 1, 2, 3],
store_chunk_ids=[0, 1, 2, 3],
evicted_keys=[],
),
)
@@ -628,36 +628,36 @@ class TestARCPolicy:
cpu_manager.complete_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
# now prepare store [5, 6, 7] should succeed
# ARC will evict blocks one at a time from T1 as needed
# ARC will evict chunks one at a time from T1 as needed
prepare_store_output = cpu_manager.prepare_store(
to_keys([5, 6, 7]), _EMPTY_REQ_CTX
)
assert prepare_store_output is not None
# Should successfully evict enough blocks to make room (at least 1)
# Should successfully evict enough chunks to make room (at least 1)
assert len(prepare_store_output.evicted_keys) >= 1
def test_adaptive_target(self):
"""
Tests ARC's adaptive target adjustment via ghost lists.
When a block in B1 (ghost list) is accessed, target_t1_size increases.
When a block in B2 is accessed, target_t1_size decreases.
When a chunk in B1 (ghost list) is accessed, target_t1_size increases.
When a chunk in B2 is accessed, target_t1_size decreases.
"""
cpu_manager, arc_policy = self._make_manager(num_blocks=2, enable_events=False)
cpu_manager, arc_policy = self._make_manager(num_chunks=2, enable_events=False)
# store blocks 1, 2 (fills cache)
# store chunks 1, 2 (fills cache)
cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
initial_target = arc_policy.target_t1_size
# store block 3, evicting block 1 (moves to B1 ghost list)
# store chunk 3, evicting chunk 1 (moves to B1 ghost list)
cpu_manager.prepare_store(to_keys([3]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([3]), _EMPTY_REQ_CTX)
# block 1 should be in B1 (ghost list)
# chunk 1 should be in B1 (ghost list)
assert to_keys([1])[0] in arc_policy.b1
# touch block 1 (cache miss, but in B1)
# touch chunk 1 (cache miss, but in B1)
# this should increase target_t1_size (favor recency)
cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
@@ -671,11 +671,11 @@ class TestARCPolicy:
"""
cpu_manager, arc_policy = self._make_manager(enable_events=False)
# store blocks 1, 2, 3, 4
# store chunks 1, 2, 3, 4
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
# promote blocks 3, 4 to T2 by touching them
# promote chunks 3, 4 to T2 by touching them
cpu_manager.touch(to_keys([3, 4]), _EMPTY_REQ_CTX)
# now: T1 = {1, 2}, T2 = {3, 4}
@@ -686,22 +686,22 @@ class TestARCPolicy:
# (when |T1| >= target, evict from T1)
arc_policy.target_t1_size = 1
# store block 5, should evict from T1 (block 1, LRU in T1)
# store chunk 5, should evict from T1 (chunk 1, LRU in T1)
output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
assert output is not None
assert to_keys([1]) == output.evicted_keys
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX)
# block 1 should be in B1 (ghost list)
# chunk 1 should be in B1 (ghost list)
assert to_keys([1])[0] in arc_policy.b1
# block 5 should be in T1
# chunk 5 should be in T1
assert to_keys([5])[0] in arc_policy.t1
def test_batch_eviction_scans_t1_and_t2_once(self):
"""ARC batch eviction must preserve order without restarting scans."""
cpu_manager, arc_policy = self._make_manager(
num_blocks=256, enable_events=False
num_chunks=256, enable_events=False
)
keys = to_keys(list(range(256)))
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
@@ -721,13 +721,13 @@ class TestARCPolicy:
t2_order = list(arc_policy.t2)
expected_t1 = [
key
for key, block in arc_policy.t1.items()
if block.ref_cnt == 0 and key not in protected
for key, chunk in arc_policy.t1.items()
if chunk.ref_cnt == 0 and key not in protected
][:num_t1_evictions]
expected_t2 = [
key
for key, block in arc_policy.t2.items()
if block.ref_cnt == 0 and key not in protected
for key, chunk in arc_policy.t2.items()
if chunk.ref_cnt == 0 and key not in protected
][:num_t2_evictions]
expected_t1_scans = t1_order.index(expected_t1[-1]) + 1
expected_t2_scans = t2_order.index(expected_t2[-1]) + 1
@@ -746,7 +746,7 @@ class TestARCPolicy:
def test_batch_eviction_falls_back_after_t1_iterator_exhausted(self):
"""An exhausted T1 scan must keep falling back to T2."""
cpu_manager, arc_policy = self._make_manager(num_blocks=8, enable_events=False)
cpu_manager, arc_policy = self._make_manager(num_chunks=8, enable_events=False)
keys = to_keys(list(range(8)))
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
@@ -764,8 +764,8 @@ class TestARCPolicy:
# the target, so each remaining selection must retry T1 then use T2.
eligible_t1 = [
key
for key, block in arc_policy.t1.items()
if block.ref_cnt == 0 and key not in protected
for key, chunk in arc_policy.t1.items()
if chunk.ref_cnt == 0 and key not in protected
]
assert eligible_t1 == t1_order[:1]
assert len(t1_order) - 1 >= int(arc_policy.target_t1_size)
@@ -839,7 +839,7 @@ class TestARCPolicy:
def test_batch_eviction_failure_is_atomic(self):
"""Finding only some candidates must not partially evict the cache."""
cpu_manager, arc_policy = self._make_manager(num_blocks=4, enable_events=False)
cpu_manager, arc_policy = self._make_manager(num_chunks=4, enable_events=False)
keys = to_keys(list(range(4)))
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
@@ -858,13 +858,13 @@ class TestARCPolicy:
Tests that ghost lists (B1, B2) don't grow unbounded.
They should be capped at cache_capacity.
"""
cpu_manager, arc_policy = self._make_manager(num_blocks=2, enable_events=False)
cpu_manager, arc_policy = self._make_manager(num_chunks=2, enable_events=False)
# fill cache with blocks 1, 2
# fill cache with chunks 1, 2
cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
# store many blocks to fill ghost lists
# store many chunks to fill ghost lists
for i in range(3, 20):
cpu_manager.prepare_store(to_keys([i]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([i]), _EMPTY_REQ_CTX)
@@ -880,7 +880,7 @@ class TestARCPolicy:
"""
cpu_manager, arc_policy = self._make_manager()
# store blocks 1, 2, 3, 4
# store chunks 1, 2, 3, 4
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
@@ -895,13 +895,13 @@ class TestARCPolicy:
assert len(arc_policy.t1) == 1
assert len(arc_policy.t2) == 3
# store block 5, should evict from T1 (block 2, only one in T1)
# store chunk 5, should evict from T1 (chunk 2, only one in T1)
prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
verify_store_output(
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[5],
store_block_ids=[1], # reuses block 2's storage
store_chunk_ids=[1], # reuses chunk 2's storage
evicted_keys=[2],
),
)
@@ -913,11 +913,11 @@ class TestARCPolicy:
"""
cpu_manager, arc_policy = self._make_manager()
# store blocks 1, 2, 3, 4
# store chunks 1, 2, 3, 4
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
# prepare store block 5 (will evict block 1)
# prepare store chunk 5 (will evict chunk 1)
prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
assert prepare_store_output is not None
assert len(prepare_store_output.evicted_keys) == 1
@@ -925,13 +925,13 @@ class TestARCPolicy:
# complete store with failure
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX, success=False)
# block 5 should not be in cache
# chunk 5 should not be in cache
assert cpu_manager.lookup(to_key(5), _EMPTY_REQ_CTX) is LookupResult.MISS
# block 5 should not be in T1 or T2
# chunk 5 should not be in T1 or T2
assert to_keys([5])[0] not in arc_policy.t1
assert to_keys([5])[0] not in arc_policy.t2
# evicted block should still be gone (in B1 ghost list)
# evicted chunk should still be gone (in B1 ghost list)
evicted_hash = prepare_store_output.evicted_keys[0]
assert evicted_hash in arc_policy.b1
@@ -954,7 +954,7 @@ class TestARCPolicy:
assert len(prepare_store_output.evicted_keys) == 1
cpu_manager.complete_store(to_keys([3, 4, 5]), _EMPTY_REQ_CTX)
# promote some blocks to T2
# promote some chunks to T2
cpu_manager.touch(to_keys([2, 3]), _EMPTY_REQ_CTX)
# T1 has {4, 5}, T2 has {2, 3}
@@ -966,7 +966,7 @@ class TestARCPolicy:
assert prepare_store_output is not None
cpu_manager.complete_store(to_keys([6]), _EMPTY_REQ_CTX)
# verify blocks 2, 3 (in T2) are still present
# verify chunks 2, 3 (in T2) are still present
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.HIT
@@ -980,7 +980,7 @@ def test_filter_reused_manager():
Tests CPUOffloadingManager reuse filtering (store_threshold=2).
"""
manager = make_cpu_manager(
num_blocks=4,
num_chunks=4,
cache_policy="lru",
enable_events=True,
store_threshold=2,
@@ -1023,7 +1023,7 @@ def test_filter_reused_manager():
def test_filter_reused_manager_oversized_offer_makes_progress():
"""An offer larger than the tracker capacity must not churn forever."""
manager = make_cpu_manager(
num_blocks=4,
num_chunks=4,
cache_policy="lru",
store_threshold=2,
max_tracker_size=3,
@@ -1042,69 +1042,69 @@ def test_filter_reused_manager_oversized_offer_makes_progress():
assert stored_keys == set(keys)
def test_evictable_cache_block_count():
def test_evictable_cache_chunk_count():
"""
Verifies _num_evictable_cache_blocks is maintained correctly through the
Verifies _num_evictable_cache_chunks is maintained correctly through the
full store/load lifecycle, eviction, failed stores, concurrent loads,
reset_cache, and the early-exit fast path in prepare_store.
"""
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
# Initially no blocks allocated.
assert manager._num_evictable_cache_blocks == 0
# Initially no chunks allocated.
assert manager._num_evictable_cache_chunks == 0
# Initial cache state [x, x, x, x]
# We get 3 blocks from the cache.
# We get 3 chunks from the cache.
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
# cache state [1', 2', 3', x] <- 1', 2', 3' are actively being used.
assert manager._num_evictable_cache_blocks == 0
assert manager._num_evictable_cache_chunks == 0
# Completing stores makes them idle.
manager.complete_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
# cache state [1, 2, 3, x] <- 1, 2, 3 blocks are idle.
assert manager._num_evictable_cache_blocks == 3
# cache state [1, 2, 3, x] <- 1, 2, 3 chunks are idle.
assert manager._num_evictable_cache_chunks == 3
# prepare_load pins a block: idle count decrements once even if the
# same block is loaded by two concurrent callers.
# prepare_load pins a chunk: idle count decrements once even if the
# same chunk is loaded by two concurrent callers.
manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX)
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
assert manager._num_evictable_cache_blocks == 2
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
assert manager._num_evictable_cache_chunks == 2
manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX) # 2nd concurrent load
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
assert manager._num_evictable_cache_blocks == 2 # no double-decrement
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
assert manager._num_evictable_cache_chunks == 2 # no double-decrement
# First complete_load does not restore idle (ref_cnt still 1).
manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX)
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
assert manager._num_evictable_cache_blocks == 2
# Second complete_load drops ref_cnt to 0 -> block becomes idle again.
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
assert manager._num_evictable_cache_chunks == 2
# Second complete_load drops ref_cnt to 0 -> chunk becomes idle again.
manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX)
# cache state [1, 2, 3, x] <- 1, 2, 3 blocks are idle.
assert manager._num_evictable_cache_blocks == 3
# cache state [1, 2, 3, x] <- 1, 2, 3 chunks are idle.
assert manager._num_evictable_cache_chunks == 3
# Eviction decrements idle count.
# Cache has 3 stored blocks and 1 free slot. Storing 3 new keys needs 2 eviction.
# Cache has 3 stored chunks and 1 free slot. Storing 3 new keys needs 2 eviction.
manager.prepare_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX)
# cache state [1, 4', 5', 6'] <- block 1 is idle
assert manager._num_evictable_cache_blocks == 1
# cache state [1, 4', 5', 6'] <- chunk 1 is idle
assert manager._num_evictable_cache_chunks == 1
# Failed store does not increment idle count (block discarded from cache).
# Failed store does not increment idle count (chunk discarded from cache).
manager.complete_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX, success=False)
# cache state [1, x, x, x] <- block 1 is idle. Other returned to cache.
assert manager._num_evictable_cache_blocks == 1
# cache state [1, x, x, x] <- chunk 1 is idle. Other returned to cache.
assert manager._num_evictable_cache_chunks == 1
# reset_cache zeroes the count unconditionally.
manager.reset_cache()
# cache state [x, x, x, x]
assert manager._num_evictable_cache_blocks == 0
assert manager._num_evictable_cache_chunks == 0
# setup 3 blocks with loads so idle count drops to 0.
# setup 3 chunks with loads so idle count drops to 0.
manager.prepare_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
manager.prepare_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
# cache state [10', 11', 12', x]
assert manager._num_evictable_cache_blocks == 0
assert manager._num_evictable_cache_chunks == 0
# prepare_store requiring eviction must return None immediately (fast exit).
# Spy on policy.evict to confirm the fast path short-circuits before calling it.
@@ -1120,24 +1120,24 @@ def test_evictable_cache_block_count():
# cache state [10', 11', 12', x] <- cannot evict anything
assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is None
assert not evict_called, (
"_num_evictable_cache_blocks==0 should short-circuit before evict()"
"_num_evictable_cache_chunks==0 should short-circuit before evict()"
)
# After releasing the loads, eviction becomes possible again.
manager.complete_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
# cache state [10, 11, 12, x] <- 10, 11, 12 are idle
assert manager._num_evictable_cache_blocks == 3
assert manager._num_evictable_cache_chunks == 3
assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is not None
# cache state [10, 11, 14', 15'] <- 10, 11 are idle
assert manager._num_evictable_cache_blocks == 2
assert manager._num_evictable_cache_chunks == 2
manager.complete_store(to_keys([14, 15]), _EMPTY_REQ_CTX)
# cache state [10, 11, 14, 15] <- all blocks idle
assert manager._num_evictable_cache_blocks == 4
# cache state [10, 11, 14, 15] <- all chunks idle
assert manager._num_evictable_cache_chunks == 4
def test_touch_forwards_req_context_to_policy(monkeypatch):
"""Regression: CPUOffloadingManager.touch forwards ReqContext to policy."""
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
received = []
def spy_touch(keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
@@ -38,7 +38,7 @@ def _set_spawn_method(monkeypatch):
def _make_region(
engine_id: str,
num_blocks: int = 4,
num_chunks: int = 4,
cpu_page_size: int = PAGE_SIZE,
num_workers: int = 1,
rank: int = 0,
@@ -47,9 +47,9 @@ def _make_region(
assert cpu_page_size % PAGE_SIZE == 0
return SharedOffloadRegion(
engine_id=engine_id,
num_blocks=num_blocks,
num_chunks=num_chunks,
rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size,
kv_bytes_per_chunk=num_workers * cpu_page_size,
cpu_page_size=cpu_page_size,
barrier=barrier,
)
@@ -99,16 +99,16 @@ def _region(engine_id: str, **kwargs):
def _multi_region(
engine_id: str,
num_workers: int,
num_blocks: int = 4,
num_chunks: int = 4,
cpu_page_size: int = PAGE_SIZE,
):
"""Context manager: create one SharedOffloadRegion per rank, clean up on exit."""
regions = [
SharedOffloadRegion(
engine_id=engine_id,
num_blocks=num_blocks,
num_chunks=num_chunks,
rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size,
kv_bytes_per_chunk=num_workers * cpu_page_size,
cpu_page_size=cpu_page_size,
)
for rank in range(num_workers)
@@ -124,7 +124,7 @@ def _multi_region(
def _race_construct(
engine_id: str,
num_workers: int,
num_blocks: int = 4,
num_chunks: int = 4,
cpu_page_size: int = PAGE_SIZE,
) -> tuple[list[SharedOffloadRegion], list[Exception]]:
"""Spawn num_workers threads that all race to construct SharedOffloadRegion."""
@@ -137,9 +137,9 @@ def _race_construct(
try:
regions[rank] = SharedOffloadRegion(
engine_id=engine_id,
num_blocks=num_blocks,
num_chunks=num_chunks,
rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size,
kv_bytes_per_chunk=num_workers * cpu_page_size,
cpu_page_size=cpu_page_size,
)
except Exception as e:
@@ -156,7 +156,7 @@ def _race_construct(
def _mp_race_construct_and_write(
engine_id: str,
num_blocks: int,
num_chunks: int,
rank: int,
num_workers: int,
cpu_page_size: int,
@@ -170,9 +170,9 @@ def _mp_race_construct_and_write(
try:
region = SharedOffloadRegion(
engine_id=engine_id,
num_blocks=num_blocks,
num_chunks=num_chunks,
rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size,
kv_bytes_per_chunk=num_workers * cpu_page_size,
cpu_page_size=cpu_page_size,
)
t = region.create_next_worker_view(cpu_page_size)
@@ -198,9 +198,9 @@ def _mp_barrier_construct_and_hold(
try:
region = SharedOffloadRegion(
engine_id=engine_id,
num_blocks=2,
num_chunks=2,
rank=rank,
kv_bytes_per_block=num_workers * PAGE_SIZE,
kv_bytes_per_chunk=num_workers * PAGE_SIZE,
cpu_page_size=PAGE_SIZE,
barrier=lambda: barrier.wait(30),
)
@@ -233,9 +233,9 @@ def iid():
def test_create_next_worker_view_shape_and_stride(iid):
"""Returned tensor must have shape (num_blocks, tensor_page_size) and
"""Returned tensor must have shape (num_chunks, tensor_page_size) and
stride (row_stride, 1) where row_stride = cpu_page_size * num_workers."""
with _region(iid, num_blocks=4, cpu_page_size=2 * PAGE_SIZE) as r:
with _region(iid, num_chunks=4, cpu_page_size=2 * PAGE_SIZE) as r:
t = r.create_next_worker_view(PAGE_SIZE)
assert t.shape == (4, PAGE_SIZE)
# num_workers=1 → row_stride = cpu_page_size
@@ -253,7 +253,7 @@ def test_create_next_worker_view_storage_offset_rank0(iid):
def test_create_next_worker_view_storage_offset_rank1(iid):
"""rank=1 worker's first tensor must start cpu_page_size bytes into the mmap."""
with _multi_region(iid, num_workers=2, num_blocks=4) as (r0, r1):
with _multi_region(iid, num_workers=2, num_chunks=4) as (r0, r1):
t1 = r1.create_next_worker_view(PAGE_SIZE)
assert t1.data_ptr() == r1._base.data_ptr() + PAGE_SIZE
del t1
@@ -261,7 +261,7 @@ def test_create_next_worker_view_storage_offset_rank1(iid):
def test_create_next_worker_view_row_stride_with_multiple_workers(iid):
"""With num_workers=4, row_stride must be 4 * cpu_page_size."""
with _region(iid, num_blocks=2, num_workers=4) as r:
with _region(iid, num_chunks=2, num_workers=4) as r:
t = r.create_next_worker_view(PAGE_SIZE)
assert t.stride(0) == 4 * PAGE_SIZE
del t
@@ -332,12 +332,12 @@ def test_create_next_worker_view_overflow_does_not_mutate_cursor(iid):
def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
"""Writes into a create_next_worker_view view must appear at the correct
raw mmap offset"""
with _region(iid, num_blocks=4) as r:
with _region(iid, num_chunks=4) as r:
t = r.create_next_worker_view(PAGE_SIZE)
t[2, :] = 42 # write to block row 2
t[2, :] = 42 # write to chunk row 2
raw = memoryview(r.mmap_obj)
# num_workers=1 → row_stride = PAGE_SIZE; block 2 starts at byte 2*PAGE_SIZE
# num_workers=1 → row_stride = PAGE_SIZE; chunk 2 starts at byte 2*PAGE_SIZE
chunk = bytes(raw[2 * PAGE_SIZE : 3 * PAGE_SIZE])
assert all(b == 42 for b in chunk)
del raw, t
@@ -345,7 +345,7 @@ def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
def test_create_next_worker_view_multi_tensor_layout(iid):
"""Two tensors from the same worker land at consecutive byte offsets per row."""
with _region(iid, num_blocks=2, cpu_page_size=2 * PAGE_SIZE) as r:
with _region(iid, num_chunks=2, cpu_page_size=2 * PAGE_SIZE) as r:
ta = r.create_next_worker_view(PAGE_SIZE)
tb = r.create_next_worker_view(PAGE_SIZE)
@@ -353,8 +353,8 @@ def test_create_next_worker_view_multi_tensor_layout(iid):
tb[:, :] = 2
raw = memoryview(r.mmap_obj)
for blk in range(2):
row_offset = blk * 2 * PAGE_SIZE # num_workers=1
for chunk in range(2):
row_offset = chunk * 2 * PAGE_SIZE # num_workers=1
assert all(b == 1 for b in raw[row_offset : row_offset + PAGE_SIZE])
assert all(
b == 2 for b in raw[row_offset + PAGE_SIZE : row_offset + 2 * PAGE_SIZE]
@@ -366,7 +366,7 @@ def test_create_next_worker_view_multiprocess_slots(iid):
"""Each worker process calls create_next_worker_view and writes distinct data;
the parent verifies each slot lands at the correct interleaved offset."""
num_workers = 2
num_blocks = 4
num_chunks = 4
ctx = get_mp_context()
done_queue = ctx.Queue()
@@ -375,9 +375,9 @@ def test_create_next_worker_view_multiprocess_slots(iid):
# Parent is rank 0 (creator); child is rank 1 (joiner).
region = SharedOffloadRegion(
engine_id=iid,
num_blocks=num_blocks,
num_chunks=num_chunks,
rank=0,
kv_bytes_per_block=num_workers * PAGE_SIZE,
kv_bytes_per_chunk=num_workers * PAGE_SIZE,
cpu_page_size=PAGE_SIZE,
)
try:
@@ -385,7 +385,7 @@ def test_create_next_worker_view_multiprocess_slots(iid):
target=_mp_race_construct_and_write,
args=(
iid,
num_blocks,
num_chunks,
1,
num_workers,
PAGE_SIZE,
@@ -403,12 +403,12 @@ def test_create_next_worker_view_multiprocess_slots(iid):
assert result["error"] is None, result["error"]
raw = memoryview(region.mmap_obj)
for blk in range(num_blocks):
row_start = blk * num_workers * PAGE_SIZE
for chunk in range(num_chunks):
row_start = chunk * num_workers * PAGE_SIZE
w0 = bytes(raw[row_start : row_start + PAGE_SIZE])
w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE])
assert all(b == 11 for b in w0), f"block {blk}: rank0 slot wrong"
assert all(b == 22 for b in w1), f"block {blk}: rank1 slot wrong"
assert all(b == 11 for b in w0), f"chunk {chunk}: rank0 slot wrong"
assert all(b == 22 for b in w1), f"chunk {chunk}: rank1 slot wrong"
del raw, t0 # release before finally triggers cleanup
cleanup_queue.put(True)
@@ -422,8 +422,8 @@ def test_create_next_worker_view_multiprocess_slots(iid):
def test_create_next_worker_view_worker_isolation(iid):
"""Writes by worker 0 must not affect worker 1's slot and vice versa."""
num_workers = 2
num_blocks = 4
with _multi_region(iid, num_workers=num_workers, num_blocks=num_blocks) as regions:
num_chunks = 4
with _multi_region(iid, num_workers=num_workers, num_chunks=num_chunks) as regions:
t0 = regions[0].create_next_worker_view(PAGE_SIZE)
t1 = regions[1].create_next_worker_view(PAGE_SIZE)
@@ -431,12 +431,12 @@ def test_create_next_worker_view_worker_isolation(iid):
t1[:, :] = 22
raw = memoryview(regions[0].mmap_obj)
for blk in range(num_blocks):
row_start = blk * num_workers * PAGE_SIZE
for chunk in range(num_chunks):
row_start = chunk * num_workers * PAGE_SIZE
w0 = bytes(raw[row_start : row_start + PAGE_SIZE])
w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE])
assert all(b == 11 for b in w0), f"block {blk}: worker0 slot corrupted"
assert all(b == 22 for b in w1), f"block {blk}: worker1 slot corrupted"
assert all(b == 11 for b in w0), f"chunk {chunk}: worker0 slot corrupted"
assert all(b == 22 for b in w1), f"chunk {chunk}: worker1 slot corrupted"
del raw, t0, t1 # release before finally triggers cleanup
@@ -466,7 +466,7 @@ def test_file_exists_after_construction(iid):
def test_file_has_correct_size(iid):
"""The mmap file size on disk must equal total_size_bytes."""
with _region(iid, num_blocks=4) as r:
with _region(iid, num_chunks=4) as r:
assert os.path.getsize(r.mmap_path) == 4 * PAGE_SIZE
@@ -491,11 +491,11 @@ def test_madvise_success_selects_madvise_population(iid, monkeypatch):
monkeypatch.setattr(sor, "_madvise_populate_write", _spy_madvise)
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
num_blocks = 3
num_chunks = 3
num_workers = 2
with _region(iid, num_blocks=num_blocks, num_workers=num_workers, rank=1):
# 1 probe (PAGESIZE) + N populate calls (one per block per worker column).
expected_populate = num_blocks # ranked path: one call per block
with _region(iid, num_chunks=num_chunks, num_workers=num_workers, rank=1):
# 1 probe (PAGESIZE) + N populate calls (one per chunk per worker column).
expected_populate = num_chunks # ranked path: one call per chunk
mmap_id = madvise_calls[0][2]
assert madvise_calls == [
(0, mmap.PAGESIZE, mmap_id),
@@ -510,7 +510,7 @@ def test_madvise_success_selects_madvise_population(iid, monkeypatch):
def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
"""An EINVAL probe must select fallback for every ranked block."""
"""An EINVAL probe must select fallback for every ranked chunk."""
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
fallback_calls: list[tuple[int, int]] = []
@@ -526,7 +526,7 @@ def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
with _region(iid, num_blocks=3, num_workers=2, rank=1):
with _region(iid, num_chunks=3, num_workers=2, rank=1):
assert fallback_calls == [
(mmap.PAGESIZE, mmap.PAGESIZE),
(3 * mmap.PAGESIZE, mmap.PAGESIZE),
@@ -551,7 +551,7 @@ def test_madvise_einval_selects_fallback_for_unranked_region(iid, monkeypatch):
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
with _region(iid, num_blocks=3, num_workers=2, rank=None):
with _region(iid, num_chunks=3, num_workers=2, rank=None):
assert fallback_calls == [(0, 6 * mmap.PAGESIZE)]
@@ -651,7 +651,7 @@ def test_multiprocess_race_construct_and_write(iid):
"""N processes race to construct the same SharedOffloadRegion, each writes
fill_value = rank+1 into their slot; parent verifies interleaved layout."""
num_workers = 4
num_blocks = 3
num_chunks = 3
ctx = get_mp_context()
done_queue = ctx.Queue()
@@ -662,7 +662,7 @@ def test_multiprocess_race_construct_and_write(iid):
target=_mp_race_construct_and_write,
args=(
iid,
num_blocks,
num_chunks,
rank,
num_workers,
PAGE_SIZE,
@@ -689,13 +689,13 @@ def test_multiprocess_race_construct_and_write(iid):
with open(mmap_path, "rb") as f:
raw = f.read()
for blk in range(num_blocks):
for chunk in range(num_chunks):
for w in range(num_workers):
slot_start = (blk * num_workers + w) * PAGE_SIZE
slot_start = (chunk * num_workers + w) * PAGE_SIZE
slot = raw[slot_start : slot_start + PAGE_SIZE]
expected = w + 1 # fill_value = rank + 1
assert all(b == expected for b in slot), (
f"block {blk}, worker {w}: expected {expected} but got wrong bytes"
f"chunk {chunk}, worker {w}: expected {expected} but got wrong bytes"
)
# Unblock all workers to clean up.
@@ -839,9 +839,9 @@ def test_insufficient_space_raises_clear_error(monkeypatch):
with pytest.raises(RuntimeError, match="Insufficient space"):
SharedOffloadRegion(
engine_id=engine_id,
num_blocks=4,
num_chunks=4,
rank=0,
kv_bytes_per_block=PAGE_SIZE,
kv_bytes_per_chunk=PAGE_SIZE,
cpu_page_size=PAGE_SIZE,
)
@@ -870,9 +870,9 @@ def test_ftruncate_failure_cleans_up_creator(monkeypatch):
with pytest.raises(OSError, match="ftruncate failed"):
SharedOffloadRegion(
engine_id=engine_id,
num_blocks=4,
num_chunks=4,
rank=0,
kv_bytes_per_block=PAGE_SIZE,
kv_bytes_per_chunk=PAGE_SIZE,
cpu_page_size=PAGE_SIZE,
)
+11 -11
View File
@@ -136,7 +136,7 @@ def test_get_spec_cls_defaults_to_cpu():
def test_create_cpu_offloading_spec():
spec = _create_spec()
assert isinstance(spec, CPUOffloadingSpec)
assert spec.num_blocks > 0
assert spec.num_chunks > 0
def test_cpu_spec_sizes_normalized_worker_layout():
@@ -156,7 +156,7 @@ def test_cpu_spec_sizes_normalized_worker_layout():
assert isinstance(spec, CPUOffloadingSpec)
assert spec.cpu_page_size_per_worker == 32
assert spec.kv_bytes_per_chunk == alignment
assert spec.num_blocks == 3
assert spec.num_chunks == 3
def test_cpu_spec_zero_worker_bytes_produces_empty_cache():
@@ -165,7 +165,7 @@ def test_cpu_spec_zero_worker_bytes_produces_empty_cache():
assert isinstance(spec, CPUOffloadingSpec)
assert spec.cpu_page_size_per_worker == 0
assert spec.kv_bytes_per_chunk == 0
assert spec.num_blocks == 0
assert spec.num_chunks == 0
def test_tiering_spec_aligns_row_size():
@@ -183,7 +183,7 @@ def test_tiering_spec_aligns_row_size():
assert isinstance(spec, TieringOffloadingSpec)
assert spec.cpu_page_size_per_worker == 32
assert spec.kv_bytes_per_chunk == alignment
assert spec.num_blocks == 3
assert spec.num_chunks == 3
@pytest.mark.parametrize("world_size", [2, 4, 8])
@@ -201,7 +201,7 @@ def test_tiering_spec_replicated_sizing_removes_world_factor(world_size: int):
assert spec.replicated_layout is True
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block
assert spec.num_blocks == 8
assert spec.num_chunks == 8
def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monkeypatch):
@@ -239,7 +239,7 @@ def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monke
spec.create_worker(kv_caches)
assert region_calls[0]["rank"] == 0
assert region_calls[0]["kv_bytes_per_block"] == worker_kv_bytes_per_block
assert region_calls[0]["kv_bytes_per_chunk"] == worker_kv_bytes_per_block
assert worker_calls[0]["kv_caches"] is kv_caches
assert worker_calls[0]["mmap_region"] is region
@@ -292,7 +292,7 @@ def test_cpu_spec_replicated_sizing_on_shared_region(monkeypatch, world_size: in
assert spec.replicated_layout is True
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block
assert spec.num_blocks == 8
assert spec.num_chunks == 8
@pytest.mark.parametrize("world_size", [2, 4, 8])
@@ -320,7 +320,7 @@ def test_cpu_spec_replicated_disabled_without_shared_region(
assert spec.replicated_layout is False
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block * world_size
assert spec.num_blocks == 2
assert spec.num_chunks == 2
@pytest.mark.parametrize("config_replicated", [True, False])
@@ -381,7 +381,7 @@ def test_cpu_spec_create_worker_uses_mmap_on_cuda_alike(monkeypatch):
spec.create_worker(kv_caches)
assert region_calls[0]["engine_id"] == "test-engine"
assert region_calls[0]["kv_bytes_per_block"] == worker_kv_bytes_per_block * 4
assert region_calls[0]["kv_bytes_per_chunk"] == worker_kv_bytes_per_block * 4
assert worker_calls[0]["kv_caches"] is kv_caches
assert worker_calls[0]["mmap_region"] is region
@@ -419,11 +419,11 @@ def test_cpu_spec_create_worker_uses_tensor_path_off_cuda_alike(monkeypatch):
def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch):
import vllm.v1.kv_offload.cpu.spec as cpu_spec_module
# worker_kv_bytes_per_block=0 yields num_blocks=0; a zero-byte region cannot
# worker_kv_bytes_per_block=0 yields num_chunks=0; a zero-byte region cannot
# be mmap'd, so even on CUDA-alike this must fall back to the tensor path.
spec = _create_spec(worker_kv_bytes_per_block=0, world_size=4)
assert isinstance(spec, CPUOffloadingSpec)
assert spec.num_blocks == 0
assert spec.num_chunks == 0
region_calls: list[dict[str, Any]] = []
worker_calls: list[dict[str, Any]] = []
+17 -17
View File
@@ -91,17 +91,17 @@ def _req_context(kv_params: dict | None = None) -> ReqContext:
def _job_metadata(
job_id: int,
keys: list[bytes] | None = None,
block_ids: list[int] | None = None,
chunk_ids: list[int] | None = None,
kv_params: dict | None = None,
) -> TransferJob:
if keys is None:
keys = [b"key1"]
if block_ids is None:
block_ids = list(range(len(keys)))
if chunk_ids is None:
chunk_ids = list(range(len(keys)))
return TransferJob(
job_id=job_id,
keys=keys,
block_ids=np.array(block_ids),
chunk_ids=np.array(chunk_ids),
is_promotion=False,
req_context=_req_context(kv_params),
)
@@ -254,9 +254,9 @@ class TestLookup:
"kv_params,expected",
[
(_remote_decoder_kv_params(), OffloadPolicy.REQUEST_LEVEL),
(None, OffloadPolicy.BLOCK_LEVEL),
(_remote_prefiller_kv_params(), OffloadPolicy.BLOCK_LEVEL),
({"remote_decoder": {}}, OffloadPolicy.BLOCK_LEVEL),
(None, OffloadPolicy.CHUNK_LEVEL),
(_remote_prefiller_kv_params(), OffloadPolicy.CHUNK_LEVEL),
({"remote_decoder": {}}, OffloadPolicy.CHUNK_LEVEL),
],
ids=["producer", "plain", "consumer", "producer_no_id"],
)
@@ -366,7 +366,7 @@ class TestSubmitStore:
job = _job_metadata(
job_id=1,
keys=[b"k1", b"k2"],
block_ids=[3, 4],
chunk_ids=[3, 4],
kv_params=_remote_decoder_kv_params(kv_request_id="req-1"),
)
mgr.submit_store(job)
@@ -397,7 +397,7 @@ class TestSubmitStore:
job = _job_metadata(
job_id=7,
keys=[b"k1", b"k2"],
block_ids=[3, 4],
chunk_ids=[3, 4],
kv_params=_remote_decoder_kv_params(kv_request_id="req-1"),
)
mgr.submit_store(job)
@@ -440,7 +440,7 @@ class TestSubmitLoad:
"""Empty key list succeeds immediately."""
mgr = _make_manager()
job = _job_metadata(
job_id=1, keys=[], block_ids=[], kv_params=_remote_prefiller_kv_params()
job_id=1, keys=[], chunk_ids=[], kv_params=_remote_prefiller_kv_params()
)
mgr.submit_load(job)
assert mgr._finished_jobs == [JobResult(job_id=1, success=True)]
@@ -463,7 +463,7 @@ class TestSubmitLoad:
job = _job_metadata(
job_id=42,
keys=[b"k1", b"k2"],
block_ids=[5, 6],
chunk_ids=[5, 6],
kv_params=_remote_prefiller_kv_params(kv_request_id="req-42"),
)
mgr.submit_load(job)
@@ -1155,7 +1155,7 @@ class TestBidirectionalManager:
_job_metadata(
job_id=100,
keys=[b"a-block"],
block_ids=[0],
chunk_ids=[0],
kv_params=a_prefiller_params,
)
)
@@ -1163,7 +1163,7 @@ class TestBidirectionalManager:
_job_metadata(
job_id=200,
keys=[b"b-block"],
block_ids=[0],
chunk_ids=[0],
kv_params=b_prefiller_params,
)
)
@@ -1173,7 +1173,7 @@ class TestBidirectionalManager:
_job_metadata(
job_id=101,
keys=[b"b-block"],
block_ids=[0],
chunk_ids=[0],
kv_params=a_decoder_params,
)
)
@@ -1181,7 +1181,7 @@ class TestBidirectionalManager:
_job_metadata(
job_id=201,
keys=[b"a-block"],
block_ids=[0],
chunk_ids=[0],
kv_params=b_decoder_params,
)
)
@@ -1573,7 +1573,7 @@ class TestConnectionDeathMidTransfer:
_job_metadata(
job_id=900,
keys=[b"a-block"],
block_ids=[0],
chunk_ids=[0],
kv_params=a_prefiller_params,
)
)
@@ -1581,7 +1581,7 @@ class TestConnectionDeathMidTransfer:
_job_metadata(
job_id=901,
keys=[b"b-block"],
block_ids=[0],
chunk_ids=[0],
kv_params=a_decoder_params,
)
)
@@ -259,13 +259,13 @@ class FakeParent:
) -> TransferJob:
keys_list = list(keys)
self.calls.append(("create_store_job", tuple(keys_list), ctx.req_id))
block_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32)
chunk_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32)
job_id = self._next_job_id
self._next_job_id += 1
return TransferJob(
job_id=job_id,
keys=keys_list,
block_ids=block_ids,
chunk_ids=chunk_ids,
is_promotion=False,
req_context=ctx,
)
+13 -13
View File
@@ -105,15 +105,15 @@ def key(n: int) -> OffloadKey:
def make_job(
job_id: int,
keys: list[OffloadKey],
block_ids: list[int] | None = None,
chunk_ids: list[int] | None = None,
is_promotion: bool = False,
) -> TransferJob:
if block_ids is None:
block_ids = list(range(len(keys)))
if chunk_ids is None:
chunk_ids = list(range(len(keys)))
return TransferJob(
job_id=job_id,
keys=keys,
block_ids=np.array(block_ids, dtype=np.int64),
chunk_ids=np.array(chunk_ids, dtype=np.int64),
is_promotion=is_promotion,
req_context=_CTX,
)
@@ -382,11 +382,11 @@ def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size):
tensor[:] = _page_aligned_rand_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS)
keys = [key(i) for i in range(batch_size)]
store_block_ids = list(range(batch_size))
load_block_ids = list(range(_NUM_BLOCKS - batch_size, _NUM_BLOCKS))
store_chunk_ids = list(range(batch_size))
load_chunk_ids = list(range(_NUM_BLOCKS - batch_size, _NUM_BLOCKS))
expected = tensor[:batch_size].clone()
tier.submit_store(make_job(1, keys, store_block_ids))
tier.submit_store(make_job(1, keys, store_chunk_ids))
store_results = drain(tier)
assert len(store_results) == 1
assert store_results[0].success
@@ -396,15 +396,15 @@ def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size):
tensor[:] = 0.0
# Load into a range disjoint by index from the store ids, to also
# exercise loading a block into a different id than it was stored from.
tier.submit_load(make_job(2, keys, load_block_ids, is_promotion=True))
# exercise loading a chunk into a different id than it was stored from.
tier.submit_load(make_job(2, keys, load_chunk_ids, is_promotion=True))
load_results = drain(tier)
assert len(load_results) == 1
assert load_results[0].success
for i, bid in enumerate(load_block_ids):
assert torch.allclose(tensor[bid], expected[i]), (
f"Block {bid} data mismatch after store+load"
for i, cid in enumerate(load_chunk_ids):
assert torch.allclose(tensor[cid], expected[i]), (
f"Chunk {cid} data mismatch after store+load"
)
@@ -895,7 +895,7 @@ def test_cascade_store_emits_fs_event_through_tiering_manager(tmp_path):
view = memoryview(tensor.numpy())
mock_region = MagicMock()
mock_region.create_kv_memoryview.return_value = view
primary = CPUPrimaryTierOffloadingManager(num_blocks=4, mmap_region=mock_region)
primary = CPUPrimaryTierOffloadingManager(num_chunks=4, mmap_region=mock_region)
tier = FileSystemTierManager(
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
primary_kv_view=primary.get_kv_memoryview(),
+22 -22
View File
@@ -25,8 +25,8 @@ def to_keys(int_ids: Iterable[int]):
def test_tiering_metrics_tracker_records_lookup_metrics():
tracker = TieringMetricsTracker(
tier_types=["fs", "p2p"],
num_primary_blocks=5,
primary_block_size=16,
num_primary_chunks=5,
primary_chunk_size=16,
)
tracker.on_new_request(_CTX)
@@ -45,18 +45,18 @@ def test_tiering_metrics_tracker_records_lookup_metrics():
stats = tracker.take_stats()
assert stats is not None
values = stats.data["data"]
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("2:p2p",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_HITS][("2:p2p",)] == 1
assert ("1:fs",) not in values[TieringOffloadingMetrics.BLOCK_HITS]
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("0:primary",)] == 1
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("1:fs",)] == 1
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("2:p2p",)] == 1
assert values[TieringOffloadingMetrics.CHUNK_HITS][("2:p2p",)] == 1
assert ("1:fs",) not in values[TieringOffloadingMetrics.CHUNK_HITS]
def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
tracker = TieringMetricsTracker(
tier_types=["fs"],
num_primary_blocks=5,
primary_block_size=16,
num_primary_chunks=5,
primary_chunk_size=16,
)
tracker.on_new_request(_CTX)
key = to_keys([1])[0]
@@ -73,9 +73,9 @@ def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
stats = tracker.take_stats()
assert stats is not None
values = stats.data["data"]
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_HITS][("1:fs",)] == 1
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("0:primary",)] == 1
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("1:fs",)] == 1
assert values[TieringOffloadingMetrics.CHUNK_HITS][("1:fs",)] == 1
tracker.on_request_allocated(_CTX)
tracker.on_lookup(
@@ -90,15 +90,15 @@ def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
stats = tracker.take_stats()
assert stats is not None
values = stats.data["data"]
assert TieringOffloadingMetrics.BLOCK_QUERIES not in values
assert TieringOffloadingMetrics.BLOCK_HITS not in values
assert TieringOffloadingMetrics.CHUNK_QUERIES not in values
assert TieringOffloadingMetrics.CHUNK_HITS not in values
def test_tiering_metrics_tracker_records_finished_job_metrics():
tracker = TieringMetricsTracker(
tier_types=["fs"],
num_primary_blocks=5,
primary_block_size=16,
num_primary_chunks=5,
primary_chunk_size=16,
)
cascade_key, promotion_key, failed_cascade_key, failed_promotion_key = to_keys(
range(4)
@@ -149,8 +149,8 @@ def test_tiering_metrics_tracker_records_finished_job_metrics():
def test_tiering_metrics_tracker_records_partial_promotion_success_bytes():
tracker = TieringMetricsTracker(
tier_types=["fs"],
num_primary_blocks=5,
primary_block_size=16,
num_primary_chunks=5,
primary_chunk_size=16,
)
keys = to_keys(range(3))
job = JobMetadata(
@@ -182,8 +182,8 @@ def test_tiering_metrics_tracker_records_partial_promotion_success_bytes():
def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges():
tracker = TieringMetricsTracker(
tier_types=["fs", "p2p"],
num_primary_blocks=6,
primary_block_size=16,
num_primary_chunks=6,
primary_chunk_size=16,
)
fs_job = JobMetadata(
TransferJob(0, to_keys([0, 1]), np.array([0, 1]), False, _CTX),
@@ -214,8 +214,8 @@ def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges():
def test_tiering_metrics_tracker_records_promotion_allocation_failures():
tracker = TieringMetricsTracker(
tier_types=["fs"],
num_primary_blocks=1,
primary_block_size=16,
num_primary_chunks=1,
primary_chunk_size=16,
)
tracker.on_promotion_allocation_failure()
+5 -5
View File
@@ -106,14 +106,14 @@ def key(n: int) -> OffloadKey:
def make_job(
job_id: int,
keys: list[OffloadKey],
block_ids: list[int] | None = None,
chunk_ids: list[int] | None = None,
) -> TransferJob:
if block_ids is None:
block_ids = list(range(len(keys)))
if chunk_ids is None:
chunk_ids = list(range(len(keys)))
return TransferJob(
job_id=job_id,
keys=keys,
block_ids=np.array(block_ids, dtype=np.int32),
chunk_ids=np.array(chunk_ids, dtype=np.int32),
is_promotion=False,
req_context=_CTX,
)
@@ -541,7 +541,7 @@ class TestMockObjTierFailures:
mmap_region = MagicMock()
mmap_region.create_kv_memoryview.return_value = primary_kv_view
primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=num_blocks, mmap_region=mmap_region
num_chunks=num_blocks, mmap_region=mmap_region
)
obj_tier, agent = _make_tier(
num_blocks=num_blocks, primary_kv_view=primary_kv_view
@@ -5,15 +5,16 @@ Unit tests for TieringOffloadingManager and ExampleSecondaryTierManager.
These tests verify:
1. Basic tiered offloading operations (store, load, lookup)
2. Cascade behavior (blocks stored to all secondary tiers)
3. Promotion behavior (blocks loaded from secondary to primary to GPU)
4. ref_cnt management (blocks protected during async transfers)
2. Cascade behavior (chunks stored to all secondary tiers)
3. Promotion behavior (chunks loaded from secondary to primary to GPU)
4. ref_cnt management (chunks protected during async transfers)
5. Eviction coordination between tiers
"""
from collections.abc import Iterable
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
@@ -57,10 +58,10 @@ _CTX = ReqContext(req_id="test")
_MOCK_OFFLOADING_SPEC = MagicMock()
def _mock_mmap_region(num_blocks: int, row_bytes: int = 16):
def _mock_mmap_region(num_chunks: int, row_bytes: int = 16):
"""Create a mock SharedOffloadRegion for testing."""
mock = MagicMock()
view = memoryview(torch.zeros((num_blocks, row_bytes), dtype=torch.int8).numpy())
view = memoryview(torch.zeros((num_chunks, row_bytes), dtype=torch.int8).numpy())
mock.create_kv_memoryview.return_value = view
return mock
@@ -160,7 +161,7 @@ def test_tiering_spec_collects_secondary_metric_definitions(monkeypatch):
metadata = metrics[MetricsSecondaryTierManager.MY_TIER_METRIC]
assert metadata.documentation == "Number of bytes served by the test tier."
assert metadata.labelnames == ("tier",)
assert metrics[TieringOffloadingMetrics.BLOCK_QUERIES].labelnames == ("tier",)
assert metrics[TieringOffloadingMetrics.CHUNK_QUERIES].labelnames == ("tier",)
assert metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY].labelnames == ("tier",)
assert metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY].labelnames == ("tier",)
assert metrics[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES].labelnames == (
@@ -183,7 +184,7 @@ def test_tiering_spec_collects_secondary_metric_definitions(monkeypatch):
def test_tiering_manager_aggregates_secondary_stats():
mock_region = _mock_mmap_region(5)
primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=5, mmap_region=mock_region
num_chunks=5, mmap_region=mock_region
)
secondary_tier = MetricsSecondaryTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC,
@@ -232,19 +233,19 @@ class TestExampleSecondaryTierManager:
)
# Initially empty
blocks = to_keys(range(3))
assert tier.lookup(blocks[0], _CTX) is LookupResult.MISS
chunks = to_keys(range(3))
assert tier.lookup(chunks[0], _CTX) is LookupResult.MISS
# Store blocks (simulate with direct insertion for testing)
tier.blocks[blocks[0]] = True
tier.blocks[blocks[1]] = True
# Store chunks (simulate with direct insertion for testing)
tier.chunks[chunks[0]] = True
tier.chunks[chunks[1]] = True
# Lookup should find first two blocks
assert tier.lookup(blocks[0], _CTX) is LookupResult.HIT
assert tier.lookup(blocks[1], _CTX) is LookupResult.HIT
# Lookup should find first two chunks
assert tier.lookup(chunks[0], _CTX) is LookupResult.HIT
assert tier.lookup(chunks[1], _CTX) is LookupResult.HIT
# Third block not present
assert tier.lookup(blocks[2], _CTX) is LookupResult.MISS
# Third chunk not present
assert tier.lookup(chunks[2], _CTX) is LookupResult.MISS
# What a request-level cascade does with a key already present in the primary
@@ -260,7 +261,7 @@ _CASCADE_SUPPLY_BEHAVIOR = {
# The same test, before it lands.
LookupResult.HIT_PENDING: "defer",
# test_cascade_drops_deferred_keys_whose_primary_write_failed, where the
# failed write frees the block.
# failed write frees the chunk.
LookupResult.MISS: "drop",
# test_cascade_rejects_retry_from_primary. The primary tier resolves every
# key it holds, so RETRY cannot reach the cascade, and parking on it would
@@ -284,7 +285,7 @@ class TestTieringOffloadingManager:
# Create primary tier (CPU-based)
mock_region = _mock_mmap_region(5)
self.primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=5, mmap_region=mock_region
num_chunks=5, mmap_region=mock_region
)
mock_view = mock_region.create_kv_memoryview()
@@ -331,7 +332,7 @@ class TestTieringOffloadingManager:
TransferJob(
job_id=job_id,
keys=to_keys([1, 2]),
block_ids=[0, 1],
chunk_ids=np.array([0, 1], dtype=np.int64),
is_promotion=True,
req_context=_CTX,
),
@@ -358,7 +359,7 @@ class TestTieringOffloadingManager:
TransferJob(
job_id=job_id,
keys=to_keys([1]),
block_ids=[0],
chunk_ids=np.array([0], dtype=np.int64),
is_promotion=True,
req_context=_CTX,
),
@@ -392,23 +393,23 @@ class TestTieringOffloadingManager:
def test_basic_store_to_primary(self, manager_setup):
"""Test basic store operation to primary tier."""
blocks = to_keys(range(3))
chunks = to_keys(range(3))
# Prepare store
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
result = self.manager.prepare_store(chunks, _CTX)
assert result is not None
assert len(result.keys_to_store) == 3
# Complete store
self.manager.complete_store(blocks, _CTX, success=True)
self.manager.complete_store(chunks, _CTX, success=True)
# Blocks should be in primary tier
assert count_hits(self.primary_tier, blocks) == 3
# Chunks should be in primary tier
assert count_hits(self.primary_tier, chunks) == 3
def test_cascade_to_all_secondary_tiers(self, manager_setup):
"""Test that blocks are cascaded to ALL secondary tiers."""
blocks = to_keys(range(3))
"""Test that chunks are cascaded to ALL secondary tiers."""
chunks = to_keys(range(3))
self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store
@@ -419,44 +420,44 @@ class TestTieringOffloadingManager:
# Store to primary
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
result = self.manager.prepare_store(chunks, _CTX)
assert result is not None
# Complete store (triggers cascade via submit_store on each tier)
self.manager.complete_store(blocks, _CTX, success=True)
self.manager.complete_store(chunks, _CTX, success=True)
# submit_store was called once per secondary tier
self.secondary_tier1.submit_store.assert_called_once()
self.secondary_tier2.submit_store.assert_called_once()
# Blocks should be in both secondary tiers
assert self.secondary_tier1.get_num_blocks() == 3
assert self.secondary_tier2.get_num_blocks() == 3
# Chunks should be in both secondary tiers
assert self.secondary_tier1.get_num_chunks() == 3
assert self.secondary_tier2.get_num_chunks() == 3
# Verify blocks are present
# Verify chunks are present
assert all(
self.secondary_tier1.lookup(b, _CTX) is LookupResult.HIT for b in blocks
self.secondary_tier1.lookup(c, _CTX) is LookupResult.HIT for c in chunks
)
assert all(
self.secondary_tier2.lookup(b, _CTX) is LookupResult.HIT for b in blocks
self.secondary_tier2.lookup(c, _CTX) is LookupResult.HIT for c in chunks
)
def test_ref_cnt_protection_during_cascade(self, manager_setup):
"""Test that ref_cnt protects blocks during cascade."""
blocks = to_keys(range(3))
"""Test that ref_cnt protects chunks during cascade."""
chunks = to_keys(range(3))
# Store to primary
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
result = self.manager.prepare_store(chunks, _CTX)
assert result is not None
self.manager.complete_store(blocks, _CTX, success=True)
self.manager.complete_store(chunks, _CTX, success=True)
# After complete_store, blocks should have ref_cnt > 0
# After complete_store, chunks should have ref_cnt > 0
# (one for each secondary tier)
for block_hash in blocks:
block = self.primary_tier._policy.get(block_hash)
for key in chunks:
chunk = self.primary_tier._policy.get(key)
# ref_cnt should be 2 (one for each secondary tier)
assert block.ref_cnt == 2
assert chunk.ref_cnt == 2
# End of step 1: _maybe_process_finished_jobs() was already called by
# prepare_store() above (setting the per-step flag), so on_schedule_end()
@@ -466,9 +467,9 @@ class TestTieringOffloadingManager:
# ref_cnt still held: cascade jobs finished (sync tier) but haven't
# been polled yet because the per-step guard skipped the second call.
for block_hash in blocks:
block = self.primary_tier._policy.get(block_hash)
assert block.ref_cnt == 2
for key in chunks:
chunk = self.primary_tier._policy.get(key)
assert chunk.ref_cnt == 2
# Secondary tiers have completed jobs waiting to be drained
assert len(self.secondary_tier1.completed_jobs) > 0
@@ -479,37 +480,37 @@ class TestTieringOffloadingManager:
self._simulate_on_schedule_end()
# After cascade completes, ref_cnt should be 0
for block_hash in blocks:
block = self.primary_tier._policy.get(block_hash)
assert block.ref_cnt == 0
for key in chunks:
chunk = self.primary_tier._policy.get(key)
assert chunk.ref_cnt == 0
# All completed jobs have been drained
assert len(self.secondary_tier1.completed_jobs) == 0
assert len(self.secondary_tier2.completed_jobs) == 0
def test_lookup_from_primary(self, manager_setup):
"""Test lookup when blocks are in primary tier."""
blocks = to_keys(range(3))
"""Test lookup when chunks are in primary tier."""
chunks = to_keys(range(3))
# Store blocks
# Store chunks
self._start_request()
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
self.manager.prepare_store(chunks, _CTX)
self.manager.complete_store(chunks, _CTX, success=True)
# Lookup should find all blocks in primary
assert count_hits(self.manager, blocks) == 3
# Lookup should find all chunks in primary
assert count_hits(self.manager, chunks) == 3
def test_promotion_from_secondary(self, manager_setup):
"""Test promotion of blocks from secondary to primary tier."""
blocks = to_keys(range(3))
"""Test promotion of chunks from secondary to primary tier."""
chunks = to_keys(range(3))
# Manually add blocks to secondary tier (simulate previous cascade)
for block in blocks:
self.secondary_tier1.blocks[block] = True
# Manually add chunks to secondary tier (simulate previous cascade)
for chunk in chunks:
self.secondary_tier1.chunks[chunk] = True
# Lookup each block to initiate promotion for all of them
for block in blocks:
result = self.manager.lookup(block, _CTX)
# Lookup each chunk to initiate promotion for all of them
for chunk in chunks:
result = self.manager.lookup(chunk, _CTX)
assert result is LookupResult.HIT_PENDING # promotion initiated
# End of step 1: flushes deferred submit_load() calls
@@ -518,11 +519,11 @@ class TestTieringOffloadingManager:
# End of step 2: processes the completed promotion jobs
self._simulate_on_schedule_end()
# Now blocks should be in primary tier
assert count_hits(self.primary_tier, blocks) == 3
# Now chunks should be in primary tier
assert count_hits(self.primary_tier, chunks) == 3
# Next lookup should succeed
assert count_hits(self.manager, blocks) == 3
assert count_hits(self.manager, chunks) == 3
@pytest.mark.parametrize(
("successful_indices", "expected_results"),
@@ -538,18 +539,18 @@ class TestTieringOffloadingManager:
],
ids=["partial", "legacy-full-failure"],
)
def test_failed_promotion_keeps_only_successful_blocks(
def test_failed_promotion_keeps_only_successful_chunks(
self, manager_setup, successful_indices, expected_results
):
blocks = to_keys(range(3))
for block in blocks:
self.secondary_tier1.blocks[block] = True
chunks = to_keys(range(3))
for chunk in chunks:
self.secondary_tier1.chunks[chunk] = True
def submit_partial(job_metadata: TransferJob) -> None:
successful_keys = (
None
if successful_indices is None
else tuple(blocks[i] for i in successful_indices)
else tuple(chunks[i] for i in successful_indices)
)
self.secondary_tier1.completed_jobs.append(
JobResult(
@@ -561,24 +562,24 @@ class TestTieringOffloadingManager:
self.secondary_tier1.submit_load = submit_partial
for block in blocks:
assert self.manager.lookup(block, _CTX) is LookupResult.HIT_PENDING
for chunk in chunks:
assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT_PENDING
self._simulate_on_schedule_end()
self._simulate_on_schedule_end()
assert [
self.primary_tier.lookup(block, _CTX) for block in blocks
self.primary_tier.lookup(chunk, _CTX) for chunk in chunks
] == expected_results
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
"""Resolved lookups report one sync delay sample per tier and block."""
"""Resolved lookups report one sync delay sample per tier and chunk."""
self._start_request()
blocks = to_keys(range(2))
chunks = to_keys(range(2))
# No tier has these blocks: they resolve immediately as misses.
for block in blocks:
assert self.manager.lookup(block, _CTX) is LookupResult.MISS
# No tier has these chunks: they resolve immediately as misses.
for chunk in chunks:
assert self.manager.lookup(chunk, _CTX) is LookupResult.MISS
stats = self.manager.get_stats()
assert stats is not None
@@ -614,11 +615,11 @@ class TestTieringOffloadingManager:
def test_lookup_does_not_report_async_delay_for_promotion(self, manager_setup):
"""Promotion time is not included in lookup async delay."""
self._start_request()
block = to_keys(range(1))[0]
self.secondary_tier1.blocks[block] = True
chunk = to_keys(range(1))[0]
self.secondary_tier1.chunks[chunk] = True
# First lookup finds the block in a secondary tier and defers.
assert self.manager.lookup(block, _CTX) is LookupResult.HIT_PENDING
# First lookup finds the chunk in a secondary tier and defers.
assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT_PENDING
# Promotion is not treated as unresolved secondary lookup time.
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
@@ -648,8 +649,8 @@ class TestTieringOffloadingManager:
# Promotion completes on the next scheduler step.
self._simulate_on_schedule_end()
# Next lookup resolves via the now-promoted primary-tier block.
assert self.manager.lookup(block, _CTX) is LookupResult.HIT
# Next lookup resolves via the now-promoted primary-tier chunk.
assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT
stats = self.manager.get_stats()
if stats is not None:
@@ -663,13 +664,13 @@ class TestTieringOffloadingManager:
"""Async delay is observed when a RETRY lookup later resolves."""
ctx = ReqContext(req_id="req_lookup_finish")
self._start_request(ctx)
block = to_keys(range(1))[0]
chunk = to_keys(range(1))[0]
self.secondary_tier1.lookup = MagicMock(
side_effect=[LookupResult.RETRY, LookupResult.HIT]
)
# First lookup is deferred by the secondary tier.
assert self.manager.lookup(block, ctx) is LookupResult.RETRY
assert self.manager.lookup(chunk, ctx) is LookupResult.RETRY
stats = self.manager.get_stats()
if stats is not None:
@@ -677,7 +678,7 @@ class TestTieringOffloadingManager:
stats.reduce(), TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
)
assert self.manager.lookup(block, ctx) is LookupResult.HIT_PENDING
assert self.manager.lookup(chunk, ctx) is LookupResult.HIT_PENDING
stats = self.manager.get_stats()
assert stats is not None
@@ -701,68 +702,68 @@ class TestTieringOffloadingManager:
def test_partial_lookup(self, manager_setup):
"""Test lookup with partial hits."""
blocks = to_keys(range(5))
chunks = to_keys(range(5))
# Store first 3 blocks to primary
# Store first 3 chunks to primary
self._start_request()
self.manager.prepare_store(blocks[:3], _CTX)
self.manager.complete_store(blocks[:3], _CTX, success=True)
self.manager.prepare_store(chunks[:3], _CTX)
self.manager.complete_store(chunks[:3], _CTX, success=True)
# Lookup all 5 blocks should return 3 (first 3 found)
assert count_hits(self.manager, blocks) == 3
# Lookup all 5 chunks should return 3 (first 3 found)
assert count_hits(self.manager, chunks) == 3
def test_eviction_in_primary_tier(self, manager_setup):
"""Test eviction in primary tier when capacity is exceeded."""
# Primary tier has capacity of 5 blocks
# Primary tier has capacity of 5 chunks
# First, fill the primary tier
blocks = to_keys(range(5))
chunks = to_keys(range(5))
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
result = self.manager.prepare_store(chunks, _CTX)
assert result is not None
assert len(result.keys_to_store) == 5
self.manager.complete_store(blocks, _CTX, success=True)
self.manager.complete_store(chunks, _CTX, success=True)
# End of step: release ref_cnt from cascade
self._simulate_on_schedule_end()
# Now try to store 2 more blocks (should trigger eviction)
more_blocks = to_keys(range(5, 7))
result = self.manager.prepare_store(more_blocks, _CTX)
# Now try to store 2 more chunks (should trigger eviction)
more_chunks = to_keys(range(5, 7))
result = self.manager.prepare_store(more_chunks, _CTX)
# Should evict 2 blocks from primary tier
# Should evict 2 chunks from primary tier
assert result is not None
assert len(result.evicted_keys) == 2
assert len(result.keys_to_store) == 2
def test_touch_propagates_to_all_tiers(self, manager_setup):
"""Test that touch() propagates to all tiers."""
blocks = to_keys(range(3))
chunks = to_keys(range(3))
# Store blocks
# Store chunks
self._start_request()
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
self.manager.prepare_store(chunks, _CTX)
self.manager.complete_store(chunks, _CTX, success=True)
self._simulate_on_schedule_end()
# for secondary tiers to drain jobs, so primary tier's blocks are evictable.
# for secondary tiers to drain jobs, so primary tier's chunks are evictable.
self._simulate_on_schedule_end()
self.secondary_tier1.touch = MagicMock(wraps=self.secondary_tier1.touch)
self.secondary_tier2.touch = MagicMock(wraps=self.secondary_tier2.touch)
# Touch blocks
self.manager.touch(blocks, _CTX)
# Touch chunks
self.manager.touch(chunks, _CTX)
# Verify touch was called on primary tier (check LRU order)
primary_keys = list(self.primary_tier._policy.evictable_blocks.keys())
assert primary_keys[-3:] == list(reversed(blocks))
primary_keys = list(self.primary_tier._policy.evictable_chunks.keys())
assert primary_keys[-3:] == list(reversed(chunks))
# Verify touch was propagated to all secondary tiers
self.secondary_tier1.touch.assert_called_once_with(blocks, _CTX)
self.secondary_tier2.touch.assert_called_once_with(blocks, _CTX)
self.secondary_tier1.touch.assert_called_once_with(chunks, _CTX)
self.secondary_tier2.touch.assert_called_once_with(chunks, _CTX)
def test_failed_store_no_cascade(self, manager_setup):
"""Test that failed GPU→primary store doesn't cascade."""
blocks = to_keys(range(3))
chunks = to_keys(range(3))
self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store
@@ -773,11 +774,11 @@ class TestTieringOffloadingManager:
# Prepare store
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
result = self.manager.prepare_store(chunks, _CTX)
assert result is not None
# Complete store with failure — cascade must not happen
self.manager.complete_store(blocks, _CTX, success=False)
self.manager.complete_store(chunks, _CTX, success=False)
# submit_store was never called on either secondary tier
self.secondary_tier1.submit_store.assert_not_called()
@@ -786,12 +787,12 @@ class TestTieringOffloadingManager:
def test_lookup_batches_submit_load_per_request(self, manager_setup):
"""lookup() defers submit_load until on_schedule_end(), one per request.
Blocks from different requests each get their own submit_load call, each
Chunks from different requests each get their own submit_load call, each
carrying the correct req_context.
"""
blocks = to_keys(range(4))
for block in blocks:
self.secondary_tier1.blocks[block] = True
chunks = to_keys(range(4))
for chunk in chunks:
self.secondary_tier1.chunks[chunk] = True
self.secondary_tier1.submit_load = MagicMock(
wraps=self.secondary_tier1.submit_load
@@ -801,10 +802,10 @@ class TestTieringOffloadingManager:
ctx_b = ReqContext(req_id="req_b")
# All lookups return HIT_PENDING: secondary hit triggers promotion
assert self.manager.lookup(blocks[0], ctx_a) is LookupResult.HIT_PENDING
assert self.manager.lookup(blocks[1], ctx_a) is LookupResult.HIT_PENDING
assert self.manager.lookup(blocks[2], ctx_b) is LookupResult.HIT_PENDING
assert self.manager.lookup(blocks[3], ctx_b) is LookupResult.HIT_PENDING
assert self.manager.lookup(chunks[0], ctx_a) is LookupResult.HIT_PENDING
assert self.manager.lookup(chunks[1], ctx_a) is LookupResult.HIT_PENDING
assert self.manager.lookup(chunks[2], ctx_b) is LookupResult.HIT_PENDING
assert self.manager.lookup(chunks[3], ctx_b) is LookupResult.HIT_PENDING
# submit_load must not fire during lookup - only at end of step
self.secondary_tier1.submit_load.assert_not_called()
@@ -816,20 +817,20 @@ class TestTieringOffloadingManager:
calls = self.secondary_tier1.submit_load.call_args_list
jm_a = calls[0].args[0]
jm_b = calls[1].args[0]
assert set(jm_a.keys) == {blocks[0], blocks[1]}
assert set(jm_a.keys) == {chunks[0], chunks[1]}
assert jm_a.req_context is ctx_a
assert set(jm_b.keys) == {blocks[2], blocks[3]}
assert set(jm_b.keys) == {chunks[2], chunks[3]}
assert jm_b.req_context is ctx_b
def test_lookup_shared_block_no_duplicate_promotion(self, manager_setup):
"""A block looked up by two requests in the same step is promoted once.
def test_lookup_shared_chunk_no_duplicate_promotion(self, manager_setup):
"""A chunk looked up by two requests in the same step is promoted once.
The first lookup initiates promotion (returns None via secondary hit).
The second lookup sees ref_cnt=-1 on the primary slot and returns None
via the primary in-flight path — without triggering a second promotion.
"""
shared_block = to_keys([0])[0]
self.secondary_tier1.blocks[shared_block] = True
shared_chunk = to_keys([0])[0]
self.secondary_tier1.chunks[shared_chunk] = True
self.secondary_tier1.submit_load = MagicMock(
wraps=self.secondary_tier1.submit_load
@@ -838,10 +839,10 @@ class TestTieringOffloadingManager:
ctx_a = ReqContext(req_id="req_a")
ctx_b = ReqContext(req_id="req_b")
result_a = self.manager.lookup(shared_block, ctx_a)
result_b = self.manager.lookup(shared_block, ctx_b)
result_a = self.manager.lookup(shared_chunk, ctx_a)
result_b = self.manager.lookup(shared_chunk, ctx_b)
# Both lookups find the shared_block and trigger promotion
# Both lookups find the shared_chunk and trigger promotion
# returning HIT_PENDING.
assert result_a is LookupResult.HIT_PENDING
assert result_b is LookupResult.HIT_PENDING
@@ -851,12 +852,12 @@ class TestTieringOffloadingManager:
# Only one submit_load call despite two lookups
self.secondary_tier1.submit_load.assert_called_once()
job_metadata = self.secondary_tier1.submit_load.call_args.args[0]
assert list(job_metadata.keys) == [shared_block]
assert list(job_metadata.keys) == [shared_chunk]
assert job_metadata.req_context is ctx_a
def test_complete_store_forwards_req_context_to_submit_store(self, manager_setup):
"""complete_store cascades to secondary tiers with the correct req_context."""
blocks = to_keys(range(2))
chunks = to_keys(range(2))
self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store
@@ -865,8 +866,8 @@ class TestTieringOffloadingManager:
ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"})
self._start_request(ctx)
self.manager.prepare_store(blocks, ctx)
self.manager.complete_store(blocks, ctx, success=True)
self.manager.prepare_store(chunks, ctx)
self.manager.complete_store(chunks, ctx, success=True)
assert self.secondary_tier1.submit_store.call_count == 1
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
@@ -876,7 +877,7 @@ class TestTieringOffloadingManager:
self, manager_setup
):
"""Manager hook is eager; secondary hooks wait for cascade submission."""
blocks = to_keys(range(2))
chunks = to_keys(range(2))
ctx = ReqContext(req_id="req_delayed_secondary")
calls: list[tuple[str, str]] = []
@@ -911,14 +912,14 @@ class TestTieringOffloadingManager:
)
self._start_request(ctx)
self.manager.prepare_store(blocks, ctx)
self.manager.prepare_store(chunks, ctx)
self.manager.on_request_finished(ctx)
assert calls == [("primary_finish", ctx.req_id)]
self.secondary_tier1.on_request_finished.assert_not_called()
self.secondary_tier2.on_request_finished.assert_not_called()
self.manager.complete_store(blocks, ctx, success=True)
self.manager.complete_store(chunks, ctx, success=True)
assert calls == [
("primary_finish", ctx.req_id),
@@ -930,7 +931,7 @@ class TestTieringOffloadingManager:
def test_failed_store_finalizes_finished_request(self, manager_setup):
"""Failed primary stores still unblock secondary finalization."""
blocks = to_keys(range(2))
chunks = to_keys(range(2))
ctx = ReqContext(req_id="req_failed_store_finalize")
self.secondary_tier1.submit_store = MagicMock(
@@ -947,13 +948,13 @@ class TestTieringOffloadingManager:
)
self._start_request(ctx)
self.manager.prepare_store(blocks, ctx)
self.manager.prepare_store(chunks, ctx)
self.manager.on_request_finished(ctx)
self.secondary_tier1.on_request_finished.assert_not_called()
self.secondary_tier2.on_request_finished.assert_not_called()
self.manager.complete_store(blocks, ctx, success=False)
self.manager.complete_store(chunks, ctx, success=False)
self.secondary_tier1.submit_store.assert_not_called()
self.secondary_tier2.submit_store.assert_not_called()
@@ -981,7 +982,7 @@ class TestTieringOffloadingManager:
def test_reset_cache_finalizes_delayed_secondary_request(self, manager_setup):
"""reset_cache abandons pending primary stores and finalizes secondaries."""
blocks = to_keys(range(2))
chunks = to_keys(range(2))
ctx = ReqContext(req_id="req_reset_finalize_secondary")
self.secondary_tier1.on_request_finished = MagicMock(
@@ -992,7 +993,7 @@ class TestTieringOffloadingManager:
)
self._start_request(ctx)
self.manager.prepare_store(blocks, ctx)
self.manager.prepare_store(chunks, ctx)
self.manager.on_request_finished(ctx)
self.secondary_tier1.on_request_finished.assert_not_called()
@@ -1008,8 +1009,8 @@ class TestTieringOffloadingManager:
self, manager_setup
):
"""reset_cache drops active pending stores so resumed requests finalize."""
initial_blocks = to_keys(range(2))
resumed_blocks = to_keys(range(2, 4))
initial_chunks = to_keys(range(2))
resumed_chunks = to_keys(range(2, 4))
ctx = ReqContext(req_id="req_reset_resume")
self.secondary_tier1.on_request_finished = MagicMock(
@@ -1020,7 +1021,7 @@ class TestTieringOffloadingManager:
)
self._start_request(ctx)
self.manager.prepare_store(initial_blocks, ctx)
self.manager.prepare_store(initial_chunks, ctx)
assert self.manager._req_state[ctx.req_id].pending_primary_stores == 1
self.manager.reset_cache()
@@ -1030,8 +1031,8 @@ class TestTieringOffloadingManager:
self.secondary_tier1.on_request_finished.assert_not_called()
self.secondary_tier2.on_request_finished.assert_not_called()
self.manager.prepare_store(resumed_blocks, ctx)
self.manager.complete_store(resumed_blocks, ctx, success=True)
self.manager.prepare_store(resumed_chunks, ctx)
self.manager.complete_store(resumed_chunks, ctx, success=True)
self.manager.on_request_finished(ctx)
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
@@ -1039,12 +1040,12 @@ class TestTieringOffloadingManager:
assert ctx.req_id not in self.manager._req_state
def test_on_new_request_lifecycle(self, manager_setup):
"""Policy defaults to BLOCK_LEVEL, escalates when a tier requests it,
"""Policy defaults to CHUNK_LEVEL, escalates when a tier requests it,
and is cleaned up on on_request_finished."""
# Default: all tiers return BLOCK_LEVEL
# Default: all tiers return CHUNK_LEVEL
ctx = ReqContext(req_id="req_policy_lifecycle")
result = self.manager.on_new_request(ctx)
assert result.policy == OffloadPolicy.BLOCK_LEVEL
assert result.policy == OffloadPolicy.CHUNK_LEVEL
assert self.manager._req_state[ctx.req_id].request_level_tiers is None
self.manager.on_request_finished(ctx)
assert ctx.req_id not in self.manager._req_state
@@ -1064,26 +1065,26 @@ class TestTieringOffloadingManager:
assert ctx.req_id not in self.manager._req_state
@pytest.mark.parametrize("new_ids", [(), (3, 4)], ids=["fully_warm", "mixed"])
def test_prepare_store_cascades_existing_blocks_to_request_level_tiers(
def test_prepare_store_cascades_existing_chunks_to_request_level_tiers(
self, manager_setup, new_ids
):
"""prepare_store cascades hit blocks to request-level tiers only.
"""prepare_store cascades hit chunks to request-level tiers only.
The fully-warm case is the producer shape from issue #52808: nothing is
new, so primary returns an empty keys_to_store and no GPU->primary
transfer runs. A request-level tier must still be handed the blocks,
transfer runs. A request-level tier must still be handed the chunks,
otherwise a peer waiting on them has nothing to fetch.
"""
# Store some blocks to primary first
existing_blocks = to_keys(range(3))
# Store some chunks to primary first
existing_chunks = to_keys(range(3))
self._start_request()
result = self.manager.prepare_store(existing_blocks, _CTX)
result = self.manager.prepare_store(existing_chunks, _CTX)
assert result is not None
self.manager.complete_store(existing_blocks, _CTX, success=True)
self.manager.complete_store(existing_chunks, _CTX, success=True)
# Drain cascade completions
self._simulate_on_schedule_end()
# Make tier1 request-level, tier2 stays block-level
# Make tier1 request-level, tier2 stays chunk-level
self.secondary_tier1.on_new_request = lambda req_context: (
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
)
@@ -1099,20 +1100,20 @@ class TestTieringOffloadingManager:
wraps=self.secondary_tier2.submit_store
)
# Call prepare_store with existing + new blocks
new_blocks = to_keys(new_ids)
all_blocks = existing_blocks + new_blocks
result = self.manager.prepare_store(all_blocks, ctx)
# Call prepare_store with existing + new chunks
new_chunks = to_keys(new_ids)
all_chunks = existing_chunks + new_chunks
result = self.manager.prepare_store(all_chunks, ctx)
assert result is not None
assert set(result.keys_to_store) == set(new_blocks)
assert set(result.keys_to_store) == set(new_chunks)
# Only tier1 (request-level) should get existing blocks cascaded now.
# New blocks are cascaded to ALL tiers later via complete_store().
# Only tier1 (request-level) should get existing chunks cascaded now.
# New chunks are cascaded to ALL tiers later via complete_store().
self.secondary_tier1.submit_store.assert_called_once()
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
assert set(job_metadata.keys) == set(existing_blocks)
assert set(job_metadata.keys) == set(existing_chunks)
# tier2 (block-level) does not get existing blocks here.
# tier2 (chunk-level) does not get existing chunks here.
self.secondary_tier2.submit_store.assert_not_called()
def _make_request_level_request(self, req_id: str) -> ReqContext:
@@ -1150,7 +1151,7 @@ class TestTieringOffloadingManager:
self.manager.primary_tier.lookup = lambda key, req_context: (LookupResult.RETRY)
with pytest.raises(AssertionError):
self.manager._cascade_existing_blocks_to_request_level_tiers(keys, ctx, {0})
self.manager._cascade_existing_chunks_to_request_level_tiers(keys, ctx, {0})
def _start_in_flight_primary_write(self, keys: list[OffloadKey]) -> ReqContext:
"""Leave a GPU->primary write for `keys` open, so they look present to
@@ -1202,7 +1203,7 @@ class TestTieringOffloadingManager:
def test_cascade_drops_deferred_keys_whose_primary_write_failed(
self, manager_setup
):
"""A failed write frees the block, so the deferred key resolves to MISS
"""A failed write frees the chunk, so the deferred key resolves to MISS
and the request finalizes instead of parking forever."""
keys = to_keys(range(3))
writer_ctx = self._start_in_flight_primary_write(keys)
@@ -1224,21 +1225,21 @@ class TestTieringOffloadingManager:
"""reset_cache wipes every kind of orchestrator state and resets
primary tier; pending submissions are dropped without being sent
to the secondary tier. Active request state is retained."""
# Cascade — populates primary blocks and leaves cascade jobs
# Cascade — populates primary chunks and leaves cascade jobs
# in _jobs (the synchronous example tier has already
# queued completions); reset_cache's drain loop will pick them up.
blocks = to_keys(range(3))
chunks = to_keys(range(3))
self._start_request()
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
self.manager.prepare_store(chunks, _CTX)
self.manager.complete_store(chunks, _CTX, success=True)
assert self.manager._jobs
# Pending promotion submission (deferred — no on_schedule_end after
# the lookup that staged it).
promo_block = to_keys([99])[0]
self.secondary_tier1.blocks[promo_block] = True
promo_chunk = to_keys([99])[0]
self.secondary_tier1.chunks[promo_chunk] = True
assert (
self.manager.lookup(promo_block, ReqContext(req_id="pending"))
self.manager.lookup(promo_chunk, ReqContext(req_id="pending"))
is LookupResult.HIT_PENDING
)
assert self.manager._pending_load_submissions
@@ -1268,10 +1269,10 @@ class TestTieringOffloadingManager:
assert self.manager._processed_jobs_this_step is False
# Primary tier reset to a fresh state.
assert self.primary_tier._num_allocated_blocks == 0
assert self.primary_tier._num_allocated_chunks == 0
assert self.primary_tier._free_list == []
for block in blocks:
assert self.primary_tier.lookup(block, _CTX) is LookupResult.MISS
for chunk in chunks:
assert self.primary_tier.lookup(chunk, _CTX) is LookupResult.MISS
# Pending submission was dropped, not submitted.
self.secondary_tier1.submit_load.assert_not_called()
@@ -1291,10 +1292,10 @@ class TestTieringOffloadingManager:
)
# Drive a cascade so a job lands in _jobs.
blocks = to_keys(range(3))
chunks = to_keys(range(3))
self._start_request()
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
self.manager.prepare_store(chunks, _CTX)
self.manager.complete_store(chunks, _CTX, success=True)
assert self.manager._jobs
self.manager.reset_cache()
@@ -1315,20 +1316,20 @@ class TestTieringOffloadingManager:
self, manager_setup, load_tier_filter
):
"""Filter excluding secondary medium returns MISS from secondaries
even when they hold the block; primary is unaffected."""
blocks = to_keys(range(2))
# Put one block in primary, one only in secondary
even when they hold the chunk; primary is unaffected."""
chunks = to_keys(range(2))
# Put one chunk in primary, one only in secondary
self._start_request()
self.manager.prepare_store(blocks[:1], _CTX)
self.manager.complete_store(blocks[:1], _CTX, success=True)
self.secondary_tier1.blocks[blocks[1]] = True
self.manager.prepare_store(chunks[:1], _CTX)
self.manager.complete_store(chunks[:1], _CTX, success=True)
self.secondary_tier1.chunks[chunks[1]] = True
# Secondaries have medium=CPU, so load_tier_filter skips them.
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
ctx = ReqContext(req_id="r1", load_tier_filter=load_tier_filter)
assert self.manager.lookup(blocks[0], ctx) is LookupResult.HIT
assert self.manager.lookup(blocks[1], ctx) is LookupResult.MISS
assert self.manager.lookup(chunks[0], ctx) is LookupResult.HIT
assert self.manager.lookup(chunks[1], ctx) is LookupResult.MISS
self.secondary_tier1.lookup.assert_not_called()
@pytest.mark.parametrize(
@@ -1344,13 +1345,13 @@ class TestTieringOffloadingManager:
self, manager_setup, load_tier_filter
):
"""Filter that matches the secondary's medium allows lookup."""
blocks = to_keys(range(1))
self.secondary_tier1.blocks[blocks[0]] = True
chunks = to_keys(range(1))
self.secondary_tier1.chunks[chunks[0]] = True
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
ctx = ReqContext(req_id="r2", load_tier_filter=load_tier_filter)
assert self.manager.lookup(blocks[0], ctx) is LookupResult.HIT_PENDING
assert self.manager.lookup(chunks[0], ctx) is LookupResult.HIT_PENDING
self.secondary_tier1.lookup.assert_called()
@@ -1360,7 +1361,7 @@ class TestTieringOffloadingWithoutSecondaryTiers:
def test_works_without_secondary_tiers(self):
"""Test that manager works with empty secondary_tiers list."""
primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=5, mmap_region=_mock_mmap_region(5)
num_chunks=5, mmap_region=_mock_mmap_region(5)
)
# Create manager with no secondary tiers
@@ -1368,15 +1369,15 @@ class TestTieringOffloadingWithoutSecondaryTiers:
primary_tier=primary_tier, secondary_tiers=[]
)
blocks = to_keys(range(3))
chunks = to_keys(range(3))
# Should work like a regular OffloadingManager
manager.on_new_request(_CTX)
result = manager.prepare_store(blocks, _CTX)
result = manager.prepare_store(chunks, _CTX)
assert result is not None
manager.complete_store(blocks, _CTX, success=True)
manager.complete_store(chunks, _CTX, success=True)
assert count_hits(manager, blocks) == 3
assert count_hits(manager, chunks) == 3
@pytest.mark.parametrize(