[Refactor][kv_offload]: rename block→chunk (#52615)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
@@ -704,7 +704,7 @@ def test_prom_metrics_registers_tiering_metrics_from_spec():
|
||||
)
|
||||
|
||||
metric = prom_metrics._offloading_metric_defs[
|
||||
TieringOffloadingMetrics.BLOCK_QUERIES
|
||||
TieringOffloadingMetrics.CHUNK_QUERIES
|
||||
]
|
||||
assert metric.kwargs["labelnames"] == ["model_name", "engine", "tier"]
|
||||
|
||||
|
||||
@@ -96,7 +96,7 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
|
||||
groups.append(
|
||||
KVCacheGroupSpec([f"layer{i}"], kv_spec, is_eagle_group=eagle and i == 1)
|
||||
)
|
||||
manager = CPUOffloadingManager(num_blocks=100)
|
||||
manager = CPUOffloadingManager(num_chunks=100)
|
||||
spec = SimpleNamespace(
|
||||
tokens_per_block=(256, 64, 8),
|
||||
tokens_per_hash=8,
|
||||
@@ -2888,7 +2888,7 @@ class TestEagle:
|
||||
req=req,
|
||||
req_context=ReqContext(req_id="test-req"),
|
||||
offloading_context=RequestOffloadingContext(
|
||||
policy=OffloadPolicy.BLOCK_LEVEL
|
||||
policy=OffloadPolicy.CHUNK_LEVEL
|
||||
),
|
||||
num_locally_computed_tokens=num_computed_tokens,
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ import pytest
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
||||
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
|
||||
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
|
||||
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
|
||||
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
|
||||
|
||||
@@ -20,10 +20,10 @@ class _DummyCachePolicy(CachePolicy):
|
||||
def __init__(self, cache_capacity: int) -> None:
|
||||
self.cache_capacity = cache_capacity
|
||||
|
||||
def get(self, key: OffloadKey) -> BlockStatus | None:
|
||||
def get(self, key: OffloadKey) -> ChunkStatus | None:
|
||||
return None
|
||||
|
||||
def insert(self, key: OffloadKey, block: BlockStatus) -> None:
|
||||
def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
|
||||
pass
|
||||
|
||||
def remove(self, key: OffloadKey) -> None:
|
||||
@@ -34,7 +34,7 @@ class _DummyCachePolicy(CachePolicy):
|
||||
|
||||
def evict(
|
||||
self, n: int, protected: set[OffloadKey]
|
||||
) -> list[tuple[OffloadKey, BlockStatus]] | None:
|
||||
) -> list[tuple[OffloadKey, ChunkStatus]] | None:
|
||||
return None
|
||||
|
||||
def clear(self) -> None:
|
||||
@@ -72,7 +72,7 @@ class TestCachePolicyFactory:
|
||||
policy_cls = CachePolicyFactory.get_cache_policy_cls("dummy")
|
||||
assert policy_cls is _DummyCachePolicy
|
||||
|
||||
manager = CPUOffloadingManager(num_blocks=4, cache_policy="dummy")
|
||||
manager = CPUOffloadingManager(num_chunks=4, cache_policy="dummy")
|
||||
assert isinstance(manager._policy, _DummyCachePolicy)
|
||||
|
||||
def test_unregistered_policy_raises(self):
|
||||
@@ -98,7 +98,7 @@ class TestCachePolicyFactory:
|
||||
"""End-to-end: CPUOffloadingManager resolves an unregistered policy
|
||||
purely from cache_policy_module_path."""
|
||||
manager = CPUOffloadingManager(
|
||||
num_blocks=4,
|
||||
num_chunks=4,
|
||||
cache_policy="_DummyCachePolicy",
|
||||
cache_policy_module_path="tests.v1.kv_offload.cpu.policies.test_factory",
|
||||
)
|
||||
|
||||
@@ -232,9 +232,9 @@ def test_cross_topology_roundtrip(writer_tp: int, reader_tp: int):
|
||||
def canonical_view(rank: int, world_size: int) -> torch.Tensor:
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_blocks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=row_stride,
|
||||
kv_bytes_per_chunk=row_stride,
|
||||
cpu_page_size=row_stride // world_size,
|
||||
)
|
||||
regions.append(region)
|
||||
|
||||
@@ -26,7 +26,7 @@ from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
|
||||
NUM_GPU_BLOCKS = [64]
|
||||
NUM_CPU_BLOCKS = [256]
|
||||
NUM_CPU_CHUNKS = [256]
|
||||
GPU_PAGE_SIZES = [512, 1024]
|
||||
BLOCKS_PER_CHUNK_VALUES = [1, 3]
|
||||
NUM_TENSORS = [4]
|
||||
@@ -225,7 +225,7 @@ def test_worker_syncs_before_cleanup_after_handler_failure(
|
||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
|
||||
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_cpu_chunks", NUM_CPU_CHUNKS)
|
||||
@pytest.mark.parametrize("num_tensors", NUM_TENSORS)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
@@ -241,7 +241,7 @@ def test_transfer(
|
||||
gpu_page_size_bytes: int,
|
||||
blocks_per_chunk: int,
|
||||
num_gpu_blocks: int,
|
||||
num_cpu_blocks: int,
|
||||
num_cpu_chunks: int,
|
||||
num_tensors: int,
|
||||
seed: int,
|
||||
device: str,
|
||||
@@ -288,41 +288,41 @@ def test_transfer(
|
||||
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
|
||||
)
|
||||
simulated_world_size = 2
|
||||
kv_bytes_per_block = (
|
||||
kv_bytes_per_chunk = (
|
||||
cpu_page_size if replicated_layout else cpu_page_size * simulated_world_size
|
||||
)
|
||||
mmap_region = SharedOffloadRegion(
|
||||
engine_id=str(uuid.uuid4()),
|
||||
num_blocks=num_cpu_blocks,
|
||||
num_chunks=num_cpu_chunks,
|
||||
rank=0,
|
||||
kv_bytes_per_block=kv_bytes_per_block,
|
||||
kv_bytes_per_chunk=kv_bytes_per_chunk,
|
||||
cpu_page_size=cpu_page_size,
|
||||
)
|
||||
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
num_cpu_chunks=num_cpu_chunks,
|
||||
mmap_region=mmap_region,
|
||||
)
|
||||
|
||||
# select block mappings
|
||||
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * blocks_per_chunk)
|
||||
cpu_blocks = random.sample(range(num_cpu_blocks), num_mappings)
|
||||
cpu_chunks = random.sample(range(num_cpu_chunks), num_mappings)
|
||||
|
||||
# expand cpu blocks to gpu-page granularity for uniform comparison:
|
||||
# each cpu block maps to blocks_per_chunk consecutive sub-blocks
|
||||
cpu_blocks_expanded = [
|
||||
cpu_block * blocks_per_chunk + j
|
||||
for cpu_block in cpu_blocks
|
||||
# expand cpu chunks to gpu-page granularity for uniform comparison:
|
||||
# each cpu chunk maps to blocks_per_chunk consecutive sub-blocks
|
||||
cpu_chunks_expanded = [
|
||||
cpu_chunk * blocks_per_chunk + j
|
||||
for cpu_chunk in cpu_chunks
|
||||
for j in range(blocks_per_chunk)
|
||||
]
|
||||
|
||||
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU block
|
||||
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU chunk
|
||||
blocks_to_skip = blocks_per_chunk - 1
|
||||
if blocks_to_skip > 0:
|
||||
gpu_blocks = gpu_blocks[blocks_to_skip:]
|
||||
cpu_blocks_expanded = cpu_blocks_expanded[blocks_to_skip:]
|
||||
cpu_chunks_expanded = cpu_chunks_expanded[blocks_to_skip:]
|
||||
|
||||
# set transfer direction
|
||||
if gpu_to_cpu:
|
||||
@@ -330,16 +330,16 @@ def test_transfer(
|
||||
src_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
|
||||
)
|
||||
dst_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
dst_to_src = dict(zip(cpu_blocks_expanded, gpu_blocks))
|
||||
dst_spec = CPULoadStoreSpec(cpu_chunks)
|
||||
dst_to_src = dict(zip(cpu_chunks_expanded, gpu_blocks))
|
||||
num_dst_sub_blocks = num_gpu_blocks
|
||||
else:
|
||||
handler = worker._load_handler
|
||||
src_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
src_spec = CPULoadStoreSpec(cpu_chunks)
|
||||
dst_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
|
||||
)
|
||||
dst_to_src = dict(zip(gpu_blocks, cpu_blocks_expanded))
|
||||
dst_to_src = dict(zip(gpu_blocks, cpu_chunks_expanded))
|
||||
num_dst_sub_blocks = num_gpu_blocks
|
||||
|
||||
# randomize src and dst tensors before transfer
|
||||
@@ -410,7 +410,7 @@ def test_transfer(
|
||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
|
||||
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_cpu_chunks", NUM_CPU_CHUNKS)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
@torch.inference_mode()
|
||||
@@ -421,7 +421,7 @@ def test_transfer_multi_group(
|
||||
gpu_page_size_bytes: int,
|
||||
blocks_per_chunk: int,
|
||||
num_gpu_blocks: int,
|
||||
num_cpu_blocks: int,
|
||||
num_cpu_chunks: int,
|
||||
seed: int,
|
||||
device: str,
|
||||
) -> None:
|
||||
@@ -470,37 +470,37 @@ def test_transfer_multi_group(
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=canonical_kv_caches,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
num_cpu_chunks=num_cpu_chunks,
|
||||
)
|
||||
|
||||
# group 0: aligned, group 1: empty, group 2: unaligned on CPU->GPU
|
||||
group_sizes_in_cpu_blocks = [num_mappings_per_group, 0, num_mappings_per_group]
|
||||
group_sizes_in_cpu_chunks = [num_mappings_per_group, 0, num_mappings_per_group]
|
||||
|
||||
total_cpu_blocks = sum(group_sizes_in_cpu_blocks)
|
||||
total_gpu_blocks_needed = total_cpu_blocks * blocks_per_chunk
|
||||
total_cpu_chunks = sum(group_sizes_in_cpu_chunks)
|
||||
total_gpu_blocks_needed = total_cpu_chunks * blocks_per_chunk
|
||||
gpu_blocks_all = random.sample(range(num_gpu_blocks), total_gpu_blocks_needed)
|
||||
cpu_blocks_all = random.sample(range(num_cpu_blocks), total_cpu_blocks)
|
||||
cpu_chunks_all = random.sample(range(num_cpu_chunks), total_cpu_chunks)
|
||||
|
||||
# split gpu/cpu blocks per group
|
||||
# split gpu blocks / cpu chunks per group
|
||||
gpu_blocks_per_group: list[list[int]] = []
|
||||
cpu_blocks_per_group: list[list[int]] = []
|
||||
cpu_chunks_per_group: list[list[int]] = []
|
||||
gpu_offset = 0
|
||||
cpu_offset = 0
|
||||
for size in group_sizes_in_cpu_blocks:
|
||||
for size in group_sizes_in_cpu_chunks:
|
||||
gpu_count = size * blocks_per_chunk
|
||||
gpu_blocks_per_group.append(gpu_blocks_all[gpu_offset : gpu_offset + gpu_count])
|
||||
cpu_blocks_per_group.append(cpu_blocks_all[cpu_offset : cpu_offset + size])
|
||||
cpu_chunks_per_group.append(cpu_chunks_all[cpu_offset : cpu_offset + size])
|
||||
gpu_offset += gpu_count
|
||||
cpu_offset += size
|
||||
|
||||
# expand cpu blocks to gpu-page granularity
|
||||
cpu_blocks_expanded_per_group = [
|
||||
# expand cpu chunks to gpu-page granularity
|
||||
cpu_chunks_expanded_per_group = [
|
||||
[
|
||||
cpu_block * blocks_per_chunk + j
|
||||
for cpu_block in cpu_blocks
|
||||
cpu_chunk * blocks_per_chunk + j
|
||||
for cpu_chunk in cpu_chunks
|
||||
for j in range(blocks_per_chunk)
|
||||
]
|
||||
for cpu_blocks in cpu_blocks_per_group
|
||||
for cpu_chunks in cpu_chunks_per_group
|
||||
]
|
||||
|
||||
# skip sub-blocks from group 2 to test unaligned transfers.
|
||||
@@ -509,7 +509,7 @@ def test_transfer_multi_group(
|
||||
gpu_blocks_per_group[2] = gpu_blocks_per_group[2][
|
||||
sub_blocks_to_skip:-sub_blocks_to_skip
|
||||
]
|
||||
cpu_blocks_expanded_per_group[2] = cpu_blocks_expanded_per_group[2][
|
||||
cpu_chunks_expanded_per_group[2] = cpu_chunks_expanded_per_group[2][
|
||||
sub_blocks_to_skip:-sub_blocks_to_skip
|
||||
]
|
||||
|
||||
@@ -520,10 +520,10 @@ def test_transfer_multi_group(
|
||||
gpu_blocks.extend(gpu_blks)
|
||||
group_sizes.append(len(gpu_blks))
|
||||
|
||||
# build flat cpu_blocks list
|
||||
cpu_blocks = []
|
||||
for cpu_blks in cpu_blocks_per_group:
|
||||
cpu_blocks.extend(cpu_blks)
|
||||
# build flat cpu_chunks list
|
||||
cpu_chunks = []
|
||||
for cpu_chnks in cpu_chunks_per_group:
|
||||
cpu_chunks.extend(cpu_chnks)
|
||||
|
||||
# block_indices: only relevant for unaligned transfers
|
||||
block_indices: list[int] = [0, 0, sub_blocks_to_skip]
|
||||
@@ -533,18 +533,18 @@ def test_transfer_multi_group(
|
||||
src_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
|
||||
)
|
||||
dst_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
dst_spec = CPULoadStoreSpec(cpu_chunks)
|
||||
# per-group mapping: cpu sub-block -> gpu sub-block
|
||||
dst_to_src_per_group = [
|
||||
dict(zip(expanded, gpu_blks))
|
||||
for expanded, gpu_blks in zip(
|
||||
cpu_blocks_expanded_per_group, gpu_blocks_per_group
|
||||
cpu_chunks_expanded_per_group, gpu_blocks_per_group
|
||||
)
|
||||
]
|
||||
num_dst_sub_blocks = num_cpu_blocks * blocks_per_chunk
|
||||
num_dst_sub_blocks = num_cpu_chunks * blocks_per_chunk
|
||||
else:
|
||||
handler = worker._load_handler
|
||||
src_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
src_spec = CPULoadStoreSpec(cpu_chunks)
|
||||
dst_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
|
||||
)
|
||||
@@ -552,7 +552,7 @@ def test_transfer_multi_group(
|
||||
dst_to_src_per_group = [
|
||||
dict(zip(gpu_blks, expanded))
|
||||
for gpu_blks, expanded in zip(
|
||||
gpu_blocks_per_group, cpu_blocks_expanded_per_group
|
||||
gpu_blocks_per_group, cpu_chunks_expanded_per_group
|
||||
)
|
||||
]
|
||||
num_dst_sub_blocks = num_gpu_blocks
|
||||
@@ -644,7 +644,7 @@ def test_load_waits_for_pending_compute_stream_writes(default_vllm_config) -> No
|
||||
],
|
||||
),
|
||||
blocks_per_chunk=1,
|
||||
num_cpu_blocks=num_blocks,
|
||||
num_cpu_chunks=num_blocks,
|
||||
)
|
||||
worker._load_handler.src_tensors[0].fill_(sentinel)
|
||||
expected = torch.full((page_size_bytes,), sentinel, dtype=torch.int8)
|
||||
|
||||
@@ -36,7 +36,7 @@ _EMPTY_REQ_CTX = make_req_context()
|
||||
|
||||
|
||||
def make_cpu_manager(
|
||||
num_blocks: int = 4,
|
||||
num_chunks: int = 4,
|
||||
cache_policy: str = "lru",
|
||||
cache_policy_module_path: str | None = None,
|
||||
enable_events: bool = False,
|
||||
@@ -44,7 +44,7 @@ def make_cpu_manager(
|
||||
max_tracker_size: int = 64_000,
|
||||
) -> CPUOffloadingManager:
|
||||
return CPUOffloadingManager(
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
cache_policy=cache_policy,
|
||||
cache_policy_module_path=cache_policy_module_path,
|
||||
enable_events=enable_events,
|
||||
@@ -56,7 +56,7 @@ def make_cpu_manager(
|
||||
@dataclass
|
||||
class ExpectedPrepareStoreOutput:
|
||||
keys_to_store: list[int]
|
||||
store_block_ids: list[int]
|
||||
store_chunk_ids: list[int]
|
||||
evicted_keys: list[int]
|
||||
|
||||
|
||||
@@ -93,9 +93,9 @@ def verify_store_output(
|
||||
store_spec = prepare_store_output.store_spec
|
||||
assert isinstance(store_spec, CPULoadStoreSpec)
|
||||
expected_array = np.array(
|
||||
expected_prepare_store_output.store_block_ids, dtype=np.int64
|
||||
expected_prepare_store_output.store_chunk_ids, dtype=np.int64
|
||||
)
|
||||
assert np.array_equal(expected_array, store_spec.block_ids)
|
||||
assert np.array_equal(expected_array, store_spec.chunk_ids)
|
||||
|
||||
|
||||
def verify_load_output(
|
||||
@@ -103,7 +103,7 @@ def verify_load_output(
|
||||
):
|
||||
assert isinstance(prepare_load_output, CPULoadStoreSpec)
|
||||
expected_array = np.array(expected_prepare_load_output, dtype=np.int64)
|
||||
assert np.array_equal(expected_array, prepare_load_output.block_ids)
|
||||
assert np.array_equal(expected_array, prepare_load_output.chunk_ids)
|
||||
|
||||
|
||||
def check_split_usage_stats(
|
||||
@@ -146,7 +146,7 @@ def verify_events(
|
||||
|
||||
def test_cpu_eviction_removed_precedes_stored():
|
||||
"""An eviction is announced before the store that reuses its capacity."""
|
||||
manager = make_cpu_manager(num_blocks=2, enable_events=True)
|
||||
manager = make_cpu_manager(num_chunks=2, enable_events=True)
|
||||
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
@@ -164,23 +164,23 @@ def test_cpu_eviction_removed_precedes_stored():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("eviction_policy", ["lru", "arc"])
|
||||
def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
|
||||
def test_already_stored_chunk_not_evicted_during_prepare_store(eviction_policy):
|
||||
"""
|
||||
Regression test: a block that is already stored must not be evicted
|
||||
by prepare_store() when it needs to make room for new blocks.
|
||||
Regression test: a chunk that is already stored must not be evicted
|
||||
by prepare_store() when it needs to make room for new chunks.
|
||||
Applies to both lru and arc policies.
|
||||
|
||||
Scenario:
|
||||
- Store blocks [1, 2] and complete.
|
||||
- touch([1]) makes block 2 the LRU candidate.
|
||||
- Store chunks [1, 2] and complete.
|
||||
- touch([1]) makes chunk 2 the LRU candidate.
|
||||
- prepare_store([2, 3, 4, 5]):
|
||||
* block 2 is filtered out as "already stored"
|
||||
* but without the fix, block 2 would be evicted as the LRU
|
||||
* chunk 2 is filtered out as "already stored"
|
||||
* but without the fix, chunk 2 would be evicted as the LRU
|
||||
candidate to make room for [3, 4, 5]
|
||||
- After complete_store([2, 3, 4, 5]), block 2 must still be present.
|
||||
- After complete_store([2, 3, 4, 5]), chunk 2 must still be present.
|
||||
"""
|
||||
manager = make_cpu_manager(
|
||||
num_blocks=4,
|
||||
num_chunks=4,
|
||||
cache_policy=eviction_policy,
|
||||
enable_events=True,
|
||||
)
|
||||
@@ -189,33 +189,33 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
|
||||
# touch [1] to make block 2 the LRU candidate
|
||||
# touch [1] to make chunk 2 the LRU candidate
|
||||
manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
|
||||
# prepare_store([2, 3, 4, 5]):
|
||||
# - block 2 is already stored -> filtered out of keys_to_store
|
||||
# - block 2 must NOT be evicted even though it is the LRU candidate
|
||||
# - block 1 (ID 0) is evicted instead; new blocks [3,4,5] get IDs 2,3,0
|
||||
# - chunk 2 is already stored -> filtered out of keys_to_store
|
||||
# - chunk 2 must NOT be evicted even though it is the LRU candidate
|
||||
# - chunk 1 (ID 0) is evicted instead; new chunks [3,4,5] get IDs 2,3,0
|
||||
prepare_store_output = manager.prepare_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
|
||||
verify_store_output(
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[3, 4, 5],
|
||||
store_block_ids=[2, 3, 0],
|
||||
evicted_keys=[1], # block 1 evicted, not block 2
|
||||
store_chunk_ids=[2, 3, 0],
|
||||
evicted_keys=[1], # chunk 1 evicted, not chunk 2
|
||||
),
|
||||
)
|
||||
|
||||
# complete_store must not silently drop block 2
|
||||
# complete_store must not silently drop chunk 2
|
||||
manager.complete_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
|
||||
|
||||
# block 2 must still be present in the cache
|
||||
# chunk 2 must still be present in the cache
|
||||
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
|
||||
|
||||
|
||||
def test_filter_reused_manager_reports_stores_skipped_counter():
|
||||
manager = make_cpu_manager(
|
||||
num_blocks=4,
|
||||
num_chunks=4,
|
||||
cache_policy="lru",
|
||||
store_threshold=2,
|
||||
)
|
||||
@@ -226,7 +226,7 @@ def test_filter_reused_manager_reports_stores_skipped_counter():
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[],
|
||||
store_block_ids=[],
|
||||
store_chunk_ids=[],
|
||||
evicted_keys=[],
|
||||
),
|
||||
)
|
||||
@@ -247,34 +247,34 @@ def test_cpu_manager_reports_cache_usage_gauge():
|
||||
] == pytest.approx(value)
|
||||
|
||||
# Zero-capacity manager always reports 0.0
|
||||
manager = make_cpu_manager(num_blocks=0)
|
||||
manager = make_cpu_manager(num_chunks=0)
|
||||
check_usage_stats(manager, 0.0)
|
||||
|
||||
# Empty manager (4 blocks, none allocated): usage = 0.0
|
||||
manager = make_cpu_manager(num_blocks=4)
|
||||
# Empty manager (4 chunks, none allocated): usage = 0.0
|
||||
manager = make_cpu_manager(num_chunks=4)
|
||||
check_usage_stats(manager, 0.0)
|
||||
|
||||
# After allocating 2 of 4 blocks: usage = 0.5
|
||||
# After allocating 2 of 4 chunks: usage = 0.5
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
check_usage_stats(manager, 0.5)
|
||||
|
||||
# After filling all 4 blocks: usage = 1.0
|
||||
# After filling all 4 chunks: usage = 1.0
|
||||
manager.prepare_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
|
||||
check_usage_stats(manager, 1.0)
|
||||
|
||||
# After completing store, the blocks becomes evictable as it is not actively used
|
||||
# After completing store, the chunks become evictable as not actively used
|
||||
# and usage drops.
|
||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
check_usage_stats(manager, 0.5)
|
||||
|
||||
# After completing store, the blocks becomes evictable as it is not actively used
|
||||
# After completing store, the chunks become evictable as not actively used
|
||||
# and usage drops.
|
||||
manager.complete_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
|
||||
check_usage_stats(manager, 0.0)
|
||||
|
||||
|
||||
def test_cpu_manager_reports_allocation_size_histogram():
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
@@ -297,12 +297,12 @@ def test_cpu_manager_reports_allocation_size_histogram():
|
||||
|
||||
|
||||
def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
|
||||
def fail_allocate_blocks(keys):
|
||||
def fail_allocate_chunks(keys):
|
||||
raise RuntimeError("allocation failed")
|
||||
|
||||
monkeypatch.setattr(manager, "_allocate_blocks", fail_allocate_blocks)
|
||||
monkeypatch.setattr(manager, "_allocate_chunks", fail_allocate_chunks)
|
||||
|
||||
with pytest.raises(RuntimeError, match="allocation failed"):
|
||||
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
||||
@@ -316,7 +316,7 @@ def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
|
||||
|
||||
|
||||
def test_cpu_manager_reports_allocation_size_on_eviction_failure():
|
||||
manager = make_cpu_manager(num_blocks=1, cache_policy="lru")
|
||||
manager = make_cpu_manager(num_chunks=1, cache_policy="lru")
|
||||
|
||||
manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
manager.get_stats()
|
||||
@@ -332,7 +332,7 @@ def test_cpu_manager_reports_allocation_size_on_eviction_failure():
|
||||
|
||||
|
||||
def test_cpu_manager_reports_cache_write_and_read_usage_gauges():
|
||||
manager = make_cpu_manager(num_blocks=4)
|
||||
manager = make_cpu_manager(num_chunks=4)
|
||||
|
||||
# Store path: pins write usage until complete_store.
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
@@ -356,7 +356,7 @@ def test_cpu_manager_reports_cache_write_and_read_usage_gauges():
|
||||
|
||||
|
||||
def test_cpu_manager_clears_write_usage_after_failed_store():
|
||||
manager = make_cpu_manager(num_blocks=4)
|
||||
manager = make_cpu_manager(num_chunks=4)
|
||||
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
check_split_usage_stats(manager, write=0.5, read=0.0, total=0.5)
|
||||
@@ -369,8 +369,8 @@ def test_cpu_manager():
|
||||
"""
|
||||
Tests CPUOffloadingManager with lru policy.
|
||||
"""
|
||||
# initialize a CPU manager with a capacity of 4 blocks
|
||||
cpu_manager = make_cpu_manager(num_blocks=4, cache_policy="lru", enable_events=True)
|
||||
# initialize a CPU manager with a capacity of 4 chunks
|
||||
cpu_manager = make_cpu_manager(num_chunks=4, cache_policy="lru", enable_events=True)
|
||||
|
||||
# prepare store [1, 2]
|
||||
prepare_store_output = cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
@@ -378,7 +378,7 @@ def test_cpu_manager():
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[1, 2],
|
||||
store_block_ids=[0, 1],
|
||||
store_chunk_ids=[0, 1],
|
||||
evicted_keys=[],
|
||||
),
|
||||
)
|
||||
@@ -407,7 +407,7 @@ def test_cpu_manager():
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[3, 4, 5],
|
||||
store_block_ids=[2, 3, 0],
|
||||
store_chunk_ids=[2, 3, 0],
|
||||
evicted_keys=[1],
|
||||
),
|
||||
)
|
||||
@@ -445,7 +445,7 @@ def test_cpu_manager():
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[6, 7, 8],
|
||||
store_block_ids=[1, 0, 3],
|
||||
store_chunk_ids=[1, 0, 3],
|
||||
evicted_keys=[4, 5, 2],
|
||||
),
|
||||
)
|
||||
@@ -462,7 +462,7 @@ def test_cpu_manager():
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[9],
|
||||
store_block_ids=[3],
|
||||
store_chunk_ids=[3],
|
||||
evicted_keys=[8],
|
||||
),
|
||||
)
|
||||
@@ -482,38 +482,38 @@ def test_cpu_manager():
|
||||
|
||||
|
||||
def test_prepare_load_preserves_key_order():
|
||||
"""block_ids[i] must correspond to keys[i] (co-indexed invariant)."""
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
"""chunk_ids[i] must correspond to keys[i] (co-indexed invariant)."""
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
|
||||
key_a, key_b, key_c = to_key(0), to_key(1), to_key(2)
|
||||
|
||||
# Store all three keys and learn their block ID assignments
|
||||
# Store all three keys and learn their chunk ID assignments
|
||||
store_output = manager.prepare_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
||||
assert store_output is not None
|
||||
assert isinstance(store_output.store_spec, CPULoadStoreSpec)
|
||||
key_to_block_id = {
|
||||
k: int(bid)
|
||||
for k, bid in zip(store_output.keys_to_store, store_output.store_spec.block_ids)
|
||||
key_to_chunk_id = {
|
||||
k: int(cid)
|
||||
for k, cid in zip(store_output.keys_to_store, store_output.store_spec.chunk_ids)
|
||||
}
|
||||
manager.complete_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
||||
|
||||
# Forward order: [a, b, c]
|
||||
spec_fwd = manager.prepare_load([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
||||
assert isinstance(spec_fwd, CPULoadStoreSpec)
|
||||
assert [int(x) for x in spec_fwd.block_ids] == [
|
||||
key_to_block_id[key_a],
|
||||
key_to_block_id[key_b],
|
||||
key_to_block_id[key_c],
|
||||
assert [int(x) for x in spec_fwd.chunk_ids] == [
|
||||
key_to_chunk_id[key_a],
|
||||
key_to_chunk_id[key_b],
|
||||
key_to_chunk_id[key_c],
|
||||
]
|
||||
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
|
||||
|
||||
# Arbitrary permutation: [b, c, a]
|
||||
spec_perm = manager.prepare_load([key_b, key_c, key_a], _EMPTY_REQ_CTX)
|
||||
assert isinstance(spec_perm, CPULoadStoreSpec)
|
||||
assert [int(x) for x in spec_perm.block_ids] == [
|
||||
key_to_block_id[key_b],
|
||||
key_to_block_id[key_c],
|
||||
key_to_block_id[key_a],
|
||||
assert [int(x) for x in spec_perm.chunk_ids] == [
|
||||
key_to_chunk_id[key_b],
|
||||
key_to_chunk_id[key_c],
|
||||
key_to_chunk_id[key_a],
|
||||
]
|
||||
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
|
||||
|
||||
@@ -522,10 +522,10 @@ class TestARCPolicy:
|
||||
"""Unit tests for CPUOffloadingManager with ARC eviction policy."""
|
||||
|
||||
def _make_manager(
|
||||
self, num_blocks: int = 4, enable_events: bool = True
|
||||
self, num_chunks: int = 4, enable_events: bool = True
|
||||
) -> tuple[CPUOffloadingManager, ARCCachePolicy]:
|
||||
manager = make_cpu_manager(
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
cache_policy="arc",
|
||||
enable_events=enable_events,
|
||||
)
|
||||
@@ -548,7 +548,7 @@ class TestARCPolicy:
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[1, 2],
|
||||
store_block_ids=[0, 1],
|
||||
store_chunk_ids=[0, 1],
|
||||
evicted_keys=[],
|
||||
),
|
||||
)
|
||||
@@ -569,36 +569,36 @@ class TestARCPolicy:
|
||||
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
|
||||
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.MISS
|
||||
|
||||
# blocks should be in T1 (recent)
|
||||
# chunks should be in T1 (recent)
|
||||
assert len(arc_policy.t1) == 2
|
||||
assert len(arc_policy.t2) == 0
|
||||
|
||||
def test_t1_to_t2_promotion(self):
|
||||
"""
|
||||
Tests that accessing a block in T1 promotes it to T2 (frequent).
|
||||
Tests that accessing a chunk in T1 promotes it to T2 (frequent).
|
||||
This is a key feature of ARC's adaptive behavior.
|
||||
"""
|
||||
cpu_manager, arc_policy = self._make_manager(enable_events=False)
|
||||
|
||||
# store and complete block 1
|
||||
# store and complete chunk 1
|
||||
cpu_manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
|
||||
# block 1 starts in T1 (recent)
|
||||
# chunk 1 starts in T1 (recent)
|
||||
assert to_keys([1])[0] in arc_policy.t1
|
||||
assert to_keys([1])[0] not in arc_policy.t2
|
||||
|
||||
# touch block 1 (simulate second access)
|
||||
# touch chunk 1 (simulate second access)
|
||||
cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
|
||||
# block 1 should now be in T2 (frequent)
|
||||
# chunk 1 should now be in T2 (frequent)
|
||||
assert to_keys([1])[0] not in arc_policy.t1
|
||||
assert to_keys([1])[0] in arc_policy.t2
|
||||
|
||||
def test_eviction_with_load(self):
|
||||
"""
|
||||
Tests ARC eviction behavior similar to LRU test.
|
||||
Verifies that blocks being loaded (ref_cnt > 0) cannot be evicted.
|
||||
Verifies that chunks being loaded (ref_cnt > 0) cannot be evicted.
|
||||
"""
|
||||
cpu_manager, _ = self._make_manager()
|
||||
|
||||
@@ -610,7 +610,7 @@ class TestARCPolicy:
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[1, 2, 3, 4],
|
||||
store_block_ids=[0, 1, 2, 3],
|
||||
store_chunk_ids=[0, 1, 2, 3],
|
||||
evicted_keys=[],
|
||||
),
|
||||
)
|
||||
@@ -628,36 +628,36 @@ class TestARCPolicy:
|
||||
cpu_manager.complete_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
|
||||
|
||||
# now prepare store [5, 6, 7] should succeed
|
||||
# ARC will evict blocks one at a time from T1 as needed
|
||||
# ARC will evict chunks one at a time from T1 as needed
|
||||
prepare_store_output = cpu_manager.prepare_store(
|
||||
to_keys([5, 6, 7]), _EMPTY_REQ_CTX
|
||||
)
|
||||
assert prepare_store_output is not None
|
||||
# Should successfully evict enough blocks to make room (at least 1)
|
||||
# Should successfully evict enough chunks to make room (at least 1)
|
||||
assert len(prepare_store_output.evicted_keys) >= 1
|
||||
|
||||
def test_adaptive_target(self):
|
||||
"""
|
||||
Tests ARC's adaptive target adjustment via ghost lists.
|
||||
When a block in B1 (ghost list) is accessed, target_t1_size increases.
|
||||
When a block in B2 is accessed, target_t1_size decreases.
|
||||
When a chunk in B1 (ghost list) is accessed, target_t1_size increases.
|
||||
When a chunk in B2 is accessed, target_t1_size decreases.
|
||||
"""
|
||||
cpu_manager, arc_policy = self._make_manager(num_blocks=2, enable_events=False)
|
||||
cpu_manager, arc_policy = self._make_manager(num_chunks=2, enable_events=False)
|
||||
|
||||
# store blocks 1, 2 (fills cache)
|
||||
# store chunks 1, 2 (fills cache)
|
||||
cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
|
||||
initial_target = arc_policy.target_t1_size
|
||||
|
||||
# store block 3, evicting block 1 (moves to B1 ghost list)
|
||||
# store chunk 3, evicting chunk 1 (moves to B1 ghost list)
|
||||
cpu_manager.prepare_store(to_keys([3]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([3]), _EMPTY_REQ_CTX)
|
||||
|
||||
# block 1 should be in B1 (ghost list)
|
||||
# chunk 1 should be in B1 (ghost list)
|
||||
assert to_keys([1])[0] in arc_policy.b1
|
||||
|
||||
# touch block 1 (cache miss, but in B1)
|
||||
# touch chunk 1 (cache miss, but in B1)
|
||||
# this should increase target_t1_size (favor recency)
|
||||
cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
|
||||
@@ -671,11 +671,11 @@ class TestARCPolicy:
|
||||
"""
|
||||
cpu_manager, arc_policy = self._make_manager(enable_events=False)
|
||||
|
||||
# store blocks 1, 2, 3, 4
|
||||
# store chunks 1, 2, 3, 4
|
||||
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||
|
||||
# promote blocks 3, 4 to T2 by touching them
|
||||
# promote chunks 3, 4 to T2 by touching them
|
||||
cpu_manager.touch(to_keys([3, 4]), _EMPTY_REQ_CTX)
|
||||
|
||||
# now: T1 = {1, 2}, T2 = {3, 4}
|
||||
@@ -686,22 +686,22 @@ class TestARCPolicy:
|
||||
# (when |T1| >= target, evict from T1)
|
||||
arc_policy.target_t1_size = 1
|
||||
|
||||
# store block 5, should evict from T1 (block 1, LRU in T1)
|
||||
# store chunk 5, should evict from T1 (chunk 1, LRU in T1)
|
||||
output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
|
||||
assert output is not None
|
||||
assert to_keys([1]) == output.evicted_keys
|
||||
|
||||
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX)
|
||||
|
||||
# block 1 should be in B1 (ghost list)
|
||||
# chunk 1 should be in B1 (ghost list)
|
||||
assert to_keys([1])[0] in arc_policy.b1
|
||||
# block 5 should be in T1
|
||||
# chunk 5 should be in T1
|
||||
assert to_keys([5])[0] in arc_policy.t1
|
||||
|
||||
def test_batch_eviction_scans_t1_and_t2_once(self):
|
||||
"""ARC batch eviction must preserve order without restarting scans."""
|
||||
cpu_manager, arc_policy = self._make_manager(
|
||||
num_blocks=256, enable_events=False
|
||||
num_chunks=256, enable_events=False
|
||||
)
|
||||
keys = to_keys(list(range(256)))
|
||||
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
||||
@@ -721,13 +721,13 @@ class TestARCPolicy:
|
||||
t2_order = list(arc_policy.t2)
|
||||
expected_t1 = [
|
||||
key
|
||||
for key, block in arc_policy.t1.items()
|
||||
if block.ref_cnt == 0 and key not in protected
|
||||
for key, chunk in arc_policy.t1.items()
|
||||
if chunk.ref_cnt == 0 and key not in protected
|
||||
][:num_t1_evictions]
|
||||
expected_t2 = [
|
||||
key
|
||||
for key, block in arc_policy.t2.items()
|
||||
if block.ref_cnt == 0 and key not in protected
|
||||
for key, chunk in arc_policy.t2.items()
|
||||
if chunk.ref_cnt == 0 and key not in protected
|
||||
][:num_t2_evictions]
|
||||
expected_t1_scans = t1_order.index(expected_t1[-1]) + 1
|
||||
expected_t2_scans = t2_order.index(expected_t2[-1]) + 1
|
||||
@@ -746,7 +746,7 @@ class TestARCPolicy:
|
||||
|
||||
def test_batch_eviction_falls_back_after_t1_iterator_exhausted(self):
|
||||
"""An exhausted T1 scan must keep falling back to T2."""
|
||||
cpu_manager, arc_policy = self._make_manager(num_blocks=8, enable_events=False)
|
||||
cpu_manager, arc_policy = self._make_manager(num_chunks=8, enable_events=False)
|
||||
keys = to_keys(list(range(8)))
|
||||
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
|
||||
@@ -764,8 +764,8 @@ class TestARCPolicy:
|
||||
# the target, so each remaining selection must retry T1 then use T2.
|
||||
eligible_t1 = [
|
||||
key
|
||||
for key, block in arc_policy.t1.items()
|
||||
if block.ref_cnt == 0 and key not in protected
|
||||
for key, chunk in arc_policy.t1.items()
|
||||
if chunk.ref_cnt == 0 and key not in protected
|
||||
]
|
||||
assert eligible_t1 == t1_order[:1]
|
||||
assert len(t1_order) - 1 >= int(arc_policy.target_t1_size)
|
||||
@@ -839,7 +839,7 @@ class TestARCPolicy:
|
||||
|
||||
def test_batch_eviction_failure_is_atomic(self):
|
||||
"""Finding only some candidates must not partially evict the cache."""
|
||||
cpu_manager, arc_policy = self._make_manager(num_blocks=4, enable_events=False)
|
||||
cpu_manager, arc_policy = self._make_manager(num_chunks=4, enable_events=False)
|
||||
keys = to_keys(list(range(4)))
|
||||
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
|
||||
@@ -858,13 +858,13 @@ class TestARCPolicy:
|
||||
Tests that ghost lists (B1, B2) don't grow unbounded.
|
||||
They should be capped at cache_capacity.
|
||||
"""
|
||||
cpu_manager, arc_policy = self._make_manager(num_blocks=2, enable_events=False)
|
||||
cpu_manager, arc_policy = self._make_manager(num_chunks=2, enable_events=False)
|
||||
|
||||
# fill cache with blocks 1, 2
|
||||
# fill cache with chunks 1, 2
|
||||
cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
|
||||
# store many blocks to fill ghost lists
|
||||
# store many chunks to fill ghost lists
|
||||
for i in range(3, 20):
|
||||
cpu_manager.prepare_store(to_keys([i]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([i]), _EMPTY_REQ_CTX)
|
||||
@@ -880,7 +880,7 @@ class TestARCPolicy:
|
||||
"""
|
||||
cpu_manager, arc_policy = self._make_manager()
|
||||
|
||||
# store blocks 1, 2, 3, 4
|
||||
# store chunks 1, 2, 3, 4
|
||||
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||
|
||||
@@ -895,13 +895,13 @@ class TestARCPolicy:
|
||||
assert len(arc_policy.t1) == 1
|
||||
assert len(arc_policy.t2) == 3
|
||||
|
||||
# store block 5, should evict from T1 (block 2, only one in T1)
|
||||
# store chunk 5, should evict from T1 (chunk 2, only one in T1)
|
||||
prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
|
||||
verify_store_output(
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[5],
|
||||
store_block_ids=[1], # reuses block 2's storage
|
||||
store_chunk_ids=[1], # reuses chunk 2's storage
|
||||
evicted_keys=[2],
|
||||
),
|
||||
)
|
||||
@@ -913,11 +913,11 @@ class TestARCPolicy:
|
||||
"""
|
||||
cpu_manager, arc_policy = self._make_manager()
|
||||
|
||||
# store blocks 1, 2, 3, 4
|
||||
# store chunks 1, 2, 3, 4
|
||||
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||
|
||||
# prepare store block 5 (will evict block 1)
|
||||
# prepare store chunk 5 (will evict chunk 1)
|
||||
prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
|
||||
assert prepare_store_output is not None
|
||||
assert len(prepare_store_output.evicted_keys) == 1
|
||||
@@ -925,13 +925,13 @@ class TestARCPolicy:
|
||||
# complete store with failure
|
||||
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX, success=False)
|
||||
|
||||
# block 5 should not be in cache
|
||||
# chunk 5 should not be in cache
|
||||
assert cpu_manager.lookup(to_key(5), _EMPTY_REQ_CTX) is LookupResult.MISS
|
||||
# block 5 should not be in T1 or T2
|
||||
# chunk 5 should not be in T1 or T2
|
||||
assert to_keys([5])[0] not in arc_policy.t1
|
||||
assert to_keys([5])[0] not in arc_policy.t2
|
||||
|
||||
# evicted block should still be gone (in B1 ghost list)
|
||||
# evicted chunk should still be gone (in B1 ghost list)
|
||||
evicted_hash = prepare_store_output.evicted_keys[0]
|
||||
assert evicted_hash in arc_policy.b1
|
||||
|
||||
@@ -954,7 +954,7 @@ class TestARCPolicy:
|
||||
assert len(prepare_store_output.evicted_keys) == 1
|
||||
cpu_manager.complete_store(to_keys([3, 4, 5]), _EMPTY_REQ_CTX)
|
||||
|
||||
# promote some blocks to T2
|
||||
# promote some chunks to T2
|
||||
cpu_manager.touch(to_keys([2, 3]), _EMPTY_REQ_CTX)
|
||||
|
||||
# T1 has {4, 5}, T2 has {2, 3}
|
||||
@@ -966,7 +966,7 @@ class TestARCPolicy:
|
||||
assert prepare_store_output is not None
|
||||
cpu_manager.complete_store(to_keys([6]), _EMPTY_REQ_CTX)
|
||||
|
||||
# verify blocks 2, 3 (in T2) are still present
|
||||
# verify chunks 2, 3 (in T2) are still present
|
||||
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
|
||||
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.HIT
|
||||
|
||||
@@ -980,7 +980,7 @@ def test_filter_reused_manager():
|
||||
Tests CPUOffloadingManager reuse filtering (store_threshold=2).
|
||||
"""
|
||||
manager = make_cpu_manager(
|
||||
num_blocks=4,
|
||||
num_chunks=4,
|
||||
cache_policy="lru",
|
||||
enable_events=True,
|
||||
store_threshold=2,
|
||||
@@ -1023,7 +1023,7 @@ def test_filter_reused_manager():
|
||||
def test_filter_reused_manager_oversized_offer_makes_progress():
|
||||
"""An offer larger than the tracker capacity must not churn forever."""
|
||||
manager = make_cpu_manager(
|
||||
num_blocks=4,
|
||||
num_chunks=4,
|
||||
cache_policy="lru",
|
||||
store_threshold=2,
|
||||
max_tracker_size=3,
|
||||
@@ -1042,69 +1042,69 @@ def test_filter_reused_manager_oversized_offer_makes_progress():
|
||||
assert stored_keys == set(keys)
|
||||
|
||||
|
||||
def test_evictable_cache_block_count():
|
||||
def test_evictable_cache_chunk_count():
|
||||
"""
|
||||
Verifies _num_evictable_cache_blocks is maintained correctly through the
|
||||
Verifies _num_evictable_cache_chunks is maintained correctly through the
|
||||
full store/load lifecycle, eviction, failed stores, concurrent loads,
|
||||
reset_cache, and the early-exit fast path in prepare_store.
|
||||
"""
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
|
||||
# Initially no blocks allocated.
|
||||
assert manager._num_evictable_cache_blocks == 0
|
||||
# Initially no chunks allocated.
|
||||
assert manager._num_evictable_cache_chunks == 0
|
||||
|
||||
# Initial cache state [x, x, x, x]
|
||||
|
||||
# We get 3 blocks from the cache.
|
||||
# We get 3 chunks from the cache.
|
||||
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
||||
# cache state [1', 2', 3', x] <- 1', 2', 3' are actively being used.
|
||||
assert manager._num_evictable_cache_blocks == 0
|
||||
assert manager._num_evictable_cache_chunks == 0
|
||||
|
||||
# Completing stores makes them idle.
|
||||
manager.complete_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
||||
# cache state [1, 2, 3, x] <- 1, 2, 3 blocks are idle.
|
||||
assert manager._num_evictable_cache_blocks == 3
|
||||
# cache state [1, 2, 3, x] <- 1, 2, 3 chunks are idle.
|
||||
assert manager._num_evictable_cache_chunks == 3
|
||||
|
||||
# prepare_load pins a block: idle count decrements once even if the
|
||||
# same block is loaded by two concurrent callers.
|
||||
# prepare_load pins a chunk: idle count decrements once even if the
|
||||
# same chunk is loaded by two concurrent callers.
|
||||
manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
|
||||
assert manager._num_evictable_cache_blocks == 2
|
||||
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
|
||||
assert manager._num_evictable_cache_chunks == 2
|
||||
manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX) # 2nd concurrent load
|
||||
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
|
||||
assert manager._num_evictable_cache_blocks == 2 # no double-decrement
|
||||
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
|
||||
assert manager._num_evictable_cache_chunks == 2 # no double-decrement
|
||||
|
||||
# First complete_load does not restore idle (ref_cnt still 1).
|
||||
manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
|
||||
assert manager._num_evictable_cache_blocks == 2
|
||||
# Second complete_load drops ref_cnt to 0 -> block becomes idle again.
|
||||
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
|
||||
assert manager._num_evictable_cache_chunks == 2
|
||||
# Second complete_load drops ref_cnt to 0 -> chunk becomes idle again.
|
||||
manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
# cache state [1, 2, 3, x] <- 1, 2, 3 blocks are idle.
|
||||
assert manager._num_evictable_cache_blocks == 3
|
||||
# cache state [1, 2, 3, x] <- 1, 2, 3 chunks are idle.
|
||||
assert manager._num_evictable_cache_chunks == 3
|
||||
|
||||
# Eviction decrements idle count.
|
||||
# Cache has 3 stored blocks and 1 free slot. Storing 3 new keys needs 2 eviction.
|
||||
# Cache has 3 stored chunks and 1 free slot. Storing 3 new keys needs 2 eviction.
|
||||
manager.prepare_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX)
|
||||
# cache state [1, 4', 5', 6'] <- block 1 is idle
|
||||
assert manager._num_evictable_cache_blocks == 1
|
||||
# cache state [1, 4', 5', 6'] <- chunk 1 is idle
|
||||
assert manager._num_evictable_cache_chunks == 1
|
||||
|
||||
# Failed store does not increment idle count (block discarded from cache).
|
||||
# Failed store does not increment idle count (chunk discarded from cache).
|
||||
manager.complete_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX, success=False)
|
||||
# cache state [1, x, x, x] <- block 1 is idle. Other returned to cache.
|
||||
assert manager._num_evictable_cache_blocks == 1
|
||||
# cache state [1, x, x, x] <- chunk 1 is idle. Other returned to cache.
|
||||
assert manager._num_evictable_cache_chunks == 1
|
||||
|
||||
# reset_cache zeroes the count unconditionally.
|
||||
manager.reset_cache()
|
||||
# cache state [x, x, x, x]
|
||||
assert manager._num_evictable_cache_blocks == 0
|
||||
assert manager._num_evictable_cache_chunks == 0
|
||||
|
||||
# setup 3 blocks with loads so idle count drops to 0.
|
||||
# setup 3 chunks with loads so idle count drops to 0.
|
||||
manager.prepare_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
||||
manager.complete_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
||||
manager.prepare_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
||||
# cache state [10', 11', 12', x]
|
||||
assert manager._num_evictable_cache_blocks == 0
|
||||
assert manager._num_evictable_cache_chunks == 0
|
||||
|
||||
# prepare_store requiring eviction must return None immediately (fast exit).
|
||||
# Spy on policy.evict to confirm the fast path short-circuits before calling it.
|
||||
@@ -1120,24 +1120,24 @@ def test_evictable_cache_block_count():
|
||||
# cache state [10', 11', 12', x] <- cannot evict anything
|
||||
assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is None
|
||||
assert not evict_called, (
|
||||
"_num_evictable_cache_blocks==0 should short-circuit before evict()"
|
||||
"_num_evictable_cache_chunks==0 should short-circuit before evict()"
|
||||
)
|
||||
|
||||
# After releasing the loads, eviction becomes possible again.
|
||||
manager.complete_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
||||
# cache state [10, 11, 12, x] <- 10, 11, 12 are idle
|
||||
assert manager._num_evictable_cache_blocks == 3
|
||||
assert manager._num_evictable_cache_chunks == 3
|
||||
assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is not None
|
||||
# cache state [10, 11, 14', 15'] <- 10, 11 are idle
|
||||
assert manager._num_evictable_cache_blocks == 2
|
||||
assert manager._num_evictable_cache_chunks == 2
|
||||
manager.complete_store(to_keys([14, 15]), _EMPTY_REQ_CTX)
|
||||
# cache state [10, 11, 14, 15] <- all blocks idle
|
||||
assert manager._num_evictable_cache_blocks == 4
|
||||
# cache state [10, 11, 14, 15] <- all chunks idle
|
||||
assert manager._num_evictable_cache_chunks == 4
|
||||
|
||||
|
||||
def test_touch_forwards_req_context_to_policy(monkeypatch):
|
||||
"""Regression: CPUOffloadingManager.touch forwards ReqContext to policy."""
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
received = []
|
||||
|
||||
def spy_touch(keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
||||
|
||||
@@ -38,7 +38,7 @@ def _set_spawn_method(monkeypatch):
|
||||
|
||||
def _make_region(
|
||||
engine_id: str,
|
||||
num_blocks: int = 4,
|
||||
num_chunks: int = 4,
|
||||
cpu_page_size: int = PAGE_SIZE,
|
||||
num_workers: int = 1,
|
||||
rank: int = 0,
|
||||
@@ -47,9 +47,9 @@ def _make_region(
|
||||
assert cpu_page_size % PAGE_SIZE == 0
|
||||
return SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
||||
kv_bytes_per_chunk=num_workers * cpu_page_size,
|
||||
cpu_page_size=cpu_page_size,
|
||||
barrier=barrier,
|
||||
)
|
||||
@@ -99,16 +99,16 @@ def _region(engine_id: str, **kwargs):
|
||||
def _multi_region(
|
||||
engine_id: str,
|
||||
num_workers: int,
|
||||
num_blocks: int = 4,
|
||||
num_chunks: int = 4,
|
||||
cpu_page_size: int = PAGE_SIZE,
|
||||
):
|
||||
"""Context manager: create one SharedOffloadRegion per rank, clean up on exit."""
|
||||
regions = [
|
||||
SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
||||
kv_bytes_per_chunk=num_workers * cpu_page_size,
|
||||
cpu_page_size=cpu_page_size,
|
||||
)
|
||||
for rank in range(num_workers)
|
||||
@@ -124,7 +124,7 @@ def _multi_region(
|
||||
def _race_construct(
|
||||
engine_id: str,
|
||||
num_workers: int,
|
||||
num_blocks: int = 4,
|
||||
num_chunks: int = 4,
|
||||
cpu_page_size: int = PAGE_SIZE,
|
||||
) -> tuple[list[SharedOffloadRegion], list[Exception]]:
|
||||
"""Spawn num_workers threads that all race to construct SharedOffloadRegion."""
|
||||
@@ -137,9 +137,9 @@ def _race_construct(
|
||||
try:
|
||||
regions[rank] = SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
||||
kv_bytes_per_chunk=num_workers * cpu_page_size,
|
||||
cpu_page_size=cpu_page_size,
|
||||
)
|
||||
except Exception as e:
|
||||
@@ -156,7 +156,7 @@ def _race_construct(
|
||||
|
||||
def _mp_race_construct_and_write(
|
||||
engine_id: str,
|
||||
num_blocks: int,
|
||||
num_chunks: int,
|
||||
rank: int,
|
||||
num_workers: int,
|
||||
cpu_page_size: int,
|
||||
@@ -170,9 +170,9 @@ def _mp_race_construct_and_write(
|
||||
try:
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
||||
kv_bytes_per_chunk=num_workers * cpu_page_size,
|
||||
cpu_page_size=cpu_page_size,
|
||||
)
|
||||
t = region.create_next_worker_view(cpu_page_size)
|
||||
@@ -198,9 +198,9 @@ def _mp_barrier_construct_and_hold(
|
||||
try:
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=2,
|
||||
num_chunks=2,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * PAGE_SIZE,
|
||||
kv_bytes_per_chunk=num_workers * PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
barrier=lambda: barrier.wait(30),
|
||||
)
|
||||
@@ -233,9 +233,9 @@ def iid():
|
||||
|
||||
|
||||
def test_create_next_worker_view_shape_and_stride(iid):
|
||||
"""Returned tensor must have shape (num_blocks, tensor_page_size) and
|
||||
"""Returned tensor must have shape (num_chunks, tensor_page_size) and
|
||||
stride (row_stride, 1) where row_stride = cpu_page_size * num_workers."""
|
||||
with _region(iid, num_blocks=4, cpu_page_size=2 * PAGE_SIZE) as r:
|
||||
with _region(iid, num_chunks=4, cpu_page_size=2 * PAGE_SIZE) as r:
|
||||
t = r.create_next_worker_view(PAGE_SIZE)
|
||||
assert t.shape == (4, PAGE_SIZE)
|
||||
# num_workers=1 → row_stride = cpu_page_size
|
||||
@@ -253,7 +253,7 @@ def test_create_next_worker_view_storage_offset_rank0(iid):
|
||||
|
||||
def test_create_next_worker_view_storage_offset_rank1(iid):
|
||||
"""rank=1 worker's first tensor must start cpu_page_size bytes into the mmap."""
|
||||
with _multi_region(iid, num_workers=2, num_blocks=4) as (r0, r1):
|
||||
with _multi_region(iid, num_workers=2, num_chunks=4) as (r0, r1):
|
||||
t1 = r1.create_next_worker_view(PAGE_SIZE)
|
||||
assert t1.data_ptr() == r1._base.data_ptr() + PAGE_SIZE
|
||||
del t1
|
||||
@@ -261,7 +261,7 @@ def test_create_next_worker_view_storage_offset_rank1(iid):
|
||||
|
||||
def test_create_next_worker_view_row_stride_with_multiple_workers(iid):
|
||||
"""With num_workers=4, row_stride must be 4 * cpu_page_size."""
|
||||
with _region(iid, num_blocks=2, num_workers=4) as r:
|
||||
with _region(iid, num_chunks=2, num_workers=4) as r:
|
||||
t = r.create_next_worker_view(PAGE_SIZE)
|
||||
assert t.stride(0) == 4 * PAGE_SIZE
|
||||
del t
|
||||
@@ -332,12 +332,12 @@ def test_create_next_worker_view_overflow_does_not_mutate_cursor(iid):
|
||||
def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
|
||||
"""Writes into a create_next_worker_view view must appear at the correct
|
||||
raw mmap offset"""
|
||||
with _region(iid, num_blocks=4) as r:
|
||||
with _region(iid, num_chunks=4) as r:
|
||||
t = r.create_next_worker_view(PAGE_SIZE)
|
||||
t[2, :] = 42 # write to block row 2
|
||||
t[2, :] = 42 # write to chunk row 2
|
||||
|
||||
raw = memoryview(r.mmap_obj)
|
||||
# num_workers=1 → row_stride = PAGE_SIZE; block 2 starts at byte 2*PAGE_SIZE
|
||||
# num_workers=1 → row_stride = PAGE_SIZE; chunk 2 starts at byte 2*PAGE_SIZE
|
||||
chunk = bytes(raw[2 * PAGE_SIZE : 3 * PAGE_SIZE])
|
||||
assert all(b == 42 for b in chunk)
|
||||
del raw, t
|
||||
@@ -345,7 +345,7 @@ def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
|
||||
|
||||
def test_create_next_worker_view_multi_tensor_layout(iid):
|
||||
"""Two tensors from the same worker land at consecutive byte offsets per row."""
|
||||
with _region(iid, num_blocks=2, cpu_page_size=2 * PAGE_SIZE) as r:
|
||||
with _region(iid, num_chunks=2, cpu_page_size=2 * PAGE_SIZE) as r:
|
||||
ta = r.create_next_worker_view(PAGE_SIZE)
|
||||
tb = r.create_next_worker_view(PAGE_SIZE)
|
||||
|
||||
@@ -353,8 +353,8 @@ def test_create_next_worker_view_multi_tensor_layout(iid):
|
||||
tb[:, :] = 2
|
||||
|
||||
raw = memoryview(r.mmap_obj)
|
||||
for blk in range(2):
|
||||
row_offset = blk * 2 * PAGE_SIZE # num_workers=1
|
||||
for chunk in range(2):
|
||||
row_offset = chunk * 2 * PAGE_SIZE # num_workers=1
|
||||
assert all(b == 1 for b in raw[row_offset : row_offset + PAGE_SIZE])
|
||||
assert all(
|
||||
b == 2 for b in raw[row_offset + PAGE_SIZE : row_offset + 2 * PAGE_SIZE]
|
||||
@@ -366,7 +366,7 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
||||
"""Each worker process calls create_next_worker_view and writes distinct data;
|
||||
the parent verifies each slot lands at the correct interleaved offset."""
|
||||
num_workers = 2
|
||||
num_blocks = 4
|
||||
num_chunks = 4
|
||||
|
||||
ctx = get_mp_context()
|
||||
done_queue = ctx.Queue()
|
||||
@@ -375,9 +375,9 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
||||
# Parent is rank 0 (creator); child is rank 1 (joiner).
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=iid,
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
rank=0,
|
||||
kv_bytes_per_block=num_workers * PAGE_SIZE,
|
||||
kv_bytes_per_chunk=num_workers * PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
)
|
||||
try:
|
||||
@@ -385,7 +385,7 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
||||
target=_mp_race_construct_and_write,
|
||||
args=(
|
||||
iid,
|
||||
num_blocks,
|
||||
num_chunks,
|
||||
1,
|
||||
num_workers,
|
||||
PAGE_SIZE,
|
||||
@@ -403,12 +403,12 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
||||
assert result["error"] is None, result["error"]
|
||||
|
||||
raw = memoryview(region.mmap_obj)
|
||||
for blk in range(num_blocks):
|
||||
row_start = blk * num_workers * PAGE_SIZE
|
||||
for chunk in range(num_chunks):
|
||||
row_start = chunk * num_workers * PAGE_SIZE
|
||||
w0 = bytes(raw[row_start : row_start + PAGE_SIZE])
|
||||
w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE])
|
||||
assert all(b == 11 for b in w0), f"block {blk}: rank0 slot wrong"
|
||||
assert all(b == 22 for b in w1), f"block {blk}: rank1 slot wrong"
|
||||
assert all(b == 11 for b in w0), f"chunk {chunk}: rank0 slot wrong"
|
||||
assert all(b == 22 for b in w1), f"chunk {chunk}: rank1 slot wrong"
|
||||
|
||||
del raw, t0 # release before finally triggers cleanup
|
||||
cleanup_queue.put(True)
|
||||
@@ -422,8 +422,8 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
||||
def test_create_next_worker_view_worker_isolation(iid):
|
||||
"""Writes by worker 0 must not affect worker 1's slot and vice versa."""
|
||||
num_workers = 2
|
||||
num_blocks = 4
|
||||
with _multi_region(iid, num_workers=num_workers, num_blocks=num_blocks) as regions:
|
||||
num_chunks = 4
|
||||
with _multi_region(iid, num_workers=num_workers, num_chunks=num_chunks) as regions:
|
||||
t0 = regions[0].create_next_worker_view(PAGE_SIZE)
|
||||
t1 = regions[1].create_next_worker_view(PAGE_SIZE)
|
||||
|
||||
@@ -431,12 +431,12 @@ def test_create_next_worker_view_worker_isolation(iid):
|
||||
t1[:, :] = 22
|
||||
|
||||
raw = memoryview(regions[0].mmap_obj)
|
||||
for blk in range(num_blocks):
|
||||
row_start = blk * num_workers * PAGE_SIZE
|
||||
for chunk in range(num_chunks):
|
||||
row_start = chunk * num_workers * PAGE_SIZE
|
||||
w0 = bytes(raw[row_start : row_start + PAGE_SIZE])
|
||||
w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE])
|
||||
assert all(b == 11 for b in w0), f"block {blk}: worker0 slot corrupted"
|
||||
assert all(b == 22 for b in w1), f"block {blk}: worker1 slot corrupted"
|
||||
assert all(b == 11 for b in w0), f"chunk {chunk}: worker0 slot corrupted"
|
||||
assert all(b == 22 for b in w1), f"chunk {chunk}: worker1 slot corrupted"
|
||||
del raw, t0, t1 # release before finally triggers cleanup
|
||||
|
||||
|
||||
@@ -466,7 +466,7 @@ def test_file_exists_after_construction(iid):
|
||||
|
||||
def test_file_has_correct_size(iid):
|
||||
"""The mmap file size on disk must equal total_size_bytes."""
|
||||
with _region(iid, num_blocks=4) as r:
|
||||
with _region(iid, num_chunks=4) as r:
|
||||
assert os.path.getsize(r.mmap_path) == 4 * PAGE_SIZE
|
||||
|
||||
|
||||
@@ -491,11 +491,11 @@ def test_madvise_success_selects_madvise_population(iid, monkeypatch):
|
||||
monkeypatch.setattr(sor, "_madvise_populate_write", _spy_madvise)
|
||||
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
||||
|
||||
num_blocks = 3
|
||||
num_chunks = 3
|
||||
num_workers = 2
|
||||
with _region(iid, num_blocks=num_blocks, num_workers=num_workers, rank=1):
|
||||
# 1 probe (PAGESIZE) + N populate calls (one per block per worker column).
|
||||
expected_populate = num_blocks # ranked path: one call per block
|
||||
with _region(iid, num_chunks=num_chunks, num_workers=num_workers, rank=1):
|
||||
# 1 probe (PAGESIZE) + N populate calls (one per chunk per worker column).
|
||||
expected_populate = num_chunks # ranked path: one call per chunk
|
||||
mmap_id = madvise_calls[0][2]
|
||||
assert madvise_calls == [
|
||||
(0, mmap.PAGESIZE, mmap_id),
|
||||
@@ -510,7 +510,7 @@ def test_madvise_success_selects_madvise_population(iid, monkeypatch):
|
||||
|
||||
|
||||
def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
|
||||
"""An EINVAL probe must select fallback for every ranked block."""
|
||||
"""An EINVAL probe must select fallback for every ranked chunk."""
|
||||
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
|
||||
|
||||
fallback_calls: list[tuple[int, int]] = []
|
||||
@@ -526,7 +526,7 @@ def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
|
||||
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
|
||||
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
||||
|
||||
with _region(iid, num_blocks=3, num_workers=2, rank=1):
|
||||
with _region(iid, num_chunks=3, num_workers=2, rank=1):
|
||||
assert fallback_calls == [
|
||||
(mmap.PAGESIZE, mmap.PAGESIZE),
|
||||
(3 * mmap.PAGESIZE, mmap.PAGESIZE),
|
||||
@@ -551,7 +551,7 @@ def test_madvise_einval_selects_fallback_for_unranked_region(iid, monkeypatch):
|
||||
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
|
||||
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
||||
|
||||
with _region(iid, num_blocks=3, num_workers=2, rank=None):
|
||||
with _region(iid, num_chunks=3, num_workers=2, rank=None):
|
||||
assert fallback_calls == [(0, 6 * mmap.PAGESIZE)]
|
||||
|
||||
|
||||
@@ -651,7 +651,7 @@ def test_multiprocess_race_construct_and_write(iid):
|
||||
"""N processes race to construct the same SharedOffloadRegion, each writes
|
||||
fill_value = rank+1 into their slot; parent verifies interleaved layout."""
|
||||
num_workers = 4
|
||||
num_blocks = 3
|
||||
num_chunks = 3
|
||||
|
||||
ctx = get_mp_context()
|
||||
done_queue = ctx.Queue()
|
||||
@@ -662,7 +662,7 @@ def test_multiprocess_race_construct_and_write(iid):
|
||||
target=_mp_race_construct_and_write,
|
||||
args=(
|
||||
iid,
|
||||
num_blocks,
|
||||
num_chunks,
|
||||
rank,
|
||||
num_workers,
|
||||
PAGE_SIZE,
|
||||
@@ -689,13 +689,13 @@ def test_multiprocess_race_construct_and_write(iid):
|
||||
with open(mmap_path, "rb") as f:
|
||||
raw = f.read()
|
||||
|
||||
for blk in range(num_blocks):
|
||||
for chunk in range(num_chunks):
|
||||
for w in range(num_workers):
|
||||
slot_start = (blk * num_workers + w) * PAGE_SIZE
|
||||
slot_start = (chunk * num_workers + w) * PAGE_SIZE
|
||||
slot = raw[slot_start : slot_start + PAGE_SIZE]
|
||||
expected = w + 1 # fill_value = rank + 1
|
||||
assert all(b == expected for b in slot), (
|
||||
f"block {blk}, worker {w}: expected {expected} but got wrong bytes"
|
||||
f"chunk {chunk}, worker {w}: expected {expected} but got wrong bytes"
|
||||
)
|
||||
|
||||
# Unblock all workers to clean up.
|
||||
@@ -839,9 +839,9 @@ def test_insufficient_space_raises_clear_error(monkeypatch):
|
||||
with pytest.raises(RuntimeError, match="Insufficient space"):
|
||||
SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=4,
|
||||
num_chunks=4,
|
||||
rank=0,
|
||||
kv_bytes_per_block=PAGE_SIZE,
|
||||
kv_bytes_per_chunk=PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
)
|
||||
|
||||
@@ -870,9 +870,9 @@ def test_ftruncate_failure_cleans_up_creator(monkeypatch):
|
||||
with pytest.raises(OSError, match="ftruncate failed"):
|
||||
SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=4,
|
||||
num_chunks=4,
|
||||
rank=0,
|
||||
kv_bytes_per_block=PAGE_SIZE,
|
||||
kv_bytes_per_chunk=PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
)
|
||||
|
||||
|
||||
@@ -136,7 +136,7 @@ def test_get_spec_cls_defaults_to_cpu():
|
||||
def test_create_cpu_offloading_spec():
|
||||
spec = _create_spec()
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.num_blocks > 0
|
||||
assert spec.num_chunks > 0
|
||||
|
||||
|
||||
def test_cpu_spec_sizes_normalized_worker_layout():
|
||||
@@ -156,7 +156,7 @@ def test_cpu_spec_sizes_normalized_worker_layout():
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 32
|
||||
assert spec.kv_bytes_per_chunk == alignment
|
||||
assert spec.num_blocks == 3
|
||||
assert spec.num_chunks == 3
|
||||
|
||||
|
||||
def test_cpu_spec_zero_worker_bytes_produces_empty_cache():
|
||||
@@ -165,7 +165,7 @@ def test_cpu_spec_zero_worker_bytes_produces_empty_cache():
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 0
|
||||
assert spec.kv_bytes_per_chunk == 0
|
||||
assert spec.num_blocks == 0
|
||||
assert spec.num_chunks == 0
|
||||
|
||||
|
||||
def test_tiering_spec_aligns_row_size():
|
||||
@@ -183,7 +183,7 @@ def test_tiering_spec_aligns_row_size():
|
||||
assert isinstance(spec, TieringOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 32
|
||||
assert spec.kv_bytes_per_chunk == alignment
|
||||
assert spec.num_blocks == 3
|
||||
assert spec.num_chunks == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize("world_size", [2, 4, 8])
|
||||
@@ -201,7 +201,7 @@ def test_tiering_spec_replicated_sizing_removes_world_factor(world_size: int):
|
||||
assert spec.replicated_layout is True
|
||||
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
|
||||
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block
|
||||
assert spec.num_blocks == 8
|
||||
assert spec.num_chunks == 8
|
||||
|
||||
|
||||
def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monkeypatch):
|
||||
@@ -239,7 +239,7 @@ def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monke
|
||||
spec.create_worker(kv_caches)
|
||||
|
||||
assert region_calls[0]["rank"] == 0
|
||||
assert region_calls[0]["kv_bytes_per_block"] == worker_kv_bytes_per_block
|
||||
assert region_calls[0]["kv_bytes_per_chunk"] == worker_kv_bytes_per_block
|
||||
assert worker_calls[0]["kv_caches"] is kv_caches
|
||||
assert worker_calls[0]["mmap_region"] is region
|
||||
|
||||
@@ -292,7 +292,7 @@ def test_cpu_spec_replicated_sizing_on_shared_region(monkeypatch, world_size: in
|
||||
assert spec.replicated_layout is True
|
||||
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
|
||||
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block
|
||||
assert spec.num_blocks == 8
|
||||
assert spec.num_chunks == 8
|
||||
|
||||
|
||||
@pytest.mark.parametrize("world_size", [2, 4, 8])
|
||||
@@ -320,7 +320,7 @@ def test_cpu_spec_replicated_disabled_without_shared_region(
|
||||
assert spec.replicated_layout is False
|
||||
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
|
||||
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block * world_size
|
||||
assert spec.num_blocks == 2
|
||||
assert spec.num_chunks == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("config_replicated", [True, False])
|
||||
@@ -381,7 +381,7 @@ def test_cpu_spec_create_worker_uses_mmap_on_cuda_alike(monkeypatch):
|
||||
spec.create_worker(kv_caches)
|
||||
|
||||
assert region_calls[0]["engine_id"] == "test-engine"
|
||||
assert region_calls[0]["kv_bytes_per_block"] == worker_kv_bytes_per_block * 4
|
||||
assert region_calls[0]["kv_bytes_per_chunk"] == worker_kv_bytes_per_block * 4
|
||||
assert worker_calls[0]["kv_caches"] is kv_caches
|
||||
assert worker_calls[0]["mmap_region"] is region
|
||||
|
||||
@@ -419,11 +419,11 @@ def test_cpu_spec_create_worker_uses_tensor_path_off_cuda_alike(monkeypatch):
|
||||
def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch):
|
||||
import vllm.v1.kv_offload.cpu.spec as cpu_spec_module
|
||||
|
||||
# worker_kv_bytes_per_block=0 yields num_blocks=0; a zero-byte region cannot
|
||||
# worker_kv_bytes_per_block=0 yields num_chunks=0; a zero-byte region cannot
|
||||
# be mmap'd, so even on CUDA-alike this must fall back to the tensor path.
|
||||
spec = _create_spec(worker_kv_bytes_per_block=0, world_size=4)
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.num_blocks == 0
|
||||
assert spec.num_chunks == 0
|
||||
|
||||
region_calls: list[dict[str, Any]] = []
|
||||
worker_calls: list[dict[str, Any]] = []
|
||||
|
||||
@@ -91,17 +91,17 @@ def _req_context(kv_params: dict | None = None) -> ReqContext:
|
||||
def _job_metadata(
|
||||
job_id: int,
|
||||
keys: list[bytes] | None = None,
|
||||
block_ids: list[int] | None = None,
|
||||
chunk_ids: list[int] | None = None,
|
||||
kv_params: dict | None = None,
|
||||
) -> TransferJob:
|
||||
if keys is None:
|
||||
keys = [b"key1"]
|
||||
if block_ids is None:
|
||||
block_ids = list(range(len(keys)))
|
||||
if chunk_ids is None:
|
||||
chunk_ids = list(range(len(keys)))
|
||||
return TransferJob(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=np.array(block_ids),
|
||||
chunk_ids=np.array(chunk_ids),
|
||||
is_promotion=False,
|
||||
req_context=_req_context(kv_params),
|
||||
)
|
||||
@@ -254,9 +254,9 @@ class TestLookup:
|
||||
"kv_params,expected",
|
||||
[
|
||||
(_remote_decoder_kv_params(), OffloadPolicy.REQUEST_LEVEL),
|
||||
(None, OffloadPolicy.BLOCK_LEVEL),
|
||||
(_remote_prefiller_kv_params(), OffloadPolicy.BLOCK_LEVEL),
|
||||
({"remote_decoder": {}}, OffloadPolicy.BLOCK_LEVEL),
|
||||
(None, OffloadPolicy.CHUNK_LEVEL),
|
||||
(_remote_prefiller_kv_params(), OffloadPolicy.CHUNK_LEVEL),
|
||||
({"remote_decoder": {}}, OffloadPolicy.CHUNK_LEVEL),
|
||||
],
|
||||
ids=["producer", "plain", "consumer", "producer_no_id"],
|
||||
)
|
||||
@@ -366,7 +366,7 @@ class TestSubmitStore:
|
||||
job = _job_metadata(
|
||||
job_id=1,
|
||||
keys=[b"k1", b"k2"],
|
||||
block_ids=[3, 4],
|
||||
chunk_ids=[3, 4],
|
||||
kv_params=_remote_decoder_kv_params(kv_request_id="req-1"),
|
||||
)
|
||||
mgr.submit_store(job)
|
||||
@@ -397,7 +397,7 @@ class TestSubmitStore:
|
||||
job = _job_metadata(
|
||||
job_id=7,
|
||||
keys=[b"k1", b"k2"],
|
||||
block_ids=[3, 4],
|
||||
chunk_ids=[3, 4],
|
||||
kv_params=_remote_decoder_kv_params(kv_request_id="req-1"),
|
||||
)
|
||||
mgr.submit_store(job)
|
||||
@@ -440,7 +440,7 @@ class TestSubmitLoad:
|
||||
"""Empty key list succeeds immediately."""
|
||||
mgr = _make_manager()
|
||||
job = _job_metadata(
|
||||
job_id=1, keys=[], block_ids=[], kv_params=_remote_prefiller_kv_params()
|
||||
job_id=1, keys=[], chunk_ids=[], kv_params=_remote_prefiller_kv_params()
|
||||
)
|
||||
mgr.submit_load(job)
|
||||
assert mgr._finished_jobs == [JobResult(job_id=1, success=True)]
|
||||
@@ -463,7 +463,7 @@ class TestSubmitLoad:
|
||||
job = _job_metadata(
|
||||
job_id=42,
|
||||
keys=[b"k1", b"k2"],
|
||||
block_ids=[5, 6],
|
||||
chunk_ids=[5, 6],
|
||||
kv_params=_remote_prefiller_kv_params(kv_request_id="req-42"),
|
||||
)
|
||||
mgr.submit_load(job)
|
||||
@@ -1155,7 +1155,7 @@ class TestBidirectionalManager:
|
||||
_job_metadata(
|
||||
job_id=100,
|
||||
keys=[b"a-block"],
|
||||
block_ids=[0],
|
||||
chunk_ids=[0],
|
||||
kv_params=a_prefiller_params,
|
||||
)
|
||||
)
|
||||
@@ -1163,7 +1163,7 @@ class TestBidirectionalManager:
|
||||
_job_metadata(
|
||||
job_id=200,
|
||||
keys=[b"b-block"],
|
||||
block_ids=[0],
|
||||
chunk_ids=[0],
|
||||
kv_params=b_prefiller_params,
|
||||
)
|
||||
)
|
||||
@@ -1173,7 +1173,7 @@ class TestBidirectionalManager:
|
||||
_job_metadata(
|
||||
job_id=101,
|
||||
keys=[b"b-block"],
|
||||
block_ids=[0],
|
||||
chunk_ids=[0],
|
||||
kv_params=a_decoder_params,
|
||||
)
|
||||
)
|
||||
@@ -1181,7 +1181,7 @@ class TestBidirectionalManager:
|
||||
_job_metadata(
|
||||
job_id=201,
|
||||
keys=[b"a-block"],
|
||||
block_ids=[0],
|
||||
chunk_ids=[0],
|
||||
kv_params=b_decoder_params,
|
||||
)
|
||||
)
|
||||
@@ -1573,7 +1573,7 @@ class TestConnectionDeathMidTransfer:
|
||||
_job_metadata(
|
||||
job_id=900,
|
||||
keys=[b"a-block"],
|
||||
block_ids=[0],
|
||||
chunk_ids=[0],
|
||||
kv_params=a_prefiller_params,
|
||||
)
|
||||
)
|
||||
@@ -1581,7 +1581,7 @@ class TestConnectionDeathMidTransfer:
|
||||
_job_metadata(
|
||||
job_id=901,
|
||||
keys=[b"b-block"],
|
||||
block_ids=[0],
|
||||
chunk_ids=[0],
|
||||
kv_params=a_decoder_params,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -259,13 +259,13 @@ class FakeParent:
|
||||
) -> TransferJob:
|
||||
keys_list = list(keys)
|
||||
self.calls.append(("create_store_job", tuple(keys_list), ctx.req_id))
|
||||
block_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32)
|
||||
chunk_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32)
|
||||
job_id = self._next_job_id
|
||||
self._next_job_id += 1
|
||||
return TransferJob(
|
||||
job_id=job_id,
|
||||
keys=keys_list,
|
||||
block_ids=block_ids,
|
||||
chunk_ids=chunk_ids,
|
||||
is_promotion=False,
|
||||
req_context=ctx,
|
||||
)
|
||||
|
||||
@@ -105,15 +105,15 @@ def key(n: int) -> OffloadKey:
|
||||
def make_job(
|
||||
job_id: int,
|
||||
keys: list[OffloadKey],
|
||||
block_ids: list[int] | None = None,
|
||||
chunk_ids: list[int] | None = None,
|
||||
is_promotion: bool = False,
|
||||
) -> TransferJob:
|
||||
if block_ids is None:
|
||||
block_ids = list(range(len(keys)))
|
||||
if chunk_ids is None:
|
||||
chunk_ids = list(range(len(keys)))
|
||||
return TransferJob(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=np.array(block_ids, dtype=np.int64),
|
||||
chunk_ids=np.array(chunk_ids, dtype=np.int64),
|
||||
is_promotion=is_promotion,
|
||||
req_context=_CTX,
|
||||
)
|
||||
@@ -382,11 +382,11 @@ def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size):
|
||||
tensor[:] = _page_aligned_rand_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS)
|
||||
|
||||
keys = [key(i) for i in range(batch_size)]
|
||||
store_block_ids = list(range(batch_size))
|
||||
load_block_ids = list(range(_NUM_BLOCKS - batch_size, _NUM_BLOCKS))
|
||||
store_chunk_ids = list(range(batch_size))
|
||||
load_chunk_ids = list(range(_NUM_BLOCKS - batch_size, _NUM_BLOCKS))
|
||||
expected = tensor[:batch_size].clone()
|
||||
|
||||
tier.submit_store(make_job(1, keys, store_block_ids))
|
||||
tier.submit_store(make_job(1, keys, store_chunk_ids))
|
||||
store_results = drain(tier)
|
||||
assert len(store_results) == 1
|
||||
assert store_results[0].success
|
||||
@@ -396,15 +396,15 @@ def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size):
|
||||
tensor[:] = 0.0
|
||||
|
||||
# Load into a range disjoint by index from the store ids, to also
|
||||
# exercise loading a block into a different id than it was stored from.
|
||||
tier.submit_load(make_job(2, keys, load_block_ids, is_promotion=True))
|
||||
# exercise loading a chunk into a different id than it was stored from.
|
||||
tier.submit_load(make_job(2, keys, load_chunk_ids, is_promotion=True))
|
||||
load_results = drain(tier)
|
||||
assert len(load_results) == 1
|
||||
assert load_results[0].success
|
||||
|
||||
for i, bid in enumerate(load_block_ids):
|
||||
assert torch.allclose(tensor[bid], expected[i]), (
|
||||
f"Block {bid} data mismatch after store+load"
|
||||
for i, cid in enumerate(load_chunk_ids):
|
||||
assert torch.allclose(tensor[cid], expected[i]), (
|
||||
f"Chunk {cid} data mismatch after store+load"
|
||||
)
|
||||
|
||||
|
||||
@@ -895,7 +895,7 @@ def test_cascade_store_emits_fs_event_through_tiering_manager(tmp_path):
|
||||
view = memoryview(tensor.numpy())
|
||||
mock_region = MagicMock()
|
||||
mock_region.create_kv_memoryview.return_value = view
|
||||
primary = CPUPrimaryTierOffloadingManager(num_blocks=4, mmap_region=mock_region)
|
||||
primary = CPUPrimaryTierOffloadingManager(num_chunks=4, mmap_region=mock_region)
|
||||
tier = FileSystemTierManager(
|
||||
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
|
||||
primary_kv_view=primary.get_kv_memoryview(),
|
||||
|
||||
@@ -25,8 +25,8 @@ def to_keys(int_ids: Iterable[int]):
|
||||
def test_tiering_metrics_tracker_records_lookup_metrics():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs", "p2p"],
|
||||
num_primary_blocks=5,
|
||||
primary_block_size=16,
|
||||
num_primary_chunks=5,
|
||||
primary_chunk_size=16,
|
||||
)
|
||||
tracker.on_new_request(_CTX)
|
||||
|
||||
@@ -45,18 +45,18 @@ def test_tiering_metrics_tracker_records_lookup_metrics():
|
||||
stats = tracker.take_stats()
|
||||
assert stats is not None
|
||||
values = stats.data["data"]
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("2:p2p",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_HITS][("2:p2p",)] == 1
|
||||
assert ("1:fs",) not in values[TieringOffloadingMetrics.BLOCK_HITS]
|
||||
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("0:primary",)] == 1
|
||||
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("1:fs",)] == 1
|
||||
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("2:p2p",)] == 1
|
||||
assert values[TieringOffloadingMetrics.CHUNK_HITS][("2:p2p",)] == 1
|
||||
assert ("1:fs",) not in values[TieringOffloadingMetrics.CHUNK_HITS]
|
||||
|
||||
|
||||
def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs"],
|
||||
num_primary_blocks=5,
|
||||
primary_block_size=16,
|
||||
num_primary_chunks=5,
|
||||
primary_chunk_size=16,
|
||||
)
|
||||
tracker.on_new_request(_CTX)
|
||||
key = to_keys([1])[0]
|
||||
@@ -73,9 +73,9 @@ def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
|
||||
stats = tracker.take_stats()
|
||||
assert stats is not None
|
||||
values = stats.data["data"]
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_HITS][("1:fs",)] == 1
|
||||
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("0:primary",)] == 1
|
||||
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("1:fs",)] == 1
|
||||
assert values[TieringOffloadingMetrics.CHUNK_HITS][("1:fs",)] == 1
|
||||
|
||||
tracker.on_request_allocated(_CTX)
|
||||
tracker.on_lookup(
|
||||
@@ -90,15 +90,15 @@ def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
|
||||
stats = tracker.take_stats()
|
||||
assert stats is not None
|
||||
values = stats.data["data"]
|
||||
assert TieringOffloadingMetrics.BLOCK_QUERIES not in values
|
||||
assert TieringOffloadingMetrics.BLOCK_HITS not in values
|
||||
assert TieringOffloadingMetrics.CHUNK_QUERIES not in values
|
||||
assert TieringOffloadingMetrics.CHUNK_HITS not in values
|
||||
|
||||
|
||||
def test_tiering_metrics_tracker_records_finished_job_metrics():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs"],
|
||||
num_primary_blocks=5,
|
||||
primary_block_size=16,
|
||||
num_primary_chunks=5,
|
||||
primary_chunk_size=16,
|
||||
)
|
||||
cascade_key, promotion_key, failed_cascade_key, failed_promotion_key = to_keys(
|
||||
range(4)
|
||||
@@ -149,8 +149,8 @@ def test_tiering_metrics_tracker_records_finished_job_metrics():
|
||||
def test_tiering_metrics_tracker_records_partial_promotion_success_bytes():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs"],
|
||||
num_primary_blocks=5,
|
||||
primary_block_size=16,
|
||||
num_primary_chunks=5,
|
||||
primary_chunk_size=16,
|
||||
)
|
||||
keys = to_keys(range(3))
|
||||
job = JobMetadata(
|
||||
@@ -182,8 +182,8 @@ def test_tiering_metrics_tracker_records_partial_promotion_success_bytes():
|
||||
def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs", "p2p"],
|
||||
num_primary_blocks=6,
|
||||
primary_block_size=16,
|
||||
num_primary_chunks=6,
|
||||
primary_chunk_size=16,
|
||||
)
|
||||
fs_job = JobMetadata(
|
||||
TransferJob(0, to_keys([0, 1]), np.array([0, 1]), False, _CTX),
|
||||
@@ -214,8 +214,8 @@ def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges():
|
||||
def test_tiering_metrics_tracker_records_promotion_allocation_failures():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs"],
|
||||
num_primary_blocks=1,
|
||||
primary_block_size=16,
|
||||
num_primary_chunks=1,
|
||||
primary_chunk_size=16,
|
||||
)
|
||||
|
||||
tracker.on_promotion_allocation_failure()
|
||||
|
||||
@@ -106,14 +106,14 @@ def key(n: int) -> OffloadKey:
|
||||
def make_job(
|
||||
job_id: int,
|
||||
keys: list[OffloadKey],
|
||||
block_ids: list[int] | None = None,
|
||||
chunk_ids: list[int] | None = None,
|
||||
) -> TransferJob:
|
||||
if block_ids is None:
|
||||
block_ids = list(range(len(keys)))
|
||||
if chunk_ids is None:
|
||||
chunk_ids = list(range(len(keys)))
|
||||
return TransferJob(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=np.array(block_ids, dtype=np.int32),
|
||||
chunk_ids=np.array(chunk_ids, dtype=np.int32),
|
||||
is_promotion=False,
|
||||
req_context=_CTX,
|
||||
)
|
||||
@@ -541,7 +541,7 @@ class TestMockObjTierFailures:
|
||||
mmap_region = MagicMock()
|
||||
mmap_region.create_kv_memoryview.return_value = primary_kv_view
|
||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=num_blocks, mmap_region=mmap_region
|
||||
num_chunks=num_blocks, mmap_region=mmap_region
|
||||
)
|
||||
obj_tier, agent = _make_tier(
|
||||
num_blocks=num_blocks, primary_kv_view=primary_kv_view
|
||||
|
||||
@@ -5,15 +5,16 @@ Unit tests for TieringOffloadingManager and ExampleSecondaryTierManager.
|
||||
|
||||
These tests verify:
|
||||
1. Basic tiered offloading operations (store, load, lookup)
|
||||
2. Cascade behavior (blocks stored to all secondary tiers)
|
||||
3. Promotion behavior (blocks loaded from secondary to primary to GPU)
|
||||
4. ref_cnt management (blocks protected during async transfers)
|
||||
2. Cascade behavior (chunks stored to all secondary tiers)
|
||||
3. Promotion behavior (chunks loaded from secondary to primary to GPU)
|
||||
4. ref_cnt management (chunks protected during async transfers)
|
||||
5. Eviction coordination between tiers
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -57,10 +58,10 @@ _CTX = ReqContext(req_id="test")
|
||||
_MOCK_OFFLOADING_SPEC = MagicMock()
|
||||
|
||||
|
||||
def _mock_mmap_region(num_blocks: int, row_bytes: int = 16):
|
||||
def _mock_mmap_region(num_chunks: int, row_bytes: int = 16):
|
||||
"""Create a mock SharedOffloadRegion for testing."""
|
||||
mock = MagicMock()
|
||||
view = memoryview(torch.zeros((num_blocks, row_bytes), dtype=torch.int8).numpy())
|
||||
view = memoryview(torch.zeros((num_chunks, row_bytes), dtype=torch.int8).numpy())
|
||||
mock.create_kv_memoryview.return_value = view
|
||||
return mock
|
||||
|
||||
@@ -160,7 +161,7 @@ def test_tiering_spec_collects_secondary_metric_definitions(monkeypatch):
|
||||
metadata = metrics[MetricsSecondaryTierManager.MY_TIER_METRIC]
|
||||
assert metadata.documentation == "Number of bytes served by the test tier."
|
||||
assert metadata.labelnames == ("tier",)
|
||||
assert metrics[TieringOffloadingMetrics.BLOCK_QUERIES].labelnames == ("tier",)
|
||||
assert metrics[TieringOffloadingMetrics.CHUNK_QUERIES].labelnames == ("tier",)
|
||||
assert metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY].labelnames == ("tier",)
|
||||
assert metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY].labelnames == ("tier",)
|
||||
assert metrics[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES].labelnames == (
|
||||
@@ -183,7 +184,7 @@ def test_tiering_spec_collects_secondary_metric_definitions(monkeypatch):
|
||||
def test_tiering_manager_aggregates_secondary_stats():
|
||||
mock_region = _mock_mmap_region(5)
|
||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=5, mmap_region=mock_region
|
||||
num_chunks=5, mmap_region=mock_region
|
||||
)
|
||||
secondary_tier = MetricsSecondaryTierManager(
|
||||
offloading_spec=_MOCK_OFFLOADING_SPEC,
|
||||
@@ -232,19 +233,19 @@ class TestExampleSecondaryTierManager:
|
||||
)
|
||||
|
||||
# Initially empty
|
||||
blocks = to_keys(range(3))
|
||||
assert tier.lookup(blocks[0], _CTX) is LookupResult.MISS
|
||||
chunks = to_keys(range(3))
|
||||
assert tier.lookup(chunks[0], _CTX) is LookupResult.MISS
|
||||
|
||||
# Store blocks (simulate with direct insertion for testing)
|
||||
tier.blocks[blocks[0]] = True
|
||||
tier.blocks[blocks[1]] = True
|
||||
# Store chunks (simulate with direct insertion for testing)
|
||||
tier.chunks[chunks[0]] = True
|
||||
tier.chunks[chunks[1]] = True
|
||||
|
||||
# Lookup should find first two blocks
|
||||
assert tier.lookup(blocks[0], _CTX) is LookupResult.HIT
|
||||
assert tier.lookup(blocks[1], _CTX) is LookupResult.HIT
|
||||
# Lookup should find first two chunks
|
||||
assert tier.lookup(chunks[0], _CTX) is LookupResult.HIT
|
||||
assert tier.lookup(chunks[1], _CTX) is LookupResult.HIT
|
||||
|
||||
# Third block not present
|
||||
assert tier.lookup(blocks[2], _CTX) is LookupResult.MISS
|
||||
# Third chunk not present
|
||||
assert tier.lookup(chunks[2], _CTX) is LookupResult.MISS
|
||||
|
||||
|
||||
# What a request-level cascade does with a key already present in the primary
|
||||
@@ -260,7 +261,7 @@ _CASCADE_SUPPLY_BEHAVIOR = {
|
||||
# The same test, before it lands.
|
||||
LookupResult.HIT_PENDING: "defer",
|
||||
# test_cascade_drops_deferred_keys_whose_primary_write_failed, where the
|
||||
# failed write frees the block.
|
||||
# failed write frees the chunk.
|
||||
LookupResult.MISS: "drop",
|
||||
# test_cascade_rejects_retry_from_primary. The primary tier resolves every
|
||||
# key it holds, so RETRY cannot reach the cascade, and parking on it would
|
||||
@@ -284,7 +285,7 @@ class TestTieringOffloadingManager:
|
||||
# Create primary tier (CPU-based)
|
||||
mock_region = _mock_mmap_region(5)
|
||||
self.primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=5, mmap_region=mock_region
|
||||
num_chunks=5, mmap_region=mock_region
|
||||
)
|
||||
|
||||
mock_view = mock_region.create_kv_memoryview()
|
||||
@@ -331,7 +332,7 @@ class TestTieringOffloadingManager:
|
||||
TransferJob(
|
||||
job_id=job_id,
|
||||
keys=to_keys([1, 2]),
|
||||
block_ids=[0, 1],
|
||||
chunk_ids=np.array([0, 1], dtype=np.int64),
|
||||
is_promotion=True,
|
||||
req_context=_CTX,
|
||||
),
|
||||
@@ -358,7 +359,7 @@ class TestTieringOffloadingManager:
|
||||
TransferJob(
|
||||
job_id=job_id,
|
||||
keys=to_keys([1]),
|
||||
block_ids=[0],
|
||||
chunk_ids=np.array([0], dtype=np.int64),
|
||||
is_promotion=True,
|
||||
req_context=_CTX,
|
||||
),
|
||||
@@ -392,23 +393,23 @@ class TestTieringOffloadingManager:
|
||||
|
||||
def test_basic_store_to_primary(self, manager_setup):
|
||||
"""Test basic store operation to primary tier."""
|
||||
blocks = to_keys(range(3))
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
# Prepare store
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
result = self.manager.prepare_store(chunks, _CTX)
|
||||
assert result is not None
|
||||
assert len(result.keys_to_store) == 3
|
||||
|
||||
# Complete store
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
|
||||
# Blocks should be in primary tier
|
||||
assert count_hits(self.primary_tier, blocks) == 3
|
||||
# Chunks should be in primary tier
|
||||
assert count_hits(self.primary_tier, chunks) == 3
|
||||
|
||||
def test_cascade_to_all_secondary_tiers(self, manager_setup):
|
||||
"""Test that blocks are cascaded to ALL secondary tiers."""
|
||||
blocks = to_keys(range(3))
|
||||
"""Test that chunks are cascaded to ALL secondary tiers."""
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_store
|
||||
@@ -419,44 +420,44 @@ class TestTieringOffloadingManager:
|
||||
|
||||
# Store to primary
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
result = self.manager.prepare_store(chunks, _CTX)
|
||||
assert result is not None
|
||||
|
||||
# Complete store (triggers cascade via submit_store on each tier)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
|
||||
# submit_store was called once per secondary tier
|
||||
self.secondary_tier1.submit_store.assert_called_once()
|
||||
self.secondary_tier2.submit_store.assert_called_once()
|
||||
|
||||
# Blocks should be in both secondary tiers
|
||||
assert self.secondary_tier1.get_num_blocks() == 3
|
||||
assert self.secondary_tier2.get_num_blocks() == 3
|
||||
# Chunks should be in both secondary tiers
|
||||
assert self.secondary_tier1.get_num_chunks() == 3
|
||||
assert self.secondary_tier2.get_num_chunks() == 3
|
||||
|
||||
# Verify blocks are present
|
||||
# Verify chunks are present
|
||||
assert all(
|
||||
self.secondary_tier1.lookup(b, _CTX) is LookupResult.HIT for b in blocks
|
||||
self.secondary_tier1.lookup(c, _CTX) is LookupResult.HIT for c in chunks
|
||||
)
|
||||
assert all(
|
||||
self.secondary_tier2.lookup(b, _CTX) is LookupResult.HIT for b in blocks
|
||||
self.secondary_tier2.lookup(c, _CTX) is LookupResult.HIT for c in chunks
|
||||
)
|
||||
|
||||
def test_ref_cnt_protection_during_cascade(self, manager_setup):
|
||||
"""Test that ref_cnt protects blocks during cascade."""
|
||||
blocks = to_keys(range(3))
|
||||
"""Test that ref_cnt protects chunks during cascade."""
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
# Store to primary
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
result = self.manager.prepare_store(chunks, _CTX)
|
||||
assert result is not None
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
|
||||
# After complete_store, blocks should have ref_cnt > 0
|
||||
# After complete_store, chunks should have ref_cnt > 0
|
||||
# (one for each secondary tier)
|
||||
for block_hash in blocks:
|
||||
block = self.primary_tier._policy.get(block_hash)
|
||||
for key in chunks:
|
||||
chunk = self.primary_tier._policy.get(key)
|
||||
# ref_cnt should be 2 (one for each secondary tier)
|
||||
assert block.ref_cnt == 2
|
||||
assert chunk.ref_cnt == 2
|
||||
|
||||
# End of step 1: _maybe_process_finished_jobs() was already called by
|
||||
# prepare_store() above (setting the per-step flag), so on_schedule_end()
|
||||
@@ -466,9 +467,9 @@ class TestTieringOffloadingManager:
|
||||
|
||||
# ref_cnt still held: cascade jobs finished (sync tier) but haven't
|
||||
# been polled yet because the per-step guard skipped the second call.
|
||||
for block_hash in blocks:
|
||||
block = self.primary_tier._policy.get(block_hash)
|
||||
assert block.ref_cnt == 2
|
||||
for key in chunks:
|
||||
chunk = self.primary_tier._policy.get(key)
|
||||
assert chunk.ref_cnt == 2
|
||||
|
||||
# Secondary tiers have completed jobs waiting to be drained
|
||||
assert len(self.secondary_tier1.completed_jobs) > 0
|
||||
@@ -479,37 +480,37 @@ class TestTieringOffloadingManager:
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# After cascade completes, ref_cnt should be 0
|
||||
for block_hash in blocks:
|
||||
block = self.primary_tier._policy.get(block_hash)
|
||||
assert block.ref_cnt == 0
|
||||
for key in chunks:
|
||||
chunk = self.primary_tier._policy.get(key)
|
||||
assert chunk.ref_cnt == 0
|
||||
|
||||
# All completed jobs have been drained
|
||||
assert len(self.secondary_tier1.completed_jobs) == 0
|
||||
assert len(self.secondary_tier2.completed_jobs) == 0
|
||||
|
||||
def test_lookup_from_primary(self, manager_setup):
|
||||
"""Test lookup when blocks are in primary tier."""
|
||||
blocks = to_keys(range(3))
|
||||
"""Test lookup when chunks are in primary tier."""
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
# Store blocks
|
||||
# Store chunks
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.prepare_store(chunks, _CTX)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
|
||||
# Lookup should find all blocks in primary
|
||||
assert count_hits(self.manager, blocks) == 3
|
||||
# Lookup should find all chunks in primary
|
||||
assert count_hits(self.manager, chunks) == 3
|
||||
|
||||
def test_promotion_from_secondary(self, manager_setup):
|
||||
"""Test promotion of blocks from secondary to primary tier."""
|
||||
blocks = to_keys(range(3))
|
||||
"""Test promotion of chunks from secondary to primary tier."""
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
# Manually add blocks to secondary tier (simulate previous cascade)
|
||||
for block in blocks:
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
# Manually add chunks to secondary tier (simulate previous cascade)
|
||||
for chunk in chunks:
|
||||
self.secondary_tier1.chunks[chunk] = True
|
||||
|
||||
# Lookup each block to initiate promotion for all of them
|
||||
for block in blocks:
|
||||
result = self.manager.lookup(block, _CTX)
|
||||
# Lookup each chunk to initiate promotion for all of them
|
||||
for chunk in chunks:
|
||||
result = self.manager.lookup(chunk, _CTX)
|
||||
assert result is LookupResult.HIT_PENDING # promotion initiated
|
||||
|
||||
# End of step 1: flushes deferred submit_load() calls
|
||||
@@ -518,11 +519,11 @@ class TestTieringOffloadingManager:
|
||||
# End of step 2: processes the completed promotion jobs
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# Now blocks should be in primary tier
|
||||
assert count_hits(self.primary_tier, blocks) == 3
|
||||
# Now chunks should be in primary tier
|
||||
assert count_hits(self.primary_tier, chunks) == 3
|
||||
|
||||
# Next lookup should succeed
|
||||
assert count_hits(self.manager, blocks) == 3
|
||||
assert count_hits(self.manager, chunks) == 3
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("successful_indices", "expected_results"),
|
||||
@@ -538,18 +539,18 @@ class TestTieringOffloadingManager:
|
||||
],
|
||||
ids=["partial", "legacy-full-failure"],
|
||||
)
|
||||
def test_failed_promotion_keeps_only_successful_blocks(
|
||||
def test_failed_promotion_keeps_only_successful_chunks(
|
||||
self, manager_setup, successful_indices, expected_results
|
||||
):
|
||||
blocks = to_keys(range(3))
|
||||
for block in blocks:
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
chunks = to_keys(range(3))
|
||||
for chunk in chunks:
|
||||
self.secondary_tier1.chunks[chunk] = True
|
||||
|
||||
def submit_partial(job_metadata: TransferJob) -> None:
|
||||
successful_keys = (
|
||||
None
|
||||
if successful_indices is None
|
||||
else tuple(blocks[i] for i in successful_indices)
|
||||
else tuple(chunks[i] for i in successful_indices)
|
||||
)
|
||||
self.secondary_tier1.completed_jobs.append(
|
||||
JobResult(
|
||||
@@ -561,24 +562,24 @@ class TestTieringOffloadingManager:
|
||||
|
||||
self.secondary_tier1.submit_load = submit_partial
|
||||
|
||||
for block in blocks:
|
||||
assert self.manager.lookup(block, _CTX) is LookupResult.HIT_PENDING
|
||||
for chunk in chunks:
|
||||
assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT_PENDING
|
||||
|
||||
self._simulate_on_schedule_end()
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
assert [
|
||||
self.primary_tier.lookup(block, _CTX) for block in blocks
|
||||
self.primary_tier.lookup(chunk, _CTX) for chunk in chunks
|
||||
] == expected_results
|
||||
|
||||
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
|
||||
"""Resolved lookups report one sync delay sample per tier and block."""
|
||||
"""Resolved lookups report one sync delay sample per tier and chunk."""
|
||||
self._start_request()
|
||||
blocks = to_keys(range(2))
|
||||
chunks = to_keys(range(2))
|
||||
|
||||
# No tier has these blocks: they resolve immediately as misses.
|
||||
for block in blocks:
|
||||
assert self.manager.lookup(block, _CTX) is LookupResult.MISS
|
||||
# No tier has these chunks: they resolve immediately as misses.
|
||||
for chunk in chunks:
|
||||
assert self.manager.lookup(chunk, _CTX) is LookupResult.MISS
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
assert stats is not None
|
||||
@@ -614,11 +615,11 @@ class TestTieringOffloadingManager:
|
||||
def test_lookup_does_not_report_async_delay_for_promotion(self, manager_setup):
|
||||
"""Promotion time is not included in lookup async delay."""
|
||||
self._start_request()
|
||||
block = to_keys(range(1))[0]
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
chunk = to_keys(range(1))[0]
|
||||
self.secondary_tier1.chunks[chunk] = True
|
||||
|
||||
# First lookup finds the block in a secondary tier and defers.
|
||||
assert self.manager.lookup(block, _CTX) is LookupResult.HIT_PENDING
|
||||
# First lookup finds the chunk in a secondary tier and defers.
|
||||
assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT_PENDING
|
||||
|
||||
# Promotion is not treated as unresolved secondary lookup time.
|
||||
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
|
||||
@@ -648,8 +649,8 @@ class TestTieringOffloadingManager:
|
||||
# Promotion completes on the next scheduler step.
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# Next lookup resolves via the now-promoted primary-tier block.
|
||||
assert self.manager.lookup(block, _CTX) is LookupResult.HIT
|
||||
# Next lookup resolves via the now-promoted primary-tier chunk.
|
||||
assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
if stats is not None:
|
||||
@@ -663,13 +664,13 @@ class TestTieringOffloadingManager:
|
||||
"""Async delay is observed when a RETRY lookup later resolves."""
|
||||
ctx = ReqContext(req_id="req_lookup_finish")
|
||||
self._start_request(ctx)
|
||||
block = to_keys(range(1))[0]
|
||||
chunk = to_keys(range(1))[0]
|
||||
self.secondary_tier1.lookup = MagicMock(
|
||||
side_effect=[LookupResult.RETRY, LookupResult.HIT]
|
||||
)
|
||||
|
||||
# First lookup is deferred by the secondary tier.
|
||||
assert self.manager.lookup(block, ctx) is LookupResult.RETRY
|
||||
assert self.manager.lookup(chunk, ctx) is LookupResult.RETRY
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
if stats is not None:
|
||||
@@ -677,7 +678,7 @@ class TestTieringOffloadingManager:
|
||||
stats.reduce(), TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
|
||||
)
|
||||
|
||||
assert self.manager.lookup(block, ctx) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(chunk, ctx) is LookupResult.HIT_PENDING
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
assert stats is not None
|
||||
@@ -701,68 +702,68 @@ class TestTieringOffloadingManager:
|
||||
|
||||
def test_partial_lookup(self, manager_setup):
|
||||
"""Test lookup with partial hits."""
|
||||
blocks = to_keys(range(5))
|
||||
chunks = to_keys(range(5))
|
||||
|
||||
# Store first 3 blocks to primary
|
||||
# Store first 3 chunks to primary
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks[:3], _CTX)
|
||||
self.manager.complete_store(blocks[:3], _CTX, success=True)
|
||||
self.manager.prepare_store(chunks[:3], _CTX)
|
||||
self.manager.complete_store(chunks[:3], _CTX, success=True)
|
||||
|
||||
# Lookup all 5 blocks should return 3 (first 3 found)
|
||||
assert count_hits(self.manager, blocks) == 3
|
||||
# Lookup all 5 chunks should return 3 (first 3 found)
|
||||
assert count_hits(self.manager, chunks) == 3
|
||||
|
||||
def test_eviction_in_primary_tier(self, manager_setup):
|
||||
"""Test eviction in primary tier when capacity is exceeded."""
|
||||
# Primary tier has capacity of 5 blocks
|
||||
# Primary tier has capacity of 5 chunks
|
||||
# First, fill the primary tier
|
||||
blocks = to_keys(range(5))
|
||||
chunks = to_keys(range(5))
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
result = self.manager.prepare_store(chunks, _CTX)
|
||||
assert result is not None
|
||||
assert len(result.keys_to_store) == 5
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
|
||||
# End of step: release ref_cnt from cascade
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# Now try to store 2 more blocks (should trigger eviction)
|
||||
more_blocks = to_keys(range(5, 7))
|
||||
result = self.manager.prepare_store(more_blocks, _CTX)
|
||||
# Now try to store 2 more chunks (should trigger eviction)
|
||||
more_chunks = to_keys(range(5, 7))
|
||||
result = self.manager.prepare_store(more_chunks, _CTX)
|
||||
|
||||
# Should evict 2 blocks from primary tier
|
||||
# Should evict 2 chunks from primary tier
|
||||
assert result is not None
|
||||
assert len(result.evicted_keys) == 2
|
||||
assert len(result.keys_to_store) == 2
|
||||
|
||||
def test_touch_propagates_to_all_tiers(self, manager_setup):
|
||||
"""Test that touch() propagates to all tiers."""
|
||||
blocks = to_keys(range(3))
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
# Store blocks
|
||||
# Store chunks
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.prepare_store(chunks, _CTX)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
self._simulate_on_schedule_end()
|
||||
# for secondary tiers to drain jobs, so primary tier's blocks are evictable.
|
||||
# for secondary tiers to drain jobs, so primary tier's chunks are evictable.
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
self.secondary_tier1.touch = MagicMock(wraps=self.secondary_tier1.touch)
|
||||
self.secondary_tier2.touch = MagicMock(wraps=self.secondary_tier2.touch)
|
||||
|
||||
# Touch blocks
|
||||
self.manager.touch(blocks, _CTX)
|
||||
# Touch chunks
|
||||
self.manager.touch(chunks, _CTX)
|
||||
|
||||
# Verify touch was called on primary tier (check LRU order)
|
||||
primary_keys = list(self.primary_tier._policy.evictable_blocks.keys())
|
||||
assert primary_keys[-3:] == list(reversed(blocks))
|
||||
primary_keys = list(self.primary_tier._policy.evictable_chunks.keys())
|
||||
assert primary_keys[-3:] == list(reversed(chunks))
|
||||
|
||||
# Verify touch was propagated to all secondary tiers
|
||||
self.secondary_tier1.touch.assert_called_once_with(blocks, _CTX)
|
||||
self.secondary_tier2.touch.assert_called_once_with(blocks, _CTX)
|
||||
self.secondary_tier1.touch.assert_called_once_with(chunks, _CTX)
|
||||
self.secondary_tier2.touch.assert_called_once_with(chunks, _CTX)
|
||||
|
||||
def test_failed_store_no_cascade(self, manager_setup):
|
||||
"""Test that failed GPU→primary store doesn't cascade."""
|
||||
blocks = to_keys(range(3))
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_store
|
||||
@@ -773,11 +774,11 @@ class TestTieringOffloadingManager:
|
||||
|
||||
# Prepare store
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
result = self.manager.prepare_store(chunks, _CTX)
|
||||
assert result is not None
|
||||
|
||||
# Complete store with failure — cascade must not happen
|
||||
self.manager.complete_store(blocks, _CTX, success=False)
|
||||
self.manager.complete_store(chunks, _CTX, success=False)
|
||||
|
||||
# submit_store was never called on either secondary tier
|
||||
self.secondary_tier1.submit_store.assert_not_called()
|
||||
@@ -786,12 +787,12 @@ class TestTieringOffloadingManager:
|
||||
def test_lookup_batches_submit_load_per_request(self, manager_setup):
|
||||
"""lookup() defers submit_load until on_schedule_end(), one per request.
|
||||
|
||||
Blocks from different requests each get their own submit_load call, each
|
||||
Chunks from different requests each get their own submit_load call, each
|
||||
carrying the correct req_context.
|
||||
"""
|
||||
blocks = to_keys(range(4))
|
||||
for block in blocks:
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
chunks = to_keys(range(4))
|
||||
for chunk in chunks:
|
||||
self.secondary_tier1.chunks[chunk] = True
|
||||
|
||||
self.secondary_tier1.submit_load = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_load
|
||||
@@ -801,10 +802,10 @@ class TestTieringOffloadingManager:
|
||||
ctx_b = ReqContext(req_id="req_b")
|
||||
|
||||
# All lookups return HIT_PENDING: secondary hit triggers promotion
|
||||
assert self.manager.lookup(blocks[0], ctx_a) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(blocks[1], ctx_a) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(blocks[2], ctx_b) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(blocks[3], ctx_b) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(chunks[0], ctx_a) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(chunks[1], ctx_a) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(chunks[2], ctx_b) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(chunks[3], ctx_b) is LookupResult.HIT_PENDING
|
||||
|
||||
# submit_load must not fire during lookup - only at end of step
|
||||
self.secondary_tier1.submit_load.assert_not_called()
|
||||
@@ -816,20 +817,20 @@ class TestTieringOffloadingManager:
|
||||
calls = self.secondary_tier1.submit_load.call_args_list
|
||||
jm_a = calls[0].args[0]
|
||||
jm_b = calls[1].args[0]
|
||||
assert set(jm_a.keys) == {blocks[0], blocks[1]}
|
||||
assert set(jm_a.keys) == {chunks[0], chunks[1]}
|
||||
assert jm_a.req_context is ctx_a
|
||||
assert set(jm_b.keys) == {blocks[2], blocks[3]}
|
||||
assert set(jm_b.keys) == {chunks[2], chunks[3]}
|
||||
assert jm_b.req_context is ctx_b
|
||||
|
||||
def test_lookup_shared_block_no_duplicate_promotion(self, manager_setup):
|
||||
"""A block looked up by two requests in the same step is promoted once.
|
||||
def test_lookup_shared_chunk_no_duplicate_promotion(self, manager_setup):
|
||||
"""A chunk looked up by two requests in the same step is promoted once.
|
||||
|
||||
The first lookup initiates promotion (returns None via secondary hit).
|
||||
The second lookup sees ref_cnt=-1 on the primary slot and returns None
|
||||
via the primary in-flight path — without triggering a second promotion.
|
||||
"""
|
||||
shared_block = to_keys([0])[0]
|
||||
self.secondary_tier1.blocks[shared_block] = True
|
||||
shared_chunk = to_keys([0])[0]
|
||||
self.secondary_tier1.chunks[shared_chunk] = True
|
||||
|
||||
self.secondary_tier1.submit_load = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_load
|
||||
@@ -838,10 +839,10 @@ class TestTieringOffloadingManager:
|
||||
ctx_a = ReqContext(req_id="req_a")
|
||||
ctx_b = ReqContext(req_id="req_b")
|
||||
|
||||
result_a = self.manager.lookup(shared_block, ctx_a)
|
||||
result_b = self.manager.lookup(shared_block, ctx_b)
|
||||
result_a = self.manager.lookup(shared_chunk, ctx_a)
|
||||
result_b = self.manager.lookup(shared_chunk, ctx_b)
|
||||
|
||||
# Both lookups find the shared_block and trigger promotion
|
||||
# Both lookups find the shared_chunk and trigger promotion
|
||||
# returning HIT_PENDING.
|
||||
assert result_a is LookupResult.HIT_PENDING
|
||||
assert result_b is LookupResult.HIT_PENDING
|
||||
@@ -851,12 +852,12 @@ class TestTieringOffloadingManager:
|
||||
# Only one submit_load call despite two lookups
|
||||
self.secondary_tier1.submit_load.assert_called_once()
|
||||
job_metadata = self.secondary_tier1.submit_load.call_args.args[0]
|
||||
assert list(job_metadata.keys) == [shared_block]
|
||||
assert list(job_metadata.keys) == [shared_chunk]
|
||||
assert job_metadata.req_context is ctx_a
|
||||
|
||||
def test_complete_store_forwards_req_context_to_submit_store(self, manager_setup):
|
||||
"""complete_store cascades to secondary tiers with the correct req_context."""
|
||||
blocks = to_keys(range(2))
|
||||
chunks = to_keys(range(2))
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_store
|
||||
@@ -865,8 +866,8 @@ class TestTieringOffloadingManager:
|
||||
ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"})
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.complete_store(blocks, ctx, success=True)
|
||||
self.manager.prepare_store(chunks, ctx)
|
||||
self.manager.complete_store(chunks, ctx, success=True)
|
||||
|
||||
assert self.secondary_tier1.submit_store.call_count == 1
|
||||
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
||||
@@ -876,7 +877,7 @@ class TestTieringOffloadingManager:
|
||||
self, manager_setup
|
||||
):
|
||||
"""Manager hook is eager; secondary hooks wait for cascade submission."""
|
||||
blocks = to_keys(range(2))
|
||||
chunks = to_keys(range(2))
|
||||
ctx = ReqContext(req_id="req_delayed_secondary")
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
@@ -911,14 +912,14 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.prepare_store(chunks, ctx)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
assert calls == [("primary_finish", ctx.req_id)]
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||
|
||||
self.manager.complete_store(blocks, ctx, success=True)
|
||||
self.manager.complete_store(chunks, ctx, success=True)
|
||||
|
||||
assert calls == [
|
||||
("primary_finish", ctx.req_id),
|
||||
@@ -930,7 +931,7 @@ class TestTieringOffloadingManager:
|
||||
|
||||
def test_failed_store_finalizes_finished_request(self, manager_setup):
|
||||
"""Failed primary stores still unblock secondary finalization."""
|
||||
blocks = to_keys(range(2))
|
||||
chunks = to_keys(range(2))
|
||||
ctx = ReqContext(req_id="req_failed_store_finalize")
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(
|
||||
@@ -947,13 +948,13 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.prepare_store(chunks, ctx)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||
|
||||
self.manager.complete_store(blocks, ctx, success=False)
|
||||
self.manager.complete_store(chunks, ctx, success=False)
|
||||
|
||||
self.secondary_tier1.submit_store.assert_not_called()
|
||||
self.secondary_tier2.submit_store.assert_not_called()
|
||||
@@ -981,7 +982,7 @@ class TestTieringOffloadingManager:
|
||||
|
||||
def test_reset_cache_finalizes_delayed_secondary_request(self, manager_setup):
|
||||
"""reset_cache abandons pending primary stores and finalizes secondaries."""
|
||||
blocks = to_keys(range(2))
|
||||
chunks = to_keys(range(2))
|
||||
ctx = ReqContext(req_id="req_reset_finalize_secondary")
|
||||
|
||||
self.secondary_tier1.on_request_finished = MagicMock(
|
||||
@@ -992,7 +993,7 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.prepare_store(chunks, ctx)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
@@ -1008,8 +1009,8 @@ class TestTieringOffloadingManager:
|
||||
self, manager_setup
|
||||
):
|
||||
"""reset_cache drops active pending stores so resumed requests finalize."""
|
||||
initial_blocks = to_keys(range(2))
|
||||
resumed_blocks = to_keys(range(2, 4))
|
||||
initial_chunks = to_keys(range(2))
|
||||
resumed_chunks = to_keys(range(2, 4))
|
||||
ctx = ReqContext(req_id="req_reset_resume")
|
||||
|
||||
self.secondary_tier1.on_request_finished = MagicMock(
|
||||
@@ -1020,7 +1021,7 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(initial_blocks, ctx)
|
||||
self.manager.prepare_store(initial_chunks, ctx)
|
||||
assert self.manager._req_state[ctx.req_id].pending_primary_stores == 1
|
||||
|
||||
self.manager.reset_cache()
|
||||
@@ -1030,8 +1031,8 @@ class TestTieringOffloadingManager:
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||
|
||||
self.manager.prepare_store(resumed_blocks, ctx)
|
||||
self.manager.complete_store(resumed_blocks, ctx, success=True)
|
||||
self.manager.prepare_store(resumed_chunks, ctx)
|
||||
self.manager.complete_store(resumed_chunks, ctx, success=True)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
|
||||
@@ -1039,12 +1040,12 @@ class TestTieringOffloadingManager:
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
def test_on_new_request_lifecycle(self, manager_setup):
|
||||
"""Policy defaults to BLOCK_LEVEL, escalates when a tier requests it,
|
||||
"""Policy defaults to CHUNK_LEVEL, escalates when a tier requests it,
|
||||
and is cleaned up on on_request_finished."""
|
||||
# Default: all tiers return BLOCK_LEVEL
|
||||
# Default: all tiers return CHUNK_LEVEL
|
||||
ctx = ReqContext(req_id="req_policy_lifecycle")
|
||||
result = self.manager.on_new_request(ctx)
|
||||
assert result.policy == OffloadPolicy.BLOCK_LEVEL
|
||||
assert result.policy == OffloadPolicy.CHUNK_LEVEL
|
||||
assert self.manager._req_state[ctx.req_id].request_level_tiers is None
|
||||
self.manager.on_request_finished(ctx)
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
@@ -1064,26 +1065,26 @@ class TestTieringOffloadingManager:
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
@pytest.mark.parametrize("new_ids", [(), (3, 4)], ids=["fully_warm", "mixed"])
|
||||
def test_prepare_store_cascades_existing_blocks_to_request_level_tiers(
|
||||
def test_prepare_store_cascades_existing_chunks_to_request_level_tiers(
|
||||
self, manager_setup, new_ids
|
||||
):
|
||||
"""prepare_store cascades hit blocks to request-level tiers only.
|
||||
"""prepare_store cascades hit chunks to request-level tiers only.
|
||||
|
||||
The fully-warm case is the producer shape from issue #52808: nothing is
|
||||
new, so primary returns an empty keys_to_store and no GPU->primary
|
||||
transfer runs. A request-level tier must still be handed the blocks,
|
||||
transfer runs. A request-level tier must still be handed the chunks,
|
||||
otherwise a peer waiting on them has nothing to fetch.
|
||||
"""
|
||||
# Store some blocks to primary first
|
||||
existing_blocks = to_keys(range(3))
|
||||
# Store some chunks to primary first
|
||||
existing_chunks = to_keys(range(3))
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(existing_blocks, _CTX)
|
||||
result = self.manager.prepare_store(existing_chunks, _CTX)
|
||||
assert result is not None
|
||||
self.manager.complete_store(existing_blocks, _CTX, success=True)
|
||||
self.manager.complete_store(existing_chunks, _CTX, success=True)
|
||||
# Drain cascade completions
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# Make tier1 request-level, tier2 stays block-level
|
||||
# Make tier1 request-level, tier2 stays chunk-level
|
||||
self.secondary_tier1.on_new_request = lambda req_context: (
|
||||
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
|
||||
)
|
||||
@@ -1099,20 +1100,20 @@ class TestTieringOffloadingManager:
|
||||
wraps=self.secondary_tier2.submit_store
|
||||
)
|
||||
|
||||
# Call prepare_store with existing + new blocks
|
||||
new_blocks = to_keys(new_ids)
|
||||
all_blocks = existing_blocks + new_blocks
|
||||
result = self.manager.prepare_store(all_blocks, ctx)
|
||||
# Call prepare_store with existing + new chunks
|
||||
new_chunks = to_keys(new_ids)
|
||||
all_chunks = existing_chunks + new_chunks
|
||||
result = self.manager.prepare_store(all_chunks, ctx)
|
||||
assert result is not None
|
||||
assert set(result.keys_to_store) == set(new_blocks)
|
||||
assert set(result.keys_to_store) == set(new_chunks)
|
||||
|
||||
# Only tier1 (request-level) should get existing blocks cascaded now.
|
||||
# New blocks are cascaded to ALL tiers later via complete_store().
|
||||
# Only tier1 (request-level) should get existing chunks cascaded now.
|
||||
# New chunks are cascaded to ALL tiers later via complete_store().
|
||||
self.secondary_tier1.submit_store.assert_called_once()
|
||||
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
||||
assert set(job_metadata.keys) == set(existing_blocks)
|
||||
assert set(job_metadata.keys) == set(existing_chunks)
|
||||
|
||||
# tier2 (block-level) does not get existing blocks here.
|
||||
# tier2 (chunk-level) does not get existing chunks here.
|
||||
self.secondary_tier2.submit_store.assert_not_called()
|
||||
|
||||
def _make_request_level_request(self, req_id: str) -> ReqContext:
|
||||
@@ -1150,7 +1151,7 @@ class TestTieringOffloadingManager:
|
||||
self.manager.primary_tier.lookup = lambda key, req_context: (LookupResult.RETRY)
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
self.manager._cascade_existing_blocks_to_request_level_tiers(keys, ctx, {0})
|
||||
self.manager._cascade_existing_chunks_to_request_level_tiers(keys, ctx, {0})
|
||||
|
||||
def _start_in_flight_primary_write(self, keys: list[OffloadKey]) -> ReqContext:
|
||||
"""Leave a GPU->primary write for `keys` open, so they look present to
|
||||
@@ -1202,7 +1203,7 @@ class TestTieringOffloadingManager:
|
||||
def test_cascade_drops_deferred_keys_whose_primary_write_failed(
|
||||
self, manager_setup
|
||||
):
|
||||
"""A failed write frees the block, so the deferred key resolves to MISS
|
||||
"""A failed write frees the chunk, so the deferred key resolves to MISS
|
||||
and the request finalizes instead of parking forever."""
|
||||
keys = to_keys(range(3))
|
||||
writer_ctx = self._start_in_flight_primary_write(keys)
|
||||
@@ -1224,21 +1225,21 @@ class TestTieringOffloadingManager:
|
||||
"""reset_cache wipes every kind of orchestrator state and resets
|
||||
primary tier; pending submissions are dropped without being sent
|
||||
to the secondary tier. Active request state is retained."""
|
||||
# Cascade — populates primary blocks and leaves cascade jobs
|
||||
# Cascade — populates primary chunks and leaves cascade jobs
|
||||
# in _jobs (the synchronous example tier has already
|
||||
# queued completions); reset_cache's drain loop will pick them up.
|
||||
blocks = to_keys(range(3))
|
||||
chunks = to_keys(range(3))
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.prepare_store(chunks, _CTX)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
assert self.manager._jobs
|
||||
|
||||
# Pending promotion submission (deferred — no on_schedule_end after
|
||||
# the lookup that staged it).
|
||||
promo_block = to_keys([99])[0]
|
||||
self.secondary_tier1.blocks[promo_block] = True
|
||||
promo_chunk = to_keys([99])[0]
|
||||
self.secondary_tier1.chunks[promo_chunk] = True
|
||||
assert (
|
||||
self.manager.lookup(promo_block, ReqContext(req_id="pending"))
|
||||
self.manager.lookup(promo_chunk, ReqContext(req_id="pending"))
|
||||
is LookupResult.HIT_PENDING
|
||||
)
|
||||
assert self.manager._pending_load_submissions
|
||||
@@ -1268,10 +1269,10 @@ class TestTieringOffloadingManager:
|
||||
assert self.manager._processed_jobs_this_step is False
|
||||
|
||||
# Primary tier reset to a fresh state.
|
||||
assert self.primary_tier._num_allocated_blocks == 0
|
||||
assert self.primary_tier._num_allocated_chunks == 0
|
||||
assert self.primary_tier._free_list == []
|
||||
for block in blocks:
|
||||
assert self.primary_tier.lookup(block, _CTX) is LookupResult.MISS
|
||||
for chunk in chunks:
|
||||
assert self.primary_tier.lookup(chunk, _CTX) is LookupResult.MISS
|
||||
|
||||
# Pending submission was dropped, not submitted.
|
||||
self.secondary_tier1.submit_load.assert_not_called()
|
||||
@@ -1291,10 +1292,10 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
# Drive a cascade so a job lands in _jobs.
|
||||
blocks = to_keys(range(3))
|
||||
chunks = to_keys(range(3))
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.prepare_store(chunks, _CTX)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
assert self.manager._jobs
|
||||
|
||||
self.manager.reset_cache()
|
||||
@@ -1315,20 +1316,20 @@ class TestTieringOffloadingManager:
|
||||
self, manager_setup, load_tier_filter
|
||||
):
|
||||
"""Filter excluding secondary medium returns MISS from secondaries
|
||||
even when they hold the block; primary is unaffected."""
|
||||
blocks = to_keys(range(2))
|
||||
# Put one block in primary, one only in secondary
|
||||
even when they hold the chunk; primary is unaffected."""
|
||||
chunks = to_keys(range(2))
|
||||
# Put one chunk in primary, one only in secondary
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks[:1], _CTX)
|
||||
self.manager.complete_store(blocks[:1], _CTX, success=True)
|
||||
self.secondary_tier1.blocks[blocks[1]] = True
|
||||
self.manager.prepare_store(chunks[:1], _CTX)
|
||||
self.manager.complete_store(chunks[:1], _CTX, success=True)
|
||||
self.secondary_tier1.chunks[chunks[1]] = True
|
||||
|
||||
# Secondaries have medium=CPU, so load_tier_filter skips them.
|
||||
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
|
||||
|
||||
ctx = ReqContext(req_id="r1", load_tier_filter=load_tier_filter)
|
||||
assert self.manager.lookup(blocks[0], ctx) is LookupResult.HIT
|
||||
assert self.manager.lookup(blocks[1], ctx) is LookupResult.MISS
|
||||
assert self.manager.lookup(chunks[0], ctx) is LookupResult.HIT
|
||||
assert self.manager.lookup(chunks[1], ctx) is LookupResult.MISS
|
||||
self.secondary_tier1.lookup.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -1344,13 +1345,13 @@ class TestTieringOffloadingManager:
|
||||
self, manager_setup, load_tier_filter
|
||||
):
|
||||
"""Filter that matches the secondary's medium allows lookup."""
|
||||
blocks = to_keys(range(1))
|
||||
self.secondary_tier1.blocks[blocks[0]] = True
|
||||
chunks = to_keys(range(1))
|
||||
self.secondary_tier1.chunks[chunks[0]] = True
|
||||
|
||||
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
|
||||
|
||||
ctx = ReqContext(req_id="r2", load_tier_filter=load_tier_filter)
|
||||
assert self.manager.lookup(blocks[0], ctx) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(chunks[0], ctx) is LookupResult.HIT_PENDING
|
||||
self.secondary_tier1.lookup.assert_called()
|
||||
|
||||
|
||||
@@ -1360,7 +1361,7 @@ class TestTieringOffloadingWithoutSecondaryTiers:
|
||||
def test_works_without_secondary_tiers(self):
|
||||
"""Test that manager works with empty secondary_tiers list."""
|
||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=5, mmap_region=_mock_mmap_region(5)
|
||||
num_chunks=5, mmap_region=_mock_mmap_region(5)
|
||||
)
|
||||
|
||||
# Create manager with no secondary tiers
|
||||
@@ -1368,15 +1369,15 @@ class TestTieringOffloadingWithoutSecondaryTiers:
|
||||
primary_tier=primary_tier, secondary_tiers=[]
|
||||
)
|
||||
|
||||
blocks = to_keys(range(3))
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
# Should work like a regular OffloadingManager
|
||||
manager.on_new_request(_CTX)
|
||||
result = manager.prepare_store(blocks, _CTX)
|
||||
result = manager.prepare_store(chunks, _CTX)
|
||||
assert result is not None
|
||||
manager.complete_store(blocks, _CTX, success=True)
|
||||
manager.complete_store(chunks, _CTX, success=True)
|
||||
|
||||
assert count_hits(manager, blocks) == 3
|
||||
assert count_hits(manager, chunks) == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
Reference in New Issue
Block a user