[Refactor][kv_offload]: rename block→chunk (#52615)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
Ronen Schaffer
2026-09-09 17:45:49 +03:00
committed by GitHub
parent 9ffb8cea96
commit 138d137b5b
34 changed files with 933 additions and 916 deletions
+4
View File
@@ -22,6 +22,10 @@ flowchart LR
CPU <--> SN["..."] CPU <--> SN["..."]
``` ```
## Terminology: Chunks
The unit of operation is a **chunk** — a fixed-size piece of KV data covering a group of tokens. By default, a chunk maps to a single accelerator block. A configurable `blocks_per_chunk` parameter allows larger chunks, yielding larger I/Os to the host and secondary tiers.
## Single-Tier Setup (CPU Only) ## Single-Tier Setup (CPU Only)
```bash ```bash
@@ -704,7 +704,7 @@ def test_prom_metrics_registers_tiering_metrics_from_spec():
) )
metric = prom_metrics._offloading_metric_defs[ metric = prom_metrics._offloading_metric_defs[
TieringOffloadingMetrics.BLOCK_QUERIES TieringOffloadingMetrics.CHUNK_QUERIES
] ]
assert metric.kwargs["labelnames"] == ["model_name", "engine", "tier"] assert metric.kwargs["labelnames"] == ["model_name", "engine", "tier"]
@@ -96,7 +96,7 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
groups.append( groups.append(
KVCacheGroupSpec([f"layer{i}"], kv_spec, is_eagle_group=eagle and i == 1) KVCacheGroupSpec([f"layer{i}"], kv_spec, is_eagle_group=eagle and i == 1)
) )
manager = CPUOffloadingManager(num_blocks=100) manager = CPUOffloadingManager(num_chunks=100)
spec = SimpleNamespace( spec = SimpleNamespace(
tokens_per_block=(256, 64, 8), tokens_per_block=(256, 64, 8),
tokens_per_hash=8, tokens_per_hash=8,
@@ -2888,7 +2888,7 @@ class TestEagle:
req=req, req=req,
req_context=ReqContext(req_id="test-req"), req_context=ReqContext(req_id="test-req"),
offloading_context=RequestOffloadingContext( offloading_context=RequestOffloadingContext(
policy=OffloadPolicy.BLOCK_LEVEL policy=OffloadPolicy.CHUNK_LEVEL
), ),
num_locally_computed_tokens=num_computed_tokens, num_locally_computed_tokens=num_computed_tokens,
) )
@@ -7,7 +7,7 @@ import pytest
from vllm.v1.kv_offload.base import OffloadKey, ReqContext from vllm.v1.kv_offload.base import OffloadKey, ReqContext
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
@@ -20,10 +20,10 @@ class _DummyCachePolicy(CachePolicy):
def __init__(self, cache_capacity: int) -> None: def __init__(self, cache_capacity: int) -> None:
self.cache_capacity = cache_capacity self.cache_capacity = cache_capacity
def get(self, key: OffloadKey) -> BlockStatus | None: def get(self, key: OffloadKey) -> ChunkStatus | None:
return None return None
def insert(self, key: OffloadKey, block: BlockStatus) -> None: def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
pass pass
def remove(self, key: OffloadKey) -> None: def remove(self, key: OffloadKey) -> None:
@@ -34,7 +34,7 @@ class _DummyCachePolicy(CachePolicy):
def evict( def evict(
self, n: int, protected: set[OffloadKey] self, n: int, protected: set[OffloadKey]
) -> list[tuple[OffloadKey, BlockStatus]] | None: ) -> list[tuple[OffloadKey, ChunkStatus]] | None:
return None return None
def clear(self) -> None: def clear(self) -> None:
@@ -72,7 +72,7 @@ class TestCachePolicyFactory:
policy_cls = CachePolicyFactory.get_cache_policy_cls("dummy") policy_cls = CachePolicyFactory.get_cache_policy_cls("dummy")
assert policy_cls is _DummyCachePolicy assert policy_cls is _DummyCachePolicy
manager = CPUOffloadingManager(num_blocks=4, cache_policy="dummy") manager = CPUOffloadingManager(num_chunks=4, cache_policy="dummy")
assert isinstance(manager._policy, _DummyCachePolicy) assert isinstance(manager._policy, _DummyCachePolicy)
def test_unregistered_policy_raises(self): def test_unregistered_policy_raises(self):
@@ -98,7 +98,7 @@ class TestCachePolicyFactory:
"""End-to-end: CPUOffloadingManager resolves an unregistered policy """End-to-end: CPUOffloadingManager resolves an unregistered policy
purely from cache_policy_module_path.""" purely from cache_policy_module_path."""
manager = CPUOffloadingManager( manager = CPUOffloadingManager(
num_blocks=4, num_chunks=4,
cache_policy="_DummyCachePolicy", cache_policy="_DummyCachePolicy",
cache_policy_module_path="tests.v1.kv_offload.cpu.policies.test_factory", cache_policy_module_path="tests.v1.kv_offload.cpu.policies.test_factory",
) )
@@ -232,9 +232,9 @@ def test_cross_topology_roundtrip(writer_tp: int, reader_tp: int):
def canonical_view(rank: int, world_size: int) -> torch.Tensor: def canonical_view(rank: int, world_size: int) -> torch.Tensor:
region = SharedOffloadRegion( region = SharedOffloadRegion(
engine_id=engine_id, engine_id=engine_id,
num_blocks=num_blocks, num_chunks=num_blocks,
rank=rank, rank=rank,
kv_bytes_per_block=row_stride, kv_bytes_per_chunk=row_stride,
cpu_page_size=row_stride // world_size, cpu_page_size=row_stride // world_size,
) )
regions.append(region) regions.append(region)
+46 -46
View File
@@ -26,7 +26,7 @@ from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
NUM_GPU_BLOCKS = [64] NUM_GPU_BLOCKS = [64]
NUM_CPU_BLOCKS = [256] NUM_CPU_CHUNKS = [256]
GPU_PAGE_SIZES = [512, 1024] GPU_PAGE_SIZES = [512, 1024]
BLOCKS_PER_CHUNK_VALUES = [1, 3] BLOCKS_PER_CHUNK_VALUES = [1, 3]
NUM_TENSORS = [4] NUM_TENSORS = [4]
@@ -225,7 +225,7 @@ def test_worker_syncs_before_cleanup_after_handler_failure(
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES) @pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES) @pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS) @pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS) @pytest.mark.parametrize("num_cpu_chunks", NUM_CPU_CHUNKS)
@pytest.mark.parametrize("num_tensors", NUM_TENSORS) @pytest.mark.parametrize("num_tensors", NUM_TENSORS)
@pytest.mark.parametrize("seed", SEEDS) @pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", DEVICES) @pytest.mark.parametrize("device", DEVICES)
@@ -241,7 +241,7 @@ def test_transfer(
gpu_page_size_bytes: int, gpu_page_size_bytes: int,
blocks_per_chunk: int, blocks_per_chunk: int,
num_gpu_blocks: int, num_gpu_blocks: int,
num_cpu_blocks: int, num_cpu_chunks: int,
num_tensors: int, num_tensors: int,
seed: int, seed: int,
device: str, device: str,
@@ -288,41 +288,41 @@ def test_transfer(
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT, SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
) )
simulated_world_size = 2 simulated_world_size = 2
kv_bytes_per_block = ( kv_bytes_per_chunk = (
cpu_page_size if replicated_layout else cpu_page_size * simulated_world_size cpu_page_size if replicated_layout else cpu_page_size * simulated_world_size
) )
mmap_region = SharedOffloadRegion( mmap_region = SharedOffloadRegion(
engine_id=str(uuid.uuid4()), engine_id=str(uuid.uuid4()),
num_blocks=num_cpu_blocks, num_chunks=num_cpu_chunks,
rank=0, rank=0,
kv_bytes_per_block=kv_bytes_per_block, kv_bytes_per_chunk=kv_bytes_per_chunk,
cpu_page_size=cpu_page_size, cpu_page_size=cpu_page_size,
) )
worker = CPUOffloadingWorker( worker = CPUOffloadingWorker(
kv_caches=kv_caches, kv_caches=kv_caches,
blocks_per_chunk=blocks_per_chunk, blocks_per_chunk=blocks_per_chunk,
num_cpu_blocks=num_cpu_blocks, num_cpu_chunks=num_cpu_chunks,
mmap_region=mmap_region, mmap_region=mmap_region,
) )
# select block mappings # select block mappings
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * blocks_per_chunk) gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * blocks_per_chunk)
cpu_blocks = random.sample(range(num_cpu_blocks), num_mappings) cpu_chunks = random.sample(range(num_cpu_chunks), num_mappings)
# expand cpu blocks to gpu-page granularity for uniform comparison: # expand cpu chunks to gpu-page granularity for uniform comparison:
# each cpu block maps to blocks_per_chunk consecutive sub-blocks # each cpu chunk maps to blocks_per_chunk consecutive sub-blocks
cpu_blocks_expanded = [ cpu_chunks_expanded = [
cpu_block * blocks_per_chunk + j cpu_chunk * blocks_per_chunk + j
for cpu_block in cpu_blocks for cpu_chunk in cpu_chunks
for j in range(blocks_per_chunk) for j in range(blocks_per_chunk)
] ]
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU block # maybe skip some GPU blocks to test reading/writing from the middle of a CPU chunk
blocks_to_skip = blocks_per_chunk - 1 blocks_to_skip = blocks_per_chunk - 1
if blocks_to_skip > 0: if blocks_to_skip > 0:
gpu_blocks = gpu_blocks[blocks_to_skip:] gpu_blocks = gpu_blocks[blocks_to_skip:]
cpu_blocks_expanded = cpu_blocks_expanded[blocks_to_skip:] cpu_chunks_expanded = cpu_chunks_expanded[blocks_to_skip:]
# set transfer direction # set transfer direction
if gpu_to_cpu: if gpu_to_cpu:
@@ -330,16 +330,16 @@ def test_transfer(
src_spec = GPULoadStoreSpec( src_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,) gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
) )
dst_spec = CPULoadStoreSpec(cpu_blocks) dst_spec = CPULoadStoreSpec(cpu_chunks)
dst_to_src = dict(zip(cpu_blocks_expanded, gpu_blocks)) dst_to_src = dict(zip(cpu_chunks_expanded, gpu_blocks))
num_dst_sub_blocks = num_gpu_blocks num_dst_sub_blocks = num_gpu_blocks
else: else:
handler = worker._load_handler handler = worker._load_handler
src_spec = CPULoadStoreSpec(cpu_blocks) src_spec = CPULoadStoreSpec(cpu_chunks)
dst_spec = GPULoadStoreSpec( dst_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,) gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
) )
dst_to_src = dict(zip(gpu_blocks, cpu_blocks_expanded)) dst_to_src = dict(zip(gpu_blocks, cpu_chunks_expanded))
num_dst_sub_blocks = num_gpu_blocks num_dst_sub_blocks = num_gpu_blocks
# randomize src and dst tensors before transfer # randomize src and dst tensors before transfer
@@ -410,7 +410,7 @@ def test_transfer(
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES) @pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES) @pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS) @pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS) @pytest.mark.parametrize("num_cpu_chunks", NUM_CPU_CHUNKS)
@pytest.mark.parametrize("seed", SEEDS) @pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", DEVICES) @pytest.mark.parametrize("device", DEVICES)
@torch.inference_mode() @torch.inference_mode()
@@ -421,7 +421,7 @@ def test_transfer_multi_group(
gpu_page_size_bytes: int, gpu_page_size_bytes: int,
blocks_per_chunk: int, blocks_per_chunk: int,
num_gpu_blocks: int, num_gpu_blocks: int,
num_cpu_blocks: int, num_cpu_chunks: int,
seed: int, seed: int,
device: str, device: str,
) -> None: ) -> None:
@@ -470,37 +470,37 @@ def test_transfer_multi_group(
worker = CPUOffloadingWorker( worker = CPUOffloadingWorker(
kv_caches=canonical_kv_caches, kv_caches=canonical_kv_caches,
blocks_per_chunk=blocks_per_chunk, blocks_per_chunk=blocks_per_chunk,
num_cpu_blocks=num_cpu_blocks, num_cpu_chunks=num_cpu_chunks,
) )
# group 0: aligned, group 1: empty, group 2: unaligned on CPU->GPU # group 0: aligned, group 1: empty, group 2: unaligned on CPU->GPU
group_sizes_in_cpu_blocks = [num_mappings_per_group, 0, num_mappings_per_group] group_sizes_in_cpu_chunks = [num_mappings_per_group, 0, num_mappings_per_group]
total_cpu_blocks = sum(group_sizes_in_cpu_blocks) total_cpu_chunks = sum(group_sizes_in_cpu_chunks)
total_gpu_blocks_needed = total_cpu_blocks * blocks_per_chunk total_gpu_blocks_needed = total_cpu_chunks * blocks_per_chunk
gpu_blocks_all = random.sample(range(num_gpu_blocks), total_gpu_blocks_needed) gpu_blocks_all = random.sample(range(num_gpu_blocks), total_gpu_blocks_needed)
cpu_blocks_all = random.sample(range(num_cpu_blocks), total_cpu_blocks) cpu_chunks_all = random.sample(range(num_cpu_chunks), total_cpu_chunks)
# split gpu/cpu blocks per group # split gpu blocks / cpu chunks per group
gpu_blocks_per_group: list[list[int]] = [] gpu_blocks_per_group: list[list[int]] = []
cpu_blocks_per_group: list[list[int]] = [] cpu_chunks_per_group: list[list[int]] = []
gpu_offset = 0 gpu_offset = 0
cpu_offset = 0 cpu_offset = 0
for size in group_sizes_in_cpu_blocks: for size in group_sizes_in_cpu_chunks:
gpu_count = size * blocks_per_chunk gpu_count = size * blocks_per_chunk
gpu_blocks_per_group.append(gpu_blocks_all[gpu_offset : gpu_offset + gpu_count]) gpu_blocks_per_group.append(gpu_blocks_all[gpu_offset : gpu_offset + gpu_count])
cpu_blocks_per_group.append(cpu_blocks_all[cpu_offset : cpu_offset + size]) cpu_chunks_per_group.append(cpu_chunks_all[cpu_offset : cpu_offset + size])
gpu_offset += gpu_count gpu_offset += gpu_count
cpu_offset += size cpu_offset += size
# expand cpu blocks to gpu-page granularity # expand cpu chunks to gpu-page granularity
cpu_blocks_expanded_per_group = [ cpu_chunks_expanded_per_group = [
[ [
cpu_block * blocks_per_chunk + j cpu_chunk * blocks_per_chunk + j
for cpu_block in cpu_blocks for cpu_chunk in cpu_chunks
for j in range(blocks_per_chunk) for j in range(blocks_per_chunk)
] ]
for cpu_blocks in cpu_blocks_per_group for cpu_chunks in cpu_chunks_per_group
] ]
# skip sub-blocks from group 2 to test unaligned transfers. # skip sub-blocks from group 2 to test unaligned transfers.
@@ -509,7 +509,7 @@ def test_transfer_multi_group(
gpu_blocks_per_group[2] = gpu_blocks_per_group[2][ gpu_blocks_per_group[2] = gpu_blocks_per_group[2][
sub_blocks_to_skip:-sub_blocks_to_skip sub_blocks_to_skip:-sub_blocks_to_skip
] ]
cpu_blocks_expanded_per_group[2] = cpu_blocks_expanded_per_group[2][ cpu_chunks_expanded_per_group[2] = cpu_chunks_expanded_per_group[2][
sub_blocks_to_skip:-sub_blocks_to_skip sub_blocks_to_skip:-sub_blocks_to_skip
] ]
@@ -520,10 +520,10 @@ def test_transfer_multi_group(
gpu_blocks.extend(gpu_blks) gpu_blocks.extend(gpu_blks)
group_sizes.append(len(gpu_blks)) group_sizes.append(len(gpu_blks))
# build flat cpu_blocks list # build flat cpu_chunks list
cpu_blocks = [] cpu_chunks = []
for cpu_blks in cpu_blocks_per_group: for cpu_chnks in cpu_chunks_per_group:
cpu_blocks.extend(cpu_blks) cpu_chunks.extend(cpu_chnks)
# block_indices: only relevant for unaligned transfers # block_indices: only relevant for unaligned transfers
block_indices: list[int] = [0, 0, sub_blocks_to_skip] block_indices: list[int] = [0, 0, sub_blocks_to_skip]
@@ -533,18 +533,18 @@ def test_transfer_multi_group(
src_spec = GPULoadStoreSpec( src_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
) )
dst_spec = CPULoadStoreSpec(cpu_blocks) dst_spec = CPULoadStoreSpec(cpu_chunks)
# per-group mapping: cpu sub-block -> gpu sub-block # per-group mapping: cpu sub-block -> gpu sub-block
dst_to_src_per_group = [ dst_to_src_per_group = [
dict(zip(expanded, gpu_blks)) dict(zip(expanded, gpu_blks))
for expanded, gpu_blks in zip( for expanded, gpu_blks in zip(
cpu_blocks_expanded_per_group, gpu_blocks_per_group cpu_chunks_expanded_per_group, gpu_blocks_per_group
) )
] ]
num_dst_sub_blocks = num_cpu_blocks * blocks_per_chunk num_dst_sub_blocks = num_cpu_chunks * blocks_per_chunk
else: else:
handler = worker._load_handler handler = worker._load_handler
src_spec = CPULoadStoreSpec(cpu_blocks) src_spec = CPULoadStoreSpec(cpu_chunks)
dst_spec = GPULoadStoreSpec( dst_spec = GPULoadStoreSpec(
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
) )
@@ -552,7 +552,7 @@ def test_transfer_multi_group(
dst_to_src_per_group = [ dst_to_src_per_group = [
dict(zip(gpu_blks, expanded)) dict(zip(gpu_blks, expanded))
for gpu_blks, expanded in zip( for gpu_blks, expanded in zip(
gpu_blocks_per_group, cpu_blocks_expanded_per_group gpu_blocks_per_group, cpu_chunks_expanded_per_group
) )
] ]
num_dst_sub_blocks = num_gpu_blocks num_dst_sub_blocks = num_gpu_blocks
@@ -644,7 +644,7 @@ def test_load_waits_for_pending_compute_stream_writes(default_vllm_config) -> No
], ],
), ),
blocks_per_chunk=1, blocks_per_chunk=1,
num_cpu_blocks=num_blocks, num_cpu_chunks=num_blocks,
) )
worker._load_handler.src_tensors[0].fill_(sentinel) worker._load_handler.src_tensors[0].fill_(sentinel)
expected = torch.full((page_size_bytes,), sentinel, dtype=torch.int8) expected = torch.full((page_size_bytes,), sentinel, dtype=torch.int8)
+144 -144
View File
@@ -36,7 +36,7 @@ _EMPTY_REQ_CTX = make_req_context()
def make_cpu_manager( def make_cpu_manager(
num_blocks: int = 4, num_chunks: int = 4,
cache_policy: str = "lru", cache_policy: str = "lru",
cache_policy_module_path: str | None = None, cache_policy_module_path: str | None = None,
enable_events: bool = False, enable_events: bool = False,
@@ -44,7 +44,7 @@ def make_cpu_manager(
max_tracker_size: int = 64_000, max_tracker_size: int = 64_000,
) -> CPUOffloadingManager: ) -> CPUOffloadingManager:
return CPUOffloadingManager( return CPUOffloadingManager(
num_blocks=num_blocks, num_chunks=num_chunks,
cache_policy=cache_policy, cache_policy=cache_policy,
cache_policy_module_path=cache_policy_module_path, cache_policy_module_path=cache_policy_module_path,
enable_events=enable_events, enable_events=enable_events,
@@ -56,7 +56,7 @@ def make_cpu_manager(
@dataclass @dataclass
class ExpectedPrepareStoreOutput: class ExpectedPrepareStoreOutput:
keys_to_store: list[int] keys_to_store: list[int]
store_block_ids: list[int] store_chunk_ids: list[int]
evicted_keys: list[int] evicted_keys: list[int]
@@ -93,9 +93,9 @@ def verify_store_output(
store_spec = prepare_store_output.store_spec store_spec = prepare_store_output.store_spec
assert isinstance(store_spec, CPULoadStoreSpec) assert isinstance(store_spec, CPULoadStoreSpec)
expected_array = np.array( expected_array = np.array(
expected_prepare_store_output.store_block_ids, dtype=np.int64 expected_prepare_store_output.store_chunk_ids, dtype=np.int64
) )
assert np.array_equal(expected_array, store_spec.block_ids) assert np.array_equal(expected_array, store_spec.chunk_ids)
def verify_load_output( def verify_load_output(
@@ -103,7 +103,7 @@ def verify_load_output(
): ):
assert isinstance(prepare_load_output, CPULoadStoreSpec) assert isinstance(prepare_load_output, CPULoadStoreSpec)
expected_array = np.array(expected_prepare_load_output, dtype=np.int64) expected_array = np.array(expected_prepare_load_output, dtype=np.int64)
assert np.array_equal(expected_array, prepare_load_output.block_ids) assert np.array_equal(expected_array, prepare_load_output.chunk_ids)
def check_split_usage_stats( def check_split_usage_stats(
@@ -146,7 +146,7 @@ def verify_events(
def test_cpu_eviction_removed_precedes_stored(): def test_cpu_eviction_removed_precedes_stored():
"""An eviction is announced before the store that reuses its capacity.""" """An eviction is announced before the store that reuses its capacity."""
manager = make_cpu_manager(num_blocks=2, enable_events=True) manager = make_cpu_manager(num_chunks=2, enable_events=True)
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX) manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX) manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -164,23 +164,23 @@ def test_cpu_eviction_removed_precedes_stored():
@pytest.mark.parametrize("eviction_policy", ["lru", "arc"]) @pytest.mark.parametrize("eviction_policy", ["lru", "arc"])
def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy): def test_already_stored_chunk_not_evicted_during_prepare_store(eviction_policy):
""" """
Regression test: a block that is already stored must not be evicted Regression test: a chunk that is already stored must not be evicted
by prepare_store() when it needs to make room for new blocks. by prepare_store() when it needs to make room for new chunks.
Applies to both lru and arc policies. Applies to both lru and arc policies.
Scenario: Scenario:
- Store blocks [1, 2] and complete. - Store chunks [1, 2] and complete.
- touch([1]) makes block 2 the LRU candidate. - touch([1]) makes chunk 2 the LRU candidate.
- prepare_store([2, 3, 4, 5]): - prepare_store([2, 3, 4, 5]):
* block 2 is filtered out as "already stored" * chunk 2 is filtered out as "already stored"
* but without the fix, block 2 would be evicted as the LRU * but without the fix, chunk 2 would be evicted as the LRU
candidate to make room for [3, 4, 5] candidate to make room for [3, 4, 5]
- After complete_store([2, 3, 4, 5]), block 2 must still be present. - After complete_store([2, 3, 4, 5]), chunk 2 must still be present.
""" """
manager = make_cpu_manager( manager = make_cpu_manager(
num_blocks=4, num_chunks=4,
cache_policy=eviction_policy, cache_policy=eviction_policy,
enable_events=True, enable_events=True,
) )
@@ -189,33 +189,33 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX) manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX) manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
# touch [1] to make block 2 the LRU candidate # touch [1] to make chunk 2 the LRU candidate
manager.touch(to_keys([1]), _EMPTY_REQ_CTX) manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
# prepare_store([2, 3, 4, 5]): # prepare_store([2, 3, 4, 5]):
# - block 2 is already stored -> filtered out of keys_to_store # - chunk 2 is already stored -> filtered out of keys_to_store
# - block 2 must NOT be evicted even though it is the LRU candidate # - chunk 2 must NOT be evicted even though it is the LRU candidate
# - block 1 (ID 0) is evicted instead; new blocks [3,4,5] get IDs 2,3,0 # - chunk 1 (ID 0) is evicted instead; new chunks [3,4,5] get IDs 2,3,0
prepare_store_output = manager.prepare_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX) prepare_store_output = manager.prepare_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
verify_store_output( verify_store_output(
prepare_store_output, prepare_store_output,
ExpectedPrepareStoreOutput( ExpectedPrepareStoreOutput(
keys_to_store=[3, 4, 5], keys_to_store=[3, 4, 5],
store_block_ids=[2, 3, 0], store_chunk_ids=[2, 3, 0],
evicted_keys=[1], # block 1 evicted, not block 2 evicted_keys=[1], # chunk 1 evicted, not chunk 2
), ),
) )
# complete_store must not silently drop block 2 # complete_store must not silently drop chunk 2
manager.complete_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX) manager.complete_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
# block 2 must still be present in the cache # chunk 2 must still be present in the cache
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
def test_filter_reused_manager_reports_stores_skipped_counter(): def test_filter_reused_manager_reports_stores_skipped_counter():
manager = make_cpu_manager( manager = make_cpu_manager(
num_blocks=4, num_chunks=4,
cache_policy="lru", cache_policy="lru",
store_threshold=2, store_threshold=2,
) )
@@ -226,7 +226,7 @@ def test_filter_reused_manager_reports_stores_skipped_counter():
prepare_store_output, prepare_store_output,
ExpectedPrepareStoreOutput( ExpectedPrepareStoreOutput(
keys_to_store=[], keys_to_store=[],
store_block_ids=[], store_chunk_ids=[],
evicted_keys=[], evicted_keys=[],
), ),
) )
@@ -247,34 +247,34 @@ def test_cpu_manager_reports_cache_usage_gauge():
] == pytest.approx(value) ] == pytest.approx(value)
# Zero-capacity manager always reports 0.0 # Zero-capacity manager always reports 0.0
manager = make_cpu_manager(num_blocks=0) manager = make_cpu_manager(num_chunks=0)
check_usage_stats(manager, 0.0) check_usage_stats(manager, 0.0)
# Empty manager (4 blocks, none allocated): usage = 0.0 # Empty manager (4 chunks, none allocated): usage = 0.0
manager = make_cpu_manager(num_blocks=4) manager = make_cpu_manager(num_chunks=4)
check_usage_stats(manager, 0.0) check_usage_stats(manager, 0.0)
# After allocating 2 of 4 blocks: usage = 0.5 # After allocating 2 of 4 chunks: usage = 0.5
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX) manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
check_usage_stats(manager, 0.5) check_usage_stats(manager, 0.5)
# After filling all 4 blocks: usage = 1.0 # After filling all 4 chunks: usage = 1.0
manager.prepare_store(to_keys([3, 4]), _EMPTY_REQ_CTX) manager.prepare_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
check_usage_stats(manager, 1.0) check_usage_stats(manager, 1.0)
# After completing store, the blocks becomes evictable as it is not actively used # After completing store, the chunks become evictable as not actively used
# and usage drops. # and usage drops.
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX) manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
check_usage_stats(manager, 0.5) check_usage_stats(manager, 0.5)
# After completing store, the blocks becomes evictable as it is not actively used # After completing store, the chunks become evictable as not actively used
# and usage drops. # and usage drops.
manager.complete_store(to_keys([3, 4]), _EMPTY_REQ_CTX) manager.complete_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
check_usage_stats(manager, 0.0) check_usage_stats(manager, 0.0)
def test_cpu_manager_reports_allocation_size_histogram(): def test_cpu_manager_reports_allocation_size_histogram():
manager = make_cpu_manager(num_blocks=4, cache_policy="lru") manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX) manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX) manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -297,12 +297,12 @@ def test_cpu_manager_reports_allocation_size_histogram():
def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch): def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
manager = make_cpu_manager(num_blocks=4, cache_policy="lru") manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
def fail_allocate_blocks(keys): def fail_allocate_chunks(keys):
raise RuntimeError("allocation failed") raise RuntimeError("allocation failed")
monkeypatch.setattr(manager, "_allocate_blocks", fail_allocate_blocks) monkeypatch.setattr(manager, "_allocate_chunks", fail_allocate_chunks)
with pytest.raises(RuntimeError, match="allocation failed"): with pytest.raises(RuntimeError, match="allocation failed"):
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX) manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
@@ -316,7 +316,7 @@ def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
def test_cpu_manager_reports_allocation_size_on_eviction_failure(): def test_cpu_manager_reports_allocation_size_on_eviction_failure():
manager = make_cpu_manager(num_blocks=1, cache_policy="lru") manager = make_cpu_manager(num_chunks=1, cache_policy="lru")
manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX) manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
manager.get_stats() manager.get_stats()
@@ -332,7 +332,7 @@ def test_cpu_manager_reports_allocation_size_on_eviction_failure():
def test_cpu_manager_reports_cache_write_and_read_usage_gauges(): def test_cpu_manager_reports_cache_write_and_read_usage_gauges():
manager = make_cpu_manager(num_blocks=4) manager = make_cpu_manager(num_chunks=4)
# Store path: pins write usage until complete_store. # Store path: pins write usage until complete_store.
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX) manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -356,7 +356,7 @@ def test_cpu_manager_reports_cache_write_and_read_usage_gauges():
def test_cpu_manager_clears_write_usage_after_failed_store(): def test_cpu_manager_clears_write_usage_after_failed_store():
manager = make_cpu_manager(num_blocks=4) manager = make_cpu_manager(num_chunks=4)
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX) manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
check_split_usage_stats(manager, write=0.5, read=0.0, total=0.5) check_split_usage_stats(manager, write=0.5, read=0.0, total=0.5)
@@ -369,8 +369,8 @@ def test_cpu_manager():
""" """
Tests CPUOffloadingManager with lru policy. Tests CPUOffloadingManager with lru policy.
""" """
# initialize a CPU manager with a capacity of 4 blocks # initialize a CPU manager with a capacity of 4 chunks
cpu_manager = make_cpu_manager(num_blocks=4, cache_policy="lru", enable_events=True) cpu_manager = make_cpu_manager(num_chunks=4, cache_policy="lru", enable_events=True)
# prepare store [1, 2] # prepare store [1, 2]
prepare_store_output = cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX) prepare_store_output = cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -378,7 +378,7 @@ def test_cpu_manager():
prepare_store_output, prepare_store_output,
ExpectedPrepareStoreOutput( ExpectedPrepareStoreOutput(
keys_to_store=[1, 2], keys_to_store=[1, 2],
store_block_ids=[0, 1], store_chunk_ids=[0, 1],
evicted_keys=[], evicted_keys=[],
), ),
) )
@@ -407,7 +407,7 @@ def test_cpu_manager():
prepare_store_output, prepare_store_output,
ExpectedPrepareStoreOutput( ExpectedPrepareStoreOutput(
keys_to_store=[3, 4, 5], keys_to_store=[3, 4, 5],
store_block_ids=[2, 3, 0], store_chunk_ids=[2, 3, 0],
evicted_keys=[1], evicted_keys=[1],
), ),
) )
@@ -445,7 +445,7 @@ def test_cpu_manager():
prepare_store_output, prepare_store_output,
ExpectedPrepareStoreOutput( ExpectedPrepareStoreOutput(
keys_to_store=[6, 7, 8], keys_to_store=[6, 7, 8],
store_block_ids=[1, 0, 3], store_chunk_ids=[1, 0, 3],
evicted_keys=[4, 5, 2], evicted_keys=[4, 5, 2],
), ),
) )
@@ -462,7 +462,7 @@ def test_cpu_manager():
prepare_store_output, prepare_store_output,
ExpectedPrepareStoreOutput( ExpectedPrepareStoreOutput(
keys_to_store=[9], keys_to_store=[9],
store_block_ids=[3], store_chunk_ids=[3],
evicted_keys=[8], evicted_keys=[8],
), ),
) )
@@ -482,38 +482,38 @@ def test_cpu_manager():
def test_prepare_load_preserves_key_order(): def test_prepare_load_preserves_key_order():
"""block_ids[i] must correspond to keys[i] (co-indexed invariant).""" """chunk_ids[i] must correspond to keys[i] (co-indexed invariant)."""
manager = make_cpu_manager(num_blocks=4, cache_policy="lru") manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
key_a, key_b, key_c = to_key(0), to_key(1), to_key(2) key_a, key_b, key_c = to_key(0), to_key(1), to_key(2)
# Store all three keys and learn their block ID assignments # Store all three keys and learn their chunk ID assignments
store_output = manager.prepare_store([key_a, key_b, key_c], _EMPTY_REQ_CTX) store_output = manager.prepare_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
assert store_output is not None assert store_output is not None
assert isinstance(store_output.store_spec, CPULoadStoreSpec) assert isinstance(store_output.store_spec, CPULoadStoreSpec)
key_to_block_id = { key_to_chunk_id = {
k: int(bid) k: int(cid)
for k, bid in zip(store_output.keys_to_store, store_output.store_spec.block_ids) for k, cid in zip(store_output.keys_to_store, store_output.store_spec.chunk_ids)
} }
manager.complete_store([key_a, key_b, key_c], _EMPTY_REQ_CTX) manager.complete_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
# Forward order: [a, b, c] # Forward order: [a, b, c]
spec_fwd = manager.prepare_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) spec_fwd = manager.prepare_load([key_a, key_b, key_c], _EMPTY_REQ_CTX)
assert isinstance(spec_fwd, CPULoadStoreSpec) assert isinstance(spec_fwd, CPULoadStoreSpec)
assert [int(x) for x in spec_fwd.block_ids] == [ assert [int(x) for x in spec_fwd.chunk_ids] == [
key_to_block_id[key_a], key_to_chunk_id[key_a],
key_to_block_id[key_b], key_to_chunk_id[key_b],
key_to_block_id[key_c], key_to_chunk_id[key_c],
] ]
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
# Arbitrary permutation: [b, c, a] # Arbitrary permutation: [b, c, a]
spec_perm = manager.prepare_load([key_b, key_c, key_a], _EMPTY_REQ_CTX) spec_perm = manager.prepare_load([key_b, key_c, key_a], _EMPTY_REQ_CTX)
assert isinstance(spec_perm, CPULoadStoreSpec) assert isinstance(spec_perm, CPULoadStoreSpec)
assert [int(x) for x in spec_perm.block_ids] == [ assert [int(x) for x in spec_perm.chunk_ids] == [
key_to_block_id[key_b], key_to_chunk_id[key_b],
key_to_block_id[key_c], key_to_chunk_id[key_c],
key_to_block_id[key_a], key_to_chunk_id[key_a],
] ]
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
@@ -522,10 +522,10 @@ class TestARCPolicy:
"""Unit tests for CPUOffloadingManager with ARC eviction policy.""" """Unit tests for CPUOffloadingManager with ARC eviction policy."""
def _make_manager( def _make_manager(
self, num_blocks: int = 4, enable_events: bool = True self, num_chunks: int = 4, enable_events: bool = True
) -> tuple[CPUOffloadingManager, ARCCachePolicy]: ) -> tuple[CPUOffloadingManager, ARCCachePolicy]:
manager = make_cpu_manager( manager = make_cpu_manager(
num_blocks=num_blocks, num_chunks=num_chunks,
cache_policy="arc", cache_policy="arc",
enable_events=enable_events, enable_events=enable_events,
) )
@@ -548,7 +548,7 @@ class TestARCPolicy:
prepare_store_output, prepare_store_output,
ExpectedPrepareStoreOutput( ExpectedPrepareStoreOutput(
keys_to_store=[1, 2], keys_to_store=[1, 2],
store_block_ids=[0, 1], store_chunk_ids=[0, 1],
evicted_keys=[], evicted_keys=[],
), ),
) )
@@ -569,36 +569,36 @@ class TestARCPolicy:
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.MISS assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.MISS
# blocks should be in T1 (recent) # chunks should be in T1 (recent)
assert len(arc_policy.t1) == 2 assert len(arc_policy.t1) == 2
assert len(arc_policy.t2) == 0 assert len(arc_policy.t2) == 0
def test_t1_to_t2_promotion(self): def test_t1_to_t2_promotion(self):
""" """
Tests that accessing a block in T1 promotes it to T2 (frequent). Tests that accessing a chunk in T1 promotes it to T2 (frequent).
This is a key feature of ARC's adaptive behavior. This is a key feature of ARC's adaptive behavior.
""" """
cpu_manager, arc_policy = self._make_manager(enable_events=False) cpu_manager, arc_policy = self._make_manager(enable_events=False)
# store and complete block 1 # store and complete chunk 1
cpu_manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX) cpu_manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1]), _EMPTY_REQ_CTX) cpu_manager.complete_store(to_keys([1]), _EMPTY_REQ_CTX)
# block 1 starts in T1 (recent) # chunk 1 starts in T1 (recent)
assert to_keys([1])[0] in arc_policy.t1 assert to_keys([1])[0] in arc_policy.t1
assert to_keys([1])[0] not in arc_policy.t2 assert to_keys([1])[0] not in arc_policy.t2
# touch block 1 (simulate second access) # touch chunk 1 (simulate second access)
cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX) cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
# block 1 should now be in T2 (frequent) # chunk 1 should now be in T2 (frequent)
assert to_keys([1])[0] not in arc_policy.t1 assert to_keys([1])[0] not in arc_policy.t1
assert to_keys([1])[0] in arc_policy.t2 assert to_keys([1])[0] in arc_policy.t2
def test_eviction_with_load(self): def test_eviction_with_load(self):
""" """
Tests ARC eviction behavior similar to LRU test. Tests ARC eviction behavior similar to LRU test.
Verifies that blocks being loaded (ref_cnt > 0) cannot be evicted. Verifies that chunks being loaded (ref_cnt > 0) cannot be evicted.
""" """
cpu_manager, _ = self._make_manager() cpu_manager, _ = self._make_manager()
@@ -610,7 +610,7 @@ class TestARCPolicy:
prepare_store_output, prepare_store_output,
ExpectedPrepareStoreOutput( ExpectedPrepareStoreOutput(
keys_to_store=[1, 2, 3, 4], keys_to_store=[1, 2, 3, 4],
store_block_ids=[0, 1, 2, 3], store_chunk_ids=[0, 1, 2, 3],
evicted_keys=[], evicted_keys=[],
), ),
) )
@@ -628,36 +628,36 @@ class TestARCPolicy:
cpu_manager.complete_load(to_keys([2, 3]), _EMPTY_REQ_CTX) cpu_manager.complete_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
# now prepare store [5, 6, 7] should succeed # now prepare store [5, 6, 7] should succeed
# ARC will evict blocks one at a time from T1 as needed # ARC will evict chunks one at a time from T1 as needed
prepare_store_output = cpu_manager.prepare_store( prepare_store_output = cpu_manager.prepare_store(
to_keys([5, 6, 7]), _EMPTY_REQ_CTX to_keys([5, 6, 7]), _EMPTY_REQ_CTX
) )
assert prepare_store_output is not None assert prepare_store_output is not None
# Should successfully evict enough blocks to make room (at least 1) # Should successfully evict enough chunks to make room (at least 1)
assert len(prepare_store_output.evicted_keys) >= 1 assert len(prepare_store_output.evicted_keys) >= 1
def test_adaptive_target(self): def test_adaptive_target(self):
""" """
Tests ARC's adaptive target adjustment via ghost lists. Tests ARC's adaptive target adjustment via ghost lists.
When a block in B1 (ghost list) is accessed, target_t1_size increases. When a chunk in B1 (ghost list) is accessed, target_t1_size increases.
When a block in B2 is accessed, target_t1_size decreases. When a chunk in B2 is accessed, target_t1_size decreases.
""" """
cpu_manager, arc_policy = self._make_manager(num_blocks=2, enable_events=False) cpu_manager, arc_policy = self._make_manager(num_chunks=2, enable_events=False)
# store blocks 1, 2 (fills cache) # store chunks 1, 2 (fills cache)
cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX) cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX) cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
initial_target = arc_policy.target_t1_size initial_target = arc_policy.target_t1_size
# store block 3, evicting block 1 (moves to B1 ghost list) # store chunk 3, evicting chunk 1 (moves to B1 ghost list)
cpu_manager.prepare_store(to_keys([3]), _EMPTY_REQ_CTX) cpu_manager.prepare_store(to_keys([3]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([3]), _EMPTY_REQ_CTX) cpu_manager.complete_store(to_keys([3]), _EMPTY_REQ_CTX)
# block 1 should be in B1 (ghost list) # chunk 1 should be in B1 (ghost list)
assert to_keys([1])[0] in arc_policy.b1 assert to_keys([1])[0] in arc_policy.b1
# touch block 1 (cache miss, but in B1) # touch chunk 1 (cache miss, but in B1)
# this should increase target_t1_size (favor recency) # this should increase target_t1_size (favor recency)
cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX) cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
@@ -671,11 +671,11 @@ class TestARCPolicy:
""" """
cpu_manager, arc_policy = self._make_manager(enable_events=False) cpu_manager, arc_policy = self._make_manager(enable_events=False)
# store blocks 1, 2, 3, 4 # store chunks 1, 2, 3, 4
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX) cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX) cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
# promote blocks 3, 4 to T2 by touching them # promote chunks 3, 4 to T2 by touching them
cpu_manager.touch(to_keys([3, 4]), _EMPTY_REQ_CTX) cpu_manager.touch(to_keys([3, 4]), _EMPTY_REQ_CTX)
# now: T1 = {1, 2}, T2 = {3, 4} # now: T1 = {1, 2}, T2 = {3, 4}
@@ -686,22 +686,22 @@ class TestARCPolicy:
# (when |T1| >= target, evict from T1) # (when |T1| >= target, evict from T1)
arc_policy.target_t1_size = 1 arc_policy.target_t1_size = 1
# store block 5, should evict from T1 (block 1, LRU in T1) # store chunk 5, should evict from T1 (chunk 1, LRU in T1)
output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX) output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
assert output is not None assert output is not None
assert to_keys([1]) == output.evicted_keys assert to_keys([1]) == output.evicted_keys
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX) cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX)
# block 1 should be in B1 (ghost list) # chunk 1 should be in B1 (ghost list)
assert to_keys([1])[0] in arc_policy.b1 assert to_keys([1])[0] in arc_policy.b1
# block 5 should be in T1 # chunk 5 should be in T1
assert to_keys([5])[0] in arc_policy.t1 assert to_keys([5])[0] in arc_policy.t1
def test_batch_eviction_scans_t1_and_t2_once(self): def test_batch_eviction_scans_t1_and_t2_once(self):
"""ARC batch eviction must preserve order without restarting scans.""" """ARC batch eviction must preserve order without restarting scans."""
cpu_manager, arc_policy = self._make_manager( cpu_manager, arc_policy = self._make_manager(
num_blocks=256, enable_events=False num_chunks=256, enable_events=False
) )
keys = to_keys(list(range(256))) keys = to_keys(list(range(256)))
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX) cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
@@ -721,13 +721,13 @@ class TestARCPolicy:
t2_order = list(arc_policy.t2) t2_order = list(arc_policy.t2)
expected_t1 = [ expected_t1 = [
key key
for key, block in arc_policy.t1.items() for key, chunk in arc_policy.t1.items()
if block.ref_cnt == 0 and key not in protected if chunk.ref_cnt == 0 and key not in protected
][:num_t1_evictions] ][:num_t1_evictions]
expected_t2 = [ expected_t2 = [
key key
for key, block in arc_policy.t2.items() for key, chunk in arc_policy.t2.items()
if block.ref_cnt == 0 and key not in protected if chunk.ref_cnt == 0 and key not in protected
][:num_t2_evictions] ][:num_t2_evictions]
expected_t1_scans = t1_order.index(expected_t1[-1]) + 1 expected_t1_scans = t1_order.index(expected_t1[-1]) + 1
expected_t2_scans = t2_order.index(expected_t2[-1]) + 1 expected_t2_scans = t2_order.index(expected_t2[-1]) + 1
@@ -746,7 +746,7 @@ class TestARCPolicy:
def test_batch_eviction_falls_back_after_t1_iterator_exhausted(self): def test_batch_eviction_falls_back_after_t1_iterator_exhausted(self):
"""An exhausted T1 scan must keep falling back to T2.""" """An exhausted T1 scan must keep falling back to T2."""
cpu_manager, arc_policy = self._make_manager(num_blocks=8, enable_events=False) cpu_manager, arc_policy = self._make_manager(num_chunks=8, enable_events=False)
keys = to_keys(list(range(8))) keys = to_keys(list(range(8)))
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX) cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX) cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
@@ -764,8 +764,8 @@ class TestARCPolicy:
# the target, so each remaining selection must retry T1 then use T2. # the target, so each remaining selection must retry T1 then use T2.
eligible_t1 = [ eligible_t1 = [
key key
for key, block in arc_policy.t1.items() for key, chunk in arc_policy.t1.items()
if block.ref_cnt == 0 and key not in protected if chunk.ref_cnt == 0 and key not in protected
] ]
assert eligible_t1 == t1_order[:1] assert eligible_t1 == t1_order[:1]
assert len(t1_order) - 1 >= int(arc_policy.target_t1_size) assert len(t1_order) - 1 >= int(arc_policy.target_t1_size)
@@ -839,7 +839,7 @@ class TestARCPolicy:
def test_batch_eviction_failure_is_atomic(self): def test_batch_eviction_failure_is_atomic(self):
"""Finding only some candidates must not partially evict the cache.""" """Finding only some candidates must not partially evict the cache."""
cpu_manager, arc_policy = self._make_manager(num_blocks=4, enable_events=False) cpu_manager, arc_policy = self._make_manager(num_chunks=4, enable_events=False)
keys = to_keys(list(range(4))) keys = to_keys(list(range(4)))
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX) cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX) cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
@@ -858,13 +858,13 @@ class TestARCPolicy:
Tests that ghost lists (B1, B2) don't grow unbounded. Tests that ghost lists (B1, B2) don't grow unbounded.
They should be capped at cache_capacity. They should be capped at cache_capacity.
""" """
cpu_manager, arc_policy = self._make_manager(num_blocks=2, enable_events=False) cpu_manager, arc_policy = self._make_manager(num_chunks=2, enable_events=False)
# fill cache with blocks 1, 2 # fill cache with chunks 1, 2
cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX) cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX) cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
# store many blocks to fill ghost lists # store many chunks to fill ghost lists
for i in range(3, 20): for i in range(3, 20):
cpu_manager.prepare_store(to_keys([i]), _EMPTY_REQ_CTX) cpu_manager.prepare_store(to_keys([i]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([i]), _EMPTY_REQ_CTX) cpu_manager.complete_store(to_keys([i]), _EMPTY_REQ_CTX)
@@ -880,7 +880,7 @@ class TestARCPolicy:
""" """
cpu_manager, arc_policy = self._make_manager() cpu_manager, arc_policy = self._make_manager()
# store blocks 1, 2, 3, 4 # store chunks 1, 2, 3, 4
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX) cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX) cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
@@ -895,13 +895,13 @@ class TestARCPolicy:
assert len(arc_policy.t1) == 1 assert len(arc_policy.t1) == 1
assert len(arc_policy.t2) == 3 assert len(arc_policy.t2) == 3
# store block 5, should evict from T1 (block 2, only one in T1) # store chunk 5, should evict from T1 (chunk 2, only one in T1)
prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX) prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
verify_store_output( verify_store_output(
prepare_store_output, prepare_store_output,
ExpectedPrepareStoreOutput( ExpectedPrepareStoreOutput(
keys_to_store=[5], keys_to_store=[5],
store_block_ids=[1], # reuses block 2's storage store_chunk_ids=[1], # reuses chunk 2's storage
evicted_keys=[2], evicted_keys=[2],
), ),
) )
@@ -913,11 +913,11 @@ class TestARCPolicy:
""" """
cpu_manager, arc_policy = self._make_manager() cpu_manager, arc_policy = self._make_manager()
# store blocks 1, 2, 3, 4 # store chunks 1, 2, 3, 4
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX) cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX) cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
# prepare store block 5 (will evict block 1) # prepare store chunk 5 (will evict chunk 1)
prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX) prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
assert prepare_store_output is not None assert prepare_store_output is not None
assert len(prepare_store_output.evicted_keys) == 1 assert len(prepare_store_output.evicted_keys) == 1
@@ -925,13 +925,13 @@ class TestARCPolicy:
# complete store with failure # complete store with failure
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX, success=False) cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX, success=False)
# block 5 should not be in cache # chunk 5 should not be in cache
assert cpu_manager.lookup(to_key(5), _EMPTY_REQ_CTX) is LookupResult.MISS assert cpu_manager.lookup(to_key(5), _EMPTY_REQ_CTX) is LookupResult.MISS
# block 5 should not be in T1 or T2 # chunk 5 should not be in T1 or T2
assert to_keys([5])[0] not in arc_policy.t1 assert to_keys([5])[0] not in arc_policy.t1
assert to_keys([5])[0] not in arc_policy.t2 assert to_keys([5])[0] not in arc_policy.t2
# evicted block should still be gone (in B1 ghost list) # evicted chunk should still be gone (in B1 ghost list)
evicted_hash = prepare_store_output.evicted_keys[0] evicted_hash = prepare_store_output.evicted_keys[0]
assert evicted_hash in arc_policy.b1 assert evicted_hash in arc_policy.b1
@@ -954,7 +954,7 @@ class TestARCPolicy:
assert len(prepare_store_output.evicted_keys) == 1 assert len(prepare_store_output.evicted_keys) == 1
cpu_manager.complete_store(to_keys([3, 4, 5]), _EMPTY_REQ_CTX) cpu_manager.complete_store(to_keys([3, 4, 5]), _EMPTY_REQ_CTX)
# promote some blocks to T2 # promote some chunks to T2
cpu_manager.touch(to_keys([2, 3]), _EMPTY_REQ_CTX) cpu_manager.touch(to_keys([2, 3]), _EMPTY_REQ_CTX)
# T1 has {4, 5}, T2 has {2, 3} # T1 has {4, 5}, T2 has {2, 3}
@@ -966,7 +966,7 @@ class TestARCPolicy:
assert prepare_store_output is not None assert prepare_store_output is not None
cpu_manager.complete_store(to_keys([6]), _EMPTY_REQ_CTX) cpu_manager.complete_store(to_keys([6]), _EMPTY_REQ_CTX)
# verify blocks 2, 3 (in T2) are still present # verify chunks 2, 3 (in T2) are still present
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.HIT assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.HIT
@@ -980,7 +980,7 @@ def test_filter_reused_manager():
Tests CPUOffloadingManager reuse filtering (store_threshold=2). Tests CPUOffloadingManager reuse filtering (store_threshold=2).
""" """
manager = make_cpu_manager( manager = make_cpu_manager(
num_blocks=4, num_chunks=4,
cache_policy="lru", cache_policy="lru",
enable_events=True, enable_events=True,
store_threshold=2, store_threshold=2,
@@ -1023,7 +1023,7 @@ def test_filter_reused_manager():
def test_filter_reused_manager_oversized_offer_makes_progress(): def test_filter_reused_manager_oversized_offer_makes_progress():
"""An offer larger than the tracker capacity must not churn forever.""" """An offer larger than the tracker capacity must not churn forever."""
manager = make_cpu_manager( manager = make_cpu_manager(
num_blocks=4, num_chunks=4,
cache_policy="lru", cache_policy="lru",
store_threshold=2, store_threshold=2,
max_tracker_size=3, max_tracker_size=3,
@@ -1042,69 +1042,69 @@ def test_filter_reused_manager_oversized_offer_makes_progress():
assert stored_keys == set(keys) assert stored_keys == set(keys)
def test_evictable_cache_block_count(): def test_evictable_cache_chunk_count():
""" """
Verifies _num_evictable_cache_blocks is maintained correctly through the Verifies _num_evictable_cache_chunks is maintained correctly through the
full store/load lifecycle, eviction, failed stores, concurrent loads, full store/load lifecycle, eviction, failed stores, concurrent loads,
reset_cache, and the early-exit fast path in prepare_store. reset_cache, and the early-exit fast path in prepare_store.
""" """
manager = make_cpu_manager(num_blocks=4, cache_policy="lru") manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
# Initially no blocks allocated. # Initially no chunks allocated.
assert manager._num_evictable_cache_blocks == 0 assert manager._num_evictable_cache_chunks == 0
# Initial cache state [x, x, x, x] # Initial cache state [x, x, x, x]
# We get 3 blocks from the cache. # We get 3 chunks from the cache.
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX) manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
# cache state [1', 2', 3', x] <- 1', 2', 3' are actively being used. # cache state [1', 2', 3', x] <- 1', 2', 3' are actively being used.
assert manager._num_evictable_cache_blocks == 0 assert manager._num_evictable_cache_chunks == 0
# Completing stores makes them idle. # Completing stores makes them idle.
manager.complete_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX) manager.complete_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
# cache state [1, 2, 3, x] <- 1, 2, 3 blocks are idle. # cache state [1, 2, 3, x] <- 1, 2, 3 chunks are idle.
assert manager._num_evictable_cache_blocks == 3 assert manager._num_evictable_cache_chunks == 3
# prepare_load pins a block: idle count decrements once even if the # prepare_load pins a chunk: idle count decrements once even if the
# same block is loaded by two concurrent callers. # same chunk is loaded by two concurrent callers.
manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX) manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX)
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle. # cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
assert manager._num_evictable_cache_blocks == 2 assert manager._num_evictable_cache_chunks == 2
manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX) # 2nd concurrent load manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX) # 2nd concurrent load
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle. # cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
assert manager._num_evictable_cache_blocks == 2 # no double-decrement assert manager._num_evictable_cache_chunks == 2 # no double-decrement
# First complete_load does not restore idle (ref_cnt still 1). # First complete_load does not restore idle (ref_cnt still 1).
manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX) manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX)
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle. # cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
assert manager._num_evictable_cache_blocks == 2 assert manager._num_evictable_cache_chunks == 2
# Second complete_load drops ref_cnt to 0 -> block becomes idle again. # Second complete_load drops ref_cnt to 0 -> chunk becomes idle again.
manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX) manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX)
# cache state [1, 2, 3, x] <- 1, 2, 3 blocks are idle. # cache state [1, 2, 3, x] <- 1, 2, 3 chunks are idle.
assert manager._num_evictable_cache_blocks == 3 assert manager._num_evictable_cache_chunks == 3
# Eviction decrements idle count. # Eviction decrements idle count.
# Cache has 3 stored blocks and 1 free slot. Storing 3 new keys needs 2 eviction. # Cache has 3 stored chunks and 1 free slot. Storing 3 new keys needs 2 eviction.
manager.prepare_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX) manager.prepare_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX)
# cache state [1, 4', 5', 6'] <- block 1 is idle # cache state [1, 4', 5', 6'] <- chunk 1 is idle
assert manager._num_evictable_cache_blocks == 1 assert manager._num_evictable_cache_chunks == 1
# Failed store does not increment idle count (block discarded from cache). # Failed store does not increment idle count (chunk discarded from cache).
manager.complete_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX, success=False) manager.complete_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX, success=False)
# cache state [1, x, x, x] <- block 1 is idle. Other returned to cache. # cache state [1, x, x, x] <- chunk 1 is idle. Other returned to cache.
assert manager._num_evictable_cache_blocks == 1 assert manager._num_evictable_cache_chunks == 1
# reset_cache zeroes the count unconditionally. # reset_cache zeroes the count unconditionally.
manager.reset_cache() manager.reset_cache()
# cache state [x, x, x, x] # cache state [x, x, x, x]
assert manager._num_evictable_cache_blocks == 0 assert manager._num_evictable_cache_chunks == 0
# setup 3 blocks with loads so idle count drops to 0. # setup 3 chunks with loads so idle count drops to 0.
manager.prepare_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX) manager.prepare_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX) manager.complete_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
manager.prepare_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX) manager.prepare_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
# cache state [10', 11', 12', x] # cache state [10', 11', 12', x]
assert manager._num_evictable_cache_blocks == 0 assert manager._num_evictable_cache_chunks == 0
# prepare_store requiring eviction must return None immediately (fast exit). # prepare_store requiring eviction must return None immediately (fast exit).
# Spy on policy.evict to confirm the fast path short-circuits before calling it. # Spy on policy.evict to confirm the fast path short-circuits before calling it.
@@ -1120,24 +1120,24 @@ def test_evictable_cache_block_count():
# cache state [10', 11', 12', x] <- cannot evict anything # cache state [10', 11', 12', x] <- cannot evict anything
assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is None assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is None
assert not evict_called, ( assert not evict_called, (
"_num_evictable_cache_blocks==0 should short-circuit before evict()" "_num_evictable_cache_chunks==0 should short-circuit before evict()"
) )
# After releasing the loads, eviction becomes possible again. # After releasing the loads, eviction becomes possible again.
manager.complete_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX) manager.complete_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
# cache state [10, 11, 12, x] <- 10, 11, 12 are idle # cache state [10, 11, 12, x] <- 10, 11, 12 are idle
assert manager._num_evictable_cache_blocks == 3 assert manager._num_evictable_cache_chunks == 3
assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is not None assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is not None
# cache state [10, 11, 14', 15'] <- 10, 11 are idle # cache state [10, 11, 14', 15'] <- 10, 11 are idle
assert manager._num_evictable_cache_blocks == 2 assert manager._num_evictable_cache_chunks == 2
manager.complete_store(to_keys([14, 15]), _EMPTY_REQ_CTX) manager.complete_store(to_keys([14, 15]), _EMPTY_REQ_CTX)
# cache state [10, 11, 14, 15] <- all blocks idle # cache state [10, 11, 14, 15] <- all chunks idle
assert manager._num_evictable_cache_blocks == 4 assert manager._num_evictable_cache_chunks == 4
def test_touch_forwards_req_context_to_policy(monkeypatch): def test_touch_forwards_req_context_to_policy(monkeypatch):
"""Regression: CPUOffloadingManager.touch forwards ReqContext to policy.""" """Regression: CPUOffloadingManager.touch forwards ReqContext to policy."""
manager = make_cpu_manager(num_blocks=4, cache_policy="lru") manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
received = [] received = []
def spy_touch(keys: Iterable[OffloadKey], req_context: ReqContext) -> None: def spy_touch(keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
@@ -38,7 +38,7 @@ def _set_spawn_method(monkeypatch):
def _make_region( def _make_region(
engine_id: str, engine_id: str,
num_blocks: int = 4, num_chunks: int = 4,
cpu_page_size: int = PAGE_SIZE, cpu_page_size: int = PAGE_SIZE,
num_workers: int = 1, num_workers: int = 1,
rank: int = 0, rank: int = 0,
@@ -47,9 +47,9 @@ def _make_region(
assert cpu_page_size % PAGE_SIZE == 0 assert cpu_page_size % PAGE_SIZE == 0
return SharedOffloadRegion( return SharedOffloadRegion(
engine_id=engine_id, engine_id=engine_id,
num_blocks=num_blocks, num_chunks=num_chunks,
rank=rank, rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size, kv_bytes_per_chunk=num_workers * cpu_page_size,
cpu_page_size=cpu_page_size, cpu_page_size=cpu_page_size,
barrier=barrier, barrier=barrier,
) )
@@ -99,16 +99,16 @@ def _region(engine_id: str, **kwargs):
def _multi_region( def _multi_region(
engine_id: str, engine_id: str,
num_workers: int, num_workers: int,
num_blocks: int = 4, num_chunks: int = 4,
cpu_page_size: int = PAGE_SIZE, cpu_page_size: int = PAGE_SIZE,
): ):
"""Context manager: create one SharedOffloadRegion per rank, clean up on exit.""" """Context manager: create one SharedOffloadRegion per rank, clean up on exit."""
regions = [ regions = [
SharedOffloadRegion( SharedOffloadRegion(
engine_id=engine_id, engine_id=engine_id,
num_blocks=num_blocks, num_chunks=num_chunks,
rank=rank, rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size, kv_bytes_per_chunk=num_workers * cpu_page_size,
cpu_page_size=cpu_page_size, cpu_page_size=cpu_page_size,
) )
for rank in range(num_workers) for rank in range(num_workers)
@@ -124,7 +124,7 @@ def _multi_region(
def _race_construct( def _race_construct(
engine_id: str, engine_id: str,
num_workers: int, num_workers: int,
num_blocks: int = 4, num_chunks: int = 4,
cpu_page_size: int = PAGE_SIZE, cpu_page_size: int = PAGE_SIZE,
) -> tuple[list[SharedOffloadRegion], list[Exception]]: ) -> tuple[list[SharedOffloadRegion], list[Exception]]:
"""Spawn num_workers threads that all race to construct SharedOffloadRegion.""" """Spawn num_workers threads that all race to construct SharedOffloadRegion."""
@@ -137,9 +137,9 @@ def _race_construct(
try: try:
regions[rank] = SharedOffloadRegion( regions[rank] = SharedOffloadRegion(
engine_id=engine_id, engine_id=engine_id,
num_blocks=num_blocks, num_chunks=num_chunks,
rank=rank, rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size, kv_bytes_per_chunk=num_workers * cpu_page_size,
cpu_page_size=cpu_page_size, cpu_page_size=cpu_page_size,
) )
except Exception as e: except Exception as e:
@@ -156,7 +156,7 @@ def _race_construct(
def _mp_race_construct_and_write( def _mp_race_construct_and_write(
engine_id: str, engine_id: str,
num_blocks: int, num_chunks: int,
rank: int, rank: int,
num_workers: int, num_workers: int,
cpu_page_size: int, cpu_page_size: int,
@@ -170,9 +170,9 @@ def _mp_race_construct_and_write(
try: try:
region = SharedOffloadRegion( region = SharedOffloadRegion(
engine_id=engine_id, engine_id=engine_id,
num_blocks=num_blocks, num_chunks=num_chunks,
rank=rank, rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size, kv_bytes_per_chunk=num_workers * cpu_page_size,
cpu_page_size=cpu_page_size, cpu_page_size=cpu_page_size,
) )
t = region.create_next_worker_view(cpu_page_size) t = region.create_next_worker_view(cpu_page_size)
@@ -198,9 +198,9 @@ def _mp_barrier_construct_and_hold(
try: try:
region = SharedOffloadRegion( region = SharedOffloadRegion(
engine_id=engine_id, engine_id=engine_id,
num_blocks=2, num_chunks=2,
rank=rank, rank=rank,
kv_bytes_per_block=num_workers * PAGE_SIZE, kv_bytes_per_chunk=num_workers * PAGE_SIZE,
cpu_page_size=PAGE_SIZE, cpu_page_size=PAGE_SIZE,
barrier=lambda: barrier.wait(30), barrier=lambda: barrier.wait(30),
) )
@@ -233,9 +233,9 @@ def iid():
def test_create_next_worker_view_shape_and_stride(iid): def test_create_next_worker_view_shape_and_stride(iid):
"""Returned tensor must have shape (num_blocks, tensor_page_size) and """Returned tensor must have shape (num_chunks, tensor_page_size) and
stride (row_stride, 1) where row_stride = cpu_page_size * num_workers.""" stride (row_stride, 1) where row_stride = cpu_page_size * num_workers."""
with _region(iid, num_blocks=4, cpu_page_size=2 * PAGE_SIZE) as r: with _region(iid, num_chunks=4, cpu_page_size=2 * PAGE_SIZE) as r:
t = r.create_next_worker_view(PAGE_SIZE) t = r.create_next_worker_view(PAGE_SIZE)
assert t.shape == (4, PAGE_SIZE) assert t.shape == (4, PAGE_SIZE)
# num_workers=1 → row_stride = cpu_page_size # num_workers=1 → row_stride = cpu_page_size
@@ -253,7 +253,7 @@ def test_create_next_worker_view_storage_offset_rank0(iid):
def test_create_next_worker_view_storage_offset_rank1(iid): def test_create_next_worker_view_storage_offset_rank1(iid):
"""rank=1 worker's first tensor must start cpu_page_size bytes into the mmap.""" """rank=1 worker's first tensor must start cpu_page_size bytes into the mmap."""
with _multi_region(iid, num_workers=2, num_blocks=4) as (r0, r1): with _multi_region(iid, num_workers=2, num_chunks=4) as (r0, r1):
t1 = r1.create_next_worker_view(PAGE_SIZE) t1 = r1.create_next_worker_view(PAGE_SIZE)
assert t1.data_ptr() == r1._base.data_ptr() + PAGE_SIZE assert t1.data_ptr() == r1._base.data_ptr() + PAGE_SIZE
del t1 del t1
@@ -261,7 +261,7 @@ def test_create_next_worker_view_storage_offset_rank1(iid):
def test_create_next_worker_view_row_stride_with_multiple_workers(iid): def test_create_next_worker_view_row_stride_with_multiple_workers(iid):
"""With num_workers=4, row_stride must be 4 * cpu_page_size.""" """With num_workers=4, row_stride must be 4 * cpu_page_size."""
with _region(iid, num_blocks=2, num_workers=4) as r: with _region(iid, num_chunks=2, num_workers=4) as r:
t = r.create_next_worker_view(PAGE_SIZE) t = r.create_next_worker_view(PAGE_SIZE)
assert t.stride(0) == 4 * PAGE_SIZE assert t.stride(0) == 4 * PAGE_SIZE
del t del t
@@ -332,12 +332,12 @@ def test_create_next_worker_view_overflow_does_not_mutate_cursor(iid):
def test_create_next_worker_view_write_visible_in_raw_mmap(iid): def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
"""Writes into a create_next_worker_view view must appear at the correct """Writes into a create_next_worker_view view must appear at the correct
raw mmap offset""" raw mmap offset"""
with _region(iid, num_blocks=4) as r: with _region(iid, num_chunks=4) as r:
t = r.create_next_worker_view(PAGE_SIZE) t = r.create_next_worker_view(PAGE_SIZE)
t[2, :] = 42 # write to block row 2 t[2, :] = 42 # write to chunk row 2
raw = memoryview(r.mmap_obj) raw = memoryview(r.mmap_obj)
# num_workers=1 → row_stride = PAGE_SIZE; block 2 starts at byte 2*PAGE_SIZE # num_workers=1 → row_stride = PAGE_SIZE; chunk 2 starts at byte 2*PAGE_SIZE
chunk = bytes(raw[2 * PAGE_SIZE : 3 * PAGE_SIZE]) chunk = bytes(raw[2 * PAGE_SIZE : 3 * PAGE_SIZE])
assert all(b == 42 for b in chunk) assert all(b == 42 for b in chunk)
del raw, t del raw, t
@@ -345,7 +345,7 @@ def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
def test_create_next_worker_view_multi_tensor_layout(iid): def test_create_next_worker_view_multi_tensor_layout(iid):
"""Two tensors from the same worker land at consecutive byte offsets per row.""" """Two tensors from the same worker land at consecutive byte offsets per row."""
with _region(iid, num_blocks=2, cpu_page_size=2 * PAGE_SIZE) as r: with _region(iid, num_chunks=2, cpu_page_size=2 * PAGE_SIZE) as r:
ta = r.create_next_worker_view(PAGE_SIZE) ta = r.create_next_worker_view(PAGE_SIZE)
tb = r.create_next_worker_view(PAGE_SIZE) tb = r.create_next_worker_view(PAGE_SIZE)
@@ -353,8 +353,8 @@ def test_create_next_worker_view_multi_tensor_layout(iid):
tb[:, :] = 2 tb[:, :] = 2
raw = memoryview(r.mmap_obj) raw = memoryview(r.mmap_obj)
for blk in range(2): for chunk in range(2):
row_offset = blk * 2 * PAGE_SIZE # num_workers=1 row_offset = chunk * 2 * PAGE_SIZE # num_workers=1
assert all(b == 1 for b in raw[row_offset : row_offset + PAGE_SIZE]) assert all(b == 1 for b in raw[row_offset : row_offset + PAGE_SIZE])
assert all( assert all(
b == 2 for b in raw[row_offset + PAGE_SIZE : row_offset + 2 * PAGE_SIZE] b == 2 for b in raw[row_offset + PAGE_SIZE : row_offset + 2 * PAGE_SIZE]
@@ -366,7 +366,7 @@ def test_create_next_worker_view_multiprocess_slots(iid):
"""Each worker process calls create_next_worker_view and writes distinct data; """Each worker process calls create_next_worker_view and writes distinct data;
the parent verifies each slot lands at the correct interleaved offset.""" the parent verifies each slot lands at the correct interleaved offset."""
num_workers = 2 num_workers = 2
num_blocks = 4 num_chunks = 4
ctx = get_mp_context() ctx = get_mp_context()
done_queue = ctx.Queue() done_queue = ctx.Queue()
@@ -375,9 +375,9 @@ def test_create_next_worker_view_multiprocess_slots(iid):
# Parent is rank 0 (creator); child is rank 1 (joiner). # Parent is rank 0 (creator); child is rank 1 (joiner).
region = SharedOffloadRegion( region = SharedOffloadRegion(
engine_id=iid, engine_id=iid,
num_blocks=num_blocks, num_chunks=num_chunks,
rank=0, rank=0,
kv_bytes_per_block=num_workers * PAGE_SIZE, kv_bytes_per_chunk=num_workers * PAGE_SIZE,
cpu_page_size=PAGE_SIZE, cpu_page_size=PAGE_SIZE,
) )
try: try:
@@ -385,7 +385,7 @@ def test_create_next_worker_view_multiprocess_slots(iid):
target=_mp_race_construct_and_write, target=_mp_race_construct_and_write,
args=( args=(
iid, iid,
num_blocks, num_chunks,
1, 1,
num_workers, num_workers,
PAGE_SIZE, PAGE_SIZE,
@@ -403,12 +403,12 @@ def test_create_next_worker_view_multiprocess_slots(iid):
assert result["error"] is None, result["error"] assert result["error"] is None, result["error"]
raw = memoryview(region.mmap_obj) raw = memoryview(region.mmap_obj)
for blk in range(num_blocks): for chunk in range(num_chunks):
row_start = blk * num_workers * PAGE_SIZE row_start = chunk * num_workers * PAGE_SIZE
w0 = bytes(raw[row_start : row_start + PAGE_SIZE]) w0 = bytes(raw[row_start : row_start + PAGE_SIZE])
w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE]) w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE])
assert all(b == 11 for b in w0), f"block {blk}: rank0 slot wrong" assert all(b == 11 for b in w0), f"chunk {chunk}: rank0 slot wrong"
assert all(b == 22 for b in w1), f"block {blk}: rank1 slot wrong" assert all(b == 22 for b in w1), f"chunk {chunk}: rank1 slot wrong"
del raw, t0 # release before finally triggers cleanup del raw, t0 # release before finally triggers cleanup
cleanup_queue.put(True) cleanup_queue.put(True)
@@ -422,8 +422,8 @@ def test_create_next_worker_view_multiprocess_slots(iid):
def test_create_next_worker_view_worker_isolation(iid): def test_create_next_worker_view_worker_isolation(iid):
"""Writes by worker 0 must not affect worker 1's slot and vice versa.""" """Writes by worker 0 must not affect worker 1's slot and vice versa."""
num_workers = 2 num_workers = 2
num_blocks = 4 num_chunks = 4
with _multi_region(iid, num_workers=num_workers, num_blocks=num_blocks) as regions: with _multi_region(iid, num_workers=num_workers, num_chunks=num_chunks) as regions:
t0 = regions[0].create_next_worker_view(PAGE_SIZE) t0 = regions[0].create_next_worker_view(PAGE_SIZE)
t1 = regions[1].create_next_worker_view(PAGE_SIZE) t1 = regions[1].create_next_worker_view(PAGE_SIZE)
@@ -431,12 +431,12 @@ def test_create_next_worker_view_worker_isolation(iid):
t1[:, :] = 22 t1[:, :] = 22
raw = memoryview(regions[0].mmap_obj) raw = memoryview(regions[0].mmap_obj)
for blk in range(num_blocks): for chunk in range(num_chunks):
row_start = blk * num_workers * PAGE_SIZE row_start = chunk * num_workers * PAGE_SIZE
w0 = bytes(raw[row_start : row_start + PAGE_SIZE]) w0 = bytes(raw[row_start : row_start + PAGE_SIZE])
w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE]) w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE])
assert all(b == 11 for b in w0), f"block {blk}: worker0 slot corrupted" assert all(b == 11 for b in w0), f"chunk {chunk}: worker0 slot corrupted"
assert all(b == 22 for b in w1), f"block {blk}: worker1 slot corrupted" assert all(b == 22 for b in w1), f"chunk {chunk}: worker1 slot corrupted"
del raw, t0, t1 # release before finally triggers cleanup del raw, t0, t1 # release before finally triggers cleanup
@@ -466,7 +466,7 @@ def test_file_exists_after_construction(iid):
def test_file_has_correct_size(iid): def test_file_has_correct_size(iid):
"""The mmap file size on disk must equal total_size_bytes.""" """The mmap file size on disk must equal total_size_bytes."""
with _region(iid, num_blocks=4) as r: with _region(iid, num_chunks=4) as r:
assert os.path.getsize(r.mmap_path) == 4 * PAGE_SIZE assert os.path.getsize(r.mmap_path) == 4 * PAGE_SIZE
@@ -491,11 +491,11 @@ def test_madvise_success_selects_madvise_population(iid, monkeypatch):
monkeypatch.setattr(sor, "_madvise_populate_write", _spy_madvise) monkeypatch.setattr(sor, "_madvise_populate_write", _spy_madvise)
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback) monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
num_blocks = 3 num_chunks = 3
num_workers = 2 num_workers = 2
with _region(iid, num_blocks=num_blocks, num_workers=num_workers, rank=1): with _region(iid, num_chunks=num_chunks, num_workers=num_workers, rank=1):
# 1 probe (PAGESIZE) + N populate calls (one per block per worker column). # 1 probe (PAGESIZE) + N populate calls (one per chunk per worker column).
expected_populate = num_blocks # ranked path: one call per block expected_populate = num_chunks # ranked path: one call per chunk
mmap_id = madvise_calls[0][2] mmap_id = madvise_calls[0][2]
assert madvise_calls == [ assert madvise_calls == [
(0, mmap.PAGESIZE, mmap_id), (0, mmap.PAGESIZE, mmap_id),
@@ -510,7 +510,7 @@ def test_madvise_success_selects_madvise_population(iid, monkeypatch):
def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch): def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
"""An EINVAL probe must select fallback for every ranked block.""" """An EINVAL probe must select fallback for every ranked chunk."""
from vllm.v1.kv_offload.cpu import shared_offload_region as sor from vllm.v1.kv_offload.cpu import shared_offload_region as sor
fallback_calls: list[tuple[int, int]] = [] fallback_calls: list[tuple[int, int]] = []
@@ -526,7 +526,7 @@ def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval) monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback) monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
with _region(iid, num_blocks=3, num_workers=2, rank=1): with _region(iid, num_chunks=3, num_workers=2, rank=1):
assert fallback_calls == [ assert fallback_calls == [
(mmap.PAGESIZE, mmap.PAGESIZE), (mmap.PAGESIZE, mmap.PAGESIZE),
(3 * mmap.PAGESIZE, mmap.PAGESIZE), (3 * mmap.PAGESIZE, mmap.PAGESIZE),
@@ -551,7 +551,7 @@ def test_madvise_einval_selects_fallback_for_unranked_region(iid, monkeypatch):
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval) monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback) monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
with _region(iid, num_blocks=3, num_workers=2, rank=None): with _region(iid, num_chunks=3, num_workers=2, rank=None):
assert fallback_calls == [(0, 6 * mmap.PAGESIZE)] assert fallback_calls == [(0, 6 * mmap.PAGESIZE)]
@@ -651,7 +651,7 @@ def test_multiprocess_race_construct_and_write(iid):
"""N processes race to construct the same SharedOffloadRegion, each writes """N processes race to construct the same SharedOffloadRegion, each writes
fill_value = rank+1 into their slot; parent verifies interleaved layout.""" fill_value = rank+1 into their slot; parent verifies interleaved layout."""
num_workers = 4 num_workers = 4
num_blocks = 3 num_chunks = 3
ctx = get_mp_context() ctx = get_mp_context()
done_queue = ctx.Queue() done_queue = ctx.Queue()
@@ -662,7 +662,7 @@ def test_multiprocess_race_construct_and_write(iid):
target=_mp_race_construct_and_write, target=_mp_race_construct_and_write,
args=( args=(
iid, iid,
num_blocks, num_chunks,
rank, rank,
num_workers, num_workers,
PAGE_SIZE, PAGE_SIZE,
@@ -689,13 +689,13 @@ def test_multiprocess_race_construct_and_write(iid):
with open(mmap_path, "rb") as f: with open(mmap_path, "rb") as f:
raw = f.read() raw = f.read()
for blk in range(num_blocks): for chunk in range(num_chunks):
for w in range(num_workers): for w in range(num_workers):
slot_start = (blk * num_workers + w) * PAGE_SIZE slot_start = (chunk * num_workers + w) * PAGE_SIZE
slot = raw[slot_start : slot_start + PAGE_SIZE] slot = raw[slot_start : slot_start + PAGE_SIZE]
expected = w + 1 # fill_value = rank + 1 expected = w + 1 # fill_value = rank + 1
assert all(b == expected for b in slot), ( assert all(b == expected for b in slot), (
f"block {blk}, worker {w}: expected {expected} but got wrong bytes" f"chunk {chunk}, worker {w}: expected {expected} but got wrong bytes"
) )
# Unblock all workers to clean up. # Unblock all workers to clean up.
@@ -839,9 +839,9 @@ def test_insufficient_space_raises_clear_error(monkeypatch):
with pytest.raises(RuntimeError, match="Insufficient space"): with pytest.raises(RuntimeError, match="Insufficient space"):
SharedOffloadRegion( SharedOffloadRegion(
engine_id=engine_id, engine_id=engine_id,
num_blocks=4, num_chunks=4,
rank=0, rank=0,
kv_bytes_per_block=PAGE_SIZE, kv_bytes_per_chunk=PAGE_SIZE,
cpu_page_size=PAGE_SIZE, cpu_page_size=PAGE_SIZE,
) )
@@ -870,9 +870,9 @@ def test_ftruncate_failure_cleans_up_creator(monkeypatch):
with pytest.raises(OSError, match="ftruncate failed"): with pytest.raises(OSError, match="ftruncate failed"):
SharedOffloadRegion( SharedOffloadRegion(
engine_id=engine_id, engine_id=engine_id,
num_blocks=4, num_chunks=4,
rank=0, rank=0,
kv_bytes_per_block=PAGE_SIZE, kv_bytes_per_chunk=PAGE_SIZE,
cpu_page_size=PAGE_SIZE, cpu_page_size=PAGE_SIZE,
) )
+11 -11
View File
@@ -136,7 +136,7 @@ def test_get_spec_cls_defaults_to_cpu():
def test_create_cpu_offloading_spec(): def test_create_cpu_offloading_spec():
spec = _create_spec() spec = _create_spec()
assert isinstance(spec, CPUOffloadingSpec) assert isinstance(spec, CPUOffloadingSpec)
assert spec.num_blocks > 0 assert spec.num_chunks > 0
def test_cpu_spec_sizes_normalized_worker_layout(): def test_cpu_spec_sizes_normalized_worker_layout():
@@ -156,7 +156,7 @@ def test_cpu_spec_sizes_normalized_worker_layout():
assert isinstance(spec, CPUOffloadingSpec) assert isinstance(spec, CPUOffloadingSpec)
assert spec.cpu_page_size_per_worker == 32 assert spec.cpu_page_size_per_worker == 32
assert spec.kv_bytes_per_chunk == alignment assert spec.kv_bytes_per_chunk == alignment
assert spec.num_blocks == 3 assert spec.num_chunks == 3
def test_cpu_spec_zero_worker_bytes_produces_empty_cache(): def test_cpu_spec_zero_worker_bytes_produces_empty_cache():
@@ -165,7 +165,7 @@ def test_cpu_spec_zero_worker_bytes_produces_empty_cache():
assert isinstance(spec, CPUOffloadingSpec) assert isinstance(spec, CPUOffloadingSpec)
assert spec.cpu_page_size_per_worker == 0 assert spec.cpu_page_size_per_worker == 0
assert spec.kv_bytes_per_chunk == 0 assert spec.kv_bytes_per_chunk == 0
assert spec.num_blocks == 0 assert spec.num_chunks == 0
def test_tiering_spec_aligns_row_size(): def test_tiering_spec_aligns_row_size():
@@ -183,7 +183,7 @@ def test_tiering_spec_aligns_row_size():
assert isinstance(spec, TieringOffloadingSpec) assert isinstance(spec, TieringOffloadingSpec)
assert spec.cpu_page_size_per_worker == 32 assert spec.cpu_page_size_per_worker == 32
assert spec.kv_bytes_per_chunk == alignment assert spec.kv_bytes_per_chunk == alignment
assert spec.num_blocks == 3 assert spec.num_chunks == 3
@pytest.mark.parametrize("world_size", [2, 4, 8]) @pytest.mark.parametrize("world_size", [2, 4, 8])
@@ -201,7 +201,7 @@ def test_tiering_spec_replicated_sizing_removes_world_factor(world_size: int):
assert spec.replicated_layout is True assert spec.replicated_layout is True
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block
assert spec.num_blocks == 8 assert spec.num_chunks == 8
def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monkeypatch): def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monkeypatch):
@@ -239,7 +239,7 @@ def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monke
spec.create_worker(kv_caches) spec.create_worker(kv_caches)
assert region_calls[0]["rank"] == 0 assert region_calls[0]["rank"] == 0
assert region_calls[0]["kv_bytes_per_block"] == worker_kv_bytes_per_block assert region_calls[0]["kv_bytes_per_chunk"] == worker_kv_bytes_per_block
assert worker_calls[0]["kv_caches"] is kv_caches assert worker_calls[0]["kv_caches"] is kv_caches
assert worker_calls[0]["mmap_region"] is region assert worker_calls[0]["mmap_region"] is region
@@ -292,7 +292,7 @@ def test_cpu_spec_replicated_sizing_on_shared_region(monkeypatch, world_size: in
assert spec.replicated_layout is True assert spec.replicated_layout is True
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block
assert spec.num_blocks == 8 assert spec.num_chunks == 8
@pytest.mark.parametrize("world_size", [2, 4, 8]) @pytest.mark.parametrize("world_size", [2, 4, 8])
@@ -320,7 +320,7 @@ def test_cpu_spec_replicated_disabled_without_shared_region(
assert spec.replicated_layout is False assert spec.replicated_layout is False
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block * world_size assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block * world_size
assert spec.num_blocks == 2 assert spec.num_chunks == 2
@pytest.mark.parametrize("config_replicated", [True, False]) @pytest.mark.parametrize("config_replicated", [True, False])
@@ -381,7 +381,7 @@ def test_cpu_spec_create_worker_uses_mmap_on_cuda_alike(monkeypatch):
spec.create_worker(kv_caches) spec.create_worker(kv_caches)
assert region_calls[0]["engine_id"] == "test-engine" assert region_calls[0]["engine_id"] == "test-engine"
assert region_calls[0]["kv_bytes_per_block"] == worker_kv_bytes_per_block * 4 assert region_calls[0]["kv_bytes_per_chunk"] == worker_kv_bytes_per_block * 4
assert worker_calls[0]["kv_caches"] is kv_caches assert worker_calls[0]["kv_caches"] is kv_caches
assert worker_calls[0]["mmap_region"] is region assert worker_calls[0]["mmap_region"] is region
@@ -419,11 +419,11 @@ def test_cpu_spec_create_worker_uses_tensor_path_off_cuda_alike(monkeypatch):
def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch): def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch):
import vllm.v1.kv_offload.cpu.spec as cpu_spec_module import vllm.v1.kv_offload.cpu.spec as cpu_spec_module
# worker_kv_bytes_per_block=0 yields num_blocks=0; a zero-byte region cannot # worker_kv_bytes_per_block=0 yields num_chunks=0; a zero-byte region cannot
# be mmap'd, so even on CUDA-alike this must fall back to the tensor path. # be mmap'd, so even on CUDA-alike this must fall back to the tensor path.
spec = _create_spec(worker_kv_bytes_per_block=0, world_size=4) spec = _create_spec(worker_kv_bytes_per_block=0, world_size=4)
assert isinstance(spec, CPUOffloadingSpec) assert isinstance(spec, CPUOffloadingSpec)
assert spec.num_blocks == 0 assert spec.num_chunks == 0
region_calls: list[dict[str, Any]] = [] region_calls: list[dict[str, Any]] = []
worker_calls: list[dict[str, Any]] = [] worker_calls: list[dict[str, Any]] = []
+17 -17
View File
@@ -91,17 +91,17 @@ def _req_context(kv_params: dict | None = None) -> ReqContext:
def _job_metadata( def _job_metadata(
job_id: int, job_id: int,
keys: list[bytes] | None = None, keys: list[bytes] | None = None,
block_ids: list[int] | None = None, chunk_ids: list[int] | None = None,
kv_params: dict | None = None, kv_params: dict | None = None,
) -> TransferJob: ) -> TransferJob:
if keys is None: if keys is None:
keys = [b"key1"] keys = [b"key1"]
if block_ids is None: if chunk_ids is None:
block_ids = list(range(len(keys))) chunk_ids = list(range(len(keys)))
return TransferJob( return TransferJob(
job_id=job_id, job_id=job_id,
keys=keys, keys=keys,
block_ids=np.array(block_ids), chunk_ids=np.array(chunk_ids),
is_promotion=False, is_promotion=False,
req_context=_req_context(kv_params), req_context=_req_context(kv_params),
) )
@@ -254,9 +254,9 @@ class TestLookup:
"kv_params,expected", "kv_params,expected",
[ [
(_remote_decoder_kv_params(), OffloadPolicy.REQUEST_LEVEL), (_remote_decoder_kv_params(), OffloadPolicy.REQUEST_LEVEL),
(None, OffloadPolicy.BLOCK_LEVEL), (None, OffloadPolicy.CHUNK_LEVEL),
(_remote_prefiller_kv_params(), OffloadPolicy.BLOCK_LEVEL), (_remote_prefiller_kv_params(), OffloadPolicy.CHUNK_LEVEL),
({"remote_decoder": {}}, OffloadPolicy.BLOCK_LEVEL), ({"remote_decoder": {}}, OffloadPolicy.CHUNK_LEVEL),
], ],
ids=["producer", "plain", "consumer", "producer_no_id"], ids=["producer", "plain", "consumer", "producer_no_id"],
) )
@@ -366,7 +366,7 @@ class TestSubmitStore:
job = _job_metadata( job = _job_metadata(
job_id=1, job_id=1,
keys=[b"k1", b"k2"], keys=[b"k1", b"k2"],
block_ids=[3, 4], chunk_ids=[3, 4],
kv_params=_remote_decoder_kv_params(kv_request_id="req-1"), kv_params=_remote_decoder_kv_params(kv_request_id="req-1"),
) )
mgr.submit_store(job) mgr.submit_store(job)
@@ -397,7 +397,7 @@ class TestSubmitStore:
job = _job_metadata( job = _job_metadata(
job_id=7, job_id=7,
keys=[b"k1", b"k2"], keys=[b"k1", b"k2"],
block_ids=[3, 4], chunk_ids=[3, 4],
kv_params=_remote_decoder_kv_params(kv_request_id="req-1"), kv_params=_remote_decoder_kv_params(kv_request_id="req-1"),
) )
mgr.submit_store(job) mgr.submit_store(job)
@@ -440,7 +440,7 @@ class TestSubmitLoad:
"""Empty key list succeeds immediately.""" """Empty key list succeeds immediately."""
mgr = _make_manager() mgr = _make_manager()
job = _job_metadata( job = _job_metadata(
job_id=1, keys=[], block_ids=[], kv_params=_remote_prefiller_kv_params() job_id=1, keys=[], chunk_ids=[], kv_params=_remote_prefiller_kv_params()
) )
mgr.submit_load(job) mgr.submit_load(job)
assert mgr._finished_jobs == [JobResult(job_id=1, success=True)] assert mgr._finished_jobs == [JobResult(job_id=1, success=True)]
@@ -463,7 +463,7 @@ class TestSubmitLoad:
job = _job_metadata( job = _job_metadata(
job_id=42, job_id=42,
keys=[b"k1", b"k2"], keys=[b"k1", b"k2"],
block_ids=[5, 6], chunk_ids=[5, 6],
kv_params=_remote_prefiller_kv_params(kv_request_id="req-42"), kv_params=_remote_prefiller_kv_params(kv_request_id="req-42"),
) )
mgr.submit_load(job) mgr.submit_load(job)
@@ -1155,7 +1155,7 @@ class TestBidirectionalManager:
_job_metadata( _job_metadata(
job_id=100, job_id=100,
keys=[b"a-block"], keys=[b"a-block"],
block_ids=[0], chunk_ids=[0],
kv_params=a_prefiller_params, kv_params=a_prefiller_params,
) )
) )
@@ -1163,7 +1163,7 @@ class TestBidirectionalManager:
_job_metadata( _job_metadata(
job_id=200, job_id=200,
keys=[b"b-block"], keys=[b"b-block"],
block_ids=[0], chunk_ids=[0],
kv_params=b_prefiller_params, kv_params=b_prefiller_params,
) )
) )
@@ -1173,7 +1173,7 @@ class TestBidirectionalManager:
_job_metadata( _job_metadata(
job_id=101, job_id=101,
keys=[b"b-block"], keys=[b"b-block"],
block_ids=[0], chunk_ids=[0],
kv_params=a_decoder_params, kv_params=a_decoder_params,
) )
) )
@@ -1181,7 +1181,7 @@ class TestBidirectionalManager:
_job_metadata( _job_metadata(
job_id=201, job_id=201,
keys=[b"a-block"], keys=[b"a-block"],
block_ids=[0], chunk_ids=[0],
kv_params=b_decoder_params, kv_params=b_decoder_params,
) )
) )
@@ -1573,7 +1573,7 @@ class TestConnectionDeathMidTransfer:
_job_metadata( _job_metadata(
job_id=900, job_id=900,
keys=[b"a-block"], keys=[b"a-block"],
block_ids=[0], chunk_ids=[0],
kv_params=a_prefiller_params, kv_params=a_prefiller_params,
) )
) )
@@ -1581,7 +1581,7 @@ class TestConnectionDeathMidTransfer:
_job_metadata( _job_metadata(
job_id=901, job_id=901,
keys=[b"b-block"], keys=[b"b-block"],
block_ids=[0], chunk_ids=[0],
kv_params=a_decoder_params, kv_params=a_decoder_params,
) )
) )
@@ -259,13 +259,13 @@ class FakeParent:
) -> TransferJob: ) -> TransferJob:
keys_list = list(keys) keys_list = list(keys)
self.calls.append(("create_store_job", tuple(keys_list), ctx.req_id)) self.calls.append(("create_store_job", tuple(keys_list), ctx.req_id))
block_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32) chunk_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32)
job_id = self._next_job_id job_id = self._next_job_id
self._next_job_id += 1 self._next_job_id += 1
return TransferJob( return TransferJob(
job_id=job_id, job_id=job_id,
keys=keys_list, keys=keys_list,
block_ids=block_ids, chunk_ids=chunk_ids,
is_promotion=False, is_promotion=False,
req_context=ctx, req_context=ctx,
) )
+13 -13
View File
@@ -105,15 +105,15 @@ def key(n: int) -> OffloadKey:
def make_job( def make_job(
job_id: int, job_id: int,
keys: list[OffloadKey], keys: list[OffloadKey],
block_ids: list[int] | None = None, chunk_ids: list[int] | None = None,
is_promotion: bool = False, is_promotion: bool = False,
) -> TransferJob: ) -> TransferJob:
if block_ids is None: if chunk_ids is None:
block_ids = list(range(len(keys))) chunk_ids = list(range(len(keys)))
return TransferJob( return TransferJob(
job_id=job_id, job_id=job_id,
keys=keys, keys=keys,
block_ids=np.array(block_ids, dtype=np.int64), chunk_ids=np.array(chunk_ids, dtype=np.int64),
is_promotion=is_promotion, is_promotion=is_promotion,
req_context=_CTX, req_context=_CTX,
) )
@@ -382,11 +382,11 @@ def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size):
tensor[:] = _page_aligned_rand_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS) tensor[:] = _page_aligned_rand_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS)
keys = [key(i) for i in range(batch_size)] keys = [key(i) for i in range(batch_size)]
store_block_ids = list(range(batch_size)) store_chunk_ids = list(range(batch_size))
load_block_ids = list(range(_NUM_BLOCKS - batch_size, _NUM_BLOCKS)) load_chunk_ids = list(range(_NUM_BLOCKS - batch_size, _NUM_BLOCKS))
expected = tensor[:batch_size].clone() expected = tensor[:batch_size].clone()
tier.submit_store(make_job(1, keys, store_block_ids)) tier.submit_store(make_job(1, keys, store_chunk_ids))
store_results = drain(tier) store_results = drain(tier)
assert len(store_results) == 1 assert len(store_results) == 1
assert store_results[0].success assert store_results[0].success
@@ -396,15 +396,15 @@ def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size):
tensor[:] = 0.0 tensor[:] = 0.0
# Load into a range disjoint by index from the store ids, to also # Load into a range disjoint by index from the store ids, to also
# exercise loading a block into a different id than it was stored from. # exercise loading a chunk into a different id than it was stored from.
tier.submit_load(make_job(2, keys, load_block_ids, is_promotion=True)) tier.submit_load(make_job(2, keys, load_chunk_ids, is_promotion=True))
load_results = drain(tier) load_results = drain(tier)
assert len(load_results) == 1 assert len(load_results) == 1
assert load_results[0].success assert load_results[0].success
for i, bid in enumerate(load_block_ids): for i, cid in enumerate(load_chunk_ids):
assert torch.allclose(tensor[bid], expected[i]), ( assert torch.allclose(tensor[cid], expected[i]), (
f"Block {bid} data mismatch after store+load" f"Chunk {cid} data mismatch after store+load"
) )
@@ -895,7 +895,7 @@ def test_cascade_store_emits_fs_event_through_tiering_manager(tmp_path):
view = memoryview(tensor.numpy()) view = memoryview(tensor.numpy())
mock_region = MagicMock() mock_region = MagicMock()
mock_region.create_kv_memoryview.return_value = view mock_region.create_kv_memoryview.return_value = view
primary = CPUPrimaryTierOffloadingManager(num_blocks=4, mmap_region=mock_region) primary = CPUPrimaryTierOffloadingManager(num_chunks=4, mmap_region=mock_region)
tier = FileSystemTierManager( tier = FileSystemTierManager(
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True), offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
primary_kv_view=primary.get_kv_memoryview(), primary_kv_view=primary.get_kv_memoryview(),
+22 -22
View File
@@ -25,8 +25,8 @@ def to_keys(int_ids: Iterable[int]):
def test_tiering_metrics_tracker_records_lookup_metrics(): def test_tiering_metrics_tracker_records_lookup_metrics():
tracker = TieringMetricsTracker( tracker = TieringMetricsTracker(
tier_types=["fs", "p2p"], tier_types=["fs", "p2p"],
num_primary_blocks=5, num_primary_chunks=5,
primary_block_size=16, primary_chunk_size=16,
) )
tracker.on_new_request(_CTX) tracker.on_new_request(_CTX)
@@ -45,18 +45,18 @@ def test_tiering_metrics_tracker_records_lookup_metrics():
stats = tracker.take_stats() stats = tracker.take_stats()
assert stats is not None assert stats is not None
values = stats.data["data"] values = stats.data["data"]
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1 assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("0:primary",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1 assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("1:fs",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("2:p2p",)] == 1 assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("2:p2p",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_HITS][("2:p2p",)] == 1 assert values[TieringOffloadingMetrics.CHUNK_HITS][("2:p2p",)] == 1
assert ("1:fs",) not in values[TieringOffloadingMetrics.BLOCK_HITS] assert ("1:fs",) not in values[TieringOffloadingMetrics.CHUNK_HITS]
def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation(): def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
tracker = TieringMetricsTracker( tracker = TieringMetricsTracker(
tier_types=["fs"], tier_types=["fs"],
num_primary_blocks=5, num_primary_chunks=5,
primary_block_size=16, primary_chunk_size=16,
) )
tracker.on_new_request(_CTX) tracker.on_new_request(_CTX)
key = to_keys([1])[0] key = to_keys([1])[0]
@@ -73,9 +73,9 @@ def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
stats = tracker.take_stats() stats = tracker.take_stats()
assert stats is not None assert stats is not None
values = stats.data["data"] values = stats.data["data"]
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1 assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("0:primary",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1 assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("1:fs",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_HITS][("1:fs",)] == 1 assert values[TieringOffloadingMetrics.CHUNK_HITS][("1:fs",)] == 1
tracker.on_request_allocated(_CTX) tracker.on_request_allocated(_CTX)
tracker.on_lookup( tracker.on_lookup(
@@ -90,15 +90,15 @@ def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
stats = tracker.take_stats() stats = tracker.take_stats()
assert stats is not None assert stats is not None
values = stats.data["data"] values = stats.data["data"]
assert TieringOffloadingMetrics.BLOCK_QUERIES not in values assert TieringOffloadingMetrics.CHUNK_QUERIES not in values
assert TieringOffloadingMetrics.BLOCK_HITS not in values assert TieringOffloadingMetrics.CHUNK_HITS not in values
def test_tiering_metrics_tracker_records_finished_job_metrics(): def test_tiering_metrics_tracker_records_finished_job_metrics():
tracker = TieringMetricsTracker( tracker = TieringMetricsTracker(
tier_types=["fs"], tier_types=["fs"],
num_primary_blocks=5, num_primary_chunks=5,
primary_block_size=16, primary_chunk_size=16,
) )
cascade_key, promotion_key, failed_cascade_key, failed_promotion_key = to_keys( cascade_key, promotion_key, failed_cascade_key, failed_promotion_key = to_keys(
range(4) range(4)
@@ -149,8 +149,8 @@ def test_tiering_metrics_tracker_records_finished_job_metrics():
def test_tiering_metrics_tracker_records_partial_promotion_success_bytes(): def test_tiering_metrics_tracker_records_partial_promotion_success_bytes():
tracker = TieringMetricsTracker( tracker = TieringMetricsTracker(
tier_types=["fs"], tier_types=["fs"],
num_primary_blocks=5, num_primary_chunks=5,
primary_block_size=16, primary_chunk_size=16,
) )
keys = to_keys(range(3)) keys = to_keys(range(3))
job = JobMetadata( job = JobMetadata(
@@ -182,8 +182,8 @@ def test_tiering_metrics_tracker_records_partial_promotion_success_bytes():
def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges(): def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges():
tracker = TieringMetricsTracker( tracker = TieringMetricsTracker(
tier_types=["fs", "p2p"], tier_types=["fs", "p2p"],
num_primary_blocks=6, num_primary_chunks=6,
primary_block_size=16, primary_chunk_size=16,
) )
fs_job = JobMetadata( fs_job = JobMetadata(
TransferJob(0, to_keys([0, 1]), np.array([0, 1]), False, _CTX), TransferJob(0, to_keys([0, 1]), np.array([0, 1]), False, _CTX),
@@ -214,8 +214,8 @@ def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges():
def test_tiering_metrics_tracker_records_promotion_allocation_failures(): def test_tiering_metrics_tracker_records_promotion_allocation_failures():
tracker = TieringMetricsTracker( tracker = TieringMetricsTracker(
tier_types=["fs"], tier_types=["fs"],
num_primary_blocks=1, num_primary_chunks=1,
primary_block_size=16, primary_chunk_size=16,
) )
tracker.on_promotion_allocation_failure() tracker.on_promotion_allocation_failure()
+5 -5
View File
@@ -106,14 +106,14 @@ def key(n: int) -> OffloadKey:
def make_job( def make_job(
job_id: int, job_id: int,
keys: list[OffloadKey], keys: list[OffloadKey],
block_ids: list[int] | None = None, chunk_ids: list[int] | None = None,
) -> TransferJob: ) -> TransferJob:
if block_ids is None: if chunk_ids is None:
block_ids = list(range(len(keys))) chunk_ids = list(range(len(keys)))
return TransferJob( return TransferJob(
job_id=job_id, job_id=job_id,
keys=keys, keys=keys,
block_ids=np.array(block_ids, dtype=np.int32), chunk_ids=np.array(chunk_ids, dtype=np.int32),
is_promotion=False, is_promotion=False,
req_context=_CTX, req_context=_CTX,
) )
@@ -541,7 +541,7 @@ class TestMockObjTierFailures:
mmap_region = MagicMock() mmap_region = MagicMock()
mmap_region.create_kv_memoryview.return_value = primary_kv_view mmap_region.create_kv_memoryview.return_value = primary_kv_view
primary_tier = CPUPrimaryTierOffloadingManager( primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=num_blocks, mmap_region=mmap_region num_chunks=num_blocks, mmap_region=mmap_region
) )
obj_tier, agent = _make_tier( obj_tier, agent = _make_tier(
num_blocks=num_blocks, primary_kv_view=primary_kv_view num_blocks=num_blocks, primary_kv_view=primary_kv_view
@@ -5,15 +5,16 @@ Unit tests for TieringOffloadingManager and ExampleSecondaryTierManager.
These tests verify: These tests verify:
1. Basic tiered offloading operations (store, load, lookup) 1. Basic tiered offloading operations (store, load, lookup)
2. Cascade behavior (blocks stored to all secondary tiers) 2. Cascade behavior (chunks stored to all secondary tiers)
3. Promotion behavior (blocks loaded from secondary to primary to GPU) 3. Promotion behavior (chunks loaded from secondary to primary to GPU)
4. ref_cnt management (blocks protected during async transfers) 4. ref_cnt management (chunks protected during async transfers)
5. Eviction coordination between tiers 5. Eviction coordination between tiers
""" """
from collections.abc import Iterable from collections.abc import Iterable
from unittest.mock import MagicMock from unittest.mock import MagicMock
import numpy as np
import pytest import pytest
import torch import torch
@@ -57,10 +58,10 @@ _CTX = ReqContext(req_id="test")
_MOCK_OFFLOADING_SPEC = MagicMock() _MOCK_OFFLOADING_SPEC = MagicMock()
def _mock_mmap_region(num_blocks: int, row_bytes: int = 16): def _mock_mmap_region(num_chunks: int, row_bytes: int = 16):
"""Create a mock SharedOffloadRegion for testing.""" """Create a mock SharedOffloadRegion for testing."""
mock = MagicMock() mock = MagicMock()
view = memoryview(torch.zeros((num_blocks, row_bytes), dtype=torch.int8).numpy()) view = memoryview(torch.zeros((num_chunks, row_bytes), dtype=torch.int8).numpy())
mock.create_kv_memoryview.return_value = view mock.create_kv_memoryview.return_value = view
return mock return mock
@@ -160,7 +161,7 @@ def test_tiering_spec_collects_secondary_metric_definitions(monkeypatch):
metadata = metrics[MetricsSecondaryTierManager.MY_TIER_METRIC] metadata = metrics[MetricsSecondaryTierManager.MY_TIER_METRIC]
assert metadata.documentation == "Number of bytes served by the test tier." assert metadata.documentation == "Number of bytes served by the test tier."
assert metadata.labelnames == ("tier",) assert metadata.labelnames == ("tier",)
assert metrics[TieringOffloadingMetrics.BLOCK_QUERIES].labelnames == ("tier",) assert metrics[TieringOffloadingMetrics.CHUNK_QUERIES].labelnames == ("tier",)
assert metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY].labelnames == ("tier",) assert metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY].labelnames == ("tier",)
assert metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY].labelnames == ("tier",) assert metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY].labelnames == ("tier",)
assert metrics[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES].labelnames == ( assert metrics[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES].labelnames == (
@@ -183,7 +184,7 @@ def test_tiering_spec_collects_secondary_metric_definitions(monkeypatch):
def test_tiering_manager_aggregates_secondary_stats(): def test_tiering_manager_aggregates_secondary_stats():
mock_region = _mock_mmap_region(5) mock_region = _mock_mmap_region(5)
primary_tier = CPUPrimaryTierOffloadingManager( primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=5, mmap_region=mock_region num_chunks=5, mmap_region=mock_region
) )
secondary_tier = MetricsSecondaryTierManager( secondary_tier = MetricsSecondaryTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC, offloading_spec=_MOCK_OFFLOADING_SPEC,
@@ -232,19 +233,19 @@ class TestExampleSecondaryTierManager:
) )
# Initially empty # Initially empty
blocks = to_keys(range(3)) chunks = to_keys(range(3))
assert tier.lookup(blocks[0], _CTX) is LookupResult.MISS assert tier.lookup(chunks[0], _CTX) is LookupResult.MISS
# Store blocks (simulate with direct insertion for testing) # Store chunks (simulate with direct insertion for testing)
tier.blocks[blocks[0]] = True tier.chunks[chunks[0]] = True
tier.blocks[blocks[1]] = True tier.chunks[chunks[1]] = True
# Lookup should find first two blocks # Lookup should find first two chunks
assert tier.lookup(blocks[0], _CTX) is LookupResult.HIT assert tier.lookup(chunks[0], _CTX) is LookupResult.HIT
assert tier.lookup(blocks[1], _CTX) is LookupResult.HIT assert tier.lookup(chunks[1], _CTX) is LookupResult.HIT
# Third block not present # Third chunk not present
assert tier.lookup(blocks[2], _CTX) is LookupResult.MISS assert tier.lookup(chunks[2], _CTX) is LookupResult.MISS
# What a request-level cascade does with a key already present in the primary # What a request-level cascade does with a key already present in the primary
@@ -260,7 +261,7 @@ _CASCADE_SUPPLY_BEHAVIOR = {
# The same test, before it lands. # The same test, before it lands.
LookupResult.HIT_PENDING: "defer", LookupResult.HIT_PENDING: "defer",
# test_cascade_drops_deferred_keys_whose_primary_write_failed, where the # test_cascade_drops_deferred_keys_whose_primary_write_failed, where the
# failed write frees the block. # failed write frees the chunk.
LookupResult.MISS: "drop", LookupResult.MISS: "drop",
# test_cascade_rejects_retry_from_primary. The primary tier resolves every # test_cascade_rejects_retry_from_primary. The primary tier resolves every
# key it holds, so RETRY cannot reach the cascade, and parking on it would # key it holds, so RETRY cannot reach the cascade, and parking on it would
@@ -284,7 +285,7 @@ class TestTieringOffloadingManager:
# Create primary tier (CPU-based) # Create primary tier (CPU-based)
mock_region = _mock_mmap_region(5) mock_region = _mock_mmap_region(5)
self.primary_tier = CPUPrimaryTierOffloadingManager( self.primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=5, mmap_region=mock_region num_chunks=5, mmap_region=mock_region
) )
mock_view = mock_region.create_kv_memoryview() mock_view = mock_region.create_kv_memoryview()
@@ -331,7 +332,7 @@ class TestTieringOffloadingManager:
TransferJob( TransferJob(
job_id=job_id, job_id=job_id,
keys=to_keys([1, 2]), keys=to_keys([1, 2]),
block_ids=[0, 1], chunk_ids=np.array([0, 1], dtype=np.int64),
is_promotion=True, is_promotion=True,
req_context=_CTX, req_context=_CTX,
), ),
@@ -358,7 +359,7 @@ class TestTieringOffloadingManager:
TransferJob( TransferJob(
job_id=job_id, job_id=job_id,
keys=to_keys([1]), keys=to_keys([1]),
block_ids=[0], chunk_ids=np.array([0], dtype=np.int64),
is_promotion=True, is_promotion=True,
req_context=_CTX, req_context=_CTX,
), ),
@@ -392,23 +393,23 @@ class TestTieringOffloadingManager:
def test_basic_store_to_primary(self, manager_setup): def test_basic_store_to_primary(self, manager_setup):
"""Test basic store operation to primary tier.""" """Test basic store operation to primary tier."""
blocks = to_keys(range(3)) chunks = to_keys(range(3))
# Prepare store # Prepare store
self._start_request() self._start_request()
result = self.manager.prepare_store(blocks, _CTX) result = self.manager.prepare_store(chunks, _CTX)
assert result is not None assert result is not None
assert len(result.keys_to_store) == 3 assert len(result.keys_to_store) == 3
# Complete store # Complete store
self.manager.complete_store(blocks, _CTX, success=True) self.manager.complete_store(chunks, _CTX, success=True)
# Blocks should be in primary tier # Chunks should be in primary tier
assert count_hits(self.primary_tier, blocks) == 3 assert count_hits(self.primary_tier, chunks) == 3
def test_cascade_to_all_secondary_tiers(self, manager_setup): def test_cascade_to_all_secondary_tiers(self, manager_setup):
"""Test that blocks are cascaded to ALL secondary tiers.""" """Test that chunks are cascaded to ALL secondary tiers."""
blocks = to_keys(range(3)) chunks = to_keys(range(3))
self.secondary_tier1.submit_store = MagicMock( self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store wraps=self.secondary_tier1.submit_store
@@ -419,44 +420,44 @@ class TestTieringOffloadingManager:
# Store to primary # Store to primary
self._start_request() self._start_request()
result = self.manager.prepare_store(blocks, _CTX) result = self.manager.prepare_store(chunks, _CTX)
assert result is not None assert result is not None
# Complete store (triggers cascade via submit_store on each tier) # Complete store (triggers cascade via submit_store on each tier)
self.manager.complete_store(blocks, _CTX, success=True) self.manager.complete_store(chunks, _CTX, success=True)
# submit_store was called once per secondary tier # submit_store was called once per secondary tier
self.secondary_tier1.submit_store.assert_called_once() self.secondary_tier1.submit_store.assert_called_once()
self.secondary_tier2.submit_store.assert_called_once() self.secondary_tier2.submit_store.assert_called_once()
# Blocks should be in both secondary tiers # Chunks should be in both secondary tiers
assert self.secondary_tier1.get_num_blocks() == 3 assert self.secondary_tier1.get_num_chunks() == 3
assert self.secondary_tier2.get_num_blocks() == 3 assert self.secondary_tier2.get_num_chunks() == 3
# Verify blocks are present # Verify chunks are present
assert all( assert all(
self.secondary_tier1.lookup(b, _CTX) is LookupResult.HIT for b in blocks self.secondary_tier1.lookup(c, _CTX) is LookupResult.HIT for c in chunks
) )
assert all( assert all(
self.secondary_tier2.lookup(b, _CTX) is LookupResult.HIT for b in blocks self.secondary_tier2.lookup(c, _CTX) is LookupResult.HIT for c in chunks
) )
def test_ref_cnt_protection_during_cascade(self, manager_setup): def test_ref_cnt_protection_during_cascade(self, manager_setup):
"""Test that ref_cnt protects blocks during cascade.""" """Test that ref_cnt protects chunks during cascade."""
blocks = to_keys(range(3)) chunks = to_keys(range(3))
# Store to primary # Store to primary
self._start_request() self._start_request()
result = self.manager.prepare_store(blocks, _CTX) result = self.manager.prepare_store(chunks, _CTX)
assert result is not None assert result is not None
self.manager.complete_store(blocks, _CTX, success=True) self.manager.complete_store(chunks, _CTX, success=True)
# After complete_store, blocks should have ref_cnt > 0 # After complete_store, chunks should have ref_cnt > 0
# (one for each secondary tier) # (one for each secondary tier)
for block_hash in blocks: for key in chunks:
block = self.primary_tier._policy.get(block_hash) chunk = self.primary_tier._policy.get(key)
# ref_cnt should be 2 (one for each secondary tier) # ref_cnt should be 2 (one for each secondary tier)
assert block.ref_cnt == 2 assert chunk.ref_cnt == 2
# End of step 1: _maybe_process_finished_jobs() was already called by # End of step 1: _maybe_process_finished_jobs() was already called by
# prepare_store() above (setting the per-step flag), so on_schedule_end() # prepare_store() above (setting the per-step flag), so on_schedule_end()
@@ -466,9 +467,9 @@ class TestTieringOffloadingManager:
# ref_cnt still held: cascade jobs finished (sync tier) but haven't # ref_cnt still held: cascade jobs finished (sync tier) but haven't
# been polled yet because the per-step guard skipped the second call. # been polled yet because the per-step guard skipped the second call.
for block_hash in blocks: for key in chunks:
block = self.primary_tier._policy.get(block_hash) chunk = self.primary_tier._policy.get(key)
assert block.ref_cnt == 2 assert chunk.ref_cnt == 2
# Secondary tiers have completed jobs waiting to be drained # Secondary tiers have completed jobs waiting to be drained
assert len(self.secondary_tier1.completed_jobs) > 0 assert len(self.secondary_tier1.completed_jobs) > 0
@@ -479,37 +480,37 @@ class TestTieringOffloadingManager:
self._simulate_on_schedule_end() self._simulate_on_schedule_end()
# After cascade completes, ref_cnt should be 0 # After cascade completes, ref_cnt should be 0
for block_hash in blocks: for key in chunks:
block = self.primary_tier._policy.get(block_hash) chunk = self.primary_tier._policy.get(key)
assert block.ref_cnt == 0 assert chunk.ref_cnt == 0
# All completed jobs have been drained # All completed jobs have been drained
assert len(self.secondary_tier1.completed_jobs) == 0 assert len(self.secondary_tier1.completed_jobs) == 0
assert len(self.secondary_tier2.completed_jobs) == 0 assert len(self.secondary_tier2.completed_jobs) == 0
def test_lookup_from_primary(self, manager_setup): def test_lookup_from_primary(self, manager_setup):
"""Test lookup when blocks are in primary tier.""" """Test lookup when chunks are in primary tier."""
blocks = to_keys(range(3)) chunks = to_keys(range(3))
# Store blocks # Store chunks
self._start_request() self._start_request()
self.manager.prepare_store(blocks, _CTX) self.manager.prepare_store(chunks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True) self.manager.complete_store(chunks, _CTX, success=True)
# Lookup should find all blocks in primary # Lookup should find all chunks in primary
assert count_hits(self.manager, blocks) == 3 assert count_hits(self.manager, chunks) == 3
def test_promotion_from_secondary(self, manager_setup): def test_promotion_from_secondary(self, manager_setup):
"""Test promotion of blocks from secondary to primary tier.""" """Test promotion of chunks from secondary to primary tier."""
blocks = to_keys(range(3)) chunks = to_keys(range(3))
# Manually add blocks to secondary tier (simulate previous cascade) # Manually add chunks to secondary tier (simulate previous cascade)
for block in blocks: for chunk in chunks:
self.secondary_tier1.blocks[block] = True self.secondary_tier1.chunks[chunk] = True
# Lookup each block to initiate promotion for all of them # Lookup each chunk to initiate promotion for all of them
for block in blocks: for chunk in chunks:
result = self.manager.lookup(block, _CTX) result = self.manager.lookup(chunk, _CTX)
assert result is LookupResult.HIT_PENDING # promotion initiated assert result is LookupResult.HIT_PENDING # promotion initiated
# End of step 1: flushes deferred submit_load() calls # End of step 1: flushes deferred submit_load() calls
@@ -518,11 +519,11 @@ class TestTieringOffloadingManager:
# End of step 2: processes the completed promotion jobs # End of step 2: processes the completed promotion jobs
self._simulate_on_schedule_end() self._simulate_on_schedule_end()
# Now blocks should be in primary tier # Now chunks should be in primary tier
assert count_hits(self.primary_tier, blocks) == 3 assert count_hits(self.primary_tier, chunks) == 3
# Next lookup should succeed # Next lookup should succeed
assert count_hits(self.manager, blocks) == 3 assert count_hits(self.manager, chunks) == 3
@pytest.mark.parametrize( @pytest.mark.parametrize(
("successful_indices", "expected_results"), ("successful_indices", "expected_results"),
@@ -538,18 +539,18 @@ class TestTieringOffloadingManager:
], ],
ids=["partial", "legacy-full-failure"], ids=["partial", "legacy-full-failure"],
) )
def test_failed_promotion_keeps_only_successful_blocks( def test_failed_promotion_keeps_only_successful_chunks(
self, manager_setup, successful_indices, expected_results self, manager_setup, successful_indices, expected_results
): ):
blocks = to_keys(range(3)) chunks = to_keys(range(3))
for block in blocks: for chunk in chunks:
self.secondary_tier1.blocks[block] = True self.secondary_tier1.chunks[chunk] = True
def submit_partial(job_metadata: TransferJob) -> None: def submit_partial(job_metadata: TransferJob) -> None:
successful_keys = ( successful_keys = (
None None
if successful_indices is None if successful_indices is None
else tuple(blocks[i] for i in successful_indices) else tuple(chunks[i] for i in successful_indices)
) )
self.secondary_tier1.completed_jobs.append( self.secondary_tier1.completed_jobs.append(
JobResult( JobResult(
@@ -561,24 +562,24 @@ class TestTieringOffloadingManager:
self.secondary_tier1.submit_load = submit_partial self.secondary_tier1.submit_load = submit_partial
for block in blocks: for chunk in chunks:
assert self.manager.lookup(block, _CTX) is LookupResult.HIT_PENDING assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT_PENDING
self._simulate_on_schedule_end() self._simulate_on_schedule_end()
self._simulate_on_schedule_end() self._simulate_on_schedule_end()
assert [ assert [
self.primary_tier.lookup(block, _CTX) for block in blocks self.primary_tier.lookup(chunk, _CTX) for chunk in chunks
] == expected_results ] == expected_results
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup): def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
"""Resolved lookups report one sync delay sample per tier and block.""" """Resolved lookups report one sync delay sample per tier and chunk."""
self._start_request() self._start_request()
blocks = to_keys(range(2)) chunks = to_keys(range(2))
# No tier has these blocks: they resolve immediately as misses. # No tier has these chunks: they resolve immediately as misses.
for block in blocks: for chunk in chunks:
assert self.manager.lookup(block, _CTX) is LookupResult.MISS assert self.manager.lookup(chunk, _CTX) is LookupResult.MISS
stats = self.manager.get_stats() stats = self.manager.get_stats()
assert stats is not None assert stats is not None
@@ -614,11 +615,11 @@ class TestTieringOffloadingManager:
def test_lookup_does_not_report_async_delay_for_promotion(self, manager_setup): def test_lookup_does_not_report_async_delay_for_promotion(self, manager_setup):
"""Promotion time is not included in lookup async delay.""" """Promotion time is not included in lookup async delay."""
self._start_request() self._start_request()
block = to_keys(range(1))[0] chunk = to_keys(range(1))[0]
self.secondary_tier1.blocks[block] = True self.secondary_tier1.chunks[chunk] = True
# First lookup finds the block in a secondary tier and defers. # First lookup finds the chunk in a secondary tier and defers.
assert self.manager.lookup(block, _CTX) is LookupResult.HIT_PENDING assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT_PENDING
# Promotion is not treated as unresolved secondary lookup time. # Promotion is not treated as unresolved secondary lookup time.
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id]) self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
@@ -648,8 +649,8 @@ class TestTieringOffloadingManager:
# Promotion completes on the next scheduler step. # Promotion completes on the next scheduler step.
self._simulate_on_schedule_end() self._simulate_on_schedule_end()
# Next lookup resolves via the now-promoted primary-tier block. # Next lookup resolves via the now-promoted primary-tier chunk.
assert self.manager.lookup(block, _CTX) is LookupResult.HIT assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT
stats = self.manager.get_stats() stats = self.manager.get_stats()
if stats is not None: if stats is not None:
@@ -663,13 +664,13 @@ class TestTieringOffloadingManager:
"""Async delay is observed when a RETRY lookup later resolves.""" """Async delay is observed when a RETRY lookup later resolves."""
ctx = ReqContext(req_id="req_lookup_finish") ctx = ReqContext(req_id="req_lookup_finish")
self._start_request(ctx) self._start_request(ctx)
block = to_keys(range(1))[0] chunk = to_keys(range(1))[0]
self.secondary_tier1.lookup = MagicMock( self.secondary_tier1.lookup = MagicMock(
side_effect=[LookupResult.RETRY, LookupResult.HIT] side_effect=[LookupResult.RETRY, LookupResult.HIT]
) )
# First lookup is deferred by the secondary tier. # First lookup is deferred by the secondary tier.
assert self.manager.lookup(block, ctx) is LookupResult.RETRY assert self.manager.lookup(chunk, ctx) is LookupResult.RETRY
stats = self.manager.get_stats() stats = self.manager.get_stats()
if stats is not None: if stats is not None:
@@ -677,7 +678,7 @@ class TestTieringOffloadingManager:
stats.reduce(), TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY stats.reduce(), TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
) )
assert self.manager.lookup(block, ctx) is LookupResult.HIT_PENDING assert self.manager.lookup(chunk, ctx) is LookupResult.HIT_PENDING
stats = self.manager.get_stats() stats = self.manager.get_stats()
assert stats is not None assert stats is not None
@@ -701,68 +702,68 @@ class TestTieringOffloadingManager:
def test_partial_lookup(self, manager_setup): def test_partial_lookup(self, manager_setup):
"""Test lookup with partial hits.""" """Test lookup with partial hits."""
blocks = to_keys(range(5)) chunks = to_keys(range(5))
# Store first 3 blocks to primary # Store first 3 chunks to primary
self._start_request() self._start_request()
self.manager.prepare_store(blocks[:3], _CTX) self.manager.prepare_store(chunks[:3], _CTX)
self.manager.complete_store(blocks[:3], _CTX, success=True) self.manager.complete_store(chunks[:3], _CTX, success=True)
# Lookup all 5 blocks should return 3 (first 3 found) # Lookup all 5 chunks should return 3 (first 3 found)
assert count_hits(self.manager, blocks) == 3 assert count_hits(self.manager, chunks) == 3
def test_eviction_in_primary_tier(self, manager_setup): def test_eviction_in_primary_tier(self, manager_setup):
"""Test eviction in primary tier when capacity is exceeded.""" """Test eviction in primary tier when capacity is exceeded."""
# Primary tier has capacity of 5 blocks # Primary tier has capacity of 5 chunks
# First, fill the primary tier # First, fill the primary tier
blocks = to_keys(range(5)) chunks = to_keys(range(5))
self._start_request() self._start_request()
result = self.manager.prepare_store(blocks, _CTX) result = self.manager.prepare_store(chunks, _CTX)
assert result is not None assert result is not None
assert len(result.keys_to_store) == 5 assert len(result.keys_to_store) == 5
self.manager.complete_store(blocks, _CTX, success=True) self.manager.complete_store(chunks, _CTX, success=True)
# End of step: release ref_cnt from cascade # End of step: release ref_cnt from cascade
self._simulate_on_schedule_end() self._simulate_on_schedule_end()
# Now try to store 2 more blocks (should trigger eviction) # Now try to store 2 more chunks (should trigger eviction)
more_blocks = to_keys(range(5, 7)) more_chunks = to_keys(range(5, 7))
result = self.manager.prepare_store(more_blocks, _CTX) result = self.manager.prepare_store(more_chunks, _CTX)
# Should evict 2 blocks from primary tier # Should evict 2 chunks from primary tier
assert result is not None assert result is not None
assert len(result.evicted_keys) == 2 assert len(result.evicted_keys) == 2
assert len(result.keys_to_store) == 2 assert len(result.keys_to_store) == 2
def test_touch_propagates_to_all_tiers(self, manager_setup): def test_touch_propagates_to_all_tiers(self, manager_setup):
"""Test that touch() propagates to all tiers.""" """Test that touch() propagates to all tiers."""
blocks = to_keys(range(3)) chunks = to_keys(range(3))
# Store blocks # Store chunks
self._start_request() self._start_request()
self.manager.prepare_store(blocks, _CTX) self.manager.prepare_store(chunks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True) self.manager.complete_store(chunks, _CTX, success=True)
self._simulate_on_schedule_end() self._simulate_on_schedule_end()
# for secondary tiers to drain jobs, so primary tier's blocks are evictable. # for secondary tiers to drain jobs, so primary tier's chunks are evictable.
self._simulate_on_schedule_end() self._simulate_on_schedule_end()
self.secondary_tier1.touch = MagicMock(wraps=self.secondary_tier1.touch) self.secondary_tier1.touch = MagicMock(wraps=self.secondary_tier1.touch)
self.secondary_tier2.touch = MagicMock(wraps=self.secondary_tier2.touch) self.secondary_tier2.touch = MagicMock(wraps=self.secondary_tier2.touch)
# Touch blocks # Touch chunks
self.manager.touch(blocks, _CTX) self.manager.touch(chunks, _CTX)
# Verify touch was called on primary tier (check LRU order) # Verify touch was called on primary tier (check LRU order)
primary_keys = list(self.primary_tier._policy.evictable_blocks.keys()) primary_keys = list(self.primary_tier._policy.evictable_chunks.keys())
assert primary_keys[-3:] == list(reversed(blocks)) assert primary_keys[-3:] == list(reversed(chunks))
# Verify touch was propagated to all secondary tiers # Verify touch was propagated to all secondary tiers
self.secondary_tier1.touch.assert_called_once_with(blocks, _CTX) self.secondary_tier1.touch.assert_called_once_with(chunks, _CTX)
self.secondary_tier2.touch.assert_called_once_with(blocks, _CTX) self.secondary_tier2.touch.assert_called_once_with(chunks, _CTX)
def test_failed_store_no_cascade(self, manager_setup): def test_failed_store_no_cascade(self, manager_setup):
"""Test that failed GPU→primary store doesn't cascade.""" """Test that failed GPU→primary store doesn't cascade."""
blocks = to_keys(range(3)) chunks = to_keys(range(3))
self.secondary_tier1.submit_store = MagicMock( self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store wraps=self.secondary_tier1.submit_store
@@ -773,11 +774,11 @@ class TestTieringOffloadingManager:
# Prepare store # Prepare store
self._start_request() self._start_request()
result = self.manager.prepare_store(blocks, _CTX) result = self.manager.prepare_store(chunks, _CTX)
assert result is not None assert result is not None
# Complete store with failure — cascade must not happen # Complete store with failure — cascade must not happen
self.manager.complete_store(blocks, _CTX, success=False) self.manager.complete_store(chunks, _CTX, success=False)
# submit_store was never called on either secondary tier # submit_store was never called on either secondary tier
self.secondary_tier1.submit_store.assert_not_called() self.secondary_tier1.submit_store.assert_not_called()
@@ -786,12 +787,12 @@ class TestTieringOffloadingManager:
def test_lookup_batches_submit_load_per_request(self, manager_setup): def test_lookup_batches_submit_load_per_request(self, manager_setup):
"""lookup() defers submit_load until on_schedule_end(), one per request. """lookup() defers submit_load until on_schedule_end(), one per request.
Blocks from different requests each get their own submit_load call, each Chunks from different requests each get their own submit_load call, each
carrying the correct req_context. carrying the correct req_context.
""" """
blocks = to_keys(range(4)) chunks = to_keys(range(4))
for block in blocks: for chunk in chunks:
self.secondary_tier1.blocks[block] = True self.secondary_tier1.chunks[chunk] = True
self.secondary_tier1.submit_load = MagicMock( self.secondary_tier1.submit_load = MagicMock(
wraps=self.secondary_tier1.submit_load wraps=self.secondary_tier1.submit_load
@@ -801,10 +802,10 @@ class TestTieringOffloadingManager:
ctx_b = ReqContext(req_id="req_b") ctx_b = ReqContext(req_id="req_b")
# All lookups return HIT_PENDING: secondary hit triggers promotion # All lookups return HIT_PENDING: secondary hit triggers promotion
assert self.manager.lookup(blocks[0], ctx_a) is LookupResult.HIT_PENDING assert self.manager.lookup(chunks[0], ctx_a) is LookupResult.HIT_PENDING
assert self.manager.lookup(blocks[1], ctx_a) is LookupResult.HIT_PENDING assert self.manager.lookup(chunks[1], ctx_a) is LookupResult.HIT_PENDING
assert self.manager.lookup(blocks[2], ctx_b) is LookupResult.HIT_PENDING assert self.manager.lookup(chunks[2], ctx_b) is LookupResult.HIT_PENDING
assert self.manager.lookup(blocks[3], ctx_b) is LookupResult.HIT_PENDING assert self.manager.lookup(chunks[3], ctx_b) is LookupResult.HIT_PENDING
# submit_load must not fire during lookup - only at end of step # submit_load must not fire during lookup - only at end of step
self.secondary_tier1.submit_load.assert_not_called() self.secondary_tier1.submit_load.assert_not_called()
@@ -816,20 +817,20 @@ class TestTieringOffloadingManager:
calls = self.secondary_tier1.submit_load.call_args_list calls = self.secondary_tier1.submit_load.call_args_list
jm_a = calls[0].args[0] jm_a = calls[0].args[0]
jm_b = calls[1].args[0] jm_b = calls[1].args[0]
assert set(jm_a.keys) == {blocks[0], blocks[1]} assert set(jm_a.keys) == {chunks[0], chunks[1]}
assert jm_a.req_context is ctx_a assert jm_a.req_context is ctx_a
assert set(jm_b.keys) == {blocks[2], blocks[3]} assert set(jm_b.keys) == {chunks[2], chunks[3]}
assert jm_b.req_context is ctx_b assert jm_b.req_context is ctx_b
def test_lookup_shared_block_no_duplicate_promotion(self, manager_setup): def test_lookup_shared_chunk_no_duplicate_promotion(self, manager_setup):
"""A block looked up by two requests in the same step is promoted once. """A chunk looked up by two requests in the same step is promoted once.
The first lookup initiates promotion (returns None via secondary hit). The first lookup initiates promotion (returns None via secondary hit).
The second lookup sees ref_cnt=-1 on the primary slot and returns None The second lookup sees ref_cnt=-1 on the primary slot and returns None
via the primary in-flight path — without triggering a second promotion. via the primary in-flight path — without triggering a second promotion.
""" """
shared_block = to_keys([0])[0] shared_chunk = to_keys([0])[0]
self.secondary_tier1.blocks[shared_block] = True self.secondary_tier1.chunks[shared_chunk] = True
self.secondary_tier1.submit_load = MagicMock( self.secondary_tier1.submit_load = MagicMock(
wraps=self.secondary_tier1.submit_load wraps=self.secondary_tier1.submit_load
@@ -838,10 +839,10 @@ class TestTieringOffloadingManager:
ctx_a = ReqContext(req_id="req_a") ctx_a = ReqContext(req_id="req_a")
ctx_b = ReqContext(req_id="req_b") ctx_b = ReqContext(req_id="req_b")
result_a = self.manager.lookup(shared_block, ctx_a) result_a = self.manager.lookup(shared_chunk, ctx_a)
result_b = self.manager.lookup(shared_block, ctx_b) result_b = self.manager.lookup(shared_chunk, ctx_b)
# Both lookups find the shared_block and trigger promotion # Both lookups find the shared_chunk and trigger promotion
# returning HIT_PENDING. # returning HIT_PENDING.
assert result_a is LookupResult.HIT_PENDING assert result_a is LookupResult.HIT_PENDING
assert result_b is LookupResult.HIT_PENDING assert result_b is LookupResult.HIT_PENDING
@@ -851,12 +852,12 @@ class TestTieringOffloadingManager:
# Only one submit_load call despite two lookups # Only one submit_load call despite two lookups
self.secondary_tier1.submit_load.assert_called_once() self.secondary_tier1.submit_load.assert_called_once()
job_metadata = self.secondary_tier1.submit_load.call_args.args[0] job_metadata = self.secondary_tier1.submit_load.call_args.args[0]
assert list(job_metadata.keys) == [shared_block] assert list(job_metadata.keys) == [shared_chunk]
assert job_metadata.req_context is ctx_a assert job_metadata.req_context is ctx_a
def test_complete_store_forwards_req_context_to_submit_store(self, manager_setup): def test_complete_store_forwards_req_context_to_submit_store(self, manager_setup):
"""complete_store cascades to secondary tiers with the correct req_context.""" """complete_store cascades to secondary tiers with the correct req_context."""
blocks = to_keys(range(2)) chunks = to_keys(range(2))
self.secondary_tier1.submit_store = MagicMock( self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store wraps=self.secondary_tier1.submit_store
@@ -865,8 +866,8 @@ class TestTieringOffloadingManager:
ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"}) ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"})
self._start_request(ctx) self._start_request(ctx)
self.manager.prepare_store(blocks, ctx) self.manager.prepare_store(chunks, ctx)
self.manager.complete_store(blocks, ctx, success=True) self.manager.complete_store(chunks, ctx, success=True)
assert self.secondary_tier1.submit_store.call_count == 1 assert self.secondary_tier1.submit_store.call_count == 1
job_metadata = self.secondary_tier1.submit_store.call_args.args[0] job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
@@ -876,7 +877,7 @@ class TestTieringOffloadingManager:
self, manager_setup self, manager_setup
): ):
"""Manager hook is eager; secondary hooks wait for cascade submission.""" """Manager hook is eager; secondary hooks wait for cascade submission."""
blocks = to_keys(range(2)) chunks = to_keys(range(2))
ctx = ReqContext(req_id="req_delayed_secondary") ctx = ReqContext(req_id="req_delayed_secondary")
calls: list[tuple[str, str]] = [] calls: list[tuple[str, str]] = []
@@ -911,14 +912,14 @@ class TestTieringOffloadingManager:
) )
self._start_request(ctx) self._start_request(ctx)
self.manager.prepare_store(blocks, ctx) self.manager.prepare_store(chunks, ctx)
self.manager.on_request_finished(ctx) self.manager.on_request_finished(ctx)
assert calls == [("primary_finish", ctx.req_id)] assert calls == [("primary_finish", ctx.req_id)]
self.secondary_tier1.on_request_finished.assert_not_called() self.secondary_tier1.on_request_finished.assert_not_called()
self.secondary_tier2.on_request_finished.assert_not_called() self.secondary_tier2.on_request_finished.assert_not_called()
self.manager.complete_store(blocks, ctx, success=True) self.manager.complete_store(chunks, ctx, success=True)
assert calls == [ assert calls == [
("primary_finish", ctx.req_id), ("primary_finish", ctx.req_id),
@@ -930,7 +931,7 @@ class TestTieringOffloadingManager:
def test_failed_store_finalizes_finished_request(self, manager_setup): def test_failed_store_finalizes_finished_request(self, manager_setup):
"""Failed primary stores still unblock secondary finalization.""" """Failed primary stores still unblock secondary finalization."""
blocks = to_keys(range(2)) chunks = to_keys(range(2))
ctx = ReqContext(req_id="req_failed_store_finalize") ctx = ReqContext(req_id="req_failed_store_finalize")
self.secondary_tier1.submit_store = MagicMock( self.secondary_tier1.submit_store = MagicMock(
@@ -947,13 +948,13 @@ class TestTieringOffloadingManager:
) )
self._start_request(ctx) self._start_request(ctx)
self.manager.prepare_store(blocks, ctx) self.manager.prepare_store(chunks, ctx)
self.manager.on_request_finished(ctx) self.manager.on_request_finished(ctx)
self.secondary_tier1.on_request_finished.assert_not_called() self.secondary_tier1.on_request_finished.assert_not_called()
self.secondary_tier2.on_request_finished.assert_not_called() self.secondary_tier2.on_request_finished.assert_not_called()
self.manager.complete_store(blocks, ctx, success=False) self.manager.complete_store(chunks, ctx, success=False)
self.secondary_tier1.submit_store.assert_not_called() self.secondary_tier1.submit_store.assert_not_called()
self.secondary_tier2.submit_store.assert_not_called() self.secondary_tier2.submit_store.assert_not_called()
@@ -981,7 +982,7 @@ class TestTieringOffloadingManager:
def test_reset_cache_finalizes_delayed_secondary_request(self, manager_setup): def test_reset_cache_finalizes_delayed_secondary_request(self, manager_setup):
"""reset_cache abandons pending primary stores and finalizes secondaries.""" """reset_cache abandons pending primary stores and finalizes secondaries."""
blocks = to_keys(range(2)) chunks = to_keys(range(2))
ctx = ReqContext(req_id="req_reset_finalize_secondary") ctx = ReqContext(req_id="req_reset_finalize_secondary")
self.secondary_tier1.on_request_finished = MagicMock( self.secondary_tier1.on_request_finished = MagicMock(
@@ -992,7 +993,7 @@ class TestTieringOffloadingManager:
) )
self._start_request(ctx) self._start_request(ctx)
self.manager.prepare_store(blocks, ctx) self.manager.prepare_store(chunks, ctx)
self.manager.on_request_finished(ctx) self.manager.on_request_finished(ctx)
self.secondary_tier1.on_request_finished.assert_not_called() self.secondary_tier1.on_request_finished.assert_not_called()
@@ -1008,8 +1009,8 @@ class TestTieringOffloadingManager:
self, manager_setup self, manager_setup
): ):
"""reset_cache drops active pending stores so resumed requests finalize.""" """reset_cache drops active pending stores so resumed requests finalize."""
initial_blocks = to_keys(range(2)) initial_chunks = to_keys(range(2))
resumed_blocks = to_keys(range(2, 4)) resumed_chunks = to_keys(range(2, 4))
ctx = ReqContext(req_id="req_reset_resume") ctx = ReqContext(req_id="req_reset_resume")
self.secondary_tier1.on_request_finished = MagicMock( self.secondary_tier1.on_request_finished = MagicMock(
@@ -1020,7 +1021,7 @@ class TestTieringOffloadingManager:
) )
self._start_request(ctx) self._start_request(ctx)
self.manager.prepare_store(initial_blocks, ctx) self.manager.prepare_store(initial_chunks, ctx)
assert self.manager._req_state[ctx.req_id].pending_primary_stores == 1 assert self.manager._req_state[ctx.req_id].pending_primary_stores == 1
self.manager.reset_cache() self.manager.reset_cache()
@@ -1030,8 +1031,8 @@ class TestTieringOffloadingManager:
self.secondary_tier1.on_request_finished.assert_not_called() self.secondary_tier1.on_request_finished.assert_not_called()
self.secondary_tier2.on_request_finished.assert_not_called() self.secondary_tier2.on_request_finished.assert_not_called()
self.manager.prepare_store(resumed_blocks, ctx) self.manager.prepare_store(resumed_chunks, ctx)
self.manager.complete_store(resumed_blocks, ctx, success=True) self.manager.complete_store(resumed_chunks, ctx, success=True)
self.manager.on_request_finished(ctx) self.manager.on_request_finished(ctx)
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx) self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
@@ -1039,12 +1040,12 @@ class TestTieringOffloadingManager:
assert ctx.req_id not in self.manager._req_state assert ctx.req_id not in self.manager._req_state
def test_on_new_request_lifecycle(self, manager_setup): def test_on_new_request_lifecycle(self, manager_setup):
"""Policy defaults to BLOCK_LEVEL, escalates when a tier requests it, """Policy defaults to CHUNK_LEVEL, escalates when a tier requests it,
and is cleaned up on on_request_finished.""" and is cleaned up on on_request_finished."""
# Default: all tiers return BLOCK_LEVEL # Default: all tiers return CHUNK_LEVEL
ctx = ReqContext(req_id="req_policy_lifecycle") ctx = ReqContext(req_id="req_policy_lifecycle")
result = self.manager.on_new_request(ctx) result = self.manager.on_new_request(ctx)
assert result.policy == OffloadPolicy.BLOCK_LEVEL assert result.policy == OffloadPolicy.CHUNK_LEVEL
assert self.manager._req_state[ctx.req_id].request_level_tiers is None assert self.manager._req_state[ctx.req_id].request_level_tiers is None
self.manager.on_request_finished(ctx) self.manager.on_request_finished(ctx)
assert ctx.req_id not in self.manager._req_state assert ctx.req_id not in self.manager._req_state
@@ -1064,26 +1065,26 @@ class TestTieringOffloadingManager:
assert ctx.req_id not in self.manager._req_state assert ctx.req_id not in self.manager._req_state
@pytest.mark.parametrize("new_ids", [(), (3, 4)], ids=["fully_warm", "mixed"]) @pytest.mark.parametrize("new_ids", [(), (3, 4)], ids=["fully_warm", "mixed"])
def test_prepare_store_cascades_existing_blocks_to_request_level_tiers( def test_prepare_store_cascades_existing_chunks_to_request_level_tiers(
self, manager_setup, new_ids self, manager_setup, new_ids
): ):
"""prepare_store cascades hit blocks to request-level tiers only. """prepare_store cascades hit chunks to request-level tiers only.
The fully-warm case is the producer shape from issue #52808: nothing is The fully-warm case is the producer shape from issue #52808: nothing is
new, so primary returns an empty keys_to_store and no GPU->primary new, so primary returns an empty keys_to_store and no GPU->primary
transfer runs. A request-level tier must still be handed the blocks, transfer runs. A request-level tier must still be handed the chunks,
otherwise a peer waiting on them has nothing to fetch. otherwise a peer waiting on them has nothing to fetch.
""" """
# Store some blocks to primary first # Store some chunks to primary first
existing_blocks = to_keys(range(3)) existing_chunks = to_keys(range(3))
self._start_request() self._start_request()
result = self.manager.prepare_store(existing_blocks, _CTX) result = self.manager.prepare_store(existing_chunks, _CTX)
assert result is not None assert result is not None
self.manager.complete_store(existing_blocks, _CTX, success=True) self.manager.complete_store(existing_chunks, _CTX, success=True)
# Drain cascade completions # Drain cascade completions
self._simulate_on_schedule_end() self._simulate_on_schedule_end()
# Make tier1 request-level, tier2 stays block-level # Make tier1 request-level, tier2 stays chunk-level
self.secondary_tier1.on_new_request = lambda req_context: ( self.secondary_tier1.on_new_request = lambda req_context: (
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL) RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
) )
@@ -1099,20 +1100,20 @@ class TestTieringOffloadingManager:
wraps=self.secondary_tier2.submit_store wraps=self.secondary_tier2.submit_store
) )
# Call prepare_store with existing + new blocks # Call prepare_store with existing + new chunks
new_blocks = to_keys(new_ids) new_chunks = to_keys(new_ids)
all_blocks = existing_blocks + new_blocks all_chunks = existing_chunks + new_chunks
result = self.manager.prepare_store(all_blocks, ctx) result = self.manager.prepare_store(all_chunks, ctx)
assert result is not None assert result is not None
assert set(result.keys_to_store) == set(new_blocks) assert set(result.keys_to_store) == set(new_chunks)
# Only tier1 (request-level) should get existing blocks cascaded now. # Only tier1 (request-level) should get existing chunks cascaded now.
# New blocks are cascaded to ALL tiers later via complete_store(). # New chunks are cascaded to ALL tiers later via complete_store().
self.secondary_tier1.submit_store.assert_called_once() self.secondary_tier1.submit_store.assert_called_once()
job_metadata = self.secondary_tier1.submit_store.call_args.args[0] job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
assert set(job_metadata.keys) == set(existing_blocks) assert set(job_metadata.keys) == set(existing_chunks)
# tier2 (block-level) does not get existing blocks here. # tier2 (chunk-level) does not get existing chunks here.
self.secondary_tier2.submit_store.assert_not_called() self.secondary_tier2.submit_store.assert_not_called()
def _make_request_level_request(self, req_id: str) -> ReqContext: def _make_request_level_request(self, req_id: str) -> ReqContext:
@@ -1150,7 +1151,7 @@ class TestTieringOffloadingManager:
self.manager.primary_tier.lookup = lambda key, req_context: (LookupResult.RETRY) self.manager.primary_tier.lookup = lambda key, req_context: (LookupResult.RETRY)
with pytest.raises(AssertionError): with pytest.raises(AssertionError):
self.manager._cascade_existing_blocks_to_request_level_tiers(keys, ctx, {0}) self.manager._cascade_existing_chunks_to_request_level_tiers(keys, ctx, {0})
def _start_in_flight_primary_write(self, keys: list[OffloadKey]) -> ReqContext: def _start_in_flight_primary_write(self, keys: list[OffloadKey]) -> ReqContext:
"""Leave a GPU->primary write for `keys` open, so they look present to """Leave a GPU->primary write for `keys` open, so they look present to
@@ -1202,7 +1203,7 @@ class TestTieringOffloadingManager:
def test_cascade_drops_deferred_keys_whose_primary_write_failed( def test_cascade_drops_deferred_keys_whose_primary_write_failed(
self, manager_setup self, manager_setup
): ):
"""A failed write frees the block, so the deferred key resolves to MISS """A failed write frees the chunk, so the deferred key resolves to MISS
and the request finalizes instead of parking forever.""" and the request finalizes instead of parking forever."""
keys = to_keys(range(3)) keys = to_keys(range(3))
writer_ctx = self._start_in_flight_primary_write(keys) writer_ctx = self._start_in_flight_primary_write(keys)
@@ -1224,21 +1225,21 @@ class TestTieringOffloadingManager:
"""reset_cache wipes every kind of orchestrator state and resets """reset_cache wipes every kind of orchestrator state and resets
primary tier; pending submissions are dropped without being sent primary tier; pending submissions are dropped without being sent
to the secondary tier. Active request state is retained.""" to the secondary tier. Active request state is retained."""
# Cascade — populates primary blocks and leaves cascade jobs # Cascade — populates primary chunks and leaves cascade jobs
# in _jobs (the synchronous example tier has already # in _jobs (the synchronous example tier has already
# queued completions); reset_cache's drain loop will pick them up. # queued completions); reset_cache's drain loop will pick them up.
blocks = to_keys(range(3)) chunks = to_keys(range(3))
self._start_request() self._start_request()
self.manager.prepare_store(blocks, _CTX) self.manager.prepare_store(chunks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True) self.manager.complete_store(chunks, _CTX, success=True)
assert self.manager._jobs assert self.manager._jobs
# Pending promotion submission (deferred — no on_schedule_end after # Pending promotion submission (deferred — no on_schedule_end after
# the lookup that staged it). # the lookup that staged it).
promo_block = to_keys([99])[0] promo_chunk = to_keys([99])[0]
self.secondary_tier1.blocks[promo_block] = True self.secondary_tier1.chunks[promo_chunk] = True
assert ( assert (
self.manager.lookup(promo_block, ReqContext(req_id="pending")) self.manager.lookup(promo_chunk, ReqContext(req_id="pending"))
is LookupResult.HIT_PENDING is LookupResult.HIT_PENDING
) )
assert self.manager._pending_load_submissions assert self.manager._pending_load_submissions
@@ -1268,10 +1269,10 @@ class TestTieringOffloadingManager:
assert self.manager._processed_jobs_this_step is False assert self.manager._processed_jobs_this_step is False
# Primary tier reset to a fresh state. # Primary tier reset to a fresh state.
assert self.primary_tier._num_allocated_blocks == 0 assert self.primary_tier._num_allocated_chunks == 0
assert self.primary_tier._free_list == [] assert self.primary_tier._free_list == []
for block in blocks: for chunk in chunks:
assert self.primary_tier.lookup(block, _CTX) is LookupResult.MISS assert self.primary_tier.lookup(chunk, _CTX) is LookupResult.MISS
# Pending submission was dropped, not submitted. # Pending submission was dropped, not submitted.
self.secondary_tier1.submit_load.assert_not_called() self.secondary_tier1.submit_load.assert_not_called()
@@ -1291,10 +1292,10 @@ class TestTieringOffloadingManager:
) )
# Drive a cascade so a job lands in _jobs. # Drive a cascade so a job lands in _jobs.
blocks = to_keys(range(3)) chunks = to_keys(range(3))
self._start_request() self._start_request()
self.manager.prepare_store(blocks, _CTX) self.manager.prepare_store(chunks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True) self.manager.complete_store(chunks, _CTX, success=True)
assert self.manager._jobs assert self.manager._jobs
self.manager.reset_cache() self.manager.reset_cache()
@@ -1315,20 +1316,20 @@ class TestTieringOffloadingManager:
self, manager_setup, load_tier_filter self, manager_setup, load_tier_filter
): ):
"""Filter excluding secondary medium returns MISS from secondaries """Filter excluding secondary medium returns MISS from secondaries
even when they hold the block; primary is unaffected.""" even when they hold the chunk; primary is unaffected."""
blocks = to_keys(range(2)) chunks = to_keys(range(2))
# Put one block in primary, one only in secondary # Put one chunk in primary, one only in secondary
self._start_request() self._start_request()
self.manager.prepare_store(blocks[:1], _CTX) self.manager.prepare_store(chunks[:1], _CTX)
self.manager.complete_store(blocks[:1], _CTX, success=True) self.manager.complete_store(chunks[:1], _CTX, success=True)
self.secondary_tier1.blocks[blocks[1]] = True self.secondary_tier1.chunks[chunks[1]] = True
# Secondaries have medium=CPU, so load_tier_filter skips them. # Secondaries have medium=CPU, so load_tier_filter skips them.
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup) self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
ctx = ReqContext(req_id="r1", load_tier_filter=load_tier_filter) ctx = ReqContext(req_id="r1", load_tier_filter=load_tier_filter)
assert self.manager.lookup(blocks[0], ctx) is LookupResult.HIT assert self.manager.lookup(chunks[0], ctx) is LookupResult.HIT
assert self.manager.lookup(blocks[1], ctx) is LookupResult.MISS assert self.manager.lookup(chunks[1], ctx) is LookupResult.MISS
self.secondary_tier1.lookup.assert_not_called() self.secondary_tier1.lookup.assert_not_called()
@pytest.mark.parametrize( @pytest.mark.parametrize(
@@ -1344,13 +1345,13 @@ class TestTieringOffloadingManager:
self, manager_setup, load_tier_filter self, manager_setup, load_tier_filter
): ):
"""Filter that matches the secondary's medium allows lookup.""" """Filter that matches the secondary's medium allows lookup."""
blocks = to_keys(range(1)) chunks = to_keys(range(1))
self.secondary_tier1.blocks[blocks[0]] = True self.secondary_tier1.chunks[chunks[0]] = True
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup) self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
ctx = ReqContext(req_id="r2", load_tier_filter=load_tier_filter) ctx = ReqContext(req_id="r2", load_tier_filter=load_tier_filter)
assert self.manager.lookup(blocks[0], ctx) is LookupResult.HIT_PENDING assert self.manager.lookup(chunks[0], ctx) is LookupResult.HIT_PENDING
self.secondary_tier1.lookup.assert_called() self.secondary_tier1.lookup.assert_called()
@@ -1360,7 +1361,7 @@ class TestTieringOffloadingWithoutSecondaryTiers:
def test_works_without_secondary_tiers(self): def test_works_without_secondary_tiers(self):
"""Test that manager works with empty secondary_tiers list.""" """Test that manager works with empty secondary_tiers list."""
primary_tier = CPUPrimaryTierOffloadingManager( primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=5, mmap_region=_mock_mmap_region(5) num_chunks=5, mmap_region=_mock_mmap_region(5)
) )
# Create manager with no secondary tiers # Create manager with no secondary tiers
@@ -1368,15 +1369,15 @@ class TestTieringOffloadingWithoutSecondaryTiers:
primary_tier=primary_tier, secondary_tiers=[] primary_tier=primary_tier, secondary_tiers=[]
) )
blocks = to_keys(range(3)) chunks = to_keys(range(3))
# Should work like a regular OffloadingManager # Should work like a regular OffloadingManager
manager.on_new_request(_CTX) manager.on_new_request(_CTX)
result = manager.prepare_store(blocks, _CTX) result = manager.prepare_store(chunks, _CTX)
assert result is not None assert result is not None
manager.complete_store(blocks, _CTX, success=True) manager.complete_store(chunks, _CTX, success=True)
assert count_hits(manager, blocks) == 3 assert count_hits(manager, chunks) == 3
@pytest.mark.parametrize( @pytest.mark.parametrize(
@@ -1131,7 +1131,7 @@ class OffloadingConnectorScheduler:
# Skip prefix-hit chunks for block-level policy; for # Skip prefix-hit chunks for block-level policy; for
# request-level, next_stored_chunk_idx stays at 0 so all # request-level, next_stored_chunk_idx stays at 0 so all
# chunks (including hits) are offloaded. # chunks (including hits) are offloaded.
if req_status.offloading_context.policy == OffloadPolicy.BLOCK_LEVEL: if req_status.offloading_context.policy == OffloadPolicy.CHUNK_LEVEL:
group_state.next_stored_chunk_idx = num_chunks group_state.next_stored_chunk_idx = num_chunks
src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context) src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context)
+8 -5
View File
@@ -114,17 +114,17 @@ class LookupResult(Enum):
class OffloadPolicy(Enum): class OffloadPolicy(Enum):
# Offload only newly-computed blocks as they arrive; prefix-hit # Offload only newly-computed chunks as they arrive; prefix-hit
# blocks (already offloaded by a prior request) are skipped. # chunks (already offloaded by a prior request) are skipped.
BLOCK_LEVEL = "block_level" CHUNK_LEVEL = "chunk_level"
# Offload all blocks for the request, including prefix hits. # Offload all chunks for the request, including prefix hits.
# Used by tiers that need the complete KV context for a request. # Used by tiers that need the complete KV context for a request.
REQUEST_LEVEL = "request_level" REQUEST_LEVEL = "request_level"
@dataclass @dataclass
class RequestOffloadingContext: class RequestOffloadingContext:
policy: OffloadPolicy = OffloadPolicy.BLOCK_LEVEL policy: OffloadPolicy = OffloadPolicy.CHUNK_LEVEL
class ScheduleEndContext(NamedTuple): class ScheduleEndContext(NamedTuple):
@@ -400,6 +400,9 @@ class OffloadingManager(ABC):
class BlockIDsLoadStoreSpec(LoadStoreSpec, ABC): class BlockIDsLoadStoreSpec(LoadStoreSpec, ABC):
""" """
Spec for loading/storing KV blocks from given block numbers. Spec for loading/storing KV blocks from given block numbers.
Subclass semantics differ: GPULoadStoreSpec.block_ids are GPU block
indices; CPULoadStoreSpec.block_ids are CPU cache chunk indices.
""" """
def __init__(self, block_ids: list[int]): def __init__(self, block_ids: list[int]):
+11 -2
View File
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
@@ -12,6 +14,13 @@ class CPUOffloadingMetrics:
class CPULoadStoreSpec(BlockIDsLoadStoreSpec): class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
"""Spec for loading/storing KV chunks to/from CPU memory.
The inherited block_ids field holds chunk indices into the
CPU cache (not GPU block IDs). The chunk_ids alias exposes
the same array under the name used by the tiering layer.
""" """
Spec for loading/storing a KV block to CPU memory.
""" @property
def chunk_ids(self) -> np.ndarray:
return self.block_ids
+10 -10
View File
@@ -257,7 +257,7 @@ class SingleDirectionOffloadingHandler:
gpu_tensors: list of GPU KV cache tensors. gpu_tensors: list of GPU KV cache tensors.
Each of shape (num_gpu_blocks, gpu_page_size_bytes) with dtype int8. Each of shape (num_gpu_blocks, gpu_page_size_bytes) with dtype int8.
cpu_tensors: list of CPU KV cache tensors. cpu_tensors: list of CPU KV cache tensors.
Each of shape (num_cpu_blocks, cpu_page_size_bytes) with dtype int8. Each of shape (num_cpu_chunks, cpu_page_size_bytes) with dtype int8.
Order should match gpu_tensors. Order should match gpu_tensors.
layer_refs_per_group: list of CanonicalKVCacheRef per group. layer_refs_per_group: list of CanonicalKVCacheRef per group.
gpu_to_cpu: if True, transfer from GPU to CPU; otherwise CPU to GPU. gpu_to_cpu: if True, transfer from GPU to CPU; otherwise CPU to GPU.
@@ -527,20 +527,20 @@ class SingleDirectionOffloadingHandler:
# 1. GPU -> CPU # 1. GPU -> CPU
# 2. CPU -> GPU # 2. CPU -> GPU
# #
# transfers are also to CPU blocks, EXCEPT MAYBE for the first and last block. # transfers are also to CPU chunks, EXCEPT MAYBE for the first and last chunk.
# i.e. the first and last CPU blocks in src_blocks can match against # i.e. the first and last CPU chunks in src_blocks can match against
# a smaller (byte-wise) set of GPU blocks in dst_blocks. # a smaller (byte-wise) set of GPU blocks in dst_blocks.
# In such cases, we may need to skip some gpu-sized sub-blocks, # In such cases, we may need to skip some gpu-sized sub-blocks,
# and start reading/writing from the middle of the first CPU block. # and start reading/writing from the middle of the first CPU chunk.
# If we have multiple KV cache groups (when using HMA with hybrid models), # If we have multiple KV cache groups (when using HMA with hybrid models),
# we may have a partial first/last CPU block per each group. # we may have a partial first/last CPU chunk per each group.
# The group_sizes parameter encodes the size of each group of blocks # The group_sizes parameter encodes the size of each group of blocks
# in the GPU dst_blocks. # in the GPU dst_blocks.
# If group_sizes is None, we assume all blocks belong to a single group. # If group_sizes is None, we assume all blocks belong to a single group.
# The logical_offset parameter maps each group of blocks to its logical # The logical_offset parameter maps each group of blocks to its logical
# offset inside the request, counting in GPU blocks. # offset inside the request, counting in GPU blocks.
# This allows us to find the correct starting position # This allows us to find the correct starting position
# in the matching first CPU block. # in the matching first CPU chunk.
# extract group_sizes from the GPU spec # extract group_sizes from the GPU spec
gpu_spec = src_spec if self.gpu_to_cpu else dst_spec gpu_spec = src_spec if self.gpu_to_cpu else dst_spec
@@ -751,7 +751,7 @@ class CPUOffloadingWorker(OffloadingWorker):
self, self,
kv_caches: CanonicalKVCaches, kv_caches: CanonicalKVCaches,
blocks_per_chunk: int, blocks_per_chunk: int,
num_cpu_blocks: int, num_cpu_chunks: int,
mmap_region: SharedOffloadRegion | None = None, mmap_region: SharedOffloadRegion | None = None,
canonical_layout: bool = False, canonical_layout: bool = False,
): ):
@@ -790,16 +790,16 @@ class CPUOffloadingWorker(OffloadingWorker):
else: else:
t0 = time.monotonic() t0 = time.monotonic()
cpu_tensor = torch.zeros( cpu_tensor = torch.zeros(
(num_cpu_blocks, cpu_page_size_bytes), (num_cpu_chunks, cpu_page_size_bytes),
dtype=torch.int8, dtype=torch.int8,
device="cpu", device="cpu",
pin_memory=pin_memory, pin_memory=pin_memory,
) )
logger.debug( logger.debug(
"torch.zeros pinned tensor %d×%d (%.2f GB): %.3f s", "torch.zeros pinned tensor %d×%d (%.2f GB): %.3f s",
num_cpu_blocks, num_cpu_chunks,
cpu_page_size_bytes, cpu_page_size_bytes,
num_cpu_blocks * cpu_page_size_bytes / 1e9, num_cpu_chunks * cpu_page_size_bytes / 1e9,
time.monotonic() - t0, time.monotonic() - t0,
) )
+87 -87
View File
@@ -23,7 +23,7 @@ from vllm.v1.kv_offload.cpu.common import (
CPULoadStoreSpec, CPULoadStoreSpec,
CPUOffloadingMetrics, CPUOffloadingMetrics,
) )
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
@@ -34,14 +34,14 @@ class CPUOffloadingManager(OffloadingManager):
register their own or be loaded out-of-tree via cache_policy_module_path). register their own or be loaded out-of-tree via cache_policy_module_path).
The manager owns all shared logic: ref-counting, event emission, The manager owns all shared logic: ref-counting, event emission,
block pool management, and the prepare_store/complete_store skeletons. chunk pool management, and the prepare_store/complete_store skeletons.
Policy-specific block organization and eviction decisions are delegated Policy-specific chunk organization and eviction decisions are delegated
to the CachePolicy implementation. to the CachePolicy implementation.
""" """
def __init__( def __init__(
self, self,
num_blocks: int, num_chunks: int,
cache_policy: str = "lru", cache_policy: str = "lru",
cache_policy_module_path: str | None = None, cache_policy_module_path: str | None = None,
enable_events: bool = False, enable_events: bool = False,
@@ -49,59 +49,59 @@ class CPUOffloadingManager(OffloadingManager):
max_tracker_size: int = 64_000, max_tracker_size: int = 64_000,
): ):
self.medium: Medium = Medium.CPU self.medium: Medium = Medium.CPU
self._num_blocks: int = num_blocks self._num_chunks: int = num_chunks
self._num_allocated_blocks: int = 0 self._num_allocated_chunks: int = 0
self._free_list: list[int] = [] self._free_list: list[int] = []
self.events: list[OffloadingEvent] | None = [] if enable_events else None self.events: list[OffloadingEvent] | None = [] if enable_events else None
policy_cls = CachePolicyFactory.get_cache_policy_cls( policy_cls = CachePolicyFactory.get_cache_policy_cls(
cache_policy, cache_policy_module_path cache_policy, cache_policy_module_path
) )
self._policy: CachePolicy = policy_cls(cache_capacity=num_blocks) self._policy: CachePolicy = policy_cls(cache_capacity=num_chunks)
# Track the number of blocks in the cache that are evictable. i.e. ref_cnt 0. # Track the number of chunks in the cache that are evictable. i.e. ref_cnt 0.
self._num_evictable_cache_blocks: int = 0 self._num_evictable_cache_chunks: int = 0
# Track blocks with an in-flight store (ref_cnt -1, not yet completed). # Track chunks with an in-flight store (ref_cnt -1, not yet completed).
self._num_write_pending_blocks: int = 0 self._num_write_pending_chunks: int = 0
self.store_threshold: int = store_threshold self.store_threshold: int = store_threshold
self.max_tracker_size: int = max_tracker_size self.max_tracker_size: int = max_tracker_size
self.stores_skipped_in_current_batch: int = 0 self.stores_skipped_in_current_batch: int = 0
self.allocation_sizes_in_current_batch: list[int] = [] self.allocation_sizes_in_current_batch: list[int] = []
# Number of block references. It is ordered so can evict the LRU entry in O(1). # Number of chunk references. It is ordered so can evict the LRU entry in O(1).
self.counts: OrderedDict[OffloadKey, int] | None = ( self.counts: OrderedDict[OffloadKey, int] | None = (
OrderedDict() if store_threshold >= 2 else None OrderedDict() if store_threshold >= 2 else None
) )
# --- block pool --- # --- chunk pool ---
def _get_num_free_blocks(self) -> int: def _get_num_free_chunks(self) -> int:
return len(self._free_list) + self._num_blocks - self._num_allocated_blocks return len(self._free_list) + self._num_chunks - self._num_allocated_chunks
def _allocate_blocks(self, keys: list[OffloadKey]) -> list[BlockStatus]: def _allocate_chunks(self, keys: list[OffloadKey]) -> list[ChunkStatus]:
num_fresh = min(len(keys), self._num_blocks - self._num_allocated_blocks) num_fresh = min(len(keys), self._num_chunks - self._num_allocated_chunks)
num_reused = len(keys) - num_fresh num_reused = len(keys) - num_fresh
assert len(self._free_list) >= num_reused assert len(self._free_list) >= num_reused
# allocate fresh blocks # allocate fresh chunks
blocks: list[BlockStatus] = [] chunks: list[ChunkStatus] = []
for _ in range(num_fresh): for _ in range(num_fresh):
blocks.append(BlockStatus(self._num_allocated_blocks)) chunks.append(ChunkStatus(self._num_allocated_chunks))
self._num_allocated_blocks += 1 self._num_allocated_chunks += 1
# allocate reused blocks # allocate reused chunks
for _ in range(num_reused): for _ in range(num_reused):
blocks.append(BlockStatus(self._free_list.pop())) chunks.append(ChunkStatus(self._free_list.pop()))
return blocks return chunks
def _free_block(self, block: BlockStatus) -> None: def _free_chunk(self, chunk: ChunkStatus) -> None:
self._free_list.append(block.block_id) self._free_list.append(chunk.chunk_id)
def _get_load_store_spec( def _get_load_store_spec(
self, self,
keys: Iterable[OffloadKey], keys: Iterable[OffloadKey],
blocks: Iterable[BlockStatus], chunks: Iterable[ChunkStatus],
) -> CPULoadStoreSpec: ) -> CPULoadStoreSpec:
return CPULoadStoreSpec([block.block_id for block in blocks]) return CPULoadStoreSpec([chunk.chunk_id for chunk in chunks])
def _record_accesses(self, keys: Collection[OffloadKey]) -> None: def _record_accesses(self, keys: Collection[OffloadKey]) -> None:
"""Record an offer without evicting its tracked candidates.""" """Record an offer without evicting its tracked candidates."""
@@ -132,10 +132,10 @@ class CPUOffloadingManager(OffloadingManager):
@override @override
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
block = self._policy.get(key) chunk = self._policy.get(key)
if block is None: if chunk is None:
return LookupResult.MISS return LookupResult.MISS
if not block.is_ready: if not chunk.is_ready:
return LookupResult.HIT_PENDING return LookupResult.HIT_PENDING
return LookupResult.HIT return LookupResult.HIT
@@ -145,18 +145,18 @@ class CPUOffloadingManager(OffloadingManager):
keys: Collection[OffloadKey], keys: Collection[OffloadKey],
req_context: ReqContext, req_context: ReqContext,
) -> LoadStoreSpec: ) -> LoadStoreSpec:
blocks = [] chunks = []
for key in keys: for key in keys:
block = self._policy.get(key) chunk = self._policy.get(key)
assert block is not None, f"Block {key!r} not found in cache" assert chunk is not None, f"Chunk {key!r} not found in cache"
assert block.is_ready, f"Block {key!r} is not ready for reading" assert chunk.is_ready, f"Chunk {key!r} is not ready for reading"
if block.ref_cnt == 0: if chunk.ref_cnt == 0:
self._policy.mark_non_evictable(key) self._policy.mark_non_evictable(key)
self._num_evictable_cache_blocks -= 1 # ref_cnt 0 -> 1 self._num_evictable_cache_chunks -= 1 # ref_cnt 0 -> 1
assert self._num_evictable_cache_blocks >= 0 assert self._num_evictable_cache_chunks >= 0
block.ref_cnt += 1 chunk.ref_cnt += 1
blocks.append(block) chunks.append(chunk)
return self._get_load_store_spec(keys, blocks) return self._get_load_store_spec(keys, chunks)
@override @override
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext) -> None: def touch(self, keys: Collection[OffloadKey], req_context: ReqContext) -> None:
@@ -167,12 +167,12 @@ class CPUOffloadingManager(OffloadingManager):
self, keys: Collection[OffloadKey], req_context: ReqContext self, keys: Collection[OffloadKey], req_context: ReqContext
) -> None: ) -> None:
for key in keys: for key in keys:
block = self._policy.get(key) chunk = self._policy.get(key)
assert block is not None, f"Block {key!r} not found" assert chunk is not None, f"Chunk {key!r} not found"
assert block.ref_cnt > 0, f"Block {key!r} ref_cnt is already 0" assert chunk.ref_cnt > 0, f"Chunk {key!r} ref_cnt is already 0"
block.ref_cnt -= 1 chunk.ref_cnt -= 1
if block.ref_cnt == 0: if chunk.ref_cnt == 0:
self._num_evictable_cache_blocks += 1 # ref_cnt 1 -> 0 self._num_evictable_cache_chunks += 1 # ref_cnt 1 -> 0
self._policy.mark_evictable(key) self._policy.mark_evictable(key)
@override @override
@@ -186,7 +186,7 @@ class CPUOffloadingManager(OffloadingManager):
self._record_accesses(keys) self._record_accesses(keys)
keys = [k for k in keys if self.counts.get(k, 0) >= self.store_threshold] keys = [k for k in keys if self.counts.get(k, 0) >= self.store_threshold]
self.stores_skipped_in_current_batch += num_keys - len(keys) self.stores_skipped_in_current_batch += num_keys - len(keys)
# filter out blocks that are already stored # filter out chunks that are already stored
keys_to_store = [k for k in keys if self._policy.get(k) is None] keys_to_store = [k for k in keys if self._policy.get(k) is None]
if not keys_to_store: if not keys_to_store:
@@ -197,29 +197,29 @@ class CPUOffloadingManager(OffloadingManager):
) )
self.allocation_sizes_in_current_batch.append(len(keys_to_store)) self.allocation_sizes_in_current_batch.append(len(keys_to_store))
num_blocks_to_evict = len(keys_to_store) - self._get_num_free_blocks() num_chunks_to_evict = len(keys_to_store) - self._get_num_free_chunks()
to_evict: list[OffloadKey] = [] to_evict: list[OffloadKey] = []
if num_blocks_to_evict > 0: if num_chunks_to_evict > 0:
if num_blocks_to_evict > self._num_evictable_cache_blocks: if num_chunks_to_evict > self._num_evictable_cache_chunks:
# Eviction will fail. # Eviction will fail.
return None return None
# There is a still a chance for eviction failure as some of the # There is a still a chance for eviction failure as some of the
# idle blocks might be in the protected list. # idle chunks might be in the protected list.
# Blocks from the original input are excluded from eviction candidates: # Chunks from the original input are excluded from eviction candidates:
# a block that was already stored must remain in the cache after this call. # a chunk that was already stored must remain in the cache after this call.
protected = set(keys) protected = set(keys)
evicted = self._policy.evict(num_blocks_to_evict, protected) evicted = self._policy.evict(num_chunks_to_evict, protected)
if evicted is None: if evicted is None:
return None return None
# cache-policy removes only idle blocks. # cache-policy removes only idle chunks.
self._num_evictable_cache_blocks -= len(evicted) self._num_evictable_cache_chunks -= len(evicted)
assert self._num_evictable_cache_blocks >= 0 assert self._num_evictable_cache_chunks >= 0
for key, block in evicted: for key, chunk in evicted:
self._free_block(block) self._free_chunk(chunk)
to_evict.append(key) to_evict.append(key)
if to_evict and self.events is not None: if to_evict and self.events is not None:
@@ -231,17 +231,17 @@ class CPUOffloadingManager(OffloadingManager):
) )
) )
blocks = self._allocate_blocks(keys_to_store) chunks = self._allocate_chunks(keys_to_store)
assert len(blocks) == len(keys_to_store), ( assert len(chunks) == len(keys_to_store), (
"Block pool did not allocate the expected number of blocks" "Chunk pool did not allocate the expected number of chunks"
) )
for key, block in zip(keys_to_store, blocks): for key, chunk in zip(keys_to_store, chunks):
self._policy.insert(key, block) self._policy.insert(key, chunk)
self._num_write_pending_blocks += len(keys_to_store) self._num_write_pending_chunks += len(keys_to_store)
# build store specs for allocated blocks # build store specs for allocated chunks
store_spec = self._get_load_store_spec(keys_to_store, blocks) store_spec = self._get_load_store_spec(keys_to_store, chunks)
return PrepareStoreOutput( return PrepareStoreOutput(
keys_to_store=keys_to_store, keys_to_store=keys_to_store,
@@ -260,20 +260,20 @@ class CPUOffloadingManager(OffloadingManager):
if success: if success:
for key in keys: for key in keys:
block = self._policy.get(key) chunk = self._policy.get(key)
if block is not None and not block.is_ready: if chunk is not None and not chunk.is_ready:
block.ref_cnt = 0 chunk.ref_cnt = 0
self._num_write_pending_blocks -= 1 self._num_write_pending_chunks -= 1
self._num_evictable_cache_blocks += 1 self._num_evictable_cache_chunks += 1
self._policy.mark_evictable(key) self._policy.mark_evictable(key)
stored_keys.append(key) stored_keys.append(key)
else: else:
for key in keys: for key in keys:
block = self._policy.get(key) chunk = self._policy.get(key)
if block is not None and not block.is_ready: if chunk is not None and not chunk.is_ready:
self._num_write_pending_blocks -= 1 self._num_write_pending_chunks -= 1
self._policy.remove(key) self._policy.remove(key)
self._free_block(block) self._free_chunk(chunk)
if stored_keys and self.events is not None: if stored_keys and self.events is not None:
self.events.append( self.events.append(
@@ -286,17 +286,17 @@ class CPUOffloadingManager(OffloadingManager):
@override @override
def reset_cache(self) -> None: def reset_cache(self) -> None:
# Clear ALL blocks unconditionally. The scheduler's _stale_job_threshold # Clear ALL chunks unconditionally. The scheduler's _stale_job_threshold
# guarantees that complete_load / complete_store are never called for # guarantees that complete_load / complete_store are never called for
# pre-reset jobs, so no lazy cleanup is needed. The scheduler also # pre-reset jobs, so no lazy cleanup is needed. The scheduler also
# flushes in-flight load job IDs to the workers before any new stores # flushes in-flight load job IDs to the workers before any new stores
# can begin, preventing a cross-direction data race on reused offload block IDs. # can begin, preventing a cross-direction data race on reused offload chunk IDs.
self._policy.clear() self._policy.clear()
self._num_evictable_cache_blocks = 0 self._num_evictable_cache_chunks = 0
self._num_write_pending_blocks = 0 self._num_write_pending_chunks = 0
self._free_list.clear() self._free_list.clear()
self._num_allocated_blocks = 0 self._num_allocated_chunks = 0
@override @override
def take_events(self) -> Iterable[OffloadingEvent]: def take_events(self) -> Iterable[OffloadingEvent]:
@@ -309,11 +309,11 @@ class CPUOffloadingManager(OffloadingManager):
# Compute cache usage. # Compute cache usage.
num_used = ( num_used = (
self._num_allocated_blocks self._num_allocated_chunks
- len(self._free_list) - len(self._free_list)
- self._num_evictable_cache_blocks - self._num_evictable_cache_chunks
) )
usage = num_used / self._num_blocks if self._num_blocks > 0 else 0.0 usage = num_used / self._num_chunks if self._num_chunks > 0 else 0.0
stats.set_gauge(CPUOffloadingMetrics.CPU_CACHE_USAGE_PERC, usage) stats.set_gauge(CPUOffloadingMetrics.CPU_CACHE_USAGE_PERC, usage)
for allocation_size in self.allocation_sizes_in_current_batch: for allocation_size in self.allocation_sizes_in_current_batch:
@@ -323,8 +323,8 @@ class CPUOffloadingManager(OffloadingManager):
self.allocation_sizes_in_current_batch.clear() self.allocation_sizes_in_current_batch.clear()
write_usage = ( write_usage = (
self._num_write_pending_blocks / self._num_blocks self._num_write_pending_chunks / self._num_chunks
if self._num_blocks > 0 if self._num_chunks > 0
else 0.0 else 0.0
) )
read_usage = max(usage - write_usage, 0.0) read_usage = max(usage - write_usage, 0.0)
+33 -33
View File
@@ -6,7 +6,7 @@ from collections.abc import Iterable, Iterator
from typing_extensions import override from typing_extensions import override
from vllm.v1.kv_offload.base import OffloadKey, ReqContext from vllm.v1.kv_offload.base import OffloadKey, ReqContext
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
class ARCCachePolicy(CachePolicy): class ARCCachePolicy(CachePolicy):
@@ -14,15 +14,15 @@ class ARCCachePolicy(CachePolicy):
ARC (Adaptive Replacement Cache) cache policy. ARC (Adaptive Replacement Cache) cache policy.
Data Structures: Data Structures:
T1: Recent cache containing blocks accessed once. T1: Recent cache containing chunks accessed once.
T2: Frequent cache containing blocks accessed multiple times. T2: Frequent cache containing chunks accessed multiple times.
B1/B2: Ghost lists tracking recently evicted blocks from T1/T2. B1/B2: Ghost lists tracking recently evicted chunks from T1/T2.
target_t1_size: Adaptive target size for the T1 partition. target_t1_size: Adaptive target size for the T1 partition.
Algorithm Flow: Algorithm Flow:
1. Cache lookup (lookup): 1. Cache lookup (lookup):
Searches T1 and T2 for block hashes and counts consecutive hits Searches T1 and T2 for chunk hashes and counts consecutive hits
until a miss or non-ready block is encountered. until a miss or non-ready chunk is encountered.
2. Cache touch (touch) - Adaptive Learning: 2. Cache touch (touch) - Adaptive Learning:
For each key (in reverse order): For each key (in reverse order):
@@ -31,15 +31,15 @@ class ARCCachePolicy(CachePolicy):
- If in B1 ghost list: Increase target_t1_size. - If in B1 ghost list: Increase target_t1_size.
- If in B2 ghost list: Decrease target_t1_size. - If in B2 ghost list: Decrease target_t1_size.
3. Block eviction (evict) - Adaptive Replacement: 3. Chunk eviction (evict) - Adaptive Replacement:
Determines eviction source based on adaptive target: Determines eviction source based on adaptive target:
- If T1 size >= target_t1_size: Evict from T1, add to B1. - If T1 size >= target_t1_size: Evict from T1, add to B1.
- Otherwise: Evict from T2, add to B2. - Otherwise: Evict from T2, add to B2.
Finally, bound each ghost list size. Finally, bound each ghost list size.
4. Block insertion (insert): 4. Chunk insertion (insert):
New blocks are always inserted into T1 and removed from B1/B2 if New chunks are always inserted into T1 and removed from B1/B2 if
present. Blocks may later be promoted to T2 during touch operations. present. Chunks may later be promoted to T2 during touch operations.
Adaptive Behavior: Adaptive Behavior:
The algorithm self-tunes the recency vs. frequency trade-off: The algorithm self-tunes the recency vs. frequency trade-off:
@@ -50,19 +50,19 @@ class ARCCachePolicy(CachePolicy):
def __init__(self, cache_capacity: int): def __init__(self, cache_capacity: int):
super().__init__(cache_capacity) super().__init__(cache_capacity)
self.target_t1_size: float = 0.0 self.target_t1_size: float = 0.0
self.t1: OrderedDict[OffloadKey, BlockStatus] = OrderedDict() self.t1: OrderedDict[OffloadKey, ChunkStatus] = OrderedDict()
self.t2: OrderedDict[OffloadKey, BlockStatus] = OrderedDict() self.t2: OrderedDict[OffloadKey, ChunkStatus] = OrderedDict()
# key -> None (only care about presence) # key -> None (only care about presence)
self.b1: OrderedDict[OffloadKey, None] = OrderedDict() self.b1: OrderedDict[OffloadKey, None] = OrderedDict()
self.b2: OrderedDict[OffloadKey, None] = OrderedDict() self.b2: OrderedDict[OffloadKey, None] = OrderedDict()
@override @override
def get(self, key: OffloadKey) -> BlockStatus | None: def get(self, key: OffloadKey) -> ChunkStatus | None:
return self.t1.get(key) or self.t2.get(key) return self.t1.get(key) or self.t2.get(key)
@override @override
def insert(self, key: OffloadKey, block: BlockStatus) -> None: def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
self.t1[key] = block self.t1[key] = chunk
self.b1.pop(key, None) self.b1.pop(key, None)
self.b2.pop(key, None) self.b2.pop(key, None)
@@ -75,13 +75,13 @@ class ARCCachePolicy(CachePolicy):
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None: def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
for key in reversed(list(keys)): for key in reversed(list(keys)):
if key in self.t1: if key in self.t1:
block = self.t1.pop(key) chunk = self.t1.pop(key)
if not block.is_ready: if not chunk.is_ready:
# block was just prepared to be stored, not really touched # chunk was just prepared to be stored, not really touched
# twice — keep it in T1 and mark as most recently used # twice — keep it in T1 and mark as most recently used
self.t1[key] = block self.t1[key] = chunk
else: else:
self.t2[key] = block self.t2[key] = chunk
elif key in self.t2: elif key in self.t2:
self.t2.move_to_end(key) self.t2.move_to_end(key)
@@ -111,31 +111,31 @@ class ARCCachePolicy(CachePolicy):
@override @override
def evict( def evict(
self, n: int, protected: set[OffloadKey] self, n: int, protected: set[OffloadKey]
) -> list[tuple[OffloadKey, BlockStatus]] | None: ) -> list[tuple[OffloadKey, ChunkStatus]] | None:
if n == 0: if n == 0:
return [] return []
# Collect candidates atomically: simulate T1 size changes as we select, # Collect candidates atomically: simulate T1 size changes as we select,
# but do not modify actual data structures until all n are found. # but do not modify actual data structures until all n are found.
candidates: list[ candidates: list[
tuple[OffloadKey, BlockStatus, bool] tuple[OffloadKey, ChunkStatus, bool]
] = [] # (key, block, from_t1) ] = [] # (key, chunk, from_t1)
virtual_t1_size = len(self.t1) virtual_t1_size = len(self.t1)
# Keep the scans monotonic: restarting from the LRU end after every # Keep the scans monotonic: restarting from the LRU end after every
# selection makes a batch eviction quadratic in the number of blocks. # selection makes a batch eviction quadratic in the number of chunks.
t1_iter = iter(self.t1.items()) t1_iter = iter(self.t1.items())
t2_iter = iter(self.t2.items()) t2_iter = iter(self.t2.items())
def next_candidate( def next_candidate(
entries: Iterator[tuple[OffloadKey, BlockStatus]], entries: Iterator[tuple[OffloadKey, ChunkStatus]],
) -> tuple[OffloadKey, BlockStatus] | None: ) -> tuple[OffloadKey, ChunkStatus] | None:
for key, block in entries: for key, chunk in entries:
if block.ref_cnt == 0 and key not in protected: if chunk.ref_cnt == 0 and key not in protected:
return key, block return key, chunk
return None return None
for _ in range(n): for _ in range(n):
candidate: tuple[OffloadKey, BlockStatus, bool] | None = None candidate: tuple[OffloadKey, ChunkStatus, bool] | None = None
if virtual_t1_size >= int(self.target_t1_size): if virtual_t1_size >= int(self.target_t1_size):
entry = next_candidate(t1_iter) entry = next_candidate(t1_iter)
@@ -158,15 +158,15 @@ class ARCCachePolicy(CachePolicy):
candidates.append(candidate) candidates.append(candidate)
# Apply all evictions now that we know n candidates exist. # Apply all evictions now that we know n candidates exist.
result: list[tuple[OffloadKey, BlockStatus]] = [] result: list[tuple[OffloadKey, ChunkStatus]] = []
for key, block, from_t1 in candidates: for key, chunk, from_t1 in candidates:
if from_t1: if from_t1:
del self.t1[key] del self.t1[key]
self.b1[key] = None self.b1[key] = None
else: else:
del self.t2[key] del self.t2[key]
self.b2[key] = None self.b2[key] = None
result.append((key, block)) result.append((key, chunk))
# Trim ghost lists to cache_capacity. # Trim ghost lists to cache_capacity.
for ghost in (self.b1, self.b2): for ghost in (self.b1, self.b2):
+26 -26
View File
@@ -7,36 +7,36 @@ from collections.abc import Iterable
from vllm.v1.kv_offload.base import OffloadKey, ReqContext from vllm.v1.kv_offload.base import OffloadKey, ReqContext
class BlockStatus(ctypes.Structure): class ChunkStatus(ctypes.Structure):
""" """
Offloading status for a single block of KV data. Offloading status for a single chunk of KV data.
Holds the following information: Holds the following information:
ref_cnt - the current number of transfers using this block as a source. ref_cnt - the current number of transfers using this chunk as a source.
A value of -1 indicates the block is not yet ready to be read. A value of -1 indicates the chunk is not yet ready to be read.
block_id - index of the physical CPU buffer slot. chunk_id - index of the physical CPU buffer slot.
""" """
_fields_ = [("ref_cnt", ctypes.c_int32), ("block_id", ctypes.c_int64)] _fields_ = [("ref_cnt", ctypes.c_int32), ("chunk_id", ctypes.c_int64)]
def __init__(self, block_id: int): def __init__(self, chunk_id: int):
super().__init__() super().__init__()
# initialize block as "not ready" (ref_cnt = -1) # initialize chunk as "not ready" (ref_cnt = -1)
self.ref_cnt = -1 self.ref_cnt = -1
self.block_id = block_id self.chunk_id = chunk_id
@property @property
def is_ready(self) -> bool: def is_ready(self) -> bool:
""" """
Returns whether the block is ready to be read. Returns whether the chunk is ready to be read.
""" """
return self.ref_cnt >= 0 return self.ref_cnt >= 0
class CachePolicy(ABC): class CachePolicy(ABC):
""" """
Encapsulates both block organization (data structures) and replacement Encapsulates both chunk organization (data structures) and replacement
decisions (which block to evict). LRU and ARC differ in both dimensions — decisions (which chunk to evict). LRU and ARC differ in both dimensions —
ARC's ghost lists and target_t1_size live at the intersection of storage ARC's ghost lists and target_t1_size live at the intersection of storage
and eviction, so they cannot be separated cleanly. and eviction, so they cannot be separated cleanly.
""" """
@@ -45,35 +45,35 @@ class CachePolicy(ABC):
self.cache_capacity = cache_capacity self.cache_capacity = cache_capacity
@abstractmethod @abstractmethod
def get(self, key: OffloadKey) -> BlockStatus | None: def get(self, key: OffloadKey) -> ChunkStatus | None:
"""Find block in data structures. Returns None if not present.""" """Find chunk in data structures. Returns None if not present."""
@abstractmethod @abstractmethod
def insert(self, key: OffloadKey, block: BlockStatus) -> None: def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
"""Add a newly allocated block. For ARC: also removes from ghost lists.""" """Add a newly allocated chunk. For ARC: also removes from ghost lists."""
@abstractmethod @abstractmethod
def remove(self, key: OffloadKey) -> None: def remove(self, key: OffloadKey) -> None:
"""Remove a block (used to clean up after a failed store).""" """Remove a chunk (used to clean up after a failed store)."""
@abstractmethod @abstractmethod
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None: def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
""" """
Mark blocks as recently used. Mark chunks as recently used.
Args: Args:
keys: Blocks to mark as recently used. keys: Chunks to mark as recently used.
req_context: Per-request context for the request touching these blocks. req_context: Per-request context for the request touching these chunks.
""" """
@abstractmethod @abstractmethod
def evict( def evict(
self, n: int, protected: set[OffloadKey] self, n: int, protected: set[OffloadKey]
) -> list[tuple[OffloadKey, BlockStatus]] | None: ) -> list[tuple[OffloadKey, ChunkStatus]] | None:
""" """
Evict exactly n blocks, skipping any in protected. Evict exactly n chunks, skipping any in protected.
Returns a list of (key, block) for the evicted blocks, Returns a list of (key, chunk) for the evicted chunks,
or None if n evictions cannot be satisfied. The operation is atomic: or None if n evictions cannot be satisfied. The operation is atomic:
if None is returned, no state changes are made. if None is returned, no state changes are made.
@@ -84,15 +84,15 @@ class CachePolicy(ABC):
@abstractmethod @abstractmethod
def clear(self) -> None: def clear(self) -> None:
""" """
Remove ALL blocks regardless of ref_cnt. Remove ALL chunks regardless of ref_cnt.
Ghost lists and adaptive state are also reset. Ghost lists and adaptive state are also reset.
""" """
def mark_evictable(self, key: OffloadKey) -> None: def mark_evictable(self, key: OffloadKey) -> None:
"""Called when a block's ref_cnt transitions to 0.""" """Called when a chunk's ref_cnt transitions to 0."""
return return
def mark_non_evictable(self, key: OffloadKey) -> None: def mark_non_evictable(self, key: OffloadKey) -> None:
"""Called when a block's ref_cnt transitions from 0.""" """Called when a chunk's ref_cnt transitions from 0."""
return return
+29 -29
View File
@@ -6,7 +6,7 @@ from collections.abc import Iterable
from typing_extensions import override from typing_extensions import override
from vllm.v1.kv_offload.base import OffloadKey, ReqContext from vllm.v1.kv_offload.base import OffloadKey, ReqContext
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
class LRUCachePolicy(CachePolicy): class LRUCachePolicy(CachePolicy):
@@ -20,71 +20,71 @@ class LRUCachePolicy(CachePolicy):
def __init__(self, cache_capacity: int): def __init__(self, cache_capacity: int):
super().__init__(cache_capacity) super().__init__(cache_capacity)
# Blocks with ref_cnt 0 (not participating in any loads/stores) ordered in LRU # Chunks with ref_cnt 0 (not participating in any loads/stores) ordered in LRU
self.evictable_blocks: OrderedDict[OffloadKey, None] = OrderedDict() self.evictable_chunks: OrderedDict[OffloadKey, None] = OrderedDict()
self.blocks: dict[OffloadKey, BlockStatus] = {} self.chunks: dict[OffloadKey, ChunkStatus] = {}
@override @override
def get(self, key: OffloadKey) -> BlockStatus | None: def get(self, key: OffloadKey) -> ChunkStatus | None:
return self.blocks.get(key) return self.chunks.get(key)
@override @override
def insert(self, key: OffloadKey, block: BlockStatus) -> None: def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
self.blocks[key] = block self.chunks[key] = chunk
if block.ref_cnt == 0: if chunk.ref_cnt == 0:
self.evictable_blocks[key] = None self.evictable_chunks[key] = None
@override @override
def remove(self, key: OffloadKey) -> None: def remove(self, key: OffloadKey) -> None:
del self.blocks[key] del self.chunks[key]
self.evictable_blocks.pop(key, None) self.evictable_chunks.pop(key, None)
@override @override
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None: def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
for key in reversed(list(keys)): for key in reversed(list(keys)):
if key in self.evictable_blocks: if key in self.evictable_chunks:
self.evictable_blocks.move_to_end(key) self.evictable_chunks.move_to_end(key)
# active blocks are untouched as they are non-evictable now. They # active chunks are untouched as they are non-evictable now. They
# will eventually reach the end of evictable_blocks when they finish. # will eventually reach the end of evictable_chunks when they finish.
@override @override
def clear(self) -> None: def clear(self) -> None:
self.evictable_blocks.clear() self.evictable_chunks.clear()
self.blocks.clear() self.chunks.clear()
@override @override
def evict( def evict(
self, n: int, protected: set[OffloadKey] self, n: int, protected: set[OffloadKey]
) -> list[tuple[OffloadKey, BlockStatus]] | None: ) -> list[tuple[OffloadKey, ChunkStatus]] | None:
if n == 0: if n == 0:
return [] return []
candidates: list[tuple[OffloadKey, BlockStatus]] = [] candidates: list[tuple[OffloadKey, ChunkStatus]] = []
for key, _ in self.evictable_blocks.items(): for key, _ in self.evictable_chunks.items():
if key in protected: if key in protected:
continue continue
block = self.blocks[key] chunk = self.chunks[key]
assert block.ref_cnt == 0 assert chunk.ref_cnt == 0
candidates.append((key, block)) candidates.append((key, chunk))
if len(candidates) == n: if len(candidates) == n:
break break
if len(candidates) < n: if len(candidates) < n:
return None return None
for key, _ in candidates: for key, _ in candidates:
del self.evictable_blocks[key] del self.evictable_chunks[key]
del self.blocks[key] del self.chunks[key]
return candidates return candidates
@override @override
def mark_evictable(self, key: OffloadKey) -> None: def mark_evictable(self, key: OffloadKey) -> None:
# blocks can become evictable when, # chunks can become evictable when,
# store completes - i.e. ref_cnt -1 -> 0 # not in evictable list # store completes - i.e. ref_cnt -1 -> 0 # not in evictable list
# all loads complete - i.e ref_cnt 1 -> 0 # not in evictable list # all loads complete - i.e ref_cnt 1 -> 0 # not in evictable list
self.evictable_blocks[key] = None self.evictable_chunks[key] = None
@override @override
def mark_non_evictable(self, key: OffloadKey) -> None: def mark_non_evictable(self, key: OffloadKey) -> None:
# key must have been in the evictable list. # key must have been in the evictable list.
del self.evictable_blocks[key] del self.evictable_chunks[key]
+26 -26
View File
@@ -82,24 +82,24 @@ class SharedOffloadRegion:
def __init__( def __init__(
self, self,
engine_id: str, engine_id: str,
num_blocks: int, num_chunks: int,
rank: int | None, rank: int | None,
kv_bytes_per_block: int, kv_bytes_per_chunk: int,
cpu_page_size: int, cpu_page_size: int,
barrier: Callable[[], None] | None = None, barrier: Callable[[], None] | None = None,
) -> None: ) -> None:
self.page_size = mmap.PAGESIZE self.page_size = mmap.PAGESIZE
assert kv_bytes_per_block % self.page_size == 0 assert kv_bytes_per_chunk % self.page_size == 0
self.num_blocks = num_blocks self.num_chunks = num_chunks
self._row_stride = kv_bytes_per_block self._row_stride = kv_bytes_per_chunk
self.total_size_bytes = self.num_blocks * self._row_stride self.total_size_bytes = self.num_chunks * self._row_stride
self.mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap" self.mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap"
self._creator = False # set True only if this worker creates the file self._creator = False # set True only if this worker creates the file
self.rank = rank self.rank = rank
if rank is not None: if rank is not None:
# byte offset to this worker's first slot within each block row # byte offset to this worker's first slot within each chunk row
self._worker_offset = rank * cpu_page_size self._worker_offset = rank * cpu_page_size
# exclusive upper bound for this worker's area within each row # exclusive upper bound for this worker's area within each row
self._worker_area_end = (rank + 1) * cpu_page_size self._worker_area_end = (rank + 1) * cpu_page_size
@@ -182,19 +182,19 @@ class SharedOffloadRegion:
populate_write_fn = _get_populate_write_fn(self.mmap_obj) populate_write_fn = _get_populate_write_fn(self.mmap_obj)
if rank is not None: if rank is not None:
# Populate only this worker's pages (one slot per block row). # Populate only this worker's pages (one slot per chunk row).
worker_offset = rank * cpu_page_size worker_offset = rank * cpu_page_size
_t0 = time.perf_counter() _t0 = time.perf_counter()
page_size = self.page_size page_size = self.page_size
for block in range(num_blocks): for chunk in range(num_chunks):
raw_offset = block * self._row_stride + worker_offset raw_offset = chunk * self._row_stride + worker_offset
aligned_offset = (raw_offset // page_size) * page_size aligned_offset = (raw_offset // page_size) * page_size
end = raw_offset + cpu_page_size end = raw_offset + cpu_page_size
aligned_length = end - aligned_offset aligned_length = end - aligned_offset
populate_write_fn(self.mmap_obj, aligned_offset, aligned_length) populate_write_fn(self.mmap_obj, aligned_offset, aligned_length)
logger.debug( logger.debug(
"MADV_POPULATE_WRITE loop: %d blocks in %.3f s", "MADV_POPULATE_WRITE loop: %d chunks in %.3f s",
num_blocks, num_chunks,
time.perf_counter() - _t0, time.perf_counter() - _t0,
) )
else: else:
@@ -215,22 +215,22 @@ class SharedOffloadRegion:
Must be called once per canonical tensor. The full mmap layout is: Must be called once per canonical tensor. The full mmap layout is:
worker0_block0 | worker1_block0 | ... | worker{M-1}_block0 worker0_chunk0 | worker1_chunk0 | ... | worker{M-1}_chunk0
worker0_block1 | worker1_block1 | ... | worker{M-1}_block1 worker0_chunk1 | worker1_chunk1 | ... | worker{M-1}_chunk1
... ...
Each worker_block cell is cpu_page_size bytes and holds all canonical Each worker_chunk cell is cpu_page_size bytes and holds all canonical
tensors for that worker and block concatenated: tensors for that worker and chunk concatenated:
[ tensor0_data | tensor1_data | ... | tensor{L-1}_data ] [ tensor0_data | tensor1_data | ... | tensor{L-1}_data ]
Consecutive rows are separated by row_stride = cpu_page_size * M. Consecutive rows are separated by row_stride = cpu_page_size * M.
Returns an int8 tensor of shape (num_blocks, tensor_page_size) with stride Returns an int8 tensor of shape (num_chunks, tensor_page_size) with stride
(row_stride, 1). Using int8 keeps stride == bytes, so swap_blocks (row_stride, 1). Using int8 keeps stride == bytes, so swap_blocks
address arithmetic works without any dtype conversion. address arithmetic works without any dtype conversion.
Args: Args:
tensor_page_size: Bytes per block for this tensor. tensor_page_size: Bytes per chunk for this tensor.
""" """
assert self.rank is not None assert self.rank is not None
new_offset = self._worker_offset + tensor_page_size new_offset = self._worker_offset + tensor_page_size
@@ -241,7 +241,7 @@ class SharedOffloadRegion:
) )
worker_layer_view = torch.as_strided( worker_layer_view = torch.as_strided(
self._base, self._base,
size=(self.num_blocks, tensor_page_size), size=(self.num_chunks, tensor_page_size),
stride=(self._row_stride, 1), stride=(self._row_stride, 1),
storage_offset=self._worker_offset, storage_offset=self._worker_offset,
) )
@@ -266,8 +266,8 @@ class SharedOffloadRegion:
_canonical_offset=0, then advances by each tensor's size _canonical_offset=0, then advances by each tensor's size
Each canonical_t{i} cell is that tensor's canonical page for the Each canonical_t{i} cell is that tensor's canonical page for the
block. Canonical areas are carved consecutively from the start of chunk. Canonical areas are carved consecutively from the start of
each block row; consecutive rows are separated by row_stride. Every each chunk row; consecutive rows are separated by row_stride. Every
worker gets the identical byte ranges and writes only its disjoint worker gets the identical byte ranges and writes only its disjoint
bytes within them, as described by its canonical mappings — unlike bytes within them, as described by its canonical mappings — unlike
create_next_worker_view, which gives each worker a private create_next_worker_view, which gives each worker a private
@@ -279,13 +279,13 @@ class SharedOffloadRegion:
where one canonical copy replaces world_size worker copies. where one canonical copy replaces world_size worker copies.
Args: Args:
tensor_page_size: Canonical bytes per block for this tensor. tensor_page_size: Canonical bytes per chunk for this tensor.
""" """
new_offset = self._canonical_offset + tensor_page_size new_offset = self._canonical_offset + tensor_page_size
assert new_offset <= self._row_stride assert new_offset <= self._row_stride
view = torch.as_strided( view = torch.as_strided(
self._base, self._base,
size=(self.num_blocks, tensor_page_size), size=(self.num_chunks, tensor_page_size),
stride=(self._row_stride, 1), stride=(self._row_stride, 1),
storage_offset=self._canonical_offset, storage_offset=self._canonical_offset,
) )
@@ -296,10 +296,10 @@ class SharedOffloadRegion:
def create_kv_memoryview(self) -> memoryview: def create_kv_memoryview(self) -> memoryview:
"""Return a zero-copy memoryview over the entire KV buffer. """Return a zero-copy memoryview over the entire KV buffer.
Shape: (num_blocks, row_stride_bytes). Secondary tiers address Shape: (num_chunks, row_stride_bytes). Secondary tiers address
block *b* as ``view[b]``. chunk *b* as ``view[b]``.
""" """
kv_tensor = self._base.view(self.num_blocks, self._row_stride) kv_tensor = self._base.view(self.num_chunks, self._row_stride)
np_arr = kv_tensor.numpy() np_arr = kv_tensor.numpy()
assert np_arr.ctypes.data == self._base.data_ptr(), ( assert np_arr.ctypes.data == self._base.data_ptr(), (
"view()/numpy() created a copy instead of sharing the mmap buffer; " "view()/numpy() created a copy instead of sharing the mmap buffer; "
+14 -14
View File
@@ -73,7 +73,7 @@ class CPUOffloadingSpec(OffloadingSpec):
), ),
CPUOffloadingMetrics.CPU_ALLOCATION_SIZE: OffloadingHistogramMetadata( CPUOffloadingMetrics.CPU_ALLOCATION_SIZE: OffloadingHistogramMetadata(
documentation=( documentation=(
"Histogram of the number of CPU blocks requested by each " "Histogram of the number of CPU chunks requested by each "
"KV offload prepare_store call." "KV offload prepare_store call."
), ),
buckets=(1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144), buckets=(1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144),
@@ -101,7 +101,7 @@ class CPUOffloadingSpec(OffloadingSpec):
) )
world_size = config.parallel.world_size world_size = config.parallel.world_size
self.num_blocks = 0 self.num_chunks = 0
self.kv_bytes_per_chunk = 0 self.kv_bytes_per_chunk = 0
self.cpu_page_size_per_worker = 0 self.cpu_page_size_per_worker = 0
self.replicated_layout = config.replicated_layout and self._uses_shared_region() self.replicated_layout = config.replicated_layout and self._uses_shared_region()
@@ -113,17 +113,17 @@ class CPUOffloadingSpec(OffloadingSpec):
# calculate cpu_page_size_per_worker # calculate cpu_page_size_per_worker
self.cpu_page_size_per_worker = kv_bytes_per_chunk // num_copies self.cpu_page_size_per_worker = kv_bytes_per_chunk // num_copies
# calculate num_blocks # calculate num_chunks
aligned_kv_bytes_per_chunk = round_up( aligned_kv_bytes_per_chunk = round_up(
kv_bytes_per_chunk, self.BLOCK_SIZE_ALIGNMENT kv_bytes_per_chunk, self.BLOCK_SIZE_ALIGNMENT
) )
self.num_blocks = int(cpu_bytes_to_use) // aligned_kv_bytes_per_chunk self.num_chunks = int(cpu_bytes_to_use) // aligned_kv_bytes_per_chunk
# Expose aligned_kv_bytes_per_chunk as # Expose aligned_kv_bytes_per_chunk as
# kv_bytes_per_chunk. Note that this might contain # kv_bytes_per_chunk. Note that this might contain
# some padding. i.e. each offloaded block is of the form, # some padding. i.e. each offloaded chunk is of the form,
# |--- W0-B0---|---- W1-B0---| ... |---- Wn-B0---| *** maybe-pad *** | # |--- W0-C0---|---- W1-C0---| ... |---- Wn-C0---| *** maybe-pad *** |
# or |--- B0 (single copy) ---| *** maybe-pad *** | # or |--- C0 (single copy) ---| *** maybe-pad *** |
self.kv_bytes_per_chunk = aligned_kv_bytes_per_chunk self.kv_bytes_per_chunk = aligned_kv_bytes_per_chunk
# scheduler-side # scheduler-side
@@ -140,7 +140,7 @@ class CPUOffloadingSpec(OffloadingSpec):
@override @override
def get_manager(self) -> OffloadingManager: def get_manager(self) -> OffloadingManager:
if not self._manager: if not self._manager:
# store_threshold: how many times a block must be offered for # store_threshold: how many times a chunk must be offered for
# storage before it is eligible for CPU offloading. Values < 2 # storage before it is eligible for CPU offloading. Values < 2
# disable filtering (a threshold of 1 equals no filter; 0 is the # disable filtering (a threshold of 1 equals no filter; 0 is the
# default). # default).
@@ -150,7 +150,7 @@ class CPUOffloadingSpec(OffloadingSpec):
max_tracker_size = int(self.extra_config.get("max_tracker_size", 64_000)) max_tracker_size = int(self.extra_config.get("max_tracker_size", 64_000))
self._manager = CPUOffloadingManager( self._manager = CPUOffloadingManager(
num_blocks=self.num_blocks, num_chunks=self.num_chunks,
cache_policy=self.eviction_policy, cache_policy=self.eviction_policy,
cache_policy_module_path=self.cache_policy_module_path, cache_policy_module_path=self.cache_policy_module_path,
enable_events=self.kv_events_config.enable_kv_cache_events, enable_events=self.kv_events_config.enable_kv_cache_events,
@@ -166,9 +166,9 @@ class CPUOffloadingSpec(OffloadingSpec):
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker: def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
mmap_region: SharedOffloadRegion | None = None mmap_region: SharedOffloadRegion | None = None
# num_blocks == 0 would size the region to zero bytes, which cannot be # num_chunks == 0 would size the region to zero bytes, which cannot be
# mmap'd; fall back to the tensor path (empty tensors) as before. # mmap'd; fall back to the tensor path (empty tensors) as before.
if self._uses_shared_region() and self.num_blocks > 0: if self._uses_shared_region() and self.num_chunks > 0:
# Replicated layout puts all ranks on slot 0 (single MLA copy); # Replicated layout puts all ranks on slot 0 (single MLA copy);
# otherwise each rank takes its own slot by physical device index. # otherwise each rank takes its own slot by physical device index.
if self.replicated_layout: if self.replicated_layout:
@@ -178,9 +178,9 @@ class CPUOffloadingSpec(OffloadingSpec):
rank = torch.accelerator.current_device_index() % world_size rank = torch.accelerator.current_device_index() % world_size
mmap_region = SharedOffloadRegion( mmap_region = SharedOffloadRegion(
engine_id=self.config.engine_id, engine_id=self.config.engine_id,
num_blocks=self.num_blocks, num_chunks=self.num_chunks,
rank=rank, rank=rank,
kv_bytes_per_block=self.kv_bytes_per_chunk, kv_bytes_per_chunk=self.kv_bytes_per_chunk,
cpu_page_size=self.cpu_page_size_per_worker, cpu_page_size=self.cpu_page_size_per_worker,
barrier=_all_workers_barrier, barrier=_all_workers_barrier,
) )
@@ -188,7 +188,7 @@ class CPUOffloadingSpec(OffloadingSpec):
return CPUOffloadingWorker( return CPUOffloadingWorker(
kv_caches=kv_caches, kv_caches=kv_caches,
blocks_per_chunk=self.blocks_per_chunk, blocks_per_chunk=self.blocks_per_chunk,
num_cpu_blocks=self.num_blocks, num_cpu_chunks=self.num_chunks,
mmap_region=mmap_region, mmap_region=mmap_region,
) )
except Exception: except Exception:
+23 -23
View File
@@ -45,8 +45,8 @@ class TieringOffloadingMetrics:
WRITE_TIME = "vllm:kv_offload_tiering_write_time" WRITE_TIME = "vllm:kv_offload_tiering_write_time"
PROMOTION_JOB_FAILURES = "vllm:kv_offload_tiering_promotion_job_failures" PROMOTION_JOB_FAILURES = "vllm:kv_offload_tiering_promotion_job_failures"
CASCADE_JOB_FAILURES = "vllm:kv_offload_tiering_cascade_job_failures" CASCADE_JOB_FAILURES = "vllm:kv_offload_tiering_cascade_job_failures"
BLOCK_QUERIES = "vllm:kv_offload_tiering_chunk_queries" CHUNK_QUERIES = "vllm:kv_offload_tiering_chunk_queries"
BLOCK_HITS = "vllm:kv_offload_tiering_chunk_hits" CHUNK_HITS = "vllm:kv_offload_tiering_chunk_hits"
PRIMARY_WRITE_USAGE_PERC = "vllm:kv_offload_tiering_primary_write_usage_perc" PRIMARY_WRITE_USAGE_PERC = "vllm:kv_offload_tiering_primary_write_usage_perc"
PRIMARY_READ_USAGE_PERC = "vllm:kv_offload_tiering_primary_read_usage_perc" PRIMARY_READ_USAGE_PERC = "vllm:kv_offload_tiering_primary_read_usage_perc"
PROMOTION_ALLOCATION_FAILURES = ( PROMOTION_ALLOCATION_FAILURES = (
@@ -62,7 +62,7 @@ class TransferJob:
job_id: JobId job_id: JobId
keys: Collection[OffloadKey] keys: Collection[OffloadKey]
block_ids: np.ndarray chunk_ids: np.ndarray
is_promotion: bool is_promotion: bool
req_context: ReqContext req_context: ReqContext
@@ -90,14 +90,14 @@ class ParentManager(ABC):
Required call sequence for each remote request: Required call sequence for each remote request:
1. on_new_request(req_context) — set up per-request state 1. on_new_request(req_context) — set up per-request state
2. lookup(key, req_context) — check block availability 2. lookup(key, req_context) — check chunk availability
(repeat per block) (repeat per chunk)
3. create_store_job(keys, req_context) — pin blocks and get a 3. create_store_job(keys, req_context) — pin chunks and get a
job handle job handle
4. on_request_finished(req_context) — clean up per-request state 4. on_request_finished(req_context) — clean up per-request state
Steps 2-3 may be interleaved. Step 4 must be called even if no Steps 2-3 may be interleaved. Step 4 must be called even if no
blocks were found, to avoid leaking async lookup state (e.g. in chunks were found, to avoid leaking async lookup state (e.g. in
the fs tier's AsyncLookupManager). the fs tier's AsyncLookupManager).
""" """
@@ -155,23 +155,23 @@ class SecondaryTierManager(ABC):
@abstractmethod @abstractmethod
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
""" """
Check whether a block exists in this secondary tier. Check whether a chunk exists in this secondary tier.
Args: Args:
key: Offload key to look up. key: Offload key to look up.
req_context: per-request context (e.g. kv_transfer_params). req_context: per-request context (e.g. kv_transfer_params).
Returns: Returns:
HIT if the block is present and ready, HIT if the chunk is present and ready,
MISS if not found, MISS if not found,
or RETRY if the block is being transferred (retry later). or RETRY if the chunk is being transferred (retry later).
""" """
pass pass
@abstractmethod @abstractmethod
def submit_store(self, job_metadata: TransferJob) -> None: def submit_store(self, job_metadata: TransferJob) -> None:
""" """
Submit an async job to store blocks from the primary tier to this Submit an async job to store chunks from the primary tier to this
secondary tier. secondary tier.
This method must be lightweight and non-blocking: allocate metadata This method must be lightweight and non-blocking: allocate metadata
@@ -179,19 +179,19 @@ class SecondaryTierManager(ABC):
calling thread. calling thread.
Preconditions (guaranteed by the framework): Preconditions (guaranteed by the framework):
- ``job_metadata.block_ids`` are valid primary-tier slots, pinned - ``job_metadata.chunk_ids`` are valid primary-tier slots, pinned
(ref-counted) for the duration of the transfer. (ref-counted) for the duration of the transfer.
The implementation is responsible for: The implementation is responsible for:
1. Filtering out blocks already present in this tier 1. Filtering out chunks already present in this tier
2. Evicting blocks if capacity is needed 2. Evicting chunks if capacity is needed
3. Allocating space in this tier 3. Allocating space in this tier
4. Submitting the async transfer (read from primary via block_ids) 4. Submitting the async transfer (read from primary via chunk_ids)
Report completion via ``get_finished_jobs()``. Report completion via ``get_finished_jobs()``.
Args: Args:
job_metadata: Job metadata including job_id, keys, and block_ids job_metadata: Job metadata including job_id, keys, and chunk_ids
identifying the primary-tier slots to read from. identifying the primary-tier slots to read from.
""" """
pass pass
@@ -199,24 +199,24 @@ class SecondaryTierManager(ABC):
@abstractmethod @abstractmethod
def submit_load(self, job_metadata: TransferJob) -> None: def submit_load(self, job_metadata: TransferJob) -> None:
""" """
Submit an async job to load blocks from this secondary tier to the Submit an async job to load chunks from this secondary tier to the
primary tier. primary tier.
This method must be lightweight and non-blocking: mark blocks as This method must be lightweight and non-blocking: mark chunks as
in-flight and submit the transfer, but do NOT perform the data copy in-flight and submit the transfer, but do NOT perform the data copy
on the calling thread. on the calling thread.
Preconditions (guaranteed by the framework): Preconditions (guaranteed by the framework):
- ``job_metadata.block_ids`` are allocated primary-tier slots - ``job_metadata.chunk_ids`` are allocated primary-tier slots
ready to receive data. ready to receive data.
The implementation must copy data from this tier into the The implementation must copy data from this tier into the
primary-tier slots identified by ``block_ids``. primary-tier slots identified by ``chunk_ids``.
Report completion via ``get_finished_jobs()``. Report completion via ``get_finished_jobs()``.
Args: Args:
job_metadata: Job metadata including job_id, keys, and block_ids job_metadata: Job metadata including job_id, keys, and chunk_ids
identifying the primary-tier slots to write into. identifying the primary-tier slots to write into.
""" """
pass pass
@@ -249,7 +249,7 @@ class SecondaryTierManager(ABC):
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext): def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
""" """
Mark blocks as recently used for eviction policy. Mark chunks as recently used for eviction policy.
Args: Args:
keys: Offload keys to mark as recently used. keys: Offload keys to mark as recently used.
@@ -263,7 +263,7 @@ class SecondaryTierManager(ABC):
Called when a new request is first seen by the scheduler. Called when a new request is first seen by the scheduler.
Returns a RequestOffloadingContext expressing this tier's preference Returns a RequestOffloadingContext expressing this tier's preference
for how blocks should be offloaded for this request. for how chunks should be offloaded for this request.
Args: Args:
req_context: Per-request context. req_context: Per-request context.
+22 -22
View File
@@ -3,7 +3,7 @@
""" """
ExampleSecondaryTierManager: A simple in-memory secondary tier. ExampleSecondaryTierManager: A simple in-memory secondary tier.
This implementation provides a minimal secondary tier that stores blocks This implementation provides a minimal secondary tier that stores chunks
in memory (using a dictionary) with immediate completion. It serves as a in memory (using a dictionary) with immediate completion. It serves as a
reference for writing new tiers and is useful for testing the reference for writing new tiers and is useful for testing the
TieringOffloadingManager without requiring actual storage or network backends. TieringOffloadingManager without requiring actual storage or network backends.
@@ -39,7 +39,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
A simple in-memory secondary tier. A simple in-memory secondary tier.
This implementation: This implementation:
- Stores blocks in a dictionary (key -> True) - Stores chunks in a dictionary (key -> True)
- Completes transfers immediately (synchronous) - Completes transfers immediately (synchronous)
""" """
@@ -69,45 +69,45 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
) )
# key -> True (only care about presence) # key -> True (only care about presence)
self.blocks: dict[OffloadKey, bool] = {} self.chunks: dict[OffloadKey, bool] = {}
# Completed jobs waiting to be retrieved by get_finished_jobs() # Completed jobs waiting to be retrieved by get_finished_jobs()
self.completed_jobs: list[JobResult] = [] self.completed_jobs: list[JobResult] = []
assert primary_kv_view.strides is not None assert primary_kv_view.strides is not None
self._block_size = primary_kv_view.strides[0] self._chunk_size = primary_kv_view.strides[0]
@override @override
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
""" """
Check whether a block exists in this secondary tier. Check whether a chunk exists in this secondary tier.
Args: Args:
key: Offload key to look up. key: Offload key to look up.
req_context: Per-request context. req_context: Per-request context.
Returns: Returns:
HIT if the block is present, MISS if not found. HIT if the chunk is present, MISS if not found.
""" """
return LookupResult.HIT if key in self.blocks else LookupResult.MISS return LookupResult.HIT if key in self.chunks else LookupResult.MISS
@override @override
def submit_store(self, job_metadata: TransferJob) -> None: def submit_store(self, job_metadata: TransferJob) -> None:
""" """
Submit a job to store blocks from primary tier to this tier. Submit a job to store chunks from primary tier to this tier.
Args: Args:
job_metadata: Job metadata including job_id, keys, and job_metadata: Job metadata including job_id, keys, and
spec for reading blocks from the primary tier. spec for reading chunks from the primary tier.
""" """
keys = job_metadata.keys keys = job_metadata.keys
block_ids = job_metadata.block_ids chunk_ids = job_metadata.chunk_ids
assert len(keys) == len(block_ids), ( assert len(keys) == len(chunk_ids), (
f"Length mismatch: {len(keys)} keys but {len(block_ids)} block_ids" f"Length mismatch: {len(keys)} keys but {len(chunk_ids)} chunk_ids"
) )
for key in keys: for key in keys:
self.blocks[key] = True self.chunks[key] = True
self.completed_jobs.append( self.completed_jobs.append(
JobResult( JobResult(
job_id=job_metadata.job_id, job_id=job_metadata.job_id,
@@ -119,21 +119,21 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
@override @override
def submit_load(self, job_metadata: TransferJob) -> None: def submit_load(self, job_metadata: TransferJob) -> None:
""" """
Submit a job to load blocks from this tier to primary tier. Submit a job to load chunks from this tier to primary tier.
Args: Args:
job_metadata: Job metadata including job_id, keys, and job_metadata: Job metadata including job_id, keys, and
spec for writing blocks into the primary tier. spec for writing chunks into the primary tier.
""" """
keys = job_metadata.keys keys = job_metadata.keys
block_ids = job_metadata.block_ids chunk_ids = job_metadata.chunk_ids
assert len(keys) == len(block_ids), ( assert len(keys) == len(chunk_ids), (
f"Length mismatch: {len(keys)} keys but {len(block_ids)} block_ids" f"Length mismatch: {len(keys)} keys but {len(chunk_ids)} chunk_ids"
) )
for key in keys: for key in keys:
if key not in self.blocks: if key not in self.chunks:
self.completed_jobs.append( self.completed_jobs.append(
JobResult(job_id=job_metadata.job_id, success=False) JobResult(job_id=job_metadata.job_id, success=False)
) )
@@ -170,6 +170,6 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
completes, so there is nothing to wait for.""" completes, so there is nothing to wait for."""
return return
def get_num_blocks(self) -> int: def get_num_chunks(self) -> int:
"""Get the number of blocks currently stored in this tier.""" """Get the number of chunks currently stored in this tier."""
return len(self.blocks) return len(self.chunks)
+2 -2
View File
@@ -223,7 +223,7 @@ class FileSystemTierManager(SecondaryTierManager):
batch_store_block, batch_store_block,
[self.file_mapper.get_file_name(key) for key in keys], [self.file_mapper.get_file_name(key) for key in keys],
self._primary_kv_view, self._primary_kv_view,
[int(bid) * self._block_size for bid in job_metadata.block_ids], [int(cid) * self._block_size for cid in job_metadata.chunk_ids],
self._block_size, self._block_size,
self._use_o_direct, self._use_o_direct,
) )
@@ -237,7 +237,7 @@ class FileSystemTierManager(SecondaryTierManager):
keys = list(job_metadata.keys) keys = list(job_metadata.keys)
self._load_job_keys[job_id] = keys self._load_job_keys[job_id] = keys
paths = [self.file_mapper.get_file_name(key) for key in keys] paths = [self.file_mapper.get_file_name(key) for key in keys]
offsets = [int(bid) * self._block_size for bid in job_metadata.block_ids] offsets = [int(cid) * self._block_size for cid in job_metadata.chunk_ids]
def load_task() -> None: def load_task() -> None:
try: try:
+64 -64
View File
@@ -8,16 +8,16 @@ and zero or more secondary tiers (Storage, Network, etc.) to provide
hierarchical KV cache offloading. hierarchical KV cache offloading.
Key Design Principles: Key Design Principles:
1. Always offload to all tiers — When a block is stored to the primary tier, 1. Always offload to all tiers — When a chunk is stored to the primary tier,
it is cascaded to ALL secondary tiers it is cascaded to ALL secondary tiers
2. Primary tier is the gateway — Secondary tiers cannot access GPU memory 2. Primary tier is the gateway — Secondary tiers cannot access GPU memory
directly; all data flows through the CPU primary tier directly; all data flows through the CPU primary tier
3. Staged promotion — Blocks in secondary tiers must be promoted to the 3. Staged promotion — Chunks in secondary tiers must be promoted to the
primary tier before GPU can access them primary tier before GPU can access them
4. Transparent retry mechanism — Return None from lookup() to signal 4. Transparent retry mechanism — Return None from lookup() to signal
"data is being promoted, try later" "data is being promoted, try later"
5. ref_cnt as eviction protection — primary.prepare_read() increments ref_cnt, 5. ref_cnt as eviction protection — primary.prepare_read() increments ref_cnt,
protecting blocks from eviction until complete_read() is called protecting chunks from eviction until complete_read() is called
""" """
import time import time
@@ -61,11 +61,11 @@ logger = init_logger(__name__)
@dataclass @dataclass
class PendingPromotion: class PendingPromotion:
"""Accumulator for blocks awaiting submit_load() for one (tier, request).""" """Accumulator for chunks awaiting submit_load() for one (tier, request)."""
req_context: ReqContext req_context: ReqContext
keys: list[OffloadKey] = field(default_factory=list) keys: list[OffloadKey] = field(default_factory=list)
block_ids: list[int] = field(default_factory=list) chunk_ids: list[int] = field(default_factory=list)
@dataclass(slots=True) @dataclass(slots=True)
@@ -94,14 +94,14 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
def __init__( def __init__(
self, self,
num_blocks: int, num_chunks: int,
mmap_region: SharedOffloadRegion, mmap_region: SharedOffloadRegion,
cache_policy: str = "lru", cache_policy: str = "lru",
cache_policy_module_path: str | None = None, cache_policy_module_path: str | None = None,
enable_events: bool = False, enable_events: bool = False,
): ):
super().__init__( super().__init__(
num_blocks=num_blocks, num_chunks=num_chunks,
cache_policy=cache_policy, cache_policy=cache_policy,
cache_policy_module_path=cache_policy_module_path, cache_policy_module_path=cache_policy_module_path,
enable_events=enable_events, enable_events=enable_events,
@@ -120,9 +120,9 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
def get_kv_memoryview(self) -> memoryview: def get_kv_memoryview(self) -> memoryview:
"""Return the memoryview over the primary tier's KV cache buffer. """Return the memoryview over the primary tier's KV cache buffer.
The view has shape (num_blocks, row_stride_bytes) and is backed by the The view has shape (num_chunks, row_stride_bytes) and is backed by the
SharedOffloadRegion mmap. Secondary tiers address block *b* as SharedOffloadRegion mmap. Secondary tiers address chunk *c* as
``view[b]``. ``view[c]``.
""" """
return self._kv_memoryview return self._kv_memoryview
@@ -206,14 +206,14 @@ class TieringOffloadingManager(OffloadingManager):
assert primary_view.strides is not None assert primary_view.strides is not None
self._metrics = TieringMetricsTracker( self._metrics = TieringMetricsTracker(
tier_types=[tier.tier_type for tier in self.secondary_tiers], tier_types=[tier.tier_type for tier in self.secondary_tiers],
num_primary_blocks=self.primary_tier._num_blocks, num_primary_chunks=self.primary_tier._num_chunks,
primary_block_size=primary_view.strides[0], primary_chunk_size=primary_view.strides[0],
) )
# Pending promotion requests accumulated during lookup() calls; flushed # Pending promotion requests accumulated during lookup() calls; flushed
# as one batched submit_load() per (tier, request) in on_schedule_end(). # as one batched submit_load() per (tier, request) in on_schedule_end().
# Outer key: tier index. Inner key: req_context.req_id — the same ReqContext # Outer key: tier index. Inner key: req_context.req_id — the same ReqContext
# object is reused for all block lookups of a given request per engine step. # object is reused for all chunk lookups of a given request per engine step.
self._pending_load_submissions: dict[int, dict[str, PendingPromotion]] = {} self._pending_load_submissions: dict[int, dict[str, PendingPromotion]] = {}
# Gate for once-per-step execution of _maybe_process_finished_jobs(). # Gate for once-per-step execution of _maybe_process_finished_jobs().
@@ -305,7 +305,7 @@ class TieringOffloadingManager(OffloadingManager):
2. For completed stores (primary→secondary): calls primary.complete_read() 2. For completed stores (primary→secondary): calls primary.complete_read()
to decrement ref_cnt to decrement ref_cnt
3. For completed loads (secondary→primary): calls primary.complete_write() 3. For completed loads (secondary→primary): calls primary.complete_write()
to make blocks available to make chunks available
""" """
for i, tier in enumerate(self.secondary_tiers): for i, tier in enumerate(self.secondary_tiers):
for completed_job in tier.get_finished_jobs(): for completed_job in tier.get_finished_jobs():
@@ -324,11 +324,11 @@ class TieringOffloadingManager(OffloadingManager):
if transfer_job.is_promotion: if transfer_job.is_promotion:
# secondary→primary transfer (promotion) completed. # secondary→primary transfer (promotion) completed.
# Make blocks available in primary tier. # Make chunks available in primary tier.
self._complete_promotion(job_metadata, completed_job) self._complete_promotion(job_metadata, completed_job)
else: else:
# primary→secondary transfer completed. # primary→secondary transfer completed.
# Decrement ref_cnt on primary blocks. # Decrement ref_cnt on primary chunks.
self.primary_tier.complete_read( self.primary_tier.complete_read(
transfer_job.keys, transfer_job.req_context transfer_job.keys, transfer_job.req_context
) )
@@ -342,7 +342,7 @@ class TieringOffloadingManager(OffloadingManager):
exclude_tier_idx: int | None = None, exclude_tier_idx: int | None = None,
) -> LookupResult: ) -> LookupResult:
""" """
Check whether a single block is offloaded and ready. Check whether a single chunk is offloaded and ready.
Algorithm: Algorithm:
1. Process any completed async jobs first. 1. Process any completed async jobs first.
@@ -351,20 +351,20 @@ class TieringOffloadingManager(OffloadingManager):
hit and initiate promotion. hit and initiate promotion.
Args: Args:
key: Block hash to look up. key: Chunk hash to look up.
req_context: Per-request context. req_context: Per-request context.
Returns: Returns:
HIT — block is ready in the primary tier. HIT — chunk is ready in the primary tier.
HIT_PENDING — block found but not yet readable (write HIT_PENDING — chunk found but not yet readable (write
in-flight on the primary tier). in-flight on the primary tier).
RETRY — promotion started or a secondary tier is busy. RETRY — promotion started or a secondary tier is busy.
MISS — block not found in any tier, or primary is full MISS — chunk not found in any tier, or primary is full
and cannot accept a promotion. and cannot accept a promotion.
""" """
# Poll first so a promotion that finished since the last call is # Poll first so a promotion that finished since the last call is
# already reflected as HIT (not stale HIT_PENDING/MISS) below, and # already reflected as HIT (not stale HIT_PENDING/MISS) below, and
# so blocks freed by cascade or promotion completions are evictable # so chunks freed by cascade or promotion completions are evictable
# in time for a promotion this lookup may initiate. # in time for a promotion this lookup may initiate.
self._maybe_process_finished_jobs() self._maybe_process_finished_jobs()
@@ -424,30 +424,30 @@ class TieringOffloadingManager(OffloadingManager):
req_context: ReqContext, req_context: ReqContext,
) -> bool: ) -> bool:
""" """
Queue a block for promotion from a secondary tier to the primary tier. Queue a chunk for promotion from a secondary tier to the primary tier.
Allocates space in the primary tier immediately (sets ref_cnt=-1 so Allocates space in the primary tier immediately (sets ref_cnt=-1 so
subsequent lookups within the same step see the slot as in-flight), subsequent lookups within the same step see the slot as in-flight),
then defers the actual submit_load() call to _flush_pending_promotions() then defers the actual submit_load() call to _flush_pending_promotions()
so all blocks queued during one engine step are submitted as a single so all chunks queued during one engine step are submitted as a single
batched job. batched job.
Args: Args:
tier_idx: The secondary tier index to promote from tier_idx: The secondary tier index to promote from
key: Block to promote key: Chunk to promote
req_context: Per-request context forwarded to primary.prepare_write(). req_context: Per-request context forwarded to primary.prepare_write().
Returns: Returns:
True if promotion was initiated, False if primary tier is full. True if promotion was initiated, False if primary tier is full.
""" """
# Allocate space in primary tier for promoted block. # Allocate space in primary tier for promoted chunk.
# Must happen immediately so primary.lookup() returns None (in-flight) # Must happen immediately so primary.lookup() returns None (in-flight)
# for this key on any subsequent lookup() call within the same step, # for this key on any subsequent lookup() call within the same step,
# preventing duplicate promotion attempts. # preventing duplicate promotion attempts.
primary_write_result = self.primary_tier.prepare_write([key], req_context) primary_write_result = self.primary_tier.prepare_write([key], req_context)
if primary_write_result is None: if primary_write_result is None:
# Primary tier is full; caller should treat the block as unavailable # Primary tier is full; caller should treat the chunk as unavailable
# rather than retrying indefinitely. # rather than retrying indefinitely.
self._metrics.on_promotion_allocation_failure() self._metrics.on_promotion_allocation_failure()
return False return False
@@ -455,16 +455,16 @@ class TieringOffloadingManager(OffloadingManager):
store_spec = primary_write_result.store_spec store_spec = primary_write_result.store_spec
assert isinstance(store_spec, CPULoadStoreSpec) assert isinstance(store_spec, CPULoadStoreSpec)
# Defer submit_load to on_schedule_end(). Group by (tier, request) so # Defer submit_load to on_schedule_end(). Group by (tier, request) so
# each request's blocks are submitted as one batched job per tier. # each request's chunks are submitted as one batched job per tier.
tier_pending = self._pending_load_submissions.setdefault(tier_idx, {}) tier_pending = self._pending_load_submissions.setdefault(tier_idx, {})
ctx_id = req_context.req_id ctx_id = req_context.req_id
if ctx_id not in tier_pending: if ctx_id not in tier_pending:
tier_pending[ctx_id] = PendingPromotion( tier_pending[ctx_id] = PendingPromotion(
keys=[], block_ids=[], req_context=req_context keys=[], chunk_ids=[], req_context=req_context
) )
entry = tier_pending[ctx_id] entry = tier_pending[ctx_id]
entry.keys.extend(primary_write_result.keys_to_store) entry.keys.extend(primary_write_result.keys_to_store)
entry.block_ids.extend(store_spec.block_ids) entry.chunk_ids.extend(store_spec.chunk_ids)
return True return True
def _flush_pending_promotions(self) -> None: def _flush_pending_promotions(self) -> None:
@@ -483,7 +483,7 @@ class TieringOffloadingManager(OffloadingManager):
job_metadata = TransferJob( job_metadata = TransferJob(
job_id=job_id, job_id=job_id,
keys=entry.keys, keys=entry.keys,
block_ids=np.array(entry.block_ids, dtype=np.int32), chunk_ids=np.array(entry.chunk_ids, dtype=np.int32),
is_promotion=True, is_promotion=True,
req_context=entry.req_context, req_context=entry.req_context,
) )
@@ -497,16 +497,16 @@ class TieringOffloadingManager(OffloadingManager):
self, keys: Collection[OffloadKey], req_context: ReqContext self, keys: Collection[OffloadKey], req_context: ReqContext
) -> LoadStoreSpec: ) -> LoadStoreSpec:
""" """
Prepare blocks to be loaded from primary tier to GPU. Prepare chunks to be loaded from primary tier to GPU.
Callers only pass keys already confirmed HIT by lookup() earlier this Callers only pass keys already confirmed HIT by lookup() earlier this
step. step.
This increments ref_cnt on the blocks in the primary tier, protecting This increments ref_cnt on the chunks in the primary tier, protecting
them from eviction during the transfer. them from eviction during the transfer.
Args: Args:
keys: Blocks to prepare for loading. keys: Chunks to prepare for loading.
req_context: Per-request context. req_context: Per-request context.
Returns: Returns:
@@ -517,10 +517,10 @@ class TieringOffloadingManager(OffloadingManager):
@override @override
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext): def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
""" """
Mark blocks as recently used in all tiers. Mark chunks as recently used in all tiers.
Args: Args:
keys: Blocks to mark as recently used. keys: Chunks to mark as recently used.
req_context: Per-request context. req_context: Per-request context.
""" """
self.primary_tier.touch(keys, req_context) self.primary_tier.touch(keys, req_context)
@@ -530,13 +530,13 @@ class TieringOffloadingManager(OffloadingManager):
@override @override
def complete_load(self, keys: Collection[OffloadKey], req_context: ReqContext): def complete_load(self, keys: Collection[OffloadKey], req_context: ReqContext):
""" """
Mark blocks as done loading from primary tier to GPU. Mark chunks as done loading from primary tier to GPU.
This decrements ref_cnt on the blocks in the primary tier, allowing This decrements ref_cnt on the chunks in the primary tier, allowing
them to be evicted again. them to be evicted again.
Args: Args:
keys: Blocks that finished loading. keys: Chunks that finished loading.
req_context: Per-request context. req_context: Per-request context.
""" """
self.primary_tier.complete_load(keys, req_context) self.primary_tier.complete_load(keys, req_context)
@@ -546,37 +546,37 @@ class TieringOffloadingManager(OffloadingManager):
self, keys: Collection[OffloadKey], req_context: ReqContext self, keys: Collection[OffloadKey], req_context: ReqContext
) -> PrepareStoreOutput | None: ) -> PrepareStoreOutput | None:
""" """
Prepare blocks to be stored from GPU to primary tier. Prepare chunks to be stored from GPU to primary tier.
CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure
that any completed async transfers have their ref_cnt decremented that any completed async transfers have their ref_cnt decremented
before the primary tier makes eviction decisions. before the primary tier makes eviction decisions.
For request-level tiers, blocks already present in the primary tier For request-level tiers, chunks already present in the primary tier
are immediately cascaded via submit_store(). are immediately cascaded via submit_store().
Args: Args:
keys: Blocks to prepare for storing. keys: Chunks to prepare for storing.
req_context: Per-request context. req_context: Per-request context.
Returns: Returns:
PrepareStoreOutput describing where to store blocks and what was PrepareStoreOutput describing where to store chunks and what was
evicted, or None if store cannot proceed. evicted, or None if store cannot proceed.
""" """
# Step 1: Poll for completed async jobs FIRST # Step 1: Poll for completed async jobs FIRST
# _process_finished_jobs() handles two kinds of completions here: # _process_finished_jobs() handles two kinds of completions here:
# - Cascade completions (store to a secondary tier, either a local # - Cascade completions (store to a secondary tier, either a local
# cascade or a store job created for a remote requester via # cascade or a store job created for a remote requester via
# create_store_job()): decrements ref_cnt on the primary blocks # create_store_job()): decrements ref_cnt on the primary chunks
# that were read, making them evictable again once ref_cnt hits 0. # that were read, making them evictable again once ref_cnt hits 0.
# - Promotion completions (secondary->primary loads): sets a # - Promotion completions (secondary->primary loads): sets a
# not-yet-ready block's ref_cnt from -1 to 0 via complete_write(), # not-yet-ready chunk's ref_cnt from -1 to 0 via complete_write(),
# making it evictable for the first time. # making it evictable for the first time.
# Both must be accounted for before the eviction decision below. # Both must be accounted for before the eviction decision below.
self._maybe_process_finished_jobs() self._maybe_process_finished_jobs()
# Step 2: Store to primary tier (new blocks only). # Step 2: Store to primary tier (new chunks only).
# Cascading of these newly-stored blocks to ALL secondary tiers # Cascading of these newly-stored chunks to ALL secondary tiers
# happens later in complete_store(), after the GPU→Primary transfer # happens later in complete_store(), after the GPU→Primary transfer
# completes. # completes.
primary_result = self.primary_tier.prepare_store(keys, req_context) primary_result = self.primary_tier.prepare_store(keys, req_context)
@@ -588,7 +588,7 @@ class TieringOffloadingManager(OffloadingManager):
state = self._req_state[req_context.req_id] state = self._req_state[req_context.req_id]
state.pending_primary_stores += 1 state.pending_primary_stores += 1
# Step 3: For request-level tiers, cascade blocks already in primary # Step 3: For request-level tiers, cascade chunks already in primary
request_level_tiers = self._req_state[req_context.req_id].request_level_tiers request_level_tiers = self._req_state[req_context.req_id].request_level_tiers
if request_level_tiers: if request_level_tiers:
keys_to_store_set = set(primary_result.keys_to_store) keys_to_store_set = set(primary_result.keys_to_store)
@@ -596,13 +596,13 @@ class TieringOffloadingManager(OffloadingManager):
k for k in keys if k not in keys_to_store_set k for k in keys if k not in keys_to_store_set
) )
if keys_already_in_primary: if keys_already_in_primary:
self._cascade_existing_blocks_to_request_level_tiers( self._cascade_existing_chunks_to_request_level_tiers(
keys_already_in_primary, req_context, request_level_tiers keys_already_in_primary, req_context, request_level_tiers
) )
return primary_result return primary_result
def _cascade_existing_blocks_to_request_level_tiers( def _cascade_existing_chunks_to_request_level_tiers(
self, self,
keys: Sequence[OffloadKey], keys: Sequence[OffloadKey],
req_context: ReqContext, req_context: ReqContext,
@@ -610,7 +610,7 @@ class TieringOffloadingManager(OffloadingManager):
) -> None: ) -> None:
""" """
For tiers that requested request-level policy, submit_store() for For tiers that requested request-level policy, submit_store() for
blocks that are already present in the primary tier. chunks that are already present in the primary tier.
A key whose primary write is still in flight (HIT_PENDING) cannot be A key whose primary write is still in flight (HIT_PENDING) cannot be
dropped: prepare_store already excluded it as present, and the dropped: prepare_store already excluded it as present, and the
@@ -653,7 +653,7 @@ class TieringOffloadingManager(OffloadingManager):
continue continue
assert state.request_level_tiers assert state.request_level_tiers
keys, state.pending_cascade_keys = state.pending_cascade_keys, [] keys, state.pending_cascade_keys = state.pending_cascade_keys, []
self._cascade_existing_blocks_to_request_level_tiers( self._cascade_existing_chunks_to_request_level_tiers(
keys, state.req_context, state.request_level_tiers keys, state.req_context, state.request_level_tiers
) )
self._maybe_finalize_request(req_id) self._maybe_finalize_request(req_id)
@@ -666,30 +666,30 @@ class TieringOffloadingManager(OffloadingManager):
success: bool = True, success: bool = True,
) -> None: ) -> None:
""" """
Mark blocks as done storing from GPU to primary tier. Mark chunks as done storing from GPU to primary tier.
This is where secondary tier cascading happens — after blocks are This is where secondary tier cascading happens — after chunks are
confirmed to be in the primary tier, they are cascaded to ALL confirmed to be in the primary tier, they are cascaded to ALL
secondary tiers. secondary tiers.
For each secondary tier: For each secondary tier:
1. Call primary.prepare_read() to get LoadStoreSpec AND increment 1. Call primary.prepare_read() to get LoadStoreSpec AND increment
ref_cnt (protecting blocks during async transfer) ref_cnt (protecting chunks during async transfer)
2. Call tier.submit_store() to start async transfer: primary→secondary 2. Call tier.submit_store() to start async transfer: primary→secondary
3. Track the job in _store_jobs dictionary 3. Track the job in _store_jobs dictionary
Args: Args:
keys: Blocks that finished storing. keys: Chunks that finished storing.
success: Whether the GPU→primary transfer succeeded. success: Whether the GPU→primary transfer succeeded.
req_context: Per-request context forwarded to primary.prepare_read(). req_context: Per-request context forwarded to primary.prepare_read().
""" """
# Step 1: Complete store in primary tier (makes blocks loadable) # Step 1: Complete store in primary tier (makes chunks loadable)
self.primary_tier.complete_store(keys, req_context, success) self.primary_tier.complete_store(keys, req_context, success)
if success: if success:
# Step 2: Cascade to ALL secondary tiers # Step 2: Cascade to ALL secondary tiers
# For each secondary tier, call primary.prepare_read() to get the # For each secondary tier, call primary.prepare_read() to get the
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from # LoadStoreSpec AND to increment ref_cnt (protecting chunks from
# eviction during the async transfer). One prepare_read() call per # eviction during the async transfer). One prepare_read() call per
# secondary tier. # secondary tier.
for tier_idx, tier in enumerate(self.secondary_tiers): for tier_idx, tier in enumerate(self.secondary_tiers):
@@ -710,22 +710,22 @@ class TieringOffloadingManager(OffloadingManager):
req_context: ReqContext, req_context: ReqContext,
tier_idx: int = 0, tier_idx: int = 0,
) -> TransferJob: ) -> TransferJob:
"""Pin blocks in the primary tier and create a tracked store job. """Pin chunks in the primary tier and create a tracked store job.
Calls prepare_read() to increment ref_cnt (protecting blocks Calls prepare_read() to increment ref_cnt (protecting chunks
from eviction during the async transfer), allocates a job ID, from eviction during the async transfer), allocates a job ID,
and registers the job in _jobs. and registers the job in _jobs.
The caller is responsible for the actual data transfer and The caller is responsible for the actual data transfer and
reporting completion via get_finished_jobs(). reporting completion via get_finished_jobs().
""" """
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context) primary_chunks_spec = self.primary_tier.prepare_read(keys, req_context)
assert isinstance(primary_blocks_spec, CPULoadStoreSpec) assert isinstance(primary_chunks_spec, CPULoadStoreSpec)
job_id = self._next_job_id() job_id = self._next_job_id()
job_metadata = TransferJob( job_metadata = TransferJob(
job_id=job_id, job_id=job_id,
keys=keys, keys=keys,
block_ids=primary_blocks_spec.block_ids, chunk_ids=primary_chunks_spec.chunk_ids,
is_promotion=False, is_promotion=False,
req_context=req_context, req_context=req_context,
) )
@@ -760,7 +760,7 @@ class TieringOffloadingManager(OffloadingManager):
policy = ( policy = (
OffloadPolicy.REQUEST_LEVEL OffloadPolicy.REQUEST_LEVEL
if state.request_level_tiers if state.request_level_tiers
else OffloadPolicy.BLOCK_LEVEL else OffloadPolicy.CHUNK_LEVEL
) )
return RequestOffloadingContext(policy=policy) return RequestOffloadingContext(policy=policy)
+23 -23
View File
@@ -39,20 +39,20 @@ class _RequestMetricsState:
class _TierState: class _TierState:
active_promotion_count: int = 0 active_promotion_count: int = 0
active_cascade_count: int = 0 active_cascade_count: int = 0
primary_write_block_count: int = 0 primary_write_chunk_count: int = 0
primary_read_block_count: int = 0 primary_read_chunk_count: int = 0
class TieringMetricsTracker: class TieringMetricsTracker:
def __init__( def __init__(
self, self,
tier_types: list[str], tier_types: list[str],
num_primary_blocks: int, num_primary_chunks: int,
primary_block_size: int, primary_chunk_size: int,
) -> None: ) -> None:
self._tier_types = tier_types self._tier_types = tier_types
self._num_primary_blocks = num_primary_blocks self._num_primary_chunks = num_primary_chunks
self._primary_block_size = primary_block_size self._primary_chunk_size = primary_chunk_size
self._request_states: dict[str, _RequestMetricsState] = {} self._request_states: dict[str, _RequestMetricsState] = {}
self._tier_states = [_TierState() for _ in tier_types] self._tier_states = [_TierState() for _ in tier_types]
self._stats = OffloadingConnectorStats() self._stats = OffloadingConnectorStats()
@@ -111,13 +111,13 @@ class TieringMetricsTracker:
def on_job_registered(self, job_metadata: _JobMetadataLike) -> None: def on_job_registered(self, job_metadata: _JobMetadataLike) -> None:
transfer_job = job_metadata.transfer_job transfer_job = job_metadata.transfer_job
state = self._tier_states[job_metadata.tier_idx] state = self._tier_states[job_metadata.tier_idx]
block_count = len(transfer_job.block_ids) chunk_count = len(transfer_job.chunk_ids)
if transfer_job.is_promotion: if transfer_job.is_promotion:
state.active_promotion_count += 1 state.active_promotion_count += 1
state.primary_write_block_count += block_count state.primary_write_chunk_count += chunk_count
else: else:
state.active_cascade_count += 1 state.active_cascade_count += 1
state.primary_read_block_count += block_count state.primary_read_chunk_count += chunk_count
def on_job_finished( def on_job_finished(
self, job_metadata: _JobMetadataLike, result: JobResult self, job_metadata: _JobMetadataLike, result: JobResult
@@ -149,25 +149,25 @@ class TieringMetricsTracker:
assert all( assert all(
state.active_promotion_count == 0 state.active_promotion_count == 0
and state.active_cascade_count == 0 and state.active_cascade_count == 0
and state.primary_write_block_count == 0 and state.primary_write_chunk_count == 0
and state.primary_read_block_count == 0 and state.primary_read_chunk_count == 0
for state in self._tier_states for state in self._tier_states
) )
def _decrement_tier_state(self, job_metadata: _JobMetadataLike) -> None: def _decrement_tier_state(self, job_metadata: _JobMetadataLike) -> None:
transfer_job = job_metadata.transfer_job transfer_job = job_metadata.transfer_job
state = self._tier_states[job_metadata.tier_idx] state = self._tier_states[job_metadata.tier_idx]
block_count = len(transfer_job.block_ids) chunk_count = len(transfer_job.chunk_ids)
if transfer_job.is_promotion: if transfer_job.is_promotion:
assert state.active_promotion_count > 0 assert state.active_promotion_count > 0
state.active_promotion_count -= 1 state.active_promotion_count -= 1
state.primary_write_block_count -= block_count state.primary_write_chunk_count -= chunk_count
assert state.primary_write_block_count >= 0 assert state.primary_write_chunk_count >= 0
else: else:
assert state.active_cascade_count > 0 assert state.active_cascade_count > 0
state.active_cascade_count -= 1 state.active_cascade_count -= 1
state.primary_read_block_count -= block_count state.primary_read_chunk_count -= chunk_count
assert state.primary_read_block_count >= 0 assert state.primary_read_chunk_count >= 0
def _observe_finished_job_stats( def _observe_finished_job_stats(
self, self,
@@ -199,7 +199,7 @@ class TieringMetricsTracker:
if transfer_job.is_promotion if transfer_job.is_promotion
else TieringOffloadingMetrics.WRITE_TIME else TieringOffloadingMetrics.WRITE_TIME
) )
transfer_size = completed_key_count * self._primary_block_size transfer_size = completed_key_count * self._primary_chunk_size
self._stats.increase_counter(bytes_metric, transfer_size, labelvalues) self._stats.increase_counter(bytes_metric, transfer_size, labelvalues)
if completed_job.transfer_time is not None: if completed_job.transfer_time is not None:
self._stats.increase_counter( self._stats.increase_counter(
@@ -210,13 +210,13 @@ class TieringMetricsTracker:
for tier_idx, state in enumerate(self._tier_states): for tier_idx, state in enumerate(self._tier_states):
labelvalues = self.tier_label(tier_idx) labelvalues = self.tier_label(tier_idx)
write_usage = ( write_usage = (
state.primary_write_block_count / self._num_primary_blocks state.primary_write_chunk_count / self._num_primary_chunks
if self._num_primary_blocks > 0 if self._num_primary_chunks > 0
else 0.0 else 0.0
) )
read_usage = ( read_usage = (
state.primary_read_block_count / self._num_primary_blocks state.primary_read_chunk_count / self._num_primary_chunks
if self._num_primary_blocks > 0 if self._num_primary_chunks > 0
else 0.0 else 0.0
) )
stats.set_gauge( stats.set_gauge(
@@ -248,12 +248,12 @@ class TieringMetricsTracker:
async_start_time: float | None, async_start_time: float | None,
) -> None: ) -> None:
self._stats.increase_counter( self._stats.increase_counter(
TieringOffloadingMetrics.BLOCK_QUERIES, TieringOffloadingMetrics.CHUNK_QUERIES,
labelvalues=tier_label, labelvalues=tier_label,
) )
if result is LookupResult.HIT: if result is LookupResult.HIT:
self._stats.increase_counter( self._stats.increase_counter(
TieringOffloadingMetrics.BLOCK_HITS, TieringOffloadingMetrics.CHUNK_HITS,
labelvalues=tier_label, labelvalues=tier_label,
) )
self._stats.observe_histogram( self._stats.observe_histogram(
+2 -2
View File
@@ -287,14 +287,14 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys) self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys)
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys) obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
self._submit_transfer( self._submit_transfer(
job_metadata.job_id, job_metadata.block_ids, obj_keys, NIXL_WRITE job_metadata.job_id, job_metadata.chunk_ids, obj_keys, NIXL_WRITE
) )
def submit_load(self, job_metadata: TransferJob) -> None: def submit_load(self, job_metadata: TransferJob) -> None:
self._load_job_keys[job_metadata.job_id] = list(job_metadata.keys) self._load_job_keys[job_metadata.job_id] = list(job_metadata.keys)
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys) obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
self._submit_transfer( self._submit_transfer(
job_metadata.job_id, job_metadata.block_ids, obj_keys, NIXL_READ job_metadata.job_id, job_metadata.chunk_ids, obj_keys, NIXL_READ
) )
def on_request_finished(self, req_context: ReqContext) -> None: def on_request_finished(self, req_context: ReqContext) -> None:
+2 -2
View File
@@ -417,7 +417,7 @@ class P2PSecondaryTierManager(SecondaryTierManager):
def submit_store(self, job_metadata: TransferJob) -> None: def submit_store(self, job_metadata: TransferJob) -> None:
job_id = job_metadata.job_id job_id = job_metadata.job_id
keys = list(job_metadata.keys) keys = list(job_metadata.keys)
block_ids = job_metadata.block_ids.tolist() block_ids = job_metadata.chunk_ids.tolist()
assert len(keys) == len(block_ids) assert len(keys) == len(block_ids)
@@ -476,7 +476,7 @@ class P2PSecondaryTierManager(SecondaryTierManager):
def submit_load(self, job_metadata: TransferJob) -> None: def submit_load(self, job_metadata: TransferJob) -> None:
job_id = job_metadata.job_id job_id = job_metadata.job_id
keys = list(job_metadata.keys) keys = list(job_metadata.keys)
block_ids = job_metadata.block_ids block_ids = job_metadata.chunk_ids
source = job_metadata.req_context.get_state(P2PSourceInfo) source = job_metadata.req_context.get_state(P2PSourceInfo)
logger.debug( logger.debug(
@@ -593,7 +593,7 @@ class ServerRole:
self.add_stored_blocks( self.add_stored_blocks(
lookup.kv_request_id, lookup.kv_request_id,
list(meta.keys), list(meta.keys),
meta.block_ids.tolist(), meta.chunk_ids.tolist(),
meta.job_id, meta.job_id,
round_seq=lookup.round_seq, round_seq=lookup.round_seq,
from_lookup=True, from_lookup=True,
+15 -15
View File
@@ -8,7 +8,7 @@ and configurable secondary tiers (e.g., Storage, Network).
Configuration via kv_connector_extra_config: Configuration via kv_connector_extra_config:
- cpu_bytes_to_use: (required) Bytes to allocate for CPU primary tier - cpu_bytes_to_use: (required) Bytes to allocate for CPU primary tier
- block_size: (optional) Block size for offloaded blocks (default: GPU block size) - block_size: (optional) Tokens per offloaded chunk (default: GPU block size)
- eviction_policy: (optional) Primary tier eviction policy: built-in "lru"/ - eviction_policy: (optional) Primary tier eviction policy: built-in "lru"/
"arc", or the name of a policy registered via CachePolicyFactory, or an "arc", or the name of a policy registered via CachePolicyFactory, or an
out-of-tree CachePolicy class name paired with cache_policy_module_path out-of-tree CachePolicy class name paired with cache_policy_module_path
@@ -105,7 +105,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY] = ( metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY] = (
OffloadingHistogramMetadata( OffloadingHistogramMetadata(
documentation=( documentation=(
"Histogram of blocking time spent in a per-block tier lookup " "Histogram of blocking time spent in a per-chunk tier lookup "
"that resolved as a hit or miss, labeled by tier, in seconds." "that resolved as a hit or miss, labeled by tier, in seconds."
), ),
labelnames=("tier",), labelnames=("tier",),
@@ -127,7 +127,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY] = ( metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY] = (
OffloadingHistogramMetadata( OffloadingHistogramMetadata(
documentation=( documentation=(
"Histogram of wall-clock time from a per-block tier lookup " "Histogram of wall-clock time from a per-chunk tier lookup "
"first returning retry until that same tier lookup resolves " "first returning retry until that same tier lookup resolves "
"as a hit or miss, labeled by tier, in seconds." "as a hit or miss, labeled by tier, in seconds."
), ),
@@ -191,14 +191,14 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
labelnames=("tier",), labelnames=("tier",),
) )
) )
metrics[TieringOffloadingMetrics.BLOCK_QUERIES] = OffloadingCounterMetadata( metrics[TieringOffloadingMetrics.CHUNK_QUERIES] = OffloadingCounterMetadata(
documentation=( documentation=(
"Number of block lookup queries sent to a tier, labeled by tier." "Number of chunk lookup queries sent to a tier, labeled by tier."
), ),
labelnames=("tier",), labelnames=("tier",),
) )
metrics[TieringOffloadingMetrics.BLOCK_HITS] = OffloadingCounterMetadata( metrics[TieringOffloadingMetrics.CHUNK_HITS] = OffloadingCounterMetadata(
documentation="Number of block lookup hits in a tier, labeled by tier.", documentation="Number of chunk lookup hits in a tier, labeled by tier.",
labelnames=("tier",), labelnames=("tier",),
) )
metrics[TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES] = ( metrics[TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES] = (
@@ -300,16 +300,16 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
# primary tier can eagerly create a memoryview over _base. # primary tier can eagerly create a memoryview over _base.
scheduler_mmap = SharedOffloadRegion( scheduler_mmap = SharedOffloadRegion(
engine_id=self._engine_id, engine_id=self._engine_id,
num_blocks=self.num_blocks, num_chunks=self.num_chunks,
rank=None, rank=None,
kv_bytes_per_block=self.kv_bytes_per_chunk, kv_bytes_per_chunk=self.kv_bytes_per_chunk,
cpu_page_size=self.cpu_page_size_per_worker, cpu_page_size=self.cpu_page_size_per_worker,
) )
self._scheduler_mmap = scheduler_mmap self._scheduler_mmap = scheduler_mmap
# Create primary tier (CPU-based) # Create primary tier (CPU-based)
primary_tier = CPUPrimaryTierOffloadingManager( primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=self.num_blocks, num_chunks=self.num_chunks,
cache_policy=self.eviction_policy, cache_policy=self.eviction_policy,
cache_policy_module_path=self.cache_policy_module_path, cache_policy_module_path=self.cache_policy_module_path,
enable_events=self.kv_events_config.enable_kv_cache_events, enable_events=self.kv_events_config.enable_kv_cache_events,
@@ -367,9 +367,9 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
logger.info( logger.info(
"Created TieringOffloadingManager with primary tier " "Created TieringOffloadingManager with primary tier "
"(%s, %s blocks) and %s secondary tier(s)", "(%s, %s chunks) and %s secondary tier(s)",
self.eviction_policy, self.eviction_policy,
self.num_blocks, self.num_chunks,
len(secondary_tiers), len(secondary_tiers),
) )
@@ -393,9 +393,9 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
rank = torch.accelerator.current_device_index() % world_size rank = torch.accelerator.current_device_index() % world_size
worker_mmap = SharedOffloadRegion( worker_mmap = SharedOffloadRegion(
engine_id=self._engine_id, engine_id=self._engine_id,
num_blocks=self.num_blocks, num_chunks=self.num_chunks,
rank=rank, rank=rank,
kv_bytes_per_block=self.kv_bytes_per_chunk, kv_bytes_per_chunk=self.kv_bytes_per_chunk,
cpu_page_size=self.cpu_page_size_per_worker, cpu_page_size=self.cpu_page_size_per_worker,
) )
try: try:
@@ -404,7 +404,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
return CPUOffloadingWorker( return CPUOffloadingWorker(
kv_caches=kv_caches, kv_caches=kv_caches,
blocks_per_chunk=self.blocks_per_chunk, blocks_per_chunk=self.blocks_per_chunk,
num_cpu_blocks=self.num_blocks, num_cpu_chunks=self.num_chunks,
mmap_region=worker_mmap, mmap_region=worker_mmap,
canonical_layout=self.config.canonical_layout, canonical_layout=self.config.canonical_layout,
) )