[Refactor][kv_offload]: rename block→chunk (#52615)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
@@ -22,6 +22,10 @@ flowchart LR
|
||||
CPU <--> SN["..."]
|
||||
```
|
||||
|
||||
## Terminology: Chunks
|
||||
|
||||
The unit of operation is a **chunk** — a fixed-size piece of KV data covering a group of tokens. By default, a chunk maps to a single accelerator block. A configurable `blocks_per_chunk` parameter allows larger chunks, yielding larger I/Os to the host and secondary tiers.
|
||||
|
||||
## Single-Tier Setup (CPU Only)
|
||||
|
||||
```bash
|
||||
|
||||
@@ -704,7 +704,7 @@ def test_prom_metrics_registers_tiering_metrics_from_spec():
|
||||
)
|
||||
|
||||
metric = prom_metrics._offloading_metric_defs[
|
||||
TieringOffloadingMetrics.BLOCK_QUERIES
|
||||
TieringOffloadingMetrics.CHUNK_QUERIES
|
||||
]
|
||||
assert metric.kwargs["labelnames"] == ["model_name", "engine", "tier"]
|
||||
|
||||
|
||||
@@ -96,7 +96,7 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
|
||||
groups.append(
|
||||
KVCacheGroupSpec([f"layer{i}"], kv_spec, is_eagle_group=eagle and i == 1)
|
||||
)
|
||||
manager = CPUOffloadingManager(num_blocks=100)
|
||||
manager = CPUOffloadingManager(num_chunks=100)
|
||||
spec = SimpleNamespace(
|
||||
tokens_per_block=(256, 64, 8),
|
||||
tokens_per_hash=8,
|
||||
@@ -2888,7 +2888,7 @@ class TestEagle:
|
||||
req=req,
|
||||
req_context=ReqContext(req_id="test-req"),
|
||||
offloading_context=RequestOffloadingContext(
|
||||
policy=OffloadPolicy.BLOCK_LEVEL
|
||||
policy=OffloadPolicy.CHUNK_LEVEL
|
||||
),
|
||||
num_locally_computed_tokens=num_computed_tokens,
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ import pytest
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
||||
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
|
||||
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
|
||||
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
|
||||
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
|
||||
|
||||
@@ -20,10 +20,10 @@ class _DummyCachePolicy(CachePolicy):
|
||||
def __init__(self, cache_capacity: int) -> None:
|
||||
self.cache_capacity = cache_capacity
|
||||
|
||||
def get(self, key: OffloadKey) -> BlockStatus | None:
|
||||
def get(self, key: OffloadKey) -> ChunkStatus | None:
|
||||
return None
|
||||
|
||||
def insert(self, key: OffloadKey, block: BlockStatus) -> None:
|
||||
def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
|
||||
pass
|
||||
|
||||
def remove(self, key: OffloadKey) -> None:
|
||||
@@ -34,7 +34,7 @@ class _DummyCachePolicy(CachePolicy):
|
||||
|
||||
def evict(
|
||||
self, n: int, protected: set[OffloadKey]
|
||||
) -> list[tuple[OffloadKey, BlockStatus]] | None:
|
||||
) -> list[tuple[OffloadKey, ChunkStatus]] | None:
|
||||
return None
|
||||
|
||||
def clear(self) -> None:
|
||||
@@ -72,7 +72,7 @@ class TestCachePolicyFactory:
|
||||
policy_cls = CachePolicyFactory.get_cache_policy_cls("dummy")
|
||||
assert policy_cls is _DummyCachePolicy
|
||||
|
||||
manager = CPUOffloadingManager(num_blocks=4, cache_policy="dummy")
|
||||
manager = CPUOffloadingManager(num_chunks=4, cache_policy="dummy")
|
||||
assert isinstance(manager._policy, _DummyCachePolicy)
|
||||
|
||||
def test_unregistered_policy_raises(self):
|
||||
@@ -98,7 +98,7 @@ class TestCachePolicyFactory:
|
||||
"""End-to-end: CPUOffloadingManager resolves an unregistered policy
|
||||
purely from cache_policy_module_path."""
|
||||
manager = CPUOffloadingManager(
|
||||
num_blocks=4,
|
||||
num_chunks=4,
|
||||
cache_policy="_DummyCachePolicy",
|
||||
cache_policy_module_path="tests.v1.kv_offload.cpu.policies.test_factory",
|
||||
)
|
||||
|
||||
@@ -232,9 +232,9 @@ def test_cross_topology_roundtrip(writer_tp: int, reader_tp: int):
|
||||
def canonical_view(rank: int, world_size: int) -> torch.Tensor:
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_blocks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=row_stride,
|
||||
kv_bytes_per_chunk=row_stride,
|
||||
cpu_page_size=row_stride // world_size,
|
||||
)
|
||||
regions.append(region)
|
||||
|
||||
@@ -26,7 +26,7 @@ from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
|
||||
NUM_GPU_BLOCKS = [64]
|
||||
NUM_CPU_BLOCKS = [256]
|
||||
NUM_CPU_CHUNKS = [256]
|
||||
GPU_PAGE_SIZES = [512, 1024]
|
||||
BLOCKS_PER_CHUNK_VALUES = [1, 3]
|
||||
NUM_TENSORS = [4]
|
||||
@@ -225,7 +225,7 @@ def test_worker_syncs_before_cleanup_after_handler_failure(
|
||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
|
||||
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_cpu_chunks", NUM_CPU_CHUNKS)
|
||||
@pytest.mark.parametrize("num_tensors", NUM_TENSORS)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
@@ -241,7 +241,7 @@ def test_transfer(
|
||||
gpu_page_size_bytes: int,
|
||||
blocks_per_chunk: int,
|
||||
num_gpu_blocks: int,
|
||||
num_cpu_blocks: int,
|
||||
num_cpu_chunks: int,
|
||||
num_tensors: int,
|
||||
seed: int,
|
||||
device: str,
|
||||
@@ -288,41 +288,41 @@ def test_transfer(
|
||||
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
|
||||
)
|
||||
simulated_world_size = 2
|
||||
kv_bytes_per_block = (
|
||||
kv_bytes_per_chunk = (
|
||||
cpu_page_size if replicated_layout else cpu_page_size * simulated_world_size
|
||||
)
|
||||
mmap_region = SharedOffloadRegion(
|
||||
engine_id=str(uuid.uuid4()),
|
||||
num_blocks=num_cpu_blocks,
|
||||
num_chunks=num_cpu_chunks,
|
||||
rank=0,
|
||||
kv_bytes_per_block=kv_bytes_per_block,
|
||||
kv_bytes_per_chunk=kv_bytes_per_chunk,
|
||||
cpu_page_size=cpu_page_size,
|
||||
)
|
||||
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
num_cpu_chunks=num_cpu_chunks,
|
||||
mmap_region=mmap_region,
|
||||
)
|
||||
|
||||
# select block mappings
|
||||
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * blocks_per_chunk)
|
||||
cpu_blocks = random.sample(range(num_cpu_blocks), num_mappings)
|
||||
cpu_chunks = random.sample(range(num_cpu_chunks), num_mappings)
|
||||
|
||||
# expand cpu blocks to gpu-page granularity for uniform comparison:
|
||||
# each cpu block maps to blocks_per_chunk consecutive sub-blocks
|
||||
cpu_blocks_expanded = [
|
||||
cpu_block * blocks_per_chunk + j
|
||||
for cpu_block in cpu_blocks
|
||||
# expand cpu chunks to gpu-page granularity for uniform comparison:
|
||||
# each cpu chunk maps to blocks_per_chunk consecutive sub-blocks
|
||||
cpu_chunks_expanded = [
|
||||
cpu_chunk * blocks_per_chunk + j
|
||||
for cpu_chunk in cpu_chunks
|
||||
for j in range(blocks_per_chunk)
|
||||
]
|
||||
|
||||
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU block
|
||||
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU chunk
|
||||
blocks_to_skip = blocks_per_chunk - 1
|
||||
if blocks_to_skip > 0:
|
||||
gpu_blocks = gpu_blocks[blocks_to_skip:]
|
||||
cpu_blocks_expanded = cpu_blocks_expanded[blocks_to_skip:]
|
||||
cpu_chunks_expanded = cpu_chunks_expanded[blocks_to_skip:]
|
||||
|
||||
# set transfer direction
|
||||
if gpu_to_cpu:
|
||||
@@ -330,16 +330,16 @@ def test_transfer(
|
||||
src_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
|
||||
)
|
||||
dst_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
dst_to_src = dict(zip(cpu_blocks_expanded, gpu_blocks))
|
||||
dst_spec = CPULoadStoreSpec(cpu_chunks)
|
||||
dst_to_src = dict(zip(cpu_chunks_expanded, gpu_blocks))
|
||||
num_dst_sub_blocks = num_gpu_blocks
|
||||
else:
|
||||
handler = worker._load_handler
|
||||
src_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
src_spec = CPULoadStoreSpec(cpu_chunks)
|
||||
dst_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
|
||||
)
|
||||
dst_to_src = dict(zip(gpu_blocks, cpu_blocks_expanded))
|
||||
dst_to_src = dict(zip(gpu_blocks, cpu_chunks_expanded))
|
||||
num_dst_sub_blocks = num_gpu_blocks
|
||||
|
||||
# randomize src and dst tensors before transfer
|
||||
@@ -410,7 +410,7 @@ def test_transfer(
|
||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
|
||||
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
|
||||
@pytest.mark.parametrize("num_cpu_chunks", NUM_CPU_CHUNKS)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
@torch.inference_mode()
|
||||
@@ -421,7 +421,7 @@ def test_transfer_multi_group(
|
||||
gpu_page_size_bytes: int,
|
||||
blocks_per_chunk: int,
|
||||
num_gpu_blocks: int,
|
||||
num_cpu_blocks: int,
|
||||
num_cpu_chunks: int,
|
||||
seed: int,
|
||||
device: str,
|
||||
) -> None:
|
||||
@@ -470,37 +470,37 @@ def test_transfer_multi_group(
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=canonical_kv_caches,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
num_cpu_chunks=num_cpu_chunks,
|
||||
)
|
||||
|
||||
# group 0: aligned, group 1: empty, group 2: unaligned on CPU->GPU
|
||||
group_sizes_in_cpu_blocks = [num_mappings_per_group, 0, num_mappings_per_group]
|
||||
group_sizes_in_cpu_chunks = [num_mappings_per_group, 0, num_mappings_per_group]
|
||||
|
||||
total_cpu_blocks = sum(group_sizes_in_cpu_blocks)
|
||||
total_gpu_blocks_needed = total_cpu_blocks * blocks_per_chunk
|
||||
total_cpu_chunks = sum(group_sizes_in_cpu_chunks)
|
||||
total_gpu_blocks_needed = total_cpu_chunks * blocks_per_chunk
|
||||
gpu_blocks_all = random.sample(range(num_gpu_blocks), total_gpu_blocks_needed)
|
||||
cpu_blocks_all = random.sample(range(num_cpu_blocks), total_cpu_blocks)
|
||||
cpu_chunks_all = random.sample(range(num_cpu_chunks), total_cpu_chunks)
|
||||
|
||||
# split gpu/cpu blocks per group
|
||||
# split gpu blocks / cpu chunks per group
|
||||
gpu_blocks_per_group: list[list[int]] = []
|
||||
cpu_blocks_per_group: list[list[int]] = []
|
||||
cpu_chunks_per_group: list[list[int]] = []
|
||||
gpu_offset = 0
|
||||
cpu_offset = 0
|
||||
for size in group_sizes_in_cpu_blocks:
|
||||
for size in group_sizes_in_cpu_chunks:
|
||||
gpu_count = size * blocks_per_chunk
|
||||
gpu_blocks_per_group.append(gpu_blocks_all[gpu_offset : gpu_offset + gpu_count])
|
||||
cpu_blocks_per_group.append(cpu_blocks_all[cpu_offset : cpu_offset + size])
|
||||
cpu_chunks_per_group.append(cpu_chunks_all[cpu_offset : cpu_offset + size])
|
||||
gpu_offset += gpu_count
|
||||
cpu_offset += size
|
||||
|
||||
# expand cpu blocks to gpu-page granularity
|
||||
cpu_blocks_expanded_per_group = [
|
||||
# expand cpu chunks to gpu-page granularity
|
||||
cpu_chunks_expanded_per_group = [
|
||||
[
|
||||
cpu_block * blocks_per_chunk + j
|
||||
for cpu_block in cpu_blocks
|
||||
cpu_chunk * blocks_per_chunk + j
|
||||
for cpu_chunk in cpu_chunks
|
||||
for j in range(blocks_per_chunk)
|
||||
]
|
||||
for cpu_blocks in cpu_blocks_per_group
|
||||
for cpu_chunks in cpu_chunks_per_group
|
||||
]
|
||||
|
||||
# skip sub-blocks from group 2 to test unaligned transfers.
|
||||
@@ -509,7 +509,7 @@ def test_transfer_multi_group(
|
||||
gpu_blocks_per_group[2] = gpu_blocks_per_group[2][
|
||||
sub_blocks_to_skip:-sub_blocks_to_skip
|
||||
]
|
||||
cpu_blocks_expanded_per_group[2] = cpu_blocks_expanded_per_group[2][
|
||||
cpu_chunks_expanded_per_group[2] = cpu_chunks_expanded_per_group[2][
|
||||
sub_blocks_to_skip:-sub_blocks_to_skip
|
||||
]
|
||||
|
||||
@@ -520,10 +520,10 @@ def test_transfer_multi_group(
|
||||
gpu_blocks.extend(gpu_blks)
|
||||
group_sizes.append(len(gpu_blks))
|
||||
|
||||
# build flat cpu_blocks list
|
||||
cpu_blocks = []
|
||||
for cpu_blks in cpu_blocks_per_group:
|
||||
cpu_blocks.extend(cpu_blks)
|
||||
# build flat cpu_chunks list
|
||||
cpu_chunks = []
|
||||
for cpu_chnks in cpu_chunks_per_group:
|
||||
cpu_chunks.extend(cpu_chnks)
|
||||
|
||||
# block_indices: only relevant for unaligned transfers
|
||||
block_indices: list[int] = [0, 0, sub_blocks_to_skip]
|
||||
@@ -533,18 +533,18 @@ def test_transfer_multi_group(
|
||||
src_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
|
||||
)
|
||||
dst_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
dst_spec = CPULoadStoreSpec(cpu_chunks)
|
||||
# per-group mapping: cpu sub-block -> gpu sub-block
|
||||
dst_to_src_per_group = [
|
||||
dict(zip(expanded, gpu_blks))
|
||||
for expanded, gpu_blks in zip(
|
||||
cpu_blocks_expanded_per_group, gpu_blocks_per_group
|
||||
cpu_chunks_expanded_per_group, gpu_blocks_per_group
|
||||
)
|
||||
]
|
||||
num_dst_sub_blocks = num_cpu_blocks * blocks_per_chunk
|
||||
num_dst_sub_blocks = num_cpu_chunks * blocks_per_chunk
|
||||
else:
|
||||
handler = worker._load_handler
|
||||
src_spec = CPULoadStoreSpec(cpu_blocks)
|
||||
src_spec = CPULoadStoreSpec(cpu_chunks)
|
||||
dst_spec = GPULoadStoreSpec(
|
||||
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
|
||||
)
|
||||
@@ -552,7 +552,7 @@ def test_transfer_multi_group(
|
||||
dst_to_src_per_group = [
|
||||
dict(zip(gpu_blks, expanded))
|
||||
for gpu_blks, expanded in zip(
|
||||
gpu_blocks_per_group, cpu_blocks_expanded_per_group
|
||||
gpu_blocks_per_group, cpu_chunks_expanded_per_group
|
||||
)
|
||||
]
|
||||
num_dst_sub_blocks = num_gpu_blocks
|
||||
@@ -644,7 +644,7 @@ def test_load_waits_for_pending_compute_stream_writes(default_vllm_config) -> No
|
||||
],
|
||||
),
|
||||
blocks_per_chunk=1,
|
||||
num_cpu_blocks=num_blocks,
|
||||
num_cpu_chunks=num_blocks,
|
||||
)
|
||||
worker._load_handler.src_tensors[0].fill_(sentinel)
|
||||
expected = torch.full((page_size_bytes,), sentinel, dtype=torch.int8)
|
||||
|
||||
@@ -36,7 +36,7 @@ _EMPTY_REQ_CTX = make_req_context()
|
||||
|
||||
|
||||
def make_cpu_manager(
|
||||
num_blocks: int = 4,
|
||||
num_chunks: int = 4,
|
||||
cache_policy: str = "lru",
|
||||
cache_policy_module_path: str | None = None,
|
||||
enable_events: bool = False,
|
||||
@@ -44,7 +44,7 @@ def make_cpu_manager(
|
||||
max_tracker_size: int = 64_000,
|
||||
) -> CPUOffloadingManager:
|
||||
return CPUOffloadingManager(
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
cache_policy=cache_policy,
|
||||
cache_policy_module_path=cache_policy_module_path,
|
||||
enable_events=enable_events,
|
||||
@@ -56,7 +56,7 @@ def make_cpu_manager(
|
||||
@dataclass
|
||||
class ExpectedPrepareStoreOutput:
|
||||
keys_to_store: list[int]
|
||||
store_block_ids: list[int]
|
||||
store_chunk_ids: list[int]
|
||||
evicted_keys: list[int]
|
||||
|
||||
|
||||
@@ -93,9 +93,9 @@ def verify_store_output(
|
||||
store_spec = prepare_store_output.store_spec
|
||||
assert isinstance(store_spec, CPULoadStoreSpec)
|
||||
expected_array = np.array(
|
||||
expected_prepare_store_output.store_block_ids, dtype=np.int64
|
||||
expected_prepare_store_output.store_chunk_ids, dtype=np.int64
|
||||
)
|
||||
assert np.array_equal(expected_array, store_spec.block_ids)
|
||||
assert np.array_equal(expected_array, store_spec.chunk_ids)
|
||||
|
||||
|
||||
def verify_load_output(
|
||||
@@ -103,7 +103,7 @@ def verify_load_output(
|
||||
):
|
||||
assert isinstance(prepare_load_output, CPULoadStoreSpec)
|
||||
expected_array = np.array(expected_prepare_load_output, dtype=np.int64)
|
||||
assert np.array_equal(expected_array, prepare_load_output.block_ids)
|
||||
assert np.array_equal(expected_array, prepare_load_output.chunk_ids)
|
||||
|
||||
|
||||
def check_split_usage_stats(
|
||||
@@ -146,7 +146,7 @@ def verify_events(
|
||||
|
||||
def test_cpu_eviction_removed_precedes_stored():
|
||||
"""An eviction is announced before the store that reuses its capacity."""
|
||||
manager = make_cpu_manager(num_blocks=2, enable_events=True)
|
||||
manager = make_cpu_manager(num_chunks=2, enable_events=True)
|
||||
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
@@ -164,23 +164,23 @@ def test_cpu_eviction_removed_precedes_stored():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("eviction_policy", ["lru", "arc"])
|
||||
def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
|
||||
def test_already_stored_chunk_not_evicted_during_prepare_store(eviction_policy):
|
||||
"""
|
||||
Regression test: a block that is already stored must not be evicted
|
||||
by prepare_store() when it needs to make room for new blocks.
|
||||
Regression test: a chunk that is already stored must not be evicted
|
||||
by prepare_store() when it needs to make room for new chunks.
|
||||
Applies to both lru and arc policies.
|
||||
|
||||
Scenario:
|
||||
- Store blocks [1, 2] and complete.
|
||||
- touch([1]) makes block 2 the LRU candidate.
|
||||
- Store chunks [1, 2] and complete.
|
||||
- touch([1]) makes chunk 2 the LRU candidate.
|
||||
- prepare_store([2, 3, 4, 5]):
|
||||
* block 2 is filtered out as "already stored"
|
||||
* but without the fix, block 2 would be evicted as the LRU
|
||||
* chunk 2 is filtered out as "already stored"
|
||||
* but without the fix, chunk 2 would be evicted as the LRU
|
||||
candidate to make room for [3, 4, 5]
|
||||
- After complete_store([2, 3, 4, 5]), block 2 must still be present.
|
||||
- After complete_store([2, 3, 4, 5]), chunk 2 must still be present.
|
||||
"""
|
||||
manager = make_cpu_manager(
|
||||
num_blocks=4,
|
||||
num_chunks=4,
|
||||
cache_policy=eviction_policy,
|
||||
enable_events=True,
|
||||
)
|
||||
@@ -189,33 +189,33 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
|
||||
# touch [1] to make block 2 the LRU candidate
|
||||
# touch [1] to make chunk 2 the LRU candidate
|
||||
manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
|
||||
# prepare_store([2, 3, 4, 5]):
|
||||
# - block 2 is already stored -> filtered out of keys_to_store
|
||||
# - block 2 must NOT be evicted even though it is the LRU candidate
|
||||
# - block 1 (ID 0) is evicted instead; new blocks [3,4,5] get IDs 2,3,0
|
||||
# - chunk 2 is already stored -> filtered out of keys_to_store
|
||||
# - chunk 2 must NOT be evicted even though it is the LRU candidate
|
||||
# - chunk 1 (ID 0) is evicted instead; new chunks [3,4,5] get IDs 2,3,0
|
||||
prepare_store_output = manager.prepare_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
|
||||
verify_store_output(
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[3, 4, 5],
|
||||
store_block_ids=[2, 3, 0],
|
||||
evicted_keys=[1], # block 1 evicted, not block 2
|
||||
store_chunk_ids=[2, 3, 0],
|
||||
evicted_keys=[1], # chunk 1 evicted, not chunk 2
|
||||
),
|
||||
)
|
||||
|
||||
# complete_store must not silently drop block 2
|
||||
# complete_store must not silently drop chunk 2
|
||||
manager.complete_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
|
||||
|
||||
# block 2 must still be present in the cache
|
||||
# chunk 2 must still be present in the cache
|
||||
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
|
||||
|
||||
|
||||
def test_filter_reused_manager_reports_stores_skipped_counter():
|
||||
manager = make_cpu_manager(
|
||||
num_blocks=4,
|
||||
num_chunks=4,
|
||||
cache_policy="lru",
|
||||
store_threshold=2,
|
||||
)
|
||||
@@ -226,7 +226,7 @@ def test_filter_reused_manager_reports_stores_skipped_counter():
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[],
|
||||
store_block_ids=[],
|
||||
store_chunk_ids=[],
|
||||
evicted_keys=[],
|
||||
),
|
||||
)
|
||||
@@ -247,34 +247,34 @@ def test_cpu_manager_reports_cache_usage_gauge():
|
||||
] == pytest.approx(value)
|
||||
|
||||
# Zero-capacity manager always reports 0.0
|
||||
manager = make_cpu_manager(num_blocks=0)
|
||||
manager = make_cpu_manager(num_chunks=0)
|
||||
check_usage_stats(manager, 0.0)
|
||||
|
||||
# Empty manager (4 blocks, none allocated): usage = 0.0
|
||||
manager = make_cpu_manager(num_blocks=4)
|
||||
# Empty manager (4 chunks, none allocated): usage = 0.0
|
||||
manager = make_cpu_manager(num_chunks=4)
|
||||
check_usage_stats(manager, 0.0)
|
||||
|
||||
# After allocating 2 of 4 blocks: usage = 0.5
|
||||
# After allocating 2 of 4 chunks: usage = 0.5
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
check_usage_stats(manager, 0.5)
|
||||
|
||||
# After filling all 4 blocks: usage = 1.0
|
||||
# After filling all 4 chunks: usage = 1.0
|
||||
manager.prepare_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
|
||||
check_usage_stats(manager, 1.0)
|
||||
|
||||
# After completing store, the blocks becomes evictable as it is not actively used
|
||||
# After completing store, the chunks become evictable as not actively used
|
||||
# and usage drops.
|
||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
check_usage_stats(manager, 0.5)
|
||||
|
||||
# After completing store, the blocks becomes evictable as it is not actively used
|
||||
# After completing store, the chunks become evictable as not actively used
|
||||
# and usage drops.
|
||||
manager.complete_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
|
||||
check_usage_stats(manager, 0.0)
|
||||
|
||||
|
||||
def test_cpu_manager_reports_allocation_size_histogram():
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
@@ -297,12 +297,12 @@ def test_cpu_manager_reports_allocation_size_histogram():
|
||||
|
||||
|
||||
def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
|
||||
def fail_allocate_blocks(keys):
|
||||
def fail_allocate_chunks(keys):
|
||||
raise RuntimeError("allocation failed")
|
||||
|
||||
monkeypatch.setattr(manager, "_allocate_blocks", fail_allocate_blocks)
|
||||
monkeypatch.setattr(manager, "_allocate_chunks", fail_allocate_chunks)
|
||||
|
||||
with pytest.raises(RuntimeError, match="allocation failed"):
|
||||
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
||||
@@ -316,7 +316,7 @@ def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
|
||||
|
||||
|
||||
def test_cpu_manager_reports_allocation_size_on_eviction_failure():
|
||||
manager = make_cpu_manager(num_blocks=1, cache_policy="lru")
|
||||
manager = make_cpu_manager(num_chunks=1, cache_policy="lru")
|
||||
|
||||
manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
manager.get_stats()
|
||||
@@ -332,7 +332,7 @@ def test_cpu_manager_reports_allocation_size_on_eviction_failure():
|
||||
|
||||
|
||||
def test_cpu_manager_reports_cache_write_and_read_usage_gauges():
|
||||
manager = make_cpu_manager(num_blocks=4)
|
||||
manager = make_cpu_manager(num_chunks=4)
|
||||
|
||||
# Store path: pins write usage until complete_store.
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
@@ -356,7 +356,7 @@ def test_cpu_manager_reports_cache_write_and_read_usage_gauges():
|
||||
|
||||
|
||||
def test_cpu_manager_clears_write_usage_after_failed_store():
|
||||
manager = make_cpu_manager(num_blocks=4)
|
||||
manager = make_cpu_manager(num_chunks=4)
|
||||
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
check_split_usage_stats(manager, write=0.5, read=0.0, total=0.5)
|
||||
@@ -369,8 +369,8 @@ def test_cpu_manager():
|
||||
"""
|
||||
Tests CPUOffloadingManager with lru policy.
|
||||
"""
|
||||
# initialize a CPU manager with a capacity of 4 blocks
|
||||
cpu_manager = make_cpu_manager(num_blocks=4, cache_policy="lru", enable_events=True)
|
||||
# initialize a CPU manager with a capacity of 4 chunks
|
||||
cpu_manager = make_cpu_manager(num_chunks=4, cache_policy="lru", enable_events=True)
|
||||
|
||||
# prepare store [1, 2]
|
||||
prepare_store_output = cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
@@ -378,7 +378,7 @@ def test_cpu_manager():
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[1, 2],
|
||||
store_block_ids=[0, 1],
|
||||
store_chunk_ids=[0, 1],
|
||||
evicted_keys=[],
|
||||
),
|
||||
)
|
||||
@@ -407,7 +407,7 @@ def test_cpu_manager():
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[3, 4, 5],
|
||||
store_block_ids=[2, 3, 0],
|
||||
store_chunk_ids=[2, 3, 0],
|
||||
evicted_keys=[1],
|
||||
),
|
||||
)
|
||||
@@ -445,7 +445,7 @@ def test_cpu_manager():
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[6, 7, 8],
|
||||
store_block_ids=[1, 0, 3],
|
||||
store_chunk_ids=[1, 0, 3],
|
||||
evicted_keys=[4, 5, 2],
|
||||
),
|
||||
)
|
||||
@@ -462,7 +462,7 @@ def test_cpu_manager():
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[9],
|
||||
store_block_ids=[3],
|
||||
store_chunk_ids=[3],
|
||||
evicted_keys=[8],
|
||||
),
|
||||
)
|
||||
@@ -482,38 +482,38 @@ def test_cpu_manager():
|
||||
|
||||
|
||||
def test_prepare_load_preserves_key_order():
|
||||
"""block_ids[i] must correspond to keys[i] (co-indexed invariant)."""
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
"""chunk_ids[i] must correspond to keys[i] (co-indexed invariant)."""
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
|
||||
key_a, key_b, key_c = to_key(0), to_key(1), to_key(2)
|
||||
|
||||
# Store all three keys and learn their block ID assignments
|
||||
# Store all three keys and learn their chunk ID assignments
|
||||
store_output = manager.prepare_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
||||
assert store_output is not None
|
||||
assert isinstance(store_output.store_spec, CPULoadStoreSpec)
|
||||
key_to_block_id = {
|
||||
k: int(bid)
|
||||
for k, bid in zip(store_output.keys_to_store, store_output.store_spec.block_ids)
|
||||
key_to_chunk_id = {
|
||||
k: int(cid)
|
||||
for k, cid in zip(store_output.keys_to_store, store_output.store_spec.chunk_ids)
|
||||
}
|
||||
manager.complete_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
||||
|
||||
# Forward order: [a, b, c]
|
||||
spec_fwd = manager.prepare_load([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
||||
assert isinstance(spec_fwd, CPULoadStoreSpec)
|
||||
assert [int(x) for x in spec_fwd.block_ids] == [
|
||||
key_to_block_id[key_a],
|
||||
key_to_block_id[key_b],
|
||||
key_to_block_id[key_c],
|
||||
assert [int(x) for x in spec_fwd.chunk_ids] == [
|
||||
key_to_chunk_id[key_a],
|
||||
key_to_chunk_id[key_b],
|
||||
key_to_chunk_id[key_c],
|
||||
]
|
||||
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
|
||||
|
||||
# Arbitrary permutation: [b, c, a]
|
||||
spec_perm = manager.prepare_load([key_b, key_c, key_a], _EMPTY_REQ_CTX)
|
||||
assert isinstance(spec_perm, CPULoadStoreSpec)
|
||||
assert [int(x) for x in spec_perm.block_ids] == [
|
||||
key_to_block_id[key_b],
|
||||
key_to_block_id[key_c],
|
||||
key_to_block_id[key_a],
|
||||
assert [int(x) for x in spec_perm.chunk_ids] == [
|
||||
key_to_chunk_id[key_b],
|
||||
key_to_chunk_id[key_c],
|
||||
key_to_chunk_id[key_a],
|
||||
]
|
||||
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
|
||||
|
||||
@@ -522,10 +522,10 @@ class TestARCPolicy:
|
||||
"""Unit tests for CPUOffloadingManager with ARC eviction policy."""
|
||||
|
||||
def _make_manager(
|
||||
self, num_blocks: int = 4, enable_events: bool = True
|
||||
self, num_chunks: int = 4, enable_events: bool = True
|
||||
) -> tuple[CPUOffloadingManager, ARCCachePolicy]:
|
||||
manager = make_cpu_manager(
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
cache_policy="arc",
|
||||
enable_events=enable_events,
|
||||
)
|
||||
@@ -548,7 +548,7 @@ class TestARCPolicy:
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[1, 2],
|
||||
store_block_ids=[0, 1],
|
||||
store_chunk_ids=[0, 1],
|
||||
evicted_keys=[],
|
||||
),
|
||||
)
|
||||
@@ -569,36 +569,36 @@ class TestARCPolicy:
|
||||
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
|
||||
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.MISS
|
||||
|
||||
# blocks should be in T1 (recent)
|
||||
# chunks should be in T1 (recent)
|
||||
assert len(arc_policy.t1) == 2
|
||||
assert len(arc_policy.t2) == 0
|
||||
|
||||
def test_t1_to_t2_promotion(self):
|
||||
"""
|
||||
Tests that accessing a block in T1 promotes it to T2 (frequent).
|
||||
Tests that accessing a chunk in T1 promotes it to T2 (frequent).
|
||||
This is a key feature of ARC's adaptive behavior.
|
||||
"""
|
||||
cpu_manager, arc_policy = self._make_manager(enable_events=False)
|
||||
|
||||
# store and complete block 1
|
||||
# store and complete chunk 1
|
||||
cpu_manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
|
||||
# block 1 starts in T1 (recent)
|
||||
# chunk 1 starts in T1 (recent)
|
||||
assert to_keys([1])[0] in arc_policy.t1
|
||||
assert to_keys([1])[0] not in arc_policy.t2
|
||||
|
||||
# touch block 1 (simulate second access)
|
||||
# touch chunk 1 (simulate second access)
|
||||
cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
|
||||
# block 1 should now be in T2 (frequent)
|
||||
# chunk 1 should now be in T2 (frequent)
|
||||
assert to_keys([1])[0] not in arc_policy.t1
|
||||
assert to_keys([1])[0] in arc_policy.t2
|
||||
|
||||
def test_eviction_with_load(self):
|
||||
"""
|
||||
Tests ARC eviction behavior similar to LRU test.
|
||||
Verifies that blocks being loaded (ref_cnt > 0) cannot be evicted.
|
||||
Verifies that chunks being loaded (ref_cnt > 0) cannot be evicted.
|
||||
"""
|
||||
cpu_manager, _ = self._make_manager()
|
||||
|
||||
@@ -610,7 +610,7 @@ class TestARCPolicy:
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[1, 2, 3, 4],
|
||||
store_block_ids=[0, 1, 2, 3],
|
||||
store_chunk_ids=[0, 1, 2, 3],
|
||||
evicted_keys=[],
|
||||
),
|
||||
)
|
||||
@@ -628,36 +628,36 @@ class TestARCPolicy:
|
||||
cpu_manager.complete_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
|
||||
|
||||
# now prepare store [5, 6, 7] should succeed
|
||||
# ARC will evict blocks one at a time from T1 as needed
|
||||
# ARC will evict chunks one at a time from T1 as needed
|
||||
prepare_store_output = cpu_manager.prepare_store(
|
||||
to_keys([5, 6, 7]), _EMPTY_REQ_CTX
|
||||
)
|
||||
assert prepare_store_output is not None
|
||||
# Should successfully evict enough blocks to make room (at least 1)
|
||||
# Should successfully evict enough chunks to make room (at least 1)
|
||||
assert len(prepare_store_output.evicted_keys) >= 1
|
||||
|
||||
def test_adaptive_target(self):
|
||||
"""
|
||||
Tests ARC's adaptive target adjustment via ghost lists.
|
||||
When a block in B1 (ghost list) is accessed, target_t1_size increases.
|
||||
When a block in B2 is accessed, target_t1_size decreases.
|
||||
When a chunk in B1 (ghost list) is accessed, target_t1_size increases.
|
||||
When a chunk in B2 is accessed, target_t1_size decreases.
|
||||
"""
|
||||
cpu_manager, arc_policy = self._make_manager(num_blocks=2, enable_events=False)
|
||||
cpu_manager, arc_policy = self._make_manager(num_chunks=2, enable_events=False)
|
||||
|
||||
# store blocks 1, 2 (fills cache)
|
||||
# store chunks 1, 2 (fills cache)
|
||||
cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
|
||||
initial_target = arc_policy.target_t1_size
|
||||
|
||||
# store block 3, evicting block 1 (moves to B1 ghost list)
|
||||
# store chunk 3, evicting chunk 1 (moves to B1 ghost list)
|
||||
cpu_manager.prepare_store(to_keys([3]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([3]), _EMPTY_REQ_CTX)
|
||||
|
||||
# block 1 should be in B1 (ghost list)
|
||||
# chunk 1 should be in B1 (ghost list)
|
||||
assert to_keys([1])[0] in arc_policy.b1
|
||||
|
||||
# touch block 1 (cache miss, but in B1)
|
||||
# touch chunk 1 (cache miss, but in B1)
|
||||
# this should increase target_t1_size (favor recency)
|
||||
cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
|
||||
@@ -671,11 +671,11 @@ class TestARCPolicy:
|
||||
"""
|
||||
cpu_manager, arc_policy = self._make_manager(enable_events=False)
|
||||
|
||||
# store blocks 1, 2, 3, 4
|
||||
# store chunks 1, 2, 3, 4
|
||||
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||
|
||||
# promote blocks 3, 4 to T2 by touching them
|
||||
# promote chunks 3, 4 to T2 by touching them
|
||||
cpu_manager.touch(to_keys([3, 4]), _EMPTY_REQ_CTX)
|
||||
|
||||
# now: T1 = {1, 2}, T2 = {3, 4}
|
||||
@@ -686,22 +686,22 @@ class TestARCPolicy:
|
||||
# (when |T1| >= target, evict from T1)
|
||||
arc_policy.target_t1_size = 1
|
||||
|
||||
# store block 5, should evict from T1 (block 1, LRU in T1)
|
||||
# store chunk 5, should evict from T1 (chunk 1, LRU in T1)
|
||||
output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
|
||||
assert output is not None
|
||||
assert to_keys([1]) == output.evicted_keys
|
||||
|
||||
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX)
|
||||
|
||||
# block 1 should be in B1 (ghost list)
|
||||
# chunk 1 should be in B1 (ghost list)
|
||||
assert to_keys([1])[0] in arc_policy.b1
|
||||
# block 5 should be in T1
|
||||
# chunk 5 should be in T1
|
||||
assert to_keys([5])[0] in arc_policy.t1
|
||||
|
||||
def test_batch_eviction_scans_t1_and_t2_once(self):
|
||||
"""ARC batch eviction must preserve order without restarting scans."""
|
||||
cpu_manager, arc_policy = self._make_manager(
|
||||
num_blocks=256, enable_events=False
|
||||
num_chunks=256, enable_events=False
|
||||
)
|
||||
keys = to_keys(list(range(256)))
|
||||
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
||||
@@ -721,13 +721,13 @@ class TestARCPolicy:
|
||||
t2_order = list(arc_policy.t2)
|
||||
expected_t1 = [
|
||||
key
|
||||
for key, block in arc_policy.t1.items()
|
||||
if block.ref_cnt == 0 and key not in protected
|
||||
for key, chunk in arc_policy.t1.items()
|
||||
if chunk.ref_cnt == 0 and key not in protected
|
||||
][:num_t1_evictions]
|
||||
expected_t2 = [
|
||||
key
|
||||
for key, block in arc_policy.t2.items()
|
||||
if block.ref_cnt == 0 and key not in protected
|
||||
for key, chunk in arc_policy.t2.items()
|
||||
if chunk.ref_cnt == 0 and key not in protected
|
||||
][:num_t2_evictions]
|
||||
expected_t1_scans = t1_order.index(expected_t1[-1]) + 1
|
||||
expected_t2_scans = t2_order.index(expected_t2[-1]) + 1
|
||||
@@ -746,7 +746,7 @@ class TestARCPolicy:
|
||||
|
||||
def test_batch_eviction_falls_back_after_t1_iterator_exhausted(self):
|
||||
"""An exhausted T1 scan must keep falling back to T2."""
|
||||
cpu_manager, arc_policy = self._make_manager(num_blocks=8, enable_events=False)
|
||||
cpu_manager, arc_policy = self._make_manager(num_chunks=8, enable_events=False)
|
||||
keys = to_keys(list(range(8)))
|
||||
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
|
||||
@@ -764,8 +764,8 @@ class TestARCPolicy:
|
||||
# the target, so each remaining selection must retry T1 then use T2.
|
||||
eligible_t1 = [
|
||||
key
|
||||
for key, block in arc_policy.t1.items()
|
||||
if block.ref_cnt == 0 and key not in protected
|
||||
for key, chunk in arc_policy.t1.items()
|
||||
if chunk.ref_cnt == 0 and key not in protected
|
||||
]
|
||||
assert eligible_t1 == t1_order[:1]
|
||||
assert len(t1_order) - 1 >= int(arc_policy.target_t1_size)
|
||||
@@ -839,7 +839,7 @@ class TestARCPolicy:
|
||||
|
||||
def test_batch_eviction_failure_is_atomic(self):
|
||||
"""Finding only some candidates must not partially evict the cache."""
|
||||
cpu_manager, arc_policy = self._make_manager(num_blocks=4, enable_events=False)
|
||||
cpu_manager, arc_policy = self._make_manager(num_chunks=4, enable_events=False)
|
||||
keys = to_keys(list(range(4)))
|
||||
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
|
||||
@@ -858,13 +858,13 @@ class TestARCPolicy:
|
||||
Tests that ghost lists (B1, B2) don't grow unbounded.
|
||||
They should be capped at cache_capacity.
|
||||
"""
|
||||
cpu_manager, arc_policy = self._make_manager(num_blocks=2, enable_events=False)
|
||||
cpu_manager, arc_policy = self._make_manager(num_chunks=2, enable_events=False)
|
||||
|
||||
# fill cache with blocks 1, 2
|
||||
# fill cache with chunks 1, 2
|
||||
cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
|
||||
# store many blocks to fill ghost lists
|
||||
# store many chunks to fill ghost lists
|
||||
for i in range(3, 20):
|
||||
cpu_manager.prepare_store(to_keys([i]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([i]), _EMPTY_REQ_CTX)
|
||||
@@ -880,7 +880,7 @@ class TestARCPolicy:
|
||||
"""
|
||||
cpu_manager, arc_policy = self._make_manager()
|
||||
|
||||
# store blocks 1, 2, 3, 4
|
||||
# store chunks 1, 2, 3, 4
|
||||
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||
|
||||
@@ -895,13 +895,13 @@ class TestARCPolicy:
|
||||
assert len(arc_policy.t1) == 1
|
||||
assert len(arc_policy.t2) == 3
|
||||
|
||||
# store block 5, should evict from T1 (block 2, only one in T1)
|
||||
# store chunk 5, should evict from T1 (chunk 2, only one in T1)
|
||||
prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
|
||||
verify_store_output(
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[5],
|
||||
store_block_ids=[1], # reuses block 2's storage
|
||||
store_chunk_ids=[1], # reuses chunk 2's storage
|
||||
evicted_keys=[2],
|
||||
),
|
||||
)
|
||||
@@ -913,11 +913,11 @@ class TestARCPolicy:
|
||||
"""
|
||||
cpu_manager, arc_policy = self._make_manager()
|
||||
|
||||
# store blocks 1, 2, 3, 4
|
||||
# store chunks 1, 2, 3, 4
|
||||
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||
|
||||
# prepare store block 5 (will evict block 1)
|
||||
# prepare store chunk 5 (will evict chunk 1)
|
||||
prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
|
||||
assert prepare_store_output is not None
|
||||
assert len(prepare_store_output.evicted_keys) == 1
|
||||
@@ -925,13 +925,13 @@ class TestARCPolicy:
|
||||
# complete store with failure
|
||||
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX, success=False)
|
||||
|
||||
# block 5 should not be in cache
|
||||
# chunk 5 should not be in cache
|
||||
assert cpu_manager.lookup(to_key(5), _EMPTY_REQ_CTX) is LookupResult.MISS
|
||||
# block 5 should not be in T1 or T2
|
||||
# chunk 5 should not be in T1 or T2
|
||||
assert to_keys([5])[0] not in arc_policy.t1
|
||||
assert to_keys([5])[0] not in arc_policy.t2
|
||||
|
||||
# evicted block should still be gone (in B1 ghost list)
|
||||
# evicted chunk should still be gone (in B1 ghost list)
|
||||
evicted_hash = prepare_store_output.evicted_keys[0]
|
||||
assert evicted_hash in arc_policy.b1
|
||||
|
||||
@@ -954,7 +954,7 @@ class TestARCPolicy:
|
||||
assert len(prepare_store_output.evicted_keys) == 1
|
||||
cpu_manager.complete_store(to_keys([3, 4, 5]), _EMPTY_REQ_CTX)
|
||||
|
||||
# promote some blocks to T2
|
||||
# promote some chunks to T2
|
||||
cpu_manager.touch(to_keys([2, 3]), _EMPTY_REQ_CTX)
|
||||
|
||||
# T1 has {4, 5}, T2 has {2, 3}
|
||||
@@ -966,7 +966,7 @@ class TestARCPolicy:
|
||||
assert prepare_store_output is not None
|
||||
cpu_manager.complete_store(to_keys([6]), _EMPTY_REQ_CTX)
|
||||
|
||||
# verify blocks 2, 3 (in T2) are still present
|
||||
# verify chunks 2, 3 (in T2) are still present
|
||||
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
|
||||
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.HIT
|
||||
|
||||
@@ -980,7 +980,7 @@ def test_filter_reused_manager():
|
||||
Tests CPUOffloadingManager reuse filtering (store_threshold=2).
|
||||
"""
|
||||
manager = make_cpu_manager(
|
||||
num_blocks=4,
|
||||
num_chunks=4,
|
||||
cache_policy="lru",
|
||||
enable_events=True,
|
||||
store_threshold=2,
|
||||
@@ -1023,7 +1023,7 @@ def test_filter_reused_manager():
|
||||
def test_filter_reused_manager_oversized_offer_makes_progress():
|
||||
"""An offer larger than the tracker capacity must not churn forever."""
|
||||
manager = make_cpu_manager(
|
||||
num_blocks=4,
|
||||
num_chunks=4,
|
||||
cache_policy="lru",
|
||||
store_threshold=2,
|
||||
max_tracker_size=3,
|
||||
@@ -1042,69 +1042,69 @@ def test_filter_reused_manager_oversized_offer_makes_progress():
|
||||
assert stored_keys == set(keys)
|
||||
|
||||
|
||||
def test_evictable_cache_block_count():
|
||||
def test_evictable_cache_chunk_count():
|
||||
"""
|
||||
Verifies _num_evictable_cache_blocks is maintained correctly through the
|
||||
Verifies _num_evictable_cache_chunks is maintained correctly through the
|
||||
full store/load lifecycle, eviction, failed stores, concurrent loads,
|
||||
reset_cache, and the early-exit fast path in prepare_store.
|
||||
"""
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
|
||||
# Initially no blocks allocated.
|
||||
assert manager._num_evictable_cache_blocks == 0
|
||||
# Initially no chunks allocated.
|
||||
assert manager._num_evictable_cache_chunks == 0
|
||||
|
||||
# Initial cache state [x, x, x, x]
|
||||
|
||||
# We get 3 blocks from the cache.
|
||||
# We get 3 chunks from the cache.
|
||||
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
||||
# cache state [1', 2', 3', x] <- 1', 2', 3' are actively being used.
|
||||
assert manager._num_evictable_cache_blocks == 0
|
||||
assert manager._num_evictable_cache_chunks == 0
|
||||
|
||||
# Completing stores makes them idle.
|
||||
manager.complete_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
||||
# cache state [1, 2, 3, x] <- 1, 2, 3 blocks are idle.
|
||||
assert manager._num_evictable_cache_blocks == 3
|
||||
# cache state [1, 2, 3, x] <- 1, 2, 3 chunks are idle.
|
||||
assert manager._num_evictable_cache_chunks == 3
|
||||
|
||||
# prepare_load pins a block: idle count decrements once even if the
|
||||
# same block is loaded by two concurrent callers.
|
||||
# prepare_load pins a chunk: idle count decrements once even if the
|
||||
# same chunk is loaded by two concurrent callers.
|
||||
manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
|
||||
assert manager._num_evictable_cache_blocks == 2
|
||||
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
|
||||
assert manager._num_evictable_cache_chunks == 2
|
||||
manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX) # 2nd concurrent load
|
||||
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
|
||||
assert manager._num_evictable_cache_blocks == 2 # no double-decrement
|
||||
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
|
||||
assert manager._num_evictable_cache_chunks == 2 # no double-decrement
|
||||
|
||||
# First complete_load does not restore idle (ref_cnt still 1).
|
||||
manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
|
||||
assert manager._num_evictable_cache_blocks == 2
|
||||
# Second complete_load drops ref_cnt to 0 -> block becomes idle again.
|
||||
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
|
||||
assert manager._num_evictable_cache_chunks == 2
|
||||
# Second complete_load drops ref_cnt to 0 -> chunk becomes idle again.
|
||||
manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
# cache state [1, 2, 3, x] <- 1, 2, 3 blocks are idle.
|
||||
assert manager._num_evictable_cache_blocks == 3
|
||||
# cache state [1, 2, 3, x] <- 1, 2, 3 chunks are idle.
|
||||
assert manager._num_evictable_cache_chunks == 3
|
||||
|
||||
# Eviction decrements idle count.
|
||||
# Cache has 3 stored blocks and 1 free slot. Storing 3 new keys needs 2 eviction.
|
||||
# Cache has 3 stored chunks and 1 free slot. Storing 3 new keys needs 2 eviction.
|
||||
manager.prepare_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX)
|
||||
# cache state [1, 4', 5', 6'] <- block 1 is idle
|
||||
assert manager._num_evictable_cache_blocks == 1
|
||||
# cache state [1, 4', 5', 6'] <- chunk 1 is idle
|
||||
assert manager._num_evictable_cache_chunks == 1
|
||||
|
||||
# Failed store does not increment idle count (block discarded from cache).
|
||||
# Failed store does not increment idle count (chunk discarded from cache).
|
||||
manager.complete_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX, success=False)
|
||||
# cache state [1, x, x, x] <- block 1 is idle. Other returned to cache.
|
||||
assert manager._num_evictable_cache_blocks == 1
|
||||
# cache state [1, x, x, x] <- chunk 1 is idle. Other returned to cache.
|
||||
assert manager._num_evictable_cache_chunks == 1
|
||||
|
||||
# reset_cache zeroes the count unconditionally.
|
||||
manager.reset_cache()
|
||||
# cache state [x, x, x, x]
|
||||
assert manager._num_evictable_cache_blocks == 0
|
||||
assert manager._num_evictable_cache_chunks == 0
|
||||
|
||||
# setup 3 blocks with loads so idle count drops to 0.
|
||||
# setup 3 chunks with loads so idle count drops to 0.
|
||||
manager.prepare_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
||||
manager.complete_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
||||
manager.prepare_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
||||
# cache state [10', 11', 12', x]
|
||||
assert manager._num_evictable_cache_blocks == 0
|
||||
assert manager._num_evictable_cache_chunks == 0
|
||||
|
||||
# prepare_store requiring eviction must return None immediately (fast exit).
|
||||
# Spy on policy.evict to confirm the fast path short-circuits before calling it.
|
||||
@@ -1120,24 +1120,24 @@ def test_evictable_cache_block_count():
|
||||
# cache state [10', 11', 12', x] <- cannot evict anything
|
||||
assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is None
|
||||
assert not evict_called, (
|
||||
"_num_evictable_cache_blocks==0 should short-circuit before evict()"
|
||||
"_num_evictable_cache_chunks==0 should short-circuit before evict()"
|
||||
)
|
||||
|
||||
# After releasing the loads, eviction becomes possible again.
|
||||
manager.complete_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
||||
# cache state [10, 11, 12, x] <- 10, 11, 12 are idle
|
||||
assert manager._num_evictable_cache_blocks == 3
|
||||
assert manager._num_evictable_cache_chunks == 3
|
||||
assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is not None
|
||||
# cache state [10, 11, 14', 15'] <- 10, 11 are idle
|
||||
assert manager._num_evictable_cache_blocks == 2
|
||||
assert manager._num_evictable_cache_chunks == 2
|
||||
manager.complete_store(to_keys([14, 15]), _EMPTY_REQ_CTX)
|
||||
# cache state [10, 11, 14, 15] <- all blocks idle
|
||||
assert manager._num_evictable_cache_blocks == 4
|
||||
# cache state [10, 11, 14, 15] <- all chunks idle
|
||||
assert manager._num_evictable_cache_chunks == 4
|
||||
|
||||
|
||||
def test_touch_forwards_req_context_to_policy(monkeypatch):
|
||||
"""Regression: CPUOffloadingManager.touch forwards ReqContext to policy."""
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
received = []
|
||||
|
||||
def spy_touch(keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
||||
|
||||
@@ -38,7 +38,7 @@ def _set_spawn_method(monkeypatch):
|
||||
|
||||
def _make_region(
|
||||
engine_id: str,
|
||||
num_blocks: int = 4,
|
||||
num_chunks: int = 4,
|
||||
cpu_page_size: int = PAGE_SIZE,
|
||||
num_workers: int = 1,
|
||||
rank: int = 0,
|
||||
@@ -47,9 +47,9 @@ def _make_region(
|
||||
assert cpu_page_size % PAGE_SIZE == 0
|
||||
return SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
||||
kv_bytes_per_chunk=num_workers * cpu_page_size,
|
||||
cpu_page_size=cpu_page_size,
|
||||
barrier=barrier,
|
||||
)
|
||||
@@ -99,16 +99,16 @@ def _region(engine_id: str, **kwargs):
|
||||
def _multi_region(
|
||||
engine_id: str,
|
||||
num_workers: int,
|
||||
num_blocks: int = 4,
|
||||
num_chunks: int = 4,
|
||||
cpu_page_size: int = PAGE_SIZE,
|
||||
):
|
||||
"""Context manager: create one SharedOffloadRegion per rank, clean up on exit."""
|
||||
regions = [
|
||||
SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
||||
kv_bytes_per_chunk=num_workers * cpu_page_size,
|
||||
cpu_page_size=cpu_page_size,
|
||||
)
|
||||
for rank in range(num_workers)
|
||||
@@ -124,7 +124,7 @@ def _multi_region(
|
||||
def _race_construct(
|
||||
engine_id: str,
|
||||
num_workers: int,
|
||||
num_blocks: int = 4,
|
||||
num_chunks: int = 4,
|
||||
cpu_page_size: int = PAGE_SIZE,
|
||||
) -> tuple[list[SharedOffloadRegion], list[Exception]]:
|
||||
"""Spawn num_workers threads that all race to construct SharedOffloadRegion."""
|
||||
@@ -137,9 +137,9 @@ def _race_construct(
|
||||
try:
|
||||
regions[rank] = SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
||||
kv_bytes_per_chunk=num_workers * cpu_page_size,
|
||||
cpu_page_size=cpu_page_size,
|
||||
)
|
||||
except Exception as e:
|
||||
@@ -156,7 +156,7 @@ def _race_construct(
|
||||
|
||||
def _mp_race_construct_and_write(
|
||||
engine_id: str,
|
||||
num_blocks: int,
|
||||
num_chunks: int,
|
||||
rank: int,
|
||||
num_workers: int,
|
||||
cpu_page_size: int,
|
||||
@@ -170,9 +170,9 @@ def _mp_race_construct_and_write(
|
||||
try:
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
||||
kv_bytes_per_chunk=num_workers * cpu_page_size,
|
||||
cpu_page_size=cpu_page_size,
|
||||
)
|
||||
t = region.create_next_worker_view(cpu_page_size)
|
||||
@@ -198,9 +198,9 @@ def _mp_barrier_construct_and_hold(
|
||||
try:
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=2,
|
||||
num_chunks=2,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * PAGE_SIZE,
|
||||
kv_bytes_per_chunk=num_workers * PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
barrier=lambda: barrier.wait(30),
|
||||
)
|
||||
@@ -233,9 +233,9 @@ def iid():
|
||||
|
||||
|
||||
def test_create_next_worker_view_shape_and_stride(iid):
|
||||
"""Returned tensor must have shape (num_blocks, tensor_page_size) and
|
||||
"""Returned tensor must have shape (num_chunks, tensor_page_size) and
|
||||
stride (row_stride, 1) where row_stride = cpu_page_size * num_workers."""
|
||||
with _region(iid, num_blocks=4, cpu_page_size=2 * PAGE_SIZE) as r:
|
||||
with _region(iid, num_chunks=4, cpu_page_size=2 * PAGE_SIZE) as r:
|
||||
t = r.create_next_worker_view(PAGE_SIZE)
|
||||
assert t.shape == (4, PAGE_SIZE)
|
||||
# num_workers=1 → row_stride = cpu_page_size
|
||||
@@ -253,7 +253,7 @@ def test_create_next_worker_view_storage_offset_rank0(iid):
|
||||
|
||||
def test_create_next_worker_view_storage_offset_rank1(iid):
|
||||
"""rank=1 worker's first tensor must start cpu_page_size bytes into the mmap."""
|
||||
with _multi_region(iid, num_workers=2, num_blocks=4) as (r0, r1):
|
||||
with _multi_region(iid, num_workers=2, num_chunks=4) as (r0, r1):
|
||||
t1 = r1.create_next_worker_view(PAGE_SIZE)
|
||||
assert t1.data_ptr() == r1._base.data_ptr() + PAGE_SIZE
|
||||
del t1
|
||||
@@ -261,7 +261,7 @@ def test_create_next_worker_view_storage_offset_rank1(iid):
|
||||
|
||||
def test_create_next_worker_view_row_stride_with_multiple_workers(iid):
|
||||
"""With num_workers=4, row_stride must be 4 * cpu_page_size."""
|
||||
with _region(iid, num_blocks=2, num_workers=4) as r:
|
||||
with _region(iid, num_chunks=2, num_workers=4) as r:
|
||||
t = r.create_next_worker_view(PAGE_SIZE)
|
||||
assert t.stride(0) == 4 * PAGE_SIZE
|
||||
del t
|
||||
@@ -332,12 +332,12 @@ def test_create_next_worker_view_overflow_does_not_mutate_cursor(iid):
|
||||
def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
|
||||
"""Writes into a create_next_worker_view view must appear at the correct
|
||||
raw mmap offset"""
|
||||
with _region(iid, num_blocks=4) as r:
|
||||
with _region(iid, num_chunks=4) as r:
|
||||
t = r.create_next_worker_view(PAGE_SIZE)
|
||||
t[2, :] = 42 # write to block row 2
|
||||
t[2, :] = 42 # write to chunk row 2
|
||||
|
||||
raw = memoryview(r.mmap_obj)
|
||||
# num_workers=1 → row_stride = PAGE_SIZE; block 2 starts at byte 2*PAGE_SIZE
|
||||
# num_workers=1 → row_stride = PAGE_SIZE; chunk 2 starts at byte 2*PAGE_SIZE
|
||||
chunk = bytes(raw[2 * PAGE_SIZE : 3 * PAGE_SIZE])
|
||||
assert all(b == 42 for b in chunk)
|
||||
del raw, t
|
||||
@@ -345,7 +345,7 @@ def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
|
||||
|
||||
def test_create_next_worker_view_multi_tensor_layout(iid):
|
||||
"""Two tensors from the same worker land at consecutive byte offsets per row."""
|
||||
with _region(iid, num_blocks=2, cpu_page_size=2 * PAGE_SIZE) as r:
|
||||
with _region(iid, num_chunks=2, cpu_page_size=2 * PAGE_SIZE) as r:
|
||||
ta = r.create_next_worker_view(PAGE_SIZE)
|
||||
tb = r.create_next_worker_view(PAGE_SIZE)
|
||||
|
||||
@@ -353,8 +353,8 @@ def test_create_next_worker_view_multi_tensor_layout(iid):
|
||||
tb[:, :] = 2
|
||||
|
||||
raw = memoryview(r.mmap_obj)
|
||||
for blk in range(2):
|
||||
row_offset = blk * 2 * PAGE_SIZE # num_workers=1
|
||||
for chunk in range(2):
|
||||
row_offset = chunk * 2 * PAGE_SIZE # num_workers=1
|
||||
assert all(b == 1 for b in raw[row_offset : row_offset + PAGE_SIZE])
|
||||
assert all(
|
||||
b == 2 for b in raw[row_offset + PAGE_SIZE : row_offset + 2 * PAGE_SIZE]
|
||||
@@ -366,7 +366,7 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
||||
"""Each worker process calls create_next_worker_view and writes distinct data;
|
||||
the parent verifies each slot lands at the correct interleaved offset."""
|
||||
num_workers = 2
|
||||
num_blocks = 4
|
||||
num_chunks = 4
|
||||
|
||||
ctx = get_mp_context()
|
||||
done_queue = ctx.Queue()
|
||||
@@ -375,9 +375,9 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
||||
# Parent is rank 0 (creator); child is rank 1 (joiner).
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=iid,
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
rank=0,
|
||||
kv_bytes_per_block=num_workers * PAGE_SIZE,
|
||||
kv_bytes_per_chunk=num_workers * PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
)
|
||||
try:
|
||||
@@ -385,7 +385,7 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
||||
target=_mp_race_construct_and_write,
|
||||
args=(
|
||||
iid,
|
||||
num_blocks,
|
||||
num_chunks,
|
||||
1,
|
||||
num_workers,
|
||||
PAGE_SIZE,
|
||||
@@ -403,12 +403,12 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
||||
assert result["error"] is None, result["error"]
|
||||
|
||||
raw = memoryview(region.mmap_obj)
|
||||
for blk in range(num_blocks):
|
||||
row_start = blk * num_workers * PAGE_SIZE
|
||||
for chunk in range(num_chunks):
|
||||
row_start = chunk * num_workers * PAGE_SIZE
|
||||
w0 = bytes(raw[row_start : row_start + PAGE_SIZE])
|
||||
w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE])
|
||||
assert all(b == 11 for b in w0), f"block {blk}: rank0 slot wrong"
|
||||
assert all(b == 22 for b in w1), f"block {blk}: rank1 slot wrong"
|
||||
assert all(b == 11 for b in w0), f"chunk {chunk}: rank0 slot wrong"
|
||||
assert all(b == 22 for b in w1), f"chunk {chunk}: rank1 slot wrong"
|
||||
|
||||
del raw, t0 # release before finally triggers cleanup
|
||||
cleanup_queue.put(True)
|
||||
@@ -422,8 +422,8 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
||||
def test_create_next_worker_view_worker_isolation(iid):
|
||||
"""Writes by worker 0 must not affect worker 1's slot and vice versa."""
|
||||
num_workers = 2
|
||||
num_blocks = 4
|
||||
with _multi_region(iid, num_workers=num_workers, num_blocks=num_blocks) as regions:
|
||||
num_chunks = 4
|
||||
with _multi_region(iid, num_workers=num_workers, num_chunks=num_chunks) as regions:
|
||||
t0 = regions[0].create_next_worker_view(PAGE_SIZE)
|
||||
t1 = regions[1].create_next_worker_view(PAGE_SIZE)
|
||||
|
||||
@@ -431,12 +431,12 @@ def test_create_next_worker_view_worker_isolation(iid):
|
||||
t1[:, :] = 22
|
||||
|
||||
raw = memoryview(regions[0].mmap_obj)
|
||||
for blk in range(num_blocks):
|
||||
row_start = blk * num_workers * PAGE_SIZE
|
||||
for chunk in range(num_chunks):
|
||||
row_start = chunk * num_workers * PAGE_SIZE
|
||||
w0 = bytes(raw[row_start : row_start + PAGE_SIZE])
|
||||
w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE])
|
||||
assert all(b == 11 for b in w0), f"block {blk}: worker0 slot corrupted"
|
||||
assert all(b == 22 for b in w1), f"block {blk}: worker1 slot corrupted"
|
||||
assert all(b == 11 for b in w0), f"chunk {chunk}: worker0 slot corrupted"
|
||||
assert all(b == 22 for b in w1), f"chunk {chunk}: worker1 slot corrupted"
|
||||
del raw, t0, t1 # release before finally triggers cleanup
|
||||
|
||||
|
||||
@@ -466,7 +466,7 @@ def test_file_exists_after_construction(iid):
|
||||
|
||||
def test_file_has_correct_size(iid):
|
||||
"""The mmap file size on disk must equal total_size_bytes."""
|
||||
with _region(iid, num_blocks=4) as r:
|
||||
with _region(iid, num_chunks=4) as r:
|
||||
assert os.path.getsize(r.mmap_path) == 4 * PAGE_SIZE
|
||||
|
||||
|
||||
@@ -491,11 +491,11 @@ def test_madvise_success_selects_madvise_population(iid, monkeypatch):
|
||||
monkeypatch.setattr(sor, "_madvise_populate_write", _spy_madvise)
|
||||
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
||||
|
||||
num_blocks = 3
|
||||
num_chunks = 3
|
||||
num_workers = 2
|
||||
with _region(iid, num_blocks=num_blocks, num_workers=num_workers, rank=1):
|
||||
# 1 probe (PAGESIZE) + N populate calls (one per block per worker column).
|
||||
expected_populate = num_blocks # ranked path: one call per block
|
||||
with _region(iid, num_chunks=num_chunks, num_workers=num_workers, rank=1):
|
||||
# 1 probe (PAGESIZE) + N populate calls (one per chunk per worker column).
|
||||
expected_populate = num_chunks # ranked path: one call per chunk
|
||||
mmap_id = madvise_calls[0][2]
|
||||
assert madvise_calls == [
|
||||
(0, mmap.PAGESIZE, mmap_id),
|
||||
@@ -510,7 +510,7 @@ def test_madvise_success_selects_madvise_population(iid, monkeypatch):
|
||||
|
||||
|
||||
def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
|
||||
"""An EINVAL probe must select fallback for every ranked block."""
|
||||
"""An EINVAL probe must select fallback for every ranked chunk."""
|
||||
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
|
||||
|
||||
fallback_calls: list[tuple[int, int]] = []
|
||||
@@ -526,7 +526,7 @@ def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
|
||||
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
|
||||
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
||||
|
||||
with _region(iid, num_blocks=3, num_workers=2, rank=1):
|
||||
with _region(iid, num_chunks=3, num_workers=2, rank=1):
|
||||
assert fallback_calls == [
|
||||
(mmap.PAGESIZE, mmap.PAGESIZE),
|
||||
(3 * mmap.PAGESIZE, mmap.PAGESIZE),
|
||||
@@ -551,7 +551,7 @@ def test_madvise_einval_selects_fallback_for_unranked_region(iid, monkeypatch):
|
||||
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
|
||||
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
||||
|
||||
with _region(iid, num_blocks=3, num_workers=2, rank=None):
|
||||
with _region(iid, num_chunks=3, num_workers=2, rank=None):
|
||||
assert fallback_calls == [(0, 6 * mmap.PAGESIZE)]
|
||||
|
||||
|
||||
@@ -651,7 +651,7 @@ def test_multiprocess_race_construct_and_write(iid):
|
||||
"""N processes race to construct the same SharedOffloadRegion, each writes
|
||||
fill_value = rank+1 into their slot; parent verifies interleaved layout."""
|
||||
num_workers = 4
|
||||
num_blocks = 3
|
||||
num_chunks = 3
|
||||
|
||||
ctx = get_mp_context()
|
||||
done_queue = ctx.Queue()
|
||||
@@ -662,7 +662,7 @@ def test_multiprocess_race_construct_and_write(iid):
|
||||
target=_mp_race_construct_and_write,
|
||||
args=(
|
||||
iid,
|
||||
num_blocks,
|
||||
num_chunks,
|
||||
rank,
|
||||
num_workers,
|
||||
PAGE_SIZE,
|
||||
@@ -689,13 +689,13 @@ def test_multiprocess_race_construct_and_write(iid):
|
||||
with open(mmap_path, "rb") as f:
|
||||
raw = f.read()
|
||||
|
||||
for blk in range(num_blocks):
|
||||
for chunk in range(num_chunks):
|
||||
for w in range(num_workers):
|
||||
slot_start = (blk * num_workers + w) * PAGE_SIZE
|
||||
slot_start = (chunk * num_workers + w) * PAGE_SIZE
|
||||
slot = raw[slot_start : slot_start + PAGE_SIZE]
|
||||
expected = w + 1 # fill_value = rank + 1
|
||||
assert all(b == expected for b in slot), (
|
||||
f"block {blk}, worker {w}: expected {expected} but got wrong bytes"
|
||||
f"chunk {chunk}, worker {w}: expected {expected} but got wrong bytes"
|
||||
)
|
||||
|
||||
# Unblock all workers to clean up.
|
||||
@@ -839,9 +839,9 @@ def test_insufficient_space_raises_clear_error(monkeypatch):
|
||||
with pytest.raises(RuntimeError, match="Insufficient space"):
|
||||
SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=4,
|
||||
num_chunks=4,
|
||||
rank=0,
|
||||
kv_bytes_per_block=PAGE_SIZE,
|
||||
kv_bytes_per_chunk=PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
)
|
||||
|
||||
@@ -870,9 +870,9 @@ def test_ftruncate_failure_cleans_up_creator(monkeypatch):
|
||||
with pytest.raises(OSError, match="ftruncate failed"):
|
||||
SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=4,
|
||||
num_chunks=4,
|
||||
rank=0,
|
||||
kv_bytes_per_block=PAGE_SIZE,
|
||||
kv_bytes_per_chunk=PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
)
|
||||
|
||||
|
||||
@@ -136,7 +136,7 @@ def test_get_spec_cls_defaults_to_cpu():
|
||||
def test_create_cpu_offloading_spec():
|
||||
spec = _create_spec()
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.num_blocks > 0
|
||||
assert spec.num_chunks > 0
|
||||
|
||||
|
||||
def test_cpu_spec_sizes_normalized_worker_layout():
|
||||
@@ -156,7 +156,7 @@ def test_cpu_spec_sizes_normalized_worker_layout():
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 32
|
||||
assert spec.kv_bytes_per_chunk == alignment
|
||||
assert spec.num_blocks == 3
|
||||
assert spec.num_chunks == 3
|
||||
|
||||
|
||||
def test_cpu_spec_zero_worker_bytes_produces_empty_cache():
|
||||
@@ -165,7 +165,7 @@ def test_cpu_spec_zero_worker_bytes_produces_empty_cache():
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 0
|
||||
assert spec.kv_bytes_per_chunk == 0
|
||||
assert spec.num_blocks == 0
|
||||
assert spec.num_chunks == 0
|
||||
|
||||
|
||||
def test_tiering_spec_aligns_row_size():
|
||||
@@ -183,7 +183,7 @@ def test_tiering_spec_aligns_row_size():
|
||||
assert isinstance(spec, TieringOffloadingSpec)
|
||||
assert spec.cpu_page_size_per_worker == 32
|
||||
assert spec.kv_bytes_per_chunk == alignment
|
||||
assert spec.num_blocks == 3
|
||||
assert spec.num_chunks == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize("world_size", [2, 4, 8])
|
||||
@@ -201,7 +201,7 @@ def test_tiering_spec_replicated_sizing_removes_world_factor(world_size: int):
|
||||
assert spec.replicated_layout is True
|
||||
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
|
||||
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block
|
||||
assert spec.num_blocks == 8
|
||||
assert spec.num_chunks == 8
|
||||
|
||||
|
||||
def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monkeypatch):
|
||||
@@ -239,7 +239,7 @@ def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monke
|
||||
spec.create_worker(kv_caches)
|
||||
|
||||
assert region_calls[0]["rank"] == 0
|
||||
assert region_calls[0]["kv_bytes_per_block"] == worker_kv_bytes_per_block
|
||||
assert region_calls[0]["kv_bytes_per_chunk"] == worker_kv_bytes_per_block
|
||||
assert worker_calls[0]["kv_caches"] is kv_caches
|
||||
assert worker_calls[0]["mmap_region"] is region
|
||||
|
||||
@@ -292,7 +292,7 @@ def test_cpu_spec_replicated_sizing_on_shared_region(monkeypatch, world_size: in
|
||||
assert spec.replicated_layout is True
|
||||
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
|
||||
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block
|
||||
assert spec.num_blocks == 8
|
||||
assert spec.num_chunks == 8
|
||||
|
||||
|
||||
@pytest.mark.parametrize("world_size", [2, 4, 8])
|
||||
@@ -320,7 +320,7 @@ def test_cpu_spec_replicated_disabled_without_shared_region(
|
||||
assert spec.replicated_layout is False
|
||||
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
|
||||
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block * world_size
|
||||
assert spec.num_blocks == 2
|
||||
assert spec.num_chunks == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("config_replicated", [True, False])
|
||||
@@ -381,7 +381,7 @@ def test_cpu_spec_create_worker_uses_mmap_on_cuda_alike(monkeypatch):
|
||||
spec.create_worker(kv_caches)
|
||||
|
||||
assert region_calls[0]["engine_id"] == "test-engine"
|
||||
assert region_calls[0]["kv_bytes_per_block"] == worker_kv_bytes_per_block * 4
|
||||
assert region_calls[0]["kv_bytes_per_chunk"] == worker_kv_bytes_per_block * 4
|
||||
assert worker_calls[0]["kv_caches"] is kv_caches
|
||||
assert worker_calls[0]["mmap_region"] is region
|
||||
|
||||
@@ -419,11 +419,11 @@ def test_cpu_spec_create_worker_uses_tensor_path_off_cuda_alike(monkeypatch):
|
||||
def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch):
|
||||
import vllm.v1.kv_offload.cpu.spec as cpu_spec_module
|
||||
|
||||
# worker_kv_bytes_per_block=0 yields num_blocks=0; a zero-byte region cannot
|
||||
# worker_kv_bytes_per_block=0 yields num_chunks=0; a zero-byte region cannot
|
||||
# be mmap'd, so even on CUDA-alike this must fall back to the tensor path.
|
||||
spec = _create_spec(worker_kv_bytes_per_block=0, world_size=4)
|
||||
assert isinstance(spec, CPUOffloadingSpec)
|
||||
assert spec.num_blocks == 0
|
||||
assert spec.num_chunks == 0
|
||||
|
||||
region_calls: list[dict[str, Any]] = []
|
||||
worker_calls: list[dict[str, Any]] = []
|
||||
|
||||
@@ -91,17 +91,17 @@ def _req_context(kv_params: dict | None = None) -> ReqContext:
|
||||
def _job_metadata(
|
||||
job_id: int,
|
||||
keys: list[bytes] | None = None,
|
||||
block_ids: list[int] | None = None,
|
||||
chunk_ids: list[int] | None = None,
|
||||
kv_params: dict | None = None,
|
||||
) -> TransferJob:
|
||||
if keys is None:
|
||||
keys = [b"key1"]
|
||||
if block_ids is None:
|
||||
block_ids = list(range(len(keys)))
|
||||
if chunk_ids is None:
|
||||
chunk_ids = list(range(len(keys)))
|
||||
return TransferJob(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=np.array(block_ids),
|
||||
chunk_ids=np.array(chunk_ids),
|
||||
is_promotion=False,
|
||||
req_context=_req_context(kv_params),
|
||||
)
|
||||
@@ -254,9 +254,9 @@ class TestLookup:
|
||||
"kv_params,expected",
|
||||
[
|
||||
(_remote_decoder_kv_params(), OffloadPolicy.REQUEST_LEVEL),
|
||||
(None, OffloadPolicy.BLOCK_LEVEL),
|
||||
(_remote_prefiller_kv_params(), OffloadPolicy.BLOCK_LEVEL),
|
||||
({"remote_decoder": {}}, OffloadPolicy.BLOCK_LEVEL),
|
||||
(None, OffloadPolicy.CHUNK_LEVEL),
|
||||
(_remote_prefiller_kv_params(), OffloadPolicy.CHUNK_LEVEL),
|
||||
({"remote_decoder": {}}, OffloadPolicy.CHUNK_LEVEL),
|
||||
],
|
||||
ids=["producer", "plain", "consumer", "producer_no_id"],
|
||||
)
|
||||
@@ -366,7 +366,7 @@ class TestSubmitStore:
|
||||
job = _job_metadata(
|
||||
job_id=1,
|
||||
keys=[b"k1", b"k2"],
|
||||
block_ids=[3, 4],
|
||||
chunk_ids=[3, 4],
|
||||
kv_params=_remote_decoder_kv_params(kv_request_id="req-1"),
|
||||
)
|
||||
mgr.submit_store(job)
|
||||
@@ -397,7 +397,7 @@ class TestSubmitStore:
|
||||
job = _job_metadata(
|
||||
job_id=7,
|
||||
keys=[b"k1", b"k2"],
|
||||
block_ids=[3, 4],
|
||||
chunk_ids=[3, 4],
|
||||
kv_params=_remote_decoder_kv_params(kv_request_id="req-1"),
|
||||
)
|
||||
mgr.submit_store(job)
|
||||
@@ -440,7 +440,7 @@ class TestSubmitLoad:
|
||||
"""Empty key list succeeds immediately."""
|
||||
mgr = _make_manager()
|
||||
job = _job_metadata(
|
||||
job_id=1, keys=[], block_ids=[], kv_params=_remote_prefiller_kv_params()
|
||||
job_id=1, keys=[], chunk_ids=[], kv_params=_remote_prefiller_kv_params()
|
||||
)
|
||||
mgr.submit_load(job)
|
||||
assert mgr._finished_jobs == [JobResult(job_id=1, success=True)]
|
||||
@@ -463,7 +463,7 @@ class TestSubmitLoad:
|
||||
job = _job_metadata(
|
||||
job_id=42,
|
||||
keys=[b"k1", b"k2"],
|
||||
block_ids=[5, 6],
|
||||
chunk_ids=[5, 6],
|
||||
kv_params=_remote_prefiller_kv_params(kv_request_id="req-42"),
|
||||
)
|
||||
mgr.submit_load(job)
|
||||
@@ -1155,7 +1155,7 @@ class TestBidirectionalManager:
|
||||
_job_metadata(
|
||||
job_id=100,
|
||||
keys=[b"a-block"],
|
||||
block_ids=[0],
|
||||
chunk_ids=[0],
|
||||
kv_params=a_prefiller_params,
|
||||
)
|
||||
)
|
||||
@@ -1163,7 +1163,7 @@ class TestBidirectionalManager:
|
||||
_job_metadata(
|
||||
job_id=200,
|
||||
keys=[b"b-block"],
|
||||
block_ids=[0],
|
||||
chunk_ids=[0],
|
||||
kv_params=b_prefiller_params,
|
||||
)
|
||||
)
|
||||
@@ -1173,7 +1173,7 @@ class TestBidirectionalManager:
|
||||
_job_metadata(
|
||||
job_id=101,
|
||||
keys=[b"b-block"],
|
||||
block_ids=[0],
|
||||
chunk_ids=[0],
|
||||
kv_params=a_decoder_params,
|
||||
)
|
||||
)
|
||||
@@ -1181,7 +1181,7 @@ class TestBidirectionalManager:
|
||||
_job_metadata(
|
||||
job_id=201,
|
||||
keys=[b"a-block"],
|
||||
block_ids=[0],
|
||||
chunk_ids=[0],
|
||||
kv_params=b_decoder_params,
|
||||
)
|
||||
)
|
||||
@@ -1573,7 +1573,7 @@ class TestConnectionDeathMidTransfer:
|
||||
_job_metadata(
|
||||
job_id=900,
|
||||
keys=[b"a-block"],
|
||||
block_ids=[0],
|
||||
chunk_ids=[0],
|
||||
kv_params=a_prefiller_params,
|
||||
)
|
||||
)
|
||||
@@ -1581,7 +1581,7 @@ class TestConnectionDeathMidTransfer:
|
||||
_job_metadata(
|
||||
job_id=901,
|
||||
keys=[b"b-block"],
|
||||
block_ids=[0],
|
||||
chunk_ids=[0],
|
||||
kv_params=a_decoder_params,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -259,13 +259,13 @@ class FakeParent:
|
||||
) -> TransferJob:
|
||||
keys_list = list(keys)
|
||||
self.calls.append(("create_store_job", tuple(keys_list), ctx.req_id))
|
||||
block_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32)
|
||||
chunk_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32)
|
||||
job_id = self._next_job_id
|
||||
self._next_job_id += 1
|
||||
return TransferJob(
|
||||
job_id=job_id,
|
||||
keys=keys_list,
|
||||
block_ids=block_ids,
|
||||
chunk_ids=chunk_ids,
|
||||
is_promotion=False,
|
||||
req_context=ctx,
|
||||
)
|
||||
|
||||
@@ -105,15 +105,15 @@ def key(n: int) -> OffloadKey:
|
||||
def make_job(
|
||||
job_id: int,
|
||||
keys: list[OffloadKey],
|
||||
block_ids: list[int] | None = None,
|
||||
chunk_ids: list[int] | None = None,
|
||||
is_promotion: bool = False,
|
||||
) -> TransferJob:
|
||||
if block_ids is None:
|
||||
block_ids = list(range(len(keys)))
|
||||
if chunk_ids is None:
|
||||
chunk_ids = list(range(len(keys)))
|
||||
return TransferJob(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=np.array(block_ids, dtype=np.int64),
|
||||
chunk_ids=np.array(chunk_ids, dtype=np.int64),
|
||||
is_promotion=is_promotion,
|
||||
req_context=_CTX,
|
||||
)
|
||||
@@ -382,11 +382,11 @@ def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size):
|
||||
tensor[:] = _page_aligned_rand_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS)
|
||||
|
||||
keys = [key(i) for i in range(batch_size)]
|
||||
store_block_ids = list(range(batch_size))
|
||||
load_block_ids = list(range(_NUM_BLOCKS - batch_size, _NUM_BLOCKS))
|
||||
store_chunk_ids = list(range(batch_size))
|
||||
load_chunk_ids = list(range(_NUM_BLOCKS - batch_size, _NUM_BLOCKS))
|
||||
expected = tensor[:batch_size].clone()
|
||||
|
||||
tier.submit_store(make_job(1, keys, store_block_ids))
|
||||
tier.submit_store(make_job(1, keys, store_chunk_ids))
|
||||
store_results = drain(tier)
|
||||
assert len(store_results) == 1
|
||||
assert store_results[0].success
|
||||
@@ -396,15 +396,15 @@ def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size):
|
||||
tensor[:] = 0.0
|
||||
|
||||
# Load into a range disjoint by index from the store ids, to also
|
||||
# exercise loading a block into a different id than it was stored from.
|
||||
tier.submit_load(make_job(2, keys, load_block_ids, is_promotion=True))
|
||||
# exercise loading a chunk into a different id than it was stored from.
|
||||
tier.submit_load(make_job(2, keys, load_chunk_ids, is_promotion=True))
|
||||
load_results = drain(tier)
|
||||
assert len(load_results) == 1
|
||||
assert load_results[0].success
|
||||
|
||||
for i, bid in enumerate(load_block_ids):
|
||||
assert torch.allclose(tensor[bid], expected[i]), (
|
||||
f"Block {bid} data mismatch after store+load"
|
||||
for i, cid in enumerate(load_chunk_ids):
|
||||
assert torch.allclose(tensor[cid], expected[i]), (
|
||||
f"Chunk {cid} data mismatch after store+load"
|
||||
)
|
||||
|
||||
|
||||
@@ -895,7 +895,7 @@ def test_cascade_store_emits_fs_event_through_tiering_manager(tmp_path):
|
||||
view = memoryview(tensor.numpy())
|
||||
mock_region = MagicMock()
|
||||
mock_region.create_kv_memoryview.return_value = view
|
||||
primary = CPUPrimaryTierOffloadingManager(num_blocks=4, mmap_region=mock_region)
|
||||
primary = CPUPrimaryTierOffloadingManager(num_chunks=4, mmap_region=mock_region)
|
||||
tier = FileSystemTierManager(
|
||||
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
|
||||
primary_kv_view=primary.get_kv_memoryview(),
|
||||
|
||||
@@ -25,8 +25,8 @@ def to_keys(int_ids: Iterable[int]):
|
||||
def test_tiering_metrics_tracker_records_lookup_metrics():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs", "p2p"],
|
||||
num_primary_blocks=5,
|
||||
primary_block_size=16,
|
||||
num_primary_chunks=5,
|
||||
primary_chunk_size=16,
|
||||
)
|
||||
tracker.on_new_request(_CTX)
|
||||
|
||||
@@ -45,18 +45,18 @@ def test_tiering_metrics_tracker_records_lookup_metrics():
|
||||
stats = tracker.take_stats()
|
||||
assert stats is not None
|
||||
values = stats.data["data"]
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("2:p2p",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_HITS][("2:p2p",)] == 1
|
||||
assert ("1:fs",) not in values[TieringOffloadingMetrics.BLOCK_HITS]
|
||||
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("0:primary",)] == 1
|
||||
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("1:fs",)] == 1
|
||||
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("2:p2p",)] == 1
|
||||
assert values[TieringOffloadingMetrics.CHUNK_HITS][("2:p2p",)] == 1
|
||||
assert ("1:fs",) not in values[TieringOffloadingMetrics.CHUNK_HITS]
|
||||
|
||||
|
||||
def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs"],
|
||||
num_primary_blocks=5,
|
||||
primary_block_size=16,
|
||||
num_primary_chunks=5,
|
||||
primary_chunk_size=16,
|
||||
)
|
||||
tracker.on_new_request(_CTX)
|
||||
key = to_keys([1])[0]
|
||||
@@ -73,9 +73,9 @@ def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
|
||||
stats = tracker.take_stats()
|
||||
assert stats is not None
|
||||
values = stats.data["data"]
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_HITS][("1:fs",)] == 1
|
||||
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("0:primary",)] == 1
|
||||
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("1:fs",)] == 1
|
||||
assert values[TieringOffloadingMetrics.CHUNK_HITS][("1:fs",)] == 1
|
||||
|
||||
tracker.on_request_allocated(_CTX)
|
||||
tracker.on_lookup(
|
||||
@@ -90,15 +90,15 @@ def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
|
||||
stats = tracker.take_stats()
|
||||
assert stats is not None
|
||||
values = stats.data["data"]
|
||||
assert TieringOffloadingMetrics.BLOCK_QUERIES not in values
|
||||
assert TieringOffloadingMetrics.BLOCK_HITS not in values
|
||||
assert TieringOffloadingMetrics.CHUNK_QUERIES not in values
|
||||
assert TieringOffloadingMetrics.CHUNK_HITS not in values
|
||||
|
||||
|
||||
def test_tiering_metrics_tracker_records_finished_job_metrics():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs"],
|
||||
num_primary_blocks=5,
|
||||
primary_block_size=16,
|
||||
num_primary_chunks=5,
|
||||
primary_chunk_size=16,
|
||||
)
|
||||
cascade_key, promotion_key, failed_cascade_key, failed_promotion_key = to_keys(
|
||||
range(4)
|
||||
@@ -149,8 +149,8 @@ def test_tiering_metrics_tracker_records_finished_job_metrics():
|
||||
def test_tiering_metrics_tracker_records_partial_promotion_success_bytes():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs"],
|
||||
num_primary_blocks=5,
|
||||
primary_block_size=16,
|
||||
num_primary_chunks=5,
|
||||
primary_chunk_size=16,
|
||||
)
|
||||
keys = to_keys(range(3))
|
||||
job = JobMetadata(
|
||||
@@ -182,8 +182,8 @@ def test_tiering_metrics_tracker_records_partial_promotion_success_bytes():
|
||||
def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs", "p2p"],
|
||||
num_primary_blocks=6,
|
||||
primary_block_size=16,
|
||||
num_primary_chunks=6,
|
||||
primary_chunk_size=16,
|
||||
)
|
||||
fs_job = JobMetadata(
|
||||
TransferJob(0, to_keys([0, 1]), np.array([0, 1]), False, _CTX),
|
||||
@@ -214,8 +214,8 @@ def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges():
|
||||
def test_tiering_metrics_tracker_records_promotion_allocation_failures():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs"],
|
||||
num_primary_blocks=1,
|
||||
primary_block_size=16,
|
||||
num_primary_chunks=1,
|
||||
primary_chunk_size=16,
|
||||
)
|
||||
|
||||
tracker.on_promotion_allocation_failure()
|
||||
|
||||
@@ -106,14 +106,14 @@ def key(n: int) -> OffloadKey:
|
||||
def make_job(
|
||||
job_id: int,
|
||||
keys: list[OffloadKey],
|
||||
block_ids: list[int] | None = None,
|
||||
chunk_ids: list[int] | None = None,
|
||||
) -> TransferJob:
|
||||
if block_ids is None:
|
||||
block_ids = list(range(len(keys)))
|
||||
if chunk_ids is None:
|
||||
chunk_ids = list(range(len(keys)))
|
||||
return TransferJob(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=np.array(block_ids, dtype=np.int32),
|
||||
chunk_ids=np.array(chunk_ids, dtype=np.int32),
|
||||
is_promotion=False,
|
||||
req_context=_CTX,
|
||||
)
|
||||
@@ -541,7 +541,7 @@ class TestMockObjTierFailures:
|
||||
mmap_region = MagicMock()
|
||||
mmap_region.create_kv_memoryview.return_value = primary_kv_view
|
||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=num_blocks, mmap_region=mmap_region
|
||||
num_chunks=num_blocks, mmap_region=mmap_region
|
||||
)
|
||||
obj_tier, agent = _make_tier(
|
||||
num_blocks=num_blocks, primary_kv_view=primary_kv_view
|
||||
|
||||
@@ -5,15 +5,16 @@ Unit tests for TieringOffloadingManager and ExampleSecondaryTierManager.
|
||||
|
||||
These tests verify:
|
||||
1. Basic tiered offloading operations (store, load, lookup)
|
||||
2. Cascade behavior (blocks stored to all secondary tiers)
|
||||
3. Promotion behavior (blocks loaded from secondary to primary to GPU)
|
||||
4. ref_cnt management (blocks protected during async transfers)
|
||||
2. Cascade behavior (chunks stored to all secondary tiers)
|
||||
3. Promotion behavior (chunks loaded from secondary to primary to GPU)
|
||||
4. ref_cnt management (chunks protected during async transfers)
|
||||
5. Eviction coordination between tiers
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -57,10 +58,10 @@ _CTX = ReqContext(req_id="test")
|
||||
_MOCK_OFFLOADING_SPEC = MagicMock()
|
||||
|
||||
|
||||
def _mock_mmap_region(num_blocks: int, row_bytes: int = 16):
|
||||
def _mock_mmap_region(num_chunks: int, row_bytes: int = 16):
|
||||
"""Create a mock SharedOffloadRegion for testing."""
|
||||
mock = MagicMock()
|
||||
view = memoryview(torch.zeros((num_blocks, row_bytes), dtype=torch.int8).numpy())
|
||||
view = memoryview(torch.zeros((num_chunks, row_bytes), dtype=torch.int8).numpy())
|
||||
mock.create_kv_memoryview.return_value = view
|
||||
return mock
|
||||
|
||||
@@ -160,7 +161,7 @@ def test_tiering_spec_collects_secondary_metric_definitions(monkeypatch):
|
||||
metadata = metrics[MetricsSecondaryTierManager.MY_TIER_METRIC]
|
||||
assert metadata.documentation == "Number of bytes served by the test tier."
|
||||
assert metadata.labelnames == ("tier",)
|
||||
assert metrics[TieringOffloadingMetrics.BLOCK_QUERIES].labelnames == ("tier",)
|
||||
assert metrics[TieringOffloadingMetrics.CHUNK_QUERIES].labelnames == ("tier",)
|
||||
assert metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY].labelnames == ("tier",)
|
||||
assert metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY].labelnames == ("tier",)
|
||||
assert metrics[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES].labelnames == (
|
||||
@@ -183,7 +184,7 @@ def test_tiering_spec_collects_secondary_metric_definitions(monkeypatch):
|
||||
def test_tiering_manager_aggregates_secondary_stats():
|
||||
mock_region = _mock_mmap_region(5)
|
||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=5, mmap_region=mock_region
|
||||
num_chunks=5, mmap_region=mock_region
|
||||
)
|
||||
secondary_tier = MetricsSecondaryTierManager(
|
||||
offloading_spec=_MOCK_OFFLOADING_SPEC,
|
||||
@@ -232,19 +233,19 @@ class TestExampleSecondaryTierManager:
|
||||
)
|
||||
|
||||
# Initially empty
|
||||
blocks = to_keys(range(3))
|
||||
assert tier.lookup(blocks[0], _CTX) is LookupResult.MISS
|
||||
chunks = to_keys(range(3))
|
||||
assert tier.lookup(chunks[0], _CTX) is LookupResult.MISS
|
||||
|
||||
# Store blocks (simulate with direct insertion for testing)
|
||||
tier.blocks[blocks[0]] = True
|
||||
tier.blocks[blocks[1]] = True
|
||||
# Store chunks (simulate with direct insertion for testing)
|
||||
tier.chunks[chunks[0]] = True
|
||||
tier.chunks[chunks[1]] = True
|
||||
|
||||
# Lookup should find first two blocks
|
||||
assert tier.lookup(blocks[0], _CTX) is LookupResult.HIT
|
||||
assert tier.lookup(blocks[1], _CTX) is LookupResult.HIT
|
||||
# Lookup should find first two chunks
|
||||
assert tier.lookup(chunks[0], _CTX) is LookupResult.HIT
|
||||
assert tier.lookup(chunks[1], _CTX) is LookupResult.HIT
|
||||
|
||||
# Third block not present
|
||||
assert tier.lookup(blocks[2], _CTX) is LookupResult.MISS
|
||||
# Third chunk not present
|
||||
assert tier.lookup(chunks[2], _CTX) is LookupResult.MISS
|
||||
|
||||
|
||||
# What a request-level cascade does with a key already present in the primary
|
||||
@@ -260,7 +261,7 @@ _CASCADE_SUPPLY_BEHAVIOR = {
|
||||
# The same test, before it lands.
|
||||
LookupResult.HIT_PENDING: "defer",
|
||||
# test_cascade_drops_deferred_keys_whose_primary_write_failed, where the
|
||||
# failed write frees the block.
|
||||
# failed write frees the chunk.
|
||||
LookupResult.MISS: "drop",
|
||||
# test_cascade_rejects_retry_from_primary. The primary tier resolves every
|
||||
# key it holds, so RETRY cannot reach the cascade, and parking on it would
|
||||
@@ -284,7 +285,7 @@ class TestTieringOffloadingManager:
|
||||
# Create primary tier (CPU-based)
|
||||
mock_region = _mock_mmap_region(5)
|
||||
self.primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=5, mmap_region=mock_region
|
||||
num_chunks=5, mmap_region=mock_region
|
||||
)
|
||||
|
||||
mock_view = mock_region.create_kv_memoryview()
|
||||
@@ -331,7 +332,7 @@ class TestTieringOffloadingManager:
|
||||
TransferJob(
|
||||
job_id=job_id,
|
||||
keys=to_keys([1, 2]),
|
||||
block_ids=[0, 1],
|
||||
chunk_ids=np.array([0, 1], dtype=np.int64),
|
||||
is_promotion=True,
|
||||
req_context=_CTX,
|
||||
),
|
||||
@@ -358,7 +359,7 @@ class TestTieringOffloadingManager:
|
||||
TransferJob(
|
||||
job_id=job_id,
|
||||
keys=to_keys([1]),
|
||||
block_ids=[0],
|
||||
chunk_ids=np.array([0], dtype=np.int64),
|
||||
is_promotion=True,
|
||||
req_context=_CTX,
|
||||
),
|
||||
@@ -392,23 +393,23 @@ class TestTieringOffloadingManager:
|
||||
|
||||
def test_basic_store_to_primary(self, manager_setup):
|
||||
"""Test basic store operation to primary tier."""
|
||||
blocks = to_keys(range(3))
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
# Prepare store
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
result = self.manager.prepare_store(chunks, _CTX)
|
||||
assert result is not None
|
||||
assert len(result.keys_to_store) == 3
|
||||
|
||||
# Complete store
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
|
||||
# Blocks should be in primary tier
|
||||
assert count_hits(self.primary_tier, blocks) == 3
|
||||
# Chunks should be in primary tier
|
||||
assert count_hits(self.primary_tier, chunks) == 3
|
||||
|
||||
def test_cascade_to_all_secondary_tiers(self, manager_setup):
|
||||
"""Test that blocks are cascaded to ALL secondary tiers."""
|
||||
blocks = to_keys(range(3))
|
||||
"""Test that chunks are cascaded to ALL secondary tiers."""
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_store
|
||||
@@ -419,44 +420,44 @@ class TestTieringOffloadingManager:
|
||||
|
||||
# Store to primary
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
result = self.manager.prepare_store(chunks, _CTX)
|
||||
assert result is not None
|
||||
|
||||
# Complete store (triggers cascade via submit_store on each tier)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
|
||||
# submit_store was called once per secondary tier
|
||||
self.secondary_tier1.submit_store.assert_called_once()
|
||||
self.secondary_tier2.submit_store.assert_called_once()
|
||||
|
||||
# Blocks should be in both secondary tiers
|
||||
assert self.secondary_tier1.get_num_blocks() == 3
|
||||
assert self.secondary_tier2.get_num_blocks() == 3
|
||||
# Chunks should be in both secondary tiers
|
||||
assert self.secondary_tier1.get_num_chunks() == 3
|
||||
assert self.secondary_tier2.get_num_chunks() == 3
|
||||
|
||||
# Verify blocks are present
|
||||
# Verify chunks are present
|
||||
assert all(
|
||||
self.secondary_tier1.lookup(b, _CTX) is LookupResult.HIT for b in blocks
|
||||
self.secondary_tier1.lookup(c, _CTX) is LookupResult.HIT for c in chunks
|
||||
)
|
||||
assert all(
|
||||
self.secondary_tier2.lookup(b, _CTX) is LookupResult.HIT for b in blocks
|
||||
self.secondary_tier2.lookup(c, _CTX) is LookupResult.HIT for c in chunks
|
||||
)
|
||||
|
||||
def test_ref_cnt_protection_during_cascade(self, manager_setup):
|
||||
"""Test that ref_cnt protects blocks during cascade."""
|
||||
blocks = to_keys(range(3))
|
||||
"""Test that ref_cnt protects chunks during cascade."""
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
# Store to primary
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
result = self.manager.prepare_store(chunks, _CTX)
|
||||
assert result is not None
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
|
||||
# After complete_store, blocks should have ref_cnt > 0
|
||||
# After complete_store, chunks should have ref_cnt > 0
|
||||
# (one for each secondary tier)
|
||||
for block_hash in blocks:
|
||||
block = self.primary_tier._policy.get(block_hash)
|
||||
for key in chunks:
|
||||
chunk = self.primary_tier._policy.get(key)
|
||||
# ref_cnt should be 2 (one for each secondary tier)
|
||||
assert block.ref_cnt == 2
|
||||
assert chunk.ref_cnt == 2
|
||||
|
||||
# End of step 1: _maybe_process_finished_jobs() was already called by
|
||||
# prepare_store() above (setting the per-step flag), so on_schedule_end()
|
||||
@@ -466,9 +467,9 @@ class TestTieringOffloadingManager:
|
||||
|
||||
# ref_cnt still held: cascade jobs finished (sync tier) but haven't
|
||||
# been polled yet because the per-step guard skipped the second call.
|
||||
for block_hash in blocks:
|
||||
block = self.primary_tier._policy.get(block_hash)
|
||||
assert block.ref_cnt == 2
|
||||
for key in chunks:
|
||||
chunk = self.primary_tier._policy.get(key)
|
||||
assert chunk.ref_cnt == 2
|
||||
|
||||
# Secondary tiers have completed jobs waiting to be drained
|
||||
assert len(self.secondary_tier1.completed_jobs) > 0
|
||||
@@ -479,37 +480,37 @@ class TestTieringOffloadingManager:
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# After cascade completes, ref_cnt should be 0
|
||||
for block_hash in blocks:
|
||||
block = self.primary_tier._policy.get(block_hash)
|
||||
assert block.ref_cnt == 0
|
||||
for key in chunks:
|
||||
chunk = self.primary_tier._policy.get(key)
|
||||
assert chunk.ref_cnt == 0
|
||||
|
||||
# All completed jobs have been drained
|
||||
assert len(self.secondary_tier1.completed_jobs) == 0
|
||||
assert len(self.secondary_tier2.completed_jobs) == 0
|
||||
|
||||
def test_lookup_from_primary(self, manager_setup):
|
||||
"""Test lookup when blocks are in primary tier."""
|
||||
blocks = to_keys(range(3))
|
||||
"""Test lookup when chunks are in primary tier."""
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
# Store blocks
|
||||
# Store chunks
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.prepare_store(chunks, _CTX)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
|
||||
# Lookup should find all blocks in primary
|
||||
assert count_hits(self.manager, blocks) == 3
|
||||
# Lookup should find all chunks in primary
|
||||
assert count_hits(self.manager, chunks) == 3
|
||||
|
||||
def test_promotion_from_secondary(self, manager_setup):
|
||||
"""Test promotion of blocks from secondary to primary tier."""
|
||||
blocks = to_keys(range(3))
|
||||
"""Test promotion of chunks from secondary to primary tier."""
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
# Manually add blocks to secondary tier (simulate previous cascade)
|
||||
for block in blocks:
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
# Manually add chunks to secondary tier (simulate previous cascade)
|
||||
for chunk in chunks:
|
||||
self.secondary_tier1.chunks[chunk] = True
|
||||
|
||||
# Lookup each block to initiate promotion for all of them
|
||||
for block in blocks:
|
||||
result = self.manager.lookup(block, _CTX)
|
||||
# Lookup each chunk to initiate promotion for all of them
|
||||
for chunk in chunks:
|
||||
result = self.manager.lookup(chunk, _CTX)
|
||||
assert result is LookupResult.HIT_PENDING # promotion initiated
|
||||
|
||||
# End of step 1: flushes deferred submit_load() calls
|
||||
@@ -518,11 +519,11 @@ class TestTieringOffloadingManager:
|
||||
# End of step 2: processes the completed promotion jobs
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# Now blocks should be in primary tier
|
||||
assert count_hits(self.primary_tier, blocks) == 3
|
||||
# Now chunks should be in primary tier
|
||||
assert count_hits(self.primary_tier, chunks) == 3
|
||||
|
||||
# Next lookup should succeed
|
||||
assert count_hits(self.manager, blocks) == 3
|
||||
assert count_hits(self.manager, chunks) == 3
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("successful_indices", "expected_results"),
|
||||
@@ -538,18 +539,18 @@ class TestTieringOffloadingManager:
|
||||
],
|
||||
ids=["partial", "legacy-full-failure"],
|
||||
)
|
||||
def test_failed_promotion_keeps_only_successful_blocks(
|
||||
def test_failed_promotion_keeps_only_successful_chunks(
|
||||
self, manager_setup, successful_indices, expected_results
|
||||
):
|
||||
blocks = to_keys(range(3))
|
||||
for block in blocks:
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
chunks = to_keys(range(3))
|
||||
for chunk in chunks:
|
||||
self.secondary_tier1.chunks[chunk] = True
|
||||
|
||||
def submit_partial(job_metadata: TransferJob) -> None:
|
||||
successful_keys = (
|
||||
None
|
||||
if successful_indices is None
|
||||
else tuple(blocks[i] for i in successful_indices)
|
||||
else tuple(chunks[i] for i in successful_indices)
|
||||
)
|
||||
self.secondary_tier1.completed_jobs.append(
|
||||
JobResult(
|
||||
@@ -561,24 +562,24 @@ class TestTieringOffloadingManager:
|
||||
|
||||
self.secondary_tier1.submit_load = submit_partial
|
||||
|
||||
for block in blocks:
|
||||
assert self.manager.lookup(block, _CTX) is LookupResult.HIT_PENDING
|
||||
for chunk in chunks:
|
||||
assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT_PENDING
|
||||
|
||||
self._simulate_on_schedule_end()
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
assert [
|
||||
self.primary_tier.lookup(block, _CTX) for block in blocks
|
||||
self.primary_tier.lookup(chunk, _CTX) for chunk in chunks
|
||||
] == expected_results
|
||||
|
||||
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
|
||||
"""Resolved lookups report one sync delay sample per tier and block."""
|
||||
"""Resolved lookups report one sync delay sample per tier and chunk."""
|
||||
self._start_request()
|
||||
blocks = to_keys(range(2))
|
||||
chunks = to_keys(range(2))
|
||||
|
||||
# No tier has these blocks: they resolve immediately as misses.
|
||||
for block in blocks:
|
||||
assert self.manager.lookup(block, _CTX) is LookupResult.MISS
|
||||
# No tier has these chunks: they resolve immediately as misses.
|
||||
for chunk in chunks:
|
||||
assert self.manager.lookup(chunk, _CTX) is LookupResult.MISS
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
assert stats is not None
|
||||
@@ -614,11 +615,11 @@ class TestTieringOffloadingManager:
|
||||
def test_lookup_does_not_report_async_delay_for_promotion(self, manager_setup):
|
||||
"""Promotion time is not included in lookup async delay."""
|
||||
self._start_request()
|
||||
block = to_keys(range(1))[0]
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
chunk = to_keys(range(1))[0]
|
||||
self.secondary_tier1.chunks[chunk] = True
|
||||
|
||||
# First lookup finds the block in a secondary tier and defers.
|
||||
assert self.manager.lookup(block, _CTX) is LookupResult.HIT_PENDING
|
||||
# First lookup finds the chunk in a secondary tier and defers.
|
||||
assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT_PENDING
|
||||
|
||||
# Promotion is not treated as unresolved secondary lookup time.
|
||||
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
|
||||
@@ -648,8 +649,8 @@ class TestTieringOffloadingManager:
|
||||
# Promotion completes on the next scheduler step.
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# Next lookup resolves via the now-promoted primary-tier block.
|
||||
assert self.manager.lookup(block, _CTX) is LookupResult.HIT
|
||||
# Next lookup resolves via the now-promoted primary-tier chunk.
|
||||
assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
if stats is not None:
|
||||
@@ -663,13 +664,13 @@ class TestTieringOffloadingManager:
|
||||
"""Async delay is observed when a RETRY lookup later resolves."""
|
||||
ctx = ReqContext(req_id="req_lookup_finish")
|
||||
self._start_request(ctx)
|
||||
block = to_keys(range(1))[0]
|
||||
chunk = to_keys(range(1))[0]
|
||||
self.secondary_tier1.lookup = MagicMock(
|
||||
side_effect=[LookupResult.RETRY, LookupResult.HIT]
|
||||
)
|
||||
|
||||
# First lookup is deferred by the secondary tier.
|
||||
assert self.manager.lookup(block, ctx) is LookupResult.RETRY
|
||||
assert self.manager.lookup(chunk, ctx) is LookupResult.RETRY
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
if stats is not None:
|
||||
@@ -677,7 +678,7 @@ class TestTieringOffloadingManager:
|
||||
stats.reduce(), TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
|
||||
)
|
||||
|
||||
assert self.manager.lookup(block, ctx) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(chunk, ctx) is LookupResult.HIT_PENDING
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
assert stats is not None
|
||||
@@ -701,68 +702,68 @@ class TestTieringOffloadingManager:
|
||||
|
||||
def test_partial_lookup(self, manager_setup):
|
||||
"""Test lookup with partial hits."""
|
||||
blocks = to_keys(range(5))
|
||||
chunks = to_keys(range(5))
|
||||
|
||||
# Store first 3 blocks to primary
|
||||
# Store first 3 chunks to primary
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks[:3], _CTX)
|
||||
self.manager.complete_store(blocks[:3], _CTX, success=True)
|
||||
self.manager.prepare_store(chunks[:3], _CTX)
|
||||
self.manager.complete_store(chunks[:3], _CTX, success=True)
|
||||
|
||||
# Lookup all 5 blocks should return 3 (first 3 found)
|
||||
assert count_hits(self.manager, blocks) == 3
|
||||
# Lookup all 5 chunks should return 3 (first 3 found)
|
||||
assert count_hits(self.manager, chunks) == 3
|
||||
|
||||
def test_eviction_in_primary_tier(self, manager_setup):
|
||||
"""Test eviction in primary tier when capacity is exceeded."""
|
||||
# Primary tier has capacity of 5 blocks
|
||||
# Primary tier has capacity of 5 chunks
|
||||
# First, fill the primary tier
|
||||
blocks = to_keys(range(5))
|
||||
chunks = to_keys(range(5))
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
result = self.manager.prepare_store(chunks, _CTX)
|
||||
assert result is not None
|
||||
assert len(result.keys_to_store) == 5
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
|
||||
# End of step: release ref_cnt from cascade
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# Now try to store 2 more blocks (should trigger eviction)
|
||||
more_blocks = to_keys(range(5, 7))
|
||||
result = self.manager.prepare_store(more_blocks, _CTX)
|
||||
# Now try to store 2 more chunks (should trigger eviction)
|
||||
more_chunks = to_keys(range(5, 7))
|
||||
result = self.manager.prepare_store(more_chunks, _CTX)
|
||||
|
||||
# Should evict 2 blocks from primary tier
|
||||
# Should evict 2 chunks from primary tier
|
||||
assert result is not None
|
||||
assert len(result.evicted_keys) == 2
|
||||
assert len(result.keys_to_store) == 2
|
||||
|
||||
def test_touch_propagates_to_all_tiers(self, manager_setup):
|
||||
"""Test that touch() propagates to all tiers."""
|
||||
blocks = to_keys(range(3))
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
# Store blocks
|
||||
# Store chunks
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.prepare_store(chunks, _CTX)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
self._simulate_on_schedule_end()
|
||||
# for secondary tiers to drain jobs, so primary tier's blocks are evictable.
|
||||
# for secondary tiers to drain jobs, so primary tier's chunks are evictable.
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
self.secondary_tier1.touch = MagicMock(wraps=self.secondary_tier1.touch)
|
||||
self.secondary_tier2.touch = MagicMock(wraps=self.secondary_tier2.touch)
|
||||
|
||||
# Touch blocks
|
||||
self.manager.touch(blocks, _CTX)
|
||||
# Touch chunks
|
||||
self.manager.touch(chunks, _CTX)
|
||||
|
||||
# Verify touch was called on primary tier (check LRU order)
|
||||
primary_keys = list(self.primary_tier._policy.evictable_blocks.keys())
|
||||
assert primary_keys[-3:] == list(reversed(blocks))
|
||||
primary_keys = list(self.primary_tier._policy.evictable_chunks.keys())
|
||||
assert primary_keys[-3:] == list(reversed(chunks))
|
||||
|
||||
# Verify touch was propagated to all secondary tiers
|
||||
self.secondary_tier1.touch.assert_called_once_with(blocks, _CTX)
|
||||
self.secondary_tier2.touch.assert_called_once_with(blocks, _CTX)
|
||||
self.secondary_tier1.touch.assert_called_once_with(chunks, _CTX)
|
||||
self.secondary_tier2.touch.assert_called_once_with(chunks, _CTX)
|
||||
|
||||
def test_failed_store_no_cascade(self, manager_setup):
|
||||
"""Test that failed GPU→primary store doesn't cascade."""
|
||||
blocks = to_keys(range(3))
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_store
|
||||
@@ -773,11 +774,11 @@ class TestTieringOffloadingManager:
|
||||
|
||||
# Prepare store
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
result = self.manager.prepare_store(chunks, _CTX)
|
||||
assert result is not None
|
||||
|
||||
# Complete store with failure — cascade must not happen
|
||||
self.manager.complete_store(blocks, _CTX, success=False)
|
||||
self.manager.complete_store(chunks, _CTX, success=False)
|
||||
|
||||
# submit_store was never called on either secondary tier
|
||||
self.secondary_tier1.submit_store.assert_not_called()
|
||||
@@ -786,12 +787,12 @@ class TestTieringOffloadingManager:
|
||||
def test_lookup_batches_submit_load_per_request(self, manager_setup):
|
||||
"""lookup() defers submit_load until on_schedule_end(), one per request.
|
||||
|
||||
Blocks from different requests each get their own submit_load call, each
|
||||
Chunks from different requests each get their own submit_load call, each
|
||||
carrying the correct req_context.
|
||||
"""
|
||||
blocks = to_keys(range(4))
|
||||
for block in blocks:
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
chunks = to_keys(range(4))
|
||||
for chunk in chunks:
|
||||
self.secondary_tier1.chunks[chunk] = True
|
||||
|
||||
self.secondary_tier1.submit_load = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_load
|
||||
@@ -801,10 +802,10 @@ class TestTieringOffloadingManager:
|
||||
ctx_b = ReqContext(req_id="req_b")
|
||||
|
||||
# All lookups return HIT_PENDING: secondary hit triggers promotion
|
||||
assert self.manager.lookup(blocks[0], ctx_a) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(blocks[1], ctx_a) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(blocks[2], ctx_b) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(blocks[3], ctx_b) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(chunks[0], ctx_a) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(chunks[1], ctx_a) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(chunks[2], ctx_b) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(chunks[3], ctx_b) is LookupResult.HIT_PENDING
|
||||
|
||||
# submit_load must not fire during lookup - only at end of step
|
||||
self.secondary_tier1.submit_load.assert_not_called()
|
||||
@@ -816,20 +817,20 @@ class TestTieringOffloadingManager:
|
||||
calls = self.secondary_tier1.submit_load.call_args_list
|
||||
jm_a = calls[0].args[0]
|
||||
jm_b = calls[1].args[0]
|
||||
assert set(jm_a.keys) == {blocks[0], blocks[1]}
|
||||
assert set(jm_a.keys) == {chunks[0], chunks[1]}
|
||||
assert jm_a.req_context is ctx_a
|
||||
assert set(jm_b.keys) == {blocks[2], blocks[3]}
|
||||
assert set(jm_b.keys) == {chunks[2], chunks[3]}
|
||||
assert jm_b.req_context is ctx_b
|
||||
|
||||
def test_lookup_shared_block_no_duplicate_promotion(self, manager_setup):
|
||||
"""A block looked up by two requests in the same step is promoted once.
|
||||
def test_lookup_shared_chunk_no_duplicate_promotion(self, manager_setup):
|
||||
"""A chunk looked up by two requests in the same step is promoted once.
|
||||
|
||||
The first lookup initiates promotion (returns None via secondary hit).
|
||||
The second lookup sees ref_cnt=-1 on the primary slot and returns None
|
||||
via the primary in-flight path — without triggering a second promotion.
|
||||
"""
|
||||
shared_block = to_keys([0])[0]
|
||||
self.secondary_tier1.blocks[shared_block] = True
|
||||
shared_chunk = to_keys([0])[0]
|
||||
self.secondary_tier1.chunks[shared_chunk] = True
|
||||
|
||||
self.secondary_tier1.submit_load = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_load
|
||||
@@ -838,10 +839,10 @@ class TestTieringOffloadingManager:
|
||||
ctx_a = ReqContext(req_id="req_a")
|
||||
ctx_b = ReqContext(req_id="req_b")
|
||||
|
||||
result_a = self.manager.lookup(shared_block, ctx_a)
|
||||
result_b = self.manager.lookup(shared_block, ctx_b)
|
||||
result_a = self.manager.lookup(shared_chunk, ctx_a)
|
||||
result_b = self.manager.lookup(shared_chunk, ctx_b)
|
||||
|
||||
# Both lookups find the shared_block and trigger promotion
|
||||
# Both lookups find the shared_chunk and trigger promotion
|
||||
# returning HIT_PENDING.
|
||||
assert result_a is LookupResult.HIT_PENDING
|
||||
assert result_b is LookupResult.HIT_PENDING
|
||||
@@ -851,12 +852,12 @@ class TestTieringOffloadingManager:
|
||||
# Only one submit_load call despite two lookups
|
||||
self.secondary_tier1.submit_load.assert_called_once()
|
||||
job_metadata = self.secondary_tier1.submit_load.call_args.args[0]
|
||||
assert list(job_metadata.keys) == [shared_block]
|
||||
assert list(job_metadata.keys) == [shared_chunk]
|
||||
assert job_metadata.req_context is ctx_a
|
||||
|
||||
def test_complete_store_forwards_req_context_to_submit_store(self, manager_setup):
|
||||
"""complete_store cascades to secondary tiers with the correct req_context."""
|
||||
blocks = to_keys(range(2))
|
||||
chunks = to_keys(range(2))
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_store
|
||||
@@ -865,8 +866,8 @@ class TestTieringOffloadingManager:
|
||||
ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"})
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.complete_store(blocks, ctx, success=True)
|
||||
self.manager.prepare_store(chunks, ctx)
|
||||
self.manager.complete_store(chunks, ctx, success=True)
|
||||
|
||||
assert self.secondary_tier1.submit_store.call_count == 1
|
||||
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
||||
@@ -876,7 +877,7 @@ class TestTieringOffloadingManager:
|
||||
self, manager_setup
|
||||
):
|
||||
"""Manager hook is eager; secondary hooks wait for cascade submission."""
|
||||
blocks = to_keys(range(2))
|
||||
chunks = to_keys(range(2))
|
||||
ctx = ReqContext(req_id="req_delayed_secondary")
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
@@ -911,14 +912,14 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.prepare_store(chunks, ctx)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
assert calls == [("primary_finish", ctx.req_id)]
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||
|
||||
self.manager.complete_store(blocks, ctx, success=True)
|
||||
self.manager.complete_store(chunks, ctx, success=True)
|
||||
|
||||
assert calls == [
|
||||
("primary_finish", ctx.req_id),
|
||||
@@ -930,7 +931,7 @@ class TestTieringOffloadingManager:
|
||||
|
||||
def test_failed_store_finalizes_finished_request(self, manager_setup):
|
||||
"""Failed primary stores still unblock secondary finalization."""
|
||||
blocks = to_keys(range(2))
|
||||
chunks = to_keys(range(2))
|
||||
ctx = ReqContext(req_id="req_failed_store_finalize")
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(
|
||||
@@ -947,13 +948,13 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.prepare_store(chunks, ctx)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||
|
||||
self.manager.complete_store(blocks, ctx, success=False)
|
||||
self.manager.complete_store(chunks, ctx, success=False)
|
||||
|
||||
self.secondary_tier1.submit_store.assert_not_called()
|
||||
self.secondary_tier2.submit_store.assert_not_called()
|
||||
@@ -981,7 +982,7 @@ class TestTieringOffloadingManager:
|
||||
|
||||
def test_reset_cache_finalizes_delayed_secondary_request(self, manager_setup):
|
||||
"""reset_cache abandons pending primary stores and finalizes secondaries."""
|
||||
blocks = to_keys(range(2))
|
||||
chunks = to_keys(range(2))
|
||||
ctx = ReqContext(req_id="req_reset_finalize_secondary")
|
||||
|
||||
self.secondary_tier1.on_request_finished = MagicMock(
|
||||
@@ -992,7 +993,7 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.prepare_store(chunks, ctx)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
@@ -1008,8 +1009,8 @@ class TestTieringOffloadingManager:
|
||||
self, manager_setup
|
||||
):
|
||||
"""reset_cache drops active pending stores so resumed requests finalize."""
|
||||
initial_blocks = to_keys(range(2))
|
||||
resumed_blocks = to_keys(range(2, 4))
|
||||
initial_chunks = to_keys(range(2))
|
||||
resumed_chunks = to_keys(range(2, 4))
|
||||
ctx = ReqContext(req_id="req_reset_resume")
|
||||
|
||||
self.secondary_tier1.on_request_finished = MagicMock(
|
||||
@@ -1020,7 +1021,7 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(initial_blocks, ctx)
|
||||
self.manager.prepare_store(initial_chunks, ctx)
|
||||
assert self.manager._req_state[ctx.req_id].pending_primary_stores == 1
|
||||
|
||||
self.manager.reset_cache()
|
||||
@@ -1030,8 +1031,8 @@ class TestTieringOffloadingManager:
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||
|
||||
self.manager.prepare_store(resumed_blocks, ctx)
|
||||
self.manager.complete_store(resumed_blocks, ctx, success=True)
|
||||
self.manager.prepare_store(resumed_chunks, ctx)
|
||||
self.manager.complete_store(resumed_chunks, ctx, success=True)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
|
||||
@@ -1039,12 +1040,12 @@ class TestTieringOffloadingManager:
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
def test_on_new_request_lifecycle(self, manager_setup):
|
||||
"""Policy defaults to BLOCK_LEVEL, escalates when a tier requests it,
|
||||
"""Policy defaults to CHUNK_LEVEL, escalates when a tier requests it,
|
||||
and is cleaned up on on_request_finished."""
|
||||
# Default: all tiers return BLOCK_LEVEL
|
||||
# Default: all tiers return CHUNK_LEVEL
|
||||
ctx = ReqContext(req_id="req_policy_lifecycle")
|
||||
result = self.manager.on_new_request(ctx)
|
||||
assert result.policy == OffloadPolicy.BLOCK_LEVEL
|
||||
assert result.policy == OffloadPolicy.CHUNK_LEVEL
|
||||
assert self.manager._req_state[ctx.req_id].request_level_tiers is None
|
||||
self.manager.on_request_finished(ctx)
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
@@ -1064,26 +1065,26 @@ class TestTieringOffloadingManager:
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
@pytest.mark.parametrize("new_ids", [(), (3, 4)], ids=["fully_warm", "mixed"])
|
||||
def test_prepare_store_cascades_existing_blocks_to_request_level_tiers(
|
||||
def test_prepare_store_cascades_existing_chunks_to_request_level_tiers(
|
||||
self, manager_setup, new_ids
|
||||
):
|
||||
"""prepare_store cascades hit blocks to request-level tiers only.
|
||||
"""prepare_store cascades hit chunks to request-level tiers only.
|
||||
|
||||
The fully-warm case is the producer shape from issue #52808: nothing is
|
||||
new, so primary returns an empty keys_to_store and no GPU->primary
|
||||
transfer runs. A request-level tier must still be handed the blocks,
|
||||
transfer runs. A request-level tier must still be handed the chunks,
|
||||
otherwise a peer waiting on them has nothing to fetch.
|
||||
"""
|
||||
# Store some blocks to primary first
|
||||
existing_blocks = to_keys(range(3))
|
||||
# Store some chunks to primary first
|
||||
existing_chunks = to_keys(range(3))
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(existing_blocks, _CTX)
|
||||
result = self.manager.prepare_store(existing_chunks, _CTX)
|
||||
assert result is not None
|
||||
self.manager.complete_store(existing_blocks, _CTX, success=True)
|
||||
self.manager.complete_store(existing_chunks, _CTX, success=True)
|
||||
# Drain cascade completions
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# Make tier1 request-level, tier2 stays block-level
|
||||
# Make tier1 request-level, tier2 stays chunk-level
|
||||
self.secondary_tier1.on_new_request = lambda req_context: (
|
||||
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
|
||||
)
|
||||
@@ -1099,20 +1100,20 @@ class TestTieringOffloadingManager:
|
||||
wraps=self.secondary_tier2.submit_store
|
||||
)
|
||||
|
||||
# Call prepare_store with existing + new blocks
|
||||
new_blocks = to_keys(new_ids)
|
||||
all_blocks = existing_blocks + new_blocks
|
||||
result = self.manager.prepare_store(all_blocks, ctx)
|
||||
# Call prepare_store with existing + new chunks
|
||||
new_chunks = to_keys(new_ids)
|
||||
all_chunks = existing_chunks + new_chunks
|
||||
result = self.manager.prepare_store(all_chunks, ctx)
|
||||
assert result is not None
|
||||
assert set(result.keys_to_store) == set(new_blocks)
|
||||
assert set(result.keys_to_store) == set(new_chunks)
|
||||
|
||||
# Only tier1 (request-level) should get existing blocks cascaded now.
|
||||
# New blocks are cascaded to ALL tiers later via complete_store().
|
||||
# Only tier1 (request-level) should get existing chunks cascaded now.
|
||||
# New chunks are cascaded to ALL tiers later via complete_store().
|
||||
self.secondary_tier1.submit_store.assert_called_once()
|
||||
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
||||
assert set(job_metadata.keys) == set(existing_blocks)
|
||||
assert set(job_metadata.keys) == set(existing_chunks)
|
||||
|
||||
# tier2 (block-level) does not get existing blocks here.
|
||||
# tier2 (chunk-level) does not get existing chunks here.
|
||||
self.secondary_tier2.submit_store.assert_not_called()
|
||||
|
||||
def _make_request_level_request(self, req_id: str) -> ReqContext:
|
||||
@@ -1150,7 +1151,7 @@ class TestTieringOffloadingManager:
|
||||
self.manager.primary_tier.lookup = lambda key, req_context: (LookupResult.RETRY)
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
self.manager._cascade_existing_blocks_to_request_level_tiers(keys, ctx, {0})
|
||||
self.manager._cascade_existing_chunks_to_request_level_tiers(keys, ctx, {0})
|
||||
|
||||
def _start_in_flight_primary_write(self, keys: list[OffloadKey]) -> ReqContext:
|
||||
"""Leave a GPU->primary write for `keys` open, so they look present to
|
||||
@@ -1202,7 +1203,7 @@ class TestTieringOffloadingManager:
|
||||
def test_cascade_drops_deferred_keys_whose_primary_write_failed(
|
||||
self, manager_setup
|
||||
):
|
||||
"""A failed write frees the block, so the deferred key resolves to MISS
|
||||
"""A failed write frees the chunk, so the deferred key resolves to MISS
|
||||
and the request finalizes instead of parking forever."""
|
||||
keys = to_keys(range(3))
|
||||
writer_ctx = self._start_in_flight_primary_write(keys)
|
||||
@@ -1224,21 +1225,21 @@ class TestTieringOffloadingManager:
|
||||
"""reset_cache wipes every kind of orchestrator state and resets
|
||||
primary tier; pending submissions are dropped without being sent
|
||||
to the secondary tier. Active request state is retained."""
|
||||
# Cascade — populates primary blocks and leaves cascade jobs
|
||||
# Cascade — populates primary chunks and leaves cascade jobs
|
||||
# in _jobs (the synchronous example tier has already
|
||||
# queued completions); reset_cache's drain loop will pick them up.
|
||||
blocks = to_keys(range(3))
|
||||
chunks = to_keys(range(3))
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.prepare_store(chunks, _CTX)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
assert self.manager._jobs
|
||||
|
||||
# Pending promotion submission (deferred — no on_schedule_end after
|
||||
# the lookup that staged it).
|
||||
promo_block = to_keys([99])[0]
|
||||
self.secondary_tier1.blocks[promo_block] = True
|
||||
promo_chunk = to_keys([99])[0]
|
||||
self.secondary_tier1.chunks[promo_chunk] = True
|
||||
assert (
|
||||
self.manager.lookup(promo_block, ReqContext(req_id="pending"))
|
||||
self.manager.lookup(promo_chunk, ReqContext(req_id="pending"))
|
||||
is LookupResult.HIT_PENDING
|
||||
)
|
||||
assert self.manager._pending_load_submissions
|
||||
@@ -1268,10 +1269,10 @@ class TestTieringOffloadingManager:
|
||||
assert self.manager._processed_jobs_this_step is False
|
||||
|
||||
# Primary tier reset to a fresh state.
|
||||
assert self.primary_tier._num_allocated_blocks == 0
|
||||
assert self.primary_tier._num_allocated_chunks == 0
|
||||
assert self.primary_tier._free_list == []
|
||||
for block in blocks:
|
||||
assert self.primary_tier.lookup(block, _CTX) is LookupResult.MISS
|
||||
for chunk in chunks:
|
||||
assert self.primary_tier.lookup(chunk, _CTX) is LookupResult.MISS
|
||||
|
||||
# Pending submission was dropped, not submitted.
|
||||
self.secondary_tier1.submit_load.assert_not_called()
|
||||
@@ -1291,10 +1292,10 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
# Drive a cascade so a job lands in _jobs.
|
||||
blocks = to_keys(range(3))
|
||||
chunks = to_keys(range(3))
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self.manager.prepare_store(chunks, _CTX)
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
assert self.manager._jobs
|
||||
|
||||
self.manager.reset_cache()
|
||||
@@ -1315,20 +1316,20 @@ class TestTieringOffloadingManager:
|
||||
self, manager_setup, load_tier_filter
|
||||
):
|
||||
"""Filter excluding secondary medium returns MISS from secondaries
|
||||
even when they hold the block; primary is unaffected."""
|
||||
blocks = to_keys(range(2))
|
||||
# Put one block in primary, one only in secondary
|
||||
even when they hold the chunk; primary is unaffected."""
|
||||
chunks = to_keys(range(2))
|
||||
# Put one chunk in primary, one only in secondary
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks[:1], _CTX)
|
||||
self.manager.complete_store(blocks[:1], _CTX, success=True)
|
||||
self.secondary_tier1.blocks[blocks[1]] = True
|
||||
self.manager.prepare_store(chunks[:1], _CTX)
|
||||
self.manager.complete_store(chunks[:1], _CTX, success=True)
|
||||
self.secondary_tier1.chunks[chunks[1]] = True
|
||||
|
||||
# Secondaries have medium=CPU, so load_tier_filter skips them.
|
||||
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
|
||||
|
||||
ctx = ReqContext(req_id="r1", load_tier_filter=load_tier_filter)
|
||||
assert self.manager.lookup(blocks[0], ctx) is LookupResult.HIT
|
||||
assert self.manager.lookup(blocks[1], ctx) is LookupResult.MISS
|
||||
assert self.manager.lookup(chunks[0], ctx) is LookupResult.HIT
|
||||
assert self.manager.lookup(chunks[1], ctx) is LookupResult.MISS
|
||||
self.secondary_tier1.lookup.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -1344,13 +1345,13 @@ class TestTieringOffloadingManager:
|
||||
self, manager_setup, load_tier_filter
|
||||
):
|
||||
"""Filter that matches the secondary's medium allows lookup."""
|
||||
blocks = to_keys(range(1))
|
||||
self.secondary_tier1.blocks[blocks[0]] = True
|
||||
chunks = to_keys(range(1))
|
||||
self.secondary_tier1.chunks[chunks[0]] = True
|
||||
|
||||
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
|
||||
|
||||
ctx = ReqContext(req_id="r2", load_tier_filter=load_tier_filter)
|
||||
assert self.manager.lookup(blocks[0], ctx) is LookupResult.HIT_PENDING
|
||||
assert self.manager.lookup(chunks[0], ctx) is LookupResult.HIT_PENDING
|
||||
self.secondary_tier1.lookup.assert_called()
|
||||
|
||||
|
||||
@@ -1360,7 +1361,7 @@ class TestTieringOffloadingWithoutSecondaryTiers:
|
||||
def test_works_without_secondary_tiers(self):
|
||||
"""Test that manager works with empty secondary_tiers list."""
|
||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=5, mmap_region=_mock_mmap_region(5)
|
||||
num_chunks=5, mmap_region=_mock_mmap_region(5)
|
||||
)
|
||||
|
||||
# Create manager with no secondary tiers
|
||||
@@ -1368,15 +1369,15 @@ class TestTieringOffloadingWithoutSecondaryTiers:
|
||||
primary_tier=primary_tier, secondary_tiers=[]
|
||||
)
|
||||
|
||||
blocks = to_keys(range(3))
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
# Should work like a regular OffloadingManager
|
||||
manager.on_new_request(_CTX)
|
||||
result = manager.prepare_store(blocks, _CTX)
|
||||
result = manager.prepare_store(chunks, _CTX)
|
||||
assert result is not None
|
||||
manager.complete_store(blocks, _CTX, success=True)
|
||||
manager.complete_store(chunks, _CTX, success=True)
|
||||
|
||||
assert count_hits(manager, blocks) == 3
|
||||
assert count_hits(manager, chunks) == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -1131,7 +1131,7 @@ class OffloadingConnectorScheduler:
|
||||
# Skip prefix-hit chunks for block-level policy; for
|
||||
# request-level, next_stored_chunk_idx stays at 0 so all
|
||||
# chunks (including hits) are offloaded.
|
||||
if req_status.offloading_context.policy == OffloadPolicy.BLOCK_LEVEL:
|
||||
if req_status.offloading_context.policy == OffloadPolicy.CHUNK_LEVEL:
|
||||
group_state.next_stored_chunk_idx = num_chunks
|
||||
|
||||
src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context)
|
||||
|
||||
@@ -114,17 +114,17 @@ class LookupResult(Enum):
|
||||
|
||||
|
||||
class OffloadPolicy(Enum):
|
||||
# Offload only newly-computed blocks as they arrive; prefix-hit
|
||||
# blocks (already offloaded by a prior request) are skipped.
|
||||
BLOCK_LEVEL = "block_level"
|
||||
# Offload all blocks for the request, including prefix hits.
|
||||
# Offload only newly-computed chunks as they arrive; prefix-hit
|
||||
# chunks (already offloaded by a prior request) are skipped.
|
||||
CHUNK_LEVEL = "chunk_level"
|
||||
# Offload all chunks for the request, including prefix hits.
|
||||
# Used by tiers that need the complete KV context for a request.
|
||||
REQUEST_LEVEL = "request_level"
|
||||
|
||||
|
||||
@dataclass
|
||||
class RequestOffloadingContext:
|
||||
policy: OffloadPolicy = OffloadPolicy.BLOCK_LEVEL
|
||||
policy: OffloadPolicy = OffloadPolicy.CHUNK_LEVEL
|
||||
|
||||
|
||||
class ScheduleEndContext(NamedTuple):
|
||||
@@ -400,6 +400,9 @@ class OffloadingManager(ABC):
|
||||
class BlockIDsLoadStoreSpec(LoadStoreSpec, ABC):
|
||||
"""
|
||||
Spec for loading/storing KV blocks from given block numbers.
|
||||
|
||||
Subclass semantics differ: GPULoadStoreSpec.block_ids are GPU block
|
||||
indices; CPULoadStoreSpec.block_ids are CPU cache chunk indices.
|
||||
"""
|
||||
|
||||
def __init__(self, block_ids: list[int]):
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import numpy as np
|
||||
|
||||
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
|
||||
|
||||
|
||||
@@ -12,6 +14,13 @@ class CPUOffloadingMetrics:
|
||||
|
||||
|
||||
class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
|
||||
"""Spec for loading/storing KV chunks to/from CPU memory.
|
||||
|
||||
The inherited block_ids field holds chunk indices into the
|
||||
CPU cache (not GPU block IDs). The chunk_ids alias exposes
|
||||
the same array under the name used by the tiering layer.
|
||||
"""
|
||||
Spec for loading/storing a KV block to CPU memory.
|
||||
"""
|
||||
|
||||
@property
|
||||
def chunk_ids(self) -> np.ndarray:
|
||||
return self.block_ids
|
||||
|
||||
@@ -257,7 +257,7 @@ class SingleDirectionOffloadingHandler:
|
||||
gpu_tensors: list of GPU KV cache tensors.
|
||||
Each of shape (num_gpu_blocks, gpu_page_size_bytes) with dtype int8.
|
||||
cpu_tensors: list of CPU KV cache tensors.
|
||||
Each of shape (num_cpu_blocks, cpu_page_size_bytes) with dtype int8.
|
||||
Each of shape (num_cpu_chunks, cpu_page_size_bytes) with dtype int8.
|
||||
Order should match gpu_tensors.
|
||||
layer_refs_per_group: list of CanonicalKVCacheRef per group.
|
||||
gpu_to_cpu: if True, transfer from GPU to CPU; otherwise CPU to GPU.
|
||||
@@ -527,20 +527,20 @@ class SingleDirectionOffloadingHandler:
|
||||
# 1. GPU -> CPU
|
||||
# 2. CPU -> GPU
|
||||
#
|
||||
# transfers are also to CPU blocks, EXCEPT MAYBE for the first and last block.
|
||||
# i.e. the first and last CPU blocks in src_blocks can match against
|
||||
# transfers are also to CPU chunks, EXCEPT MAYBE for the first and last chunk.
|
||||
# i.e. the first and last CPU chunks in src_blocks can match against
|
||||
# a smaller (byte-wise) set of GPU blocks in dst_blocks.
|
||||
# In such cases, we may need to skip some gpu-sized sub-blocks,
|
||||
# and start reading/writing from the middle of the first CPU block.
|
||||
# and start reading/writing from the middle of the first CPU chunk.
|
||||
# If we have multiple KV cache groups (when using HMA with hybrid models),
|
||||
# we may have a partial first/last CPU block per each group.
|
||||
# we may have a partial first/last CPU chunk per each group.
|
||||
# The group_sizes parameter encodes the size of each group of blocks
|
||||
# in the GPU dst_blocks.
|
||||
# If group_sizes is None, we assume all blocks belong to a single group.
|
||||
# The logical_offset parameter maps each group of blocks to its logical
|
||||
# offset inside the request, counting in GPU blocks.
|
||||
# This allows us to find the correct starting position
|
||||
# in the matching first CPU block.
|
||||
# in the matching first CPU chunk.
|
||||
|
||||
# extract group_sizes from the GPU spec
|
||||
gpu_spec = src_spec if self.gpu_to_cpu else dst_spec
|
||||
@@ -751,7 +751,7 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
self,
|
||||
kv_caches: CanonicalKVCaches,
|
||||
blocks_per_chunk: int,
|
||||
num_cpu_blocks: int,
|
||||
num_cpu_chunks: int,
|
||||
mmap_region: SharedOffloadRegion | None = None,
|
||||
canonical_layout: bool = False,
|
||||
):
|
||||
@@ -790,16 +790,16 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
else:
|
||||
t0 = time.monotonic()
|
||||
cpu_tensor = torch.zeros(
|
||||
(num_cpu_blocks, cpu_page_size_bytes),
|
||||
(num_cpu_chunks, cpu_page_size_bytes),
|
||||
dtype=torch.int8,
|
||||
device="cpu",
|
||||
pin_memory=pin_memory,
|
||||
)
|
||||
logger.debug(
|
||||
"torch.zeros pinned tensor %d×%d (%.2f GB): %.3f s",
|
||||
num_cpu_blocks,
|
||||
num_cpu_chunks,
|
||||
cpu_page_size_bytes,
|
||||
num_cpu_blocks * cpu_page_size_bytes / 1e9,
|
||||
num_cpu_chunks * cpu_page_size_bytes / 1e9,
|
||||
time.monotonic() - t0,
|
||||
)
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ from vllm.v1.kv_offload.cpu.common import (
|
||||
CPULoadStoreSpec,
|
||||
CPUOffloadingMetrics,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
|
||||
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
|
||||
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
|
||||
|
||||
|
||||
@@ -34,14 +34,14 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
register their own or be loaded out-of-tree via cache_policy_module_path).
|
||||
|
||||
The manager owns all shared logic: ref-counting, event emission,
|
||||
block pool management, and the prepare_store/complete_store skeletons.
|
||||
Policy-specific block organization and eviction decisions are delegated
|
||||
chunk pool management, and the prepare_store/complete_store skeletons.
|
||||
Policy-specific chunk organization and eviction decisions are delegated
|
||||
to the CachePolicy implementation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_blocks: int,
|
||||
num_chunks: int,
|
||||
cache_policy: str = "lru",
|
||||
cache_policy_module_path: str | None = None,
|
||||
enable_events: bool = False,
|
||||
@@ -49,59 +49,59 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
max_tracker_size: int = 64_000,
|
||||
):
|
||||
self.medium: Medium = Medium.CPU
|
||||
self._num_blocks: int = num_blocks
|
||||
self._num_allocated_blocks: int = 0
|
||||
self._num_chunks: int = num_chunks
|
||||
self._num_allocated_chunks: int = 0
|
||||
self._free_list: list[int] = []
|
||||
self.events: list[OffloadingEvent] | None = [] if enable_events else None
|
||||
policy_cls = CachePolicyFactory.get_cache_policy_cls(
|
||||
cache_policy, cache_policy_module_path
|
||||
)
|
||||
self._policy: CachePolicy = policy_cls(cache_capacity=num_blocks)
|
||||
# Track the number of blocks in the cache that are evictable. i.e. ref_cnt 0.
|
||||
self._num_evictable_cache_blocks: int = 0
|
||||
# Track blocks with an in-flight store (ref_cnt -1, not yet completed).
|
||||
self._num_write_pending_blocks: int = 0
|
||||
self._policy: CachePolicy = policy_cls(cache_capacity=num_chunks)
|
||||
# Track the number of chunks in the cache that are evictable. i.e. ref_cnt 0.
|
||||
self._num_evictable_cache_chunks: int = 0
|
||||
# Track chunks with an in-flight store (ref_cnt -1, not yet completed).
|
||||
self._num_write_pending_chunks: int = 0
|
||||
|
||||
self.store_threshold: int = store_threshold
|
||||
self.max_tracker_size: int = max_tracker_size
|
||||
self.stores_skipped_in_current_batch: int = 0
|
||||
self.allocation_sizes_in_current_batch: list[int] = []
|
||||
|
||||
# Number of block references. It is ordered so can evict the LRU entry in O(1).
|
||||
# Number of chunk references. It is ordered so can evict the LRU entry in O(1).
|
||||
self.counts: OrderedDict[OffloadKey, int] | None = (
|
||||
OrderedDict() if store_threshold >= 2 else None
|
||||
)
|
||||
|
||||
# --- block pool ---
|
||||
# --- chunk pool ---
|
||||
|
||||
def _get_num_free_blocks(self) -> int:
|
||||
return len(self._free_list) + self._num_blocks - self._num_allocated_blocks
|
||||
def _get_num_free_chunks(self) -> int:
|
||||
return len(self._free_list) + self._num_chunks - self._num_allocated_chunks
|
||||
|
||||
def _allocate_blocks(self, keys: list[OffloadKey]) -> list[BlockStatus]:
|
||||
num_fresh = min(len(keys), self._num_blocks - self._num_allocated_blocks)
|
||||
def _allocate_chunks(self, keys: list[OffloadKey]) -> list[ChunkStatus]:
|
||||
num_fresh = min(len(keys), self._num_chunks - self._num_allocated_chunks)
|
||||
num_reused = len(keys) - num_fresh
|
||||
assert len(self._free_list) >= num_reused
|
||||
|
||||
# allocate fresh blocks
|
||||
blocks: list[BlockStatus] = []
|
||||
# allocate fresh chunks
|
||||
chunks: list[ChunkStatus] = []
|
||||
for _ in range(num_fresh):
|
||||
blocks.append(BlockStatus(self._num_allocated_blocks))
|
||||
self._num_allocated_blocks += 1
|
||||
chunks.append(ChunkStatus(self._num_allocated_chunks))
|
||||
self._num_allocated_chunks += 1
|
||||
|
||||
# allocate reused blocks
|
||||
# allocate reused chunks
|
||||
for _ in range(num_reused):
|
||||
blocks.append(BlockStatus(self._free_list.pop()))
|
||||
return blocks
|
||||
chunks.append(ChunkStatus(self._free_list.pop()))
|
||||
return chunks
|
||||
|
||||
def _free_block(self, block: BlockStatus) -> None:
|
||||
self._free_list.append(block.block_id)
|
||||
def _free_chunk(self, chunk: ChunkStatus) -> None:
|
||||
self._free_list.append(chunk.chunk_id)
|
||||
|
||||
def _get_load_store_spec(
|
||||
self,
|
||||
keys: Iterable[OffloadKey],
|
||||
blocks: Iterable[BlockStatus],
|
||||
chunks: Iterable[ChunkStatus],
|
||||
) -> CPULoadStoreSpec:
|
||||
return CPULoadStoreSpec([block.block_id for block in blocks])
|
||||
return CPULoadStoreSpec([chunk.chunk_id for chunk in chunks])
|
||||
|
||||
def _record_accesses(self, keys: Collection[OffloadKey]) -> None:
|
||||
"""Record an offer without evicting its tracked candidates."""
|
||||
@@ -132,10 +132,10 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
|
||||
@override
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
block = self._policy.get(key)
|
||||
if block is None:
|
||||
chunk = self._policy.get(key)
|
||||
if chunk is None:
|
||||
return LookupResult.MISS
|
||||
if not block.is_ready:
|
||||
if not chunk.is_ready:
|
||||
return LookupResult.HIT_PENDING
|
||||
return LookupResult.HIT
|
||||
|
||||
@@ -145,18 +145,18 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
keys: Collection[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
) -> LoadStoreSpec:
|
||||
blocks = []
|
||||
chunks = []
|
||||
for key in keys:
|
||||
block = self._policy.get(key)
|
||||
assert block is not None, f"Block {key!r} not found in cache"
|
||||
assert block.is_ready, f"Block {key!r} is not ready for reading"
|
||||
if block.ref_cnt == 0:
|
||||
chunk = self._policy.get(key)
|
||||
assert chunk is not None, f"Chunk {key!r} not found in cache"
|
||||
assert chunk.is_ready, f"Chunk {key!r} is not ready for reading"
|
||||
if chunk.ref_cnt == 0:
|
||||
self._policy.mark_non_evictable(key)
|
||||
self._num_evictable_cache_blocks -= 1 # ref_cnt 0 -> 1
|
||||
assert self._num_evictable_cache_blocks >= 0
|
||||
block.ref_cnt += 1
|
||||
blocks.append(block)
|
||||
return self._get_load_store_spec(keys, blocks)
|
||||
self._num_evictable_cache_chunks -= 1 # ref_cnt 0 -> 1
|
||||
assert self._num_evictable_cache_chunks >= 0
|
||||
chunk.ref_cnt += 1
|
||||
chunks.append(chunk)
|
||||
return self._get_load_store_spec(keys, chunks)
|
||||
|
||||
@override
|
||||
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext) -> None:
|
||||
@@ -167,12 +167,12 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||
) -> None:
|
||||
for key in keys:
|
||||
block = self._policy.get(key)
|
||||
assert block is not None, f"Block {key!r} not found"
|
||||
assert block.ref_cnt > 0, f"Block {key!r} ref_cnt is already 0"
|
||||
block.ref_cnt -= 1
|
||||
if block.ref_cnt == 0:
|
||||
self._num_evictable_cache_blocks += 1 # ref_cnt 1 -> 0
|
||||
chunk = self._policy.get(key)
|
||||
assert chunk is not None, f"Chunk {key!r} not found"
|
||||
assert chunk.ref_cnt > 0, f"Chunk {key!r} ref_cnt is already 0"
|
||||
chunk.ref_cnt -= 1
|
||||
if chunk.ref_cnt == 0:
|
||||
self._num_evictable_cache_chunks += 1 # ref_cnt 1 -> 0
|
||||
self._policy.mark_evictable(key)
|
||||
|
||||
@override
|
||||
@@ -186,7 +186,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
self._record_accesses(keys)
|
||||
keys = [k for k in keys if self.counts.get(k, 0) >= self.store_threshold]
|
||||
self.stores_skipped_in_current_batch += num_keys - len(keys)
|
||||
# filter out blocks that are already stored
|
||||
# filter out chunks that are already stored
|
||||
keys_to_store = [k for k in keys if self._policy.get(k) is None]
|
||||
|
||||
if not keys_to_store:
|
||||
@@ -197,29 +197,29 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
)
|
||||
|
||||
self.allocation_sizes_in_current_batch.append(len(keys_to_store))
|
||||
num_blocks_to_evict = len(keys_to_store) - self._get_num_free_blocks()
|
||||
num_chunks_to_evict = len(keys_to_store) - self._get_num_free_chunks()
|
||||
|
||||
to_evict: list[OffloadKey] = []
|
||||
if num_blocks_to_evict > 0:
|
||||
if num_blocks_to_evict > self._num_evictable_cache_blocks:
|
||||
if num_chunks_to_evict > 0:
|
||||
if num_chunks_to_evict > self._num_evictable_cache_chunks:
|
||||
# Eviction will fail.
|
||||
return None
|
||||
# There is a still a chance for eviction failure as some of the
|
||||
# idle blocks might be in the protected list.
|
||||
# idle chunks might be in the protected list.
|
||||
|
||||
# Blocks from the original input are excluded from eviction candidates:
|
||||
# a block that was already stored must remain in the cache after this call.
|
||||
# Chunks from the original input are excluded from eviction candidates:
|
||||
# a chunk that was already stored must remain in the cache after this call.
|
||||
protected = set(keys)
|
||||
evicted = self._policy.evict(num_blocks_to_evict, protected)
|
||||
evicted = self._policy.evict(num_chunks_to_evict, protected)
|
||||
if evicted is None:
|
||||
return None
|
||||
|
||||
# cache-policy removes only idle blocks.
|
||||
self._num_evictable_cache_blocks -= len(evicted)
|
||||
assert self._num_evictable_cache_blocks >= 0
|
||||
# cache-policy removes only idle chunks.
|
||||
self._num_evictable_cache_chunks -= len(evicted)
|
||||
assert self._num_evictable_cache_chunks >= 0
|
||||
|
||||
for key, block in evicted:
|
||||
self._free_block(block)
|
||||
for key, chunk in evicted:
|
||||
self._free_chunk(chunk)
|
||||
to_evict.append(key)
|
||||
|
||||
if to_evict and self.events is not None:
|
||||
@@ -231,17 +231,17 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
)
|
||||
)
|
||||
|
||||
blocks = self._allocate_blocks(keys_to_store)
|
||||
assert len(blocks) == len(keys_to_store), (
|
||||
"Block pool did not allocate the expected number of blocks"
|
||||
chunks = self._allocate_chunks(keys_to_store)
|
||||
assert len(chunks) == len(keys_to_store), (
|
||||
"Chunk pool did not allocate the expected number of chunks"
|
||||
)
|
||||
|
||||
for key, block in zip(keys_to_store, blocks):
|
||||
self._policy.insert(key, block)
|
||||
self._num_write_pending_blocks += len(keys_to_store)
|
||||
for key, chunk in zip(keys_to_store, chunks):
|
||||
self._policy.insert(key, chunk)
|
||||
self._num_write_pending_chunks += len(keys_to_store)
|
||||
|
||||
# build store specs for allocated blocks
|
||||
store_spec = self._get_load_store_spec(keys_to_store, blocks)
|
||||
# build store specs for allocated chunks
|
||||
store_spec = self._get_load_store_spec(keys_to_store, chunks)
|
||||
|
||||
return PrepareStoreOutput(
|
||||
keys_to_store=keys_to_store,
|
||||
@@ -260,20 +260,20 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
|
||||
if success:
|
||||
for key in keys:
|
||||
block = self._policy.get(key)
|
||||
if block is not None and not block.is_ready:
|
||||
block.ref_cnt = 0
|
||||
self._num_write_pending_blocks -= 1
|
||||
self._num_evictable_cache_blocks += 1
|
||||
chunk = self._policy.get(key)
|
||||
if chunk is not None and not chunk.is_ready:
|
||||
chunk.ref_cnt = 0
|
||||
self._num_write_pending_chunks -= 1
|
||||
self._num_evictable_cache_chunks += 1
|
||||
self._policy.mark_evictable(key)
|
||||
stored_keys.append(key)
|
||||
else:
|
||||
for key in keys:
|
||||
block = self._policy.get(key)
|
||||
if block is not None and not block.is_ready:
|
||||
self._num_write_pending_blocks -= 1
|
||||
chunk = self._policy.get(key)
|
||||
if chunk is not None and not chunk.is_ready:
|
||||
self._num_write_pending_chunks -= 1
|
||||
self._policy.remove(key)
|
||||
self._free_block(block)
|
||||
self._free_chunk(chunk)
|
||||
|
||||
if stored_keys and self.events is not None:
|
||||
self.events.append(
|
||||
@@ -286,17 +286,17 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
|
||||
@override
|
||||
def reset_cache(self) -> None:
|
||||
# Clear ALL blocks unconditionally. The scheduler's _stale_job_threshold
|
||||
# Clear ALL chunks unconditionally. The scheduler's _stale_job_threshold
|
||||
# guarantees that complete_load / complete_store are never called for
|
||||
# pre-reset jobs, so no lazy cleanup is needed. The scheduler also
|
||||
# flushes in-flight load job IDs to the workers before any new stores
|
||||
# can begin, preventing a cross-direction data race on reused offload block IDs.
|
||||
# can begin, preventing a cross-direction data race on reused offload chunk IDs.
|
||||
self._policy.clear()
|
||||
self._num_evictable_cache_blocks = 0
|
||||
self._num_write_pending_blocks = 0
|
||||
self._num_evictable_cache_chunks = 0
|
||||
self._num_write_pending_chunks = 0
|
||||
|
||||
self._free_list.clear()
|
||||
self._num_allocated_blocks = 0
|
||||
self._num_allocated_chunks = 0
|
||||
|
||||
@override
|
||||
def take_events(self) -> Iterable[OffloadingEvent]:
|
||||
@@ -309,11 +309,11 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
|
||||
# Compute cache usage.
|
||||
num_used = (
|
||||
self._num_allocated_blocks
|
||||
self._num_allocated_chunks
|
||||
- len(self._free_list)
|
||||
- self._num_evictable_cache_blocks
|
||||
- self._num_evictable_cache_chunks
|
||||
)
|
||||
usage = num_used / self._num_blocks if self._num_blocks > 0 else 0.0
|
||||
usage = num_used / self._num_chunks if self._num_chunks > 0 else 0.0
|
||||
stats.set_gauge(CPUOffloadingMetrics.CPU_CACHE_USAGE_PERC, usage)
|
||||
|
||||
for allocation_size in self.allocation_sizes_in_current_batch:
|
||||
@@ -323,8 +323,8 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
self.allocation_sizes_in_current_batch.clear()
|
||||
|
||||
write_usage = (
|
||||
self._num_write_pending_blocks / self._num_blocks
|
||||
if self._num_blocks > 0
|
||||
self._num_write_pending_chunks / self._num_chunks
|
||||
if self._num_chunks > 0
|
||||
else 0.0
|
||||
)
|
||||
read_usage = max(usage - write_usage, 0.0)
|
||||
|
||||
@@ -6,7 +6,7 @@ from collections.abc import Iterable, Iterator
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
|
||||
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
|
||||
|
||||
|
||||
class ARCCachePolicy(CachePolicy):
|
||||
@@ -14,15 +14,15 @@ class ARCCachePolicy(CachePolicy):
|
||||
ARC (Adaptive Replacement Cache) cache policy.
|
||||
|
||||
Data Structures:
|
||||
T1: Recent cache containing blocks accessed once.
|
||||
T2: Frequent cache containing blocks accessed multiple times.
|
||||
B1/B2: Ghost lists tracking recently evicted blocks from T1/T2.
|
||||
T1: Recent cache containing chunks accessed once.
|
||||
T2: Frequent cache containing chunks accessed multiple times.
|
||||
B1/B2: Ghost lists tracking recently evicted chunks from T1/T2.
|
||||
target_t1_size: Adaptive target size for the T1 partition.
|
||||
|
||||
Algorithm Flow:
|
||||
1. Cache lookup (lookup):
|
||||
Searches T1 and T2 for block hashes and counts consecutive hits
|
||||
until a miss or non-ready block is encountered.
|
||||
Searches T1 and T2 for chunk hashes and counts consecutive hits
|
||||
until a miss or non-ready chunk is encountered.
|
||||
|
||||
2. Cache touch (touch) - Adaptive Learning:
|
||||
For each key (in reverse order):
|
||||
@@ -31,15 +31,15 @@ class ARCCachePolicy(CachePolicy):
|
||||
- If in B1 ghost list: Increase target_t1_size.
|
||||
- If in B2 ghost list: Decrease target_t1_size.
|
||||
|
||||
3. Block eviction (evict) - Adaptive Replacement:
|
||||
3. Chunk eviction (evict) - Adaptive Replacement:
|
||||
Determines eviction source based on adaptive target:
|
||||
- If T1 size >= target_t1_size: Evict from T1, add to B1.
|
||||
- Otherwise: Evict from T2, add to B2.
|
||||
Finally, bound each ghost list size.
|
||||
|
||||
4. Block insertion (insert):
|
||||
New blocks are always inserted into T1 and removed from B1/B2 if
|
||||
present. Blocks may later be promoted to T2 during touch operations.
|
||||
4. Chunk insertion (insert):
|
||||
New chunks are always inserted into T1 and removed from B1/B2 if
|
||||
present. Chunks may later be promoted to T2 during touch operations.
|
||||
|
||||
Adaptive Behavior:
|
||||
The algorithm self-tunes the recency vs. frequency trade-off:
|
||||
@@ -50,19 +50,19 @@ class ARCCachePolicy(CachePolicy):
|
||||
def __init__(self, cache_capacity: int):
|
||||
super().__init__(cache_capacity)
|
||||
self.target_t1_size: float = 0.0
|
||||
self.t1: OrderedDict[OffloadKey, BlockStatus] = OrderedDict()
|
||||
self.t2: OrderedDict[OffloadKey, BlockStatus] = OrderedDict()
|
||||
self.t1: OrderedDict[OffloadKey, ChunkStatus] = OrderedDict()
|
||||
self.t2: OrderedDict[OffloadKey, ChunkStatus] = OrderedDict()
|
||||
# key -> None (only care about presence)
|
||||
self.b1: OrderedDict[OffloadKey, None] = OrderedDict()
|
||||
self.b2: OrderedDict[OffloadKey, None] = OrderedDict()
|
||||
|
||||
@override
|
||||
def get(self, key: OffloadKey) -> BlockStatus | None:
|
||||
def get(self, key: OffloadKey) -> ChunkStatus | None:
|
||||
return self.t1.get(key) or self.t2.get(key)
|
||||
|
||||
@override
|
||||
def insert(self, key: OffloadKey, block: BlockStatus) -> None:
|
||||
self.t1[key] = block
|
||||
def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
|
||||
self.t1[key] = chunk
|
||||
self.b1.pop(key, None)
|
||||
self.b2.pop(key, None)
|
||||
|
||||
@@ -75,13 +75,13 @@ class ARCCachePolicy(CachePolicy):
|
||||
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
||||
for key in reversed(list(keys)):
|
||||
if key in self.t1:
|
||||
block = self.t1.pop(key)
|
||||
if not block.is_ready:
|
||||
# block was just prepared to be stored, not really touched
|
||||
chunk = self.t1.pop(key)
|
||||
if not chunk.is_ready:
|
||||
# chunk was just prepared to be stored, not really touched
|
||||
# twice — keep it in T1 and mark as most recently used
|
||||
self.t1[key] = block
|
||||
self.t1[key] = chunk
|
||||
else:
|
||||
self.t2[key] = block
|
||||
self.t2[key] = chunk
|
||||
|
||||
elif key in self.t2:
|
||||
self.t2.move_to_end(key)
|
||||
@@ -111,31 +111,31 @@ class ARCCachePolicy(CachePolicy):
|
||||
@override
|
||||
def evict(
|
||||
self, n: int, protected: set[OffloadKey]
|
||||
) -> list[tuple[OffloadKey, BlockStatus]] | None:
|
||||
) -> list[tuple[OffloadKey, ChunkStatus]] | None:
|
||||
if n == 0:
|
||||
return []
|
||||
|
||||
# Collect candidates atomically: simulate T1 size changes as we select,
|
||||
# but do not modify actual data structures until all n are found.
|
||||
candidates: list[
|
||||
tuple[OffloadKey, BlockStatus, bool]
|
||||
] = [] # (key, block, from_t1)
|
||||
tuple[OffloadKey, ChunkStatus, bool]
|
||||
] = [] # (key, chunk, from_t1)
|
||||
virtual_t1_size = len(self.t1)
|
||||
# Keep the scans monotonic: restarting from the LRU end after every
|
||||
# selection makes a batch eviction quadratic in the number of blocks.
|
||||
# selection makes a batch eviction quadratic in the number of chunks.
|
||||
t1_iter = iter(self.t1.items())
|
||||
t2_iter = iter(self.t2.items())
|
||||
|
||||
def next_candidate(
|
||||
entries: Iterator[tuple[OffloadKey, BlockStatus]],
|
||||
) -> tuple[OffloadKey, BlockStatus] | None:
|
||||
for key, block in entries:
|
||||
if block.ref_cnt == 0 and key not in protected:
|
||||
return key, block
|
||||
entries: Iterator[tuple[OffloadKey, ChunkStatus]],
|
||||
) -> tuple[OffloadKey, ChunkStatus] | None:
|
||||
for key, chunk in entries:
|
||||
if chunk.ref_cnt == 0 and key not in protected:
|
||||
return key, chunk
|
||||
return None
|
||||
|
||||
for _ in range(n):
|
||||
candidate: tuple[OffloadKey, BlockStatus, bool] | None = None
|
||||
candidate: tuple[OffloadKey, ChunkStatus, bool] | None = None
|
||||
|
||||
if virtual_t1_size >= int(self.target_t1_size):
|
||||
entry = next_candidate(t1_iter)
|
||||
@@ -158,15 +158,15 @@ class ARCCachePolicy(CachePolicy):
|
||||
candidates.append(candidate)
|
||||
|
||||
# Apply all evictions now that we know n candidates exist.
|
||||
result: list[tuple[OffloadKey, BlockStatus]] = []
|
||||
for key, block, from_t1 in candidates:
|
||||
result: list[tuple[OffloadKey, ChunkStatus]] = []
|
||||
for key, chunk, from_t1 in candidates:
|
||||
if from_t1:
|
||||
del self.t1[key]
|
||||
self.b1[key] = None
|
||||
else:
|
||||
del self.t2[key]
|
||||
self.b2[key] = None
|
||||
result.append((key, block))
|
||||
result.append((key, chunk))
|
||||
|
||||
# Trim ghost lists to cache_capacity.
|
||||
for ghost in (self.b1, self.b2):
|
||||
|
||||
@@ -7,36 +7,36 @@ from collections.abc import Iterable
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||
|
||||
|
||||
class BlockStatus(ctypes.Structure):
|
||||
class ChunkStatus(ctypes.Structure):
|
||||
"""
|
||||
Offloading status for a single block of KV data.
|
||||
Offloading status for a single chunk of KV data.
|
||||
Holds the following information:
|
||||
|
||||
ref_cnt - the current number of transfers using this block as a source.
|
||||
A value of -1 indicates the block is not yet ready to be read.
|
||||
block_id - index of the physical CPU buffer slot.
|
||||
ref_cnt - the current number of transfers using this chunk as a source.
|
||||
A value of -1 indicates the chunk is not yet ready to be read.
|
||||
chunk_id - index of the physical CPU buffer slot.
|
||||
"""
|
||||
|
||||
_fields_ = [("ref_cnt", ctypes.c_int32), ("block_id", ctypes.c_int64)]
|
||||
_fields_ = [("ref_cnt", ctypes.c_int32), ("chunk_id", ctypes.c_int64)]
|
||||
|
||||
def __init__(self, block_id: int):
|
||||
def __init__(self, chunk_id: int):
|
||||
super().__init__()
|
||||
# initialize block as "not ready" (ref_cnt = -1)
|
||||
# initialize chunk as "not ready" (ref_cnt = -1)
|
||||
self.ref_cnt = -1
|
||||
self.block_id = block_id
|
||||
self.chunk_id = chunk_id
|
||||
|
||||
@property
|
||||
def is_ready(self) -> bool:
|
||||
"""
|
||||
Returns whether the block is ready to be read.
|
||||
Returns whether the chunk is ready to be read.
|
||||
"""
|
||||
return self.ref_cnt >= 0
|
||||
|
||||
|
||||
class CachePolicy(ABC):
|
||||
"""
|
||||
Encapsulates both block organization (data structures) and replacement
|
||||
decisions (which block to evict). LRU and ARC differ in both dimensions —
|
||||
Encapsulates both chunk organization (data structures) and replacement
|
||||
decisions (which chunk to evict). LRU and ARC differ in both dimensions —
|
||||
ARC's ghost lists and target_t1_size live at the intersection of storage
|
||||
and eviction, so they cannot be separated cleanly.
|
||||
"""
|
||||
@@ -45,35 +45,35 @@ class CachePolicy(ABC):
|
||||
self.cache_capacity = cache_capacity
|
||||
|
||||
@abstractmethod
|
||||
def get(self, key: OffloadKey) -> BlockStatus | None:
|
||||
"""Find block in data structures. Returns None if not present."""
|
||||
def get(self, key: OffloadKey) -> ChunkStatus | None:
|
||||
"""Find chunk in data structures. Returns None if not present."""
|
||||
|
||||
@abstractmethod
|
||||
def insert(self, key: OffloadKey, block: BlockStatus) -> None:
|
||||
"""Add a newly allocated block. For ARC: also removes from ghost lists."""
|
||||
def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
|
||||
"""Add a newly allocated chunk. For ARC: also removes from ghost lists."""
|
||||
|
||||
@abstractmethod
|
||||
def remove(self, key: OffloadKey) -> None:
|
||||
"""Remove a block (used to clean up after a failed store)."""
|
||||
"""Remove a chunk (used to clean up after a failed store)."""
|
||||
|
||||
@abstractmethod
|
||||
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
||||
"""
|
||||
Mark blocks as recently used.
|
||||
Mark chunks as recently used.
|
||||
|
||||
Args:
|
||||
keys: Blocks to mark as recently used.
|
||||
req_context: Per-request context for the request touching these blocks.
|
||||
keys: Chunks to mark as recently used.
|
||||
req_context: Per-request context for the request touching these chunks.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def evict(
|
||||
self, n: int, protected: set[OffloadKey]
|
||||
) -> list[tuple[OffloadKey, BlockStatus]] | None:
|
||||
) -> list[tuple[OffloadKey, ChunkStatus]] | None:
|
||||
"""
|
||||
Evict exactly n blocks, skipping any in protected.
|
||||
Evict exactly n chunks, skipping any in protected.
|
||||
|
||||
Returns a list of (key, block) for the evicted blocks,
|
||||
Returns a list of (key, chunk) for the evicted chunks,
|
||||
or None if n evictions cannot be satisfied. The operation is atomic:
|
||||
if None is returned, no state changes are made.
|
||||
|
||||
@@ -84,15 +84,15 @@ class CachePolicy(ABC):
|
||||
@abstractmethod
|
||||
def clear(self) -> None:
|
||||
"""
|
||||
Remove ALL blocks regardless of ref_cnt.
|
||||
Remove ALL chunks regardless of ref_cnt.
|
||||
|
||||
Ghost lists and adaptive state are also reset.
|
||||
"""
|
||||
|
||||
def mark_evictable(self, key: OffloadKey) -> None:
|
||||
"""Called when a block's ref_cnt transitions to 0."""
|
||||
"""Called when a chunk's ref_cnt transitions to 0."""
|
||||
return
|
||||
|
||||
def mark_non_evictable(self, key: OffloadKey) -> None:
|
||||
"""Called when a block's ref_cnt transitions from 0."""
|
||||
"""Called when a chunk's ref_cnt transitions from 0."""
|
||||
return
|
||||
|
||||
@@ -6,7 +6,7 @@ from collections.abc import Iterable
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
|
||||
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
|
||||
|
||||
|
||||
class LRUCachePolicy(CachePolicy):
|
||||
@@ -20,71 +20,71 @@ class LRUCachePolicy(CachePolicy):
|
||||
|
||||
def __init__(self, cache_capacity: int):
|
||||
super().__init__(cache_capacity)
|
||||
# Blocks with ref_cnt 0 (not participating in any loads/stores) ordered in LRU
|
||||
self.evictable_blocks: OrderedDict[OffloadKey, None] = OrderedDict()
|
||||
self.blocks: dict[OffloadKey, BlockStatus] = {}
|
||||
# Chunks with ref_cnt 0 (not participating in any loads/stores) ordered in LRU
|
||||
self.evictable_chunks: OrderedDict[OffloadKey, None] = OrderedDict()
|
||||
self.chunks: dict[OffloadKey, ChunkStatus] = {}
|
||||
|
||||
@override
|
||||
def get(self, key: OffloadKey) -> BlockStatus | None:
|
||||
return self.blocks.get(key)
|
||||
def get(self, key: OffloadKey) -> ChunkStatus | None:
|
||||
return self.chunks.get(key)
|
||||
|
||||
@override
|
||||
def insert(self, key: OffloadKey, block: BlockStatus) -> None:
|
||||
self.blocks[key] = block
|
||||
if block.ref_cnt == 0:
|
||||
self.evictable_blocks[key] = None
|
||||
def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
|
||||
self.chunks[key] = chunk
|
||||
if chunk.ref_cnt == 0:
|
||||
self.evictable_chunks[key] = None
|
||||
|
||||
@override
|
||||
def remove(self, key: OffloadKey) -> None:
|
||||
del self.blocks[key]
|
||||
self.evictable_blocks.pop(key, None)
|
||||
del self.chunks[key]
|
||||
self.evictable_chunks.pop(key, None)
|
||||
|
||||
@override
|
||||
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
||||
for key in reversed(list(keys)):
|
||||
if key in self.evictable_blocks:
|
||||
self.evictable_blocks.move_to_end(key)
|
||||
# active blocks are untouched as they are non-evictable now. They
|
||||
# will eventually reach the end of evictable_blocks when they finish.
|
||||
if key in self.evictable_chunks:
|
||||
self.evictable_chunks.move_to_end(key)
|
||||
# active chunks are untouched as they are non-evictable now. They
|
||||
# will eventually reach the end of evictable_chunks when they finish.
|
||||
|
||||
@override
|
||||
def clear(self) -> None:
|
||||
self.evictable_blocks.clear()
|
||||
self.blocks.clear()
|
||||
self.evictable_chunks.clear()
|
||||
self.chunks.clear()
|
||||
|
||||
@override
|
||||
def evict(
|
||||
self, n: int, protected: set[OffloadKey]
|
||||
) -> list[tuple[OffloadKey, BlockStatus]] | None:
|
||||
) -> list[tuple[OffloadKey, ChunkStatus]] | None:
|
||||
if n == 0:
|
||||
return []
|
||||
|
||||
candidates: list[tuple[OffloadKey, BlockStatus]] = []
|
||||
for key, _ in self.evictable_blocks.items():
|
||||
candidates: list[tuple[OffloadKey, ChunkStatus]] = []
|
||||
for key, _ in self.evictable_chunks.items():
|
||||
if key in protected:
|
||||
continue
|
||||
|
||||
block = self.blocks[key]
|
||||
assert block.ref_cnt == 0
|
||||
candidates.append((key, block))
|
||||
chunk = self.chunks[key]
|
||||
assert chunk.ref_cnt == 0
|
||||
candidates.append((key, chunk))
|
||||
if len(candidates) == n:
|
||||
break
|
||||
|
||||
if len(candidates) < n:
|
||||
return None
|
||||
for key, _ in candidates:
|
||||
del self.evictable_blocks[key]
|
||||
del self.blocks[key]
|
||||
del self.evictable_chunks[key]
|
||||
del self.chunks[key]
|
||||
return candidates
|
||||
|
||||
@override
|
||||
def mark_evictable(self, key: OffloadKey) -> None:
|
||||
# blocks can become evictable when,
|
||||
# chunks can become evictable when,
|
||||
# store completes - i.e. ref_cnt -1 -> 0 # not in evictable list
|
||||
# all loads complete - i.e ref_cnt 1 -> 0 # not in evictable list
|
||||
self.evictable_blocks[key] = None
|
||||
self.evictable_chunks[key] = None
|
||||
|
||||
@override
|
||||
def mark_non_evictable(self, key: OffloadKey) -> None:
|
||||
# key must have been in the evictable list.
|
||||
del self.evictable_blocks[key]
|
||||
del self.evictable_chunks[key]
|
||||
|
||||
@@ -82,24 +82,24 @@ class SharedOffloadRegion:
|
||||
def __init__(
|
||||
self,
|
||||
engine_id: str,
|
||||
num_blocks: int,
|
||||
num_chunks: int,
|
||||
rank: int | None,
|
||||
kv_bytes_per_block: int,
|
||||
kv_bytes_per_chunk: int,
|
||||
cpu_page_size: int,
|
||||
barrier: Callable[[], None] | None = None,
|
||||
) -> None:
|
||||
self.page_size = mmap.PAGESIZE
|
||||
assert kv_bytes_per_block % self.page_size == 0
|
||||
assert kv_bytes_per_chunk % self.page_size == 0
|
||||
|
||||
self.num_blocks = num_blocks
|
||||
self._row_stride = kv_bytes_per_block
|
||||
self.total_size_bytes = self.num_blocks * self._row_stride
|
||||
self.num_chunks = num_chunks
|
||||
self._row_stride = kv_bytes_per_chunk
|
||||
self.total_size_bytes = self.num_chunks * self._row_stride
|
||||
|
||||
self.mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap"
|
||||
self._creator = False # set True only if this worker creates the file
|
||||
self.rank = rank
|
||||
if rank is not None:
|
||||
# byte offset to this worker's first slot within each block row
|
||||
# byte offset to this worker's first slot within each chunk row
|
||||
self._worker_offset = rank * cpu_page_size
|
||||
# exclusive upper bound for this worker's area within each row
|
||||
self._worker_area_end = (rank + 1) * cpu_page_size
|
||||
@@ -182,19 +182,19 @@ class SharedOffloadRegion:
|
||||
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
|
||||
|
||||
if rank is not None:
|
||||
# Populate only this worker's pages (one slot per block row).
|
||||
# Populate only this worker's pages (one slot per chunk row).
|
||||
worker_offset = rank * cpu_page_size
|
||||
_t0 = time.perf_counter()
|
||||
page_size = self.page_size
|
||||
for block in range(num_blocks):
|
||||
raw_offset = block * self._row_stride + worker_offset
|
||||
for chunk in range(num_chunks):
|
||||
raw_offset = chunk * self._row_stride + worker_offset
|
||||
aligned_offset = (raw_offset // page_size) * page_size
|
||||
end = raw_offset + cpu_page_size
|
||||
aligned_length = end - aligned_offset
|
||||
populate_write_fn(self.mmap_obj, aligned_offset, aligned_length)
|
||||
logger.debug(
|
||||
"MADV_POPULATE_WRITE loop: %d blocks in %.3f s",
|
||||
num_blocks,
|
||||
"MADV_POPULATE_WRITE loop: %d chunks in %.3f s",
|
||||
num_chunks,
|
||||
time.perf_counter() - _t0,
|
||||
)
|
||||
else:
|
||||
@@ -215,22 +215,22 @@ class SharedOffloadRegion:
|
||||
|
||||
Must be called once per canonical tensor. The full mmap layout is:
|
||||
|
||||
worker0_block0 | worker1_block0 | ... | worker{M-1}_block0
|
||||
worker0_block1 | worker1_block1 | ... | worker{M-1}_block1
|
||||
worker0_chunk0 | worker1_chunk0 | ... | worker{M-1}_chunk0
|
||||
worker0_chunk1 | worker1_chunk1 | ... | worker{M-1}_chunk1
|
||||
...
|
||||
|
||||
Each worker_block cell is cpu_page_size bytes and holds all canonical
|
||||
tensors for that worker and block concatenated:
|
||||
Each worker_chunk cell is cpu_page_size bytes and holds all canonical
|
||||
tensors for that worker and chunk concatenated:
|
||||
[ tensor0_data | tensor1_data | ... | tensor{L-1}_data ]
|
||||
|
||||
Consecutive rows are separated by row_stride = cpu_page_size * M.
|
||||
|
||||
Returns an int8 tensor of shape (num_blocks, tensor_page_size) with stride
|
||||
Returns an int8 tensor of shape (num_chunks, tensor_page_size) with stride
|
||||
(row_stride, 1). Using int8 keeps stride == bytes, so swap_blocks
|
||||
address arithmetic works without any dtype conversion.
|
||||
|
||||
Args:
|
||||
tensor_page_size: Bytes per block for this tensor.
|
||||
tensor_page_size: Bytes per chunk for this tensor.
|
||||
"""
|
||||
assert self.rank is not None
|
||||
new_offset = self._worker_offset + tensor_page_size
|
||||
@@ -241,7 +241,7 @@ class SharedOffloadRegion:
|
||||
)
|
||||
worker_layer_view = torch.as_strided(
|
||||
self._base,
|
||||
size=(self.num_blocks, tensor_page_size),
|
||||
size=(self.num_chunks, tensor_page_size),
|
||||
stride=(self._row_stride, 1),
|
||||
storage_offset=self._worker_offset,
|
||||
)
|
||||
@@ -266,8 +266,8 @@ class SharedOffloadRegion:
|
||||
_canonical_offset=0, then advances by each tensor's size
|
||||
|
||||
Each canonical_t{i} cell is that tensor's canonical page for the
|
||||
block. Canonical areas are carved consecutively from the start of
|
||||
each block row; consecutive rows are separated by row_stride. Every
|
||||
chunk. Canonical areas are carved consecutively from the start of
|
||||
each chunk row; consecutive rows are separated by row_stride. Every
|
||||
worker gets the identical byte ranges and writes only its disjoint
|
||||
bytes within them, as described by its canonical mappings — unlike
|
||||
create_next_worker_view, which gives each worker a private
|
||||
@@ -279,13 +279,13 @@ class SharedOffloadRegion:
|
||||
where one canonical copy replaces world_size worker copies.
|
||||
|
||||
Args:
|
||||
tensor_page_size: Canonical bytes per block for this tensor.
|
||||
tensor_page_size: Canonical bytes per chunk for this tensor.
|
||||
"""
|
||||
new_offset = self._canonical_offset + tensor_page_size
|
||||
assert new_offset <= self._row_stride
|
||||
view = torch.as_strided(
|
||||
self._base,
|
||||
size=(self.num_blocks, tensor_page_size),
|
||||
size=(self.num_chunks, tensor_page_size),
|
||||
stride=(self._row_stride, 1),
|
||||
storage_offset=self._canonical_offset,
|
||||
)
|
||||
@@ -296,10 +296,10 @@ class SharedOffloadRegion:
|
||||
def create_kv_memoryview(self) -> memoryview:
|
||||
"""Return a zero-copy memoryview over the entire KV buffer.
|
||||
|
||||
Shape: (num_blocks, row_stride_bytes). Secondary tiers address
|
||||
block *b* as ``view[b]``.
|
||||
Shape: (num_chunks, row_stride_bytes). Secondary tiers address
|
||||
chunk *b* as ``view[b]``.
|
||||
"""
|
||||
kv_tensor = self._base.view(self.num_blocks, self._row_stride)
|
||||
kv_tensor = self._base.view(self.num_chunks, self._row_stride)
|
||||
np_arr = kv_tensor.numpy()
|
||||
assert np_arr.ctypes.data == self._base.data_ptr(), (
|
||||
"view()/numpy() created a copy instead of sharing the mmap buffer; "
|
||||
|
||||
@@ -73,7 +73,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
),
|
||||
CPUOffloadingMetrics.CPU_ALLOCATION_SIZE: OffloadingHistogramMetadata(
|
||||
documentation=(
|
||||
"Histogram of the number of CPU blocks requested by each "
|
||||
"Histogram of the number of CPU chunks requested by each "
|
||||
"KV offload prepare_store call."
|
||||
),
|
||||
buckets=(1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144),
|
||||
@@ -101,7 +101,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
)
|
||||
|
||||
world_size = config.parallel.world_size
|
||||
self.num_blocks = 0
|
||||
self.num_chunks = 0
|
||||
self.kv_bytes_per_chunk = 0
|
||||
self.cpu_page_size_per_worker = 0
|
||||
self.replicated_layout = config.replicated_layout and self._uses_shared_region()
|
||||
@@ -113,17 +113,17 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
# calculate cpu_page_size_per_worker
|
||||
self.cpu_page_size_per_worker = kv_bytes_per_chunk // num_copies
|
||||
|
||||
# calculate num_blocks
|
||||
# calculate num_chunks
|
||||
aligned_kv_bytes_per_chunk = round_up(
|
||||
kv_bytes_per_chunk, self.BLOCK_SIZE_ALIGNMENT
|
||||
)
|
||||
self.num_blocks = int(cpu_bytes_to_use) // aligned_kv_bytes_per_chunk
|
||||
self.num_chunks = int(cpu_bytes_to_use) // aligned_kv_bytes_per_chunk
|
||||
|
||||
# Expose aligned_kv_bytes_per_chunk as
|
||||
# kv_bytes_per_chunk. Note that this might contain
|
||||
# some padding. i.e. each offloaded block is of the form,
|
||||
# |--- W0-B0---|---- W1-B0---| ... |---- Wn-B0---| *** maybe-pad *** |
|
||||
# or |--- B0 (single copy) ---| *** maybe-pad *** |
|
||||
# some padding. i.e. each offloaded chunk is of the form,
|
||||
# |--- W0-C0---|---- W1-C0---| ... |---- Wn-C0---| *** maybe-pad *** |
|
||||
# or |--- C0 (single copy) ---| *** maybe-pad *** |
|
||||
self.kv_bytes_per_chunk = aligned_kv_bytes_per_chunk
|
||||
|
||||
# scheduler-side
|
||||
@@ -140,7 +140,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
@override
|
||||
def get_manager(self) -> OffloadingManager:
|
||||
if not self._manager:
|
||||
# store_threshold: how many times a block must be offered for
|
||||
# store_threshold: how many times a chunk must be offered for
|
||||
# storage before it is eligible for CPU offloading. Values < 2
|
||||
# disable filtering (a threshold of 1 equals no filter; 0 is the
|
||||
# default).
|
||||
@@ -150,7 +150,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
max_tracker_size = int(self.extra_config.get("max_tracker_size", 64_000))
|
||||
|
||||
self._manager = CPUOffloadingManager(
|
||||
num_blocks=self.num_blocks,
|
||||
num_chunks=self.num_chunks,
|
||||
cache_policy=self.eviction_policy,
|
||||
cache_policy_module_path=self.cache_policy_module_path,
|
||||
enable_events=self.kv_events_config.enable_kv_cache_events,
|
||||
@@ -166,9 +166,9 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
|
||||
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
|
||||
mmap_region: SharedOffloadRegion | None = None
|
||||
# num_blocks == 0 would size the region to zero bytes, which cannot be
|
||||
# num_chunks == 0 would size the region to zero bytes, which cannot be
|
||||
# mmap'd; fall back to the tensor path (empty tensors) as before.
|
||||
if self._uses_shared_region() and self.num_blocks > 0:
|
||||
if self._uses_shared_region() and self.num_chunks > 0:
|
||||
# Replicated layout puts all ranks on slot 0 (single MLA copy);
|
||||
# otherwise each rank takes its own slot by physical device index.
|
||||
if self.replicated_layout:
|
||||
@@ -178,9 +178,9 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
rank = torch.accelerator.current_device_index() % world_size
|
||||
mmap_region = SharedOffloadRegion(
|
||||
engine_id=self.config.engine_id,
|
||||
num_blocks=self.num_blocks,
|
||||
num_chunks=self.num_chunks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=self.kv_bytes_per_chunk,
|
||||
kv_bytes_per_chunk=self.kv_bytes_per_chunk,
|
||||
cpu_page_size=self.cpu_page_size_per_worker,
|
||||
barrier=_all_workers_barrier,
|
||||
)
|
||||
@@ -188,7 +188,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
return CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
blocks_per_chunk=self.blocks_per_chunk,
|
||||
num_cpu_blocks=self.num_blocks,
|
||||
num_cpu_chunks=self.num_chunks,
|
||||
mmap_region=mmap_region,
|
||||
)
|
||||
except Exception:
|
||||
|
||||
@@ -45,8 +45,8 @@ class TieringOffloadingMetrics:
|
||||
WRITE_TIME = "vllm:kv_offload_tiering_write_time"
|
||||
PROMOTION_JOB_FAILURES = "vllm:kv_offload_tiering_promotion_job_failures"
|
||||
CASCADE_JOB_FAILURES = "vllm:kv_offload_tiering_cascade_job_failures"
|
||||
BLOCK_QUERIES = "vllm:kv_offload_tiering_chunk_queries"
|
||||
BLOCK_HITS = "vllm:kv_offload_tiering_chunk_hits"
|
||||
CHUNK_QUERIES = "vllm:kv_offload_tiering_chunk_queries"
|
||||
CHUNK_HITS = "vllm:kv_offload_tiering_chunk_hits"
|
||||
PRIMARY_WRITE_USAGE_PERC = "vllm:kv_offload_tiering_primary_write_usage_perc"
|
||||
PRIMARY_READ_USAGE_PERC = "vllm:kv_offload_tiering_primary_read_usage_perc"
|
||||
PROMOTION_ALLOCATION_FAILURES = (
|
||||
@@ -62,7 +62,7 @@ class TransferJob:
|
||||
|
||||
job_id: JobId
|
||||
keys: Collection[OffloadKey]
|
||||
block_ids: np.ndarray
|
||||
chunk_ids: np.ndarray
|
||||
is_promotion: bool
|
||||
req_context: ReqContext
|
||||
|
||||
@@ -90,14 +90,14 @@ class ParentManager(ABC):
|
||||
|
||||
Required call sequence for each remote request:
|
||||
1. on_new_request(req_context) — set up per-request state
|
||||
2. lookup(key, req_context) — check block availability
|
||||
(repeat per block)
|
||||
3. create_store_job(keys, req_context) — pin blocks and get a
|
||||
2. lookup(key, req_context) — check chunk availability
|
||||
(repeat per chunk)
|
||||
3. create_store_job(keys, req_context) — pin chunks and get a
|
||||
job handle
|
||||
4. on_request_finished(req_context) — clean up per-request state
|
||||
|
||||
Steps 2-3 may be interleaved. Step 4 must be called even if no
|
||||
blocks were found, to avoid leaking async lookup state (e.g. in
|
||||
chunks were found, to avoid leaking async lookup state (e.g. in
|
||||
the fs tier's AsyncLookupManager).
|
||||
"""
|
||||
|
||||
@@ -155,23 +155,23 @@ class SecondaryTierManager(ABC):
|
||||
@abstractmethod
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
"""
|
||||
Check whether a block exists in this secondary tier.
|
||||
Check whether a chunk exists in this secondary tier.
|
||||
|
||||
Args:
|
||||
key: Offload key to look up.
|
||||
req_context: per-request context (e.g. kv_transfer_params).
|
||||
|
||||
Returns:
|
||||
HIT if the block is present and ready,
|
||||
HIT if the chunk is present and ready,
|
||||
MISS if not found,
|
||||
or RETRY if the block is being transferred (retry later).
|
||||
or RETRY if the chunk is being transferred (retry later).
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def submit_store(self, job_metadata: TransferJob) -> None:
|
||||
"""
|
||||
Submit an async job to store blocks from the primary tier to this
|
||||
Submit an async job to store chunks from the primary tier to this
|
||||
secondary tier.
|
||||
|
||||
This method must be lightweight and non-blocking: allocate metadata
|
||||
@@ -179,19 +179,19 @@ class SecondaryTierManager(ABC):
|
||||
calling thread.
|
||||
|
||||
Preconditions (guaranteed by the framework):
|
||||
- ``job_metadata.block_ids`` are valid primary-tier slots, pinned
|
||||
- ``job_metadata.chunk_ids`` are valid primary-tier slots, pinned
|
||||
(ref-counted) for the duration of the transfer.
|
||||
|
||||
The implementation is responsible for:
|
||||
1. Filtering out blocks already present in this tier
|
||||
2. Evicting blocks if capacity is needed
|
||||
1. Filtering out chunks already present in this tier
|
||||
2. Evicting chunks if capacity is needed
|
||||
3. Allocating space in this tier
|
||||
4. Submitting the async transfer (read from primary via block_ids)
|
||||
4. Submitting the async transfer (read from primary via chunk_ids)
|
||||
|
||||
Report completion via ``get_finished_jobs()``.
|
||||
|
||||
Args:
|
||||
job_metadata: Job metadata including job_id, keys, and block_ids
|
||||
job_metadata: Job metadata including job_id, keys, and chunk_ids
|
||||
identifying the primary-tier slots to read from.
|
||||
"""
|
||||
pass
|
||||
@@ -199,24 +199,24 @@ class SecondaryTierManager(ABC):
|
||||
@abstractmethod
|
||||
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||
"""
|
||||
Submit an async job to load blocks from this secondary tier to the
|
||||
Submit an async job to load chunks from this secondary tier to the
|
||||
primary tier.
|
||||
|
||||
This method must be lightweight and non-blocking: mark blocks as
|
||||
This method must be lightweight and non-blocking: mark chunks as
|
||||
in-flight and submit the transfer, but do NOT perform the data copy
|
||||
on the calling thread.
|
||||
|
||||
Preconditions (guaranteed by the framework):
|
||||
- ``job_metadata.block_ids`` are allocated primary-tier slots
|
||||
- ``job_metadata.chunk_ids`` are allocated primary-tier slots
|
||||
ready to receive data.
|
||||
|
||||
The implementation must copy data from this tier into the
|
||||
primary-tier slots identified by ``block_ids``.
|
||||
primary-tier slots identified by ``chunk_ids``.
|
||||
|
||||
Report completion via ``get_finished_jobs()``.
|
||||
|
||||
Args:
|
||||
job_metadata: Job metadata including job_id, keys, and block_ids
|
||||
job_metadata: Job metadata including job_id, keys, and chunk_ids
|
||||
identifying the primary-tier slots to write into.
|
||||
"""
|
||||
pass
|
||||
@@ -249,7 +249,7 @@ class SecondaryTierManager(ABC):
|
||||
|
||||
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
||||
"""
|
||||
Mark blocks as recently used for eviction policy.
|
||||
Mark chunks as recently used for eviction policy.
|
||||
|
||||
Args:
|
||||
keys: Offload keys to mark as recently used.
|
||||
@@ -263,7 +263,7 @@ class SecondaryTierManager(ABC):
|
||||
Called when a new request is first seen by the scheduler.
|
||||
|
||||
Returns a RequestOffloadingContext expressing this tier's preference
|
||||
for how blocks should be offloaded for this request.
|
||||
for how chunks should be offloaded for this request.
|
||||
|
||||
Args:
|
||||
req_context: Per-request context.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
ExampleSecondaryTierManager: A simple in-memory secondary tier.
|
||||
|
||||
This implementation provides a minimal secondary tier that stores blocks
|
||||
This implementation provides a minimal secondary tier that stores chunks
|
||||
in memory (using a dictionary) with immediate completion. It serves as a
|
||||
reference for writing new tiers and is useful for testing the
|
||||
TieringOffloadingManager without requiring actual storage or network backends.
|
||||
@@ -39,7 +39,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
A simple in-memory secondary tier.
|
||||
|
||||
This implementation:
|
||||
- Stores blocks in a dictionary (key -> True)
|
||||
- Stores chunks in a dictionary (key -> True)
|
||||
- Completes transfers immediately (synchronous)
|
||||
"""
|
||||
|
||||
@@ -69,45 +69,45 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
)
|
||||
|
||||
# key -> True (only care about presence)
|
||||
self.blocks: dict[OffloadKey, bool] = {}
|
||||
self.chunks: dict[OffloadKey, bool] = {}
|
||||
|
||||
# Completed jobs waiting to be retrieved by get_finished_jobs()
|
||||
self.completed_jobs: list[JobResult] = []
|
||||
assert primary_kv_view.strides is not None
|
||||
self._block_size = primary_kv_view.strides[0]
|
||||
self._chunk_size = primary_kv_view.strides[0]
|
||||
|
||||
@override
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
"""
|
||||
Check whether a block exists in this secondary tier.
|
||||
Check whether a chunk exists in this secondary tier.
|
||||
|
||||
Args:
|
||||
key: Offload key to look up.
|
||||
req_context: Per-request context.
|
||||
|
||||
Returns:
|
||||
HIT if the block is present, MISS if not found.
|
||||
HIT if the chunk is present, MISS if not found.
|
||||
"""
|
||||
return LookupResult.HIT if key in self.blocks else LookupResult.MISS
|
||||
return LookupResult.HIT if key in self.chunks else LookupResult.MISS
|
||||
|
||||
@override
|
||||
def submit_store(self, job_metadata: TransferJob) -> None:
|
||||
"""
|
||||
Submit a job to store blocks from primary tier to this tier.
|
||||
Submit a job to store chunks from primary tier to this tier.
|
||||
|
||||
Args:
|
||||
job_metadata: Job metadata including job_id, keys, and
|
||||
spec for reading blocks from the primary tier.
|
||||
spec for reading chunks from the primary tier.
|
||||
"""
|
||||
keys = job_metadata.keys
|
||||
block_ids = job_metadata.block_ids
|
||||
chunk_ids = job_metadata.chunk_ids
|
||||
|
||||
assert len(keys) == len(block_ids), (
|
||||
f"Length mismatch: {len(keys)} keys but {len(block_ids)} block_ids"
|
||||
assert len(keys) == len(chunk_ids), (
|
||||
f"Length mismatch: {len(keys)} keys but {len(chunk_ids)} chunk_ids"
|
||||
)
|
||||
|
||||
for key in keys:
|
||||
self.blocks[key] = True
|
||||
self.chunks[key] = True
|
||||
self.completed_jobs.append(
|
||||
JobResult(
|
||||
job_id=job_metadata.job_id,
|
||||
@@ -119,21 +119,21 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
@override
|
||||
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||
"""
|
||||
Submit a job to load blocks from this tier to primary tier.
|
||||
Submit a job to load chunks from this tier to primary tier.
|
||||
|
||||
Args:
|
||||
job_metadata: Job metadata including job_id, keys, and
|
||||
spec for writing blocks into the primary tier.
|
||||
spec for writing chunks into the primary tier.
|
||||
"""
|
||||
keys = job_metadata.keys
|
||||
block_ids = job_metadata.block_ids
|
||||
chunk_ids = job_metadata.chunk_ids
|
||||
|
||||
assert len(keys) == len(block_ids), (
|
||||
f"Length mismatch: {len(keys)} keys but {len(block_ids)} block_ids"
|
||||
assert len(keys) == len(chunk_ids), (
|
||||
f"Length mismatch: {len(keys)} keys but {len(chunk_ids)} chunk_ids"
|
||||
)
|
||||
|
||||
for key in keys:
|
||||
if key not in self.blocks:
|
||||
if key not in self.chunks:
|
||||
self.completed_jobs.append(
|
||||
JobResult(job_id=job_metadata.job_id, success=False)
|
||||
)
|
||||
@@ -170,6 +170,6 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
completes, so there is nothing to wait for."""
|
||||
return
|
||||
|
||||
def get_num_blocks(self) -> int:
|
||||
"""Get the number of blocks currently stored in this tier."""
|
||||
return len(self.blocks)
|
||||
def get_num_chunks(self) -> int:
|
||||
"""Get the number of chunks currently stored in this tier."""
|
||||
return len(self.chunks)
|
||||
|
||||
@@ -223,7 +223,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
batch_store_block,
|
||||
[self.file_mapper.get_file_name(key) for key in keys],
|
||||
self._primary_kv_view,
|
||||
[int(bid) * self._block_size for bid in job_metadata.block_ids],
|
||||
[int(cid) * self._block_size for cid in job_metadata.chunk_ids],
|
||||
self._block_size,
|
||||
self._use_o_direct,
|
||||
)
|
||||
@@ -237,7 +237,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
keys = list(job_metadata.keys)
|
||||
self._load_job_keys[job_id] = keys
|
||||
paths = [self.file_mapper.get_file_name(key) for key in keys]
|
||||
offsets = [int(bid) * self._block_size for bid in job_metadata.block_ids]
|
||||
offsets = [int(cid) * self._block_size for cid in job_metadata.chunk_ids]
|
||||
|
||||
def load_task() -> None:
|
||||
try:
|
||||
|
||||
@@ -8,16 +8,16 @@ and zero or more secondary tiers (Storage, Network, etc.) to provide
|
||||
hierarchical KV cache offloading.
|
||||
|
||||
Key Design Principles:
|
||||
1. Always offload to all tiers — When a block is stored to the primary tier,
|
||||
1. Always offload to all tiers — When a chunk is stored to the primary tier,
|
||||
it is cascaded to ALL secondary tiers
|
||||
2. Primary tier is the gateway — Secondary tiers cannot access GPU memory
|
||||
directly; all data flows through the CPU primary tier
|
||||
3. Staged promotion — Blocks in secondary tiers must be promoted to the
|
||||
3. Staged promotion — Chunks in secondary tiers must be promoted to the
|
||||
primary tier before GPU can access them
|
||||
4. Transparent retry mechanism — Return None from lookup() to signal
|
||||
"data is being promoted, try later"
|
||||
5. ref_cnt as eviction protection — primary.prepare_read() increments ref_cnt,
|
||||
protecting blocks from eviction until complete_read() is called
|
||||
protecting chunks from eviction until complete_read() is called
|
||||
"""
|
||||
|
||||
import time
|
||||
@@ -61,11 +61,11 @@ logger = init_logger(__name__)
|
||||
|
||||
@dataclass
|
||||
class PendingPromotion:
|
||||
"""Accumulator for blocks awaiting submit_load() for one (tier, request)."""
|
||||
"""Accumulator for chunks awaiting submit_load() for one (tier, request)."""
|
||||
|
||||
req_context: ReqContext
|
||||
keys: list[OffloadKey] = field(default_factory=list)
|
||||
block_ids: list[int] = field(default_factory=list)
|
||||
chunk_ids: list[int] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -94,14 +94,14 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_blocks: int,
|
||||
num_chunks: int,
|
||||
mmap_region: SharedOffloadRegion,
|
||||
cache_policy: str = "lru",
|
||||
cache_policy_module_path: str | None = None,
|
||||
enable_events: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
num_blocks=num_blocks,
|
||||
num_chunks=num_chunks,
|
||||
cache_policy=cache_policy,
|
||||
cache_policy_module_path=cache_policy_module_path,
|
||||
enable_events=enable_events,
|
||||
@@ -120,9 +120,9 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
|
||||
def get_kv_memoryview(self) -> memoryview:
|
||||
"""Return the memoryview over the primary tier's KV cache buffer.
|
||||
|
||||
The view has shape (num_blocks, row_stride_bytes) and is backed by the
|
||||
SharedOffloadRegion mmap. Secondary tiers address block *b* as
|
||||
``view[b]``.
|
||||
The view has shape (num_chunks, row_stride_bytes) and is backed by the
|
||||
SharedOffloadRegion mmap. Secondary tiers address chunk *c* as
|
||||
``view[c]``.
|
||||
"""
|
||||
return self._kv_memoryview
|
||||
|
||||
@@ -206,14 +206,14 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
assert primary_view.strides is not None
|
||||
self._metrics = TieringMetricsTracker(
|
||||
tier_types=[tier.tier_type for tier in self.secondary_tiers],
|
||||
num_primary_blocks=self.primary_tier._num_blocks,
|
||||
primary_block_size=primary_view.strides[0],
|
||||
num_primary_chunks=self.primary_tier._num_chunks,
|
||||
primary_chunk_size=primary_view.strides[0],
|
||||
)
|
||||
|
||||
# Pending promotion requests accumulated during lookup() calls; flushed
|
||||
# as one batched submit_load() per (tier, request) in on_schedule_end().
|
||||
# Outer key: tier index. Inner key: req_context.req_id — the same ReqContext
|
||||
# object is reused for all block lookups of a given request per engine step.
|
||||
# object is reused for all chunk lookups of a given request per engine step.
|
||||
self._pending_load_submissions: dict[int, dict[str, PendingPromotion]] = {}
|
||||
|
||||
# Gate for once-per-step execution of _maybe_process_finished_jobs().
|
||||
@@ -305,7 +305,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
2. For completed stores (primary→secondary): calls primary.complete_read()
|
||||
to decrement ref_cnt
|
||||
3. For completed loads (secondary→primary): calls primary.complete_write()
|
||||
to make blocks available
|
||||
to make chunks available
|
||||
"""
|
||||
for i, tier in enumerate(self.secondary_tiers):
|
||||
for completed_job in tier.get_finished_jobs():
|
||||
@@ -324,11 +324,11 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
|
||||
if transfer_job.is_promotion:
|
||||
# secondary→primary transfer (promotion) completed.
|
||||
# Make blocks available in primary tier.
|
||||
# Make chunks available in primary tier.
|
||||
self._complete_promotion(job_metadata, completed_job)
|
||||
else:
|
||||
# primary→secondary transfer completed.
|
||||
# Decrement ref_cnt on primary blocks.
|
||||
# Decrement ref_cnt on primary chunks.
|
||||
self.primary_tier.complete_read(
|
||||
transfer_job.keys, transfer_job.req_context
|
||||
)
|
||||
@@ -342,7 +342,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
exclude_tier_idx: int | None = None,
|
||||
) -> LookupResult:
|
||||
"""
|
||||
Check whether a single block is offloaded and ready.
|
||||
Check whether a single chunk is offloaded and ready.
|
||||
|
||||
Algorithm:
|
||||
1. Process any completed async jobs first.
|
||||
@@ -351,20 +351,20 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
hit and initiate promotion.
|
||||
|
||||
Args:
|
||||
key: Block hash to look up.
|
||||
key: Chunk hash to look up.
|
||||
req_context: Per-request context.
|
||||
|
||||
Returns:
|
||||
HIT — block is ready in the primary tier.
|
||||
HIT_PENDING — block found but not yet readable (write
|
||||
HIT — chunk is ready in the primary tier.
|
||||
HIT_PENDING — chunk found but not yet readable (write
|
||||
in-flight on the primary tier).
|
||||
RETRY — promotion started or a secondary tier is busy.
|
||||
MISS — block not found in any tier, or primary is full
|
||||
MISS — chunk not found in any tier, or primary is full
|
||||
and cannot accept a promotion.
|
||||
"""
|
||||
# Poll first so a promotion that finished since the last call is
|
||||
# already reflected as HIT (not stale HIT_PENDING/MISS) below, and
|
||||
# so blocks freed by cascade or promotion completions are evictable
|
||||
# so chunks freed by cascade or promotion completions are evictable
|
||||
# in time for a promotion this lookup may initiate.
|
||||
self._maybe_process_finished_jobs()
|
||||
|
||||
@@ -424,30 +424,30 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
req_context: ReqContext,
|
||||
) -> bool:
|
||||
"""
|
||||
Queue a block for promotion from a secondary tier to the primary tier.
|
||||
Queue a chunk for promotion from a secondary tier to the primary tier.
|
||||
|
||||
Allocates space in the primary tier immediately (sets ref_cnt=-1 so
|
||||
subsequent lookups within the same step see the slot as in-flight),
|
||||
then defers the actual submit_load() call to _flush_pending_promotions()
|
||||
so all blocks queued during one engine step are submitted as a single
|
||||
so all chunks queued during one engine step are submitted as a single
|
||||
batched job.
|
||||
|
||||
Args:
|
||||
tier_idx: The secondary tier index to promote from
|
||||
key: Block to promote
|
||||
key: Chunk to promote
|
||||
req_context: Per-request context forwarded to primary.prepare_write().
|
||||
|
||||
Returns:
|
||||
True if promotion was initiated, False if primary tier is full.
|
||||
"""
|
||||
# Allocate space in primary tier for promoted block.
|
||||
# Allocate space in primary tier for promoted chunk.
|
||||
# Must happen immediately so primary.lookup() returns None (in-flight)
|
||||
# for this key on any subsequent lookup() call within the same step,
|
||||
# preventing duplicate promotion attempts.
|
||||
primary_write_result = self.primary_tier.prepare_write([key], req_context)
|
||||
|
||||
if primary_write_result is None:
|
||||
# Primary tier is full; caller should treat the block as unavailable
|
||||
# Primary tier is full; caller should treat the chunk as unavailable
|
||||
# rather than retrying indefinitely.
|
||||
self._metrics.on_promotion_allocation_failure()
|
||||
return False
|
||||
@@ -455,16 +455,16 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
store_spec = primary_write_result.store_spec
|
||||
assert isinstance(store_spec, CPULoadStoreSpec)
|
||||
# Defer submit_load to on_schedule_end(). Group by (tier, request) so
|
||||
# each request's blocks are submitted as one batched job per tier.
|
||||
# each request's chunks are submitted as one batched job per tier.
|
||||
tier_pending = self._pending_load_submissions.setdefault(tier_idx, {})
|
||||
ctx_id = req_context.req_id
|
||||
if ctx_id not in tier_pending:
|
||||
tier_pending[ctx_id] = PendingPromotion(
|
||||
keys=[], block_ids=[], req_context=req_context
|
||||
keys=[], chunk_ids=[], req_context=req_context
|
||||
)
|
||||
entry = tier_pending[ctx_id]
|
||||
entry.keys.extend(primary_write_result.keys_to_store)
|
||||
entry.block_ids.extend(store_spec.block_ids)
|
||||
entry.chunk_ids.extend(store_spec.chunk_ids)
|
||||
return True
|
||||
|
||||
def _flush_pending_promotions(self) -> None:
|
||||
@@ -483,7 +483,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
job_metadata = TransferJob(
|
||||
job_id=job_id,
|
||||
keys=entry.keys,
|
||||
block_ids=np.array(entry.block_ids, dtype=np.int32),
|
||||
chunk_ids=np.array(entry.chunk_ids, dtype=np.int32),
|
||||
is_promotion=True,
|
||||
req_context=entry.req_context,
|
||||
)
|
||||
@@ -497,16 +497,16 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||
) -> LoadStoreSpec:
|
||||
"""
|
||||
Prepare blocks to be loaded from primary tier to GPU.
|
||||
Prepare chunks to be loaded from primary tier to GPU.
|
||||
|
||||
Callers only pass keys already confirmed HIT by lookup() earlier this
|
||||
step.
|
||||
|
||||
This increments ref_cnt on the blocks in the primary tier, protecting
|
||||
This increments ref_cnt on the chunks in the primary tier, protecting
|
||||
them from eviction during the transfer.
|
||||
|
||||
Args:
|
||||
keys: Blocks to prepare for loading.
|
||||
keys: Chunks to prepare for loading.
|
||||
req_context: Per-request context.
|
||||
|
||||
Returns:
|
||||
@@ -517,10 +517,10 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
@override
|
||||
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
||||
"""
|
||||
Mark blocks as recently used in all tiers.
|
||||
Mark chunks as recently used in all tiers.
|
||||
|
||||
Args:
|
||||
keys: Blocks to mark as recently used.
|
||||
keys: Chunks to mark as recently used.
|
||||
req_context: Per-request context.
|
||||
"""
|
||||
self.primary_tier.touch(keys, req_context)
|
||||
@@ -530,13 +530,13 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
@override
|
||||
def complete_load(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
||||
"""
|
||||
Mark blocks as done loading from primary tier to GPU.
|
||||
Mark chunks as done loading from primary tier to GPU.
|
||||
|
||||
This decrements ref_cnt on the blocks in the primary tier, allowing
|
||||
This decrements ref_cnt on the chunks in the primary tier, allowing
|
||||
them to be evicted again.
|
||||
|
||||
Args:
|
||||
keys: Blocks that finished loading.
|
||||
keys: Chunks that finished loading.
|
||||
req_context: Per-request context.
|
||||
"""
|
||||
self.primary_tier.complete_load(keys, req_context)
|
||||
@@ -546,37 +546,37 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||
) -> PrepareStoreOutput | None:
|
||||
"""
|
||||
Prepare blocks to be stored from GPU to primary tier.
|
||||
Prepare chunks to be stored from GPU to primary tier.
|
||||
|
||||
CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure
|
||||
that any completed async transfers have their ref_cnt decremented
|
||||
before the primary tier makes eviction decisions.
|
||||
|
||||
For request-level tiers, blocks already present in the primary tier
|
||||
For request-level tiers, chunks already present in the primary tier
|
||||
are immediately cascaded via submit_store().
|
||||
|
||||
Args:
|
||||
keys: Blocks to prepare for storing.
|
||||
keys: Chunks to prepare for storing.
|
||||
req_context: Per-request context.
|
||||
|
||||
Returns:
|
||||
PrepareStoreOutput describing where to store blocks and what was
|
||||
PrepareStoreOutput describing where to store chunks and what was
|
||||
evicted, or None if store cannot proceed.
|
||||
"""
|
||||
# Step 1: Poll for completed async jobs FIRST
|
||||
# _process_finished_jobs() handles two kinds of completions here:
|
||||
# - Cascade completions (store to a secondary tier, either a local
|
||||
# cascade or a store job created for a remote requester via
|
||||
# create_store_job()): decrements ref_cnt on the primary blocks
|
||||
# create_store_job()): decrements ref_cnt on the primary chunks
|
||||
# that were read, making them evictable again once ref_cnt hits 0.
|
||||
# - Promotion completions (secondary->primary loads): sets a
|
||||
# not-yet-ready block's ref_cnt from -1 to 0 via complete_write(),
|
||||
# not-yet-ready chunk's ref_cnt from -1 to 0 via complete_write(),
|
||||
# making it evictable for the first time.
|
||||
# Both must be accounted for before the eviction decision below.
|
||||
self._maybe_process_finished_jobs()
|
||||
|
||||
# Step 2: Store to primary tier (new blocks only).
|
||||
# Cascading of these newly-stored blocks to ALL secondary tiers
|
||||
# Step 2: Store to primary tier (new chunks only).
|
||||
# Cascading of these newly-stored chunks to ALL secondary tiers
|
||||
# happens later in complete_store(), after the GPU→Primary transfer
|
||||
# completes.
|
||||
primary_result = self.primary_tier.prepare_store(keys, req_context)
|
||||
@@ -588,7 +588,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
state = self._req_state[req_context.req_id]
|
||||
state.pending_primary_stores += 1
|
||||
|
||||
# Step 3: For request-level tiers, cascade blocks already in primary
|
||||
# Step 3: For request-level tiers, cascade chunks already in primary
|
||||
request_level_tiers = self._req_state[req_context.req_id].request_level_tiers
|
||||
if request_level_tiers:
|
||||
keys_to_store_set = set(primary_result.keys_to_store)
|
||||
@@ -596,13 +596,13 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
k for k in keys if k not in keys_to_store_set
|
||||
)
|
||||
if keys_already_in_primary:
|
||||
self._cascade_existing_blocks_to_request_level_tiers(
|
||||
self._cascade_existing_chunks_to_request_level_tiers(
|
||||
keys_already_in_primary, req_context, request_level_tiers
|
||||
)
|
||||
|
||||
return primary_result
|
||||
|
||||
def _cascade_existing_blocks_to_request_level_tiers(
|
||||
def _cascade_existing_chunks_to_request_level_tiers(
|
||||
self,
|
||||
keys: Sequence[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
@@ -610,7 +610,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
) -> None:
|
||||
"""
|
||||
For tiers that requested request-level policy, submit_store() for
|
||||
blocks that are already present in the primary tier.
|
||||
chunks that are already present in the primary tier.
|
||||
|
||||
A key whose primary write is still in flight (HIT_PENDING) cannot be
|
||||
dropped: prepare_store already excluded it as present, and the
|
||||
@@ -653,7 +653,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
continue
|
||||
assert state.request_level_tiers
|
||||
keys, state.pending_cascade_keys = state.pending_cascade_keys, []
|
||||
self._cascade_existing_blocks_to_request_level_tiers(
|
||||
self._cascade_existing_chunks_to_request_level_tiers(
|
||||
keys, state.req_context, state.request_level_tiers
|
||||
)
|
||||
self._maybe_finalize_request(req_id)
|
||||
@@ -666,30 +666,30 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
success: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
Mark blocks as done storing from GPU to primary tier.
|
||||
Mark chunks as done storing from GPU to primary tier.
|
||||
|
||||
This is where secondary tier cascading happens — after blocks are
|
||||
This is where secondary tier cascading happens — after chunks are
|
||||
confirmed to be in the primary tier, they are cascaded to ALL
|
||||
secondary tiers.
|
||||
|
||||
For each secondary tier:
|
||||
1. Call primary.prepare_read() to get LoadStoreSpec AND increment
|
||||
ref_cnt (protecting blocks during async transfer)
|
||||
ref_cnt (protecting chunks during async transfer)
|
||||
2. Call tier.submit_store() to start async transfer: primary→secondary
|
||||
3. Track the job in _store_jobs dictionary
|
||||
|
||||
Args:
|
||||
keys: Blocks that finished storing.
|
||||
keys: Chunks that finished storing.
|
||||
success: Whether the GPU→primary transfer succeeded.
|
||||
req_context: Per-request context forwarded to primary.prepare_read().
|
||||
"""
|
||||
# Step 1: Complete store in primary tier (makes blocks loadable)
|
||||
# Step 1: Complete store in primary tier (makes chunks loadable)
|
||||
self.primary_tier.complete_store(keys, req_context, success)
|
||||
|
||||
if success:
|
||||
# Step 2: Cascade to ALL secondary tiers
|
||||
# For each secondary tier, call primary.prepare_read() to get the
|
||||
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from
|
||||
# LoadStoreSpec AND to increment ref_cnt (protecting chunks from
|
||||
# eviction during the async transfer). One prepare_read() call per
|
||||
# secondary tier.
|
||||
for tier_idx, tier in enumerate(self.secondary_tiers):
|
||||
@@ -710,22 +710,22 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
req_context: ReqContext,
|
||||
tier_idx: int = 0,
|
||||
) -> TransferJob:
|
||||
"""Pin blocks in the primary tier and create a tracked store job.
|
||||
"""Pin chunks in the primary tier and create a tracked store job.
|
||||
|
||||
Calls prepare_read() to increment ref_cnt (protecting blocks
|
||||
Calls prepare_read() to increment ref_cnt (protecting chunks
|
||||
from eviction during the async transfer), allocates a job ID,
|
||||
and registers the job in _jobs.
|
||||
|
||||
The caller is responsible for the actual data transfer and
|
||||
reporting completion via get_finished_jobs().
|
||||
"""
|
||||
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
|
||||
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
|
||||
primary_chunks_spec = self.primary_tier.prepare_read(keys, req_context)
|
||||
assert isinstance(primary_chunks_spec, CPULoadStoreSpec)
|
||||
job_id = self._next_job_id()
|
||||
job_metadata = TransferJob(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=primary_blocks_spec.block_ids,
|
||||
chunk_ids=primary_chunks_spec.chunk_ids,
|
||||
is_promotion=False,
|
||||
req_context=req_context,
|
||||
)
|
||||
@@ -760,7 +760,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
policy = (
|
||||
OffloadPolicy.REQUEST_LEVEL
|
||||
if state.request_level_tiers
|
||||
else OffloadPolicy.BLOCK_LEVEL
|
||||
else OffloadPolicy.CHUNK_LEVEL
|
||||
)
|
||||
return RequestOffloadingContext(policy=policy)
|
||||
|
||||
|
||||
@@ -39,20 +39,20 @@ class _RequestMetricsState:
|
||||
class _TierState:
|
||||
active_promotion_count: int = 0
|
||||
active_cascade_count: int = 0
|
||||
primary_write_block_count: int = 0
|
||||
primary_read_block_count: int = 0
|
||||
primary_write_chunk_count: int = 0
|
||||
primary_read_chunk_count: int = 0
|
||||
|
||||
|
||||
class TieringMetricsTracker:
|
||||
def __init__(
|
||||
self,
|
||||
tier_types: list[str],
|
||||
num_primary_blocks: int,
|
||||
primary_block_size: int,
|
||||
num_primary_chunks: int,
|
||||
primary_chunk_size: int,
|
||||
) -> None:
|
||||
self._tier_types = tier_types
|
||||
self._num_primary_blocks = num_primary_blocks
|
||||
self._primary_block_size = primary_block_size
|
||||
self._num_primary_chunks = num_primary_chunks
|
||||
self._primary_chunk_size = primary_chunk_size
|
||||
self._request_states: dict[str, _RequestMetricsState] = {}
|
||||
self._tier_states = [_TierState() for _ in tier_types]
|
||||
self._stats = OffloadingConnectorStats()
|
||||
@@ -111,13 +111,13 @@ class TieringMetricsTracker:
|
||||
def on_job_registered(self, job_metadata: _JobMetadataLike) -> None:
|
||||
transfer_job = job_metadata.transfer_job
|
||||
state = self._tier_states[job_metadata.tier_idx]
|
||||
block_count = len(transfer_job.block_ids)
|
||||
chunk_count = len(transfer_job.chunk_ids)
|
||||
if transfer_job.is_promotion:
|
||||
state.active_promotion_count += 1
|
||||
state.primary_write_block_count += block_count
|
||||
state.primary_write_chunk_count += chunk_count
|
||||
else:
|
||||
state.active_cascade_count += 1
|
||||
state.primary_read_block_count += block_count
|
||||
state.primary_read_chunk_count += chunk_count
|
||||
|
||||
def on_job_finished(
|
||||
self, job_metadata: _JobMetadataLike, result: JobResult
|
||||
@@ -149,25 +149,25 @@ class TieringMetricsTracker:
|
||||
assert all(
|
||||
state.active_promotion_count == 0
|
||||
and state.active_cascade_count == 0
|
||||
and state.primary_write_block_count == 0
|
||||
and state.primary_read_block_count == 0
|
||||
and state.primary_write_chunk_count == 0
|
||||
and state.primary_read_chunk_count == 0
|
||||
for state in self._tier_states
|
||||
)
|
||||
|
||||
def _decrement_tier_state(self, job_metadata: _JobMetadataLike) -> None:
|
||||
transfer_job = job_metadata.transfer_job
|
||||
state = self._tier_states[job_metadata.tier_idx]
|
||||
block_count = len(transfer_job.block_ids)
|
||||
chunk_count = len(transfer_job.chunk_ids)
|
||||
if transfer_job.is_promotion:
|
||||
assert state.active_promotion_count > 0
|
||||
state.active_promotion_count -= 1
|
||||
state.primary_write_block_count -= block_count
|
||||
assert state.primary_write_block_count >= 0
|
||||
state.primary_write_chunk_count -= chunk_count
|
||||
assert state.primary_write_chunk_count >= 0
|
||||
else:
|
||||
assert state.active_cascade_count > 0
|
||||
state.active_cascade_count -= 1
|
||||
state.primary_read_block_count -= block_count
|
||||
assert state.primary_read_block_count >= 0
|
||||
state.primary_read_chunk_count -= chunk_count
|
||||
assert state.primary_read_chunk_count >= 0
|
||||
|
||||
def _observe_finished_job_stats(
|
||||
self,
|
||||
@@ -199,7 +199,7 @@ class TieringMetricsTracker:
|
||||
if transfer_job.is_promotion
|
||||
else TieringOffloadingMetrics.WRITE_TIME
|
||||
)
|
||||
transfer_size = completed_key_count * self._primary_block_size
|
||||
transfer_size = completed_key_count * self._primary_chunk_size
|
||||
self._stats.increase_counter(bytes_metric, transfer_size, labelvalues)
|
||||
if completed_job.transfer_time is not None:
|
||||
self._stats.increase_counter(
|
||||
@@ -210,13 +210,13 @@ class TieringMetricsTracker:
|
||||
for tier_idx, state in enumerate(self._tier_states):
|
||||
labelvalues = self.tier_label(tier_idx)
|
||||
write_usage = (
|
||||
state.primary_write_block_count / self._num_primary_blocks
|
||||
if self._num_primary_blocks > 0
|
||||
state.primary_write_chunk_count / self._num_primary_chunks
|
||||
if self._num_primary_chunks > 0
|
||||
else 0.0
|
||||
)
|
||||
read_usage = (
|
||||
state.primary_read_block_count / self._num_primary_blocks
|
||||
if self._num_primary_blocks > 0
|
||||
state.primary_read_chunk_count / self._num_primary_chunks
|
||||
if self._num_primary_chunks > 0
|
||||
else 0.0
|
||||
)
|
||||
stats.set_gauge(
|
||||
@@ -248,12 +248,12 @@ class TieringMetricsTracker:
|
||||
async_start_time: float | None,
|
||||
) -> None:
|
||||
self._stats.increase_counter(
|
||||
TieringOffloadingMetrics.BLOCK_QUERIES,
|
||||
TieringOffloadingMetrics.CHUNK_QUERIES,
|
||||
labelvalues=tier_label,
|
||||
)
|
||||
if result is LookupResult.HIT:
|
||||
self._stats.increase_counter(
|
||||
TieringOffloadingMetrics.BLOCK_HITS,
|
||||
TieringOffloadingMetrics.CHUNK_HITS,
|
||||
labelvalues=tier_label,
|
||||
)
|
||||
self._stats.observe_histogram(
|
||||
|
||||
@@ -287,14 +287,14 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
||||
self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
||||
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
|
||||
self._submit_transfer(
|
||||
job_metadata.job_id, job_metadata.block_ids, obj_keys, NIXL_WRITE
|
||||
job_metadata.job_id, job_metadata.chunk_ids, obj_keys, NIXL_WRITE
|
||||
)
|
||||
|
||||
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||
self._load_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
||||
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
|
||||
self._submit_transfer(
|
||||
job_metadata.job_id, job_metadata.block_ids, obj_keys, NIXL_READ
|
||||
job_metadata.job_id, job_metadata.chunk_ids, obj_keys, NIXL_READ
|
||||
)
|
||||
|
||||
def on_request_finished(self, req_context: ReqContext) -> None:
|
||||
|
||||
@@ -417,7 +417,7 @@ class P2PSecondaryTierManager(SecondaryTierManager):
|
||||
def submit_store(self, job_metadata: TransferJob) -> None:
|
||||
job_id = job_metadata.job_id
|
||||
keys = list(job_metadata.keys)
|
||||
block_ids = job_metadata.block_ids.tolist()
|
||||
block_ids = job_metadata.chunk_ids.tolist()
|
||||
|
||||
assert len(keys) == len(block_ids)
|
||||
|
||||
@@ -476,7 +476,7 @@ class P2PSecondaryTierManager(SecondaryTierManager):
|
||||
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||
job_id = job_metadata.job_id
|
||||
keys = list(job_metadata.keys)
|
||||
block_ids = job_metadata.block_ids
|
||||
block_ids = job_metadata.chunk_ids
|
||||
|
||||
source = job_metadata.req_context.get_state(P2PSourceInfo)
|
||||
logger.debug(
|
||||
|
||||
@@ -593,7 +593,7 @@ class ServerRole:
|
||||
self.add_stored_blocks(
|
||||
lookup.kv_request_id,
|
||||
list(meta.keys),
|
||||
meta.block_ids.tolist(),
|
||||
meta.chunk_ids.tolist(),
|
||||
meta.job_id,
|
||||
round_seq=lookup.round_seq,
|
||||
from_lookup=True,
|
||||
|
||||
@@ -8,7 +8,7 @@ and configurable secondary tiers (e.g., Storage, Network).
|
||||
|
||||
Configuration via kv_connector_extra_config:
|
||||
- cpu_bytes_to_use: (required) Bytes to allocate for CPU primary tier
|
||||
- block_size: (optional) Block size for offloaded blocks (default: GPU block size)
|
||||
- block_size: (optional) Tokens per offloaded chunk (default: GPU block size)
|
||||
- eviction_policy: (optional) Primary tier eviction policy: built-in "lru"/
|
||||
"arc", or the name of a policy registered via CachePolicyFactory, or an
|
||||
out-of-tree CachePolicy class name paired with cache_policy_module_path
|
||||
@@ -105,7 +105,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY] = (
|
||||
OffloadingHistogramMetadata(
|
||||
documentation=(
|
||||
"Histogram of blocking time spent in a per-block tier lookup "
|
||||
"Histogram of blocking time spent in a per-chunk tier lookup "
|
||||
"that resolved as a hit or miss, labeled by tier, in seconds."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
@@ -127,7 +127,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY] = (
|
||||
OffloadingHistogramMetadata(
|
||||
documentation=(
|
||||
"Histogram of wall-clock time from a per-block tier lookup "
|
||||
"Histogram of wall-clock time from a per-chunk tier lookup "
|
||||
"first returning retry until that same tier lookup resolves "
|
||||
"as a hit or miss, labeled by tier, in seconds."
|
||||
),
|
||||
@@ -191,14 +191,14 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
labelnames=("tier",),
|
||||
)
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.BLOCK_QUERIES] = OffloadingCounterMetadata(
|
||||
metrics[TieringOffloadingMetrics.CHUNK_QUERIES] = OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Number of block lookup queries sent to a tier, labeled by tier."
|
||||
"Number of chunk lookup queries sent to a tier, labeled by tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.BLOCK_HITS] = OffloadingCounterMetadata(
|
||||
documentation="Number of block lookup hits in a tier, labeled by tier.",
|
||||
metrics[TieringOffloadingMetrics.CHUNK_HITS] = OffloadingCounterMetadata(
|
||||
documentation="Number of chunk lookup hits in a tier, labeled by tier.",
|
||||
labelnames=("tier",),
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES] = (
|
||||
@@ -300,16 +300,16 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
# primary tier can eagerly create a memoryview over _base.
|
||||
scheduler_mmap = SharedOffloadRegion(
|
||||
engine_id=self._engine_id,
|
||||
num_blocks=self.num_blocks,
|
||||
num_chunks=self.num_chunks,
|
||||
rank=None,
|
||||
kv_bytes_per_block=self.kv_bytes_per_chunk,
|
||||
kv_bytes_per_chunk=self.kv_bytes_per_chunk,
|
||||
cpu_page_size=self.cpu_page_size_per_worker,
|
||||
)
|
||||
self._scheduler_mmap = scheduler_mmap
|
||||
|
||||
# Create primary tier (CPU-based)
|
||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=self.num_blocks,
|
||||
num_chunks=self.num_chunks,
|
||||
cache_policy=self.eviction_policy,
|
||||
cache_policy_module_path=self.cache_policy_module_path,
|
||||
enable_events=self.kv_events_config.enable_kv_cache_events,
|
||||
@@ -367,9 +367,9 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
|
||||
logger.info(
|
||||
"Created TieringOffloadingManager with primary tier "
|
||||
"(%s, %s blocks) and %s secondary tier(s)",
|
||||
"(%s, %s chunks) and %s secondary tier(s)",
|
||||
self.eviction_policy,
|
||||
self.num_blocks,
|
||||
self.num_chunks,
|
||||
len(secondary_tiers),
|
||||
)
|
||||
|
||||
@@ -393,9 +393,9 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
rank = torch.accelerator.current_device_index() % world_size
|
||||
worker_mmap = SharedOffloadRegion(
|
||||
engine_id=self._engine_id,
|
||||
num_blocks=self.num_blocks,
|
||||
num_chunks=self.num_chunks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=self.kv_bytes_per_chunk,
|
||||
kv_bytes_per_chunk=self.kv_bytes_per_chunk,
|
||||
cpu_page_size=self.cpu_page_size_per_worker,
|
||||
)
|
||||
try:
|
||||
@@ -404,7 +404,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
return CPUOffloadingWorker(
|
||||
kv_caches=kv_caches,
|
||||
blocks_per_chunk=self.blocks_per_chunk,
|
||||
num_cpu_blocks=self.num_blocks,
|
||||
num_cpu_chunks=self.num_chunks,
|
||||
mmap_region=worker_mmap,
|
||||
canonical_layout=self.config.canonical_layout,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user