[Refactor][kv_offload]: rename block→chunk (#52615)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
@@ -22,6 +22,10 @@ flowchart LR
|
|||||||
CPU <--> SN["..."]
|
CPU <--> SN["..."]
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Terminology: Chunks
|
||||||
|
|
||||||
|
The unit of operation is a **chunk** — a fixed-size piece of KV data covering a group of tokens. By default, a chunk maps to a single accelerator block. A configurable `blocks_per_chunk` parameter allows larger chunks, yielding larger I/Os to the host and secondary tiers.
|
||||||
|
|
||||||
## Single-Tier Setup (CPU Only)
|
## Single-Tier Setup (CPU Only)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -704,7 +704,7 @@ def test_prom_metrics_registers_tiering_metrics_from_spec():
|
|||||||
)
|
)
|
||||||
|
|
||||||
metric = prom_metrics._offloading_metric_defs[
|
metric = prom_metrics._offloading_metric_defs[
|
||||||
TieringOffloadingMetrics.BLOCK_QUERIES
|
TieringOffloadingMetrics.CHUNK_QUERIES
|
||||||
]
|
]
|
||||||
assert metric.kwargs["labelnames"] == ["model_name", "engine", "tier"]
|
assert metric.kwargs["labelnames"] == ["model_name", "engine", "tier"]
|
||||||
|
|
||||||
|
|||||||
@@ -96,7 +96,7 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
|
|||||||
groups.append(
|
groups.append(
|
||||||
KVCacheGroupSpec([f"layer{i}"], kv_spec, is_eagle_group=eagle and i == 1)
|
KVCacheGroupSpec([f"layer{i}"], kv_spec, is_eagle_group=eagle and i == 1)
|
||||||
)
|
)
|
||||||
manager = CPUOffloadingManager(num_blocks=100)
|
manager = CPUOffloadingManager(num_chunks=100)
|
||||||
spec = SimpleNamespace(
|
spec = SimpleNamespace(
|
||||||
tokens_per_block=(256, 64, 8),
|
tokens_per_block=(256, 64, 8),
|
||||||
tokens_per_hash=8,
|
tokens_per_hash=8,
|
||||||
@@ -2888,7 +2888,7 @@ class TestEagle:
|
|||||||
req=req,
|
req=req,
|
||||||
req_context=ReqContext(req_id="test-req"),
|
req_context=ReqContext(req_id="test-req"),
|
||||||
offloading_context=RequestOffloadingContext(
|
offloading_context=RequestOffloadingContext(
|
||||||
policy=OffloadPolicy.BLOCK_LEVEL
|
policy=OffloadPolicy.CHUNK_LEVEL
|
||||||
),
|
),
|
||||||
num_locally_computed_tokens=num_computed_tokens,
|
num_locally_computed_tokens=num_computed_tokens,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import pytest
|
|||||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||||
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
||||||
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
|
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
|
||||||
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
|
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
|
||||||
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
|
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
|
||||||
|
|
||||||
@@ -20,10 +20,10 @@ class _DummyCachePolicy(CachePolicy):
|
|||||||
def __init__(self, cache_capacity: int) -> None:
|
def __init__(self, cache_capacity: int) -> None:
|
||||||
self.cache_capacity = cache_capacity
|
self.cache_capacity = cache_capacity
|
||||||
|
|
||||||
def get(self, key: OffloadKey) -> BlockStatus | None:
|
def get(self, key: OffloadKey) -> ChunkStatus | None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def insert(self, key: OffloadKey, block: BlockStatus) -> None:
|
def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def remove(self, key: OffloadKey) -> None:
|
def remove(self, key: OffloadKey) -> None:
|
||||||
@@ -34,7 +34,7 @@ class _DummyCachePolicy(CachePolicy):
|
|||||||
|
|
||||||
def evict(
|
def evict(
|
||||||
self, n: int, protected: set[OffloadKey]
|
self, n: int, protected: set[OffloadKey]
|
||||||
) -> list[tuple[OffloadKey, BlockStatus]] | None:
|
) -> list[tuple[OffloadKey, ChunkStatus]] | None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def clear(self) -> None:
|
def clear(self) -> None:
|
||||||
@@ -72,7 +72,7 @@ class TestCachePolicyFactory:
|
|||||||
policy_cls = CachePolicyFactory.get_cache_policy_cls("dummy")
|
policy_cls = CachePolicyFactory.get_cache_policy_cls("dummy")
|
||||||
assert policy_cls is _DummyCachePolicy
|
assert policy_cls is _DummyCachePolicy
|
||||||
|
|
||||||
manager = CPUOffloadingManager(num_blocks=4, cache_policy="dummy")
|
manager = CPUOffloadingManager(num_chunks=4, cache_policy="dummy")
|
||||||
assert isinstance(manager._policy, _DummyCachePolicy)
|
assert isinstance(manager._policy, _DummyCachePolicy)
|
||||||
|
|
||||||
def test_unregistered_policy_raises(self):
|
def test_unregistered_policy_raises(self):
|
||||||
@@ -98,7 +98,7 @@ class TestCachePolicyFactory:
|
|||||||
"""End-to-end: CPUOffloadingManager resolves an unregistered policy
|
"""End-to-end: CPUOffloadingManager resolves an unregistered policy
|
||||||
purely from cache_policy_module_path."""
|
purely from cache_policy_module_path."""
|
||||||
manager = CPUOffloadingManager(
|
manager = CPUOffloadingManager(
|
||||||
num_blocks=4,
|
num_chunks=4,
|
||||||
cache_policy="_DummyCachePolicy",
|
cache_policy="_DummyCachePolicy",
|
||||||
cache_policy_module_path="tests.v1.kv_offload.cpu.policies.test_factory",
|
cache_policy_module_path="tests.v1.kv_offload.cpu.policies.test_factory",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -232,9 +232,9 @@ def test_cross_topology_roundtrip(writer_tp: int, reader_tp: int):
|
|||||||
def canonical_view(rank: int, world_size: int) -> torch.Tensor:
|
def canonical_view(rank: int, world_size: int) -> torch.Tensor:
|
||||||
region = SharedOffloadRegion(
|
region = SharedOffloadRegion(
|
||||||
engine_id=engine_id,
|
engine_id=engine_id,
|
||||||
num_blocks=num_blocks,
|
num_chunks=num_blocks,
|
||||||
rank=rank,
|
rank=rank,
|
||||||
kv_bytes_per_block=row_stride,
|
kv_bytes_per_chunk=row_stride,
|
||||||
cpu_page_size=row_stride // world_size,
|
cpu_page_size=row_stride // world_size,
|
||||||
)
|
)
|
||||||
regions.append(region)
|
regions.append(region)
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
|
|||||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||||
|
|
||||||
NUM_GPU_BLOCKS = [64]
|
NUM_GPU_BLOCKS = [64]
|
||||||
NUM_CPU_BLOCKS = [256]
|
NUM_CPU_CHUNKS = [256]
|
||||||
GPU_PAGE_SIZES = [512, 1024]
|
GPU_PAGE_SIZES = [512, 1024]
|
||||||
BLOCKS_PER_CHUNK_VALUES = [1, 3]
|
BLOCKS_PER_CHUNK_VALUES = [1, 3]
|
||||||
NUM_TENSORS = [4]
|
NUM_TENSORS = [4]
|
||||||
@@ -225,7 +225,7 @@ def test_worker_syncs_before_cleanup_after_handler_failure(
|
|||||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||||
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
|
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
|
||||||
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
|
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
|
||||||
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
|
@pytest.mark.parametrize("num_cpu_chunks", NUM_CPU_CHUNKS)
|
||||||
@pytest.mark.parametrize("num_tensors", NUM_TENSORS)
|
@pytest.mark.parametrize("num_tensors", NUM_TENSORS)
|
||||||
@pytest.mark.parametrize("seed", SEEDS)
|
@pytest.mark.parametrize("seed", SEEDS)
|
||||||
@pytest.mark.parametrize("device", DEVICES)
|
@pytest.mark.parametrize("device", DEVICES)
|
||||||
@@ -241,7 +241,7 @@ def test_transfer(
|
|||||||
gpu_page_size_bytes: int,
|
gpu_page_size_bytes: int,
|
||||||
blocks_per_chunk: int,
|
blocks_per_chunk: int,
|
||||||
num_gpu_blocks: int,
|
num_gpu_blocks: int,
|
||||||
num_cpu_blocks: int,
|
num_cpu_chunks: int,
|
||||||
num_tensors: int,
|
num_tensors: int,
|
||||||
seed: int,
|
seed: int,
|
||||||
device: str,
|
device: str,
|
||||||
@@ -288,41 +288,41 @@ def test_transfer(
|
|||||||
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
|
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
|
||||||
)
|
)
|
||||||
simulated_world_size = 2
|
simulated_world_size = 2
|
||||||
kv_bytes_per_block = (
|
kv_bytes_per_chunk = (
|
||||||
cpu_page_size if replicated_layout else cpu_page_size * simulated_world_size
|
cpu_page_size if replicated_layout else cpu_page_size * simulated_world_size
|
||||||
)
|
)
|
||||||
mmap_region = SharedOffloadRegion(
|
mmap_region = SharedOffloadRegion(
|
||||||
engine_id=str(uuid.uuid4()),
|
engine_id=str(uuid.uuid4()),
|
||||||
num_blocks=num_cpu_blocks,
|
num_chunks=num_cpu_chunks,
|
||||||
rank=0,
|
rank=0,
|
||||||
kv_bytes_per_block=kv_bytes_per_block,
|
kv_bytes_per_chunk=kv_bytes_per_chunk,
|
||||||
cpu_page_size=cpu_page_size,
|
cpu_page_size=cpu_page_size,
|
||||||
)
|
)
|
||||||
|
|
||||||
worker = CPUOffloadingWorker(
|
worker = CPUOffloadingWorker(
|
||||||
kv_caches=kv_caches,
|
kv_caches=kv_caches,
|
||||||
blocks_per_chunk=blocks_per_chunk,
|
blocks_per_chunk=blocks_per_chunk,
|
||||||
num_cpu_blocks=num_cpu_blocks,
|
num_cpu_chunks=num_cpu_chunks,
|
||||||
mmap_region=mmap_region,
|
mmap_region=mmap_region,
|
||||||
)
|
)
|
||||||
|
|
||||||
# select block mappings
|
# select block mappings
|
||||||
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * blocks_per_chunk)
|
gpu_blocks = random.sample(range(num_gpu_blocks), num_mappings * blocks_per_chunk)
|
||||||
cpu_blocks = random.sample(range(num_cpu_blocks), num_mappings)
|
cpu_chunks = random.sample(range(num_cpu_chunks), num_mappings)
|
||||||
|
|
||||||
# expand cpu blocks to gpu-page granularity for uniform comparison:
|
# expand cpu chunks to gpu-page granularity for uniform comparison:
|
||||||
# each cpu block maps to blocks_per_chunk consecutive sub-blocks
|
# each cpu chunk maps to blocks_per_chunk consecutive sub-blocks
|
||||||
cpu_blocks_expanded = [
|
cpu_chunks_expanded = [
|
||||||
cpu_block * blocks_per_chunk + j
|
cpu_chunk * blocks_per_chunk + j
|
||||||
for cpu_block in cpu_blocks
|
for cpu_chunk in cpu_chunks
|
||||||
for j in range(blocks_per_chunk)
|
for j in range(blocks_per_chunk)
|
||||||
]
|
]
|
||||||
|
|
||||||
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU block
|
# maybe skip some GPU blocks to test reading/writing from the middle of a CPU chunk
|
||||||
blocks_to_skip = blocks_per_chunk - 1
|
blocks_to_skip = blocks_per_chunk - 1
|
||||||
if blocks_to_skip > 0:
|
if blocks_to_skip > 0:
|
||||||
gpu_blocks = gpu_blocks[blocks_to_skip:]
|
gpu_blocks = gpu_blocks[blocks_to_skip:]
|
||||||
cpu_blocks_expanded = cpu_blocks_expanded[blocks_to_skip:]
|
cpu_chunks_expanded = cpu_chunks_expanded[blocks_to_skip:]
|
||||||
|
|
||||||
# set transfer direction
|
# set transfer direction
|
||||||
if gpu_to_cpu:
|
if gpu_to_cpu:
|
||||||
@@ -330,16 +330,16 @@ def test_transfer(
|
|||||||
src_spec = GPULoadStoreSpec(
|
src_spec = GPULoadStoreSpec(
|
||||||
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
|
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
|
||||||
)
|
)
|
||||||
dst_spec = CPULoadStoreSpec(cpu_blocks)
|
dst_spec = CPULoadStoreSpec(cpu_chunks)
|
||||||
dst_to_src = dict(zip(cpu_blocks_expanded, gpu_blocks))
|
dst_to_src = dict(zip(cpu_chunks_expanded, gpu_blocks))
|
||||||
num_dst_sub_blocks = num_gpu_blocks
|
num_dst_sub_blocks = num_gpu_blocks
|
||||||
else:
|
else:
|
||||||
handler = worker._load_handler
|
handler = worker._load_handler
|
||||||
src_spec = CPULoadStoreSpec(cpu_blocks)
|
src_spec = CPULoadStoreSpec(cpu_chunks)
|
||||||
dst_spec = GPULoadStoreSpec(
|
dst_spec = GPULoadStoreSpec(
|
||||||
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
|
gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,)
|
||||||
)
|
)
|
||||||
dst_to_src = dict(zip(gpu_blocks, cpu_blocks_expanded))
|
dst_to_src = dict(zip(gpu_blocks, cpu_chunks_expanded))
|
||||||
num_dst_sub_blocks = num_gpu_blocks
|
num_dst_sub_blocks = num_gpu_blocks
|
||||||
|
|
||||||
# randomize src and dst tensors before transfer
|
# randomize src and dst tensors before transfer
|
||||||
@@ -410,7 +410,7 @@ def test_transfer(
|
|||||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||||
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
|
@pytest.mark.parametrize("blocks_per_chunk", BLOCKS_PER_CHUNK_VALUES)
|
||||||
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
|
@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS)
|
||||||
@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS)
|
@pytest.mark.parametrize("num_cpu_chunks", NUM_CPU_CHUNKS)
|
||||||
@pytest.mark.parametrize("seed", SEEDS)
|
@pytest.mark.parametrize("seed", SEEDS)
|
||||||
@pytest.mark.parametrize("device", DEVICES)
|
@pytest.mark.parametrize("device", DEVICES)
|
||||||
@torch.inference_mode()
|
@torch.inference_mode()
|
||||||
@@ -421,7 +421,7 @@ def test_transfer_multi_group(
|
|||||||
gpu_page_size_bytes: int,
|
gpu_page_size_bytes: int,
|
||||||
blocks_per_chunk: int,
|
blocks_per_chunk: int,
|
||||||
num_gpu_blocks: int,
|
num_gpu_blocks: int,
|
||||||
num_cpu_blocks: int,
|
num_cpu_chunks: int,
|
||||||
seed: int,
|
seed: int,
|
||||||
device: str,
|
device: str,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -470,37 +470,37 @@ def test_transfer_multi_group(
|
|||||||
worker = CPUOffloadingWorker(
|
worker = CPUOffloadingWorker(
|
||||||
kv_caches=canonical_kv_caches,
|
kv_caches=canonical_kv_caches,
|
||||||
blocks_per_chunk=blocks_per_chunk,
|
blocks_per_chunk=blocks_per_chunk,
|
||||||
num_cpu_blocks=num_cpu_blocks,
|
num_cpu_chunks=num_cpu_chunks,
|
||||||
)
|
)
|
||||||
|
|
||||||
# group 0: aligned, group 1: empty, group 2: unaligned on CPU->GPU
|
# group 0: aligned, group 1: empty, group 2: unaligned on CPU->GPU
|
||||||
group_sizes_in_cpu_blocks = [num_mappings_per_group, 0, num_mappings_per_group]
|
group_sizes_in_cpu_chunks = [num_mappings_per_group, 0, num_mappings_per_group]
|
||||||
|
|
||||||
total_cpu_blocks = sum(group_sizes_in_cpu_blocks)
|
total_cpu_chunks = sum(group_sizes_in_cpu_chunks)
|
||||||
total_gpu_blocks_needed = total_cpu_blocks * blocks_per_chunk
|
total_gpu_blocks_needed = total_cpu_chunks * blocks_per_chunk
|
||||||
gpu_blocks_all = random.sample(range(num_gpu_blocks), total_gpu_blocks_needed)
|
gpu_blocks_all = random.sample(range(num_gpu_blocks), total_gpu_blocks_needed)
|
||||||
cpu_blocks_all = random.sample(range(num_cpu_blocks), total_cpu_blocks)
|
cpu_chunks_all = random.sample(range(num_cpu_chunks), total_cpu_chunks)
|
||||||
|
|
||||||
# split gpu/cpu blocks per group
|
# split gpu blocks / cpu chunks per group
|
||||||
gpu_blocks_per_group: list[list[int]] = []
|
gpu_blocks_per_group: list[list[int]] = []
|
||||||
cpu_blocks_per_group: list[list[int]] = []
|
cpu_chunks_per_group: list[list[int]] = []
|
||||||
gpu_offset = 0
|
gpu_offset = 0
|
||||||
cpu_offset = 0
|
cpu_offset = 0
|
||||||
for size in group_sizes_in_cpu_blocks:
|
for size in group_sizes_in_cpu_chunks:
|
||||||
gpu_count = size * blocks_per_chunk
|
gpu_count = size * blocks_per_chunk
|
||||||
gpu_blocks_per_group.append(gpu_blocks_all[gpu_offset : gpu_offset + gpu_count])
|
gpu_blocks_per_group.append(gpu_blocks_all[gpu_offset : gpu_offset + gpu_count])
|
||||||
cpu_blocks_per_group.append(cpu_blocks_all[cpu_offset : cpu_offset + size])
|
cpu_chunks_per_group.append(cpu_chunks_all[cpu_offset : cpu_offset + size])
|
||||||
gpu_offset += gpu_count
|
gpu_offset += gpu_count
|
||||||
cpu_offset += size
|
cpu_offset += size
|
||||||
|
|
||||||
# expand cpu blocks to gpu-page granularity
|
# expand cpu chunks to gpu-page granularity
|
||||||
cpu_blocks_expanded_per_group = [
|
cpu_chunks_expanded_per_group = [
|
||||||
[
|
[
|
||||||
cpu_block * blocks_per_chunk + j
|
cpu_chunk * blocks_per_chunk + j
|
||||||
for cpu_block in cpu_blocks
|
for cpu_chunk in cpu_chunks
|
||||||
for j in range(blocks_per_chunk)
|
for j in range(blocks_per_chunk)
|
||||||
]
|
]
|
||||||
for cpu_blocks in cpu_blocks_per_group
|
for cpu_chunks in cpu_chunks_per_group
|
||||||
]
|
]
|
||||||
|
|
||||||
# skip sub-blocks from group 2 to test unaligned transfers.
|
# skip sub-blocks from group 2 to test unaligned transfers.
|
||||||
@@ -509,7 +509,7 @@ def test_transfer_multi_group(
|
|||||||
gpu_blocks_per_group[2] = gpu_blocks_per_group[2][
|
gpu_blocks_per_group[2] = gpu_blocks_per_group[2][
|
||||||
sub_blocks_to_skip:-sub_blocks_to_skip
|
sub_blocks_to_skip:-sub_blocks_to_skip
|
||||||
]
|
]
|
||||||
cpu_blocks_expanded_per_group[2] = cpu_blocks_expanded_per_group[2][
|
cpu_chunks_expanded_per_group[2] = cpu_chunks_expanded_per_group[2][
|
||||||
sub_blocks_to_skip:-sub_blocks_to_skip
|
sub_blocks_to_skip:-sub_blocks_to_skip
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -520,10 +520,10 @@ def test_transfer_multi_group(
|
|||||||
gpu_blocks.extend(gpu_blks)
|
gpu_blocks.extend(gpu_blks)
|
||||||
group_sizes.append(len(gpu_blks))
|
group_sizes.append(len(gpu_blks))
|
||||||
|
|
||||||
# build flat cpu_blocks list
|
# build flat cpu_chunks list
|
||||||
cpu_blocks = []
|
cpu_chunks = []
|
||||||
for cpu_blks in cpu_blocks_per_group:
|
for cpu_chnks in cpu_chunks_per_group:
|
||||||
cpu_blocks.extend(cpu_blks)
|
cpu_chunks.extend(cpu_chnks)
|
||||||
|
|
||||||
# block_indices: only relevant for unaligned transfers
|
# block_indices: only relevant for unaligned transfers
|
||||||
block_indices: list[int] = [0, 0, sub_blocks_to_skip]
|
block_indices: list[int] = [0, 0, sub_blocks_to_skip]
|
||||||
@@ -533,18 +533,18 @@ def test_transfer_multi_group(
|
|||||||
src_spec = GPULoadStoreSpec(
|
src_spec = GPULoadStoreSpec(
|
||||||
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
|
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
|
||||||
)
|
)
|
||||||
dst_spec = CPULoadStoreSpec(cpu_blocks)
|
dst_spec = CPULoadStoreSpec(cpu_chunks)
|
||||||
# per-group mapping: cpu sub-block -> gpu sub-block
|
# per-group mapping: cpu sub-block -> gpu sub-block
|
||||||
dst_to_src_per_group = [
|
dst_to_src_per_group = [
|
||||||
dict(zip(expanded, gpu_blks))
|
dict(zip(expanded, gpu_blks))
|
||||||
for expanded, gpu_blks in zip(
|
for expanded, gpu_blks in zip(
|
||||||
cpu_blocks_expanded_per_group, gpu_blocks_per_group
|
cpu_chunks_expanded_per_group, gpu_blocks_per_group
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
num_dst_sub_blocks = num_cpu_blocks * blocks_per_chunk
|
num_dst_sub_blocks = num_cpu_chunks * blocks_per_chunk
|
||||||
else:
|
else:
|
||||||
handler = worker._load_handler
|
handler = worker._load_handler
|
||||||
src_spec = CPULoadStoreSpec(cpu_blocks)
|
src_spec = CPULoadStoreSpec(cpu_chunks)
|
||||||
dst_spec = GPULoadStoreSpec(
|
dst_spec = GPULoadStoreSpec(
|
||||||
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
|
gpu_blocks, group_sizes=group_sizes, block_indices=block_indices
|
||||||
)
|
)
|
||||||
@@ -552,7 +552,7 @@ def test_transfer_multi_group(
|
|||||||
dst_to_src_per_group = [
|
dst_to_src_per_group = [
|
||||||
dict(zip(gpu_blks, expanded))
|
dict(zip(gpu_blks, expanded))
|
||||||
for gpu_blks, expanded in zip(
|
for gpu_blks, expanded in zip(
|
||||||
gpu_blocks_per_group, cpu_blocks_expanded_per_group
|
gpu_blocks_per_group, cpu_chunks_expanded_per_group
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
num_dst_sub_blocks = num_gpu_blocks
|
num_dst_sub_blocks = num_gpu_blocks
|
||||||
@@ -644,7 +644,7 @@ def test_load_waits_for_pending_compute_stream_writes(default_vllm_config) -> No
|
|||||||
],
|
],
|
||||||
),
|
),
|
||||||
blocks_per_chunk=1,
|
blocks_per_chunk=1,
|
||||||
num_cpu_blocks=num_blocks,
|
num_cpu_chunks=num_blocks,
|
||||||
)
|
)
|
||||||
worker._load_handler.src_tensors[0].fill_(sentinel)
|
worker._load_handler.src_tensors[0].fill_(sentinel)
|
||||||
expected = torch.full((page_size_bytes,), sentinel, dtype=torch.int8)
|
expected = torch.full((page_size_bytes,), sentinel, dtype=torch.int8)
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ _EMPTY_REQ_CTX = make_req_context()
|
|||||||
|
|
||||||
|
|
||||||
def make_cpu_manager(
|
def make_cpu_manager(
|
||||||
num_blocks: int = 4,
|
num_chunks: int = 4,
|
||||||
cache_policy: str = "lru",
|
cache_policy: str = "lru",
|
||||||
cache_policy_module_path: str | None = None,
|
cache_policy_module_path: str | None = None,
|
||||||
enable_events: bool = False,
|
enable_events: bool = False,
|
||||||
@@ -44,7 +44,7 @@ def make_cpu_manager(
|
|||||||
max_tracker_size: int = 64_000,
|
max_tracker_size: int = 64_000,
|
||||||
) -> CPUOffloadingManager:
|
) -> CPUOffloadingManager:
|
||||||
return CPUOffloadingManager(
|
return CPUOffloadingManager(
|
||||||
num_blocks=num_blocks,
|
num_chunks=num_chunks,
|
||||||
cache_policy=cache_policy,
|
cache_policy=cache_policy,
|
||||||
cache_policy_module_path=cache_policy_module_path,
|
cache_policy_module_path=cache_policy_module_path,
|
||||||
enable_events=enable_events,
|
enable_events=enable_events,
|
||||||
@@ -56,7 +56,7 @@ def make_cpu_manager(
|
|||||||
@dataclass
|
@dataclass
|
||||||
class ExpectedPrepareStoreOutput:
|
class ExpectedPrepareStoreOutput:
|
||||||
keys_to_store: list[int]
|
keys_to_store: list[int]
|
||||||
store_block_ids: list[int]
|
store_chunk_ids: list[int]
|
||||||
evicted_keys: list[int]
|
evicted_keys: list[int]
|
||||||
|
|
||||||
|
|
||||||
@@ -93,9 +93,9 @@ def verify_store_output(
|
|||||||
store_spec = prepare_store_output.store_spec
|
store_spec = prepare_store_output.store_spec
|
||||||
assert isinstance(store_spec, CPULoadStoreSpec)
|
assert isinstance(store_spec, CPULoadStoreSpec)
|
||||||
expected_array = np.array(
|
expected_array = np.array(
|
||||||
expected_prepare_store_output.store_block_ids, dtype=np.int64
|
expected_prepare_store_output.store_chunk_ids, dtype=np.int64
|
||||||
)
|
)
|
||||||
assert np.array_equal(expected_array, store_spec.block_ids)
|
assert np.array_equal(expected_array, store_spec.chunk_ids)
|
||||||
|
|
||||||
|
|
||||||
def verify_load_output(
|
def verify_load_output(
|
||||||
@@ -103,7 +103,7 @@ def verify_load_output(
|
|||||||
):
|
):
|
||||||
assert isinstance(prepare_load_output, CPULoadStoreSpec)
|
assert isinstance(prepare_load_output, CPULoadStoreSpec)
|
||||||
expected_array = np.array(expected_prepare_load_output, dtype=np.int64)
|
expected_array = np.array(expected_prepare_load_output, dtype=np.int64)
|
||||||
assert np.array_equal(expected_array, prepare_load_output.block_ids)
|
assert np.array_equal(expected_array, prepare_load_output.chunk_ids)
|
||||||
|
|
||||||
|
|
||||||
def check_split_usage_stats(
|
def check_split_usage_stats(
|
||||||
@@ -146,7 +146,7 @@ def verify_events(
|
|||||||
|
|
||||||
def test_cpu_eviction_removed_precedes_stored():
|
def test_cpu_eviction_removed_precedes_stored():
|
||||||
"""An eviction is announced before the store that reuses its capacity."""
|
"""An eviction is announced before the store that reuses its capacity."""
|
||||||
manager = make_cpu_manager(num_blocks=2, enable_events=True)
|
manager = make_cpu_manager(num_chunks=2, enable_events=True)
|
||||||
|
|
||||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
@@ -164,23 +164,23 @@ def test_cpu_eviction_removed_precedes_stored():
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("eviction_policy", ["lru", "arc"])
|
@pytest.mark.parametrize("eviction_policy", ["lru", "arc"])
|
||||||
def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
|
def test_already_stored_chunk_not_evicted_during_prepare_store(eviction_policy):
|
||||||
"""
|
"""
|
||||||
Regression test: a block that is already stored must not be evicted
|
Regression test: a chunk that is already stored must not be evicted
|
||||||
by prepare_store() when it needs to make room for new blocks.
|
by prepare_store() when it needs to make room for new chunks.
|
||||||
Applies to both lru and arc policies.
|
Applies to both lru and arc policies.
|
||||||
|
|
||||||
Scenario:
|
Scenario:
|
||||||
- Store blocks [1, 2] and complete.
|
- Store chunks [1, 2] and complete.
|
||||||
- touch([1]) makes block 2 the LRU candidate.
|
- touch([1]) makes chunk 2 the LRU candidate.
|
||||||
- prepare_store([2, 3, 4, 5]):
|
- prepare_store([2, 3, 4, 5]):
|
||||||
* block 2 is filtered out as "already stored"
|
* chunk 2 is filtered out as "already stored"
|
||||||
* but without the fix, block 2 would be evicted as the LRU
|
* but without the fix, chunk 2 would be evicted as the LRU
|
||||||
candidate to make room for [3, 4, 5]
|
candidate to make room for [3, 4, 5]
|
||||||
- After complete_store([2, 3, 4, 5]), block 2 must still be present.
|
- After complete_store([2, 3, 4, 5]), chunk 2 must still be present.
|
||||||
"""
|
"""
|
||||||
manager = make_cpu_manager(
|
manager = make_cpu_manager(
|
||||||
num_blocks=4,
|
num_chunks=4,
|
||||||
cache_policy=eviction_policy,
|
cache_policy=eviction_policy,
|
||||||
enable_events=True,
|
enable_events=True,
|
||||||
)
|
)
|
||||||
@@ -189,33 +189,33 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
|
|||||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# touch [1] to make block 2 the LRU candidate
|
# touch [1] to make chunk 2 the LRU candidate
|
||||||
manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
|
manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# prepare_store([2, 3, 4, 5]):
|
# prepare_store([2, 3, 4, 5]):
|
||||||
# - block 2 is already stored -> filtered out of keys_to_store
|
# - chunk 2 is already stored -> filtered out of keys_to_store
|
||||||
# - block 2 must NOT be evicted even though it is the LRU candidate
|
# - chunk 2 must NOT be evicted even though it is the LRU candidate
|
||||||
# - block 1 (ID 0) is evicted instead; new blocks [3,4,5] get IDs 2,3,0
|
# - chunk 1 (ID 0) is evicted instead; new chunks [3,4,5] get IDs 2,3,0
|
||||||
prepare_store_output = manager.prepare_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
|
prepare_store_output = manager.prepare_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
|
||||||
verify_store_output(
|
verify_store_output(
|
||||||
prepare_store_output,
|
prepare_store_output,
|
||||||
ExpectedPrepareStoreOutput(
|
ExpectedPrepareStoreOutput(
|
||||||
keys_to_store=[3, 4, 5],
|
keys_to_store=[3, 4, 5],
|
||||||
store_block_ids=[2, 3, 0],
|
store_chunk_ids=[2, 3, 0],
|
||||||
evicted_keys=[1], # block 1 evicted, not block 2
|
evicted_keys=[1], # chunk 1 evicted, not chunk 2
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# complete_store must not silently drop block 2
|
# complete_store must not silently drop chunk 2
|
||||||
manager.complete_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
|
manager.complete_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# block 2 must still be present in the cache
|
# chunk 2 must still be present in the cache
|
||||||
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
|
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
|
||||||
|
|
||||||
|
|
||||||
def test_filter_reused_manager_reports_stores_skipped_counter():
|
def test_filter_reused_manager_reports_stores_skipped_counter():
|
||||||
manager = make_cpu_manager(
|
manager = make_cpu_manager(
|
||||||
num_blocks=4,
|
num_chunks=4,
|
||||||
cache_policy="lru",
|
cache_policy="lru",
|
||||||
store_threshold=2,
|
store_threshold=2,
|
||||||
)
|
)
|
||||||
@@ -226,7 +226,7 @@ def test_filter_reused_manager_reports_stores_skipped_counter():
|
|||||||
prepare_store_output,
|
prepare_store_output,
|
||||||
ExpectedPrepareStoreOutput(
|
ExpectedPrepareStoreOutput(
|
||||||
keys_to_store=[],
|
keys_to_store=[],
|
||||||
store_block_ids=[],
|
store_chunk_ids=[],
|
||||||
evicted_keys=[],
|
evicted_keys=[],
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -247,34 +247,34 @@ def test_cpu_manager_reports_cache_usage_gauge():
|
|||||||
] == pytest.approx(value)
|
] == pytest.approx(value)
|
||||||
|
|
||||||
# Zero-capacity manager always reports 0.0
|
# Zero-capacity manager always reports 0.0
|
||||||
manager = make_cpu_manager(num_blocks=0)
|
manager = make_cpu_manager(num_chunks=0)
|
||||||
check_usage_stats(manager, 0.0)
|
check_usage_stats(manager, 0.0)
|
||||||
|
|
||||||
# Empty manager (4 blocks, none allocated): usage = 0.0
|
# Empty manager (4 chunks, none allocated): usage = 0.0
|
||||||
manager = make_cpu_manager(num_blocks=4)
|
manager = make_cpu_manager(num_chunks=4)
|
||||||
check_usage_stats(manager, 0.0)
|
check_usage_stats(manager, 0.0)
|
||||||
|
|
||||||
# After allocating 2 of 4 blocks: usage = 0.5
|
# After allocating 2 of 4 chunks: usage = 0.5
|
||||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
check_usage_stats(manager, 0.5)
|
check_usage_stats(manager, 0.5)
|
||||||
|
|
||||||
# After filling all 4 blocks: usage = 1.0
|
# After filling all 4 chunks: usage = 1.0
|
||||||
manager.prepare_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
|
manager.prepare_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
|
||||||
check_usage_stats(manager, 1.0)
|
check_usage_stats(manager, 1.0)
|
||||||
|
|
||||||
# After completing store, the blocks becomes evictable as it is not actively used
|
# After completing store, the chunks become evictable as not actively used
|
||||||
# and usage drops.
|
# and usage drops.
|
||||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
check_usage_stats(manager, 0.5)
|
check_usage_stats(manager, 0.5)
|
||||||
|
|
||||||
# After completing store, the blocks becomes evictable as it is not actively used
|
# After completing store, the chunks become evictable as not actively used
|
||||||
# and usage drops.
|
# and usage drops.
|
||||||
manager.complete_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
|
manager.complete_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
|
||||||
check_usage_stats(manager, 0.0)
|
check_usage_stats(manager, 0.0)
|
||||||
|
|
||||||
|
|
||||||
def test_cpu_manager_reports_allocation_size_histogram():
|
def test_cpu_manager_reports_allocation_size_histogram():
|
||||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||||
|
|
||||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
@@ -297,12 +297,12 @@ def test_cpu_manager_reports_allocation_size_histogram():
|
|||||||
|
|
||||||
|
|
||||||
def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
|
def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
|
||||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||||
|
|
||||||
def fail_allocate_blocks(keys):
|
def fail_allocate_chunks(keys):
|
||||||
raise RuntimeError("allocation failed")
|
raise RuntimeError("allocation failed")
|
||||||
|
|
||||||
monkeypatch.setattr(manager, "_allocate_blocks", fail_allocate_blocks)
|
monkeypatch.setattr(manager, "_allocate_chunks", fail_allocate_chunks)
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match="allocation failed"):
|
with pytest.raises(RuntimeError, match="allocation failed"):
|
||||||
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
||||||
@@ -316,7 +316,7 @@ def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
|
|||||||
|
|
||||||
|
|
||||||
def test_cpu_manager_reports_allocation_size_on_eviction_failure():
|
def test_cpu_manager_reports_allocation_size_on_eviction_failure():
|
||||||
manager = make_cpu_manager(num_blocks=1, cache_policy="lru")
|
manager = make_cpu_manager(num_chunks=1, cache_policy="lru")
|
||||||
|
|
||||||
manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
|
manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
|
||||||
manager.get_stats()
|
manager.get_stats()
|
||||||
@@ -332,7 +332,7 @@ def test_cpu_manager_reports_allocation_size_on_eviction_failure():
|
|||||||
|
|
||||||
|
|
||||||
def test_cpu_manager_reports_cache_write_and_read_usage_gauges():
|
def test_cpu_manager_reports_cache_write_and_read_usage_gauges():
|
||||||
manager = make_cpu_manager(num_blocks=4)
|
manager = make_cpu_manager(num_chunks=4)
|
||||||
|
|
||||||
# Store path: pins write usage until complete_store.
|
# Store path: pins write usage until complete_store.
|
||||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
@@ -356,7 +356,7 @@ def test_cpu_manager_reports_cache_write_and_read_usage_gauges():
|
|||||||
|
|
||||||
|
|
||||||
def test_cpu_manager_clears_write_usage_after_failed_store():
|
def test_cpu_manager_clears_write_usage_after_failed_store():
|
||||||
manager = make_cpu_manager(num_blocks=4)
|
manager = make_cpu_manager(num_chunks=4)
|
||||||
|
|
||||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
check_split_usage_stats(manager, write=0.5, read=0.0, total=0.5)
|
check_split_usage_stats(manager, write=0.5, read=0.0, total=0.5)
|
||||||
@@ -369,8 +369,8 @@ def test_cpu_manager():
|
|||||||
"""
|
"""
|
||||||
Tests CPUOffloadingManager with lru policy.
|
Tests CPUOffloadingManager with lru policy.
|
||||||
"""
|
"""
|
||||||
# initialize a CPU manager with a capacity of 4 blocks
|
# initialize a CPU manager with a capacity of 4 chunks
|
||||||
cpu_manager = make_cpu_manager(num_blocks=4, cache_policy="lru", enable_events=True)
|
cpu_manager = make_cpu_manager(num_chunks=4, cache_policy="lru", enable_events=True)
|
||||||
|
|
||||||
# prepare store [1, 2]
|
# prepare store [1, 2]
|
||||||
prepare_store_output = cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
prepare_store_output = cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
@@ -378,7 +378,7 @@ def test_cpu_manager():
|
|||||||
prepare_store_output,
|
prepare_store_output,
|
||||||
ExpectedPrepareStoreOutput(
|
ExpectedPrepareStoreOutput(
|
||||||
keys_to_store=[1, 2],
|
keys_to_store=[1, 2],
|
||||||
store_block_ids=[0, 1],
|
store_chunk_ids=[0, 1],
|
||||||
evicted_keys=[],
|
evicted_keys=[],
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -407,7 +407,7 @@ def test_cpu_manager():
|
|||||||
prepare_store_output,
|
prepare_store_output,
|
||||||
ExpectedPrepareStoreOutput(
|
ExpectedPrepareStoreOutput(
|
||||||
keys_to_store=[3, 4, 5],
|
keys_to_store=[3, 4, 5],
|
||||||
store_block_ids=[2, 3, 0],
|
store_chunk_ids=[2, 3, 0],
|
||||||
evicted_keys=[1],
|
evicted_keys=[1],
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -445,7 +445,7 @@ def test_cpu_manager():
|
|||||||
prepare_store_output,
|
prepare_store_output,
|
||||||
ExpectedPrepareStoreOutput(
|
ExpectedPrepareStoreOutput(
|
||||||
keys_to_store=[6, 7, 8],
|
keys_to_store=[6, 7, 8],
|
||||||
store_block_ids=[1, 0, 3],
|
store_chunk_ids=[1, 0, 3],
|
||||||
evicted_keys=[4, 5, 2],
|
evicted_keys=[4, 5, 2],
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -462,7 +462,7 @@ def test_cpu_manager():
|
|||||||
prepare_store_output,
|
prepare_store_output,
|
||||||
ExpectedPrepareStoreOutput(
|
ExpectedPrepareStoreOutput(
|
||||||
keys_to_store=[9],
|
keys_to_store=[9],
|
||||||
store_block_ids=[3],
|
store_chunk_ids=[3],
|
||||||
evicted_keys=[8],
|
evicted_keys=[8],
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -482,38 +482,38 @@ def test_cpu_manager():
|
|||||||
|
|
||||||
|
|
||||||
def test_prepare_load_preserves_key_order():
|
def test_prepare_load_preserves_key_order():
|
||||||
"""block_ids[i] must correspond to keys[i] (co-indexed invariant)."""
|
"""chunk_ids[i] must correspond to keys[i] (co-indexed invariant)."""
|
||||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||||
|
|
||||||
key_a, key_b, key_c = to_key(0), to_key(1), to_key(2)
|
key_a, key_b, key_c = to_key(0), to_key(1), to_key(2)
|
||||||
|
|
||||||
# Store all three keys and learn their block ID assignments
|
# Store all three keys and learn their chunk ID assignments
|
||||||
store_output = manager.prepare_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
store_output = manager.prepare_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
||||||
assert store_output is not None
|
assert store_output is not None
|
||||||
assert isinstance(store_output.store_spec, CPULoadStoreSpec)
|
assert isinstance(store_output.store_spec, CPULoadStoreSpec)
|
||||||
key_to_block_id = {
|
key_to_chunk_id = {
|
||||||
k: int(bid)
|
k: int(cid)
|
||||||
for k, bid in zip(store_output.keys_to_store, store_output.store_spec.block_ids)
|
for k, cid in zip(store_output.keys_to_store, store_output.store_spec.chunk_ids)
|
||||||
}
|
}
|
||||||
manager.complete_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
manager.complete_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# Forward order: [a, b, c]
|
# Forward order: [a, b, c]
|
||||||
spec_fwd = manager.prepare_load([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
spec_fwd = manager.prepare_load([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
||||||
assert isinstance(spec_fwd, CPULoadStoreSpec)
|
assert isinstance(spec_fwd, CPULoadStoreSpec)
|
||||||
assert [int(x) for x in spec_fwd.block_ids] == [
|
assert [int(x) for x in spec_fwd.chunk_ids] == [
|
||||||
key_to_block_id[key_a],
|
key_to_chunk_id[key_a],
|
||||||
key_to_block_id[key_b],
|
key_to_chunk_id[key_b],
|
||||||
key_to_block_id[key_c],
|
key_to_chunk_id[key_c],
|
||||||
]
|
]
|
||||||
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
|
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
|
||||||
|
|
||||||
# Arbitrary permutation: [b, c, a]
|
# Arbitrary permutation: [b, c, a]
|
||||||
spec_perm = manager.prepare_load([key_b, key_c, key_a], _EMPTY_REQ_CTX)
|
spec_perm = manager.prepare_load([key_b, key_c, key_a], _EMPTY_REQ_CTX)
|
||||||
assert isinstance(spec_perm, CPULoadStoreSpec)
|
assert isinstance(spec_perm, CPULoadStoreSpec)
|
||||||
assert [int(x) for x in spec_perm.block_ids] == [
|
assert [int(x) for x in spec_perm.chunk_ids] == [
|
||||||
key_to_block_id[key_b],
|
key_to_chunk_id[key_b],
|
||||||
key_to_block_id[key_c],
|
key_to_chunk_id[key_c],
|
||||||
key_to_block_id[key_a],
|
key_to_chunk_id[key_a],
|
||||||
]
|
]
|
||||||
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
|
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
|
||||||
|
|
||||||
@@ -522,10 +522,10 @@ class TestARCPolicy:
|
|||||||
"""Unit tests for CPUOffloadingManager with ARC eviction policy."""
|
"""Unit tests for CPUOffloadingManager with ARC eviction policy."""
|
||||||
|
|
||||||
def _make_manager(
|
def _make_manager(
|
||||||
self, num_blocks: int = 4, enable_events: bool = True
|
self, num_chunks: int = 4, enable_events: bool = True
|
||||||
) -> tuple[CPUOffloadingManager, ARCCachePolicy]:
|
) -> tuple[CPUOffloadingManager, ARCCachePolicy]:
|
||||||
manager = make_cpu_manager(
|
manager = make_cpu_manager(
|
||||||
num_blocks=num_blocks,
|
num_chunks=num_chunks,
|
||||||
cache_policy="arc",
|
cache_policy="arc",
|
||||||
enable_events=enable_events,
|
enable_events=enable_events,
|
||||||
)
|
)
|
||||||
@@ -548,7 +548,7 @@ class TestARCPolicy:
|
|||||||
prepare_store_output,
|
prepare_store_output,
|
||||||
ExpectedPrepareStoreOutput(
|
ExpectedPrepareStoreOutput(
|
||||||
keys_to_store=[1, 2],
|
keys_to_store=[1, 2],
|
||||||
store_block_ids=[0, 1],
|
store_chunk_ids=[0, 1],
|
||||||
evicted_keys=[],
|
evicted_keys=[],
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -569,36 +569,36 @@ class TestARCPolicy:
|
|||||||
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
|
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
|
||||||
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.MISS
|
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.MISS
|
||||||
|
|
||||||
# blocks should be in T1 (recent)
|
# chunks should be in T1 (recent)
|
||||||
assert len(arc_policy.t1) == 2
|
assert len(arc_policy.t1) == 2
|
||||||
assert len(arc_policy.t2) == 0
|
assert len(arc_policy.t2) == 0
|
||||||
|
|
||||||
def test_t1_to_t2_promotion(self):
|
def test_t1_to_t2_promotion(self):
|
||||||
"""
|
"""
|
||||||
Tests that accessing a block in T1 promotes it to T2 (frequent).
|
Tests that accessing a chunk in T1 promotes it to T2 (frequent).
|
||||||
This is a key feature of ARC's adaptive behavior.
|
This is a key feature of ARC's adaptive behavior.
|
||||||
"""
|
"""
|
||||||
cpu_manager, arc_policy = self._make_manager(enable_events=False)
|
cpu_manager, arc_policy = self._make_manager(enable_events=False)
|
||||||
|
|
||||||
# store and complete block 1
|
# store and complete chunk 1
|
||||||
cpu_manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
|
cpu_manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
|
||||||
cpu_manager.complete_store(to_keys([1]), _EMPTY_REQ_CTX)
|
cpu_manager.complete_store(to_keys([1]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# block 1 starts in T1 (recent)
|
# chunk 1 starts in T1 (recent)
|
||||||
assert to_keys([1])[0] in arc_policy.t1
|
assert to_keys([1])[0] in arc_policy.t1
|
||||||
assert to_keys([1])[0] not in arc_policy.t2
|
assert to_keys([1])[0] not in arc_policy.t2
|
||||||
|
|
||||||
# touch block 1 (simulate second access)
|
# touch chunk 1 (simulate second access)
|
||||||
cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
|
cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# block 1 should now be in T2 (frequent)
|
# chunk 1 should now be in T2 (frequent)
|
||||||
assert to_keys([1])[0] not in arc_policy.t1
|
assert to_keys([1])[0] not in arc_policy.t1
|
||||||
assert to_keys([1])[0] in arc_policy.t2
|
assert to_keys([1])[0] in arc_policy.t2
|
||||||
|
|
||||||
def test_eviction_with_load(self):
|
def test_eviction_with_load(self):
|
||||||
"""
|
"""
|
||||||
Tests ARC eviction behavior similar to LRU test.
|
Tests ARC eviction behavior similar to LRU test.
|
||||||
Verifies that blocks being loaded (ref_cnt > 0) cannot be evicted.
|
Verifies that chunks being loaded (ref_cnt > 0) cannot be evicted.
|
||||||
"""
|
"""
|
||||||
cpu_manager, _ = self._make_manager()
|
cpu_manager, _ = self._make_manager()
|
||||||
|
|
||||||
@@ -610,7 +610,7 @@ class TestARCPolicy:
|
|||||||
prepare_store_output,
|
prepare_store_output,
|
||||||
ExpectedPrepareStoreOutput(
|
ExpectedPrepareStoreOutput(
|
||||||
keys_to_store=[1, 2, 3, 4],
|
keys_to_store=[1, 2, 3, 4],
|
||||||
store_block_ids=[0, 1, 2, 3],
|
store_chunk_ids=[0, 1, 2, 3],
|
||||||
evicted_keys=[],
|
evicted_keys=[],
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -628,36 +628,36 @@ class TestARCPolicy:
|
|||||||
cpu_manager.complete_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
|
cpu_manager.complete_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# now prepare store [5, 6, 7] should succeed
|
# now prepare store [5, 6, 7] should succeed
|
||||||
# ARC will evict blocks one at a time from T1 as needed
|
# ARC will evict chunks one at a time from T1 as needed
|
||||||
prepare_store_output = cpu_manager.prepare_store(
|
prepare_store_output = cpu_manager.prepare_store(
|
||||||
to_keys([5, 6, 7]), _EMPTY_REQ_CTX
|
to_keys([5, 6, 7]), _EMPTY_REQ_CTX
|
||||||
)
|
)
|
||||||
assert prepare_store_output is not None
|
assert prepare_store_output is not None
|
||||||
# Should successfully evict enough blocks to make room (at least 1)
|
# Should successfully evict enough chunks to make room (at least 1)
|
||||||
assert len(prepare_store_output.evicted_keys) >= 1
|
assert len(prepare_store_output.evicted_keys) >= 1
|
||||||
|
|
||||||
def test_adaptive_target(self):
|
def test_adaptive_target(self):
|
||||||
"""
|
"""
|
||||||
Tests ARC's adaptive target adjustment via ghost lists.
|
Tests ARC's adaptive target adjustment via ghost lists.
|
||||||
When a block in B1 (ghost list) is accessed, target_t1_size increases.
|
When a chunk in B1 (ghost list) is accessed, target_t1_size increases.
|
||||||
When a block in B2 is accessed, target_t1_size decreases.
|
When a chunk in B2 is accessed, target_t1_size decreases.
|
||||||
"""
|
"""
|
||||||
cpu_manager, arc_policy = self._make_manager(num_blocks=2, enable_events=False)
|
cpu_manager, arc_policy = self._make_manager(num_chunks=2, enable_events=False)
|
||||||
|
|
||||||
# store blocks 1, 2 (fills cache)
|
# store chunks 1, 2 (fills cache)
|
||||||
cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
initial_target = arc_policy.target_t1_size
|
initial_target = arc_policy.target_t1_size
|
||||||
|
|
||||||
# store block 3, evicting block 1 (moves to B1 ghost list)
|
# store chunk 3, evicting chunk 1 (moves to B1 ghost list)
|
||||||
cpu_manager.prepare_store(to_keys([3]), _EMPTY_REQ_CTX)
|
cpu_manager.prepare_store(to_keys([3]), _EMPTY_REQ_CTX)
|
||||||
cpu_manager.complete_store(to_keys([3]), _EMPTY_REQ_CTX)
|
cpu_manager.complete_store(to_keys([3]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# block 1 should be in B1 (ghost list)
|
# chunk 1 should be in B1 (ghost list)
|
||||||
assert to_keys([1])[0] in arc_policy.b1
|
assert to_keys([1])[0] in arc_policy.b1
|
||||||
|
|
||||||
# touch block 1 (cache miss, but in B1)
|
# touch chunk 1 (cache miss, but in B1)
|
||||||
# this should increase target_t1_size (favor recency)
|
# this should increase target_t1_size (favor recency)
|
||||||
cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
|
cpu_manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
@@ -671,11 +671,11 @@ class TestARCPolicy:
|
|||||||
"""
|
"""
|
||||||
cpu_manager, arc_policy = self._make_manager(enable_events=False)
|
cpu_manager, arc_policy = self._make_manager(enable_events=False)
|
||||||
|
|
||||||
# store blocks 1, 2, 3, 4
|
# store chunks 1, 2, 3, 4
|
||||||
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||||
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# promote blocks 3, 4 to T2 by touching them
|
# promote chunks 3, 4 to T2 by touching them
|
||||||
cpu_manager.touch(to_keys([3, 4]), _EMPTY_REQ_CTX)
|
cpu_manager.touch(to_keys([3, 4]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# now: T1 = {1, 2}, T2 = {3, 4}
|
# now: T1 = {1, 2}, T2 = {3, 4}
|
||||||
@@ -686,22 +686,22 @@ class TestARCPolicy:
|
|||||||
# (when |T1| >= target, evict from T1)
|
# (when |T1| >= target, evict from T1)
|
||||||
arc_policy.target_t1_size = 1
|
arc_policy.target_t1_size = 1
|
||||||
|
|
||||||
# store block 5, should evict from T1 (block 1, LRU in T1)
|
# store chunk 5, should evict from T1 (chunk 1, LRU in T1)
|
||||||
output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
|
output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
|
||||||
assert output is not None
|
assert output is not None
|
||||||
assert to_keys([1]) == output.evicted_keys
|
assert to_keys([1]) == output.evicted_keys
|
||||||
|
|
||||||
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX)
|
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# block 1 should be in B1 (ghost list)
|
# chunk 1 should be in B1 (ghost list)
|
||||||
assert to_keys([1])[0] in arc_policy.b1
|
assert to_keys([1])[0] in arc_policy.b1
|
||||||
# block 5 should be in T1
|
# chunk 5 should be in T1
|
||||||
assert to_keys([5])[0] in arc_policy.t1
|
assert to_keys([5])[0] in arc_policy.t1
|
||||||
|
|
||||||
def test_batch_eviction_scans_t1_and_t2_once(self):
|
def test_batch_eviction_scans_t1_and_t2_once(self):
|
||||||
"""ARC batch eviction must preserve order without restarting scans."""
|
"""ARC batch eviction must preserve order without restarting scans."""
|
||||||
cpu_manager, arc_policy = self._make_manager(
|
cpu_manager, arc_policy = self._make_manager(
|
||||||
num_blocks=256, enable_events=False
|
num_chunks=256, enable_events=False
|
||||||
)
|
)
|
||||||
keys = to_keys(list(range(256)))
|
keys = to_keys(list(range(256)))
|
||||||
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
||||||
@@ -721,13 +721,13 @@ class TestARCPolicy:
|
|||||||
t2_order = list(arc_policy.t2)
|
t2_order = list(arc_policy.t2)
|
||||||
expected_t1 = [
|
expected_t1 = [
|
||||||
key
|
key
|
||||||
for key, block in arc_policy.t1.items()
|
for key, chunk in arc_policy.t1.items()
|
||||||
if block.ref_cnt == 0 and key not in protected
|
if chunk.ref_cnt == 0 and key not in protected
|
||||||
][:num_t1_evictions]
|
][:num_t1_evictions]
|
||||||
expected_t2 = [
|
expected_t2 = [
|
||||||
key
|
key
|
||||||
for key, block in arc_policy.t2.items()
|
for key, chunk in arc_policy.t2.items()
|
||||||
if block.ref_cnt == 0 and key not in protected
|
if chunk.ref_cnt == 0 and key not in protected
|
||||||
][:num_t2_evictions]
|
][:num_t2_evictions]
|
||||||
expected_t1_scans = t1_order.index(expected_t1[-1]) + 1
|
expected_t1_scans = t1_order.index(expected_t1[-1]) + 1
|
||||||
expected_t2_scans = t2_order.index(expected_t2[-1]) + 1
|
expected_t2_scans = t2_order.index(expected_t2[-1]) + 1
|
||||||
@@ -746,7 +746,7 @@ class TestARCPolicy:
|
|||||||
|
|
||||||
def test_batch_eviction_falls_back_after_t1_iterator_exhausted(self):
|
def test_batch_eviction_falls_back_after_t1_iterator_exhausted(self):
|
||||||
"""An exhausted T1 scan must keep falling back to T2."""
|
"""An exhausted T1 scan must keep falling back to T2."""
|
||||||
cpu_manager, arc_policy = self._make_manager(num_blocks=8, enable_events=False)
|
cpu_manager, arc_policy = self._make_manager(num_chunks=8, enable_events=False)
|
||||||
keys = to_keys(list(range(8)))
|
keys = to_keys(list(range(8)))
|
||||||
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
||||||
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
|
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
|
||||||
@@ -764,8 +764,8 @@ class TestARCPolicy:
|
|||||||
# the target, so each remaining selection must retry T1 then use T2.
|
# the target, so each remaining selection must retry T1 then use T2.
|
||||||
eligible_t1 = [
|
eligible_t1 = [
|
||||||
key
|
key
|
||||||
for key, block in arc_policy.t1.items()
|
for key, chunk in arc_policy.t1.items()
|
||||||
if block.ref_cnt == 0 and key not in protected
|
if chunk.ref_cnt == 0 and key not in protected
|
||||||
]
|
]
|
||||||
assert eligible_t1 == t1_order[:1]
|
assert eligible_t1 == t1_order[:1]
|
||||||
assert len(t1_order) - 1 >= int(arc_policy.target_t1_size)
|
assert len(t1_order) - 1 >= int(arc_policy.target_t1_size)
|
||||||
@@ -839,7 +839,7 @@ class TestARCPolicy:
|
|||||||
|
|
||||||
def test_batch_eviction_failure_is_atomic(self):
|
def test_batch_eviction_failure_is_atomic(self):
|
||||||
"""Finding only some candidates must not partially evict the cache."""
|
"""Finding only some candidates must not partially evict the cache."""
|
||||||
cpu_manager, arc_policy = self._make_manager(num_blocks=4, enable_events=False)
|
cpu_manager, arc_policy = self._make_manager(num_chunks=4, enable_events=False)
|
||||||
keys = to_keys(list(range(4)))
|
keys = to_keys(list(range(4)))
|
||||||
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
||||||
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
|
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
|
||||||
@@ -858,13 +858,13 @@ class TestARCPolicy:
|
|||||||
Tests that ghost lists (B1, B2) don't grow unbounded.
|
Tests that ghost lists (B1, B2) don't grow unbounded.
|
||||||
They should be capped at cache_capacity.
|
They should be capped at cache_capacity.
|
||||||
"""
|
"""
|
||||||
cpu_manager, arc_policy = self._make_manager(num_blocks=2, enable_events=False)
|
cpu_manager, arc_policy = self._make_manager(num_chunks=2, enable_events=False)
|
||||||
|
|
||||||
# fill cache with blocks 1, 2
|
# fill cache with chunks 1, 2
|
||||||
cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
cpu_manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# store many blocks to fill ghost lists
|
# store many chunks to fill ghost lists
|
||||||
for i in range(3, 20):
|
for i in range(3, 20):
|
||||||
cpu_manager.prepare_store(to_keys([i]), _EMPTY_REQ_CTX)
|
cpu_manager.prepare_store(to_keys([i]), _EMPTY_REQ_CTX)
|
||||||
cpu_manager.complete_store(to_keys([i]), _EMPTY_REQ_CTX)
|
cpu_manager.complete_store(to_keys([i]), _EMPTY_REQ_CTX)
|
||||||
@@ -880,7 +880,7 @@ class TestARCPolicy:
|
|||||||
"""
|
"""
|
||||||
cpu_manager, arc_policy = self._make_manager()
|
cpu_manager, arc_policy = self._make_manager()
|
||||||
|
|
||||||
# store blocks 1, 2, 3, 4
|
# store chunks 1, 2, 3, 4
|
||||||
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||||
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
@@ -895,13 +895,13 @@ class TestARCPolicy:
|
|||||||
assert len(arc_policy.t1) == 1
|
assert len(arc_policy.t1) == 1
|
||||||
assert len(arc_policy.t2) == 3
|
assert len(arc_policy.t2) == 3
|
||||||
|
|
||||||
# store block 5, should evict from T1 (block 2, only one in T1)
|
# store chunk 5, should evict from T1 (chunk 2, only one in T1)
|
||||||
prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
|
prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
|
||||||
verify_store_output(
|
verify_store_output(
|
||||||
prepare_store_output,
|
prepare_store_output,
|
||||||
ExpectedPrepareStoreOutput(
|
ExpectedPrepareStoreOutput(
|
||||||
keys_to_store=[5],
|
keys_to_store=[5],
|
||||||
store_block_ids=[1], # reuses block 2's storage
|
store_chunk_ids=[1], # reuses chunk 2's storage
|
||||||
evicted_keys=[2],
|
evicted_keys=[2],
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -913,11 +913,11 @@ class TestARCPolicy:
|
|||||||
"""
|
"""
|
||||||
cpu_manager, arc_policy = self._make_manager()
|
cpu_manager, arc_policy = self._make_manager()
|
||||||
|
|
||||||
# store blocks 1, 2, 3, 4
|
# store chunks 1, 2, 3, 4
|
||||||
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
cpu_manager.prepare_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||||
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
cpu_manager.complete_store(to_keys([1, 2, 3, 4]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# prepare store block 5 (will evict block 1)
|
# prepare store chunk 5 (will evict chunk 1)
|
||||||
prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
|
prepare_store_output = cpu_manager.prepare_store(to_keys([5]), _EMPTY_REQ_CTX)
|
||||||
assert prepare_store_output is not None
|
assert prepare_store_output is not None
|
||||||
assert len(prepare_store_output.evicted_keys) == 1
|
assert len(prepare_store_output.evicted_keys) == 1
|
||||||
@@ -925,13 +925,13 @@ class TestARCPolicy:
|
|||||||
# complete store with failure
|
# complete store with failure
|
||||||
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX, success=False)
|
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX, success=False)
|
||||||
|
|
||||||
# block 5 should not be in cache
|
# chunk 5 should not be in cache
|
||||||
assert cpu_manager.lookup(to_key(5), _EMPTY_REQ_CTX) is LookupResult.MISS
|
assert cpu_manager.lookup(to_key(5), _EMPTY_REQ_CTX) is LookupResult.MISS
|
||||||
# block 5 should not be in T1 or T2
|
# chunk 5 should not be in T1 or T2
|
||||||
assert to_keys([5])[0] not in arc_policy.t1
|
assert to_keys([5])[0] not in arc_policy.t1
|
||||||
assert to_keys([5])[0] not in arc_policy.t2
|
assert to_keys([5])[0] not in arc_policy.t2
|
||||||
|
|
||||||
# evicted block should still be gone (in B1 ghost list)
|
# evicted chunk should still be gone (in B1 ghost list)
|
||||||
evicted_hash = prepare_store_output.evicted_keys[0]
|
evicted_hash = prepare_store_output.evicted_keys[0]
|
||||||
assert evicted_hash in arc_policy.b1
|
assert evicted_hash in arc_policy.b1
|
||||||
|
|
||||||
@@ -954,7 +954,7 @@ class TestARCPolicy:
|
|||||||
assert len(prepare_store_output.evicted_keys) == 1
|
assert len(prepare_store_output.evicted_keys) == 1
|
||||||
cpu_manager.complete_store(to_keys([3, 4, 5]), _EMPTY_REQ_CTX)
|
cpu_manager.complete_store(to_keys([3, 4, 5]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# promote some blocks to T2
|
# promote some chunks to T2
|
||||||
cpu_manager.touch(to_keys([2, 3]), _EMPTY_REQ_CTX)
|
cpu_manager.touch(to_keys([2, 3]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# T1 has {4, 5}, T2 has {2, 3}
|
# T1 has {4, 5}, T2 has {2, 3}
|
||||||
@@ -966,7 +966,7 @@ class TestARCPolicy:
|
|||||||
assert prepare_store_output is not None
|
assert prepare_store_output is not None
|
||||||
cpu_manager.complete_store(to_keys([6]), _EMPTY_REQ_CTX)
|
cpu_manager.complete_store(to_keys([6]), _EMPTY_REQ_CTX)
|
||||||
|
|
||||||
# verify blocks 2, 3 (in T2) are still present
|
# verify chunks 2, 3 (in T2) are still present
|
||||||
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
|
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
|
||||||
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.HIT
|
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.HIT
|
||||||
|
|
||||||
@@ -980,7 +980,7 @@ def test_filter_reused_manager():
|
|||||||
Tests CPUOffloadingManager reuse filtering (store_threshold=2).
|
Tests CPUOffloadingManager reuse filtering (store_threshold=2).
|
||||||
"""
|
"""
|
||||||
manager = make_cpu_manager(
|
manager = make_cpu_manager(
|
||||||
num_blocks=4,
|
num_chunks=4,
|
||||||
cache_policy="lru",
|
cache_policy="lru",
|
||||||
enable_events=True,
|
enable_events=True,
|
||||||
store_threshold=2,
|
store_threshold=2,
|
||||||
@@ -1023,7 +1023,7 @@ def test_filter_reused_manager():
|
|||||||
def test_filter_reused_manager_oversized_offer_makes_progress():
|
def test_filter_reused_manager_oversized_offer_makes_progress():
|
||||||
"""An offer larger than the tracker capacity must not churn forever."""
|
"""An offer larger than the tracker capacity must not churn forever."""
|
||||||
manager = make_cpu_manager(
|
manager = make_cpu_manager(
|
||||||
num_blocks=4,
|
num_chunks=4,
|
||||||
cache_policy="lru",
|
cache_policy="lru",
|
||||||
store_threshold=2,
|
store_threshold=2,
|
||||||
max_tracker_size=3,
|
max_tracker_size=3,
|
||||||
@@ -1042,69 +1042,69 @@ def test_filter_reused_manager_oversized_offer_makes_progress():
|
|||||||
assert stored_keys == set(keys)
|
assert stored_keys == set(keys)
|
||||||
|
|
||||||
|
|
||||||
def test_evictable_cache_block_count():
|
def test_evictable_cache_chunk_count():
|
||||||
"""
|
"""
|
||||||
Verifies _num_evictable_cache_blocks is maintained correctly through the
|
Verifies _num_evictable_cache_chunks is maintained correctly through the
|
||||||
full store/load lifecycle, eviction, failed stores, concurrent loads,
|
full store/load lifecycle, eviction, failed stores, concurrent loads,
|
||||||
reset_cache, and the early-exit fast path in prepare_store.
|
reset_cache, and the early-exit fast path in prepare_store.
|
||||||
"""
|
"""
|
||||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||||
|
|
||||||
# Initially no blocks allocated.
|
# Initially no chunks allocated.
|
||||||
assert manager._num_evictable_cache_blocks == 0
|
assert manager._num_evictable_cache_chunks == 0
|
||||||
|
|
||||||
# Initial cache state [x, x, x, x]
|
# Initial cache state [x, x, x, x]
|
||||||
|
|
||||||
# We get 3 blocks from the cache.
|
# We get 3 chunks from the cache.
|
||||||
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
||||||
# cache state [1', 2', 3', x] <- 1', 2', 3' are actively being used.
|
# cache state [1', 2', 3', x] <- 1', 2', 3' are actively being used.
|
||||||
assert manager._num_evictable_cache_blocks == 0
|
assert manager._num_evictable_cache_chunks == 0
|
||||||
|
|
||||||
# Completing stores makes them idle.
|
# Completing stores makes them idle.
|
||||||
manager.complete_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
manager.complete_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
||||||
# cache state [1, 2, 3, x] <- 1, 2, 3 blocks are idle.
|
# cache state [1, 2, 3, x] <- 1, 2, 3 chunks are idle.
|
||||||
assert manager._num_evictable_cache_blocks == 3
|
assert manager._num_evictable_cache_chunks == 3
|
||||||
|
|
||||||
# prepare_load pins a block: idle count decrements once even if the
|
# prepare_load pins a chunk: idle count decrements once even if the
|
||||||
# same block is loaded by two concurrent callers.
|
# same chunk is loaded by two concurrent callers.
|
||||||
manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX)
|
manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX)
|
||||||
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
|
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
|
||||||
assert manager._num_evictable_cache_blocks == 2
|
assert manager._num_evictable_cache_chunks == 2
|
||||||
manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX) # 2nd concurrent load
|
manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX) # 2nd concurrent load
|
||||||
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
|
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
|
||||||
assert manager._num_evictable_cache_blocks == 2 # no double-decrement
|
assert manager._num_evictable_cache_chunks == 2 # no double-decrement
|
||||||
|
|
||||||
# First complete_load does not restore idle (ref_cnt still 1).
|
# First complete_load does not restore idle (ref_cnt still 1).
|
||||||
manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX)
|
manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX)
|
||||||
# cache state [1', 2, 3, x] <- 2, 3 blocks are idle.
|
# cache state [1', 2, 3, x] <- 2, 3 chunks are idle.
|
||||||
assert manager._num_evictable_cache_blocks == 2
|
assert manager._num_evictable_cache_chunks == 2
|
||||||
# Second complete_load drops ref_cnt to 0 -> block becomes idle again.
|
# Second complete_load drops ref_cnt to 0 -> chunk becomes idle again.
|
||||||
manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX)
|
manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX)
|
||||||
# cache state [1, 2, 3, x] <- 1, 2, 3 blocks are idle.
|
# cache state [1, 2, 3, x] <- 1, 2, 3 chunks are idle.
|
||||||
assert manager._num_evictable_cache_blocks == 3
|
assert manager._num_evictable_cache_chunks == 3
|
||||||
|
|
||||||
# Eviction decrements idle count.
|
# Eviction decrements idle count.
|
||||||
# Cache has 3 stored blocks and 1 free slot. Storing 3 new keys needs 2 eviction.
|
# Cache has 3 stored chunks and 1 free slot. Storing 3 new keys needs 2 eviction.
|
||||||
manager.prepare_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX)
|
manager.prepare_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX)
|
||||||
# cache state [1, 4', 5', 6'] <- block 1 is idle
|
# cache state [1, 4', 5', 6'] <- chunk 1 is idle
|
||||||
assert manager._num_evictable_cache_blocks == 1
|
assert manager._num_evictable_cache_chunks == 1
|
||||||
|
|
||||||
# Failed store does not increment idle count (block discarded from cache).
|
# Failed store does not increment idle count (chunk discarded from cache).
|
||||||
manager.complete_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX, success=False)
|
manager.complete_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX, success=False)
|
||||||
# cache state [1, x, x, x] <- block 1 is idle. Other returned to cache.
|
# cache state [1, x, x, x] <- chunk 1 is idle. Other returned to cache.
|
||||||
assert manager._num_evictable_cache_blocks == 1
|
assert manager._num_evictable_cache_chunks == 1
|
||||||
|
|
||||||
# reset_cache zeroes the count unconditionally.
|
# reset_cache zeroes the count unconditionally.
|
||||||
manager.reset_cache()
|
manager.reset_cache()
|
||||||
# cache state [x, x, x, x]
|
# cache state [x, x, x, x]
|
||||||
assert manager._num_evictable_cache_blocks == 0
|
assert manager._num_evictable_cache_chunks == 0
|
||||||
|
|
||||||
# setup 3 blocks with loads so idle count drops to 0.
|
# setup 3 chunks with loads so idle count drops to 0.
|
||||||
manager.prepare_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
manager.prepare_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
||||||
manager.complete_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
manager.complete_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
||||||
manager.prepare_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
manager.prepare_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
||||||
# cache state [10', 11', 12', x]
|
# cache state [10', 11', 12', x]
|
||||||
assert manager._num_evictable_cache_blocks == 0
|
assert manager._num_evictable_cache_chunks == 0
|
||||||
|
|
||||||
# prepare_store requiring eviction must return None immediately (fast exit).
|
# prepare_store requiring eviction must return None immediately (fast exit).
|
||||||
# Spy on policy.evict to confirm the fast path short-circuits before calling it.
|
# Spy on policy.evict to confirm the fast path short-circuits before calling it.
|
||||||
@@ -1120,24 +1120,24 @@ def test_evictable_cache_block_count():
|
|||||||
# cache state [10', 11', 12', x] <- cannot evict anything
|
# cache state [10', 11', 12', x] <- cannot evict anything
|
||||||
assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is None
|
assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is None
|
||||||
assert not evict_called, (
|
assert not evict_called, (
|
||||||
"_num_evictable_cache_blocks==0 should short-circuit before evict()"
|
"_num_evictable_cache_chunks==0 should short-circuit before evict()"
|
||||||
)
|
)
|
||||||
|
|
||||||
# After releasing the loads, eviction becomes possible again.
|
# After releasing the loads, eviction becomes possible again.
|
||||||
manager.complete_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
manager.complete_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX)
|
||||||
# cache state [10, 11, 12, x] <- 10, 11, 12 are idle
|
# cache state [10, 11, 12, x] <- 10, 11, 12 are idle
|
||||||
assert manager._num_evictable_cache_blocks == 3
|
assert manager._num_evictable_cache_chunks == 3
|
||||||
assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is not None
|
assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is not None
|
||||||
# cache state [10, 11, 14', 15'] <- 10, 11 are idle
|
# cache state [10, 11, 14', 15'] <- 10, 11 are idle
|
||||||
assert manager._num_evictable_cache_blocks == 2
|
assert manager._num_evictable_cache_chunks == 2
|
||||||
manager.complete_store(to_keys([14, 15]), _EMPTY_REQ_CTX)
|
manager.complete_store(to_keys([14, 15]), _EMPTY_REQ_CTX)
|
||||||
# cache state [10, 11, 14, 15] <- all blocks idle
|
# cache state [10, 11, 14, 15] <- all chunks idle
|
||||||
assert manager._num_evictable_cache_blocks == 4
|
assert manager._num_evictable_cache_chunks == 4
|
||||||
|
|
||||||
|
|
||||||
def test_touch_forwards_req_context_to_policy(monkeypatch):
|
def test_touch_forwards_req_context_to_policy(monkeypatch):
|
||||||
"""Regression: CPUOffloadingManager.touch forwards ReqContext to policy."""
|
"""Regression: CPUOffloadingManager.touch forwards ReqContext to policy."""
|
||||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||||
received = []
|
received = []
|
||||||
|
|
||||||
def spy_touch(keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
def spy_touch(keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ def _set_spawn_method(monkeypatch):
|
|||||||
|
|
||||||
def _make_region(
|
def _make_region(
|
||||||
engine_id: str,
|
engine_id: str,
|
||||||
num_blocks: int = 4,
|
num_chunks: int = 4,
|
||||||
cpu_page_size: int = PAGE_SIZE,
|
cpu_page_size: int = PAGE_SIZE,
|
||||||
num_workers: int = 1,
|
num_workers: int = 1,
|
||||||
rank: int = 0,
|
rank: int = 0,
|
||||||
@@ -47,9 +47,9 @@ def _make_region(
|
|||||||
assert cpu_page_size % PAGE_SIZE == 0
|
assert cpu_page_size % PAGE_SIZE == 0
|
||||||
return SharedOffloadRegion(
|
return SharedOffloadRegion(
|
||||||
engine_id=engine_id,
|
engine_id=engine_id,
|
||||||
num_blocks=num_blocks,
|
num_chunks=num_chunks,
|
||||||
rank=rank,
|
rank=rank,
|
||||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
kv_bytes_per_chunk=num_workers * cpu_page_size,
|
||||||
cpu_page_size=cpu_page_size,
|
cpu_page_size=cpu_page_size,
|
||||||
barrier=barrier,
|
barrier=barrier,
|
||||||
)
|
)
|
||||||
@@ -99,16 +99,16 @@ def _region(engine_id: str, **kwargs):
|
|||||||
def _multi_region(
|
def _multi_region(
|
||||||
engine_id: str,
|
engine_id: str,
|
||||||
num_workers: int,
|
num_workers: int,
|
||||||
num_blocks: int = 4,
|
num_chunks: int = 4,
|
||||||
cpu_page_size: int = PAGE_SIZE,
|
cpu_page_size: int = PAGE_SIZE,
|
||||||
):
|
):
|
||||||
"""Context manager: create one SharedOffloadRegion per rank, clean up on exit."""
|
"""Context manager: create one SharedOffloadRegion per rank, clean up on exit."""
|
||||||
regions = [
|
regions = [
|
||||||
SharedOffloadRegion(
|
SharedOffloadRegion(
|
||||||
engine_id=engine_id,
|
engine_id=engine_id,
|
||||||
num_blocks=num_blocks,
|
num_chunks=num_chunks,
|
||||||
rank=rank,
|
rank=rank,
|
||||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
kv_bytes_per_chunk=num_workers * cpu_page_size,
|
||||||
cpu_page_size=cpu_page_size,
|
cpu_page_size=cpu_page_size,
|
||||||
)
|
)
|
||||||
for rank in range(num_workers)
|
for rank in range(num_workers)
|
||||||
@@ -124,7 +124,7 @@ def _multi_region(
|
|||||||
def _race_construct(
|
def _race_construct(
|
||||||
engine_id: str,
|
engine_id: str,
|
||||||
num_workers: int,
|
num_workers: int,
|
||||||
num_blocks: int = 4,
|
num_chunks: int = 4,
|
||||||
cpu_page_size: int = PAGE_SIZE,
|
cpu_page_size: int = PAGE_SIZE,
|
||||||
) -> tuple[list[SharedOffloadRegion], list[Exception]]:
|
) -> tuple[list[SharedOffloadRegion], list[Exception]]:
|
||||||
"""Spawn num_workers threads that all race to construct SharedOffloadRegion."""
|
"""Spawn num_workers threads that all race to construct SharedOffloadRegion."""
|
||||||
@@ -137,9 +137,9 @@ def _race_construct(
|
|||||||
try:
|
try:
|
||||||
regions[rank] = SharedOffloadRegion(
|
regions[rank] = SharedOffloadRegion(
|
||||||
engine_id=engine_id,
|
engine_id=engine_id,
|
||||||
num_blocks=num_blocks,
|
num_chunks=num_chunks,
|
||||||
rank=rank,
|
rank=rank,
|
||||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
kv_bytes_per_chunk=num_workers * cpu_page_size,
|
||||||
cpu_page_size=cpu_page_size,
|
cpu_page_size=cpu_page_size,
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -156,7 +156,7 @@ def _race_construct(
|
|||||||
|
|
||||||
def _mp_race_construct_and_write(
|
def _mp_race_construct_and_write(
|
||||||
engine_id: str,
|
engine_id: str,
|
||||||
num_blocks: int,
|
num_chunks: int,
|
||||||
rank: int,
|
rank: int,
|
||||||
num_workers: int,
|
num_workers: int,
|
||||||
cpu_page_size: int,
|
cpu_page_size: int,
|
||||||
@@ -170,9 +170,9 @@ def _mp_race_construct_and_write(
|
|||||||
try:
|
try:
|
||||||
region = SharedOffloadRegion(
|
region = SharedOffloadRegion(
|
||||||
engine_id=engine_id,
|
engine_id=engine_id,
|
||||||
num_blocks=num_blocks,
|
num_chunks=num_chunks,
|
||||||
rank=rank,
|
rank=rank,
|
||||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
kv_bytes_per_chunk=num_workers * cpu_page_size,
|
||||||
cpu_page_size=cpu_page_size,
|
cpu_page_size=cpu_page_size,
|
||||||
)
|
)
|
||||||
t = region.create_next_worker_view(cpu_page_size)
|
t = region.create_next_worker_view(cpu_page_size)
|
||||||
@@ -198,9 +198,9 @@ def _mp_barrier_construct_and_hold(
|
|||||||
try:
|
try:
|
||||||
region = SharedOffloadRegion(
|
region = SharedOffloadRegion(
|
||||||
engine_id=engine_id,
|
engine_id=engine_id,
|
||||||
num_blocks=2,
|
num_chunks=2,
|
||||||
rank=rank,
|
rank=rank,
|
||||||
kv_bytes_per_block=num_workers * PAGE_SIZE,
|
kv_bytes_per_chunk=num_workers * PAGE_SIZE,
|
||||||
cpu_page_size=PAGE_SIZE,
|
cpu_page_size=PAGE_SIZE,
|
||||||
barrier=lambda: barrier.wait(30),
|
barrier=lambda: barrier.wait(30),
|
||||||
)
|
)
|
||||||
@@ -233,9 +233,9 @@ def iid():
|
|||||||
|
|
||||||
|
|
||||||
def test_create_next_worker_view_shape_and_stride(iid):
|
def test_create_next_worker_view_shape_and_stride(iid):
|
||||||
"""Returned tensor must have shape (num_blocks, tensor_page_size) and
|
"""Returned tensor must have shape (num_chunks, tensor_page_size) and
|
||||||
stride (row_stride, 1) where row_stride = cpu_page_size * num_workers."""
|
stride (row_stride, 1) where row_stride = cpu_page_size * num_workers."""
|
||||||
with _region(iid, num_blocks=4, cpu_page_size=2 * PAGE_SIZE) as r:
|
with _region(iid, num_chunks=4, cpu_page_size=2 * PAGE_SIZE) as r:
|
||||||
t = r.create_next_worker_view(PAGE_SIZE)
|
t = r.create_next_worker_view(PAGE_SIZE)
|
||||||
assert t.shape == (4, PAGE_SIZE)
|
assert t.shape == (4, PAGE_SIZE)
|
||||||
# num_workers=1 → row_stride = cpu_page_size
|
# num_workers=1 → row_stride = cpu_page_size
|
||||||
@@ -253,7 +253,7 @@ def test_create_next_worker_view_storage_offset_rank0(iid):
|
|||||||
|
|
||||||
def test_create_next_worker_view_storage_offset_rank1(iid):
|
def test_create_next_worker_view_storage_offset_rank1(iid):
|
||||||
"""rank=1 worker's first tensor must start cpu_page_size bytes into the mmap."""
|
"""rank=1 worker's first tensor must start cpu_page_size bytes into the mmap."""
|
||||||
with _multi_region(iid, num_workers=2, num_blocks=4) as (r0, r1):
|
with _multi_region(iid, num_workers=2, num_chunks=4) as (r0, r1):
|
||||||
t1 = r1.create_next_worker_view(PAGE_SIZE)
|
t1 = r1.create_next_worker_view(PAGE_SIZE)
|
||||||
assert t1.data_ptr() == r1._base.data_ptr() + PAGE_SIZE
|
assert t1.data_ptr() == r1._base.data_ptr() + PAGE_SIZE
|
||||||
del t1
|
del t1
|
||||||
@@ -261,7 +261,7 @@ def test_create_next_worker_view_storage_offset_rank1(iid):
|
|||||||
|
|
||||||
def test_create_next_worker_view_row_stride_with_multiple_workers(iid):
|
def test_create_next_worker_view_row_stride_with_multiple_workers(iid):
|
||||||
"""With num_workers=4, row_stride must be 4 * cpu_page_size."""
|
"""With num_workers=4, row_stride must be 4 * cpu_page_size."""
|
||||||
with _region(iid, num_blocks=2, num_workers=4) as r:
|
with _region(iid, num_chunks=2, num_workers=4) as r:
|
||||||
t = r.create_next_worker_view(PAGE_SIZE)
|
t = r.create_next_worker_view(PAGE_SIZE)
|
||||||
assert t.stride(0) == 4 * PAGE_SIZE
|
assert t.stride(0) == 4 * PAGE_SIZE
|
||||||
del t
|
del t
|
||||||
@@ -332,12 +332,12 @@ def test_create_next_worker_view_overflow_does_not_mutate_cursor(iid):
|
|||||||
def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
|
def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
|
||||||
"""Writes into a create_next_worker_view view must appear at the correct
|
"""Writes into a create_next_worker_view view must appear at the correct
|
||||||
raw mmap offset"""
|
raw mmap offset"""
|
||||||
with _region(iid, num_blocks=4) as r:
|
with _region(iid, num_chunks=4) as r:
|
||||||
t = r.create_next_worker_view(PAGE_SIZE)
|
t = r.create_next_worker_view(PAGE_SIZE)
|
||||||
t[2, :] = 42 # write to block row 2
|
t[2, :] = 42 # write to chunk row 2
|
||||||
|
|
||||||
raw = memoryview(r.mmap_obj)
|
raw = memoryview(r.mmap_obj)
|
||||||
# num_workers=1 → row_stride = PAGE_SIZE; block 2 starts at byte 2*PAGE_SIZE
|
# num_workers=1 → row_stride = PAGE_SIZE; chunk 2 starts at byte 2*PAGE_SIZE
|
||||||
chunk = bytes(raw[2 * PAGE_SIZE : 3 * PAGE_SIZE])
|
chunk = bytes(raw[2 * PAGE_SIZE : 3 * PAGE_SIZE])
|
||||||
assert all(b == 42 for b in chunk)
|
assert all(b == 42 for b in chunk)
|
||||||
del raw, t
|
del raw, t
|
||||||
@@ -345,7 +345,7 @@ def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
|
|||||||
|
|
||||||
def test_create_next_worker_view_multi_tensor_layout(iid):
|
def test_create_next_worker_view_multi_tensor_layout(iid):
|
||||||
"""Two tensors from the same worker land at consecutive byte offsets per row."""
|
"""Two tensors from the same worker land at consecutive byte offsets per row."""
|
||||||
with _region(iid, num_blocks=2, cpu_page_size=2 * PAGE_SIZE) as r:
|
with _region(iid, num_chunks=2, cpu_page_size=2 * PAGE_SIZE) as r:
|
||||||
ta = r.create_next_worker_view(PAGE_SIZE)
|
ta = r.create_next_worker_view(PAGE_SIZE)
|
||||||
tb = r.create_next_worker_view(PAGE_SIZE)
|
tb = r.create_next_worker_view(PAGE_SIZE)
|
||||||
|
|
||||||
@@ -353,8 +353,8 @@ def test_create_next_worker_view_multi_tensor_layout(iid):
|
|||||||
tb[:, :] = 2
|
tb[:, :] = 2
|
||||||
|
|
||||||
raw = memoryview(r.mmap_obj)
|
raw = memoryview(r.mmap_obj)
|
||||||
for blk in range(2):
|
for chunk in range(2):
|
||||||
row_offset = blk * 2 * PAGE_SIZE # num_workers=1
|
row_offset = chunk * 2 * PAGE_SIZE # num_workers=1
|
||||||
assert all(b == 1 for b in raw[row_offset : row_offset + PAGE_SIZE])
|
assert all(b == 1 for b in raw[row_offset : row_offset + PAGE_SIZE])
|
||||||
assert all(
|
assert all(
|
||||||
b == 2 for b in raw[row_offset + PAGE_SIZE : row_offset + 2 * PAGE_SIZE]
|
b == 2 for b in raw[row_offset + PAGE_SIZE : row_offset + 2 * PAGE_SIZE]
|
||||||
@@ -366,7 +366,7 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
|||||||
"""Each worker process calls create_next_worker_view and writes distinct data;
|
"""Each worker process calls create_next_worker_view and writes distinct data;
|
||||||
the parent verifies each slot lands at the correct interleaved offset."""
|
the parent verifies each slot lands at the correct interleaved offset."""
|
||||||
num_workers = 2
|
num_workers = 2
|
||||||
num_blocks = 4
|
num_chunks = 4
|
||||||
|
|
||||||
ctx = get_mp_context()
|
ctx = get_mp_context()
|
||||||
done_queue = ctx.Queue()
|
done_queue = ctx.Queue()
|
||||||
@@ -375,9 +375,9 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
|||||||
# Parent is rank 0 (creator); child is rank 1 (joiner).
|
# Parent is rank 0 (creator); child is rank 1 (joiner).
|
||||||
region = SharedOffloadRegion(
|
region = SharedOffloadRegion(
|
||||||
engine_id=iid,
|
engine_id=iid,
|
||||||
num_blocks=num_blocks,
|
num_chunks=num_chunks,
|
||||||
rank=0,
|
rank=0,
|
||||||
kv_bytes_per_block=num_workers * PAGE_SIZE,
|
kv_bytes_per_chunk=num_workers * PAGE_SIZE,
|
||||||
cpu_page_size=PAGE_SIZE,
|
cpu_page_size=PAGE_SIZE,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
@@ -385,7 +385,7 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
|||||||
target=_mp_race_construct_and_write,
|
target=_mp_race_construct_and_write,
|
||||||
args=(
|
args=(
|
||||||
iid,
|
iid,
|
||||||
num_blocks,
|
num_chunks,
|
||||||
1,
|
1,
|
||||||
num_workers,
|
num_workers,
|
||||||
PAGE_SIZE,
|
PAGE_SIZE,
|
||||||
@@ -403,12 +403,12 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
|||||||
assert result["error"] is None, result["error"]
|
assert result["error"] is None, result["error"]
|
||||||
|
|
||||||
raw = memoryview(region.mmap_obj)
|
raw = memoryview(region.mmap_obj)
|
||||||
for blk in range(num_blocks):
|
for chunk in range(num_chunks):
|
||||||
row_start = blk * num_workers * PAGE_SIZE
|
row_start = chunk * num_workers * PAGE_SIZE
|
||||||
w0 = bytes(raw[row_start : row_start + PAGE_SIZE])
|
w0 = bytes(raw[row_start : row_start + PAGE_SIZE])
|
||||||
w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE])
|
w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE])
|
||||||
assert all(b == 11 for b in w0), f"block {blk}: rank0 slot wrong"
|
assert all(b == 11 for b in w0), f"chunk {chunk}: rank0 slot wrong"
|
||||||
assert all(b == 22 for b in w1), f"block {blk}: rank1 slot wrong"
|
assert all(b == 22 for b in w1), f"chunk {chunk}: rank1 slot wrong"
|
||||||
|
|
||||||
del raw, t0 # release before finally triggers cleanup
|
del raw, t0 # release before finally triggers cleanup
|
||||||
cleanup_queue.put(True)
|
cleanup_queue.put(True)
|
||||||
@@ -422,8 +422,8 @@ def test_create_next_worker_view_multiprocess_slots(iid):
|
|||||||
def test_create_next_worker_view_worker_isolation(iid):
|
def test_create_next_worker_view_worker_isolation(iid):
|
||||||
"""Writes by worker 0 must not affect worker 1's slot and vice versa."""
|
"""Writes by worker 0 must not affect worker 1's slot and vice versa."""
|
||||||
num_workers = 2
|
num_workers = 2
|
||||||
num_blocks = 4
|
num_chunks = 4
|
||||||
with _multi_region(iid, num_workers=num_workers, num_blocks=num_blocks) as regions:
|
with _multi_region(iid, num_workers=num_workers, num_chunks=num_chunks) as regions:
|
||||||
t0 = regions[0].create_next_worker_view(PAGE_SIZE)
|
t0 = regions[0].create_next_worker_view(PAGE_SIZE)
|
||||||
t1 = regions[1].create_next_worker_view(PAGE_SIZE)
|
t1 = regions[1].create_next_worker_view(PAGE_SIZE)
|
||||||
|
|
||||||
@@ -431,12 +431,12 @@ def test_create_next_worker_view_worker_isolation(iid):
|
|||||||
t1[:, :] = 22
|
t1[:, :] = 22
|
||||||
|
|
||||||
raw = memoryview(regions[0].mmap_obj)
|
raw = memoryview(regions[0].mmap_obj)
|
||||||
for blk in range(num_blocks):
|
for chunk in range(num_chunks):
|
||||||
row_start = blk * num_workers * PAGE_SIZE
|
row_start = chunk * num_workers * PAGE_SIZE
|
||||||
w0 = bytes(raw[row_start : row_start + PAGE_SIZE])
|
w0 = bytes(raw[row_start : row_start + PAGE_SIZE])
|
||||||
w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE])
|
w1 = bytes(raw[row_start + PAGE_SIZE : row_start + 2 * PAGE_SIZE])
|
||||||
assert all(b == 11 for b in w0), f"block {blk}: worker0 slot corrupted"
|
assert all(b == 11 for b in w0), f"chunk {chunk}: worker0 slot corrupted"
|
||||||
assert all(b == 22 for b in w1), f"block {blk}: worker1 slot corrupted"
|
assert all(b == 22 for b in w1), f"chunk {chunk}: worker1 slot corrupted"
|
||||||
del raw, t0, t1 # release before finally triggers cleanup
|
del raw, t0, t1 # release before finally triggers cleanup
|
||||||
|
|
||||||
|
|
||||||
@@ -466,7 +466,7 @@ def test_file_exists_after_construction(iid):
|
|||||||
|
|
||||||
def test_file_has_correct_size(iid):
|
def test_file_has_correct_size(iid):
|
||||||
"""The mmap file size on disk must equal total_size_bytes."""
|
"""The mmap file size on disk must equal total_size_bytes."""
|
||||||
with _region(iid, num_blocks=4) as r:
|
with _region(iid, num_chunks=4) as r:
|
||||||
assert os.path.getsize(r.mmap_path) == 4 * PAGE_SIZE
|
assert os.path.getsize(r.mmap_path) == 4 * PAGE_SIZE
|
||||||
|
|
||||||
|
|
||||||
@@ -491,11 +491,11 @@ def test_madvise_success_selects_madvise_population(iid, monkeypatch):
|
|||||||
monkeypatch.setattr(sor, "_madvise_populate_write", _spy_madvise)
|
monkeypatch.setattr(sor, "_madvise_populate_write", _spy_madvise)
|
||||||
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
||||||
|
|
||||||
num_blocks = 3
|
num_chunks = 3
|
||||||
num_workers = 2
|
num_workers = 2
|
||||||
with _region(iid, num_blocks=num_blocks, num_workers=num_workers, rank=1):
|
with _region(iid, num_chunks=num_chunks, num_workers=num_workers, rank=1):
|
||||||
# 1 probe (PAGESIZE) + N populate calls (one per block per worker column).
|
# 1 probe (PAGESIZE) + N populate calls (one per chunk per worker column).
|
||||||
expected_populate = num_blocks # ranked path: one call per block
|
expected_populate = num_chunks # ranked path: one call per chunk
|
||||||
mmap_id = madvise_calls[0][2]
|
mmap_id = madvise_calls[0][2]
|
||||||
assert madvise_calls == [
|
assert madvise_calls == [
|
||||||
(0, mmap.PAGESIZE, mmap_id),
|
(0, mmap.PAGESIZE, mmap_id),
|
||||||
@@ -510,7 +510,7 @@ def test_madvise_success_selects_madvise_population(iid, monkeypatch):
|
|||||||
|
|
||||||
|
|
||||||
def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
|
def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
|
||||||
"""An EINVAL probe must select fallback for every ranked block."""
|
"""An EINVAL probe must select fallback for every ranked chunk."""
|
||||||
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
|
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
|
||||||
|
|
||||||
fallback_calls: list[tuple[int, int]] = []
|
fallback_calls: list[tuple[int, int]] = []
|
||||||
@@ -526,7 +526,7 @@ def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
|
|||||||
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
|
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
|
||||||
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
||||||
|
|
||||||
with _region(iid, num_blocks=3, num_workers=2, rank=1):
|
with _region(iid, num_chunks=3, num_workers=2, rank=1):
|
||||||
assert fallback_calls == [
|
assert fallback_calls == [
|
||||||
(mmap.PAGESIZE, mmap.PAGESIZE),
|
(mmap.PAGESIZE, mmap.PAGESIZE),
|
||||||
(3 * mmap.PAGESIZE, mmap.PAGESIZE),
|
(3 * mmap.PAGESIZE, mmap.PAGESIZE),
|
||||||
@@ -551,7 +551,7 @@ def test_madvise_einval_selects_fallback_for_unranked_region(iid, monkeypatch):
|
|||||||
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
|
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
|
||||||
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
||||||
|
|
||||||
with _region(iid, num_blocks=3, num_workers=2, rank=None):
|
with _region(iid, num_chunks=3, num_workers=2, rank=None):
|
||||||
assert fallback_calls == [(0, 6 * mmap.PAGESIZE)]
|
assert fallback_calls == [(0, 6 * mmap.PAGESIZE)]
|
||||||
|
|
||||||
|
|
||||||
@@ -651,7 +651,7 @@ def test_multiprocess_race_construct_and_write(iid):
|
|||||||
"""N processes race to construct the same SharedOffloadRegion, each writes
|
"""N processes race to construct the same SharedOffloadRegion, each writes
|
||||||
fill_value = rank+1 into their slot; parent verifies interleaved layout."""
|
fill_value = rank+1 into their slot; parent verifies interleaved layout."""
|
||||||
num_workers = 4
|
num_workers = 4
|
||||||
num_blocks = 3
|
num_chunks = 3
|
||||||
|
|
||||||
ctx = get_mp_context()
|
ctx = get_mp_context()
|
||||||
done_queue = ctx.Queue()
|
done_queue = ctx.Queue()
|
||||||
@@ -662,7 +662,7 @@ def test_multiprocess_race_construct_and_write(iid):
|
|||||||
target=_mp_race_construct_and_write,
|
target=_mp_race_construct_and_write,
|
||||||
args=(
|
args=(
|
||||||
iid,
|
iid,
|
||||||
num_blocks,
|
num_chunks,
|
||||||
rank,
|
rank,
|
||||||
num_workers,
|
num_workers,
|
||||||
PAGE_SIZE,
|
PAGE_SIZE,
|
||||||
@@ -689,13 +689,13 @@ def test_multiprocess_race_construct_and_write(iid):
|
|||||||
with open(mmap_path, "rb") as f:
|
with open(mmap_path, "rb") as f:
|
||||||
raw = f.read()
|
raw = f.read()
|
||||||
|
|
||||||
for blk in range(num_blocks):
|
for chunk in range(num_chunks):
|
||||||
for w in range(num_workers):
|
for w in range(num_workers):
|
||||||
slot_start = (blk * num_workers + w) * PAGE_SIZE
|
slot_start = (chunk * num_workers + w) * PAGE_SIZE
|
||||||
slot = raw[slot_start : slot_start + PAGE_SIZE]
|
slot = raw[slot_start : slot_start + PAGE_SIZE]
|
||||||
expected = w + 1 # fill_value = rank + 1
|
expected = w + 1 # fill_value = rank + 1
|
||||||
assert all(b == expected for b in slot), (
|
assert all(b == expected for b in slot), (
|
||||||
f"block {blk}, worker {w}: expected {expected} but got wrong bytes"
|
f"chunk {chunk}, worker {w}: expected {expected} but got wrong bytes"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Unblock all workers to clean up.
|
# Unblock all workers to clean up.
|
||||||
@@ -839,9 +839,9 @@ def test_insufficient_space_raises_clear_error(monkeypatch):
|
|||||||
with pytest.raises(RuntimeError, match="Insufficient space"):
|
with pytest.raises(RuntimeError, match="Insufficient space"):
|
||||||
SharedOffloadRegion(
|
SharedOffloadRegion(
|
||||||
engine_id=engine_id,
|
engine_id=engine_id,
|
||||||
num_blocks=4,
|
num_chunks=4,
|
||||||
rank=0,
|
rank=0,
|
||||||
kv_bytes_per_block=PAGE_SIZE,
|
kv_bytes_per_chunk=PAGE_SIZE,
|
||||||
cpu_page_size=PAGE_SIZE,
|
cpu_page_size=PAGE_SIZE,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -870,9 +870,9 @@ def test_ftruncate_failure_cleans_up_creator(monkeypatch):
|
|||||||
with pytest.raises(OSError, match="ftruncate failed"):
|
with pytest.raises(OSError, match="ftruncate failed"):
|
||||||
SharedOffloadRegion(
|
SharedOffloadRegion(
|
||||||
engine_id=engine_id,
|
engine_id=engine_id,
|
||||||
num_blocks=4,
|
num_chunks=4,
|
||||||
rank=0,
|
rank=0,
|
||||||
kv_bytes_per_block=PAGE_SIZE,
|
kv_bytes_per_chunk=PAGE_SIZE,
|
||||||
cpu_page_size=PAGE_SIZE,
|
cpu_page_size=PAGE_SIZE,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -136,7 +136,7 @@ def test_get_spec_cls_defaults_to_cpu():
|
|||||||
def test_create_cpu_offloading_spec():
|
def test_create_cpu_offloading_spec():
|
||||||
spec = _create_spec()
|
spec = _create_spec()
|
||||||
assert isinstance(spec, CPUOffloadingSpec)
|
assert isinstance(spec, CPUOffloadingSpec)
|
||||||
assert spec.num_blocks > 0
|
assert spec.num_chunks > 0
|
||||||
|
|
||||||
|
|
||||||
def test_cpu_spec_sizes_normalized_worker_layout():
|
def test_cpu_spec_sizes_normalized_worker_layout():
|
||||||
@@ -156,7 +156,7 @@ def test_cpu_spec_sizes_normalized_worker_layout():
|
|||||||
assert isinstance(spec, CPUOffloadingSpec)
|
assert isinstance(spec, CPUOffloadingSpec)
|
||||||
assert spec.cpu_page_size_per_worker == 32
|
assert spec.cpu_page_size_per_worker == 32
|
||||||
assert spec.kv_bytes_per_chunk == alignment
|
assert spec.kv_bytes_per_chunk == alignment
|
||||||
assert spec.num_blocks == 3
|
assert spec.num_chunks == 3
|
||||||
|
|
||||||
|
|
||||||
def test_cpu_spec_zero_worker_bytes_produces_empty_cache():
|
def test_cpu_spec_zero_worker_bytes_produces_empty_cache():
|
||||||
@@ -165,7 +165,7 @@ def test_cpu_spec_zero_worker_bytes_produces_empty_cache():
|
|||||||
assert isinstance(spec, CPUOffloadingSpec)
|
assert isinstance(spec, CPUOffloadingSpec)
|
||||||
assert spec.cpu_page_size_per_worker == 0
|
assert spec.cpu_page_size_per_worker == 0
|
||||||
assert spec.kv_bytes_per_chunk == 0
|
assert spec.kv_bytes_per_chunk == 0
|
||||||
assert spec.num_blocks == 0
|
assert spec.num_chunks == 0
|
||||||
|
|
||||||
|
|
||||||
def test_tiering_spec_aligns_row_size():
|
def test_tiering_spec_aligns_row_size():
|
||||||
@@ -183,7 +183,7 @@ def test_tiering_spec_aligns_row_size():
|
|||||||
assert isinstance(spec, TieringOffloadingSpec)
|
assert isinstance(spec, TieringOffloadingSpec)
|
||||||
assert spec.cpu_page_size_per_worker == 32
|
assert spec.cpu_page_size_per_worker == 32
|
||||||
assert spec.kv_bytes_per_chunk == alignment
|
assert spec.kv_bytes_per_chunk == alignment
|
||||||
assert spec.num_blocks == 3
|
assert spec.num_chunks == 3
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("world_size", [2, 4, 8])
|
@pytest.mark.parametrize("world_size", [2, 4, 8])
|
||||||
@@ -201,7 +201,7 @@ def test_tiering_spec_replicated_sizing_removes_world_factor(world_size: int):
|
|||||||
assert spec.replicated_layout is True
|
assert spec.replicated_layout is True
|
||||||
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
|
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
|
||||||
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block
|
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block
|
||||||
assert spec.num_blocks == 8
|
assert spec.num_chunks == 8
|
||||||
|
|
||||||
|
|
||||||
def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monkeypatch):
|
def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monkeypatch):
|
||||||
@@ -239,7 +239,7 @@ def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monke
|
|||||||
spec.create_worker(kv_caches)
|
spec.create_worker(kv_caches)
|
||||||
|
|
||||||
assert region_calls[0]["rank"] == 0
|
assert region_calls[0]["rank"] == 0
|
||||||
assert region_calls[0]["kv_bytes_per_block"] == worker_kv_bytes_per_block
|
assert region_calls[0]["kv_bytes_per_chunk"] == worker_kv_bytes_per_block
|
||||||
assert worker_calls[0]["kv_caches"] is kv_caches
|
assert worker_calls[0]["kv_caches"] is kv_caches
|
||||||
assert worker_calls[0]["mmap_region"] is region
|
assert worker_calls[0]["mmap_region"] is region
|
||||||
|
|
||||||
@@ -292,7 +292,7 @@ def test_cpu_spec_replicated_sizing_on_shared_region(monkeypatch, world_size: in
|
|||||||
assert spec.replicated_layout is True
|
assert spec.replicated_layout is True
|
||||||
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
|
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
|
||||||
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block
|
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block
|
||||||
assert spec.num_blocks == 8
|
assert spec.num_chunks == 8
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("world_size", [2, 4, 8])
|
@pytest.mark.parametrize("world_size", [2, 4, 8])
|
||||||
@@ -320,7 +320,7 @@ def test_cpu_spec_replicated_disabled_without_shared_region(
|
|||||||
assert spec.replicated_layout is False
|
assert spec.replicated_layout is False
|
||||||
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
|
assert spec.cpu_page_size_per_worker == worker_kv_bytes_per_block
|
||||||
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block * world_size
|
assert spec.kv_bytes_per_chunk == worker_kv_bytes_per_block * world_size
|
||||||
assert spec.num_blocks == 2
|
assert spec.num_chunks == 2
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("config_replicated", [True, False])
|
@pytest.mark.parametrize("config_replicated", [True, False])
|
||||||
@@ -381,7 +381,7 @@ def test_cpu_spec_create_worker_uses_mmap_on_cuda_alike(monkeypatch):
|
|||||||
spec.create_worker(kv_caches)
|
spec.create_worker(kv_caches)
|
||||||
|
|
||||||
assert region_calls[0]["engine_id"] == "test-engine"
|
assert region_calls[0]["engine_id"] == "test-engine"
|
||||||
assert region_calls[0]["kv_bytes_per_block"] == worker_kv_bytes_per_block * 4
|
assert region_calls[0]["kv_bytes_per_chunk"] == worker_kv_bytes_per_block * 4
|
||||||
assert worker_calls[0]["kv_caches"] is kv_caches
|
assert worker_calls[0]["kv_caches"] is kv_caches
|
||||||
assert worker_calls[0]["mmap_region"] is region
|
assert worker_calls[0]["mmap_region"] is region
|
||||||
|
|
||||||
@@ -419,11 +419,11 @@ def test_cpu_spec_create_worker_uses_tensor_path_off_cuda_alike(monkeypatch):
|
|||||||
def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch):
|
def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch):
|
||||||
import vllm.v1.kv_offload.cpu.spec as cpu_spec_module
|
import vllm.v1.kv_offload.cpu.spec as cpu_spec_module
|
||||||
|
|
||||||
# worker_kv_bytes_per_block=0 yields num_blocks=0; a zero-byte region cannot
|
# worker_kv_bytes_per_block=0 yields num_chunks=0; a zero-byte region cannot
|
||||||
# be mmap'd, so even on CUDA-alike this must fall back to the tensor path.
|
# be mmap'd, so even on CUDA-alike this must fall back to the tensor path.
|
||||||
spec = _create_spec(worker_kv_bytes_per_block=0, world_size=4)
|
spec = _create_spec(worker_kv_bytes_per_block=0, world_size=4)
|
||||||
assert isinstance(spec, CPUOffloadingSpec)
|
assert isinstance(spec, CPUOffloadingSpec)
|
||||||
assert spec.num_blocks == 0
|
assert spec.num_chunks == 0
|
||||||
|
|
||||||
region_calls: list[dict[str, Any]] = []
|
region_calls: list[dict[str, Any]] = []
|
||||||
worker_calls: list[dict[str, Any]] = []
|
worker_calls: list[dict[str, Any]] = []
|
||||||
|
|||||||
@@ -91,17 +91,17 @@ def _req_context(kv_params: dict | None = None) -> ReqContext:
|
|||||||
def _job_metadata(
|
def _job_metadata(
|
||||||
job_id: int,
|
job_id: int,
|
||||||
keys: list[bytes] | None = None,
|
keys: list[bytes] | None = None,
|
||||||
block_ids: list[int] | None = None,
|
chunk_ids: list[int] | None = None,
|
||||||
kv_params: dict | None = None,
|
kv_params: dict | None = None,
|
||||||
) -> TransferJob:
|
) -> TransferJob:
|
||||||
if keys is None:
|
if keys is None:
|
||||||
keys = [b"key1"]
|
keys = [b"key1"]
|
||||||
if block_ids is None:
|
if chunk_ids is None:
|
||||||
block_ids = list(range(len(keys)))
|
chunk_ids = list(range(len(keys)))
|
||||||
return TransferJob(
|
return TransferJob(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
keys=keys,
|
keys=keys,
|
||||||
block_ids=np.array(block_ids),
|
chunk_ids=np.array(chunk_ids),
|
||||||
is_promotion=False,
|
is_promotion=False,
|
||||||
req_context=_req_context(kv_params),
|
req_context=_req_context(kv_params),
|
||||||
)
|
)
|
||||||
@@ -254,9 +254,9 @@ class TestLookup:
|
|||||||
"kv_params,expected",
|
"kv_params,expected",
|
||||||
[
|
[
|
||||||
(_remote_decoder_kv_params(), OffloadPolicy.REQUEST_LEVEL),
|
(_remote_decoder_kv_params(), OffloadPolicy.REQUEST_LEVEL),
|
||||||
(None, OffloadPolicy.BLOCK_LEVEL),
|
(None, OffloadPolicy.CHUNK_LEVEL),
|
||||||
(_remote_prefiller_kv_params(), OffloadPolicy.BLOCK_LEVEL),
|
(_remote_prefiller_kv_params(), OffloadPolicy.CHUNK_LEVEL),
|
||||||
({"remote_decoder": {}}, OffloadPolicy.BLOCK_LEVEL),
|
({"remote_decoder": {}}, OffloadPolicy.CHUNK_LEVEL),
|
||||||
],
|
],
|
||||||
ids=["producer", "plain", "consumer", "producer_no_id"],
|
ids=["producer", "plain", "consumer", "producer_no_id"],
|
||||||
)
|
)
|
||||||
@@ -366,7 +366,7 @@ class TestSubmitStore:
|
|||||||
job = _job_metadata(
|
job = _job_metadata(
|
||||||
job_id=1,
|
job_id=1,
|
||||||
keys=[b"k1", b"k2"],
|
keys=[b"k1", b"k2"],
|
||||||
block_ids=[3, 4],
|
chunk_ids=[3, 4],
|
||||||
kv_params=_remote_decoder_kv_params(kv_request_id="req-1"),
|
kv_params=_remote_decoder_kv_params(kv_request_id="req-1"),
|
||||||
)
|
)
|
||||||
mgr.submit_store(job)
|
mgr.submit_store(job)
|
||||||
@@ -397,7 +397,7 @@ class TestSubmitStore:
|
|||||||
job = _job_metadata(
|
job = _job_metadata(
|
||||||
job_id=7,
|
job_id=7,
|
||||||
keys=[b"k1", b"k2"],
|
keys=[b"k1", b"k2"],
|
||||||
block_ids=[3, 4],
|
chunk_ids=[3, 4],
|
||||||
kv_params=_remote_decoder_kv_params(kv_request_id="req-1"),
|
kv_params=_remote_decoder_kv_params(kv_request_id="req-1"),
|
||||||
)
|
)
|
||||||
mgr.submit_store(job)
|
mgr.submit_store(job)
|
||||||
@@ -440,7 +440,7 @@ class TestSubmitLoad:
|
|||||||
"""Empty key list succeeds immediately."""
|
"""Empty key list succeeds immediately."""
|
||||||
mgr = _make_manager()
|
mgr = _make_manager()
|
||||||
job = _job_metadata(
|
job = _job_metadata(
|
||||||
job_id=1, keys=[], block_ids=[], kv_params=_remote_prefiller_kv_params()
|
job_id=1, keys=[], chunk_ids=[], kv_params=_remote_prefiller_kv_params()
|
||||||
)
|
)
|
||||||
mgr.submit_load(job)
|
mgr.submit_load(job)
|
||||||
assert mgr._finished_jobs == [JobResult(job_id=1, success=True)]
|
assert mgr._finished_jobs == [JobResult(job_id=1, success=True)]
|
||||||
@@ -463,7 +463,7 @@ class TestSubmitLoad:
|
|||||||
job = _job_metadata(
|
job = _job_metadata(
|
||||||
job_id=42,
|
job_id=42,
|
||||||
keys=[b"k1", b"k2"],
|
keys=[b"k1", b"k2"],
|
||||||
block_ids=[5, 6],
|
chunk_ids=[5, 6],
|
||||||
kv_params=_remote_prefiller_kv_params(kv_request_id="req-42"),
|
kv_params=_remote_prefiller_kv_params(kv_request_id="req-42"),
|
||||||
)
|
)
|
||||||
mgr.submit_load(job)
|
mgr.submit_load(job)
|
||||||
@@ -1155,7 +1155,7 @@ class TestBidirectionalManager:
|
|||||||
_job_metadata(
|
_job_metadata(
|
||||||
job_id=100,
|
job_id=100,
|
||||||
keys=[b"a-block"],
|
keys=[b"a-block"],
|
||||||
block_ids=[0],
|
chunk_ids=[0],
|
||||||
kv_params=a_prefiller_params,
|
kv_params=a_prefiller_params,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -1163,7 +1163,7 @@ class TestBidirectionalManager:
|
|||||||
_job_metadata(
|
_job_metadata(
|
||||||
job_id=200,
|
job_id=200,
|
||||||
keys=[b"b-block"],
|
keys=[b"b-block"],
|
||||||
block_ids=[0],
|
chunk_ids=[0],
|
||||||
kv_params=b_prefiller_params,
|
kv_params=b_prefiller_params,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -1173,7 +1173,7 @@ class TestBidirectionalManager:
|
|||||||
_job_metadata(
|
_job_metadata(
|
||||||
job_id=101,
|
job_id=101,
|
||||||
keys=[b"b-block"],
|
keys=[b"b-block"],
|
||||||
block_ids=[0],
|
chunk_ids=[0],
|
||||||
kv_params=a_decoder_params,
|
kv_params=a_decoder_params,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -1181,7 +1181,7 @@ class TestBidirectionalManager:
|
|||||||
_job_metadata(
|
_job_metadata(
|
||||||
job_id=201,
|
job_id=201,
|
||||||
keys=[b"a-block"],
|
keys=[b"a-block"],
|
||||||
block_ids=[0],
|
chunk_ids=[0],
|
||||||
kv_params=b_decoder_params,
|
kv_params=b_decoder_params,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -1573,7 +1573,7 @@ class TestConnectionDeathMidTransfer:
|
|||||||
_job_metadata(
|
_job_metadata(
|
||||||
job_id=900,
|
job_id=900,
|
||||||
keys=[b"a-block"],
|
keys=[b"a-block"],
|
||||||
block_ids=[0],
|
chunk_ids=[0],
|
||||||
kv_params=a_prefiller_params,
|
kv_params=a_prefiller_params,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -1581,7 +1581,7 @@ class TestConnectionDeathMidTransfer:
|
|||||||
_job_metadata(
|
_job_metadata(
|
||||||
job_id=901,
|
job_id=901,
|
||||||
keys=[b"b-block"],
|
keys=[b"b-block"],
|
||||||
block_ids=[0],
|
chunk_ids=[0],
|
||||||
kv_params=a_decoder_params,
|
kv_params=a_decoder_params,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -259,13 +259,13 @@ class FakeParent:
|
|||||||
) -> TransferJob:
|
) -> TransferJob:
|
||||||
keys_list = list(keys)
|
keys_list = list(keys)
|
||||||
self.calls.append(("create_store_job", tuple(keys_list), ctx.req_id))
|
self.calls.append(("create_store_job", tuple(keys_list), ctx.req_id))
|
||||||
block_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32)
|
chunk_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32)
|
||||||
job_id = self._next_job_id
|
job_id = self._next_job_id
|
||||||
self._next_job_id += 1
|
self._next_job_id += 1
|
||||||
return TransferJob(
|
return TransferJob(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
keys=keys_list,
|
keys=keys_list,
|
||||||
block_ids=block_ids,
|
chunk_ids=chunk_ids,
|
||||||
is_promotion=False,
|
is_promotion=False,
|
||||||
req_context=ctx,
|
req_context=ctx,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -105,15 +105,15 @@ def key(n: int) -> OffloadKey:
|
|||||||
def make_job(
|
def make_job(
|
||||||
job_id: int,
|
job_id: int,
|
||||||
keys: list[OffloadKey],
|
keys: list[OffloadKey],
|
||||||
block_ids: list[int] | None = None,
|
chunk_ids: list[int] | None = None,
|
||||||
is_promotion: bool = False,
|
is_promotion: bool = False,
|
||||||
) -> TransferJob:
|
) -> TransferJob:
|
||||||
if block_ids is None:
|
if chunk_ids is None:
|
||||||
block_ids = list(range(len(keys)))
|
chunk_ids = list(range(len(keys)))
|
||||||
return TransferJob(
|
return TransferJob(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
keys=keys,
|
keys=keys,
|
||||||
block_ids=np.array(block_ids, dtype=np.int64),
|
chunk_ids=np.array(chunk_ids, dtype=np.int64),
|
||||||
is_promotion=is_promotion,
|
is_promotion=is_promotion,
|
||||||
req_context=_CTX,
|
req_context=_CTX,
|
||||||
)
|
)
|
||||||
@@ -382,11 +382,11 @@ def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size):
|
|||||||
tensor[:] = _page_aligned_rand_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS)
|
tensor[:] = _page_aligned_rand_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS)
|
||||||
|
|
||||||
keys = [key(i) for i in range(batch_size)]
|
keys = [key(i) for i in range(batch_size)]
|
||||||
store_block_ids = list(range(batch_size))
|
store_chunk_ids = list(range(batch_size))
|
||||||
load_block_ids = list(range(_NUM_BLOCKS - batch_size, _NUM_BLOCKS))
|
load_chunk_ids = list(range(_NUM_BLOCKS - batch_size, _NUM_BLOCKS))
|
||||||
expected = tensor[:batch_size].clone()
|
expected = tensor[:batch_size].clone()
|
||||||
|
|
||||||
tier.submit_store(make_job(1, keys, store_block_ids))
|
tier.submit_store(make_job(1, keys, store_chunk_ids))
|
||||||
store_results = drain(tier)
|
store_results = drain(tier)
|
||||||
assert len(store_results) == 1
|
assert len(store_results) == 1
|
||||||
assert store_results[0].success
|
assert store_results[0].success
|
||||||
@@ -396,15 +396,15 @@ def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size):
|
|||||||
tensor[:] = 0.0
|
tensor[:] = 0.0
|
||||||
|
|
||||||
# Load into a range disjoint by index from the store ids, to also
|
# Load into a range disjoint by index from the store ids, to also
|
||||||
# exercise loading a block into a different id than it was stored from.
|
# exercise loading a chunk into a different id than it was stored from.
|
||||||
tier.submit_load(make_job(2, keys, load_block_ids, is_promotion=True))
|
tier.submit_load(make_job(2, keys, load_chunk_ids, is_promotion=True))
|
||||||
load_results = drain(tier)
|
load_results = drain(tier)
|
||||||
assert len(load_results) == 1
|
assert len(load_results) == 1
|
||||||
assert load_results[0].success
|
assert load_results[0].success
|
||||||
|
|
||||||
for i, bid in enumerate(load_block_ids):
|
for i, cid in enumerate(load_chunk_ids):
|
||||||
assert torch.allclose(tensor[bid], expected[i]), (
|
assert torch.allclose(tensor[cid], expected[i]), (
|
||||||
f"Block {bid} data mismatch after store+load"
|
f"Chunk {cid} data mismatch after store+load"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -895,7 +895,7 @@ def test_cascade_store_emits_fs_event_through_tiering_manager(tmp_path):
|
|||||||
view = memoryview(tensor.numpy())
|
view = memoryview(tensor.numpy())
|
||||||
mock_region = MagicMock()
|
mock_region = MagicMock()
|
||||||
mock_region.create_kv_memoryview.return_value = view
|
mock_region.create_kv_memoryview.return_value = view
|
||||||
primary = CPUPrimaryTierOffloadingManager(num_blocks=4, mmap_region=mock_region)
|
primary = CPUPrimaryTierOffloadingManager(num_chunks=4, mmap_region=mock_region)
|
||||||
tier = FileSystemTierManager(
|
tier = FileSystemTierManager(
|
||||||
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
|
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
|
||||||
primary_kv_view=primary.get_kv_memoryview(),
|
primary_kv_view=primary.get_kv_memoryview(),
|
||||||
|
|||||||
@@ -25,8 +25,8 @@ def to_keys(int_ids: Iterable[int]):
|
|||||||
def test_tiering_metrics_tracker_records_lookup_metrics():
|
def test_tiering_metrics_tracker_records_lookup_metrics():
|
||||||
tracker = TieringMetricsTracker(
|
tracker = TieringMetricsTracker(
|
||||||
tier_types=["fs", "p2p"],
|
tier_types=["fs", "p2p"],
|
||||||
num_primary_blocks=5,
|
num_primary_chunks=5,
|
||||||
primary_block_size=16,
|
primary_chunk_size=16,
|
||||||
)
|
)
|
||||||
tracker.on_new_request(_CTX)
|
tracker.on_new_request(_CTX)
|
||||||
|
|
||||||
@@ -45,18 +45,18 @@ def test_tiering_metrics_tracker_records_lookup_metrics():
|
|||||||
stats = tracker.take_stats()
|
stats = tracker.take_stats()
|
||||||
assert stats is not None
|
assert stats is not None
|
||||||
values = stats.data["data"]
|
values = stats.data["data"]
|
||||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1
|
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("0:primary",)] == 1
|
||||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1
|
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("1:fs",)] == 1
|
||||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("2:p2p",)] == 1
|
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("2:p2p",)] == 1
|
||||||
assert values[TieringOffloadingMetrics.BLOCK_HITS][("2:p2p",)] == 1
|
assert values[TieringOffloadingMetrics.CHUNK_HITS][("2:p2p",)] == 1
|
||||||
assert ("1:fs",) not in values[TieringOffloadingMetrics.BLOCK_HITS]
|
assert ("1:fs",) not in values[TieringOffloadingMetrics.CHUNK_HITS]
|
||||||
|
|
||||||
|
|
||||||
def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
|
def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
|
||||||
tracker = TieringMetricsTracker(
|
tracker = TieringMetricsTracker(
|
||||||
tier_types=["fs"],
|
tier_types=["fs"],
|
||||||
num_primary_blocks=5,
|
num_primary_chunks=5,
|
||||||
primary_block_size=16,
|
primary_chunk_size=16,
|
||||||
)
|
)
|
||||||
tracker.on_new_request(_CTX)
|
tracker.on_new_request(_CTX)
|
||||||
key = to_keys([1])[0]
|
key = to_keys([1])[0]
|
||||||
@@ -73,9 +73,9 @@ def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
|
|||||||
stats = tracker.take_stats()
|
stats = tracker.take_stats()
|
||||||
assert stats is not None
|
assert stats is not None
|
||||||
values = stats.data["data"]
|
values = stats.data["data"]
|
||||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1
|
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("0:primary",)] == 1
|
||||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1
|
assert values[TieringOffloadingMetrics.CHUNK_QUERIES][("1:fs",)] == 1
|
||||||
assert values[TieringOffloadingMetrics.BLOCK_HITS][("1:fs",)] == 1
|
assert values[TieringOffloadingMetrics.CHUNK_HITS][("1:fs",)] == 1
|
||||||
|
|
||||||
tracker.on_request_allocated(_CTX)
|
tracker.on_request_allocated(_CTX)
|
||||||
tracker.on_lookup(
|
tracker.on_lookup(
|
||||||
@@ -90,15 +90,15 @@ def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
|
|||||||
stats = tracker.take_stats()
|
stats = tracker.take_stats()
|
||||||
assert stats is not None
|
assert stats is not None
|
||||||
values = stats.data["data"]
|
values = stats.data["data"]
|
||||||
assert TieringOffloadingMetrics.BLOCK_QUERIES not in values
|
assert TieringOffloadingMetrics.CHUNK_QUERIES not in values
|
||||||
assert TieringOffloadingMetrics.BLOCK_HITS not in values
|
assert TieringOffloadingMetrics.CHUNK_HITS not in values
|
||||||
|
|
||||||
|
|
||||||
def test_tiering_metrics_tracker_records_finished_job_metrics():
|
def test_tiering_metrics_tracker_records_finished_job_metrics():
|
||||||
tracker = TieringMetricsTracker(
|
tracker = TieringMetricsTracker(
|
||||||
tier_types=["fs"],
|
tier_types=["fs"],
|
||||||
num_primary_blocks=5,
|
num_primary_chunks=5,
|
||||||
primary_block_size=16,
|
primary_chunk_size=16,
|
||||||
)
|
)
|
||||||
cascade_key, promotion_key, failed_cascade_key, failed_promotion_key = to_keys(
|
cascade_key, promotion_key, failed_cascade_key, failed_promotion_key = to_keys(
|
||||||
range(4)
|
range(4)
|
||||||
@@ -149,8 +149,8 @@ def test_tiering_metrics_tracker_records_finished_job_metrics():
|
|||||||
def test_tiering_metrics_tracker_records_partial_promotion_success_bytes():
|
def test_tiering_metrics_tracker_records_partial_promotion_success_bytes():
|
||||||
tracker = TieringMetricsTracker(
|
tracker = TieringMetricsTracker(
|
||||||
tier_types=["fs"],
|
tier_types=["fs"],
|
||||||
num_primary_blocks=5,
|
num_primary_chunks=5,
|
||||||
primary_block_size=16,
|
primary_chunk_size=16,
|
||||||
)
|
)
|
||||||
keys = to_keys(range(3))
|
keys = to_keys(range(3))
|
||||||
job = JobMetadata(
|
job = JobMetadata(
|
||||||
@@ -182,8 +182,8 @@ def test_tiering_metrics_tracker_records_partial_promotion_success_bytes():
|
|||||||
def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges():
|
def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges():
|
||||||
tracker = TieringMetricsTracker(
|
tracker = TieringMetricsTracker(
|
||||||
tier_types=["fs", "p2p"],
|
tier_types=["fs", "p2p"],
|
||||||
num_primary_blocks=6,
|
num_primary_chunks=6,
|
||||||
primary_block_size=16,
|
primary_chunk_size=16,
|
||||||
)
|
)
|
||||||
fs_job = JobMetadata(
|
fs_job = JobMetadata(
|
||||||
TransferJob(0, to_keys([0, 1]), np.array([0, 1]), False, _CTX),
|
TransferJob(0, to_keys([0, 1]), np.array([0, 1]), False, _CTX),
|
||||||
@@ -214,8 +214,8 @@ def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges():
|
|||||||
def test_tiering_metrics_tracker_records_promotion_allocation_failures():
|
def test_tiering_metrics_tracker_records_promotion_allocation_failures():
|
||||||
tracker = TieringMetricsTracker(
|
tracker = TieringMetricsTracker(
|
||||||
tier_types=["fs"],
|
tier_types=["fs"],
|
||||||
num_primary_blocks=1,
|
num_primary_chunks=1,
|
||||||
primary_block_size=16,
|
primary_chunk_size=16,
|
||||||
)
|
)
|
||||||
|
|
||||||
tracker.on_promotion_allocation_failure()
|
tracker.on_promotion_allocation_failure()
|
||||||
|
|||||||
@@ -106,14 +106,14 @@ def key(n: int) -> OffloadKey:
|
|||||||
def make_job(
|
def make_job(
|
||||||
job_id: int,
|
job_id: int,
|
||||||
keys: list[OffloadKey],
|
keys: list[OffloadKey],
|
||||||
block_ids: list[int] | None = None,
|
chunk_ids: list[int] | None = None,
|
||||||
) -> TransferJob:
|
) -> TransferJob:
|
||||||
if block_ids is None:
|
if chunk_ids is None:
|
||||||
block_ids = list(range(len(keys)))
|
chunk_ids = list(range(len(keys)))
|
||||||
return TransferJob(
|
return TransferJob(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
keys=keys,
|
keys=keys,
|
||||||
block_ids=np.array(block_ids, dtype=np.int32),
|
chunk_ids=np.array(chunk_ids, dtype=np.int32),
|
||||||
is_promotion=False,
|
is_promotion=False,
|
||||||
req_context=_CTX,
|
req_context=_CTX,
|
||||||
)
|
)
|
||||||
@@ -541,7 +541,7 @@ class TestMockObjTierFailures:
|
|||||||
mmap_region = MagicMock()
|
mmap_region = MagicMock()
|
||||||
mmap_region.create_kv_memoryview.return_value = primary_kv_view
|
mmap_region.create_kv_memoryview.return_value = primary_kv_view
|
||||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||||
num_blocks=num_blocks, mmap_region=mmap_region
|
num_chunks=num_blocks, mmap_region=mmap_region
|
||||||
)
|
)
|
||||||
obj_tier, agent = _make_tier(
|
obj_tier, agent = _make_tier(
|
||||||
num_blocks=num_blocks, primary_kv_view=primary_kv_view
|
num_blocks=num_blocks, primary_kv_view=primary_kv_view
|
||||||
|
|||||||
@@ -5,15 +5,16 @@ Unit tests for TieringOffloadingManager and ExampleSecondaryTierManager.
|
|||||||
|
|
||||||
These tests verify:
|
These tests verify:
|
||||||
1. Basic tiered offloading operations (store, load, lookup)
|
1. Basic tiered offloading operations (store, load, lookup)
|
||||||
2. Cascade behavior (blocks stored to all secondary tiers)
|
2. Cascade behavior (chunks stored to all secondary tiers)
|
||||||
3. Promotion behavior (blocks loaded from secondary to primary to GPU)
|
3. Promotion behavior (chunks loaded from secondary to primary to GPU)
|
||||||
4. ref_cnt management (blocks protected during async transfers)
|
4. ref_cnt management (chunks protected during async transfers)
|
||||||
5. Eviction coordination between tiers
|
5. Eviction coordination between tiers
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -57,10 +58,10 @@ _CTX = ReqContext(req_id="test")
|
|||||||
_MOCK_OFFLOADING_SPEC = MagicMock()
|
_MOCK_OFFLOADING_SPEC = MagicMock()
|
||||||
|
|
||||||
|
|
||||||
def _mock_mmap_region(num_blocks: int, row_bytes: int = 16):
|
def _mock_mmap_region(num_chunks: int, row_bytes: int = 16):
|
||||||
"""Create a mock SharedOffloadRegion for testing."""
|
"""Create a mock SharedOffloadRegion for testing."""
|
||||||
mock = MagicMock()
|
mock = MagicMock()
|
||||||
view = memoryview(torch.zeros((num_blocks, row_bytes), dtype=torch.int8).numpy())
|
view = memoryview(torch.zeros((num_chunks, row_bytes), dtype=torch.int8).numpy())
|
||||||
mock.create_kv_memoryview.return_value = view
|
mock.create_kv_memoryview.return_value = view
|
||||||
return mock
|
return mock
|
||||||
|
|
||||||
@@ -160,7 +161,7 @@ def test_tiering_spec_collects_secondary_metric_definitions(monkeypatch):
|
|||||||
metadata = metrics[MetricsSecondaryTierManager.MY_TIER_METRIC]
|
metadata = metrics[MetricsSecondaryTierManager.MY_TIER_METRIC]
|
||||||
assert metadata.documentation == "Number of bytes served by the test tier."
|
assert metadata.documentation == "Number of bytes served by the test tier."
|
||||||
assert metadata.labelnames == ("tier",)
|
assert metadata.labelnames == ("tier",)
|
||||||
assert metrics[TieringOffloadingMetrics.BLOCK_QUERIES].labelnames == ("tier",)
|
assert metrics[TieringOffloadingMetrics.CHUNK_QUERIES].labelnames == ("tier",)
|
||||||
assert metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY].labelnames == ("tier",)
|
assert metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY].labelnames == ("tier",)
|
||||||
assert metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY].labelnames == ("tier",)
|
assert metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY].labelnames == ("tier",)
|
||||||
assert metrics[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES].labelnames == (
|
assert metrics[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES].labelnames == (
|
||||||
@@ -183,7 +184,7 @@ def test_tiering_spec_collects_secondary_metric_definitions(monkeypatch):
|
|||||||
def test_tiering_manager_aggregates_secondary_stats():
|
def test_tiering_manager_aggregates_secondary_stats():
|
||||||
mock_region = _mock_mmap_region(5)
|
mock_region = _mock_mmap_region(5)
|
||||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||||
num_blocks=5, mmap_region=mock_region
|
num_chunks=5, mmap_region=mock_region
|
||||||
)
|
)
|
||||||
secondary_tier = MetricsSecondaryTierManager(
|
secondary_tier = MetricsSecondaryTierManager(
|
||||||
offloading_spec=_MOCK_OFFLOADING_SPEC,
|
offloading_spec=_MOCK_OFFLOADING_SPEC,
|
||||||
@@ -232,19 +233,19 @@ class TestExampleSecondaryTierManager:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Initially empty
|
# Initially empty
|
||||||
blocks = to_keys(range(3))
|
chunks = to_keys(range(3))
|
||||||
assert tier.lookup(blocks[0], _CTX) is LookupResult.MISS
|
assert tier.lookup(chunks[0], _CTX) is LookupResult.MISS
|
||||||
|
|
||||||
# Store blocks (simulate with direct insertion for testing)
|
# Store chunks (simulate with direct insertion for testing)
|
||||||
tier.blocks[blocks[0]] = True
|
tier.chunks[chunks[0]] = True
|
||||||
tier.blocks[blocks[1]] = True
|
tier.chunks[chunks[1]] = True
|
||||||
|
|
||||||
# Lookup should find first two blocks
|
# Lookup should find first two chunks
|
||||||
assert tier.lookup(blocks[0], _CTX) is LookupResult.HIT
|
assert tier.lookup(chunks[0], _CTX) is LookupResult.HIT
|
||||||
assert tier.lookup(blocks[1], _CTX) is LookupResult.HIT
|
assert tier.lookup(chunks[1], _CTX) is LookupResult.HIT
|
||||||
|
|
||||||
# Third block not present
|
# Third chunk not present
|
||||||
assert tier.lookup(blocks[2], _CTX) is LookupResult.MISS
|
assert tier.lookup(chunks[2], _CTX) is LookupResult.MISS
|
||||||
|
|
||||||
|
|
||||||
# What a request-level cascade does with a key already present in the primary
|
# What a request-level cascade does with a key already present in the primary
|
||||||
@@ -260,7 +261,7 @@ _CASCADE_SUPPLY_BEHAVIOR = {
|
|||||||
# The same test, before it lands.
|
# The same test, before it lands.
|
||||||
LookupResult.HIT_PENDING: "defer",
|
LookupResult.HIT_PENDING: "defer",
|
||||||
# test_cascade_drops_deferred_keys_whose_primary_write_failed, where the
|
# test_cascade_drops_deferred_keys_whose_primary_write_failed, where the
|
||||||
# failed write frees the block.
|
# failed write frees the chunk.
|
||||||
LookupResult.MISS: "drop",
|
LookupResult.MISS: "drop",
|
||||||
# test_cascade_rejects_retry_from_primary. The primary tier resolves every
|
# test_cascade_rejects_retry_from_primary. The primary tier resolves every
|
||||||
# key it holds, so RETRY cannot reach the cascade, and parking on it would
|
# key it holds, so RETRY cannot reach the cascade, and parking on it would
|
||||||
@@ -284,7 +285,7 @@ class TestTieringOffloadingManager:
|
|||||||
# Create primary tier (CPU-based)
|
# Create primary tier (CPU-based)
|
||||||
mock_region = _mock_mmap_region(5)
|
mock_region = _mock_mmap_region(5)
|
||||||
self.primary_tier = CPUPrimaryTierOffloadingManager(
|
self.primary_tier = CPUPrimaryTierOffloadingManager(
|
||||||
num_blocks=5, mmap_region=mock_region
|
num_chunks=5, mmap_region=mock_region
|
||||||
)
|
)
|
||||||
|
|
||||||
mock_view = mock_region.create_kv_memoryview()
|
mock_view = mock_region.create_kv_memoryview()
|
||||||
@@ -331,7 +332,7 @@ class TestTieringOffloadingManager:
|
|||||||
TransferJob(
|
TransferJob(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
keys=to_keys([1, 2]),
|
keys=to_keys([1, 2]),
|
||||||
block_ids=[0, 1],
|
chunk_ids=np.array([0, 1], dtype=np.int64),
|
||||||
is_promotion=True,
|
is_promotion=True,
|
||||||
req_context=_CTX,
|
req_context=_CTX,
|
||||||
),
|
),
|
||||||
@@ -358,7 +359,7 @@ class TestTieringOffloadingManager:
|
|||||||
TransferJob(
|
TransferJob(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
keys=to_keys([1]),
|
keys=to_keys([1]),
|
||||||
block_ids=[0],
|
chunk_ids=np.array([0], dtype=np.int64),
|
||||||
is_promotion=True,
|
is_promotion=True,
|
||||||
req_context=_CTX,
|
req_context=_CTX,
|
||||||
),
|
),
|
||||||
@@ -392,23 +393,23 @@ class TestTieringOffloadingManager:
|
|||||||
|
|
||||||
def test_basic_store_to_primary(self, manager_setup):
|
def test_basic_store_to_primary(self, manager_setup):
|
||||||
"""Test basic store operation to primary tier."""
|
"""Test basic store operation to primary tier."""
|
||||||
blocks = to_keys(range(3))
|
chunks = to_keys(range(3))
|
||||||
|
|
||||||
# Prepare store
|
# Prepare store
|
||||||
self._start_request()
|
self._start_request()
|
||||||
result = self.manager.prepare_store(blocks, _CTX)
|
result = self.manager.prepare_store(chunks, _CTX)
|
||||||
assert result is not None
|
assert result is not None
|
||||||
assert len(result.keys_to_store) == 3
|
assert len(result.keys_to_store) == 3
|
||||||
|
|
||||||
# Complete store
|
# Complete store
|
||||||
self.manager.complete_store(blocks, _CTX, success=True)
|
self.manager.complete_store(chunks, _CTX, success=True)
|
||||||
|
|
||||||
# Blocks should be in primary tier
|
# Chunks should be in primary tier
|
||||||
assert count_hits(self.primary_tier, blocks) == 3
|
assert count_hits(self.primary_tier, chunks) == 3
|
||||||
|
|
||||||
def test_cascade_to_all_secondary_tiers(self, manager_setup):
|
def test_cascade_to_all_secondary_tiers(self, manager_setup):
|
||||||
"""Test that blocks are cascaded to ALL secondary tiers."""
|
"""Test that chunks are cascaded to ALL secondary tiers."""
|
||||||
blocks = to_keys(range(3))
|
chunks = to_keys(range(3))
|
||||||
|
|
||||||
self.secondary_tier1.submit_store = MagicMock(
|
self.secondary_tier1.submit_store = MagicMock(
|
||||||
wraps=self.secondary_tier1.submit_store
|
wraps=self.secondary_tier1.submit_store
|
||||||
@@ -419,44 +420,44 @@ class TestTieringOffloadingManager:
|
|||||||
|
|
||||||
# Store to primary
|
# Store to primary
|
||||||
self._start_request()
|
self._start_request()
|
||||||
result = self.manager.prepare_store(blocks, _CTX)
|
result = self.manager.prepare_store(chunks, _CTX)
|
||||||
assert result is not None
|
assert result is not None
|
||||||
|
|
||||||
# Complete store (triggers cascade via submit_store on each tier)
|
# Complete store (triggers cascade via submit_store on each tier)
|
||||||
self.manager.complete_store(blocks, _CTX, success=True)
|
self.manager.complete_store(chunks, _CTX, success=True)
|
||||||
|
|
||||||
# submit_store was called once per secondary tier
|
# submit_store was called once per secondary tier
|
||||||
self.secondary_tier1.submit_store.assert_called_once()
|
self.secondary_tier1.submit_store.assert_called_once()
|
||||||
self.secondary_tier2.submit_store.assert_called_once()
|
self.secondary_tier2.submit_store.assert_called_once()
|
||||||
|
|
||||||
# Blocks should be in both secondary tiers
|
# Chunks should be in both secondary tiers
|
||||||
assert self.secondary_tier1.get_num_blocks() == 3
|
assert self.secondary_tier1.get_num_chunks() == 3
|
||||||
assert self.secondary_tier2.get_num_blocks() == 3
|
assert self.secondary_tier2.get_num_chunks() == 3
|
||||||
|
|
||||||
# Verify blocks are present
|
# Verify chunks are present
|
||||||
assert all(
|
assert all(
|
||||||
self.secondary_tier1.lookup(b, _CTX) is LookupResult.HIT for b in blocks
|
self.secondary_tier1.lookup(c, _CTX) is LookupResult.HIT for c in chunks
|
||||||
)
|
)
|
||||||
assert all(
|
assert all(
|
||||||
self.secondary_tier2.lookup(b, _CTX) is LookupResult.HIT for b in blocks
|
self.secondary_tier2.lookup(c, _CTX) is LookupResult.HIT for c in chunks
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_ref_cnt_protection_during_cascade(self, manager_setup):
|
def test_ref_cnt_protection_during_cascade(self, manager_setup):
|
||||||
"""Test that ref_cnt protects blocks during cascade."""
|
"""Test that ref_cnt protects chunks during cascade."""
|
||||||
blocks = to_keys(range(3))
|
chunks = to_keys(range(3))
|
||||||
|
|
||||||
# Store to primary
|
# Store to primary
|
||||||
self._start_request()
|
self._start_request()
|
||||||
result = self.manager.prepare_store(blocks, _CTX)
|
result = self.manager.prepare_store(chunks, _CTX)
|
||||||
assert result is not None
|
assert result is not None
|
||||||
self.manager.complete_store(blocks, _CTX, success=True)
|
self.manager.complete_store(chunks, _CTX, success=True)
|
||||||
|
|
||||||
# After complete_store, blocks should have ref_cnt > 0
|
# After complete_store, chunks should have ref_cnt > 0
|
||||||
# (one for each secondary tier)
|
# (one for each secondary tier)
|
||||||
for block_hash in blocks:
|
for key in chunks:
|
||||||
block = self.primary_tier._policy.get(block_hash)
|
chunk = self.primary_tier._policy.get(key)
|
||||||
# ref_cnt should be 2 (one for each secondary tier)
|
# ref_cnt should be 2 (one for each secondary tier)
|
||||||
assert block.ref_cnt == 2
|
assert chunk.ref_cnt == 2
|
||||||
|
|
||||||
# End of step 1: _maybe_process_finished_jobs() was already called by
|
# End of step 1: _maybe_process_finished_jobs() was already called by
|
||||||
# prepare_store() above (setting the per-step flag), so on_schedule_end()
|
# prepare_store() above (setting the per-step flag), so on_schedule_end()
|
||||||
@@ -466,9 +467,9 @@ class TestTieringOffloadingManager:
|
|||||||
|
|
||||||
# ref_cnt still held: cascade jobs finished (sync tier) but haven't
|
# ref_cnt still held: cascade jobs finished (sync tier) but haven't
|
||||||
# been polled yet because the per-step guard skipped the second call.
|
# been polled yet because the per-step guard skipped the second call.
|
||||||
for block_hash in blocks:
|
for key in chunks:
|
||||||
block = self.primary_tier._policy.get(block_hash)
|
chunk = self.primary_tier._policy.get(key)
|
||||||
assert block.ref_cnt == 2
|
assert chunk.ref_cnt == 2
|
||||||
|
|
||||||
# Secondary tiers have completed jobs waiting to be drained
|
# Secondary tiers have completed jobs waiting to be drained
|
||||||
assert len(self.secondary_tier1.completed_jobs) > 0
|
assert len(self.secondary_tier1.completed_jobs) > 0
|
||||||
@@ -479,37 +480,37 @@ class TestTieringOffloadingManager:
|
|||||||
self._simulate_on_schedule_end()
|
self._simulate_on_schedule_end()
|
||||||
|
|
||||||
# After cascade completes, ref_cnt should be 0
|
# After cascade completes, ref_cnt should be 0
|
||||||
for block_hash in blocks:
|
for key in chunks:
|
||||||
block = self.primary_tier._policy.get(block_hash)
|
chunk = self.primary_tier._policy.get(key)
|
||||||
assert block.ref_cnt == 0
|
assert chunk.ref_cnt == 0
|
||||||
|
|
||||||
# All completed jobs have been drained
|
# All completed jobs have been drained
|
||||||
assert len(self.secondary_tier1.completed_jobs) == 0
|
assert len(self.secondary_tier1.completed_jobs) == 0
|
||||||
assert len(self.secondary_tier2.completed_jobs) == 0
|
assert len(self.secondary_tier2.completed_jobs) == 0
|
||||||
|
|
||||||
def test_lookup_from_primary(self, manager_setup):
|
def test_lookup_from_primary(self, manager_setup):
|
||||||
"""Test lookup when blocks are in primary tier."""
|
"""Test lookup when chunks are in primary tier."""
|
||||||
blocks = to_keys(range(3))
|
chunks = to_keys(range(3))
|
||||||
|
|
||||||
# Store blocks
|
# Store chunks
|
||||||
self._start_request()
|
self._start_request()
|
||||||
self.manager.prepare_store(blocks, _CTX)
|
self.manager.prepare_store(chunks, _CTX)
|
||||||
self.manager.complete_store(blocks, _CTX, success=True)
|
self.manager.complete_store(chunks, _CTX, success=True)
|
||||||
|
|
||||||
# Lookup should find all blocks in primary
|
# Lookup should find all chunks in primary
|
||||||
assert count_hits(self.manager, blocks) == 3
|
assert count_hits(self.manager, chunks) == 3
|
||||||
|
|
||||||
def test_promotion_from_secondary(self, manager_setup):
|
def test_promotion_from_secondary(self, manager_setup):
|
||||||
"""Test promotion of blocks from secondary to primary tier."""
|
"""Test promotion of chunks from secondary to primary tier."""
|
||||||
blocks = to_keys(range(3))
|
chunks = to_keys(range(3))
|
||||||
|
|
||||||
# Manually add blocks to secondary tier (simulate previous cascade)
|
# Manually add chunks to secondary tier (simulate previous cascade)
|
||||||
for block in blocks:
|
for chunk in chunks:
|
||||||
self.secondary_tier1.blocks[block] = True
|
self.secondary_tier1.chunks[chunk] = True
|
||||||
|
|
||||||
# Lookup each block to initiate promotion for all of them
|
# Lookup each chunk to initiate promotion for all of them
|
||||||
for block in blocks:
|
for chunk in chunks:
|
||||||
result = self.manager.lookup(block, _CTX)
|
result = self.manager.lookup(chunk, _CTX)
|
||||||
assert result is LookupResult.HIT_PENDING # promotion initiated
|
assert result is LookupResult.HIT_PENDING # promotion initiated
|
||||||
|
|
||||||
# End of step 1: flushes deferred submit_load() calls
|
# End of step 1: flushes deferred submit_load() calls
|
||||||
@@ -518,11 +519,11 @@ class TestTieringOffloadingManager:
|
|||||||
# End of step 2: processes the completed promotion jobs
|
# End of step 2: processes the completed promotion jobs
|
||||||
self._simulate_on_schedule_end()
|
self._simulate_on_schedule_end()
|
||||||
|
|
||||||
# Now blocks should be in primary tier
|
# Now chunks should be in primary tier
|
||||||
assert count_hits(self.primary_tier, blocks) == 3
|
assert count_hits(self.primary_tier, chunks) == 3
|
||||||
|
|
||||||
# Next lookup should succeed
|
# Next lookup should succeed
|
||||||
assert count_hits(self.manager, blocks) == 3
|
assert count_hits(self.manager, chunks) == 3
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("successful_indices", "expected_results"),
|
("successful_indices", "expected_results"),
|
||||||
@@ -538,18 +539,18 @@ class TestTieringOffloadingManager:
|
|||||||
],
|
],
|
||||||
ids=["partial", "legacy-full-failure"],
|
ids=["partial", "legacy-full-failure"],
|
||||||
)
|
)
|
||||||
def test_failed_promotion_keeps_only_successful_blocks(
|
def test_failed_promotion_keeps_only_successful_chunks(
|
||||||
self, manager_setup, successful_indices, expected_results
|
self, manager_setup, successful_indices, expected_results
|
||||||
):
|
):
|
||||||
blocks = to_keys(range(3))
|
chunks = to_keys(range(3))
|
||||||
for block in blocks:
|
for chunk in chunks:
|
||||||
self.secondary_tier1.blocks[block] = True
|
self.secondary_tier1.chunks[chunk] = True
|
||||||
|
|
||||||
def submit_partial(job_metadata: TransferJob) -> None:
|
def submit_partial(job_metadata: TransferJob) -> None:
|
||||||
successful_keys = (
|
successful_keys = (
|
||||||
None
|
None
|
||||||
if successful_indices is None
|
if successful_indices is None
|
||||||
else tuple(blocks[i] for i in successful_indices)
|
else tuple(chunks[i] for i in successful_indices)
|
||||||
)
|
)
|
||||||
self.secondary_tier1.completed_jobs.append(
|
self.secondary_tier1.completed_jobs.append(
|
||||||
JobResult(
|
JobResult(
|
||||||
@@ -561,24 +562,24 @@ class TestTieringOffloadingManager:
|
|||||||
|
|
||||||
self.secondary_tier1.submit_load = submit_partial
|
self.secondary_tier1.submit_load = submit_partial
|
||||||
|
|
||||||
for block in blocks:
|
for chunk in chunks:
|
||||||
assert self.manager.lookup(block, _CTX) is LookupResult.HIT_PENDING
|
assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT_PENDING
|
||||||
|
|
||||||
self._simulate_on_schedule_end()
|
self._simulate_on_schedule_end()
|
||||||
self._simulate_on_schedule_end()
|
self._simulate_on_schedule_end()
|
||||||
|
|
||||||
assert [
|
assert [
|
||||||
self.primary_tier.lookup(block, _CTX) for block in blocks
|
self.primary_tier.lookup(chunk, _CTX) for chunk in chunks
|
||||||
] == expected_results
|
] == expected_results
|
||||||
|
|
||||||
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
|
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
|
||||||
"""Resolved lookups report one sync delay sample per tier and block."""
|
"""Resolved lookups report one sync delay sample per tier and chunk."""
|
||||||
self._start_request()
|
self._start_request()
|
||||||
blocks = to_keys(range(2))
|
chunks = to_keys(range(2))
|
||||||
|
|
||||||
# No tier has these blocks: they resolve immediately as misses.
|
# No tier has these chunks: they resolve immediately as misses.
|
||||||
for block in blocks:
|
for chunk in chunks:
|
||||||
assert self.manager.lookup(block, _CTX) is LookupResult.MISS
|
assert self.manager.lookup(chunk, _CTX) is LookupResult.MISS
|
||||||
|
|
||||||
stats = self.manager.get_stats()
|
stats = self.manager.get_stats()
|
||||||
assert stats is not None
|
assert stats is not None
|
||||||
@@ -614,11 +615,11 @@ class TestTieringOffloadingManager:
|
|||||||
def test_lookup_does_not_report_async_delay_for_promotion(self, manager_setup):
|
def test_lookup_does_not_report_async_delay_for_promotion(self, manager_setup):
|
||||||
"""Promotion time is not included in lookup async delay."""
|
"""Promotion time is not included in lookup async delay."""
|
||||||
self._start_request()
|
self._start_request()
|
||||||
block = to_keys(range(1))[0]
|
chunk = to_keys(range(1))[0]
|
||||||
self.secondary_tier1.blocks[block] = True
|
self.secondary_tier1.chunks[chunk] = True
|
||||||
|
|
||||||
# First lookup finds the block in a secondary tier and defers.
|
# First lookup finds the chunk in a secondary tier and defers.
|
||||||
assert self.manager.lookup(block, _CTX) is LookupResult.HIT_PENDING
|
assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT_PENDING
|
||||||
|
|
||||||
# Promotion is not treated as unresolved secondary lookup time.
|
# Promotion is not treated as unresolved secondary lookup time.
|
||||||
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
|
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
|
||||||
@@ -648,8 +649,8 @@ class TestTieringOffloadingManager:
|
|||||||
# Promotion completes on the next scheduler step.
|
# Promotion completes on the next scheduler step.
|
||||||
self._simulate_on_schedule_end()
|
self._simulate_on_schedule_end()
|
||||||
|
|
||||||
# Next lookup resolves via the now-promoted primary-tier block.
|
# Next lookup resolves via the now-promoted primary-tier chunk.
|
||||||
assert self.manager.lookup(block, _CTX) is LookupResult.HIT
|
assert self.manager.lookup(chunk, _CTX) is LookupResult.HIT
|
||||||
|
|
||||||
stats = self.manager.get_stats()
|
stats = self.manager.get_stats()
|
||||||
if stats is not None:
|
if stats is not None:
|
||||||
@@ -663,13 +664,13 @@ class TestTieringOffloadingManager:
|
|||||||
"""Async delay is observed when a RETRY lookup later resolves."""
|
"""Async delay is observed when a RETRY lookup later resolves."""
|
||||||
ctx = ReqContext(req_id="req_lookup_finish")
|
ctx = ReqContext(req_id="req_lookup_finish")
|
||||||
self._start_request(ctx)
|
self._start_request(ctx)
|
||||||
block = to_keys(range(1))[0]
|
chunk = to_keys(range(1))[0]
|
||||||
self.secondary_tier1.lookup = MagicMock(
|
self.secondary_tier1.lookup = MagicMock(
|
||||||
side_effect=[LookupResult.RETRY, LookupResult.HIT]
|
side_effect=[LookupResult.RETRY, LookupResult.HIT]
|
||||||
)
|
)
|
||||||
|
|
||||||
# First lookup is deferred by the secondary tier.
|
# First lookup is deferred by the secondary tier.
|
||||||
assert self.manager.lookup(block, ctx) is LookupResult.RETRY
|
assert self.manager.lookup(chunk, ctx) is LookupResult.RETRY
|
||||||
|
|
||||||
stats = self.manager.get_stats()
|
stats = self.manager.get_stats()
|
||||||
if stats is not None:
|
if stats is not None:
|
||||||
@@ -677,7 +678,7 @@ class TestTieringOffloadingManager:
|
|||||||
stats.reduce(), TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
|
stats.reduce(), TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
|
||||||
)
|
)
|
||||||
|
|
||||||
assert self.manager.lookup(block, ctx) is LookupResult.HIT_PENDING
|
assert self.manager.lookup(chunk, ctx) is LookupResult.HIT_PENDING
|
||||||
|
|
||||||
stats = self.manager.get_stats()
|
stats = self.manager.get_stats()
|
||||||
assert stats is not None
|
assert stats is not None
|
||||||
@@ -701,68 +702,68 @@ class TestTieringOffloadingManager:
|
|||||||
|
|
||||||
def test_partial_lookup(self, manager_setup):
|
def test_partial_lookup(self, manager_setup):
|
||||||
"""Test lookup with partial hits."""
|
"""Test lookup with partial hits."""
|
||||||
blocks = to_keys(range(5))
|
chunks = to_keys(range(5))
|
||||||
|
|
||||||
# Store first 3 blocks to primary
|
# Store first 3 chunks to primary
|
||||||
self._start_request()
|
self._start_request()
|
||||||
self.manager.prepare_store(blocks[:3], _CTX)
|
self.manager.prepare_store(chunks[:3], _CTX)
|
||||||
self.manager.complete_store(blocks[:3], _CTX, success=True)
|
self.manager.complete_store(chunks[:3], _CTX, success=True)
|
||||||
|
|
||||||
# Lookup all 5 blocks should return 3 (first 3 found)
|
# Lookup all 5 chunks should return 3 (first 3 found)
|
||||||
assert count_hits(self.manager, blocks) == 3
|
assert count_hits(self.manager, chunks) == 3
|
||||||
|
|
||||||
def test_eviction_in_primary_tier(self, manager_setup):
|
def test_eviction_in_primary_tier(self, manager_setup):
|
||||||
"""Test eviction in primary tier when capacity is exceeded."""
|
"""Test eviction in primary tier when capacity is exceeded."""
|
||||||
# Primary tier has capacity of 5 blocks
|
# Primary tier has capacity of 5 chunks
|
||||||
# First, fill the primary tier
|
# First, fill the primary tier
|
||||||
blocks = to_keys(range(5))
|
chunks = to_keys(range(5))
|
||||||
self._start_request()
|
self._start_request()
|
||||||
result = self.manager.prepare_store(blocks, _CTX)
|
result = self.manager.prepare_store(chunks, _CTX)
|
||||||
assert result is not None
|
assert result is not None
|
||||||
assert len(result.keys_to_store) == 5
|
assert len(result.keys_to_store) == 5
|
||||||
self.manager.complete_store(blocks, _CTX, success=True)
|
self.manager.complete_store(chunks, _CTX, success=True)
|
||||||
|
|
||||||
# End of step: release ref_cnt from cascade
|
# End of step: release ref_cnt from cascade
|
||||||
self._simulate_on_schedule_end()
|
self._simulate_on_schedule_end()
|
||||||
|
|
||||||
# Now try to store 2 more blocks (should trigger eviction)
|
# Now try to store 2 more chunks (should trigger eviction)
|
||||||
more_blocks = to_keys(range(5, 7))
|
more_chunks = to_keys(range(5, 7))
|
||||||
result = self.manager.prepare_store(more_blocks, _CTX)
|
result = self.manager.prepare_store(more_chunks, _CTX)
|
||||||
|
|
||||||
# Should evict 2 blocks from primary tier
|
# Should evict 2 chunks from primary tier
|
||||||
assert result is not None
|
assert result is not None
|
||||||
assert len(result.evicted_keys) == 2
|
assert len(result.evicted_keys) == 2
|
||||||
assert len(result.keys_to_store) == 2
|
assert len(result.keys_to_store) == 2
|
||||||
|
|
||||||
def test_touch_propagates_to_all_tiers(self, manager_setup):
|
def test_touch_propagates_to_all_tiers(self, manager_setup):
|
||||||
"""Test that touch() propagates to all tiers."""
|
"""Test that touch() propagates to all tiers."""
|
||||||
blocks = to_keys(range(3))
|
chunks = to_keys(range(3))
|
||||||
|
|
||||||
# Store blocks
|
# Store chunks
|
||||||
self._start_request()
|
self._start_request()
|
||||||
self.manager.prepare_store(blocks, _CTX)
|
self.manager.prepare_store(chunks, _CTX)
|
||||||
self.manager.complete_store(blocks, _CTX, success=True)
|
self.manager.complete_store(chunks, _CTX, success=True)
|
||||||
self._simulate_on_schedule_end()
|
self._simulate_on_schedule_end()
|
||||||
# for secondary tiers to drain jobs, so primary tier's blocks are evictable.
|
# for secondary tiers to drain jobs, so primary tier's chunks are evictable.
|
||||||
self._simulate_on_schedule_end()
|
self._simulate_on_schedule_end()
|
||||||
|
|
||||||
self.secondary_tier1.touch = MagicMock(wraps=self.secondary_tier1.touch)
|
self.secondary_tier1.touch = MagicMock(wraps=self.secondary_tier1.touch)
|
||||||
self.secondary_tier2.touch = MagicMock(wraps=self.secondary_tier2.touch)
|
self.secondary_tier2.touch = MagicMock(wraps=self.secondary_tier2.touch)
|
||||||
|
|
||||||
# Touch blocks
|
# Touch chunks
|
||||||
self.manager.touch(blocks, _CTX)
|
self.manager.touch(chunks, _CTX)
|
||||||
|
|
||||||
# Verify touch was called on primary tier (check LRU order)
|
# Verify touch was called on primary tier (check LRU order)
|
||||||
primary_keys = list(self.primary_tier._policy.evictable_blocks.keys())
|
primary_keys = list(self.primary_tier._policy.evictable_chunks.keys())
|
||||||
assert primary_keys[-3:] == list(reversed(blocks))
|
assert primary_keys[-3:] == list(reversed(chunks))
|
||||||
|
|
||||||
# Verify touch was propagated to all secondary tiers
|
# Verify touch was propagated to all secondary tiers
|
||||||
self.secondary_tier1.touch.assert_called_once_with(blocks, _CTX)
|
self.secondary_tier1.touch.assert_called_once_with(chunks, _CTX)
|
||||||
self.secondary_tier2.touch.assert_called_once_with(blocks, _CTX)
|
self.secondary_tier2.touch.assert_called_once_with(chunks, _CTX)
|
||||||
|
|
||||||
def test_failed_store_no_cascade(self, manager_setup):
|
def test_failed_store_no_cascade(self, manager_setup):
|
||||||
"""Test that failed GPU→primary store doesn't cascade."""
|
"""Test that failed GPU→primary store doesn't cascade."""
|
||||||
blocks = to_keys(range(3))
|
chunks = to_keys(range(3))
|
||||||
|
|
||||||
self.secondary_tier1.submit_store = MagicMock(
|
self.secondary_tier1.submit_store = MagicMock(
|
||||||
wraps=self.secondary_tier1.submit_store
|
wraps=self.secondary_tier1.submit_store
|
||||||
@@ -773,11 +774,11 @@ class TestTieringOffloadingManager:
|
|||||||
|
|
||||||
# Prepare store
|
# Prepare store
|
||||||
self._start_request()
|
self._start_request()
|
||||||
result = self.manager.prepare_store(blocks, _CTX)
|
result = self.manager.prepare_store(chunks, _CTX)
|
||||||
assert result is not None
|
assert result is not None
|
||||||
|
|
||||||
# Complete store with failure — cascade must not happen
|
# Complete store with failure — cascade must not happen
|
||||||
self.manager.complete_store(blocks, _CTX, success=False)
|
self.manager.complete_store(chunks, _CTX, success=False)
|
||||||
|
|
||||||
# submit_store was never called on either secondary tier
|
# submit_store was never called on either secondary tier
|
||||||
self.secondary_tier1.submit_store.assert_not_called()
|
self.secondary_tier1.submit_store.assert_not_called()
|
||||||
@@ -786,12 +787,12 @@ class TestTieringOffloadingManager:
|
|||||||
def test_lookup_batches_submit_load_per_request(self, manager_setup):
|
def test_lookup_batches_submit_load_per_request(self, manager_setup):
|
||||||
"""lookup() defers submit_load until on_schedule_end(), one per request.
|
"""lookup() defers submit_load until on_schedule_end(), one per request.
|
||||||
|
|
||||||
Blocks from different requests each get their own submit_load call, each
|
Chunks from different requests each get their own submit_load call, each
|
||||||
carrying the correct req_context.
|
carrying the correct req_context.
|
||||||
"""
|
"""
|
||||||
blocks = to_keys(range(4))
|
chunks = to_keys(range(4))
|
||||||
for block in blocks:
|
for chunk in chunks:
|
||||||
self.secondary_tier1.blocks[block] = True
|
self.secondary_tier1.chunks[chunk] = True
|
||||||
|
|
||||||
self.secondary_tier1.submit_load = MagicMock(
|
self.secondary_tier1.submit_load = MagicMock(
|
||||||
wraps=self.secondary_tier1.submit_load
|
wraps=self.secondary_tier1.submit_load
|
||||||
@@ -801,10 +802,10 @@ class TestTieringOffloadingManager:
|
|||||||
ctx_b = ReqContext(req_id="req_b")
|
ctx_b = ReqContext(req_id="req_b")
|
||||||
|
|
||||||
# All lookups return HIT_PENDING: secondary hit triggers promotion
|
# All lookups return HIT_PENDING: secondary hit triggers promotion
|
||||||
assert self.manager.lookup(blocks[0], ctx_a) is LookupResult.HIT_PENDING
|
assert self.manager.lookup(chunks[0], ctx_a) is LookupResult.HIT_PENDING
|
||||||
assert self.manager.lookup(blocks[1], ctx_a) is LookupResult.HIT_PENDING
|
assert self.manager.lookup(chunks[1], ctx_a) is LookupResult.HIT_PENDING
|
||||||
assert self.manager.lookup(blocks[2], ctx_b) is LookupResult.HIT_PENDING
|
assert self.manager.lookup(chunks[2], ctx_b) is LookupResult.HIT_PENDING
|
||||||
assert self.manager.lookup(blocks[3], ctx_b) is LookupResult.HIT_PENDING
|
assert self.manager.lookup(chunks[3], ctx_b) is LookupResult.HIT_PENDING
|
||||||
|
|
||||||
# submit_load must not fire during lookup - only at end of step
|
# submit_load must not fire during lookup - only at end of step
|
||||||
self.secondary_tier1.submit_load.assert_not_called()
|
self.secondary_tier1.submit_load.assert_not_called()
|
||||||
@@ -816,20 +817,20 @@ class TestTieringOffloadingManager:
|
|||||||
calls = self.secondary_tier1.submit_load.call_args_list
|
calls = self.secondary_tier1.submit_load.call_args_list
|
||||||
jm_a = calls[0].args[0]
|
jm_a = calls[0].args[0]
|
||||||
jm_b = calls[1].args[0]
|
jm_b = calls[1].args[0]
|
||||||
assert set(jm_a.keys) == {blocks[0], blocks[1]}
|
assert set(jm_a.keys) == {chunks[0], chunks[1]}
|
||||||
assert jm_a.req_context is ctx_a
|
assert jm_a.req_context is ctx_a
|
||||||
assert set(jm_b.keys) == {blocks[2], blocks[3]}
|
assert set(jm_b.keys) == {chunks[2], chunks[3]}
|
||||||
assert jm_b.req_context is ctx_b
|
assert jm_b.req_context is ctx_b
|
||||||
|
|
||||||
def test_lookup_shared_block_no_duplicate_promotion(self, manager_setup):
|
def test_lookup_shared_chunk_no_duplicate_promotion(self, manager_setup):
|
||||||
"""A block looked up by two requests in the same step is promoted once.
|
"""A chunk looked up by two requests in the same step is promoted once.
|
||||||
|
|
||||||
The first lookup initiates promotion (returns None via secondary hit).
|
The first lookup initiates promotion (returns None via secondary hit).
|
||||||
The second lookup sees ref_cnt=-1 on the primary slot and returns None
|
The second lookup sees ref_cnt=-1 on the primary slot and returns None
|
||||||
via the primary in-flight path — without triggering a second promotion.
|
via the primary in-flight path — without triggering a second promotion.
|
||||||
"""
|
"""
|
||||||
shared_block = to_keys([0])[0]
|
shared_chunk = to_keys([0])[0]
|
||||||
self.secondary_tier1.blocks[shared_block] = True
|
self.secondary_tier1.chunks[shared_chunk] = True
|
||||||
|
|
||||||
self.secondary_tier1.submit_load = MagicMock(
|
self.secondary_tier1.submit_load = MagicMock(
|
||||||
wraps=self.secondary_tier1.submit_load
|
wraps=self.secondary_tier1.submit_load
|
||||||
@@ -838,10 +839,10 @@ class TestTieringOffloadingManager:
|
|||||||
ctx_a = ReqContext(req_id="req_a")
|
ctx_a = ReqContext(req_id="req_a")
|
||||||
ctx_b = ReqContext(req_id="req_b")
|
ctx_b = ReqContext(req_id="req_b")
|
||||||
|
|
||||||
result_a = self.manager.lookup(shared_block, ctx_a)
|
result_a = self.manager.lookup(shared_chunk, ctx_a)
|
||||||
result_b = self.manager.lookup(shared_block, ctx_b)
|
result_b = self.manager.lookup(shared_chunk, ctx_b)
|
||||||
|
|
||||||
# Both lookups find the shared_block and trigger promotion
|
# Both lookups find the shared_chunk and trigger promotion
|
||||||
# returning HIT_PENDING.
|
# returning HIT_PENDING.
|
||||||
assert result_a is LookupResult.HIT_PENDING
|
assert result_a is LookupResult.HIT_PENDING
|
||||||
assert result_b is LookupResult.HIT_PENDING
|
assert result_b is LookupResult.HIT_PENDING
|
||||||
@@ -851,12 +852,12 @@ class TestTieringOffloadingManager:
|
|||||||
# Only one submit_load call despite two lookups
|
# Only one submit_load call despite two lookups
|
||||||
self.secondary_tier1.submit_load.assert_called_once()
|
self.secondary_tier1.submit_load.assert_called_once()
|
||||||
job_metadata = self.secondary_tier1.submit_load.call_args.args[0]
|
job_metadata = self.secondary_tier1.submit_load.call_args.args[0]
|
||||||
assert list(job_metadata.keys) == [shared_block]
|
assert list(job_metadata.keys) == [shared_chunk]
|
||||||
assert job_metadata.req_context is ctx_a
|
assert job_metadata.req_context is ctx_a
|
||||||
|
|
||||||
def test_complete_store_forwards_req_context_to_submit_store(self, manager_setup):
|
def test_complete_store_forwards_req_context_to_submit_store(self, manager_setup):
|
||||||
"""complete_store cascades to secondary tiers with the correct req_context."""
|
"""complete_store cascades to secondary tiers with the correct req_context."""
|
||||||
blocks = to_keys(range(2))
|
chunks = to_keys(range(2))
|
||||||
|
|
||||||
self.secondary_tier1.submit_store = MagicMock(
|
self.secondary_tier1.submit_store = MagicMock(
|
||||||
wraps=self.secondary_tier1.submit_store
|
wraps=self.secondary_tier1.submit_store
|
||||||
@@ -865,8 +866,8 @@ class TestTieringOffloadingManager:
|
|||||||
ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"})
|
ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"})
|
||||||
|
|
||||||
self._start_request(ctx)
|
self._start_request(ctx)
|
||||||
self.manager.prepare_store(blocks, ctx)
|
self.manager.prepare_store(chunks, ctx)
|
||||||
self.manager.complete_store(blocks, ctx, success=True)
|
self.manager.complete_store(chunks, ctx, success=True)
|
||||||
|
|
||||||
assert self.secondary_tier1.submit_store.call_count == 1
|
assert self.secondary_tier1.submit_store.call_count == 1
|
||||||
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
||||||
@@ -876,7 +877,7 @@ class TestTieringOffloadingManager:
|
|||||||
self, manager_setup
|
self, manager_setup
|
||||||
):
|
):
|
||||||
"""Manager hook is eager; secondary hooks wait for cascade submission."""
|
"""Manager hook is eager; secondary hooks wait for cascade submission."""
|
||||||
blocks = to_keys(range(2))
|
chunks = to_keys(range(2))
|
||||||
ctx = ReqContext(req_id="req_delayed_secondary")
|
ctx = ReqContext(req_id="req_delayed_secondary")
|
||||||
calls: list[tuple[str, str]] = []
|
calls: list[tuple[str, str]] = []
|
||||||
|
|
||||||
@@ -911,14 +912,14 @@ class TestTieringOffloadingManager:
|
|||||||
)
|
)
|
||||||
|
|
||||||
self._start_request(ctx)
|
self._start_request(ctx)
|
||||||
self.manager.prepare_store(blocks, ctx)
|
self.manager.prepare_store(chunks, ctx)
|
||||||
self.manager.on_request_finished(ctx)
|
self.manager.on_request_finished(ctx)
|
||||||
|
|
||||||
assert calls == [("primary_finish", ctx.req_id)]
|
assert calls == [("primary_finish", ctx.req_id)]
|
||||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||||
|
|
||||||
self.manager.complete_store(blocks, ctx, success=True)
|
self.manager.complete_store(chunks, ctx, success=True)
|
||||||
|
|
||||||
assert calls == [
|
assert calls == [
|
||||||
("primary_finish", ctx.req_id),
|
("primary_finish", ctx.req_id),
|
||||||
@@ -930,7 +931,7 @@ class TestTieringOffloadingManager:
|
|||||||
|
|
||||||
def test_failed_store_finalizes_finished_request(self, manager_setup):
|
def test_failed_store_finalizes_finished_request(self, manager_setup):
|
||||||
"""Failed primary stores still unblock secondary finalization."""
|
"""Failed primary stores still unblock secondary finalization."""
|
||||||
blocks = to_keys(range(2))
|
chunks = to_keys(range(2))
|
||||||
ctx = ReqContext(req_id="req_failed_store_finalize")
|
ctx = ReqContext(req_id="req_failed_store_finalize")
|
||||||
|
|
||||||
self.secondary_tier1.submit_store = MagicMock(
|
self.secondary_tier1.submit_store = MagicMock(
|
||||||
@@ -947,13 +948,13 @@ class TestTieringOffloadingManager:
|
|||||||
)
|
)
|
||||||
|
|
||||||
self._start_request(ctx)
|
self._start_request(ctx)
|
||||||
self.manager.prepare_store(blocks, ctx)
|
self.manager.prepare_store(chunks, ctx)
|
||||||
self.manager.on_request_finished(ctx)
|
self.manager.on_request_finished(ctx)
|
||||||
|
|
||||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||||
|
|
||||||
self.manager.complete_store(blocks, ctx, success=False)
|
self.manager.complete_store(chunks, ctx, success=False)
|
||||||
|
|
||||||
self.secondary_tier1.submit_store.assert_not_called()
|
self.secondary_tier1.submit_store.assert_not_called()
|
||||||
self.secondary_tier2.submit_store.assert_not_called()
|
self.secondary_tier2.submit_store.assert_not_called()
|
||||||
@@ -981,7 +982,7 @@ class TestTieringOffloadingManager:
|
|||||||
|
|
||||||
def test_reset_cache_finalizes_delayed_secondary_request(self, manager_setup):
|
def test_reset_cache_finalizes_delayed_secondary_request(self, manager_setup):
|
||||||
"""reset_cache abandons pending primary stores and finalizes secondaries."""
|
"""reset_cache abandons pending primary stores and finalizes secondaries."""
|
||||||
blocks = to_keys(range(2))
|
chunks = to_keys(range(2))
|
||||||
ctx = ReqContext(req_id="req_reset_finalize_secondary")
|
ctx = ReqContext(req_id="req_reset_finalize_secondary")
|
||||||
|
|
||||||
self.secondary_tier1.on_request_finished = MagicMock(
|
self.secondary_tier1.on_request_finished = MagicMock(
|
||||||
@@ -992,7 +993,7 @@ class TestTieringOffloadingManager:
|
|||||||
)
|
)
|
||||||
|
|
||||||
self._start_request(ctx)
|
self._start_request(ctx)
|
||||||
self.manager.prepare_store(blocks, ctx)
|
self.manager.prepare_store(chunks, ctx)
|
||||||
self.manager.on_request_finished(ctx)
|
self.manager.on_request_finished(ctx)
|
||||||
|
|
||||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||||
@@ -1008,8 +1009,8 @@ class TestTieringOffloadingManager:
|
|||||||
self, manager_setup
|
self, manager_setup
|
||||||
):
|
):
|
||||||
"""reset_cache drops active pending stores so resumed requests finalize."""
|
"""reset_cache drops active pending stores so resumed requests finalize."""
|
||||||
initial_blocks = to_keys(range(2))
|
initial_chunks = to_keys(range(2))
|
||||||
resumed_blocks = to_keys(range(2, 4))
|
resumed_chunks = to_keys(range(2, 4))
|
||||||
ctx = ReqContext(req_id="req_reset_resume")
|
ctx = ReqContext(req_id="req_reset_resume")
|
||||||
|
|
||||||
self.secondary_tier1.on_request_finished = MagicMock(
|
self.secondary_tier1.on_request_finished = MagicMock(
|
||||||
@@ -1020,7 +1021,7 @@ class TestTieringOffloadingManager:
|
|||||||
)
|
)
|
||||||
|
|
||||||
self._start_request(ctx)
|
self._start_request(ctx)
|
||||||
self.manager.prepare_store(initial_blocks, ctx)
|
self.manager.prepare_store(initial_chunks, ctx)
|
||||||
assert self.manager._req_state[ctx.req_id].pending_primary_stores == 1
|
assert self.manager._req_state[ctx.req_id].pending_primary_stores == 1
|
||||||
|
|
||||||
self.manager.reset_cache()
|
self.manager.reset_cache()
|
||||||
@@ -1030,8 +1031,8 @@ class TestTieringOffloadingManager:
|
|||||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||||
|
|
||||||
self.manager.prepare_store(resumed_blocks, ctx)
|
self.manager.prepare_store(resumed_chunks, ctx)
|
||||||
self.manager.complete_store(resumed_blocks, ctx, success=True)
|
self.manager.complete_store(resumed_chunks, ctx, success=True)
|
||||||
self.manager.on_request_finished(ctx)
|
self.manager.on_request_finished(ctx)
|
||||||
|
|
||||||
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
|
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
|
||||||
@@ -1039,12 +1040,12 @@ class TestTieringOffloadingManager:
|
|||||||
assert ctx.req_id not in self.manager._req_state
|
assert ctx.req_id not in self.manager._req_state
|
||||||
|
|
||||||
def test_on_new_request_lifecycle(self, manager_setup):
|
def test_on_new_request_lifecycle(self, manager_setup):
|
||||||
"""Policy defaults to BLOCK_LEVEL, escalates when a tier requests it,
|
"""Policy defaults to CHUNK_LEVEL, escalates when a tier requests it,
|
||||||
and is cleaned up on on_request_finished."""
|
and is cleaned up on on_request_finished."""
|
||||||
# Default: all tiers return BLOCK_LEVEL
|
# Default: all tiers return CHUNK_LEVEL
|
||||||
ctx = ReqContext(req_id="req_policy_lifecycle")
|
ctx = ReqContext(req_id="req_policy_lifecycle")
|
||||||
result = self.manager.on_new_request(ctx)
|
result = self.manager.on_new_request(ctx)
|
||||||
assert result.policy == OffloadPolicy.BLOCK_LEVEL
|
assert result.policy == OffloadPolicy.CHUNK_LEVEL
|
||||||
assert self.manager._req_state[ctx.req_id].request_level_tiers is None
|
assert self.manager._req_state[ctx.req_id].request_level_tiers is None
|
||||||
self.manager.on_request_finished(ctx)
|
self.manager.on_request_finished(ctx)
|
||||||
assert ctx.req_id not in self.manager._req_state
|
assert ctx.req_id not in self.manager._req_state
|
||||||
@@ -1064,26 +1065,26 @@ class TestTieringOffloadingManager:
|
|||||||
assert ctx.req_id not in self.manager._req_state
|
assert ctx.req_id not in self.manager._req_state
|
||||||
|
|
||||||
@pytest.mark.parametrize("new_ids", [(), (3, 4)], ids=["fully_warm", "mixed"])
|
@pytest.mark.parametrize("new_ids", [(), (3, 4)], ids=["fully_warm", "mixed"])
|
||||||
def test_prepare_store_cascades_existing_blocks_to_request_level_tiers(
|
def test_prepare_store_cascades_existing_chunks_to_request_level_tiers(
|
||||||
self, manager_setup, new_ids
|
self, manager_setup, new_ids
|
||||||
):
|
):
|
||||||
"""prepare_store cascades hit blocks to request-level tiers only.
|
"""prepare_store cascades hit chunks to request-level tiers only.
|
||||||
|
|
||||||
The fully-warm case is the producer shape from issue #52808: nothing is
|
The fully-warm case is the producer shape from issue #52808: nothing is
|
||||||
new, so primary returns an empty keys_to_store and no GPU->primary
|
new, so primary returns an empty keys_to_store and no GPU->primary
|
||||||
transfer runs. A request-level tier must still be handed the blocks,
|
transfer runs. A request-level tier must still be handed the chunks,
|
||||||
otherwise a peer waiting on them has nothing to fetch.
|
otherwise a peer waiting on them has nothing to fetch.
|
||||||
"""
|
"""
|
||||||
# Store some blocks to primary first
|
# Store some chunks to primary first
|
||||||
existing_blocks = to_keys(range(3))
|
existing_chunks = to_keys(range(3))
|
||||||
self._start_request()
|
self._start_request()
|
||||||
result = self.manager.prepare_store(existing_blocks, _CTX)
|
result = self.manager.prepare_store(existing_chunks, _CTX)
|
||||||
assert result is not None
|
assert result is not None
|
||||||
self.manager.complete_store(existing_blocks, _CTX, success=True)
|
self.manager.complete_store(existing_chunks, _CTX, success=True)
|
||||||
# Drain cascade completions
|
# Drain cascade completions
|
||||||
self._simulate_on_schedule_end()
|
self._simulate_on_schedule_end()
|
||||||
|
|
||||||
# Make tier1 request-level, tier2 stays block-level
|
# Make tier1 request-level, tier2 stays chunk-level
|
||||||
self.secondary_tier1.on_new_request = lambda req_context: (
|
self.secondary_tier1.on_new_request = lambda req_context: (
|
||||||
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
|
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
|
||||||
)
|
)
|
||||||
@@ -1099,20 +1100,20 @@ class TestTieringOffloadingManager:
|
|||||||
wraps=self.secondary_tier2.submit_store
|
wraps=self.secondary_tier2.submit_store
|
||||||
)
|
)
|
||||||
|
|
||||||
# Call prepare_store with existing + new blocks
|
# Call prepare_store with existing + new chunks
|
||||||
new_blocks = to_keys(new_ids)
|
new_chunks = to_keys(new_ids)
|
||||||
all_blocks = existing_blocks + new_blocks
|
all_chunks = existing_chunks + new_chunks
|
||||||
result = self.manager.prepare_store(all_blocks, ctx)
|
result = self.manager.prepare_store(all_chunks, ctx)
|
||||||
assert result is not None
|
assert result is not None
|
||||||
assert set(result.keys_to_store) == set(new_blocks)
|
assert set(result.keys_to_store) == set(new_chunks)
|
||||||
|
|
||||||
# Only tier1 (request-level) should get existing blocks cascaded now.
|
# Only tier1 (request-level) should get existing chunks cascaded now.
|
||||||
# New blocks are cascaded to ALL tiers later via complete_store().
|
# New chunks are cascaded to ALL tiers later via complete_store().
|
||||||
self.secondary_tier1.submit_store.assert_called_once()
|
self.secondary_tier1.submit_store.assert_called_once()
|
||||||
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
||||||
assert set(job_metadata.keys) == set(existing_blocks)
|
assert set(job_metadata.keys) == set(existing_chunks)
|
||||||
|
|
||||||
# tier2 (block-level) does not get existing blocks here.
|
# tier2 (chunk-level) does not get existing chunks here.
|
||||||
self.secondary_tier2.submit_store.assert_not_called()
|
self.secondary_tier2.submit_store.assert_not_called()
|
||||||
|
|
||||||
def _make_request_level_request(self, req_id: str) -> ReqContext:
|
def _make_request_level_request(self, req_id: str) -> ReqContext:
|
||||||
@@ -1150,7 +1151,7 @@ class TestTieringOffloadingManager:
|
|||||||
self.manager.primary_tier.lookup = lambda key, req_context: (LookupResult.RETRY)
|
self.manager.primary_tier.lookup = lambda key, req_context: (LookupResult.RETRY)
|
||||||
|
|
||||||
with pytest.raises(AssertionError):
|
with pytest.raises(AssertionError):
|
||||||
self.manager._cascade_existing_blocks_to_request_level_tiers(keys, ctx, {0})
|
self.manager._cascade_existing_chunks_to_request_level_tiers(keys, ctx, {0})
|
||||||
|
|
||||||
def _start_in_flight_primary_write(self, keys: list[OffloadKey]) -> ReqContext:
|
def _start_in_flight_primary_write(self, keys: list[OffloadKey]) -> ReqContext:
|
||||||
"""Leave a GPU->primary write for `keys` open, so they look present to
|
"""Leave a GPU->primary write for `keys` open, so they look present to
|
||||||
@@ -1202,7 +1203,7 @@ class TestTieringOffloadingManager:
|
|||||||
def test_cascade_drops_deferred_keys_whose_primary_write_failed(
|
def test_cascade_drops_deferred_keys_whose_primary_write_failed(
|
||||||
self, manager_setup
|
self, manager_setup
|
||||||
):
|
):
|
||||||
"""A failed write frees the block, so the deferred key resolves to MISS
|
"""A failed write frees the chunk, so the deferred key resolves to MISS
|
||||||
and the request finalizes instead of parking forever."""
|
and the request finalizes instead of parking forever."""
|
||||||
keys = to_keys(range(3))
|
keys = to_keys(range(3))
|
||||||
writer_ctx = self._start_in_flight_primary_write(keys)
|
writer_ctx = self._start_in_flight_primary_write(keys)
|
||||||
@@ -1224,21 +1225,21 @@ class TestTieringOffloadingManager:
|
|||||||
"""reset_cache wipes every kind of orchestrator state and resets
|
"""reset_cache wipes every kind of orchestrator state and resets
|
||||||
primary tier; pending submissions are dropped without being sent
|
primary tier; pending submissions are dropped without being sent
|
||||||
to the secondary tier. Active request state is retained."""
|
to the secondary tier. Active request state is retained."""
|
||||||
# Cascade — populates primary blocks and leaves cascade jobs
|
# Cascade — populates primary chunks and leaves cascade jobs
|
||||||
# in _jobs (the synchronous example tier has already
|
# in _jobs (the synchronous example tier has already
|
||||||
# queued completions); reset_cache's drain loop will pick them up.
|
# queued completions); reset_cache's drain loop will pick them up.
|
||||||
blocks = to_keys(range(3))
|
chunks = to_keys(range(3))
|
||||||
self._start_request()
|
self._start_request()
|
||||||
self.manager.prepare_store(blocks, _CTX)
|
self.manager.prepare_store(chunks, _CTX)
|
||||||
self.manager.complete_store(blocks, _CTX, success=True)
|
self.manager.complete_store(chunks, _CTX, success=True)
|
||||||
assert self.manager._jobs
|
assert self.manager._jobs
|
||||||
|
|
||||||
# Pending promotion submission (deferred — no on_schedule_end after
|
# Pending promotion submission (deferred — no on_schedule_end after
|
||||||
# the lookup that staged it).
|
# the lookup that staged it).
|
||||||
promo_block = to_keys([99])[0]
|
promo_chunk = to_keys([99])[0]
|
||||||
self.secondary_tier1.blocks[promo_block] = True
|
self.secondary_tier1.chunks[promo_chunk] = True
|
||||||
assert (
|
assert (
|
||||||
self.manager.lookup(promo_block, ReqContext(req_id="pending"))
|
self.manager.lookup(promo_chunk, ReqContext(req_id="pending"))
|
||||||
is LookupResult.HIT_PENDING
|
is LookupResult.HIT_PENDING
|
||||||
)
|
)
|
||||||
assert self.manager._pending_load_submissions
|
assert self.manager._pending_load_submissions
|
||||||
@@ -1268,10 +1269,10 @@ class TestTieringOffloadingManager:
|
|||||||
assert self.manager._processed_jobs_this_step is False
|
assert self.manager._processed_jobs_this_step is False
|
||||||
|
|
||||||
# Primary tier reset to a fresh state.
|
# Primary tier reset to a fresh state.
|
||||||
assert self.primary_tier._num_allocated_blocks == 0
|
assert self.primary_tier._num_allocated_chunks == 0
|
||||||
assert self.primary_tier._free_list == []
|
assert self.primary_tier._free_list == []
|
||||||
for block in blocks:
|
for chunk in chunks:
|
||||||
assert self.primary_tier.lookup(block, _CTX) is LookupResult.MISS
|
assert self.primary_tier.lookup(chunk, _CTX) is LookupResult.MISS
|
||||||
|
|
||||||
# Pending submission was dropped, not submitted.
|
# Pending submission was dropped, not submitted.
|
||||||
self.secondary_tier1.submit_load.assert_not_called()
|
self.secondary_tier1.submit_load.assert_not_called()
|
||||||
@@ -1291,10 +1292,10 @@ class TestTieringOffloadingManager:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Drive a cascade so a job lands in _jobs.
|
# Drive a cascade so a job lands in _jobs.
|
||||||
blocks = to_keys(range(3))
|
chunks = to_keys(range(3))
|
||||||
self._start_request()
|
self._start_request()
|
||||||
self.manager.prepare_store(blocks, _CTX)
|
self.manager.prepare_store(chunks, _CTX)
|
||||||
self.manager.complete_store(blocks, _CTX, success=True)
|
self.manager.complete_store(chunks, _CTX, success=True)
|
||||||
assert self.manager._jobs
|
assert self.manager._jobs
|
||||||
|
|
||||||
self.manager.reset_cache()
|
self.manager.reset_cache()
|
||||||
@@ -1315,20 +1316,20 @@ class TestTieringOffloadingManager:
|
|||||||
self, manager_setup, load_tier_filter
|
self, manager_setup, load_tier_filter
|
||||||
):
|
):
|
||||||
"""Filter excluding secondary medium returns MISS from secondaries
|
"""Filter excluding secondary medium returns MISS from secondaries
|
||||||
even when they hold the block; primary is unaffected."""
|
even when they hold the chunk; primary is unaffected."""
|
||||||
blocks = to_keys(range(2))
|
chunks = to_keys(range(2))
|
||||||
# Put one block in primary, one only in secondary
|
# Put one chunk in primary, one only in secondary
|
||||||
self._start_request()
|
self._start_request()
|
||||||
self.manager.prepare_store(blocks[:1], _CTX)
|
self.manager.prepare_store(chunks[:1], _CTX)
|
||||||
self.manager.complete_store(blocks[:1], _CTX, success=True)
|
self.manager.complete_store(chunks[:1], _CTX, success=True)
|
||||||
self.secondary_tier1.blocks[blocks[1]] = True
|
self.secondary_tier1.chunks[chunks[1]] = True
|
||||||
|
|
||||||
# Secondaries have medium=CPU, so load_tier_filter skips them.
|
# Secondaries have medium=CPU, so load_tier_filter skips them.
|
||||||
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
|
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
|
||||||
|
|
||||||
ctx = ReqContext(req_id="r1", load_tier_filter=load_tier_filter)
|
ctx = ReqContext(req_id="r1", load_tier_filter=load_tier_filter)
|
||||||
assert self.manager.lookup(blocks[0], ctx) is LookupResult.HIT
|
assert self.manager.lookup(chunks[0], ctx) is LookupResult.HIT
|
||||||
assert self.manager.lookup(blocks[1], ctx) is LookupResult.MISS
|
assert self.manager.lookup(chunks[1], ctx) is LookupResult.MISS
|
||||||
self.secondary_tier1.lookup.assert_not_called()
|
self.secondary_tier1.lookup.assert_not_called()
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
@@ -1344,13 +1345,13 @@ class TestTieringOffloadingManager:
|
|||||||
self, manager_setup, load_tier_filter
|
self, manager_setup, load_tier_filter
|
||||||
):
|
):
|
||||||
"""Filter that matches the secondary's medium allows lookup."""
|
"""Filter that matches the secondary's medium allows lookup."""
|
||||||
blocks = to_keys(range(1))
|
chunks = to_keys(range(1))
|
||||||
self.secondary_tier1.blocks[blocks[0]] = True
|
self.secondary_tier1.chunks[chunks[0]] = True
|
||||||
|
|
||||||
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
|
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
|
||||||
|
|
||||||
ctx = ReqContext(req_id="r2", load_tier_filter=load_tier_filter)
|
ctx = ReqContext(req_id="r2", load_tier_filter=load_tier_filter)
|
||||||
assert self.manager.lookup(blocks[0], ctx) is LookupResult.HIT_PENDING
|
assert self.manager.lookup(chunks[0], ctx) is LookupResult.HIT_PENDING
|
||||||
self.secondary_tier1.lookup.assert_called()
|
self.secondary_tier1.lookup.assert_called()
|
||||||
|
|
||||||
|
|
||||||
@@ -1360,7 +1361,7 @@ class TestTieringOffloadingWithoutSecondaryTiers:
|
|||||||
def test_works_without_secondary_tiers(self):
|
def test_works_without_secondary_tiers(self):
|
||||||
"""Test that manager works with empty secondary_tiers list."""
|
"""Test that manager works with empty secondary_tiers list."""
|
||||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||||
num_blocks=5, mmap_region=_mock_mmap_region(5)
|
num_chunks=5, mmap_region=_mock_mmap_region(5)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Create manager with no secondary tiers
|
# Create manager with no secondary tiers
|
||||||
@@ -1368,15 +1369,15 @@ class TestTieringOffloadingWithoutSecondaryTiers:
|
|||||||
primary_tier=primary_tier, secondary_tiers=[]
|
primary_tier=primary_tier, secondary_tiers=[]
|
||||||
)
|
)
|
||||||
|
|
||||||
blocks = to_keys(range(3))
|
chunks = to_keys(range(3))
|
||||||
|
|
||||||
# Should work like a regular OffloadingManager
|
# Should work like a regular OffloadingManager
|
||||||
manager.on_new_request(_CTX)
|
manager.on_new_request(_CTX)
|
||||||
result = manager.prepare_store(blocks, _CTX)
|
result = manager.prepare_store(chunks, _CTX)
|
||||||
assert result is not None
|
assert result is not None
|
||||||
manager.complete_store(blocks, _CTX, success=True)
|
manager.complete_store(chunks, _CTX, success=True)
|
||||||
|
|
||||||
assert count_hits(manager, blocks) == 3
|
assert count_hits(manager, chunks) == 3
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
|
|||||||
@@ -1131,7 +1131,7 @@ class OffloadingConnectorScheduler:
|
|||||||
# Skip prefix-hit chunks for block-level policy; for
|
# Skip prefix-hit chunks for block-level policy; for
|
||||||
# request-level, next_stored_chunk_idx stays at 0 so all
|
# request-level, next_stored_chunk_idx stays at 0 so all
|
||||||
# chunks (including hits) are offloaded.
|
# chunks (including hits) are offloaded.
|
||||||
if req_status.offloading_context.policy == OffloadPolicy.BLOCK_LEVEL:
|
if req_status.offloading_context.policy == OffloadPolicy.CHUNK_LEVEL:
|
||||||
group_state.next_stored_chunk_idx = num_chunks
|
group_state.next_stored_chunk_idx = num_chunks
|
||||||
|
|
||||||
src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context)
|
src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context)
|
||||||
|
|||||||
@@ -114,17 +114,17 @@ class LookupResult(Enum):
|
|||||||
|
|
||||||
|
|
||||||
class OffloadPolicy(Enum):
|
class OffloadPolicy(Enum):
|
||||||
# Offload only newly-computed blocks as they arrive; prefix-hit
|
# Offload only newly-computed chunks as they arrive; prefix-hit
|
||||||
# blocks (already offloaded by a prior request) are skipped.
|
# chunks (already offloaded by a prior request) are skipped.
|
||||||
BLOCK_LEVEL = "block_level"
|
CHUNK_LEVEL = "chunk_level"
|
||||||
# Offload all blocks for the request, including prefix hits.
|
# Offload all chunks for the request, including prefix hits.
|
||||||
# Used by tiers that need the complete KV context for a request.
|
# Used by tiers that need the complete KV context for a request.
|
||||||
REQUEST_LEVEL = "request_level"
|
REQUEST_LEVEL = "request_level"
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class RequestOffloadingContext:
|
class RequestOffloadingContext:
|
||||||
policy: OffloadPolicy = OffloadPolicy.BLOCK_LEVEL
|
policy: OffloadPolicy = OffloadPolicy.CHUNK_LEVEL
|
||||||
|
|
||||||
|
|
||||||
class ScheduleEndContext(NamedTuple):
|
class ScheduleEndContext(NamedTuple):
|
||||||
@@ -400,6 +400,9 @@ class OffloadingManager(ABC):
|
|||||||
class BlockIDsLoadStoreSpec(LoadStoreSpec, ABC):
|
class BlockIDsLoadStoreSpec(LoadStoreSpec, ABC):
|
||||||
"""
|
"""
|
||||||
Spec for loading/storing KV blocks from given block numbers.
|
Spec for loading/storing KV blocks from given block numbers.
|
||||||
|
|
||||||
|
Subclass semantics differ: GPULoadStoreSpec.block_ids are GPU block
|
||||||
|
indices; CPULoadStoreSpec.block_ids are CPU cache chunk indices.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, block_ids: list[int]):
|
def __init__(self, block_ids: list[int]):
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
|
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
|
||||||
|
|
||||||
|
|
||||||
@@ -12,6 +14,13 @@ class CPUOffloadingMetrics:
|
|||||||
|
|
||||||
|
|
||||||
class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
|
class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
|
||||||
|
"""Spec for loading/storing KV chunks to/from CPU memory.
|
||||||
|
|
||||||
|
The inherited block_ids field holds chunk indices into the
|
||||||
|
CPU cache (not GPU block IDs). The chunk_ids alias exposes
|
||||||
|
the same array under the name used by the tiering layer.
|
||||||
"""
|
"""
|
||||||
Spec for loading/storing a KV block to CPU memory.
|
|
||||||
"""
|
@property
|
||||||
|
def chunk_ids(self) -> np.ndarray:
|
||||||
|
return self.block_ids
|
||||||
|
|||||||
@@ -257,7 +257,7 @@ class SingleDirectionOffloadingHandler:
|
|||||||
gpu_tensors: list of GPU KV cache tensors.
|
gpu_tensors: list of GPU KV cache tensors.
|
||||||
Each of shape (num_gpu_blocks, gpu_page_size_bytes) with dtype int8.
|
Each of shape (num_gpu_blocks, gpu_page_size_bytes) with dtype int8.
|
||||||
cpu_tensors: list of CPU KV cache tensors.
|
cpu_tensors: list of CPU KV cache tensors.
|
||||||
Each of shape (num_cpu_blocks, cpu_page_size_bytes) with dtype int8.
|
Each of shape (num_cpu_chunks, cpu_page_size_bytes) with dtype int8.
|
||||||
Order should match gpu_tensors.
|
Order should match gpu_tensors.
|
||||||
layer_refs_per_group: list of CanonicalKVCacheRef per group.
|
layer_refs_per_group: list of CanonicalKVCacheRef per group.
|
||||||
gpu_to_cpu: if True, transfer from GPU to CPU; otherwise CPU to GPU.
|
gpu_to_cpu: if True, transfer from GPU to CPU; otherwise CPU to GPU.
|
||||||
@@ -527,20 +527,20 @@ class SingleDirectionOffloadingHandler:
|
|||||||
# 1. GPU -> CPU
|
# 1. GPU -> CPU
|
||||||
# 2. CPU -> GPU
|
# 2. CPU -> GPU
|
||||||
#
|
#
|
||||||
# transfers are also to CPU blocks, EXCEPT MAYBE for the first and last block.
|
# transfers are also to CPU chunks, EXCEPT MAYBE for the first and last chunk.
|
||||||
# i.e. the first and last CPU blocks in src_blocks can match against
|
# i.e. the first and last CPU chunks in src_blocks can match against
|
||||||
# a smaller (byte-wise) set of GPU blocks in dst_blocks.
|
# a smaller (byte-wise) set of GPU blocks in dst_blocks.
|
||||||
# In such cases, we may need to skip some gpu-sized sub-blocks,
|
# In such cases, we may need to skip some gpu-sized sub-blocks,
|
||||||
# and start reading/writing from the middle of the first CPU block.
|
# and start reading/writing from the middle of the first CPU chunk.
|
||||||
# If we have multiple KV cache groups (when using HMA with hybrid models),
|
# If we have multiple KV cache groups (when using HMA with hybrid models),
|
||||||
# we may have a partial first/last CPU block per each group.
|
# we may have a partial first/last CPU chunk per each group.
|
||||||
# The group_sizes parameter encodes the size of each group of blocks
|
# The group_sizes parameter encodes the size of each group of blocks
|
||||||
# in the GPU dst_blocks.
|
# in the GPU dst_blocks.
|
||||||
# If group_sizes is None, we assume all blocks belong to a single group.
|
# If group_sizes is None, we assume all blocks belong to a single group.
|
||||||
# The logical_offset parameter maps each group of blocks to its logical
|
# The logical_offset parameter maps each group of blocks to its logical
|
||||||
# offset inside the request, counting in GPU blocks.
|
# offset inside the request, counting in GPU blocks.
|
||||||
# This allows us to find the correct starting position
|
# This allows us to find the correct starting position
|
||||||
# in the matching first CPU block.
|
# in the matching first CPU chunk.
|
||||||
|
|
||||||
# extract group_sizes from the GPU spec
|
# extract group_sizes from the GPU spec
|
||||||
gpu_spec = src_spec if self.gpu_to_cpu else dst_spec
|
gpu_spec = src_spec if self.gpu_to_cpu else dst_spec
|
||||||
@@ -751,7 +751,7 @@ class CPUOffloadingWorker(OffloadingWorker):
|
|||||||
self,
|
self,
|
||||||
kv_caches: CanonicalKVCaches,
|
kv_caches: CanonicalKVCaches,
|
||||||
blocks_per_chunk: int,
|
blocks_per_chunk: int,
|
||||||
num_cpu_blocks: int,
|
num_cpu_chunks: int,
|
||||||
mmap_region: SharedOffloadRegion | None = None,
|
mmap_region: SharedOffloadRegion | None = None,
|
||||||
canonical_layout: bool = False,
|
canonical_layout: bool = False,
|
||||||
):
|
):
|
||||||
@@ -790,16 +790,16 @@ class CPUOffloadingWorker(OffloadingWorker):
|
|||||||
else:
|
else:
|
||||||
t0 = time.monotonic()
|
t0 = time.monotonic()
|
||||||
cpu_tensor = torch.zeros(
|
cpu_tensor = torch.zeros(
|
||||||
(num_cpu_blocks, cpu_page_size_bytes),
|
(num_cpu_chunks, cpu_page_size_bytes),
|
||||||
dtype=torch.int8,
|
dtype=torch.int8,
|
||||||
device="cpu",
|
device="cpu",
|
||||||
pin_memory=pin_memory,
|
pin_memory=pin_memory,
|
||||||
)
|
)
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"torch.zeros pinned tensor %d×%d (%.2f GB): %.3f s",
|
"torch.zeros pinned tensor %d×%d (%.2f GB): %.3f s",
|
||||||
num_cpu_blocks,
|
num_cpu_chunks,
|
||||||
cpu_page_size_bytes,
|
cpu_page_size_bytes,
|
||||||
num_cpu_blocks * cpu_page_size_bytes / 1e9,
|
num_cpu_chunks * cpu_page_size_bytes / 1e9,
|
||||||
time.monotonic() - t0,
|
time.monotonic() - t0,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ from vllm.v1.kv_offload.cpu.common import (
|
|||||||
CPULoadStoreSpec,
|
CPULoadStoreSpec,
|
||||||
CPUOffloadingMetrics,
|
CPUOffloadingMetrics,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
|
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
|
||||||
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
|
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
|
||||||
|
|
||||||
|
|
||||||
@@ -34,14 +34,14 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
register their own or be loaded out-of-tree via cache_policy_module_path).
|
register their own or be loaded out-of-tree via cache_policy_module_path).
|
||||||
|
|
||||||
The manager owns all shared logic: ref-counting, event emission,
|
The manager owns all shared logic: ref-counting, event emission,
|
||||||
block pool management, and the prepare_store/complete_store skeletons.
|
chunk pool management, and the prepare_store/complete_store skeletons.
|
||||||
Policy-specific block organization and eviction decisions are delegated
|
Policy-specific chunk organization and eviction decisions are delegated
|
||||||
to the CachePolicy implementation.
|
to the CachePolicy implementation.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
num_blocks: int,
|
num_chunks: int,
|
||||||
cache_policy: str = "lru",
|
cache_policy: str = "lru",
|
||||||
cache_policy_module_path: str | None = None,
|
cache_policy_module_path: str | None = None,
|
||||||
enable_events: bool = False,
|
enable_events: bool = False,
|
||||||
@@ -49,59 +49,59 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
max_tracker_size: int = 64_000,
|
max_tracker_size: int = 64_000,
|
||||||
):
|
):
|
||||||
self.medium: Medium = Medium.CPU
|
self.medium: Medium = Medium.CPU
|
||||||
self._num_blocks: int = num_blocks
|
self._num_chunks: int = num_chunks
|
||||||
self._num_allocated_blocks: int = 0
|
self._num_allocated_chunks: int = 0
|
||||||
self._free_list: list[int] = []
|
self._free_list: list[int] = []
|
||||||
self.events: list[OffloadingEvent] | None = [] if enable_events else None
|
self.events: list[OffloadingEvent] | None = [] if enable_events else None
|
||||||
policy_cls = CachePolicyFactory.get_cache_policy_cls(
|
policy_cls = CachePolicyFactory.get_cache_policy_cls(
|
||||||
cache_policy, cache_policy_module_path
|
cache_policy, cache_policy_module_path
|
||||||
)
|
)
|
||||||
self._policy: CachePolicy = policy_cls(cache_capacity=num_blocks)
|
self._policy: CachePolicy = policy_cls(cache_capacity=num_chunks)
|
||||||
# Track the number of blocks in the cache that are evictable. i.e. ref_cnt 0.
|
# Track the number of chunks in the cache that are evictable. i.e. ref_cnt 0.
|
||||||
self._num_evictable_cache_blocks: int = 0
|
self._num_evictable_cache_chunks: int = 0
|
||||||
# Track blocks with an in-flight store (ref_cnt -1, not yet completed).
|
# Track chunks with an in-flight store (ref_cnt -1, not yet completed).
|
||||||
self._num_write_pending_blocks: int = 0
|
self._num_write_pending_chunks: int = 0
|
||||||
|
|
||||||
self.store_threshold: int = store_threshold
|
self.store_threshold: int = store_threshold
|
||||||
self.max_tracker_size: int = max_tracker_size
|
self.max_tracker_size: int = max_tracker_size
|
||||||
self.stores_skipped_in_current_batch: int = 0
|
self.stores_skipped_in_current_batch: int = 0
|
||||||
self.allocation_sizes_in_current_batch: list[int] = []
|
self.allocation_sizes_in_current_batch: list[int] = []
|
||||||
|
|
||||||
# Number of block references. It is ordered so can evict the LRU entry in O(1).
|
# Number of chunk references. It is ordered so can evict the LRU entry in O(1).
|
||||||
self.counts: OrderedDict[OffloadKey, int] | None = (
|
self.counts: OrderedDict[OffloadKey, int] | None = (
|
||||||
OrderedDict() if store_threshold >= 2 else None
|
OrderedDict() if store_threshold >= 2 else None
|
||||||
)
|
)
|
||||||
|
|
||||||
# --- block pool ---
|
# --- chunk pool ---
|
||||||
|
|
||||||
def _get_num_free_blocks(self) -> int:
|
def _get_num_free_chunks(self) -> int:
|
||||||
return len(self._free_list) + self._num_blocks - self._num_allocated_blocks
|
return len(self._free_list) + self._num_chunks - self._num_allocated_chunks
|
||||||
|
|
||||||
def _allocate_blocks(self, keys: list[OffloadKey]) -> list[BlockStatus]:
|
def _allocate_chunks(self, keys: list[OffloadKey]) -> list[ChunkStatus]:
|
||||||
num_fresh = min(len(keys), self._num_blocks - self._num_allocated_blocks)
|
num_fresh = min(len(keys), self._num_chunks - self._num_allocated_chunks)
|
||||||
num_reused = len(keys) - num_fresh
|
num_reused = len(keys) - num_fresh
|
||||||
assert len(self._free_list) >= num_reused
|
assert len(self._free_list) >= num_reused
|
||||||
|
|
||||||
# allocate fresh blocks
|
# allocate fresh chunks
|
||||||
blocks: list[BlockStatus] = []
|
chunks: list[ChunkStatus] = []
|
||||||
for _ in range(num_fresh):
|
for _ in range(num_fresh):
|
||||||
blocks.append(BlockStatus(self._num_allocated_blocks))
|
chunks.append(ChunkStatus(self._num_allocated_chunks))
|
||||||
self._num_allocated_blocks += 1
|
self._num_allocated_chunks += 1
|
||||||
|
|
||||||
# allocate reused blocks
|
# allocate reused chunks
|
||||||
for _ in range(num_reused):
|
for _ in range(num_reused):
|
||||||
blocks.append(BlockStatus(self._free_list.pop()))
|
chunks.append(ChunkStatus(self._free_list.pop()))
|
||||||
return blocks
|
return chunks
|
||||||
|
|
||||||
def _free_block(self, block: BlockStatus) -> None:
|
def _free_chunk(self, chunk: ChunkStatus) -> None:
|
||||||
self._free_list.append(block.block_id)
|
self._free_list.append(chunk.chunk_id)
|
||||||
|
|
||||||
def _get_load_store_spec(
|
def _get_load_store_spec(
|
||||||
self,
|
self,
|
||||||
keys: Iterable[OffloadKey],
|
keys: Iterable[OffloadKey],
|
||||||
blocks: Iterable[BlockStatus],
|
chunks: Iterable[ChunkStatus],
|
||||||
) -> CPULoadStoreSpec:
|
) -> CPULoadStoreSpec:
|
||||||
return CPULoadStoreSpec([block.block_id for block in blocks])
|
return CPULoadStoreSpec([chunk.chunk_id for chunk in chunks])
|
||||||
|
|
||||||
def _record_accesses(self, keys: Collection[OffloadKey]) -> None:
|
def _record_accesses(self, keys: Collection[OffloadKey]) -> None:
|
||||||
"""Record an offer without evicting its tracked candidates."""
|
"""Record an offer without evicting its tracked candidates."""
|
||||||
@@ -132,10 +132,10 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
|
|
||||||
@override
|
@override
|
||||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||||
block = self._policy.get(key)
|
chunk = self._policy.get(key)
|
||||||
if block is None:
|
if chunk is None:
|
||||||
return LookupResult.MISS
|
return LookupResult.MISS
|
||||||
if not block.is_ready:
|
if not chunk.is_ready:
|
||||||
return LookupResult.HIT_PENDING
|
return LookupResult.HIT_PENDING
|
||||||
return LookupResult.HIT
|
return LookupResult.HIT
|
||||||
|
|
||||||
@@ -145,18 +145,18 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
keys: Collection[OffloadKey],
|
keys: Collection[OffloadKey],
|
||||||
req_context: ReqContext,
|
req_context: ReqContext,
|
||||||
) -> LoadStoreSpec:
|
) -> LoadStoreSpec:
|
||||||
blocks = []
|
chunks = []
|
||||||
for key in keys:
|
for key in keys:
|
||||||
block = self._policy.get(key)
|
chunk = self._policy.get(key)
|
||||||
assert block is not None, f"Block {key!r} not found in cache"
|
assert chunk is not None, f"Chunk {key!r} not found in cache"
|
||||||
assert block.is_ready, f"Block {key!r} is not ready for reading"
|
assert chunk.is_ready, f"Chunk {key!r} is not ready for reading"
|
||||||
if block.ref_cnt == 0:
|
if chunk.ref_cnt == 0:
|
||||||
self._policy.mark_non_evictable(key)
|
self._policy.mark_non_evictable(key)
|
||||||
self._num_evictable_cache_blocks -= 1 # ref_cnt 0 -> 1
|
self._num_evictable_cache_chunks -= 1 # ref_cnt 0 -> 1
|
||||||
assert self._num_evictable_cache_blocks >= 0
|
assert self._num_evictable_cache_chunks >= 0
|
||||||
block.ref_cnt += 1
|
chunk.ref_cnt += 1
|
||||||
blocks.append(block)
|
chunks.append(chunk)
|
||||||
return self._get_load_store_spec(keys, blocks)
|
return self._get_load_store_spec(keys, chunks)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext) -> None:
|
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext) -> None:
|
||||||
@@ -167,12 +167,12 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||||
) -> None:
|
) -> None:
|
||||||
for key in keys:
|
for key in keys:
|
||||||
block = self._policy.get(key)
|
chunk = self._policy.get(key)
|
||||||
assert block is not None, f"Block {key!r} not found"
|
assert chunk is not None, f"Chunk {key!r} not found"
|
||||||
assert block.ref_cnt > 0, f"Block {key!r} ref_cnt is already 0"
|
assert chunk.ref_cnt > 0, f"Chunk {key!r} ref_cnt is already 0"
|
||||||
block.ref_cnt -= 1
|
chunk.ref_cnt -= 1
|
||||||
if block.ref_cnt == 0:
|
if chunk.ref_cnt == 0:
|
||||||
self._num_evictable_cache_blocks += 1 # ref_cnt 1 -> 0
|
self._num_evictable_cache_chunks += 1 # ref_cnt 1 -> 0
|
||||||
self._policy.mark_evictable(key)
|
self._policy.mark_evictable(key)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
@@ -186,7 +186,7 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
self._record_accesses(keys)
|
self._record_accesses(keys)
|
||||||
keys = [k for k in keys if self.counts.get(k, 0) >= self.store_threshold]
|
keys = [k for k in keys if self.counts.get(k, 0) >= self.store_threshold]
|
||||||
self.stores_skipped_in_current_batch += num_keys - len(keys)
|
self.stores_skipped_in_current_batch += num_keys - len(keys)
|
||||||
# filter out blocks that are already stored
|
# filter out chunks that are already stored
|
||||||
keys_to_store = [k for k in keys if self._policy.get(k) is None]
|
keys_to_store = [k for k in keys if self._policy.get(k) is None]
|
||||||
|
|
||||||
if not keys_to_store:
|
if not keys_to_store:
|
||||||
@@ -197,29 +197,29 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
)
|
)
|
||||||
|
|
||||||
self.allocation_sizes_in_current_batch.append(len(keys_to_store))
|
self.allocation_sizes_in_current_batch.append(len(keys_to_store))
|
||||||
num_blocks_to_evict = len(keys_to_store) - self._get_num_free_blocks()
|
num_chunks_to_evict = len(keys_to_store) - self._get_num_free_chunks()
|
||||||
|
|
||||||
to_evict: list[OffloadKey] = []
|
to_evict: list[OffloadKey] = []
|
||||||
if num_blocks_to_evict > 0:
|
if num_chunks_to_evict > 0:
|
||||||
if num_blocks_to_evict > self._num_evictable_cache_blocks:
|
if num_chunks_to_evict > self._num_evictable_cache_chunks:
|
||||||
# Eviction will fail.
|
# Eviction will fail.
|
||||||
return None
|
return None
|
||||||
# There is a still a chance for eviction failure as some of the
|
# There is a still a chance for eviction failure as some of the
|
||||||
# idle blocks might be in the protected list.
|
# idle chunks might be in the protected list.
|
||||||
|
|
||||||
# Blocks from the original input are excluded from eviction candidates:
|
# Chunks from the original input are excluded from eviction candidates:
|
||||||
# a block that was already stored must remain in the cache after this call.
|
# a chunk that was already stored must remain in the cache after this call.
|
||||||
protected = set(keys)
|
protected = set(keys)
|
||||||
evicted = self._policy.evict(num_blocks_to_evict, protected)
|
evicted = self._policy.evict(num_chunks_to_evict, protected)
|
||||||
if evicted is None:
|
if evicted is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# cache-policy removes only idle blocks.
|
# cache-policy removes only idle chunks.
|
||||||
self._num_evictable_cache_blocks -= len(evicted)
|
self._num_evictable_cache_chunks -= len(evicted)
|
||||||
assert self._num_evictable_cache_blocks >= 0
|
assert self._num_evictable_cache_chunks >= 0
|
||||||
|
|
||||||
for key, block in evicted:
|
for key, chunk in evicted:
|
||||||
self._free_block(block)
|
self._free_chunk(chunk)
|
||||||
to_evict.append(key)
|
to_evict.append(key)
|
||||||
|
|
||||||
if to_evict and self.events is not None:
|
if to_evict and self.events is not None:
|
||||||
@@ -231,17 +231,17 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
blocks = self._allocate_blocks(keys_to_store)
|
chunks = self._allocate_chunks(keys_to_store)
|
||||||
assert len(blocks) == len(keys_to_store), (
|
assert len(chunks) == len(keys_to_store), (
|
||||||
"Block pool did not allocate the expected number of blocks"
|
"Chunk pool did not allocate the expected number of chunks"
|
||||||
)
|
)
|
||||||
|
|
||||||
for key, block in zip(keys_to_store, blocks):
|
for key, chunk in zip(keys_to_store, chunks):
|
||||||
self._policy.insert(key, block)
|
self._policy.insert(key, chunk)
|
||||||
self._num_write_pending_blocks += len(keys_to_store)
|
self._num_write_pending_chunks += len(keys_to_store)
|
||||||
|
|
||||||
# build store specs for allocated blocks
|
# build store specs for allocated chunks
|
||||||
store_spec = self._get_load_store_spec(keys_to_store, blocks)
|
store_spec = self._get_load_store_spec(keys_to_store, chunks)
|
||||||
|
|
||||||
return PrepareStoreOutput(
|
return PrepareStoreOutput(
|
||||||
keys_to_store=keys_to_store,
|
keys_to_store=keys_to_store,
|
||||||
@@ -260,20 +260,20 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
|
|
||||||
if success:
|
if success:
|
||||||
for key in keys:
|
for key in keys:
|
||||||
block = self._policy.get(key)
|
chunk = self._policy.get(key)
|
||||||
if block is not None and not block.is_ready:
|
if chunk is not None and not chunk.is_ready:
|
||||||
block.ref_cnt = 0
|
chunk.ref_cnt = 0
|
||||||
self._num_write_pending_blocks -= 1
|
self._num_write_pending_chunks -= 1
|
||||||
self._num_evictable_cache_blocks += 1
|
self._num_evictable_cache_chunks += 1
|
||||||
self._policy.mark_evictable(key)
|
self._policy.mark_evictable(key)
|
||||||
stored_keys.append(key)
|
stored_keys.append(key)
|
||||||
else:
|
else:
|
||||||
for key in keys:
|
for key in keys:
|
||||||
block = self._policy.get(key)
|
chunk = self._policy.get(key)
|
||||||
if block is not None and not block.is_ready:
|
if chunk is not None and not chunk.is_ready:
|
||||||
self._num_write_pending_blocks -= 1
|
self._num_write_pending_chunks -= 1
|
||||||
self._policy.remove(key)
|
self._policy.remove(key)
|
||||||
self._free_block(block)
|
self._free_chunk(chunk)
|
||||||
|
|
||||||
if stored_keys and self.events is not None:
|
if stored_keys and self.events is not None:
|
||||||
self.events.append(
|
self.events.append(
|
||||||
@@ -286,17 +286,17 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
|
|
||||||
@override
|
@override
|
||||||
def reset_cache(self) -> None:
|
def reset_cache(self) -> None:
|
||||||
# Clear ALL blocks unconditionally. The scheduler's _stale_job_threshold
|
# Clear ALL chunks unconditionally. The scheduler's _stale_job_threshold
|
||||||
# guarantees that complete_load / complete_store are never called for
|
# guarantees that complete_load / complete_store are never called for
|
||||||
# pre-reset jobs, so no lazy cleanup is needed. The scheduler also
|
# pre-reset jobs, so no lazy cleanup is needed. The scheduler also
|
||||||
# flushes in-flight load job IDs to the workers before any new stores
|
# flushes in-flight load job IDs to the workers before any new stores
|
||||||
# can begin, preventing a cross-direction data race on reused offload block IDs.
|
# can begin, preventing a cross-direction data race on reused offload chunk IDs.
|
||||||
self._policy.clear()
|
self._policy.clear()
|
||||||
self._num_evictable_cache_blocks = 0
|
self._num_evictable_cache_chunks = 0
|
||||||
self._num_write_pending_blocks = 0
|
self._num_write_pending_chunks = 0
|
||||||
|
|
||||||
self._free_list.clear()
|
self._free_list.clear()
|
||||||
self._num_allocated_blocks = 0
|
self._num_allocated_chunks = 0
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def take_events(self) -> Iterable[OffloadingEvent]:
|
def take_events(self) -> Iterable[OffloadingEvent]:
|
||||||
@@ -309,11 +309,11 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
|
|
||||||
# Compute cache usage.
|
# Compute cache usage.
|
||||||
num_used = (
|
num_used = (
|
||||||
self._num_allocated_blocks
|
self._num_allocated_chunks
|
||||||
- len(self._free_list)
|
- len(self._free_list)
|
||||||
- self._num_evictable_cache_blocks
|
- self._num_evictable_cache_chunks
|
||||||
)
|
)
|
||||||
usage = num_used / self._num_blocks if self._num_blocks > 0 else 0.0
|
usage = num_used / self._num_chunks if self._num_chunks > 0 else 0.0
|
||||||
stats.set_gauge(CPUOffloadingMetrics.CPU_CACHE_USAGE_PERC, usage)
|
stats.set_gauge(CPUOffloadingMetrics.CPU_CACHE_USAGE_PERC, usage)
|
||||||
|
|
||||||
for allocation_size in self.allocation_sizes_in_current_batch:
|
for allocation_size in self.allocation_sizes_in_current_batch:
|
||||||
@@ -323,8 +323,8 @@ class CPUOffloadingManager(OffloadingManager):
|
|||||||
self.allocation_sizes_in_current_batch.clear()
|
self.allocation_sizes_in_current_batch.clear()
|
||||||
|
|
||||||
write_usage = (
|
write_usage = (
|
||||||
self._num_write_pending_blocks / self._num_blocks
|
self._num_write_pending_chunks / self._num_chunks
|
||||||
if self._num_blocks > 0
|
if self._num_chunks > 0
|
||||||
else 0.0
|
else 0.0
|
||||||
)
|
)
|
||||||
read_usage = max(usage - write_usage, 0.0)
|
read_usage = max(usage - write_usage, 0.0)
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from collections.abc import Iterable, Iterator
|
|||||||
from typing_extensions import override
|
from typing_extensions import override
|
||||||
|
|
||||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||||
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
|
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
|
||||||
|
|
||||||
|
|
||||||
class ARCCachePolicy(CachePolicy):
|
class ARCCachePolicy(CachePolicy):
|
||||||
@@ -14,15 +14,15 @@ class ARCCachePolicy(CachePolicy):
|
|||||||
ARC (Adaptive Replacement Cache) cache policy.
|
ARC (Adaptive Replacement Cache) cache policy.
|
||||||
|
|
||||||
Data Structures:
|
Data Structures:
|
||||||
T1: Recent cache containing blocks accessed once.
|
T1: Recent cache containing chunks accessed once.
|
||||||
T2: Frequent cache containing blocks accessed multiple times.
|
T2: Frequent cache containing chunks accessed multiple times.
|
||||||
B1/B2: Ghost lists tracking recently evicted blocks from T1/T2.
|
B1/B2: Ghost lists tracking recently evicted chunks from T1/T2.
|
||||||
target_t1_size: Adaptive target size for the T1 partition.
|
target_t1_size: Adaptive target size for the T1 partition.
|
||||||
|
|
||||||
Algorithm Flow:
|
Algorithm Flow:
|
||||||
1. Cache lookup (lookup):
|
1. Cache lookup (lookup):
|
||||||
Searches T1 and T2 for block hashes and counts consecutive hits
|
Searches T1 and T2 for chunk hashes and counts consecutive hits
|
||||||
until a miss or non-ready block is encountered.
|
until a miss or non-ready chunk is encountered.
|
||||||
|
|
||||||
2. Cache touch (touch) - Adaptive Learning:
|
2. Cache touch (touch) - Adaptive Learning:
|
||||||
For each key (in reverse order):
|
For each key (in reverse order):
|
||||||
@@ -31,15 +31,15 @@ class ARCCachePolicy(CachePolicy):
|
|||||||
- If in B1 ghost list: Increase target_t1_size.
|
- If in B1 ghost list: Increase target_t1_size.
|
||||||
- If in B2 ghost list: Decrease target_t1_size.
|
- If in B2 ghost list: Decrease target_t1_size.
|
||||||
|
|
||||||
3. Block eviction (evict) - Adaptive Replacement:
|
3. Chunk eviction (evict) - Adaptive Replacement:
|
||||||
Determines eviction source based on adaptive target:
|
Determines eviction source based on adaptive target:
|
||||||
- If T1 size >= target_t1_size: Evict from T1, add to B1.
|
- If T1 size >= target_t1_size: Evict from T1, add to B1.
|
||||||
- Otherwise: Evict from T2, add to B2.
|
- Otherwise: Evict from T2, add to B2.
|
||||||
Finally, bound each ghost list size.
|
Finally, bound each ghost list size.
|
||||||
|
|
||||||
4. Block insertion (insert):
|
4. Chunk insertion (insert):
|
||||||
New blocks are always inserted into T1 and removed from B1/B2 if
|
New chunks are always inserted into T1 and removed from B1/B2 if
|
||||||
present. Blocks may later be promoted to T2 during touch operations.
|
present. Chunks may later be promoted to T2 during touch operations.
|
||||||
|
|
||||||
Adaptive Behavior:
|
Adaptive Behavior:
|
||||||
The algorithm self-tunes the recency vs. frequency trade-off:
|
The algorithm self-tunes the recency vs. frequency trade-off:
|
||||||
@@ -50,19 +50,19 @@ class ARCCachePolicy(CachePolicy):
|
|||||||
def __init__(self, cache_capacity: int):
|
def __init__(self, cache_capacity: int):
|
||||||
super().__init__(cache_capacity)
|
super().__init__(cache_capacity)
|
||||||
self.target_t1_size: float = 0.0
|
self.target_t1_size: float = 0.0
|
||||||
self.t1: OrderedDict[OffloadKey, BlockStatus] = OrderedDict()
|
self.t1: OrderedDict[OffloadKey, ChunkStatus] = OrderedDict()
|
||||||
self.t2: OrderedDict[OffloadKey, BlockStatus] = OrderedDict()
|
self.t2: OrderedDict[OffloadKey, ChunkStatus] = OrderedDict()
|
||||||
# key -> None (only care about presence)
|
# key -> None (only care about presence)
|
||||||
self.b1: OrderedDict[OffloadKey, None] = OrderedDict()
|
self.b1: OrderedDict[OffloadKey, None] = OrderedDict()
|
||||||
self.b2: OrderedDict[OffloadKey, None] = OrderedDict()
|
self.b2: OrderedDict[OffloadKey, None] = OrderedDict()
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def get(self, key: OffloadKey) -> BlockStatus | None:
|
def get(self, key: OffloadKey) -> ChunkStatus | None:
|
||||||
return self.t1.get(key) or self.t2.get(key)
|
return self.t1.get(key) or self.t2.get(key)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def insert(self, key: OffloadKey, block: BlockStatus) -> None:
|
def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
|
||||||
self.t1[key] = block
|
self.t1[key] = chunk
|
||||||
self.b1.pop(key, None)
|
self.b1.pop(key, None)
|
||||||
self.b2.pop(key, None)
|
self.b2.pop(key, None)
|
||||||
|
|
||||||
@@ -75,13 +75,13 @@ class ARCCachePolicy(CachePolicy):
|
|||||||
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
||||||
for key in reversed(list(keys)):
|
for key in reversed(list(keys)):
|
||||||
if key in self.t1:
|
if key in self.t1:
|
||||||
block = self.t1.pop(key)
|
chunk = self.t1.pop(key)
|
||||||
if not block.is_ready:
|
if not chunk.is_ready:
|
||||||
# block was just prepared to be stored, not really touched
|
# chunk was just prepared to be stored, not really touched
|
||||||
# twice — keep it in T1 and mark as most recently used
|
# twice — keep it in T1 and mark as most recently used
|
||||||
self.t1[key] = block
|
self.t1[key] = chunk
|
||||||
else:
|
else:
|
||||||
self.t2[key] = block
|
self.t2[key] = chunk
|
||||||
|
|
||||||
elif key in self.t2:
|
elif key in self.t2:
|
||||||
self.t2.move_to_end(key)
|
self.t2.move_to_end(key)
|
||||||
@@ -111,31 +111,31 @@ class ARCCachePolicy(CachePolicy):
|
|||||||
@override
|
@override
|
||||||
def evict(
|
def evict(
|
||||||
self, n: int, protected: set[OffloadKey]
|
self, n: int, protected: set[OffloadKey]
|
||||||
) -> list[tuple[OffloadKey, BlockStatus]] | None:
|
) -> list[tuple[OffloadKey, ChunkStatus]] | None:
|
||||||
if n == 0:
|
if n == 0:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
# Collect candidates atomically: simulate T1 size changes as we select,
|
# Collect candidates atomically: simulate T1 size changes as we select,
|
||||||
# but do not modify actual data structures until all n are found.
|
# but do not modify actual data structures until all n are found.
|
||||||
candidates: list[
|
candidates: list[
|
||||||
tuple[OffloadKey, BlockStatus, bool]
|
tuple[OffloadKey, ChunkStatus, bool]
|
||||||
] = [] # (key, block, from_t1)
|
] = [] # (key, chunk, from_t1)
|
||||||
virtual_t1_size = len(self.t1)
|
virtual_t1_size = len(self.t1)
|
||||||
# Keep the scans monotonic: restarting from the LRU end after every
|
# Keep the scans monotonic: restarting from the LRU end after every
|
||||||
# selection makes a batch eviction quadratic in the number of blocks.
|
# selection makes a batch eviction quadratic in the number of chunks.
|
||||||
t1_iter = iter(self.t1.items())
|
t1_iter = iter(self.t1.items())
|
||||||
t2_iter = iter(self.t2.items())
|
t2_iter = iter(self.t2.items())
|
||||||
|
|
||||||
def next_candidate(
|
def next_candidate(
|
||||||
entries: Iterator[tuple[OffloadKey, BlockStatus]],
|
entries: Iterator[tuple[OffloadKey, ChunkStatus]],
|
||||||
) -> tuple[OffloadKey, BlockStatus] | None:
|
) -> tuple[OffloadKey, ChunkStatus] | None:
|
||||||
for key, block in entries:
|
for key, chunk in entries:
|
||||||
if block.ref_cnt == 0 and key not in protected:
|
if chunk.ref_cnt == 0 and key not in protected:
|
||||||
return key, block
|
return key, chunk
|
||||||
return None
|
return None
|
||||||
|
|
||||||
for _ in range(n):
|
for _ in range(n):
|
||||||
candidate: tuple[OffloadKey, BlockStatus, bool] | None = None
|
candidate: tuple[OffloadKey, ChunkStatus, bool] | None = None
|
||||||
|
|
||||||
if virtual_t1_size >= int(self.target_t1_size):
|
if virtual_t1_size >= int(self.target_t1_size):
|
||||||
entry = next_candidate(t1_iter)
|
entry = next_candidate(t1_iter)
|
||||||
@@ -158,15 +158,15 @@ class ARCCachePolicy(CachePolicy):
|
|||||||
candidates.append(candidate)
|
candidates.append(candidate)
|
||||||
|
|
||||||
# Apply all evictions now that we know n candidates exist.
|
# Apply all evictions now that we know n candidates exist.
|
||||||
result: list[tuple[OffloadKey, BlockStatus]] = []
|
result: list[tuple[OffloadKey, ChunkStatus]] = []
|
||||||
for key, block, from_t1 in candidates:
|
for key, chunk, from_t1 in candidates:
|
||||||
if from_t1:
|
if from_t1:
|
||||||
del self.t1[key]
|
del self.t1[key]
|
||||||
self.b1[key] = None
|
self.b1[key] = None
|
||||||
else:
|
else:
|
||||||
del self.t2[key]
|
del self.t2[key]
|
||||||
self.b2[key] = None
|
self.b2[key] = None
|
||||||
result.append((key, block))
|
result.append((key, chunk))
|
||||||
|
|
||||||
# Trim ghost lists to cache_capacity.
|
# Trim ghost lists to cache_capacity.
|
||||||
for ghost in (self.b1, self.b2):
|
for ghost in (self.b1, self.b2):
|
||||||
|
|||||||
@@ -7,36 +7,36 @@ from collections.abc import Iterable
|
|||||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||||
|
|
||||||
|
|
||||||
class BlockStatus(ctypes.Structure):
|
class ChunkStatus(ctypes.Structure):
|
||||||
"""
|
"""
|
||||||
Offloading status for a single block of KV data.
|
Offloading status for a single chunk of KV data.
|
||||||
Holds the following information:
|
Holds the following information:
|
||||||
|
|
||||||
ref_cnt - the current number of transfers using this block as a source.
|
ref_cnt - the current number of transfers using this chunk as a source.
|
||||||
A value of -1 indicates the block is not yet ready to be read.
|
A value of -1 indicates the chunk is not yet ready to be read.
|
||||||
block_id - index of the physical CPU buffer slot.
|
chunk_id - index of the physical CPU buffer slot.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_fields_ = [("ref_cnt", ctypes.c_int32), ("block_id", ctypes.c_int64)]
|
_fields_ = [("ref_cnt", ctypes.c_int32), ("chunk_id", ctypes.c_int64)]
|
||||||
|
|
||||||
def __init__(self, block_id: int):
|
def __init__(self, chunk_id: int):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
# initialize block as "not ready" (ref_cnt = -1)
|
# initialize chunk as "not ready" (ref_cnt = -1)
|
||||||
self.ref_cnt = -1
|
self.ref_cnt = -1
|
||||||
self.block_id = block_id
|
self.chunk_id = chunk_id
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_ready(self) -> bool:
|
def is_ready(self) -> bool:
|
||||||
"""
|
"""
|
||||||
Returns whether the block is ready to be read.
|
Returns whether the chunk is ready to be read.
|
||||||
"""
|
"""
|
||||||
return self.ref_cnt >= 0
|
return self.ref_cnt >= 0
|
||||||
|
|
||||||
|
|
||||||
class CachePolicy(ABC):
|
class CachePolicy(ABC):
|
||||||
"""
|
"""
|
||||||
Encapsulates both block organization (data structures) and replacement
|
Encapsulates both chunk organization (data structures) and replacement
|
||||||
decisions (which block to evict). LRU and ARC differ in both dimensions —
|
decisions (which chunk to evict). LRU and ARC differ in both dimensions —
|
||||||
ARC's ghost lists and target_t1_size live at the intersection of storage
|
ARC's ghost lists and target_t1_size live at the intersection of storage
|
||||||
and eviction, so they cannot be separated cleanly.
|
and eviction, so they cannot be separated cleanly.
|
||||||
"""
|
"""
|
||||||
@@ -45,35 +45,35 @@ class CachePolicy(ABC):
|
|||||||
self.cache_capacity = cache_capacity
|
self.cache_capacity = cache_capacity
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get(self, key: OffloadKey) -> BlockStatus | None:
|
def get(self, key: OffloadKey) -> ChunkStatus | None:
|
||||||
"""Find block in data structures. Returns None if not present."""
|
"""Find chunk in data structures. Returns None if not present."""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def insert(self, key: OffloadKey, block: BlockStatus) -> None:
|
def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
|
||||||
"""Add a newly allocated block. For ARC: also removes from ghost lists."""
|
"""Add a newly allocated chunk. For ARC: also removes from ghost lists."""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def remove(self, key: OffloadKey) -> None:
|
def remove(self, key: OffloadKey) -> None:
|
||||||
"""Remove a block (used to clean up after a failed store)."""
|
"""Remove a chunk (used to clean up after a failed store)."""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
||||||
"""
|
"""
|
||||||
Mark blocks as recently used.
|
Mark chunks as recently used.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
keys: Blocks to mark as recently used.
|
keys: Chunks to mark as recently used.
|
||||||
req_context: Per-request context for the request touching these blocks.
|
req_context: Per-request context for the request touching these chunks.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def evict(
|
def evict(
|
||||||
self, n: int, protected: set[OffloadKey]
|
self, n: int, protected: set[OffloadKey]
|
||||||
) -> list[tuple[OffloadKey, BlockStatus]] | None:
|
) -> list[tuple[OffloadKey, ChunkStatus]] | None:
|
||||||
"""
|
"""
|
||||||
Evict exactly n blocks, skipping any in protected.
|
Evict exactly n chunks, skipping any in protected.
|
||||||
|
|
||||||
Returns a list of (key, block) for the evicted blocks,
|
Returns a list of (key, chunk) for the evicted chunks,
|
||||||
or None if n evictions cannot be satisfied. The operation is atomic:
|
or None if n evictions cannot be satisfied. The operation is atomic:
|
||||||
if None is returned, no state changes are made.
|
if None is returned, no state changes are made.
|
||||||
|
|
||||||
@@ -84,15 +84,15 @@ class CachePolicy(ABC):
|
|||||||
@abstractmethod
|
@abstractmethod
|
||||||
def clear(self) -> None:
|
def clear(self) -> None:
|
||||||
"""
|
"""
|
||||||
Remove ALL blocks regardless of ref_cnt.
|
Remove ALL chunks regardless of ref_cnt.
|
||||||
|
|
||||||
Ghost lists and adaptive state are also reset.
|
Ghost lists and adaptive state are also reset.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def mark_evictable(self, key: OffloadKey) -> None:
|
def mark_evictable(self, key: OffloadKey) -> None:
|
||||||
"""Called when a block's ref_cnt transitions to 0."""
|
"""Called when a chunk's ref_cnt transitions to 0."""
|
||||||
return
|
return
|
||||||
|
|
||||||
def mark_non_evictable(self, key: OffloadKey) -> None:
|
def mark_non_evictable(self, key: OffloadKey) -> None:
|
||||||
"""Called when a block's ref_cnt transitions from 0."""
|
"""Called when a chunk's ref_cnt transitions from 0."""
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from collections.abc import Iterable
|
|||||||
from typing_extensions import override
|
from typing_extensions import override
|
||||||
|
|
||||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||||
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
|
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
|
||||||
|
|
||||||
|
|
||||||
class LRUCachePolicy(CachePolicy):
|
class LRUCachePolicy(CachePolicy):
|
||||||
@@ -20,71 +20,71 @@ class LRUCachePolicy(CachePolicy):
|
|||||||
|
|
||||||
def __init__(self, cache_capacity: int):
|
def __init__(self, cache_capacity: int):
|
||||||
super().__init__(cache_capacity)
|
super().__init__(cache_capacity)
|
||||||
# Blocks with ref_cnt 0 (not participating in any loads/stores) ordered in LRU
|
# Chunks with ref_cnt 0 (not participating in any loads/stores) ordered in LRU
|
||||||
self.evictable_blocks: OrderedDict[OffloadKey, None] = OrderedDict()
|
self.evictable_chunks: OrderedDict[OffloadKey, None] = OrderedDict()
|
||||||
self.blocks: dict[OffloadKey, BlockStatus] = {}
|
self.chunks: dict[OffloadKey, ChunkStatus] = {}
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def get(self, key: OffloadKey) -> BlockStatus | None:
|
def get(self, key: OffloadKey) -> ChunkStatus | None:
|
||||||
return self.blocks.get(key)
|
return self.chunks.get(key)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def insert(self, key: OffloadKey, block: BlockStatus) -> None:
|
def insert(self, key: OffloadKey, chunk: ChunkStatus) -> None:
|
||||||
self.blocks[key] = block
|
self.chunks[key] = chunk
|
||||||
if block.ref_cnt == 0:
|
if chunk.ref_cnt == 0:
|
||||||
self.evictable_blocks[key] = None
|
self.evictable_chunks[key] = None
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def remove(self, key: OffloadKey) -> None:
|
def remove(self, key: OffloadKey) -> None:
|
||||||
del self.blocks[key]
|
del self.chunks[key]
|
||||||
self.evictable_blocks.pop(key, None)
|
self.evictable_chunks.pop(key, None)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
||||||
for key in reversed(list(keys)):
|
for key in reversed(list(keys)):
|
||||||
if key in self.evictable_blocks:
|
if key in self.evictable_chunks:
|
||||||
self.evictable_blocks.move_to_end(key)
|
self.evictable_chunks.move_to_end(key)
|
||||||
# active blocks are untouched as they are non-evictable now. They
|
# active chunks are untouched as they are non-evictable now. They
|
||||||
# will eventually reach the end of evictable_blocks when they finish.
|
# will eventually reach the end of evictable_chunks when they finish.
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def clear(self) -> None:
|
def clear(self) -> None:
|
||||||
self.evictable_blocks.clear()
|
self.evictable_chunks.clear()
|
||||||
self.blocks.clear()
|
self.chunks.clear()
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def evict(
|
def evict(
|
||||||
self, n: int, protected: set[OffloadKey]
|
self, n: int, protected: set[OffloadKey]
|
||||||
) -> list[tuple[OffloadKey, BlockStatus]] | None:
|
) -> list[tuple[OffloadKey, ChunkStatus]] | None:
|
||||||
if n == 0:
|
if n == 0:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
candidates: list[tuple[OffloadKey, BlockStatus]] = []
|
candidates: list[tuple[OffloadKey, ChunkStatus]] = []
|
||||||
for key, _ in self.evictable_blocks.items():
|
for key, _ in self.evictable_chunks.items():
|
||||||
if key in protected:
|
if key in protected:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
block = self.blocks[key]
|
chunk = self.chunks[key]
|
||||||
assert block.ref_cnt == 0
|
assert chunk.ref_cnt == 0
|
||||||
candidates.append((key, block))
|
candidates.append((key, chunk))
|
||||||
if len(candidates) == n:
|
if len(candidates) == n:
|
||||||
break
|
break
|
||||||
|
|
||||||
if len(candidates) < n:
|
if len(candidates) < n:
|
||||||
return None
|
return None
|
||||||
for key, _ in candidates:
|
for key, _ in candidates:
|
||||||
del self.evictable_blocks[key]
|
del self.evictable_chunks[key]
|
||||||
del self.blocks[key]
|
del self.chunks[key]
|
||||||
return candidates
|
return candidates
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def mark_evictable(self, key: OffloadKey) -> None:
|
def mark_evictable(self, key: OffloadKey) -> None:
|
||||||
# blocks can become evictable when,
|
# chunks can become evictable when,
|
||||||
# store completes - i.e. ref_cnt -1 -> 0 # not in evictable list
|
# store completes - i.e. ref_cnt -1 -> 0 # not in evictable list
|
||||||
# all loads complete - i.e ref_cnt 1 -> 0 # not in evictable list
|
# all loads complete - i.e ref_cnt 1 -> 0 # not in evictable list
|
||||||
self.evictable_blocks[key] = None
|
self.evictable_chunks[key] = None
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def mark_non_evictable(self, key: OffloadKey) -> None:
|
def mark_non_evictable(self, key: OffloadKey) -> None:
|
||||||
# key must have been in the evictable list.
|
# key must have been in the evictable list.
|
||||||
del self.evictable_blocks[key]
|
del self.evictable_chunks[key]
|
||||||
|
|||||||
@@ -82,24 +82,24 @@ class SharedOffloadRegion:
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
engine_id: str,
|
engine_id: str,
|
||||||
num_blocks: int,
|
num_chunks: int,
|
||||||
rank: int | None,
|
rank: int | None,
|
||||||
kv_bytes_per_block: int,
|
kv_bytes_per_chunk: int,
|
||||||
cpu_page_size: int,
|
cpu_page_size: int,
|
||||||
barrier: Callable[[], None] | None = None,
|
barrier: Callable[[], None] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.page_size = mmap.PAGESIZE
|
self.page_size = mmap.PAGESIZE
|
||||||
assert kv_bytes_per_block % self.page_size == 0
|
assert kv_bytes_per_chunk % self.page_size == 0
|
||||||
|
|
||||||
self.num_blocks = num_blocks
|
self.num_chunks = num_chunks
|
||||||
self._row_stride = kv_bytes_per_block
|
self._row_stride = kv_bytes_per_chunk
|
||||||
self.total_size_bytes = self.num_blocks * self._row_stride
|
self.total_size_bytes = self.num_chunks * self._row_stride
|
||||||
|
|
||||||
self.mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap"
|
self.mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap"
|
||||||
self._creator = False # set True only if this worker creates the file
|
self._creator = False # set True only if this worker creates the file
|
||||||
self.rank = rank
|
self.rank = rank
|
||||||
if rank is not None:
|
if rank is not None:
|
||||||
# byte offset to this worker's first slot within each block row
|
# byte offset to this worker's first slot within each chunk row
|
||||||
self._worker_offset = rank * cpu_page_size
|
self._worker_offset = rank * cpu_page_size
|
||||||
# exclusive upper bound for this worker's area within each row
|
# exclusive upper bound for this worker's area within each row
|
||||||
self._worker_area_end = (rank + 1) * cpu_page_size
|
self._worker_area_end = (rank + 1) * cpu_page_size
|
||||||
@@ -182,19 +182,19 @@ class SharedOffloadRegion:
|
|||||||
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
|
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
|
||||||
|
|
||||||
if rank is not None:
|
if rank is not None:
|
||||||
# Populate only this worker's pages (one slot per block row).
|
# Populate only this worker's pages (one slot per chunk row).
|
||||||
worker_offset = rank * cpu_page_size
|
worker_offset = rank * cpu_page_size
|
||||||
_t0 = time.perf_counter()
|
_t0 = time.perf_counter()
|
||||||
page_size = self.page_size
|
page_size = self.page_size
|
||||||
for block in range(num_blocks):
|
for chunk in range(num_chunks):
|
||||||
raw_offset = block * self._row_stride + worker_offset
|
raw_offset = chunk * self._row_stride + worker_offset
|
||||||
aligned_offset = (raw_offset // page_size) * page_size
|
aligned_offset = (raw_offset // page_size) * page_size
|
||||||
end = raw_offset + cpu_page_size
|
end = raw_offset + cpu_page_size
|
||||||
aligned_length = end - aligned_offset
|
aligned_length = end - aligned_offset
|
||||||
populate_write_fn(self.mmap_obj, aligned_offset, aligned_length)
|
populate_write_fn(self.mmap_obj, aligned_offset, aligned_length)
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"MADV_POPULATE_WRITE loop: %d blocks in %.3f s",
|
"MADV_POPULATE_WRITE loop: %d chunks in %.3f s",
|
||||||
num_blocks,
|
num_chunks,
|
||||||
time.perf_counter() - _t0,
|
time.perf_counter() - _t0,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
@@ -215,22 +215,22 @@ class SharedOffloadRegion:
|
|||||||
|
|
||||||
Must be called once per canonical tensor. The full mmap layout is:
|
Must be called once per canonical tensor. The full mmap layout is:
|
||||||
|
|
||||||
worker0_block0 | worker1_block0 | ... | worker{M-1}_block0
|
worker0_chunk0 | worker1_chunk0 | ... | worker{M-1}_chunk0
|
||||||
worker0_block1 | worker1_block1 | ... | worker{M-1}_block1
|
worker0_chunk1 | worker1_chunk1 | ... | worker{M-1}_chunk1
|
||||||
...
|
...
|
||||||
|
|
||||||
Each worker_block cell is cpu_page_size bytes and holds all canonical
|
Each worker_chunk cell is cpu_page_size bytes and holds all canonical
|
||||||
tensors for that worker and block concatenated:
|
tensors for that worker and chunk concatenated:
|
||||||
[ tensor0_data | tensor1_data | ... | tensor{L-1}_data ]
|
[ tensor0_data | tensor1_data | ... | tensor{L-1}_data ]
|
||||||
|
|
||||||
Consecutive rows are separated by row_stride = cpu_page_size * M.
|
Consecutive rows are separated by row_stride = cpu_page_size * M.
|
||||||
|
|
||||||
Returns an int8 tensor of shape (num_blocks, tensor_page_size) with stride
|
Returns an int8 tensor of shape (num_chunks, tensor_page_size) with stride
|
||||||
(row_stride, 1). Using int8 keeps stride == bytes, so swap_blocks
|
(row_stride, 1). Using int8 keeps stride == bytes, so swap_blocks
|
||||||
address arithmetic works without any dtype conversion.
|
address arithmetic works without any dtype conversion.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
tensor_page_size: Bytes per block for this tensor.
|
tensor_page_size: Bytes per chunk for this tensor.
|
||||||
"""
|
"""
|
||||||
assert self.rank is not None
|
assert self.rank is not None
|
||||||
new_offset = self._worker_offset + tensor_page_size
|
new_offset = self._worker_offset + tensor_page_size
|
||||||
@@ -241,7 +241,7 @@ class SharedOffloadRegion:
|
|||||||
)
|
)
|
||||||
worker_layer_view = torch.as_strided(
|
worker_layer_view = torch.as_strided(
|
||||||
self._base,
|
self._base,
|
||||||
size=(self.num_blocks, tensor_page_size),
|
size=(self.num_chunks, tensor_page_size),
|
||||||
stride=(self._row_stride, 1),
|
stride=(self._row_stride, 1),
|
||||||
storage_offset=self._worker_offset,
|
storage_offset=self._worker_offset,
|
||||||
)
|
)
|
||||||
@@ -266,8 +266,8 @@ class SharedOffloadRegion:
|
|||||||
_canonical_offset=0, then advances by each tensor's size
|
_canonical_offset=0, then advances by each tensor's size
|
||||||
|
|
||||||
Each canonical_t{i} cell is that tensor's canonical page for the
|
Each canonical_t{i} cell is that tensor's canonical page for the
|
||||||
block. Canonical areas are carved consecutively from the start of
|
chunk. Canonical areas are carved consecutively from the start of
|
||||||
each block row; consecutive rows are separated by row_stride. Every
|
each chunk row; consecutive rows are separated by row_stride. Every
|
||||||
worker gets the identical byte ranges and writes only its disjoint
|
worker gets the identical byte ranges and writes only its disjoint
|
||||||
bytes within them, as described by its canonical mappings — unlike
|
bytes within them, as described by its canonical mappings — unlike
|
||||||
create_next_worker_view, which gives each worker a private
|
create_next_worker_view, which gives each worker a private
|
||||||
@@ -279,13 +279,13 @@ class SharedOffloadRegion:
|
|||||||
where one canonical copy replaces world_size worker copies.
|
where one canonical copy replaces world_size worker copies.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
tensor_page_size: Canonical bytes per block for this tensor.
|
tensor_page_size: Canonical bytes per chunk for this tensor.
|
||||||
"""
|
"""
|
||||||
new_offset = self._canonical_offset + tensor_page_size
|
new_offset = self._canonical_offset + tensor_page_size
|
||||||
assert new_offset <= self._row_stride
|
assert new_offset <= self._row_stride
|
||||||
view = torch.as_strided(
|
view = torch.as_strided(
|
||||||
self._base,
|
self._base,
|
||||||
size=(self.num_blocks, tensor_page_size),
|
size=(self.num_chunks, tensor_page_size),
|
||||||
stride=(self._row_stride, 1),
|
stride=(self._row_stride, 1),
|
||||||
storage_offset=self._canonical_offset,
|
storage_offset=self._canonical_offset,
|
||||||
)
|
)
|
||||||
@@ -296,10 +296,10 @@ class SharedOffloadRegion:
|
|||||||
def create_kv_memoryview(self) -> memoryview:
|
def create_kv_memoryview(self) -> memoryview:
|
||||||
"""Return a zero-copy memoryview over the entire KV buffer.
|
"""Return a zero-copy memoryview over the entire KV buffer.
|
||||||
|
|
||||||
Shape: (num_blocks, row_stride_bytes). Secondary tiers address
|
Shape: (num_chunks, row_stride_bytes). Secondary tiers address
|
||||||
block *b* as ``view[b]``.
|
chunk *b* as ``view[b]``.
|
||||||
"""
|
"""
|
||||||
kv_tensor = self._base.view(self.num_blocks, self._row_stride)
|
kv_tensor = self._base.view(self.num_chunks, self._row_stride)
|
||||||
np_arr = kv_tensor.numpy()
|
np_arr = kv_tensor.numpy()
|
||||||
assert np_arr.ctypes.data == self._base.data_ptr(), (
|
assert np_arr.ctypes.data == self._base.data_ptr(), (
|
||||||
"view()/numpy() created a copy instead of sharing the mmap buffer; "
|
"view()/numpy() created a copy instead of sharing the mmap buffer; "
|
||||||
|
|||||||
@@ -73,7 +73,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
|||||||
),
|
),
|
||||||
CPUOffloadingMetrics.CPU_ALLOCATION_SIZE: OffloadingHistogramMetadata(
|
CPUOffloadingMetrics.CPU_ALLOCATION_SIZE: OffloadingHistogramMetadata(
|
||||||
documentation=(
|
documentation=(
|
||||||
"Histogram of the number of CPU blocks requested by each "
|
"Histogram of the number of CPU chunks requested by each "
|
||||||
"KV offload prepare_store call."
|
"KV offload prepare_store call."
|
||||||
),
|
),
|
||||||
buckets=(1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144),
|
buckets=(1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144),
|
||||||
@@ -101,7 +101,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
|||||||
)
|
)
|
||||||
|
|
||||||
world_size = config.parallel.world_size
|
world_size = config.parallel.world_size
|
||||||
self.num_blocks = 0
|
self.num_chunks = 0
|
||||||
self.kv_bytes_per_chunk = 0
|
self.kv_bytes_per_chunk = 0
|
||||||
self.cpu_page_size_per_worker = 0
|
self.cpu_page_size_per_worker = 0
|
||||||
self.replicated_layout = config.replicated_layout and self._uses_shared_region()
|
self.replicated_layout = config.replicated_layout and self._uses_shared_region()
|
||||||
@@ -113,17 +113,17 @@ class CPUOffloadingSpec(OffloadingSpec):
|
|||||||
# calculate cpu_page_size_per_worker
|
# calculate cpu_page_size_per_worker
|
||||||
self.cpu_page_size_per_worker = kv_bytes_per_chunk // num_copies
|
self.cpu_page_size_per_worker = kv_bytes_per_chunk // num_copies
|
||||||
|
|
||||||
# calculate num_blocks
|
# calculate num_chunks
|
||||||
aligned_kv_bytes_per_chunk = round_up(
|
aligned_kv_bytes_per_chunk = round_up(
|
||||||
kv_bytes_per_chunk, self.BLOCK_SIZE_ALIGNMENT
|
kv_bytes_per_chunk, self.BLOCK_SIZE_ALIGNMENT
|
||||||
)
|
)
|
||||||
self.num_blocks = int(cpu_bytes_to_use) // aligned_kv_bytes_per_chunk
|
self.num_chunks = int(cpu_bytes_to_use) // aligned_kv_bytes_per_chunk
|
||||||
|
|
||||||
# Expose aligned_kv_bytes_per_chunk as
|
# Expose aligned_kv_bytes_per_chunk as
|
||||||
# kv_bytes_per_chunk. Note that this might contain
|
# kv_bytes_per_chunk. Note that this might contain
|
||||||
# some padding. i.e. each offloaded block is of the form,
|
# some padding. i.e. each offloaded chunk is of the form,
|
||||||
# |--- W0-B0---|---- W1-B0---| ... |---- Wn-B0---| *** maybe-pad *** |
|
# |--- W0-C0---|---- W1-C0---| ... |---- Wn-C0---| *** maybe-pad *** |
|
||||||
# or |--- B0 (single copy) ---| *** maybe-pad *** |
|
# or |--- C0 (single copy) ---| *** maybe-pad *** |
|
||||||
self.kv_bytes_per_chunk = aligned_kv_bytes_per_chunk
|
self.kv_bytes_per_chunk = aligned_kv_bytes_per_chunk
|
||||||
|
|
||||||
# scheduler-side
|
# scheduler-side
|
||||||
@@ -140,7 +140,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
|||||||
@override
|
@override
|
||||||
def get_manager(self) -> OffloadingManager:
|
def get_manager(self) -> OffloadingManager:
|
||||||
if not self._manager:
|
if not self._manager:
|
||||||
# store_threshold: how many times a block must be offered for
|
# store_threshold: how many times a chunk must be offered for
|
||||||
# storage before it is eligible for CPU offloading. Values < 2
|
# storage before it is eligible for CPU offloading. Values < 2
|
||||||
# disable filtering (a threshold of 1 equals no filter; 0 is the
|
# disable filtering (a threshold of 1 equals no filter; 0 is the
|
||||||
# default).
|
# default).
|
||||||
@@ -150,7 +150,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
|||||||
max_tracker_size = int(self.extra_config.get("max_tracker_size", 64_000))
|
max_tracker_size = int(self.extra_config.get("max_tracker_size", 64_000))
|
||||||
|
|
||||||
self._manager = CPUOffloadingManager(
|
self._manager = CPUOffloadingManager(
|
||||||
num_blocks=self.num_blocks,
|
num_chunks=self.num_chunks,
|
||||||
cache_policy=self.eviction_policy,
|
cache_policy=self.eviction_policy,
|
||||||
cache_policy_module_path=self.cache_policy_module_path,
|
cache_policy_module_path=self.cache_policy_module_path,
|
||||||
enable_events=self.kv_events_config.enable_kv_cache_events,
|
enable_events=self.kv_events_config.enable_kv_cache_events,
|
||||||
@@ -166,9 +166,9 @@ class CPUOffloadingSpec(OffloadingSpec):
|
|||||||
|
|
||||||
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
|
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
|
||||||
mmap_region: SharedOffloadRegion | None = None
|
mmap_region: SharedOffloadRegion | None = None
|
||||||
# num_blocks == 0 would size the region to zero bytes, which cannot be
|
# num_chunks == 0 would size the region to zero bytes, which cannot be
|
||||||
# mmap'd; fall back to the tensor path (empty tensors) as before.
|
# mmap'd; fall back to the tensor path (empty tensors) as before.
|
||||||
if self._uses_shared_region() and self.num_blocks > 0:
|
if self._uses_shared_region() and self.num_chunks > 0:
|
||||||
# Replicated layout puts all ranks on slot 0 (single MLA copy);
|
# Replicated layout puts all ranks on slot 0 (single MLA copy);
|
||||||
# otherwise each rank takes its own slot by physical device index.
|
# otherwise each rank takes its own slot by physical device index.
|
||||||
if self.replicated_layout:
|
if self.replicated_layout:
|
||||||
@@ -178,9 +178,9 @@ class CPUOffloadingSpec(OffloadingSpec):
|
|||||||
rank = torch.accelerator.current_device_index() % world_size
|
rank = torch.accelerator.current_device_index() % world_size
|
||||||
mmap_region = SharedOffloadRegion(
|
mmap_region = SharedOffloadRegion(
|
||||||
engine_id=self.config.engine_id,
|
engine_id=self.config.engine_id,
|
||||||
num_blocks=self.num_blocks,
|
num_chunks=self.num_chunks,
|
||||||
rank=rank,
|
rank=rank,
|
||||||
kv_bytes_per_block=self.kv_bytes_per_chunk,
|
kv_bytes_per_chunk=self.kv_bytes_per_chunk,
|
||||||
cpu_page_size=self.cpu_page_size_per_worker,
|
cpu_page_size=self.cpu_page_size_per_worker,
|
||||||
barrier=_all_workers_barrier,
|
barrier=_all_workers_barrier,
|
||||||
)
|
)
|
||||||
@@ -188,7 +188,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
|||||||
return CPUOffloadingWorker(
|
return CPUOffloadingWorker(
|
||||||
kv_caches=kv_caches,
|
kv_caches=kv_caches,
|
||||||
blocks_per_chunk=self.blocks_per_chunk,
|
blocks_per_chunk=self.blocks_per_chunk,
|
||||||
num_cpu_blocks=self.num_blocks,
|
num_cpu_chunks=self.num_chunks,
|
||||||
mmap_region=mmap_region,
|
mmap_region=mmap_region,
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
@@ -45,8 +45,8 @@ class TieringOffloadingMetrics:
|
|||||||
WRITE_TIME = "vllm:kv_offload_tiering_write_time"
|
WRITE_TIME = "vllm:kv_offload_tiering_write_time"
|
||||||
PROMOTION_JOB_FAILURES = "vllm:kv_offload_tiering_promotion_job_failures"
|
PROMOTION_JOB_FAILURES = "vllm:kv_offload_tiering_promotion_job_failures"
|
||||||
CASCADE_JOB_FAILURES = "vllm:kv_offload_tiering_cascade_job_failures"
|
CASCADE_JOB_FAILURES = "vllm:kv_offload_tiering_cascade_job_failures"
|
||||||
BLOCK_QUERIES = "vllm:kv_offload_tiering_chunk_queries"
|
CHUNK_QUERIES = "vllm:kv_offload_tiering_chunk_queries"
|
||||||
BLOCK_HITS = "vllm:kv_offload_tiering_chunk_hits"
|
CHUNK_HITS = "vllm:kv_offload_tiering_chunk_hits"
|
||||||
PRIMARY_WRITE_USAGE_PERC = "vllm:kv_offload_tiering_primary_write_usage_perc"
|
PRIMARY_WRITE_USAGE_PERC = "vllm:kv_offload_tiering_primary_write_usage_perc"
|
||||||
PRIMARY_READ_USAGE_PERC = "vllm:kv_offload_tiering_primary_read_usage_perc"
|
PRIMARY_READ_USAGE_PERC = "vllm:kv_offload_tiering_primary_read_usage_perc"
|
||||||
PROMOTION_ALLOCATION_FAILURES = (
|
PROMOTION_ALLOCATION_FAILURES = (
|
||||||
@@ -62,7 +62,7 @@ class TransferJob:
|
|||||||
|
|
||||||
job_id: JobId
|
job_id: JobId
|
||||||
keys: Collection[OffloadKey]
|
keys: Collection[OffloadKey]
|
||||||
block_ids: np.ndarray
|
chunk_ids: np.ndarray
|
||||||
is_promotion: bool
|
is_promotion: bool
|
||||||
req_context: ReqContext
|
req_context: ReqContext
|
||||||
|
|
||||||
@@ -90,14 +90,14 @@ class ParentManager(ABC):
|
|||||||
|
|
||||||
Required call sequence for each remote request:
|
Required call sequence for each remote request:
|
||||||
1. on_new_request(req_context) — set up per-request state
|
1. on_new_request(req_context) — set up per-request state
|
||||||
2. lookup(key, req_context) — check block availability
|
2. lookup(key, req_context) — check chunk availability
|
||||||
(repeat per block)
|
(repeat per chunk)
|
||||||
3. create_store_job(keys, req_context) — pin blocks and get a
|
3. create_store_job(keys, req_context) — pin chunks and get a
|
||||||
job handle
|
job handle
|
||||||
4. on_request_finished(req_context) — clean up per-request state
|
4. on_request_finished(req_context) — clean up per-request state
|
||||||
|
|
||||||
Steps 2-3 may be interleaved. Step 4 must be called even if no
|
Steps 2-3 may be interleaved. Step 4 must be called even if no
|
||||||
blocks were found, to avoid leaking async lookup state (e.g. in
|
chunks were found, to avoid leaking async lookup state (e.g. in
|
||||||
the fs tier's AsyncLookupManager).
|
the fs tier's AsyncLookupManager).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -155,23 +155,23 @@ class SecondaryTierManager(ABC):
|
|||||||
@abstractmethod
|
@abstractmethod
|
||||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||||
"""
|
"""
|
||||||
Check whether a block exists in this secondary tier.
|
Check whether a chunk exists in this secondary tier.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
key: Offload key to look up.
|
key: Offload key to look up.
|
||||||
req_context: per-request context (e.g. kv_transfer_params).
|
req_context: per-request context (e.g. kv_transfer_params).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
HIT if the block is present and ready,
|
HIT if the chunk is present and ready,
|
||||||
MISS if not found,
|
MISS if not found,
|
||||||
or RETRY if the block is being transferred (retry later).
|
or RETRY if the chunk is being transferred (retry later).
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def submit_store(self, job_metadata: TransferJob) -> None:
|
def submit_store(self, job_metadata: TransferJob) -> None:
|
||||||
"""
|
"""
|
||||||
Submit an async job to store blocks from the primary tier to this
|
Submit an async job to store chunks from the primary tier to this
|
||||||
secondary tier.
|
secondary tier.
|
||||||
|
|
||||||
This method must be lightweight and non-blocking: allocate metadata
|
This method must be lightweight and non-blocking: allocate metadata
|
||||||
@@ -179,19 +179,19 @@ class SecondaryTierManager(ABC):
|
|||||||
calling thread.
|
calling thread.
|
||||||
|
|
||||||
Preconditions (guaranteed by the framework):
|
Preconditions (guaranteed by the framework):
|
||||||
- ``job_metadata.block_ids`` are valid primary-tier slots, pinned
|
- ``job_metadata.chunk_ids`` are valid primary-tier slots, pinned
|
||||||
(ref-counted) for the duration of the transfer.
|
(ref-counted) for the duration of the transfer.
|
||||||
|
|
||||||
The implementation is responsible for:
|
The implementation is responsible for:
|
||||||
1. Filtering out blocks already present in this tier
|
1. Filtering out chunks already present in this tier
|
||||||
2. Evicting blocks if capacity is needed
|
2. Evicting chunks if capacity is needed
|
||||||
3. Allocating space in this tier
|
3. Allocating space in this tier
|
||||||
4. Submitting the async transfer (read from primary via block_ids)
|
4. Submitting the async transfer (read from primary via chunk_ids)
|
||||||
|
|
||||||
Report completion via ``get_finished_jobs()``.
|
Report completion via ``get_finished_jobs()``.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
job_metadata: Job metadata including job_id, keys, and block_ids
|
job_metadata: Job metadata including job_id, keys, and chunk_ids
|
||||||
identifying the primary-tier slots to read from.
|
identifying the primary-tier slots to read from.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
@@ -199,24 +199,24 @@ class SecondaryTierManager(ABC):
|
|||||||
@abstractmethod
|
@abstractmethod
|
||||||
def submit_load(self, job_metadata: TransferJob) -> None:
|
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||||
"""
|
"""
|
||||||
Submit an async job to load blocks from this secondary tier to the
|
Submit an async job to load chunks from this secondary tier to the
|
||||||
primary tier.
|
primary tier.
|
||||||
|
|
||||||
This method must be lightweight and non-blocking: mark blocks as
|
This method must be lightweight and non-blocking: mark chunks as
|
||||||
in-flight and submit the transfer, but do NOT perform the data copy
|
in-flight and submit the transfer, but do NOT perform the data copy
|
||||||
on the calling thread.
|
on the calling thread.
|
||||||
|
|
||||||
Preconditions (guaranteed by the framework):
|
Preconditions (guaranteed by the framework):
|
||||||
- ``job_metadata.block_ids`` are allocated primary-tier slots
|
- ``job_metadata.chunk_ids`` are allocated primary-tier slots
|
||||||
ready to receive data.
|
ready to receive data.
|
||||||
|
|
||||||
The implementation must copy data from this tier into the
|
The implementation must copy data from this tier into the
|
||||||
primary-tier slots identified by ``block_ids``.
|
primary-tier slots identified by ``chunk_ids``.
|
||||||
|
|
||||||
Report completion via ``get_finished_jobs()``.
|
Report completion via ``get_finished_jobs()``.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
job_metadata: Job metadata including job_id, keys, and block_ids
|
job_metadata: Job metadata including job_id, keys, and chunk_ids
|
||||||
identifying the primary-tier slots to write into.
|
identifying the primary-tier slots to write into.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
@@ -249,7 +249,7 @@ class SecondaryTierManager(ABC):
|
|||||||
|
|
||||||
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
||||||
"""
|
"""
|
||||||
Mark blocks as recently used for eviction policy.
|
Mark chunks as recently used for eviction policy.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
keys: Offload keys to mark as recently used.
|
keys: Offload keys to mark as recently used.
|
||||||
@@ -263,7 +263,7 @@ class SecondaryTierManager(ABC):
|
|||||||
Called when a new request is first seen by the scheduler.
|
Called when a new request is first seen by the scheduler.
|
||||||
|
|
||||||
Returns a RequestOffloadingContext expressing this tier's preference
|
Returns a RequestOffloadingContext expressing this tier's preference
|
||||||
for how blocks should be offloaded for this request.
|
for how chunks should be offloaded for this request.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
req_context: Per-request context.
|
req_context: Per-request context.
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
"""
|
"""
|
||||||
ExampleSecondaryTierManager: A simple in-memory secondary tier.
|
ExampleSecondaryTierManager: A simple in-memory secondary tier.
|
||||||
|
|
||||||
This implementation provides a minimal secondary tier that stores blocks
|
This implementation provides a minimal secondary tier that stores chunks
|
||||||
in memory (using a dictionary) with immediate completion. It serves as a
|
in memory (using a dictionary) with immediate completion. It serves as a
|
||||||
reference for writing new tiers and is useful for testing the
|
reference for writing new tiers and is useful for testing the
|
||||||
TieringOffloadingManager without requiring actual storage or network backends.
|
TieringOffloadingManager without requiring actual storage or network backends.
|
||||||
@@ -39,7 +39,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
|||||||
A simple in-memory secondary tier.
|
A simple in-memory secondary tier.
|
||||||
|
|
||||||
This implementation:
|
This implementation:
|
||||||
- Stores blocks in a dictionary (key -> True)
|
- Stores chunks in a dictionary (key -> True)
|
||||||
- Completes transfers immediately (synchronous)
|
- Completes transfers immediately (synchronous)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -69,45 +69,45 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# key -> True (only care about presence)
|
# key -> True (only care about presence)
|
||||||
self.blocks: dict[OffloadKey, bool] = {}
|
self.chunks: dict[OffloadKey, bool] = {}
|
||||||
|
|
||||||
# Completed jobs waiting to be retrieved by get_finished_jobs()
|
# Completed jobs waiting to be retrieved by get_finished_jobs()
|
||||||
self.completed_jobs: list[JobResult] = []
|
self.completed_jobs: list[JobResult] = []
|
||||||
assert primary_kv_view.strides is not None
|
assert primary_kv_view.strides is not None
|
||||||
self._block_size = primary_kv_view.strides[0]
|
self._chunk_size = primary_kv_view.strides[0]
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||||
"""
|
"""
|
||||||
Check whether a block exists in this secondary tier.
|
Check whether a chunk exists in this secondary tier.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
key: Offload key to look up.
|
key: Offload key to look up.
|
||||||
req_context: Per-request context.
|
req_context: Per-request context.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
HIT if the block is present, MISS if not found.
|
HIT if the chunk is present, MISS if not found.
|
||||||
"""
|
"""
|
||||||
return LookupResult.HIT if key in self.blocks else LookupResult.MISS
|
return LookupResult.HIT if key in self.chunks else LookupResult.MISS
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def submit_store(self, job_metadata: TransferJob) -> None:
|
def submit_store(self, job_metadata: TransferJob) -> None:
|
||||||
"""
|
"""
|
||||||
Submit a job to store blocks from primary tier to this tier.
|
Submit a job to store chunks from primary tier to this tier.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
job_metadata: Job metadata including job_id, keys, and
|
job_metadata: Job metadata including job_id, keys, and
|
||||||
spec for reading blocks from the primary tier.
|
spec for reading chunks from the primary tier.
|
||||||
"""
|
"""
|
||||||
keys = job_metadata.keys
|
keys = job_metadata.keys
|
||||||
block_ids = job_metadata.block_ids
|
chunk_ids = job_metadata.chunk_ids
|
||||||
|
|
||||||
assert len(keys) == len(block_ids), (
|
assert len(keys) == len(chunk_ids), (
|
||||||
f"Length mismatch: {len(keys)} keys but {len(block_ids)} block_ids"
|
f"Length mismatch: {len(keys)} keys but {len(chunk_ids)} chunk_ids"
|
||||||
)
|
)
|
||||||
|
|
||||||
for key in keys:
|
for key in keys:
|
||||||
self.blocks[key] = True
|
self.chunks[key] = True
|
||||||
self.completed_jobs.append(
|
self.completed_jobs.append(
|
||||||
JobResult(
|
JobResult(
|
||||||
job_id=job_metadata.job_id,
|
job_id=job_metadata.job_id,
|
||||||
@@ -119,21 +119,21 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
|||||||
@override
|
@override
|
||||||
def submit_load(self, job_metadata: TransferJob) -> None:
|
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||||
"""
|
"""
|
||||||
Submit a job to load blocks from this tier to primary tier.
|
Submit a job to load chunks from this tier to primary tier.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
job_metadata: Job metadata including job_id, keys, and
|
job_metadata: Job metadata including job_id, keys, and
|
||||||
spec for writing blocks into the primary tier.
|
spec for writing chunks into the primary tier.
|
||||||
"""
|
"""
|
||||||
keys = job_metadata.keys
|
keys = job_metadata.keys
|
||||||
block_ids = job_metadata.block_ids
|
chunk_ids = job_metadata.chunk_ids
|
||||||
|
|
||||||
assert len(keys) == len(block_ids), (
|
assert len(keys) == len(chunk_ids), (
|
||||||
f"Length mismatch: {len(keys)} keys but {len(block_ids)} block_ids"
|
f"Length mismatch: {len(keys)} keys but {len(chunk_ids)} chunk_ids"
|
||||||
)
|
)
|
||||||
|
|
||||||
for key in keys:
|
for key in keys:
|
||||||
if key not in self.blocks:
|
if key not in self.chunks:
|
||||||
self.completed_jobs.append(
|
self.completed_jobs.append(
|
||||||
JobResult(job_id=job_metadata.job_id, success=False)
|
JobResult(job_id=job_metadata.job_id, success=False)
|
||||||
)
|
)
|
||||||
@@ -170,6 +170,6 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
|||||||
completes, so there is nothing to wait for."""
|
completes, so there is nothing to wait for."""
|
||||||
return
|
return
|
||||||
|
|
||||||
def get_num_blocks(self) -> int:
|
def get_num_chunks(self) -> int:
|
||||||
"""Get the number of blocks currently stored in this tier."""
|
"""Get the number of chunks currently stored in this tier."""
|
||||||
return len(self.blocks)
|
return len(self.chunks)
|
||||||
|
|||||||
@@ -223,7 +223,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
|||||||
batch_store_block,
|
batch_store_block,
|
||||||
[self.file_mapper.get_file_name(key) for key in keys],
|
[self.file_mapper.get_file_name(key) for key in keys],
|
||||||
self._primary_kv_view,
|
self._primary_kv_view,
|
||||||
[int(bid) * self._block_size for bid in job_metadata.block_ids],
|
[int(cid) * self._block_size for cid in job_metadata.chunk_ids],
|
||||||
self._block_size,
|
self._block_size,
|
||||||
self._use_o_direct,
|
self._use_o_direct,
|
||||||
)
|
)
|
||||||
@@ -237,7 +237,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
|||||||
keys = list(job_metadata.keys)
|
keys = list(job_metadata.keys)
|
||||||
self._load_job_keys[job_id] = keys
|
self._load_job_keys[job_id] = keys
|
||||||
paths = [self.file_mapper.get_file_name(key) for key in keys]
|
paths = [self.file_mapper.get_file_name(key) for key in keys]
|
||||||
offsets = [int(bid) * self._block_size for bid in job_metadata.block_ids]
|
offsets = [int(cid) * self._block_size for cid in job_metadata.chunk_ids]
|
||||||
|
|
||||||
def load_task() -> None:
|
def load_task() -> None:
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -8,16 +8,16 @@ and zero or more secondary tiers (Storage, Network, etc.) to provide
|
|||||||
hierarchical KV cache offloading.
|
hierarchical KV cache offloading.
|
||||||
|
|
||||||
Key Design Principles:
|
Key Design Principles:
|
||||||
1. Always offload to all tiers — When a block is stored to the primary tier,
|
1. Always offload to all tiers — When a chunk is stored to the primary tier,
|
||||||
it is cascaded to ALL secondary tiers
|
it is cascaded to ALL secondary tiers
|
||||||
2. Primary tier is the gateway — Secondary tiers cannot access GPU memory
|
2. Primary tier is the gateway — Secondary tiers cannot access GPU memory
|
||||||
directly; all data flows through the CPU primary tier
|
directly; all data flows through the CPU primary tier
|
||||||
3. Staged promotion — Blocks in secondary tiers must be promoted to the
|
3. Staged promotion — Chunks in secondary tiers must be promoted to the
|
||||||
primary tier before GPU can access them
|
primary tier before GPU can access them
|
||||||
4. Transparent retry mechanism — Return None from lookup() to signal
|
4. Transparent retry mechanism — Return None from lookup() to signal
|
||||||
"data is being promoted, try later"
|
"data is being promoted, try later"
|
||||||
5. ref_cnt as eviction protection — primary.prepare_read() increments ref_cnt,
|
5. ref_cnt as eviction protection — primary.prepare_read() increments ref_cnt,
|
||||||
protecting blocks from eviction until complete_read() is called
|
protecting chunks from eviction until complete_read() is called
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import time
|
import time
|
||||||
@@ -61,11 +61,11 @@ logger = init_logger(__name__)
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class PendingPromotion:
|
class PendingPromotion:
|
||||||
"""Accumulator for blocks awaiting submit_load() for one (tier, request)."""
|
"""Accumulator for chunks awaiting submit_load() for one (tier, request)."""
|
||||||
|
|
||||||
req_context: ReqContext
|
req_context: ReqContext
|
||||||
keys: list[OffloadKey] = field(default_factory=list)
|
keys: list[OffloadKey] = field(default_factory=list)
|
||||||
block_ids: list[int] = field(default_factory=list)
|
chunk_ids: list[int] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
@dataclass(slots=True)
|
||||||
@@ -94,14 +94,14 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
|
|||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
num_blocks: int,
|
num_chunks: int,
|
||||||
mmap_region: SharedOffloadRegion,
|
mmap_region: SharedOffloadRegion,
|
||||||
cache_policy: str = "lru",
|
cache_policy: str = "lru",
|
||||||
cache_policy_module_path: str | None = None,
|
cache_policy_module_path: str | None = None,
|
||||||
enable_events: bool = False,
|
enable_events: bool = False,
|
||||||
):
|
):
|
||||||
super().__init__(
|
super().__init__(
|
||||||
num_blocks=num_blocks,
|
num_chunks=num_chunks,
|
||||||
cache_policy=cache_policy,
|
cache_policy=cache_policy,
|
||||||
cache_policy_module_path=cache_policy_module_path,
|
cache_policy_module_path=cache_policy_module_path,
|
||||||
enable_events=enable_events,
|
enable_events=enable_events,
|
||||||
@@ -120,9 +120,9 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
|
|||||||
def get_kv_memoryview(self) -> memoryview:
|
def get_kv_memoryview(self) -> memoryview:
|
||||||
"""Return the memoryview over the primary tier's KV cache buffer.
|
"""Return the memoryview over the primary tier's KV cache buffer.
|
||||||
|
|
||||||
The view has shape (num_blocks, row_stride_bytes) and is backed by the
|
The view has shape (num_chunks, row_stride_bytes) and is backed by the
|
||||||
SharedOffloadRegion mmap. Secondary tiers address block *b* as
|
SharedOffloadRegion mmap. Secondary tiers address chunk *c* as
|
||||||
``view[b]``.
|
``view[c]``.
|
||||||
"""
|
"""
|
||||||
return self._kv_memoryview
|
return self._kv_memoryview
|
||||||
|
|
||||||
@@ -206,14 +206,14 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
assert primary_view.strides is not None
|
assert primary_view.strides is not None
|
||||||
self._metrics = TieringMetricsTracker(
|
self._metrics = TieringMetricsTracker(
|
||||||
tier_types=[tier.tier_type for tier in self.secondary_tiers],
|
tier_types=[tier.tier_type for tier in self.secondary_tiers],
|
||||||
num_primary_blocks=self.primary_tier._num_blocks,
|
num_primary_chunks=self.primary_tier._num_chunks,
|
||||||
primary_block_size=primary_view.strides[0],
|
primary_chunk_size=primary_view.strides[0],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Pending promotion requests accumulated during lookup() calls; flushed
|
# Pending promotion requests accumulated during lookup() calls; flushed
|
||||||
# as one batched submit_load() per (tier, request) in on_schedule_end().
|
# as one batched submit_load() per (tier, request) in on_schedule_end().
|
||||||
# Outer key: tier index. Inner key: req_context.req_id — the same ReqContext
|
# Outer key: tier index. Inner key: req_context.req_id — the same ReqContext
|
||||||
# object is reused for all block lookups of a given request per engine step.
|
# object is reused for all chunk lookups of a given request per engine step.
|
||||||
self._pending_load_submissions: dict[int, dict[str, PendingPromotion]] = {}
|
self._pending_load_submissions: dict[int, dict[str, PendingPromotion]] = {}
|
||||||
|
|
||||||
# Gate for once-per-step execution of _maybe_process_finished_jobs().
|
# Gate for once-per-step execution of _maybe_process_finished_jobs().
|
||||||
@@ -305,7 +305,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
2. For completed stores (primary→secondary): calls primary.complete_read()
|
2. For completed stores (primary→secondary): calls primary.complete_read()
|
||||||
to decrement ref_cnt
|
to decrement ref_cnt
|
||||||
3. For completed loads (secondary→primary): calls primary.complete_write()
|
3. For completed loads (secondary→primary): calls primary.complete_write()
|
||||||
to make blocks available
|
to make chunks available
|
||||||
"""
|
"""
|
||||||
for i, tier in enumerate(self.secondary_tiers):
|
for i, tier in enumerate(self.secondary_tiers):
|
||||||
for completed_job in tier.get_finished_jobs():
|
for completed_job in tier.get_finished_jobs():
|
||||||
@@ -324,11 +324,11 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
|
|
||||||
if transfer_job.is_promotion:
|
if transfer_job.is_promotion:
|
||||||
# secondary→primary transfer (promotion) completed.
|
# secondary→primary transfer (promotion) completed.
|
||||||
# Make blocks available in primary tier.
|
# Make chunks available in primary tier.
|
||||||
self._complete_promotion(job_metadata, completed_job)
|
self._complete_promotion(job_metadata, completed_job)
|
||||||
else:
|
else:
|
||||||
# primary→secondary transfer completed.
|
# primary→secondary transfer completed.
|
||||||
# Decrement ref_cnt on primary blocks.
|
# Decrement ref_cnt on primary chunks.
|
||||||
self.primary_tier.complete_read(
|
self.primary_tier.complete_read(
|
||||||
transfer_job.keys, transfer_job.req_context
|
transfer_job.keys, transfer_job.req_context
|
||||||
)
|
)
|
||||||
@@ -342,7 +342,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
exclude_tier_idx: int | None = None,
|
exclude_tier_idx: int | None = None,
|
||||||
) -> LookupResult:
|
) -> LookupResult:
|
||||||
"""
|
"""
|
||||||
Check whether a single block is offloaded and ready.
|
Check whether a single chunk is offloaded and ready.
|
||||||
|
|
||||||
Algorithm:
|
Algorithm:
|
||||||
1. Process any completed async jobs first.
|
1. Process any completed async jobs first.
|
||||||
@@ -351,20 +351,20 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
hit and initiate promotion.
|
hit and initiate promotion.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
key: Block hash to look up.
|
key: Chunk hash to look up.
|
||||||
req_context: Per-request context.
|
req_context: Per-request context.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
HIT — block is ready in the primary tier.
|
HIT — chunk is ready in the primary tier.
|
||||||
HIT_PENDING — block found but not yet readable (write
|
HIT_PENDING — chunk found but not yet readable (write
|
||||||
in-flight on the primary tier).
|
in-flight on the primary tier).
|
||||||
RETRY — promotion started or a secondary tier is busy.
|
RETRY — promotion started or a secondary tier is busy.
|
||||||
MISS — block not found in any tier, or primary is full
|
MISS — chunk not found in any tier, or primary is full
|
||||||
and cannot accept a promotion.
|
and cannot accept a promotion.
|
||||||
"""
|
"""
|
||||||
# Poll first so a promotion that finished since the last call is
|
# Poll first so a promotion that finished since the last call is
|
||||||
# already reflected as HIT (not stale HIT_PENDING/MISS) below, and
|
# already reflected as HIT (not stale HIT_PENDING/MISS) below, and
|
||||||
# so blocks freed by cascade or promotion completions are evictable
|
# so chunks freed by cascade or promotion completions are evictable
|
||||||
# in time for a promotion this lookup may initiate.
|
# in time for a promotion this lookup may initiate.
|
||||||
self._maybe_process_finished_jobs()
|
self._maybe_process_finished_jobs()
|
||||||
|
|
||||||
@@ -424,30 +424,30 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
req_context: ReqContext,
|
req_context: ReqContext,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""
|
"""
|
||||||
Queue a block for promotion from a secondary tier to the primary tier.
|
Queue a chunk for promotion from a secondary tier to the primary tier.
|
||||||
|
|
||||||
Allocates space in the primary tier immediately (sets ref_cnt=-1 so
|
Allocates space in the primary tier immediately (sets ref_cnt=-1 so
|
||||||
subsequent lookups within the same step see the slot as in-flight),
|
subsequent lookups within the same step see the slot as in-flight),
|
||||||
then defers the actual submit_load() call to _flush_pending_promotions()
|
then defers the actual submit_load() call to _flush_pending_promotions()
|
||||||
so all blocks queued during one engine step are submitted as a single
|
so all chunks queued during one engine step are submitted as a single
|
||||||
batched job.
|
batched job.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
tier_idx: The secondary tier index to promote from
|
tier_idx: The secondary tier index to promote from
|
||||||
key: Block to promote
|
key: Chunk to promote
|
||||||
req_context: Per-request context forwarded to primary.prepare_write().
|
req_context: Per-request context forwarded to primary.prepare_write().
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True if promotion was initiated, False if primary tier is full.
|
True if promotion was initiated, False if primary tier is full.
|
||||||
"""
|
"""
|
||||||
# Allocate space in primary tier for promoted block.
|
# Allocate space in primary tier for promoted chunk.
|
||||||
# Must happen immediately so primary.lookup() returns None (in-flight)
|
# Must happen immediately so primary.lookup() returns None (in-flight)
|
||||||
# for this key on any subsequent lookup() call within the same step,
|
# for this key on any subsequent lookup() call within the same step,
|
||||||
# preventing duplicate promotion attempts.
|
# preventing duplicate promotion attempts.
|
||||||
primary_write_result = self.primary_tier.prepare_write([key], req_context)
|
primary_write_result = self.primary_tier.prepare_write([key], req_context)
|
||||||
|
|
||||||
if primary_write_result is None:
|
if primary_write_result is None:
|
||||||
# Primary tier is full; caller should treat the block as unavailable
|
# Primary tier is full; caller should treat the chunk as unavailable
|
||||||
# rather than retrying indefinitely.
|
# rather than retrying indefinitely.
|
||||||
self._metrics.on_promotion_allocation_failure()
|
self._metrics.on_promotion_allocation_failure()
|
||||||
return False
|
return False
|
||||||
@@ -455,16 +455,16 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
store_spec = primary_write_result.store_spec
|
store_spec = primary_write_result.store_spec
|
||||||
assert isinstance(store_spec, CPULoadStoreSpec)
|
assert isinstance(store_spec, CPULoadStoreSpec)
|
||||||
# Defer submit_load to on_schedule_end(). Group by (tier, request) so
|
# Defer submit_load to on_schedule_end(). Group by (tier, request) so
|
||||||
# each request's blocks are submitted as one batched job per tier.
|
# each request's chunks are submitted as one batched job per tier.
|
||||||
tier_pending = self._pending_load_submissions.setdefault(tier_idx, {})
|
tier_pending = self._pending_load_submissions.setdefault(tier_idx, {})
|
||||||
ctx_id = req_context.req_id
|
ctx_id = req_context.req_id
|
||||||
if ctx_id not in tier_pending:
|
if ctx_id not in tier_pending:
|
||||||
tier_pending[ctx_id] = PendingPromotion(
|
tier_pending[ctx_id] = PendingPromotion(
|
||||||
keys=[], block_ids=[], req_context=req_context
|
keys=[], chunk_ids=[], req_context=req_context
|
||||||
)
|
)
|
||||||
entry = tier_pending[ctx_id]
|
entry = tier_pending[ctx_id]
|
||||||
entry.keys.extend(primary_write_result.keys_to_store)
|
entry.keys.extend(primary_write_result.keys_to_store)
|
||||||
entry.block_ids.extend(store_spec.block_ids)
|
entry.chunk_ids.extend(store_spec.chunk_ids)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def _flush_pending_promotions(self) -> None:
|
def _flush_pending_promotions(self) -> None:
|
||||||
@@ -483,7 +483,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
job_metadata = TransferJob(
|
job_metadata = TransferJob(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
keys=entry.keys,
|
keys=entry.keys,
|
||||||
block_ids=np.array(entry.block_ids, dtype=np.int32),
|
chunk_ids=np.array(entry.chunk_ids, dtype=np.int32),
|
||||||
is_promotion=True,
|
is_promotion=True,
|
||||||
req_context=entry.req_context,
|
req_context=entry.req_context,
|
||||||
)
|
)
|
||||||
@@ -497,16 +497,16 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||||
) -> LoadStoreSpec:
|
) -> LoadStoreSpec:
|
||||||
"""
|
"""
|
||||||
Prepare blocks to be loaded from primary tier to GPU.
|
Prepare chunks to be loaded from primary tier to GPU.
|
||||||
|
|
||||||
Callers only pass keys already confirmed HIT by lookup() earlier this
|
Callers only pass keys already confirmed HIT by lookup() earlier this
|
||||||
step.
|
step.
|
||||||
|
|
||||||
This increments ref_cnt on the blocks in the primary tier, protecting
|
This increments ref_cnt on the chunks in the primary tier, protecting
|
||||||
them from eviction during the transfer.
|
them from eviction during the transfer.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
keys: Blocks to prepare for loading.
|
keys: Chunks to prepare for loading.
|
||||||
req_context: Per-request context.
|
req_context: Per-request context.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
@@ -517,10 +517,10 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
@override
|
@override
|
||||||
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
||||||
"""
|
"""
|
||||||
Mark blocks as recently used in all tiers.
|
Mark chunks as recently used in all tiers.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
keys: Blocks to mark as recently used.
|
keys: Chunks to mark as recently used.
|
||||||
req_context: Per-request context.
|
req_context: Per-request context.
|
||||||
"""
|
"""
|
||||||
self.primary_tier.touch(keys, req_context)
|
self.primary_tier.touch(keys, req_context)
|
||||||
@@ -530,13 +530,13 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
@override
|
@override
|
||||||
def complete_load(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
def complete_load(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
||||||
"""
|
"""
|
||||||
Mark blocks as done loading from primary tier to GPU.
|
Mark chunks as done loading from primary tier to GPU.
|
||||||
|
|
||||||
This decrements ref_cnt on the blocks in the primary tier, allowing
|
This decrements ref_cnt on the chunks in the primary tier, allowing
|
||||||
them to be evicted again.
|
them to be evicted again.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
keys: Blocks that finished loading.
|
keys: Chunks that finished loading.
|
||||||
req_context: Per-request context.
|
req_context: Per-request context.
|
||||||
"""
|
"""
|
||||||
self.primary_tier.complete_load(keys, req_context)
|
self.primary_tier.complete_load(keys, req_context)
|
||||||
@@ -546,37 +546,37 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||||
) -> PrepareStoreOutput | None:
|
) -> PrepareStoreOutput | None:
|
||||||
"""
|
"""
|
||||||
Prepare blocks to be stored from GPU to primary tier.
|
Prepare chunks to be stored from GPU to primary tier.
|
||||||
|
|
||||||
CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure
|
CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure
|
||||||
that any completed async transfers have their ref_cnt decremented
|
that any completed async transfers have their ref_cnt decremented
|
||||||
before the primary tier makes eviction decisions.
|
before the primary tier makes eviction decisions.
|
||||||
|
|
||||||
For request-level tiers, blocks already present in the primary tier
|
For request-level tiers, chunks already present in the primary tier
|
||||||
are immediately cascaded via submit_store().
|
are immediately cascaded via submit_store().
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
keys: Blocks to prepare for storing.
|
keys: Chunks to prepare for storing.
|
||||||
req_context: Per-request context.
|
req_context: Per-request context.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
PrepareStoreOutput describing where to store blocks and what was
|
PrepareStoreOutput describing where to store chunks and what was
|
||||||
evicted, or None if store cannot proceed.
|
evicted, or None if store cannot proceed.
|
||||||
"""
|
"""
|
||||||
# Step 1: Poll for completed async jobs FIRST
|
# Step 1: Poll for completed async jobs FIRST
|
||||||
# _process_finished_jobs() handles two kinds of completions here:
|
# _process_finished_jobs() handles two kinds of completions here:
|
||||||
# - Cascade completions (store to a secondary tier, either a local
|
# - Cascade completions (store to a secondary tier, either a local
|
||||||
# cascade or a store job created for a remote requester via
|
# cascade or a store job created for a remote requester via
|
||||||
# create_store_job()): decrements ref_cnt on the primary blocks
|
# create_store_job()): decrements ref_cnt on the primary chunks
|
||||||
# that were read, making them evictable again once ref_cnt hits 0.
|
# that were read, making them evictable again once ref_cnt hits 0.
|
||||||
# - Promotion completions (secondary->primary loads): sets a
|
# - Promotion completions (secondary->primary loads): sets a
|
||||||
# not-yet-ready block's ref_cnt from -1 to 0 via complete_write(),
|
# not-yet-ready chunk's ref_cnt from -1 to 0 via complete_write(),
|
||||||
# making it evictable for the first time.
|
# making it evictable for the first time.
|
||||||
# Both must be accounted for before the eviction decision below.
|
# Both must be accounted for before the eviction decision below.
|
||||||
self._maybe_process_finished_jobs()
|
self._maybe_process_finished_jobs()
|
||||||
|
|
||||||
# Step 2: Store to primary tier (new blocks only).
|
# Step 2: Store to primary tier (new chunks only).
|
||||||
# Cascading of these newly-stored blocks to ALL secondary tiers
|
# Cascading of these newly-stored chunks to ALL secondary tiers
|
||||||
# happens later in complete_store(), after the GPU→Primary transfer
|
# happens later in complete_store(), after the GPU→Primary transfer
|
||||||
# completes.
|
# completes.
|
||||||
primary_result = self.primary_tier.prepare_store(keys, req_context)
|
primary_result = self.primary_tier.prepare_store(keys, req_context)
|
||||||
@@ -588,7 +588,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
state = self._req_state[req_context.req_id]
|
state = self._req_state[req_context.req_id]
|
||||||
state.pending_primary_stores += 1
|
state.pending_primary_stores += 1
|
||||||
|
|
||||||
# Step 3: For request-level tiers, cascade blocks already in primary
|
# Step 3: For request-level tiers, cascade chunks already in primary
|
||||||
request_level_tiers = self._req_state[req_context.req_id].request_level_tiers
|
request_level_tiers = self._req_state[req_context.req_id].request_level_tiers
|
||||||
if request_level_tiers:
|
if request_level_tiers:
|
||||||
keys_to_store_set = set(primary_result.keys_to_store)
|
keys_to_store_set = set(primary_result.keys_to_store)
|
||||||
@@ -596,13 +596,13 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
k for k in keys if k not in keys_to_store_set
|
k for k in keys if k not in keys_to_store_set
|
||||||
)
|
)
|
||||||
if keys_already_in_primary:
|
if keys_already_in_primary:
|
||||||
self._cascade_existing_blocks_to_request_level_tiers(
|
self._cascade_existing_chunks_to_request_level_tiers(
|
||||||
keys_already_in_primary, req_context, request_level_tiers
|
keys_already_in_primary, req_context, request_level_tiers
|
||||||
)
|
)
|
||||||
|
|
||||||
return primary_result
|
return primary_result
|
||||||
|
|
||||||
def _cascade_existing_blocks_to_request_level_tiers(
|
def _cascade_existing_chunks_to_request_level_tiers(
|
||||||
self,
|
self,
|
||||||
keys: Sequence[OffloadKey],
|
keys: Sequence[OffloadKey],
|
||||||
req_context: ReqContext,
|
req_context: ReqContext,
|
||||||
@@ -610,7 +610,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
For tiers that requested request-level policy, submit_store() for
|
For tiers that requested request-level policy, submit_store() for
|
||||||
blocks that are already present in the primary tier.
|
chunks that are already present in the primary tier.
|
||||||
|
|
||||||
A key whose primary write is still in flight (HIT_PENDING) cannot be
|
A key whose primary write is still in flight (HIT_PENDING) cannot be
|
||||||
dropped: prepare_store already excluded it as present, and the
|
dropped: prepare_store already excluded it as present, and the
|
||||||
@@ -653,7 +653,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
continue
|
continue
|
||||||
assert state.request_level_tiers
|
assert state.request_level_tiers
|
||||||
keys, state.pending_cascade_keys = state.pending_cascade_keys, []
|
keys, state.pending_cascade_keys = state.pending_cascade_keys, []
|
||||||
self._cascade_existing_blocks_to_request_level_tiers(
|
self._cascade_existing_chunks_to_request_level_tiers(
|
||||||
keys, state.req_context, state.request_level_tiers
|
keys, state.req_context, state.request_level_tiers
|
||||||
)
|
)
|
||||||
self._maybe_finalize_request(req_id)
|
self._maybe_finalize_request(req_id)
|
||||||
@@ -666,30 +666,30 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
success: bool = True,
|
success: bool = True,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Mark blocks as done storing from GPU to primary tier.
|
Mark chunks as done storing from GPU to primary tier.
|
||||||
|
|
||||||
This is where secondary tier cascading happens — after blocks are
|
This is where secondary tier cascading happens — after chunks are
|
||||||
confirmed to be in the primary tier, they are cascaded to ALL
|
confirmed to be in the primary tier, they are cascaded to ALL
|
||||||
secondary tiers.
|
secondary tiers.
|
||||||
|
|
||||||
For each secondary tier:
|
For each secondary tier:
|
||||||
1. Call primary.prepare_read() to get LoadStoreSpec AND increment
|
1. Call primary.prepare_read() to get LoadStoreSpec AND increment
|
||||||
ref_cnt (protecting blocks during async transfer)
|
ref_cnt (protecting chunks during async transfer)
|
||||||
2. Call tier.submit_store() to start async transfer: primary→secondary
|
2. Call tier.submit_store() to start async transfer: primary→secondary
|
||||||
3. Track the job in _store_jobs dictionary
|
3. Track the job in _store_jobs dictionary
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
keys: Blocks that finished storing.
|
keys: Chunks that finished storing.
|
||||||
success: Whether the GPU→primary transfer succeeded.
|
success: Whether the GPU→primary transfer succeeded.
|
||||||
req_context: Per-request context forwarded to primary.prepare_read().
|
req_context: Per-request context forwarded to primary.prepare_read().
|
||||||
"""
|
"""
|
||||||
# Step 1: Complete store in primary tier (makes blocks loadable)
|
# Step 1: Complete store in primary tier (makes chunks loadable)
|
||||||
self.primary_tier.complete_store(keys, req_context, success)
|
self.primary_tier.complete_store(keys, req_context, success)
|
||||||
|
|
||||||
if success:
|
if success:
|
||||||
# Step 2: Cascade to ALL secondary tiers
|
# Step 2: Cascade to ALL secondary tiers
|
||||||
# For each secondary tier, call primary.prepare_read() to get the
|
# For each secondary tier, call primary.prepare_read() to get the
|
||||||
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from
|
# LoadStoreSpec AND to increment ref_cnt (protecting chunks from
|
||||||
# eviction during the async transfer). One prepare_read() call per
|
# eviction during the async transfer). One prepare_read() call per
|
||||||
# secondary tier.
|
# secondary tier.
|
||||||
for tier_idx, tier in enumerate(self.secondary_tiers):
|
for tier_idx, tier in enumerate(self.secondary_tiers):
|
||||||
@@ -710,22 +710,22 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
req_context: ReqContext,
|
req_context: ReqContext,
|
||||||
tier_idx: int = 0,
|
tier_idx: int = 0,
|
||||||
) -> TransferJob:
|
) -> TransferJob:
|
||||||
"""Pin blocks in the primary tier and create a tracked store job.
|
"""Pin chunks in the primary tier and create a tracked store job.
|
||||||
|
|
||||||
Calls prepare_read() to increment ref_cnt (protecting blocks
|
Calls prepare_read() to increment ref_cnt (protecting chunks
|
||||||
from eviction during the async transfer), allocates a job ID,
|
from eviction during the async transfer), allocates a job ID,
|
||||||
and registers the job in _jobs.
|
and registers the job in _jobs.
|
||||||
|
|
||||||
The caller is responsible for the actual data transfer and
|
The caller is responsible for the actual data transfer and
|
||||||
reporting completion via get_finished_jobs().
|
reporting completion via get_finished_jobs().
|
||||||
"""
|
"""
|
||||||
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
|
primary_chunks_spec = self.primary_tier.prepare_read(keys, req_context)
|
||||||
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
|
assert isinstance(primary_chunks_spec, CPULoadStoreSpec)
|
||||||
job_id = self._next_job_id()
|
job_id = self._next_job_id()
|
||||||
job_metadata = TransferJob(
|
job_metadata = TransferJob(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
keys=keys,
|
keys=keys,
|
||||||
block_ids=primary_blocks_spec.block_ids,
|
chunk_ids=primary_chunks_spec.chunk_ids,
|
||||||
is_promotion=False,
|
is_promotion=False,
|
||||||
req_context=req_context,
|
req_context=req_context,
|
||||||
)
|
)
|
||||||
@@ -760,7 +760,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
policy = (
|
policy = (
|
||||||
OffloadPolicy.REQUEST_LEVEL
|
OffloadPolicy.REQUEST_LEVEL
|
||||||
if state.request_level_tiers
|
if state.request_level_tiers
|
||||||
else OffloadPolicy.BLOCK_LEVEL
|
else OffloadPolicy.CHUNK_LEVEL
|
||||||
)
|
)
|
||||||
return RequestOffloadingContext(policy=policy)
|
return RequestOffloadingContext(policy=policy)
|
||||||
|
|
||||||
|
|||||||
@@ -39,20 +39,20 @@ class _RequestMetricsState:
|
|||||||
class _TierState:
|
class _TierState:
|
||||||
active_promotion_count: int = 0
|
active_promotion_count: int = 0
|
||||||
active_cascade_count: int = 0
|
active_cascade_count: int = 0
|
||||||
primary_write_block_count: int = 0
|
primary_write_chunk_count: int = 0
|
||||||
primary_read_block_count: int = 0
|
primary_read_chunk_count: int = 0
|
||||||
|
|
||||||
|
|
||||||
class TieringMetricsTracker:
|
class TieringMetricsTracker:
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
tier_types: list[str],
|
tier_types: list[str],
|
||||||
num_primary_blocks: int,
|
num_primary_chunks: int,
|
||||||
primary_block_size: int,
|
primary_chunk_size: int,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._tier_types = tier_types
|
self._tier_types = tier_types
|
||||||
self._num_primary_blocks = num_primary_blocks
|
self._num_primary_chunks = num_primary_chunks
|
||||||
self._primary_block_size = primary_block_size
|
self._primary_chunk_size = primary_chunk_size
|
||||||
self._request_states: dict[str, _RequestMetricsState] = {}
|
self._request_states: dict[str, _RequestMetricsState] = {}
|
||||||
self._tier_states = [_TierState() for _ in tier_types]
|
self._tier_states = [_TierState() for _ in tier_types]
|
||||||
self._stats = OffloadingConnectorStats()
|
self._stats = OffloadingConnectorStats()
|
||||||
@@ -111,13 +111,13 @@ class TieringMetricsTracker:
|
|||||||
def on_job_registered(self, job_metadata: _JobMetadataLike) -> None:
|
def on_job_registered(self, job_metadata: _JobMetadataLike) -> None:
|
||||||
transfer_job = job_metadata.transfer_job
|
transfer_job = job_metadata.transfer_job
|
||||||
state = self._tier_states[job_metadata.tier_idx]
|
state = self._tier_states[job_metadata.tier_idx]
|
||||||
block_count = len(transfer_job.block_ids)
|
chunk_count = len(transfer_job.chunk_ids)
|
||||||
if transfer_job.is_promotion:
|
if transfer_job.is_promotion:
|
||||||
state.active_promotion_count += 1
|
state.active_promotion_count += 1
|
||||||
state.primary_write_block_count += block_count
|
state.primary_write_chunk_count += chunk_count
|
||||||
else:
|
else:
|
||||||
state.active_cascade_count += 1
|
state.active_cascade_count += 1
|
||||||
state.primary_read_block_count += block_count
|
state.primary_read_chunk_count += chunk_count
|
||||||
|
|
||||||
def on_job_finished(
|
def on_job_finished(
|
||||||
self, job_metadata: _JobMetadataLike, result: JobResult
|
self, job_metadata: _JobMetadataLike, result: JobResult
|
||||||
@@ -149,25 +149,25 @@ class TieringMetricsTracker:
|
|||||||
assert all(
|
assert all(
|
||||||
state.active_promotion_count == 0
|
state.active_promotion_count == 0
|
||||||
and state.active_cascade_count == 0
|
and state.active_cascade_count == 0
|
||||||
and state.primary_write_block_count == 0
|
and state.primary_write_chunk_count == 0
|
||||||
and state.primary_read_block_count == 0
|
and state.primary_read_chunk_count == 0
|
||||||
for state in self._tier_states
|
for state in self._tier_states
|
||||||
)
|
)
|
||||||
|
|
||||||
def _decrement_tier_state(self, job_metadata: _JobMetadataLike) -> None:
|
def _decrement_tier_state(self, job_metadata: _JobMetadataLike) -> None:
|
||||||
transfer_job = job_metadata.transfer_job
|
transfer_job = job_metadata.transfer_job
|
||||||
state = self._tier_states[job_metadata.tier_idx]
|
state = self._tier_states[job_metadata.tier_idx]
|
||||||
block_count = len(transfer_job.block_ids)
|
chunk_count = len(transfer_job.chunk_ids)
|
||||||
if transfer_job.is_promotion:
|
if transfer_job.is_promotion:
|
||||||
assert state.active_promotion_count > 0
|
assert state.active_promotion_count > 0
|
||||||
state.active_promotion_count -= 1
|
state.active_promotion_count -= 1
|
||||||
state.primary_write_block_count -= block_count
|
state.primary_write_chunk_count -= chunk_count
|
||||||
assert state.primary_write_block_count >= 0
|
assert state.primary_write_chunk_count >= 0
|
||||||
else:
|
else:
|
||||||
assert state.active_cascade_count > 0
|
assert state.active_cascade_count > 0
|
||||||
state.active_cascade_count -= 1
|
state.active_cascade_count -= 1
|
||||||
state.primary_read_block_count -= block_count
|
state.primary_read_chunk_count -= chunk_count
|
||||||
assert state.primary_read_block_count >= 0
|
assert state.primary_read_chunk_count >= 0
|
||||||
|
|
||||||
def _observe_finished_job_stats(
|
def _observe_finished_job_stats(
|
||||||
self,
|
self,
|
||||||
@@ -199,7 +199,7 @@ class TieringMetricsTracker:
|
|||||||
if transfer_job.is_promotion
|
if transfer_job.is_promotion
|
||||||
else TieringOffloadingMetrics.WRITE_TIME
|
else TieringOffloadingMetrics.WRITE_TIME
|
||||||
)
|
)
|
||||||
transfer_size = completed_key_count * self._primary_block_size
|
transfer_size = completed_key_count * self._primary_chunk_size
|
||||||
self._stats.increase_counter(bytes_metric, transfer_size, labelvalues)
|
self._stats.increase_counter(bytes_metric, transfer_size, labelvalues)
|
||||||
if completed_job.transfer_time is not None:
|
if completed_job.transfer_time is not None:
|
||||||
self._stats.increase_counter(
|
self._stats.increase_counter(
|
||||||
@@ -210,13 +210,13 @@ class TieringMetricsTracker:
|
|||||||
for tier_idx, state in enumerate(self._tier_states):
|
for tier_idx, state in enumerate(self._tier_states):
|
||||||
labelvalues = self.tier_label(tier_idx)
|
labelvalues = self.tier_label(tier_idx)
|
||||||
write_usage = (
|
write_usage = (
|
||||||
state.primary_write_block_count / self._num_primary_blocks
|
state.primary_write_chunk_count / self._num_primary_chunks
|
||||||
if self._num_primary_blocks > 0
|
if self._num_primary_chunks > 0
|
||||||
else 0.0
|
else 0.0
|
||||||
)
|
)
|
||||||
read_usage = (
|
read_usage = (
|
||||||
state.primary_read_block_count / self._num_primary_blocks
|
state.primary_read_chunk_count / self._num_primary_chunks
|
||||||
if self._num_primary_blocks > 0
|
if self._num_primary_chunks > 0
|
||||||
else 0.0
|
else 0.0
|
||||||
)
|
)
|
||||||
stats.set_gauge(
|
stats.set_gauge(
|
||||||
@@ -248,12 +248,12 @@ class TieringMetricsTracker:
|
|||||||
async_start_time: float | None,
|
async_start_time: float | None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._stats.increase_counter(
|
self._stats.increase_counter(
|
||||||
TieringOffloadingMetrics.BLOCK_QUERIES,
|
TieringOffloadingMetrics.CHUNK_QUERIES,
|
||||||
labelvalues=tier_label,
|
labelvalues=tier_label,
|
||||||
)
|
)
|
||||||
if result is LookupResult.HIT:
|
if result is LookupResult.HIT:
|
||||||
self._stats.increase_counter(
|
self._stats.increase_counter(
|
||||||
TieringOffloadingMetrics.BLOCK_HITS,
|
TieringOffloadingMetrics.CHUNK_HITS,
|
||||||
labelvalues=tier_label,
|
labelvalues=tier_label,
|
||||||
)
|
)
|
||||||
self._stats.observe_histogram(
|
self._stats.observe_histogram(
|
||||||
|
|||||||
@@ -287,14 +287,14 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
|||||||
self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
||||||
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
|
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
|
||||||
self._submit_transfer(
|
self._submit_transfer(
|
||||||
job_metadata.job_id, job_metadata.block_ids, obj_keys, NIXL_WRITE
|
job_metadata.job_id, job_metadata.chunk_ids, obj_keys, NIXL_WRITE
|
||||||
)
|
)
|
||||||
|
|
||||||
def submit_load(self, job_metadata: TransferJob) -> None:
|
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||||
self._load_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
self._load_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
||||||
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
|
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
|
||||||
self._submit_transfer(
|
self._submit_transfer(
|
||||||
job_metadata.job_id, job_metadata.block_ids, obj_keys, NIXL_READ
|
job_metadata.job_id, job_metadata.chunk_ids, obj_keys, NIXL_READ
|
||||||
)
|
)
|
||||||
|
|
||||||
def on_request_finished(self, req_context: ReqContext) -> None:
|
def on_request_finished(self, req_context: ReqContext) -> None:
|
||||||
|
|||||||
@@ -417,7 +417,7 @@ class P2PSecondaryTierManager(SecondaryTierManager):
|
|||||||
def submit_store(self, job_metadata: TransferJob) -> None:
|
def submit_store(self, job_metadata: TransferJob) -> None:
|
||||||
job_id = job_metadata.job_id
|
job_id = job_metadata.job_id
|
||||||
keys = list(job_metadata.keys)
|
keys = list(job_metadata.keys)
|
||||||
block_ids = job_metadata.block_ids.tolist()
|
block_ids = job_metadata.chunk_ids.tolist()
|
||||||
|
|
||||||
assert len(keys) == len(block_ids)
|
assert len(keys) == len(block_ids)
|
||||||
|
|
||||||
@@ -476,7 +476,7 @@ class P2PSecondaryTierManager(SecondaryTierManager):
|
|||||||
def submit_load(self, job_metadata: TransferJob) -> None:
|
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||||
job_id = job_metadata.job_id
|
job_id = job_metadata.job_id
|
||||||
keys = list(job_metadata.keys)
|
keys = list(job_metadata.keys)
|
||||||
block_ids = job_metadata.block_ids
|
block_ids = job_metadata.chunk_ids
|
||||||
|
|
||||||
source = job_metadata.req_context.get_state(P2PSourceInfo)
|
source = job_metadata.req_context.get_state(P2PSourceInfo)
|
||||||
logger.debug(
|
logger.debug(
|
||||||
|
|||||||
@@ -593,7 +593,7 @@ class ServerRole:
|
|||||||
self.add_stored_blocks(
|
self.add_stored_blocks(
|
||||||
lookup.kv_request_id,
|
lookup.kv_request_id,
|
||||||
list(meta.keys),
|
list(meta.keys),
|
||||||
meta.block_ids.tolist(),
|
meta.chunk_ids.tolist(),
|
||||||
meta.job_id,
|
meta.job_id,
|
||||||
round_seq=lookup.round_seq,
|
round_seq=lookup.round_seq,
|
||||||
from_lookup=True,
|
from_lookup=True,
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ and configurable secondary tiers (e.g., Storage, Network).
|
|||||||
|
|
||||||
Configuration via kv_connector_extra_config:
|
Configuration via kv_connector_extra_config:
|
||||||
- cpu_bytes_to_use: (required) Bytes to allocate for CPU primary tier
|
- cpu_bytes_to_use: (required) Bytes to allocate for CPU primary tier
|
||||||
- block_size: (optional) Block size for offloaded blocks (default: GPU block size)
|
- block_size: (optional) Tokens per offloaded chunk (default: GPU block size)
|
||||||
- eviction_policy: (optional) Primary tier eviction policy: built-in "lru"/
|
- eviction_policy: (optional) Primary tier eviction policy: built-in "lru"/
|
||||||
"arc", or the name of a policy registered via CachePolicyFactory, or an
|
"arc", or the name of a policy registered via CachePolicyFactory, or an
|
||||||
out-of-tree CachePolicy class name paired with cache_policy_module_path
|
out-of-tree CachePolicy class name paired with cache_policy_module_path
|
||||||
@@ -105,7 +105,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
|||||||
metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY] = (
|
metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY] = (
|
||||||
OffloadingHistogramMetadata(
|
OffloadingHistogramMetadata(
|
||||||
documentation=(
|
documentation=(
|
||||||
"Histogram of blocking time spent in a per-block tier lookup "
|
"Histogram of blocking time spent in a per-chunk tier lookup "
|
||||||
"that resolved as a hit or miss, labeled by tier, in seconds."
|
"that resolved as a hit or miss, labeled by tier, in seconds."
|
||||||
),
|
),
|
||||||
labelnames=("tier",),
|
labelnames=("tier",),
|
||||||
@@ -127,7 +127,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
|||||||
metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY] = (
|
metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY] = (
|
||||||
OffloadingHistogramMetadata(
|
OffloadingHistogramMetadata(
|
||||||
documentation=(
|
documentation=(
|
||||||
"Histogram of wall-clock time from a per-block tier lookup "
|
"Histogram of wall-clock time from a per-chunk tier lookup "
|
||||||
"first returning retry until that same tier lookup resolves "
|
"first returning retry until that same tier lookup resolves "
|
||||||
"as a hit or miss, labeled by tier, in seconds."
|
"as a hit or miss, labeled by tier, in seconds."
|
||||||
),
|
),
|
||||||
@@ -191,14 +191,14 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
|||||||
labelnames=("tier",),
|
labelnames=("tier",),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
metrics[TieringOffloadingMetrics.BLOCK_QUERIES] = OffloadingCounterMetadata(
|
metrics[TieringOffloadingMetrics.CHUNK_QUERIES] = OffloadingCounterMetadata(
|
||||||
documentation=(
|
documentation=(
|
||||||
"Number of block lookup queries sent to a tier, labeled by tier."
|
"Number of chunk lookup queries sent to a tier, labeled by tier."
|
||||||
),
|
),
|
||||||
labelnames=("tier",),
|
labelnames=("tier",),
|
||||||
)
|
)
|
||||||
metrics[TieringOffloadingMetrics.BLOCK_HITS] = OffloadingCounterMetadata(
|
metrics[TieringOffloadingMetrics.CHUNK_HITS] = OffloadingCounterMetadata(
|
||||||
documentation="Number of block lookup hits in a tier, labeled by tier.",
|
documentation="Number of chunk lookup hits in a tier, labeled by tier.",
|
||||||
labelnames=("tier",),
|
labelnames=("tier",),
|
||||||
)
|
)
|
||||||
metrics[TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES] = (
|
metrics[TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES] = (
|
||||||
@@ -300,16 +300,16 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
|||||||
# primary tier can eagerly create a memoryview over _base.
|
# primary tier can eagerly create a memoryview over _base.
|
||||||
scheduler_mmap = SharedOffloadRegion(
|
scheduler_mmap = SharedOffloadRegion(
|
||||||
engine_id=self._engine_id,
|
engine_id=self._engine_id,
|
||||||
num_blocks=self.num_blocks,
|
num_chunks=self.num_chunks,
|
||||||
rank=None,
|
rank=None,
|
||||||
kv_bytes_per_block=self.kv_bytes_per_chunk,
|
kv_bytes_per_chunk=self.kv_bytes_per_chunk,
|
||||||
cpu_page_size=self.cpu_page_size_per_worker,
|
cpu_page_size=self.cpu_page_size_per_worker,
|
||||||
)
|
)
|
||||||
self._scheduler_mmap = scheduler_mmap
|
self._scheduler_mmap = scheduler_mmap
|
||||||
|
|
||||||
# Create primary tier (CPU-based)
|
# Create primary tier (CPU-based)
|
||||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||||
num_blocks=self.num_blocks,
|
num_chunks=self.num_chunks,
|
||||||
cache_policy=self.eviction_policy,
|
cache_policy=self.eviction_policy,
|
||||||
cache_policy_module_path=self.cache_policy_module_path,
|
cache_policy_module_path=self.cache_policy_module_path,
|
||||||
enable_events=self.kv_events_config.enable_kv_cache_events,
|
enable_events=self.kv_events_config.enable_kv_cache_events,
|
||||||
@@ -367,9 +367,9 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
|||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"Created TieringOffloadingManager with primary tier "
|
"Created TieringOffloadingManager with primary tier "
|
||||||
"(%s, %s blocks) and %s secondary tier(s)",
|
"(%s, %s chunks) and %s secondary tier(s)",
|
||||||
self.eviction_policy,
|
self.eviction_policy,
|
||||||
self.num_blocks,
|
self.num_chunks,
|
||||||
len(secondary_tiers),
|
len(secondary_tiers),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -393,9 +393,9 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
|||||||
rank = torch.accelerator.current_device_index() % world_size
|
rank = torch.accelerator.current_device_index() % world_size
|
||||||
worker_mmap = SharedOffloadRegion(
|
worker_mmap = SharedOffloadRegion(
|
||||||
engine_id=self._engine_id,
|
engine_id=self._engine_id,
|
||||||
num_blocks=self.num_blocks,
|
num_chunks=self.num_chunks,
|
||||||
rank=rank,
|
rank=rank,
|
||||||
kv_bytes_per_block=self.kv_bytes_per_chunk,
|
kv_bytes_per_chunk=self.kv_bytes_per_chunk,
|
||||||
cpu_page_size=self.cpu_page_size_per_worker,
|
cpu_page_size=self.cpu_page_size_per_worker,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
@@ -404,7 +404,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
|||||||
return CPUOffloadingWorker(
|
return CPUOffloadingWorker(
|
||||||
kv_caches=kv_caches,
|
kv_caches=kv_caches,
|
||||||
blocks_per_chunk=self.blocks_per_chunk,
|
blocks_per_chunk=self.blocks_per_chunk,
|
||||||
num_cpu_blocks=self.num_blocks,
|
num_cpu_chunks=self.num_chunks,
|
||||||
mmap_region=worker_mmap,
|
mmap_region=worker_mmap,
|
||||||
canonical_layout=self.config.canonical_layout,
|
canonical_layout=self.config.canonical_layout,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user