[kv_offload] Add multi-tier KV cache offloading framework (#40020)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
Ronen Schaffer
2026-05-13 17:21:43 +03:00
committed by GitHub
parent a8887c208f
commit 11f6b545d4
14 changed files with 1841 additions and 11 deletions
@@ -292,3 +292,48 @@ def test_cpu_offloading(
if subscriber is not None:
subscriber.close()
del llm
def test_tiering_offloading() -> None:
"""Tests OffloadingConnector with TieringOffloadingSpec."""
extra_config: dict = {
"cpu_bytes_to_use": 500 << 20,
"block_size": 48,
"spec_name": "TieringOffloadingSpec",
"secondary_tiers": [{"type": "example"}],
}
kv_transfer_config = KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
kv_connector_extra_config=extra_config,
)
port: int
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("0.0.0.0", 0))
port = s.getsockname()[1]
events_endpoint = f"tcp://*:{port}"
kv_events_config = KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
endpoint=events_endpoint,
topic="test",
)
llm = LLM(
model="meta-llama/Llama-3.2-1B-Instruct",
max_model_len=4096,
gpu_memory_utilization=0.5,
kv_events_config=kv_events_config,
kv_transfer_config=kv_transfer_config,
)
subscriber = MockSubscriber(
events_endpoint.replace("*", "127.0.0.1"),
topic=kv_events_config.topic,
)
try:
_latency_test(llm, subscriber)
_accuracy_test(llm, subscriber)
finally:
subscriber.close()
del llm
+37
View File
@@ -262,6 +262,43 @@ def test_cpu_manager():
)
def test_prepare_load_preserves_key_order():
"""block_ids[i] must correspond to keys[i] (co-indexed invariant)."""
manager = CPUOffloadingManager(num_blocks=4, cache_policy="lru")
key_a, key_b, key_c = to_key(0), to_key(1), to_key(2)
# Store all three keys and learn their block ID assignments
store_output = manager.prepare_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
assert store_output is not None
assert isinstance(store_output.store_spec, CPULoadStoreSpec)
key_to_block_id = {
k: int(bid)
for k, bid in zip(store_output.keys_to_store, store_output.store_spec.block_ids)
}
manager.complete_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
# Forward order: [a, b, c]
spec_fwd = manager.prepare_load([key_a, key_b, key_c], _EMPTY_REQ_CTX)
assert isinstance(spec_fwd, CPULoadStoreSpec)
assert [int(x) for x in spec_fwd.block_ids] == [
key_to_block_id[key_a],
key_to_block_id[key_b],
key_to_block_id[key_c],
]
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
# Arbitrary permutation: [b, c, a]
spec_perm = manager.prepare_load([key_b, key_c, key_a], _EMPTY_REQ_CTX)
assert isinstance(spec_perm, CPULoadStoreSpec)
assert [int(x) for x in spec_perm.block_ids] == [
key_to_block_id[key_b],
key_to_block_id[key_c],
key_to_block_id[key_a],
]
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
class TestARCPolicy:
"""Unit tests for CPUOffloadingManager with ARC eviction policy."""
@@ -0,0 +1,565 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for TieringOffloadingManager and ExampleSecondaryTier.
These tests verify:
1. Basic tiered offloading operations (store, load, lookup)
2. Cascade behavior (blocks stored to all secondary tiers)
3. Promotion behavior (blocks loaded from secondary to primary to GPU)
4. ref_cnt management (blocks protected during async transfers)
5. Eviction coordination between tiers
"""
from collections.abc import Iterable
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
from vllm.v1.kv_offload.base import (
OffloadKey,
ReqContext,
make_offload_key,
)
from vllm.v1.kv_offload.tiering.base import JobMetadata
from vllm.v1.kv_offload.tiering.example import ExampleSecondaryTier
from vllm.v1.kv_offload.tiering.manager import (
CPUPrimaryTierOffloadingManager,
TieringOffloadingManager,
)
_CTX = ReqContext(req_id="test")
_MOCK_VLLM_CONFIG = MagicMock()
def _mock_mmap_region(num_blocks: int, row_bytes: int = 16):
"""Create a mock SharedOffloadRegion for testing."""
mock = MagicMock()
view = memoryview(torch.zeros((num_blocks, row_bytes), dtype=torch.int8).numpy())
mock.create_kv_memoryview.return_value = view
return mock
def to_keys(int_ids: Iterable[int]) -> list[OffloadKey]:
return [make_offload_key(str(i).encode(), 0) for i in int_ids]
def count_hits(manager, keys: list[OffloadKey]) -> int | None:
"""Count consecutive lookup hits from the start of keys.
Returns the count of leading True results, or None if any lookup
returns None (retry-later signal).
"""
count = 0
for key in keys:
result = manager.lookup(key, _CTX)
if result is None:
return None
if not result:
break
count += 1
return count
class TestExampleSecondaryTier:
"""Tests for ExampleSecondaryTier implementation."""
def test_basic_store_and_lookup(self):
"""Test basic store and lookup operations."""
mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy())
tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
)
# Initially empty
blocks = to_keys(range(3))
assert tier.lookup(blocks[0], _CTX) is False
# Store blocks (simulate with direct insertion for testing)
tier.blocks[blocks[0]] = True
tier.blocks[blocks[1]] = True
# Lookup should find first two blocks
assert tier.lookup(blocks[0], _CTX) is True
assert tier.lookup(blocks[1], _CTX) is True
# Third block not present
assert tier.lookup(blocks[2], _CTX) is False
def test_lru_eviction(self):
"""Test LRU eviction policy."""
mock_view = memoryview(torch.zeros((4, 16), dtype=torch.int8).numpy())
tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=3
)
# Fill tier to capacity
blocks = to_keys(range(3))
for block in blocks:
tier.blocks[block] = True
assert tier.get_num_blocks() == 3
# Touch first block (make it most recently used)
tier.touch([blocks[0]], _CTX)
# Store new block should evict blocks[1] (least recently used)
new_block = to_keys([3])[0]
tier.submit_store(
JobMetadata(
job_id=1,
keys=[new_block],
block_ids=np.array([0], dtype=np.int64),
is_promotion=False,
req_context=_CTX,
)
)
# Complete the job
tier.get_finished()
# Verify new block is stored and blocks[1] was evicted (LRU)
assert new_block in tier.blocks
assert blocks[1] not in tier.blocks
# blocks[0] and blocks[2] should still be present
assert blocks[0] in tier.blocks
assert blocks[2] in tier.blocks
def test_async_simulation(self):
"""Test simulated async behavior."""
mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy())
tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG,
primary_kv_view=mock_view,
max_blocks=10,
simulate_async=True,
)
blocks = to_keys(range(2))
# Submit store job
tier.submit_store(
JobMetadata(
job_id=1,
keys=blocks,
block_ids=np.array([0, 1], dtype=np.int64),
is_promotion=False,
req_context=_CTX,
)
)
# Blocks should not yet be stored (pending async completion)
assert tier.get_num_blocks() == 0
# First get_finished() should complete the job
completed = list(tier.get_finished())
assert len(completed) == 1
assert completed[0].job_id == 1
assert completed[0].success is True
# Blocks should now be stored
assert tier.get_num_blocks() == 2
class TestTieringOffloadingManager:
"""Tests for TieringOffloadingManager."""
@pytest.fixture
def manager_setup(self):
# Create primary tier (CPU-based)
mock_region = _mock_mmap_region(5)
self.primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=5, mmap_region=mock_region
)
mock_view = mock_region.create_kv_memoryview()
# Create secondary tiers with the primary view
self.secondary_tier1 = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
)
self.secondary_tier2 = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
)
# Create tiered manager
self.manager = TieringOffloadingManager(
primary_tier=self.primary_tier,
secondary_tiers=[self.secondary_tier1, self.secondary_tier2],
)
def test_basic_store_to_primary(self, manager_setup):
"""Test basic store operation to primary tier."""
blocks = to_keys(range(3))
# Prepare store
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
assert len(result.keys_to_store) == 3
# Complete store
self.manager.complete_store(blocks, _CTX, success=True)
# Blocks should be in primary tier
assert count_hits(self.primary_tier, blocks) == 3
def test_cascade_to_all_secondary_tiers(self, manager_setup):
"""Test that blocks are cascaded to ALL secondary tiers."""
blocks = to_keys(range(3))
self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store
)
self.secondary_tier2.submit_store = MagicMock(
wraps=self.secondary_tier2.submit_store
)
# Store to primary
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
# Complete store (triggers cascade via submit_store on each tier)
self.manager.complete_store(blocks, _CTX, success=True)
# submit_store was called once per secondary tier
self.secondary_tier1.submit_store.assert_called_once()
self.secondary_tier2.submit_store.assert_called_once()
# Blocks should be in both secondary tiers
assert self.secondary_tier1.get_num_blocks() == 3
assert self.secondary_tier2.get_num_blocks() == 3
# Verify blocks are present
assert all(self.secondary_tier1.lookup(b, _CTX) for b in blocks)
assert all(self.secondary_tier2.lookup(b, _CTX) for b in blocks)
def test_ref_cnt_protection_during_cascade(self, manager_setup):
"""Test that ref_cnt protects blocks during cascade."""
blocks = to_keys(range(3))
# Store to primary
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
self.manager.complete_store(blocks, _CTX, success=True)
# After complete_store, blocks should have ref_cnt > 0
# (one for each secondary tier)
for block_hash in blocks:
block = self.primary_tier._policy.get(block_hash)
# ref_cnt should be 2 (one for each secondary tier)
assert block.ref_cnt == 2
# End of step 1: _maybe_process_finished_jobs() was already called by
# prepare_store() above (setting the per-step flag), so take_events()
# does NOT poll get_finished() again — cascade completions remain
# unprocessed until the next step.
list(self.manager.take_events())
# ref_cnt still held: cascade jobs finished (sync tier) but haven't
# been polled yet because the per-step guard skipped the second call.
for block_hash in blocks:
block = self.primary_tier._policy.get(block_hash)
assert block.ref_cnt == 2
# Secondary tiers have completed jobs waiting to be drained
assert len(self.secondary_tier1.completed_jobs) > 0
assert len(self.secondary_tier2.completed_jobs) > 0
# End of step 2: flag was reset, so _maybe_process_finished_jobs()
# runs and processes the cascade completions (complete_read → ref_cnt--)
list(self.manager.take_events())
# After cascade completes, ref_cnt should be 0
for block_hash in blocks:
block = self.primary_tier._policy.get(block_hash)
assert block.ref_cnt == 0
# All completed jobs have been drained
assert len(self.secondary_tier1.completed_jobs) == 0
assert len(self.secondary_tier2.completed_jobs) == 0
def test_lookup_from_primary(self, manager_setup):
"""Test lookup when blocks are in primary tier."""
blocks = to_keys(range(3))
# Store blocks
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
# Lookup should find all blocks in primary
assert count_hits(self.manager, blocks) == 3
def test_promotion_from_secondary(self, manager_setup):
"""Test promotion of blocks from secondary to primary tier."""
blocks = to_keys(range(3))
# Manually add blocks to secondary tier (simulate previous cascade)
for block in blocks:
self.secondary_tier1.blocks[block] = True
# Lookup each block to initiate promotion for all of them
for block in blocks:
result = self.manager.lookup(block, _CTX)
assert result is None # Retry later (promotion initiated)
# End of step 1: flushes deferred submit_load() calls
list(self.manager.take_events())
# End of step 2: processes the completed promotion jobs
list(self.manager.take_events())
# Now blocks should be in primary tier
assert count_hits(self.primary_tier, blocks) == 3
# Next lookup should succeed
assert count_hits(self.manager, blocks) == 3
def test_partial_lookup(self, manager_setup):
"""Test lookup with partial hits."""
blocks = to_keys(range(5))
# Store first 3 blocks to primary
self.manager.prepare_store(blocks[:3], _CTX)
self.manager.complete_store(blocks[:3], _CTX, success=True)
# Lookup all 5 blocks should return 3 (first 3 found)
assert count_hits(self.manager, blocks) == 3
def test_eviction_in_primary_tier(self, manager_setup):
"""Test eviction in primary tier when capacity is exceeded."""
# Primary tier has capacity of 5 blocks
# First, fill the primary tier
blocks = to_keys(range(5))
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
assert len(result.keys_to_store) == 5
self.manager.complete_store(blocks, _CTX, success=True)
# End of step: release ref_cnt from cascade
list(self.manager.take_events())
# Now try to store 2 more blocks (should trigger eviction)
more_blocks = to_keys(range(5, 7))
result = self.manager.prepare_store(more_blocks, _CTX)
# Should evict 2 blocks from primary tier
assert result is not None
assert len(result.evicted_keys) == 2
assert len(result.keys_to_store) == 2
def test_touch_propagates_to_all_tiers(self, manager_setup):
"""Test that touch() propagates to all tiers."""
blocks = to_keys(range(3))
# Store blocks
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
list(self.manager.take_events())
# Touch blocks
self.manager.touch(blocks, _CTX)
# Verify touch was called on primary tier (check LRU order)
# In LRU, touched blocks should be at the end
primary_keys = list(self.primary_tier._policy.blocks.keys())
assert primary_keys[-3:] == list(reversed(blocks))
# Verify touch was called on all secondary tiers
secondary1_keys = list(self.secondary_tier1.blocks.keys())
assert secondary1_keys[-3:] == list(reversed(blocks))
secondary2_keys = list(self.secondary_tier2.blocks.keys())
assert secondary2_keys[-3:] == list(reversed(blocks))
def test_failed_store_no_cascade(self, manager_setup):
"""Test that failed GPU→primary store doesn't cascade."""
blocks = to_keys(range(3))
self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store
)
self.secondary_tier2.submit_store = MagicMock(
wraps=self.secondary_tier2.submit_store
)
# Prepare store
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
# Complete store with failure — cascade must not happen
self.manager.complete_store(blocks, _CTX, success=False)
# submit_store was never called on either secondary tier
self.secondary_tier1.submit_store.assert_not_called()
self.secondary_tier2.submit_store.assert_not_called()
def test_multiple_secondary_tiers_independent_eviction(self):
"""Test that secondary tiers manage their own evictions."""
mock_region = _mock_mmap_region(10)
mock_view = mock_region.create_kv_memoryview()
# Create tier with small capacity
small_tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG,
primary_kv_view=mock_view,
max_blocks=5,
simulate_async=False,
)
large_tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG,
primary_kv_view=mock_view,
max_blocks=10,
simulate_async=False,
)
# Create a fresh primary tier for this test
primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=10, mmap_region=mock_region
)
manager = TieringOffloadingManager(
primary_tier=primary_tier,
secondary_tiers=[small_tier, large_tier],
)
# First, store 5 blocks to fill the small tier
blocks1 = to_keys(range(5))
result = manager.prepare_store(blocks1, _CTX)
assert result is not None
manager.complete_store(blocks1, _CTX, success=True)
list(manager.take_events())
# Both tiers should have 5 blocks
assert small_tier.get_num_blocks() == 5
assert large_tier.get_num_blocks() == 5
# Now store 3 more blocks - small tier should evict 3 blocks
blocks2 = to_keys(range(5, 8))
result = manager.prepare_store(blocks2, _CTX)
assert result is not None
manager.complete_store(blocks2, _CTX, success=True)
list(manager.take_events())
# Small tier should still have 5 blocks (evicted 3, added 3)
assert small_tier.get_num_blocks() == 5
# Large tier should have all 8 blocks
assert large_tier.get_num_blocks() == 8
def test_lookup_batches_submit_load_per_request(self, manager_setup):
"""lookup() defers submit_load until take_events(), one call per request.
Blocks from different requests each get their own submit_load call, each
carrying the correct req_context.
"""
blocks = to_keys(range(4))
for block in blocks:
self.secondary_tier1.blocks[block] = True
self.secondary_tier1.submit_load = MagicMock(
wraps=self.secondary_tier1.submit_load
)
ctx_a = ReqContext(req_id="req_a")
ctx_b = ReqContext(req_id="req_b")
# All lookups return None: secondary hit triggers promotion (in-flight)
assert self.manager.lookup(blocks[0], ctx_a) is None
assert self.manager.lookup(blocks[1], ctx_a) is None
assert self.manager.lookup(blocks[2], ctx_b) is None
assert self.manager.lookup(blocks[3], ctx_b) is None
# submit_load must not fire during lookup - only at end of step
self.secondary_tier1.submit_load.assert_not_called()
# simulate end of step
list(self.manager.take_events())
assert self.secondary_tier1.submit_load.call_count == 2
calls = self.secondary_tier1.submit_load.call_args_list
jm_a = calls[0].args[0]
jm_b = calls[1].args[0]
assert set(jm_a.keys) == {blocks[0], blocks[1]}
assert jm_a.req_context is ctx_a
assert set(jm_b.keys) == {blocks[2], blocks[3]}
assert jm_b.req_context is ctx_b
def test_lookup_shared_block_no_duplicate_promotion(self, manager_setup):
"""A block looked up by two requests in the same step is promoted once.
The first lookup initiates promotion (returns None via secondary hit).
The second lookup sees ref_cnt=-1 on the primary slot and returns None
via the primary in-flight path — without triggering a second promotion.
"""
shared_block = to_keys([0])[0]
self.secondary_tier1.blocks[shared_block] = True
self.secondary_tier1.submit_load = MagicMock(
wraps=self.secondary_tier1.submit_load
)
ctx_a = ReqContext(req_id="req_a")
ctx_b = ReqContext(req_id="req_b")
result_a = self.manager.lookup(shared_block, ctx_a)
result_b = self.manager.lookup(shared_block, ctx_b)
# Both see None (in-flight), but promotion is only queued once
assert result_a is None
assert result_b is None
list(self.manager.take_events())
# Only one submit_load call despite two lookups
self.secondary_tier1.submit_load.assert_called_once()
job_metadata = self.secondary_tier1.submit_load.call_args.args[0]
assert list(job_metadata.keys) == [shared_block]
assert job_metadata.req_context is ctx_a
def test_complete_store_forwards_req_context_to_submit_store(self, manager_setup):
"""complete_store cascades to secondary tiers with the correct req_context."""
blocks = to_keys(range(2))
self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store
)
ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"})
self.manager.prepare_store(blocks, ctx)
self.manager.complete_store(blocks, ctx, success=True)
assert self.secondary_tier1.submit_store.call_count == 1
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
assert job_metadata.req_context is ctx
class TestTieringOffloadingWithoutSecondaryTiers:
"""Test TieringOffloadingManager with no secondary tiers (backward compat)."""
def test_works_without_secondary_tiers(self):
"""Test that manager works with empty secondary_tiers list."""
primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=5, mmap_region=_mock_mmap_region(5)
)
# Create manager with no secondary tiers
manager = TieringOffloadingManager(
primary_tier=primary_tier, secondary_tiers=[]
)
blocks = to_keys(range(3))
# Should work like a regular OffloadingManager
result = manager.prepare_store(blocks, _CTX)
assert result is not None
manager.complete_store(blocks, _CTX, success=True)
assert count_hits(manager, blocks) == 3
if __name__ == "__main__":
pytest.main([__file__, "-v"])
+3 -5
View File
@@ -4,8 +4,6 @@
Core abstractions for KV cache offloading in vLLM v1.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import Collection, Iterable, Iterator, Sequence
from dataclasses import dataclass
@@ -201,7 +199,7 @@ class OffloadingManager(ABC):
"""
Marks blocks which were previously prepared to be stored, as stored.
Following this call, the blocks become loadable.
If if_success is False, blocks that were not marked as stored will be
If success is False, blocks that were not marked as stored will be
removed.
Args:
@@ -328,7 +326,7 @@ class CanonicalKVCaches:
class OffloadingSpec(ABC):
"""Spec for an offloading connector"""
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig):
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
logger.warning(
"Initializing OffloadingSpec. This API is experimental and "
"subject to change in the future as we iterate the design."
@@ -394,7 +392,7 @@ class OffloadingSpec(ABC):
@abstractmethod
def get_handlers(
self, kv_caches: CanonicalKVCaches
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], "OffloadingHandler"]]:
"""
Get offloading handlers along with their respective src and dst types.
+7
View File
@@ -124,6 +124,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
block_size_factor: int,
kv_cache_groups_data_refs: list[list[CanonicalKVCacheRef]],
gpu_to_cpu: bool,
mmap_region: SharedOffloadRegion | None = None,
):
"""
Initialize a SingleDirectionOffloadingHandler.
@@ -167,6 +168,8 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
self.dst_block_size_factor = block_size_factor if self.gpu_to_cpu else 1
self.transfer_type = ("GPU", "CPU") if self.gpu_to_cpu else ("CPU", "GPU")
# mmap_region to clean up on shutdown (gpu_to_cpu handler owns it)
self._mmap_region = mmap_region
# job_id -> event
self._transfer_events: dict[int, torch.Event] = {}
# queue of transfers (job_id, stream, event)
@@ -382,6 +385,9 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
self._event_pool.clear()
self.src_tensors.clear()
self.dst_tensors.clear()
if self._mmap_region is not None:
self._mmap_region.cleanup()
self._mmap_region = None
class CpuGpuOffloadingHandlers:
@@ -434,6 +440,7 @@ class CpuGpuOffloadingHandlers:
block_size_factor=block_size_factor,
kv_cache_groups_data_refs=kv_caches.group_data_refs,
gpu_to_cpu=True,
mmap_region=mmap_region,
)
self.cpu_to_gpu_handler = SingleDirectionOffloadingHandler(
@@ -153,6 +153,20 @@ class SharedOffloadRegion:
self._views.append(worker_layer_view)
return worker_layer_view
def create_kv_memoryview(self) -> memoryview:
"""Return a zero-copy memoryview over the entire KV buffer.
Shape: (num_blocks, row_stride_bytes). Secondary tiers address
block *b* as ``view[b]``.
"""
kv_tensor = self._base.view(self.num_blocks, self._row_stride)
np_arr = kv_tensor.numpy()
assert np_arr.ctypes.data == self._base.data_ptr(), (
"view()/numpy() created a copy instead of sharing the mmap buffer; "
"secondary tiers require zero-copy access to primary KV data"
)
return memoryview(np_arr)
def cleanup(self) -> None:
if self.is_pinned and self._base is not None:
base_ptr = self._base.data_ptr()
+14 -6
View File
@@ -44,6 +44,10 @@ class CPUOffloadingSpec(OffloadingSpec):
if kv_bytes_per_offloaded_block > 0
else 0
)
world_size = vllm_config.parallel_config.world_size
self.cpu_page_size_per_worker: int = (
kv_bytes_per_offloaded_block // world_size if world_size > 0 else 0
)
# scheduler-side
self._manager: OffloadingManager | None = None
@@ -77,6 +81,15 @@ class CPUOffloadingSpec(OffloadingSpec):
)
return self._manager
def _create_handlers(
self, kv_caches: CanonicalKVCaches
) -> CpuGpuOffloadingHandlers:
return CpuGpuOffloadingHandlers(
kv_caches=kv_caches,
block_size_factor=self.block_size_factor,
num_cpu_blocks=self.num_blocks,
)
def get_handlers(
self, kv_caches: CanonicalKVCaches
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
@@ -85,12 +98,7 @@ class CPUOffloadingSpec(OffloadingSpec):
raise Exception(
"CPU Offloading is currently only supported on CUDA-alike GPUs"
)
self._handlers = CpuGpuOffloadingHandlers(
kv_caches=kv_caches,
block_size_factor=self.block_size_factor,
num_cpu_blocks=self.num_blocks,
)
self._handlers = self._create_handlers(kv_caches)
assert self._handlers is not None
yield GPULoadStoreSpec, CPULoadStoreSpec, self._handlers.gpu_to_cpu_handler
+5
View File
@@ -56,3 +56,8 @@ class OffloadingSpecFactory:
OffloadingSpecFactory.register_spec(
"CPUOffloadingSpec", "vllm.v1.kv_offload.cpu.spec", "CPUOffloadingSpec"
)
OffloadingSpecFactory.register_spec(
"TieringOffloadingSpec",
"vllm.v1.kv_offload.tiering.spec",
"TieringOffloadingSpec",
)
+168
View File
@@ -0,0 +1,168 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Abstract interfaces and data types for the secondary tiering layer.
"""
from abc import ABC, abstractmethod
from collections.abc import Collection, Iterable
from dataclasses import dataclass
from typing import TYPE_CHECKING
import numpy as np
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
if TYPE_CHECKING:
from vllm.config import VllmConfig
# Type alias for job IDs used in async transfer tracking
JobId = int
@dataclass
class JobMetadata:
"""Metadata for an in-flight async transfer job."""
job_id: JobId
keys: Collection[OffloadKey]
block_ids: np.ndarray
is_promotion: bool
req_context: ReqContext
@dataclass
class JobResult:
"""Result of an async transfer job (successful or failed)."""
job_id: JobId
success: bool
class SecondaryTierManager(ABC):
"""
Abstract interface for managing a single non-primary offloading tier.
Secondary tiers cannot directly access GPU memory. All data transfers
must go through the CPU (primary) tier:
- Store: GPU → CPU (primary) → secondary (cascade)
- Load: secondary → CPU (primary) → GPU (promotion)
IMPORTANT: All methods run in the Scheduler process and must be
lightweight and non-blocking. submit_load() and submit_store() submit
async jobs; get_finished() polls for completion.
"""
def __init__(self, vllm_config: "VllmConfig", primary_kv_view: memoryview) -> None:
self._vllm_config = vllm_config
self._primary_kv_view: memoryview = primary_kv_view
@abstractmethod
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
"""
Check whether a block exists in this secondary tier.
Args:
key: Offload key to look up.
req_context: per-request context (e.g. kv_transfer_params).
Returns:
True if the block is present and ready,
False if not found,
or None if the block is being transferred (retry later).
"""
pass
@abstractmethod
def submit_store(self, job_metadata: JobMetadata) -> None:
"""
Submit an async job to store blocks from the primary tier to this
secondary tier.
This method must be lightweight and non-blocking: allocate metadata
and submit the transfer, but do NOT perform the data copy on the
calling thread.
Preconditions (guaranteed by the framework):
- ``job_metadata.block_ids`` are valid primary-tier slots, pinned
(ref-counted) for the duration of the transfer.
The implementation is responsible for:
1. Filtering out blocks already present in this tier
2. Evicting blocks if capacity is needed
3. Allocating space in this tier
4. Submitting the async transfer (read from primary via block_ids)
Report completion via ``get_finished()``.
Args:
job_metadata: Job metadata including job_id, keys, and block_ids
identifying the primary-tier slots to read from.
"""
pass
@abstractmethod
def submit_load(self, job_metadata: JobMetadata) -> None:
"""
Submit an async job to load blocks from this secondary tier to the
primary tier.
This method must be lightweight and non-blocking: mark blocks as
in-flight and submit the transfer, but do NOT perform the data copy
on the calling thread.
Preconditions (guaranteed by the framework):
- ``job_metadata.block_ids`` are allocated primary-tier slots
ready to receive data.
The implementation must copy data from this tier into the
primary-tier slots identified by ``block_ids``.
Report completion via ``get_finished()``.
Args:
job_metadata: Job metadata including job_id, keys, and block_ids
identifying the primary-tier slots to write into.
"""
pass
@abstractmethod
def get_finished(self) -> Iterable[JobResult]:
"""
Return all jobs (loads and stores) that completed since the last call.
The framework uses these results to release resources and finalize
transfers.
Returns:
Iterable of JobResult objects for jobs finished since the
last call.
"""
pass
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
"""
Mark blocks as recently used for eviction policy.
Args:
keys: Offload keys to mark as recently used.
req_context: Per-request context.
"""
return
def shutdown(self) -> None:
"""Release resources held by this tier (threads, connections, etc.)."""
return
@staticmethod
@abstractmethod
def get_tier_type() -> str:
"""
Get the type identifier of this tier (e.g., "example", "storage").
Must match the "type" field in the tier config dict.
Returns:
Tier type string.
"""
pass
@@ -0,0 +1,235 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
ExampleSecondaryTier: A simple in-memory secondary tier for testing.
This implementation provides a minimal secondary tier that stores blocks
in memory (using a dictionary) and simulates async transfers with immediate
completion. It's useful for testing the TieringOffloadingManager without
requiring actual storage or network backends.
"""
from collections import OrderedDict
from collections.abc import Collection, Iterable
from dataclasses import dataclass
from typing import TYPE_CHECKING
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
from vllm.v1.kv_offload.tiering.base import (
JobId,
JobMetadata,
JobResult,
SecondaryTierManager,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
@dataclass
class _JobMetadata:
"""Internal metadata for tracking job details."""
job_id: JobId
keys: Collection[OffloadKey]
is_store: bool # True for store jobs, False for load jobs
class ExampleSecondaryTier(SecondaryTierManager):
"""
A simple in-memory secondary tier for testing.
This implementation:
- Stores blocks in a dictionary (key -> True)
- Simulates async transfers with immediate completion
- Uses LRU eviction policy
"""
def __init__(
self,
vllm_config: "VllmConfig",
primary_kv_view: memoryview,
max_blocks: int = 1000,
simulate_async: bool = False,
):
"""
Initialize the example secondary tier.
Args:
vllm_config: Global vLLM configuration.
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
max_blocks: Maximum number of blocks this tier can store
simulate_async: If True, jobs complete on next get_finished() call.
If False, jobs complete immediately.
"""
super().__init__(vllm_config, primary_kv_view)
self.max_blocks = max_blocks
self.simulate_async = simulate_async
# key -> True (only care about presence)
self.blocks: OrderedDict[OffloadKey, bool] = OrderedDict()
# Completed jobs waiting to be retrieved by get_finished()
self.completed_jobs: list[JobResult] = []
# Pending jobs (for simulated async mode)
self.pending_jobs: list[_JobMetadata] = []
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
"""
Check whether a block exists in this secondary tier.
Args:
key: Offload key to look up.
req_context: Per-request context.
Returns:
True if the block is present, False if not found.
"""
return key in self.blocks
def submit_store(self, job_metadata: JobMetadata) -> None:
"""
Submit an async job to store blocks from primary tier to this tier.
Args:
job_metadata: Job metadata including job_id, keys, and
spec for reading blocks from the primary tier.
"""
job_id = job_metadata.job_id
keys = job_metadata.keys
block_ids = job_metadata.block_ids
assert len(keys) == len(block_ids), (
f"Length mismatch: {len(keys)} keys but {len(block_ids)} block_ids"
)
# Filter out blocks already present
blocks_to_store = [bh for bh in keys if bh not in self.blocks]
if not blocks_to_store:
# All blocks already present
return
# Evict blocks if needed (LRU policy)
num_blocks_to_evict = len(blocks_to_store) - (
self.max_blocks - len(self.blocks)
)
evicted = []
if num_blocks_to_evict > 0:
# Collect eviction candidates first (LRU order), then delete atomically
protected = set(keys)
for key in self.blocks:
if key not in protected:
evicted.append(key)
if len(evicted) == num_blocks_to_evict:
break
else:
# Could not collect enough eviction candidates
return
for key in evicted:
del self.blocks[key]
# Create internal job metadata
internal_job_metadata = _JobMetadata(
job_id=job_id, keys=blocks_to_store, is_store=True
)
if self.simulate_async:
# Job will complete on next get_finished() call
self.pending_jobs.append(internal_job_metadata)
else:
# Job completes immediately
self._complete_store_job(internal_job_metadata)
def submit_load(self, job_metadata: JobMetadata) -> None:
"""
Submit an async job to load blocks from this tier to primary tier.
Args:
job_metadata: Job metadata including job_id, keys, and
spec for writing blocks into the primary tier.
"""
job_id = job_metadata.job_id
keys = job_metadata.keys
block_ids = job_metadata.block_ids
assert len(keys) == len(block_ids), (
f"Length mismatch: {len(keys)} keys but {len(block_ids)} block_ids"
)
# Verify all blocks exist
for key in keys:
if key not in self.blocks:
return
# Create internal job metadata
internal_job_metadata = _JobMetadata(job_id=job_id, keys=keys, is_store=False)
if self.simulate_async:
# Job will complete on next get_finished() call
self.pending_jobs.append(internal_job_metadata)
else:
# Job completes immediately
self._complete_load_job(internal_job_metadata)
def get_finished(self) -> Iterable[JobResult]:
"""
Poll for finished async jobs.
Returns:
Iterable of JobResult objects for all jobs that have
finished since the last call.
"""
# Move pending jobs to completed
if self.simulate_async and self.pending_jobs:
for job_metadata in self.pending_jobs:
if job_metadata.is_store:
self._complete_store_job(job_metadata)
else:
self._complete_load_job(job_metadata)
self.pending_jobs.clear()
# Return completed jobs
result = self.completed_jobs
self.completed_jobs = []
return result
def _complete_store_job(self, job_metadata: _JobMetadata):
"""Complete a store job by adding blocks to storage."""
for key in job_metadata.keys:
self.blocks[key] = True
# Return simplified JobResult (only job_id and success)
self.completed_jobs.append(JobResult(job_id=job_metadata.job_id, success=True))
def _complete_load_job(self, job_metadata: _JobMetadata):
"""Complete a load job."""
# Return simplified JobResult (only job_id and success)
self.completed_jobs.append(JobResult(job_id=job_metadata.job_id, success=True))
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
"""
Mark blocks as recently used (move to end of LRU list).
Args:
keys: Blocks to mark as recently used.
req_context: Per-request context.
"""
for key in reversed(list(keys)):
if key in self.blocks:
self.blocks.move_to_end(key)
@staticmethod
def get_tier_type() -> str:
return "example"
def get_num_blocks(self) -> int:
"""Get the number of blocks currently stored in this tier."""
return len(self.blocks)
def clear(self):
"""Clear all blocks and pending jobs (for testing)."""
self.blocks.clear()
self.completed_jobs.clear()
self.pending_jobs.clear()
+56
View File
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Factory for creating secondary tier implementations.
"""
from typing import TYPE_CHECKING
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
from vllm.v1.kv_offload.tiering.example import ExampleSecondaryTier
if TYPE_CHECKING:
from vllm.config import VllmConfig
SUPPORTED_TIERS: tuple[type[SecondaryTierManager], ...] = (ExampleSecondaryTier,)
_TIER_REGISTRY: dict[str, type[SecondaryTierManager]] = {
cls.get_tier_type(): cls for cls in SUPPORTED_TIERS
}
def create_secondary_tier(
tier_config: dict,
primary_kv_view: memoryview,
vllm_config: "VllmConfig",
) -> SecondaryTierManager:
"""
Create a secondary tier from configuration.
Args:
tier_config: Dictionary with tier configuration containing:
- type (required): Type of secondary tier (e.g., "example")
- Additional tier-specific parameters are passed directly
to the tier constructor
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
vllm_config: Global vLLM configuration.
Returns:
SecondaryTierManager instance
Raises:
ValueError: If tier type is unknown or configuration is invalid
"""
config = tier_config.copy()
tier_type = config.pop("type", None)
if not tier_type:
raise ValueError("Secondary tier configuration must include 'type'")
cls = _TIER_REGISTRY.get(tier_type)
if cls is None:
raise ValueError(
f"Unknown secondary tier type: {tier_type!r}. "
f"Supported types: {list(_TIER_REGISTRY)}"
)
return cls(vllm_config=vllm_config, primary_kv_view=primary_kv_view, **config)
+505
View File
@@ -0,0 +1,505 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
TieringOffloadingManager: Multi-tier KV cache offloading orchestrator.
This manager coordinates between a CPU primary tier (with direct GPU access)
and zero or more secondary tiers (Storage, Network, etc.) to provide
hierarchical KV cache offloading.
Key Design Principles:
1. Always offload to all tiers — When a block is stored to the primary tier,
it is cascaded to ALL secondary tiers
2. Primary tier is the gateway — Secondary tiers cannot access GPU memory
directly; all data flows through the CPU primary tier
3. Staged promotion — Blocks in secondary tiers must be promoted to the
primary tier before GPU can access them
4. Transparent retry mechanism — Return None from lookup() to signal
"data is being promoted, try later"
5. ref_cnt as eviction protection — primary.prepare_read() increments ref_cnt,
protecting blocks from eviction until complete_read() is called
"""
from collections.abc import Collection, Iterable
from dataclasses import dataclass, field
import numpy as np
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import (
LoadStoreSpec,
OffloadingEvent,
OffloadingManager,
OffloadKey,
PrepareStoreOutput,
ReqContext,
)
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.kv_offload.tiering.base import (
JobId,
JobMetadata,
SecondaryTierManager,
)
logger = init_logger(__name__)
@dataclass
class PendingPromotion:
"""Accumulator for blocks awaiting submit_load() for one (tier, request)."""
req_context: ReqContext
keys: list[OffloadKey] = field(default_factory=list)
block_ids: list[int] = field(default_factory=list)
class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
"""CPUOffloadingManager with a primary/secondary transfer interface.
The inherited prepare_store/complete_store/prepare_load/complete_load are the
GPU-facing OffloadingManager interface. These aliases expose the same operations
from the secondary tier perspective, where read/write refers to secondary
accessing primary. This avoids confusion when reading TieringOffloadingManager
code (e.g. calling prepare_load inside a cascade/store path would be misleading).
"""
def __init__(
self,
num_blocks: int,
mmap_region: SharedOffloadRegion,
cache_policy: str = "lru",
enable_events: bool = False,
):
super().__init__(
num_blocks=num_blocks,
cache_policy=cache_policy, # type: ignore[arg-type]
enable_events=enable_events,
)
self._mmap_region = mmap_region
# read/write is for CPU<->secondary transfers,
# load/store is for CPU<->GPU transfers.
# These aliases avoid calling prepare_load inside a store path.
self.prepare_read = self.prepare_load
self.complete_read = self.complete_load
self.prepare_write = self.prepare_store
self.complete_write = self.complete_store
self._kv_memoryview = mmap_region.create_kv_memoryview()
def get_kv_memoryview(self) -> memoryview:
"""Return the memoryview over the primary tier's KV cache buffer.
The view has shape (num_blocks, row_stride_bytes) and is backed by the
SharedOffloadRegion mmap. Secondary tiers address block *b* as
``view[b]``.
"""
return self._kv_memoryview
def shutdown(self) -> None:
super().shutdown()
self._kv_memoryview.release()
self._mmap_region.cleanup()
class TieringOffloadingManager(OffloadingManager):
"""
Orchestrates multi-tier KV cache offloading.
This manager coordinates between a CPU primary tier (with direct GPU access)
and zero or more secondary tiers (Storage, Network, etc.) to provide
hierarchical KV cache offloading.
Key internal state:
- Minimal state tracking; relies on secondary tiers to report completion
via get_finished()
- Secondary tiers return JobResult objects containing all necessary
information
- job_id_counter: monotonically increasing counter for job IDs
"""
def __init__(
self,
primary_tier: CPUPrimaryTierOffloadingManager,
secondary_tiers: list[SecondaryTierManager] | None = None,
enable_events: bool = False,
):
"""
Initialize the TieringOffloadingManager.
Args:
primary_tier: The primary tier manager (CPU-based).
secondary_tiers: List of secondary tier managers (e.g., Storage,
Network). Can be None or empty list.
enable_events: Whether to track offloading events
"""
self.primary_tier: CPUPrimaryTierOffloadingManager = primary_tier
self.secondary_tiers = secondary_tiers or []
self._job_id_counter: int = 0
self.events: list[OffloadingEvent] | None = [] if enable_events else None
# Job tracking: maps job_id to metadata for all in-flight transfers.
# JobMetadata.is_promotion distinguishes direction:
# True: secondary → primary (promotion)
# False: primary → secondary (cascade)
self._transfer_jobs: dict[JobId, JobMetadata] = {}
# Pending promotion requests accumulated during lookup() calls; flushed
# as one batched submit_load() per (tier, request) in take_events().
# Outer key: tier. Inner key: req_context.req_id — the same ReqContext
# object is reused for all block lookups of a given request per engine step.
self._pending_load_submissions: dict[
SecondaryTierManager, dict[str, PendingPromotion]
] = {}
# Gate for once-per-step execution of _maybe_process_finished_jobs().
# Reset at the end of each step in take_events().
self._processed_jobs_this_step: bool = False
def _next_job_id(self) -> JobId:
"""Generate a unique job ID for async transfer tracking."""
job_id = self._job_id_counter
self._job_id_counter += 1
return job_id
def _maybe_process_finished_jobs(self):
"""
Poll secondary tiers for completed jobs (at most once per step).
Guarded by _processed_jobs_this_step: the first call in an engine step
does the actual polling; subsequent calls are no-ops. The flag is reset
in take_events() at the end of each step.
"""
if self._processed_jobs_this_step:
return
self._processed_jobs_this_step = True
self._process_finished_jobs()
def _process_finished_jobs(self):
"""
Unconditionally poll all secondary tiers for completed jobs.
This method:
1. Calls get_finished() on each secondary tier
2. For completed stores (primary→secondary): calls primary.complete_read()
to decrement ref_cnt
3. For completed loads (secondary→primary): calls primary.complete_write()
to make blocks available
"""
for i, tier in enumerate(self.secondary_tiers):
for completed_job in tier.get_finished():
job_id = completed_job.job_id
job_metadata = self._transfer_jobs.pop(job_id, None)
assert job_metadata is not None, (
f"Finished job_id {job_id} from tier #{i}"
f" ({tier.get_tier_type()}) not in _transfer_jobs"
)
if job_metadata.is_promotion:
# secondary→primary transfer (promotion) completed.
# Make blocks available in primary tier.
self.primary_tier.complete_write(
job_metadata.keys,
job_metadata.req_context,
completed_job.success,
)
else:
# primary→secondary transfer completed.
# Decrement ref_cnt on primary blocks.
self.primary_tier.complete_read(
job_metadata.keys, job_metadata.req_context
)
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
"""
Check whether a single block is offloaded and ready.
Algorithm:
1. Process any completed async jobs first.
2. Query primary tier — short-circuit on hit or in-flight.
3. On primary miss, query secondary tiers — stop on first
hit and initiate promotion.
Args:
key: Block hash to look up.
req_context: Per-request context.
Returns:
True — block is ready in the primary tier.
None — block found but not yet ready (primary in-flight,
promotion started, or a secondary tier is busy).
False — block not found in any tier, or primary is full
and cannot accept a promotion.
"""
self._maybe_process_finished_jobs()
primary_hit = self.primary_tier.lookup(key, req_context)
if primary_hit is True:
return True
if primary_hit is None:
return None
any_none = False
for tier in self.secondary_tiers:
result = tier.lookup(key, req_context)
if result is True:
if not self._initiate_promotion(tier, key, req_context):
return False # primary full, block unavailable
return None # promotion started, retry later
if result is None:
any_none = True
if any_none:
return None
return False
def _initiate_promotion(
self,
tier: SecondaryTierManager,
key: OffloadKey,
req_context: ReqContext,
) -> bool:
"""
Queue a block for promotion from a secondary tier to the primary tier.
Allocates space in the primary tier immediately (sets ref_cnt=-1 so
subsequent lookups within the same step see the slot as in-flight),
then defers the actual submit_load() call to _flush_pending_promotions()
so all blocks queued during one engine step are submitted as a single
batched job.
Args:
tier: The secondary tier to promote from
key: Block to promote
req_context: Per-request context forwarded to primary.prepare_write().
Returns:
True if promotion was initiated, False if primary tier is full.
"""
# Allocate space in primary tier for promoted block.
# Must happen immediately so primary.lookup() returns None (in-flight)
# for this key on any subsequent lookup() call within the same step,
# preventing duplicate promotion attempts.
primary_write_result = self.primary_tier.prepare_write([key], req_context)
if primary_write_result is None:
# Primary tier is full; caller should treat the block as unavailable
# rather than retrying indefinitely.
return False
store_spec = primary_write_result.store_spec
assert isinstance(store_spec, CPULoadStoreSpec)
# Defer submit_load to take_events(). Group by (tier, request) so each
# request's blocks are submitted as one batched job per tier.
tier_pending = self._pending_load_submissions.setdefault(tier, {})
ctx_id = req_context.req_id
if ctx_id not in tier_pending:
tier_pending[ctx_id] = PendingPromotion(
keys=[], block_ids=[], req_context=req_context
)
entry = tier_pending[ctx_id]
entry.keys.extend(primary_write_result.keys_to_store)
entry.block_ids.extend(store_spec.block_ids)
return True
def _flush_pending_promotions(self) -> None:
"""Submit one batched submit_load() per (tier, request).
Called from take_events() at the end of each engine step, flushing
all promotion requests deferred during lookup().
"""
if not self._pending_load_submissions:
return
for tier, pending_by_ctx in self._pending_load_submissions.items():
for entry in pending_by_ctx.values():
job_id = self._next_job_id()
job_metadata = JobMetadata(
job_id=job_id,
keys=entry.keys,
block_ids=np.array(entry.block_ids, dtype=np.int64),
is_promotion=True,
req_context=entry.req_context,
)
self._transfer_jobs[job_id] = job_metadata
tier.submit_load(job_metadata)
self._pending_load_submissions.clear()
def prepare_load(
self, keys: Collection[OffloadKey], req_context: ReqContext
) -> LoadStoreSpec:
"""
Prepare blocks to be loaded from primary tier to GPU.
CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure
that any completed promotions have been finalized and blocks are ready.
This increments ref_cnt on the blocks in the primary tier, protecting
them from eviction during the transfer.
Args:
keys: Blocks to prepare for loading.
req_context: Per-request context.
Returns:
LoadStoreSpec for reading from primary tier.
"""
# Process completed promotions to ensure blocks are ready
self._maybe_process_finished_jobs()
return self.primary_tier.prepare_load(keys, req_context)
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
"""
Mark blocks as recently used in all tiers.
Args:
keys: Blocks to mark as recently used.
req_context: Per-request context.
"""
self.primary_tier.touch(keys, req_context)
for tier in self.secondary_tiers:
tier.touch(keys, req_context)
def complete_load(self, keys: Collection[OffloadKey], req_context: ReqContext):
"""
Mark blocks as done loading from primary tier to GPU.
This decrements ref_cnt on the blocks in the primary tier, allowing
them to be evicted again.
Args:
keys: Blocks that finished loading.
req_context: Per-request context.
"""
self.primary_tier.complete_load(keys, req_context)
def prepare_store(
self, keys: Collection[OffloadKey], req_context: ReqContext
) -> PrepareStoreOutput | None:
"""
Prepare blocks to be stored from GPU to primary tier.
CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure
that any completed async transfers have their ref_cnt decremented
before the primary tier makes eviction decisions.
Args:
keys: Blocks to prepare for storing.
req_context: Per-request context.
Returns:
PrepareStoreOutput describing where to store blocks and what was
evicted, or None if store cannot proceed.
"""
# Step 1: Poll for completed async jobs FIRST
# This decrements ref_cnt on primary blocks that have been
# successfully transferred to secondary tiers.
self._maybe_process_finished_jobs()
# Step 2: Store to primary tier
primary_result = self.primary_tier.prepare_store(keys, req_context)
# Note: Secondary tier cascading will happen in complete_store()
# after the GPU→Primary transfer completes and blocks are ready.
return primary_result
def complete_store(
self,
keys: Collection[OffloadKey],
req_context: ReqContext,
success: bool = True,
):
"""
Mark blocks as done storing from GPU to primary tier.
This is where secondary tier cascading happens — after blocks are
confirmed to be in the primary tier, they are cascaded to ALL
secondary tiers.
For each secondary tier:
1. Call primary.prepare_read() to get LoadStoreSpec AND increment
ref_cnt (protecting blocks during async transfer)
2. Call tier.submit_store() to start async transfer: primary→secondary
3. Track the job in _store_jobs dictionary
Args:
keys: Blocks that finished storing.
success: Whether the GPU→primary transfer succeeded.
req_context: Per-request context forwarded to primary.prepare_read().
"""
# Step 1: Complete store in primary tier (makes blocks loadable)
self.primary_tier.complete_store(keys, req_context, success)
if not success:
# If GPU→Primary transfer failed, don't cascade to secondary tiers
return
# Step 2: Cascade to ALL secondary tiers
# For each secondary tier, call primary.prepare_read() to get the
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from
# eviction during the async transfer). One prepare_read() call per
# secondary tier.
for tier in self.secondary_tiers:
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
# Submit async store job: primary→secondary
job_id = self._next_job_id()
# Track this store job
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
job_metadata = JobMetadata(
job_id=job_id,
keys=keys,
block_ids=primary_blocks_spec.block_ids,
is_promotion=False,
req_context=req_context,
)
self._transfer_jobs[job_id] = job_metadata
tier.submit_store(job_metadata)
# Note: The async transfers are now in flight. Their completion is
# tracked via get_finished() / _maybe_process_finished_jobs().
def take_events(self) -> Iterable[OffloadingEvent]:
"""
End-of-step hook: flush deferred work, yield events, reset per-step state.
Called once per engine step from Scheduler.update_from_output() →
connector.take_events(). Ensures _maybe_process_finished_jobs() has run
at least once this step, flushes pending promotions, yields collected
events, and resets the per-step flag.
Yields:
New OffloadingEvents collected since the last call.
"""
# TODO: Move _flush_pending_promotions() to a dedicated end_of_batch()
# hook once one exists. For now, take_events() serves as the flush
# point under the assumption that it is called at the end of each
# engine step (Scheduler.update_from_output() → connector.take_events()).
# When the dedicated hook is added, update tests that rely on
# take_events() to signal end of step.
self._maybe_process_finished_jobs()
self._flush_pending_promotions()
# Reset the per-step gate so next step's first call does real work.
self._processed_jobs_this_step = False
if self.events is not None:
yield from self.events
self.events.clear()
yield from self.primary_tier.take_events()
def shutdown(self) -> None:
"""Shutdown all tiers and release resources."""
for tier in self.secondary_tiers:
tier.shutdown()
self.primary_tier.shutdown()
+187
View File
@@ -0,0 +1,187 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
TieringOffloadingSpec: Spec for multi-tier KV cache offloading.
This spec creates a TieringOffloadingManager with a CPU primary tier
and configurable secondary tiers (e.g., Storage, Network).
Configuration via kv_connector_extra_config:
- cpu_bytes_to_use: (required) Bytes to allocate for CPU primary tier
- block_size: (optional) Block size for offloaded blocks (default: GPU block size)
- eviction_policy: (optional) Primary tier eviction policy: "lru" or
"arc" (default: "lru")
- secondary_tiers: (optional) List of secondary tier configurations
Each secondary tier config is a dict with:
- type: (required) Type of secondary tier (e.g., "example", "storage", "network")
- Additional tier-specific parameters are passed directly to the tier
constructor. See each tier's documentation for supported parameters.
Example configuration:
{
"cpu_bytes_to_use": 10737418240, # 10 GB
"block_size": 16,
"eviction_policy": "lru",
"secondary_tiers": [
{
"type": "example",
# Tier-specific parameters (for ExampleSecondaryTier):
"max_blocks": 10000,
"simulate_async": False
}
]
}
"""
import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.kv_offload.base import CanonicalKVCaches, OffloadingManager
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
from vllm.v1.kv_offload.tiering.factory import create_secondary_tier
from vllm.v1.kv_offload.tiering.manager import (
CPUPrimaryTierOffloadingManager,
TieringOffloadingManager,
)
logger = init_logger(__name__)
class TieringOffloadingSpec(CPUOffloadingSpec):
"""
Spec for multi-tier KV cache offloading.
Creates a TieringOffloadingManager with:
- Primary tier: CPU (LRU or ARC eviction policy)
- Secondary tiers: Configurable via extra_config
The CPU primary tier has direct GPU access and serves as the gateway for
all GPU↔offload operations. Secondary tiers cannot directly access GPU
memory and must transfer data through the primary tier.
"""
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig):
super().__init__(vllm_config, kv_cache_config)
# Redeclare for mypy: parent sets this but `--follow-imports skip` hides it
self._manager: OffloadingManager | None = None
# Parse secondary tier configurations
self.secondary_tier_configs = self.extra_config.get("secondary_tiers", [])
if not isinstance(self.secondary_tier_configs, list):
raise ValueError("secondary_tiers must be a list of tier configurations")
# Scheduler-side mmap (rank=None); kept for cleanup
self._scheduler_mmap: SharedOffloadRegion | None = None
def get_manager(self) -> OffloadingManager:
"""
Get the TieringOffloadingManager.
Creates a TieringOffloadingManager with:
- Primary tier: CPU (LRU or ARC)
- Secondary tiers: As configured in extra_config
Returns:
TieringOffloadingManager instance
"""
if not self._manager:
kv_events_config = self.vllm_config.kv_events_config
enable_events = (
kv_events_config is not None and kv_events_config.enable_kv_cache_events
)
# Create scheduler-side SharedOffloadRegion (rank=None) so the
# primary tier can eagerly create a memoryview over _base.
world_size = self.vllm_config.parallel_config.world_size
scheduler_mmap = SharedOffloadRegion(
instance_id=self.vllm_config.instance_id,
total_size_bytes=self.cpu_page_size_per_worker
* world_size
* self.num_blocks,
num_blocks=self.num_blocks,
rank=None,
num_workers=world_size,
cpu_page_size=self.cpu_page_size_per_worker,
)
self._scheduler_mmap = scheduler_mmap
# Create primary tier (CPU-based)
assert len(self.gpu_block_size) == 1
primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=self.num_blocks,
cache_policy=self.eviction_policy, # type: ignore[arg-type]
enable_events=enable_events,
mmap_region=scheduler_mmap,
)
# Create secondary tiers
primary_kv_view = primary_tier.get_kv_memoryview()
secondary_tiers = []
for i, tier_config in enumerate(self.secondary_tier_configs):
try:
tier = create_secondary_tier(
tier_config, primary_kv_view, self.vllm_config
)
secondary_tiers.append(tier)
logger.info(
"Created secondary tier #%d (%s)",
i,
tier.get_tier_type(),
)
except Exception as e:
logger.error(
"Failed to create secondary tier from config %s: %s",
tier_config,
e,
)
raise
# Create TieringOffloadingManager. GPU↔CPU transfers use the inherited
# get_handlers(); secondary tier transfers are handled by the
# secondary tier managers and need no additional handlers here.
tiering_manager = TieringOffloadingManager(
primary_tier=primary_tier,
secondary_tiers=secondary_tiers,
enable_events=enable_events,
)
if int(self.extra_config.get("store_threshold", 0)) >= 2:
raise ValueError(
"store_threshold is not supported for TieringOffloadingSpec"
)
self._manager = tiering_manager
logger.info(
"Created TieringOffloadingManager with primary tier "
"(%s, %s blocks) and %s secondary tier(s)",
self.eviction_policy,
self.num_blocks,
len(secondary_tiers),
)
return self._manager
def _create_handlers(
self, kv_caches: CanonicalKVCaches
) -> CpuGpuOffloadingHandlers:
world_size = self.vllm_config.parallel_config.world_size
rank = torch.accelerator.current_device_index()
worker_mmap = SharedOffloadRegion(
instance_id=self.vllm_config.instance_id,
total_size_bytes=self.cpu_page_size_per_worker
* world_size
* self.num_blocks,
num_blocks=self.num_blocks,
rank=rank,
num_workers=world_size,
cpu_page_size=self.cpu_page_size_per_worker,
)
return CpuGpuOffloadingHandlers(
kv_caches=kv_caches,
block_size_factor=self.block_size_factor,
num_cpu_blocks=self.num_blocks,
mmap_region=worker_mmap,
)