[kv_offload] Add multi-tier KV cache offloading framework (#40020)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
Ronen Schaffer
2026-05-13 17:21:43 +03:00
committed by GitHub
parent a8887c208f
commit 11f6b545d4
14 changed files with 1841 additions and 11 deletions
@@ -292,3 +292,48 @@ def test_cpu_offloading(
if subscriber is not None:
subscriber.close()
del llm
def test_tiering_offloading() -> None:
"""Tests OffloadingConnector with TieringOffloadingSpec."""
extra_config: dict = {
"cpu_bytes_to_use": 500 << 20,
"block_size": 48,
"spec_name": "TieringOffloadingSpec",
"secondary_tiers": [{"type": "example"}],
}
kv_transfer_config = KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
kv_connector_extra_config=extra_config,
)
port: int
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("0.0.0.0", 0))
port = s.getsockname()[1]
events_endpoint = f"tcp://*:{port}"
kv_events_config = KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
endpoint=events_endpoint,
topic="test",
)
llm = LLM(
model="meta-llama/Llama-3.2-1B-Instruct",
max_model_len=4096,
gpu_memory_utilization=0.5,
kv_events_config=kv_events_config,
kv_transfer_config=kv_transfer_config,
)
subscriber = MockSubscriber(
events_endpoint.replace("*", "127.0.0.1"),
topic=kv_events_config.topic,
)
try:
_latency_test(llm, subscriber)
_accuracy_test(llm, subscriber)
finally:
subscriber.close()
del llm
+37
View File
@@ -262,6 +262,43 @@ def test_cpu_manager():
)
def test_prepare_load_preserves_key_order():
"""block_ids[i] must correspond to keys[i] (co-indexed invariant)."""
manager = CPUOffloadingManager(num_blocks=4, cache_policy="lru")
key_a, key_b, key_c = to_key(0), to_key(1), to_key(2)
# Store all three keys and learn their block ID assignments
store_output = manager.prepare_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
assert store_output is not None
assert isinstance(store_output.store_spec, CPULoadStoreSpec)
key_to_block_id = {
k: int(bid)
for k, bid in zip(store_output.keys_to_store, store_output.store_spec.block_ids)
}
manager.complete_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
# Forward order: [a, b, c]
spec_fwd = manager.prepare_load([key_a, key_b, key_c], _EMPTY_REQ_CTX)
assert isinstance(spec_fwd, CPULoadStoreSpec)
assert [int(x) for x in spec_fwd.block_ids] == [
key_to_block_id[key_a],
key_to_block_id[key_b],
key_to_block_id[key_c],
]
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
# Arbitrary permutation: [b, c, a]
spec_perm = manager.prepare_load([key_b, key_c, key_a], _EMPTY_REQ_CTX)
assert isinstance(spec_perm, CPULoadStoreSpec)
assert [int(x) for x in spec_perm.block_ids] == [
key_to_block_id[key_b],
key_to_block_id[key_c],
key_to_block_id[key_a],
]
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
class TestARCPolicy:
"""Unit tests for CPUOffloadingManager with ARC eviction policy."""
@@ -0,0 +1,565 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for TieringOffloadingManager and ExampleSecondaryTier.
These tests verify:
1. Basic tiered offloading operations (store, load, lookup)
2. Cascade behavior (blocks stored to all secondary tiers)
3. Promotion behavior (blocks loaded from secondary to primary to GPU)
4. ref_cnt management (blocks protected during async transfers)
5. Eviction coordination between tiers
"""
from collections.abc import Iterable
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
from vllm.v1.kv_offload.base import (
OffloadKey,
ReqContext,
make_offload_key,
)
from vllm.v1.kv_offload.tiering.base import JobMetadata
from vllm.v1.kv_offload.tiering.example import ExampleSecondaryTier
from vllm.v1.kv_offload.tiering.manager import (
CPUPrimaryTierOffloadingManager,
TieringOffloadingManager,
)
_CTX = ReqContext(req_id="test")
_MOCK_VLLM_CONFIG = MagicMock()
def _mock_mmap_region(num_blocks: int, row_bytes: int = 16):
"""Create a mock SharedOffloadRegion for testing."""
mock = MagicMock()
view = memoryview(torch.zeros((num_blocks, row_bytes), dtype=torch.int8).numpy())
mock.create_kv_memoryview.return_value = view
return mock
def to_keys(int_ids: Iterable[int]) -> list[OffloadKey]:
return [make_offload_key(str(i).encode(), 0) for i in int_ids]
def count_hits(manager, keys: list[OffloadKey]) -> int | None:
"""Count consecutive lookup hits from the start of keys.
Returns the count of leading True results, or None if any lookup
returns None (retry-later signal).
"""
count = 0
for key in keys:
result = manager.lookup(key, _CTX)
if result is None:
return None
if not result:
break
count += 1
return count
class TestExampleSecondaryTier:
"""Tests for ExampleSecondaryTier implementation."""
def test_basic_store_and_lookup(self):
"""Test basic store and lookup operations."""
mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy())
tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
)
# Initially empty
blocks = to_keys(range(3))
assert tier.lookup(blocks[0], _CTX) is False
# Store blocks (simulate with direct insertion for testing)
tier.blocks[blocks[0]] = True
tier.blocks[blocks[1]] = True
# Lookup should find first two blocks
assert tier.lookup(blocks[0], _CTX) is True
assert tier.lookup(blocks[1], _CTX) is True
# Third block not present
assert tier.lookup(blocks[2], _CTX) is False
def test_lru_eviction(self):
"""Test LRU eviction policy."""
mock_view = memoryview(torch.zeros((4, 16), dtype=torch.int8).numpy())
tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=3
)
# Fill tier to capacity
blocks = to_keys(range(3))
for block in blocks:
tier.blocks[block] = True
assert tier.get_num_blocks() == 3
# Touch first block (make it most recently used)
tier.touch([blocks[0]], _CTX)
# Store new block should evict blocks[1] (least recently used)
new_block = to_keys([3])[0]
tier.submit_store(
JobMetadata(
job_id=1,
keys=[new_block],
block_ids=np.array([0], dtype=np.int64),
is_promotion=False,
req_context=_CTX,
)
)
# Complete the job
tier.get_finished()
# Verify new block is stored and blocks[1] was evicted (LRU)
assert new_block in tier.blocks
assert blocks[1] not in tier.blocks
# blocks[0] and blocks[2] should still be present
assert blocks[0] in tier.blocks
assert blocks[2] in tier.blocks
def test_async_simulation(self):
"""Test simulated async behavior."""
mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy())
tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG,
primary_kv_view=mock_view,
max_blocks=10,
simulate_async=True,
)
blocks = to_keys(range(2))
# Submit store job
tier.submit_store(
JobMetadata(
job_id=1,
keys=blocks,
block_ids=np.array([0, 1], dtype=np.int64),
is_promotion=False,
req_context=_CTX,
)
)
# Blocks should not yet be stored (pending async completion)
assert tier.get_num_blocks() == 0
# First get_finished() should complete the job
completed = list(tier.get_finished())
assert len(completed) == 1
assert completed[0].job_id == 1
assert completed[0].success is True
# Blocks should now be stored
assert tier.get_num_blocks() == 2
class TestTieringOffloadingManager:
"""Tests for TieringOffloadingManager."""
@pytest.fixture
def manager_setup(self):
# Create primary tier (CPU-based)
mock_region = _mock_mmap_region(5)
self.primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=5, mmap_region=mock_region
)
mock_view = mock_region.create_kv_memoryview()
# Create secondary tiers with the primary view
self.secondary_tier1 = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
)
self.secondary_tier2 = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
)
# Create tiered manager
self.manager = TieringOffloadingManager(
primary_tier=self.primary_tier,
secondary_tiers=[self.secondary_tier1, self.secondary_tier2],
)
def test_basic_store_to_primary(self, manager_setup):
"""Test basic store operation to primary tier."""
blocks = to_keys(range(3))
# Prepare store
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
assert len(result.keys_to_store) == 3
# Complete store
self.manager.complete_store(blocks, _CTX, success=True)
# Blocks should be in primary tier
assert count_hits(self.primary_tier, blocks) == 3
def test_cascade_to_all_secondary_tiers(self, manager_setup):
"""Test that blocks are cascaded to ALL secondary tiers."""
blocks = to_keys(range(3))
self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store
)
self.secondary_tier2.submit_store = MagicMock(
wraps=self.secondary_tier2.submit_store
)
# Store to primary
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
# Complete store (triggers cascade via submit_store on each tier)
self.manager.complete_store(blocks, _CTX, success=True)
# submit_store was called once per secondary tier
self.secondary_tier1.submit_store.assert_called_once()
self.secondary_tier2.submit_store.assert_called_once()
# Blocks should be in both secondary tiers
assert self.secondary_tier1.get_num_blocks() == 3
assert self.secondary_tier2.get_num_blocks() == 3
# Verify blocks are present
assert all(self.secondary_tier1.lookup(b, _CTX) for b in blocks)
assert all(self.secondary_tier2.lookup(b, _CTX) for b in blocks)
def test_ref_cnt_protection_during_cascade(self, manager_setup):
"""Test that ref_cnt protects blocks during cascade."""
blocks = to_keys(range(3))
# Store to primary
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
self.manager.complete_store(blocks, _CTX, success=True)
# After complete_store, blocks should have ref_cnt > 0
# (one for each secondary tier)
for block_hash in blocks:
block = self.primary_tier._policy.get(block_hash)
# ref_cnt should be 2 (one for each secondary tier)
assert block.ref_cnt == 2
# End of step 1: _maybe_process_finished_jobs() was already called by
# prepare_store() above (setting the per-step flag), so take_events()
# does NOT poll get_finished() again — cascade completions remain
# unprocessed until the next step.
list(self.manager.take_events())
# ref_cnt still held: cascade jobs finished (sync tier) but haven't
# been polled yet because the per-step guard skipped the second call.
for block_hash in blocks:
block = self.primary_tier._policy.get(block_hash)
assert block.ref_cnt == 2
# Secondary tiers have completed jobs waiting to be drained
assert len(self.secondary_tier1.completed_jobs) > 0
assert len(self.secondary_tier2.completed_jobs) > 0
# End of step 2: flag was reset, so _maybe_process_finished_jobs()
# runs and processes the cascade completions (complete_read → ref_cnt--)
list(self.manager.take_events())
# After cascade completes, ref_cnt should be 0
for block_hash in blocks:
block = self.primary_tier._policy.get(block_hash)
assert block.ref_cnt == 0
# All completed jobs have been drained
assert len(self.secondary_tier1.completed_jobs) == 0
assert len(self.secondary_tier2.completed_jobs) == 0
def test_lookup_from_primary(self, manager_setup):
"""Test lookup when blocks are in primary tier."""
blocks = to_keys(range(3))
# Store blocks
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
# Lookup should find all blocks in primary
assert count_hits(self.manager, blocks) == 3
def test_promotion_from_secondary(self, manager_setup):
"""Test promotion of blocks from secondary to primary tier."""
blocks = to_keys(range(3))
# Manually add blocks to secondary tier (simulate previous cascade)
for block in blocks:
self.secondary_tier1.blocks[block] = True
# Lookup each block to initiate promotion for all of them
for block in blocks:
result = self.manager.lookup(block, _CTX)
assert result is None # Retry later (promotion initiated)
# End of step 1: flushes deferred submit_load() calls
list(self.manager.take_events())
# End of step 2: processes the completed promotion jobs
list(self.manager.take_events())
# Now blocks should be in primary tier
assert count_hits(self.primary_tier, blocks) == 3
# Next lookup should succeed
assert count_hits(self.manager, blocks) == 3
def test_partial_lookup(self, manager_setup):
"""Test lookup with partial hits."""
blocks = to_keys(range(5))
# Store first 3 blocks to primary
self.manager.prepare_store(blocks[:3], _CTX)
self.manager.complete_store(blocks[:3], _CTX, success=True)
# Lookup all 5 blocks should return 3 (first 3 found)
assert count_hits(self.manager, blocks) == 3
def test_eviction_in_primary_tier(self, manager_setup):
"""Test eviction in primary tier when capacity is exceeded."""
# Primary tier has capacity of 5 blocks
# First, fill the primary tier
blocks = to_keys(range(5))
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
assert len(result.keys_to_store) == 5
self.manager.complete_store(blocks, _CTX, success=True)
# End of step: release ref_cnt from cascade
list(self.manager.take_events())
# Now try to store 2 more blocks (should trigger eviction)
more_blocks = to_keys(range(5, 7))
result = self.manager.prepare_store(more_blocks, _CTX)
# Should evict 2 blocks from primary tier
assert result is not None
assert len(result.evicted_keys) == 2
assert len(result.keys_to_store) == 2
def test_touch_propagates_to_all_tiers(self, manager_setup):
"""Test that touch() propagates to all tiers."""
blocks = to_keys(range(3))
# Store blocks
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
list(self.manager.take_events())
# Touch blocks
self.manager.touch(blocks, _CTX)
# Verify touch was called on primary tier (check LRU order)
# In LRU, touched blocks should be at the end
primary_keys = list(self.primary_tier._policy.blocks.keys())
assert primary_keys[-3:] == list(reversed(blocks))
# Verify touch was called on all secondary tiers
secondary1_keys = list(self.secondary_tier1.blocks.keys())
assert secondary1_keys[-3:] == list(reversed(blocks))
secondary2_keys = list(self.secondary_tier2.blocks.keys())
assert secondary2_keys[-3:] == list(reversed(blocks))
def test_failed_store_no_cascade(self, manager_setup):
"""Test that failed GPU→primary store doesn't cascade."""
blocks = to_keys(range(3))
self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store
)
self.secondary_tier2.submit_store = MagicMock(
wraps=self.secondary_tier2.submit_store
)
# Prepare store
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
# Complete store with failure — cascade must not happen
self.manager.complete_store(blocks, _CTX, success=False)
# submit_store was never called on either secondary tier
self.secondary_tier1.submit_store.assert_not_called()
self.secondary_tier2.submit_store.assert_not_called()
def test_multiple_secondary_tiers_independent_eviction(self):
"""Test that secondary tiers manage their own evictions."""
mock_region = _mock_mmap_region(10)
mock_view = mock_region.create_kv_memoryview()
# Create tier with small capacity
small_tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG,
primary_kv_view=mock_view,
max_blocks=5,
simulate_async=False,
)
large_tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG,
primary_kv_view=mock_view,
max_blocks=10,
simulate_async=False,
)
# Create a fresh primary tier for this test
primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=10, mmap_region=mock_region
)
manager = TieringOffloadingManager(
primary_tier=primary_tier,
secondary_tiers=[small_tier, large_tier],
)
# First, store 5 blocks to fill the small tier
blocks1 = to_keys(range(5))
result = manager.prepare_store(blocks1, _CTX)
assert result is not None
manager.complete_store(blocks1, _CTX, success=True)
list(manager.take_events())
# Both tiers should have 5 blocks
assert small_tier.get_num_blocks() == 5
assert large_tier.get_num_blocks() == 5
# Now store 3 more blocks - small tier should evict 3 blocks
blocks2 = to_keys(range(5, 8))
result = manager.prepare_store(blocks2, _CTX)
assert result is not None
manager.complete_store(blocks2, _CTX, success=True)
list(manager.take_events())
# Small tier should still have 5 blocks (evicted 3, added 3)
assert small_tier.get_num_blocks() == 5
# Large tier should have all 8 blocks
assert large_tier.get_num_blocks() == 8
def test_lookup_batches_submit_load_per_request(self, manager_setup):
"""lookup() defers submit_load until take_events(), one call per request.
Blocks from different requests each get their own submit_load call, each
carrying the correct req_context.
"""
blocks = to_keys(range(4))
for block in blocks:
self.secondary_tier1.blocks[block] = True
self.secondary_tier1.submit_load = MagicMock(
wraps=self.secondary_tier1.submit_load
)
ctx_a = ReqContext(req_id="req_a")
ctx_b = ReqContext(req_id="req_b")
# All lookups return None: secondary hit triggers promotion (in-flight)
assert self.manager.lookup(blocks[0], ctx_a) is None
assert self.manager.lookup(blocks[1], ctx_a) is None
assert self.manager.lookup(blocks[2], ctx_b) is None
assert self.manager.lookup(blocks[3], ctx_b) is None
# submit_load must not fire during lookup - only at end of step
self.secondary_tier1.submit_load.assert_not_called()
# simulate end of step
list(self.manager.take_events())
assert self.secondary_tier1.submit_load.call_count == 2
calls = self.secondary_tier1.submit_load.call_args_list
jm_a = calls[0].args[0]
jm_b = calls[1].args[0]
assert set(jm_a.keys) == {blocks[0], blocks[1]}
assert jm_a.req_context is ctx_a
assert set(jm_b.keys) == {blocks[2], blocks[3]}
assert jm_b.req_context is ctx_b
def test_lookup_shared_block_no_duplicate_promotion(self, manager_setup):
"""A block looked up by two requests in the same step is promoted once.
The first lookup initiates promotion (returns None via secondary hit).
The second lookup sees ref_cnt=-1 on the primary slot and returns None
via the primary in-flight path — without triggering a second promotion.
"""
shared_block = to_keys([0])[0]
self.secondary_tier1.blocks[shared_block] = True
self.secondary_tier1.submit_load = MagicMock(
wraps=self.secondary_tier1.submit_load
)
ctx_a = ReqContext(req_id="req_a")
ctx_b = ReqContext(req_id="req_b")
result_a = self.manager.lookup(shared_block, ctx_a)
result_b = self.manager.lookup(shared_block, ctx_b)
# Both see None (in-flight), but promotion is only queued once
assert result_a is None
assert result_b is None
list(self.manager.take_events())
# Only one submit_load call despite two lookups
self.secondary_tier1.submit_load.assert_called_once()
job_metadata = self.secondary_tier1.submit_load.call_args.args[0]
assert list(job_metadata.keys) == [shared_block]
assert job_metadata.req_context is ctx_a
def test_complete_store_forwards_req_context_to_submit_store(self, manager_setup):
"""complete_store cascades to secondary tiers with the correct req_context."""
blocks = to_keys(range(2))
self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store
)
ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"})
self.manager.prepare_store(blocks, ctx)
self.manager.complete_store(blocks, ctx, success=True)
assert self.secondary_tier1.submit_store.call_count == 1
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
assert job_metadata.req_context is ctx
class TestTieringOffloadingWithoutSecondaryTiers:
"""Test TieringOffloadingManager with no secondary tiers (backward compat)."""
def test_works_without_secondary_tiers(self):
"""Test that manager works with empty secondary_tiers list."""
primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=5, mmap_region=_mock_mmap_region(5)
)
# Create manager with no secondary tiers
manager = TieringOffloadingManager(
primary_tier=primary_tier, secondary_tiers=[]
)
blocks = to_keys(range(3))
# Should work like a regular OffloadingManager
result = manager.prepare_store(blocks, _CTX)
assert result is not None
manager.complete_store(blocks, _CTX, success=True)
assert count_hits(manager, blocks) == 3
if __name__ == "__main__":
pytest.main([__file__, "-v"])