[kv_offload] Add multi-tier KV cache offloading framework (#40020)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
@@ -292,3 +292,48 @@ def test_cpu_offloading(
|
||||
if subscriber is not None:
|
||||
subscriber.close()
|
||||
del llm
|
||||
|
||||
|
||||
def test_tiering_offloading() -> None:
|
||||
"""Tests OffloadingConnector with TieringOffloadingSpec."""
|
||||
extra_config: dict = {
|
||||
"cpu_bytes_to_use": 500 << 20,
|
||||
"block_size": 48,
|
||||
"spec_name": "TieringOffloadingSpec",
|
||||
"secondary_tiers": [{"type": "example"}],
|
||||
}
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="OffloadingConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config=extra_config,
|
||||
)
|
||||
|
||||
port: int
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("0.0.0.0", 0))
|
||||
port = s.getsockname()[1]
|
||||
events_endpoint = f"tcp://*:{port}"
|
||||
kv_events_config = KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
endpoint=events_endpoint,
|
||||
topic="test",
|
||||
)
|
||||
|
||||
llm = LLM(
|
||||
model="meta-llama/Llama-3.2-1B-Instruct",
|
||||
max_model_len=4096,
|
||||
gpu_memory_utilization=0.5,
|
||||
kv_events_config=kv_events_config,
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
)
|
||||
subscriber = MockSubscriber(
|
||||
events_endpoint.replace("*", "127.0.0.1"),
|
||||
topic=kv_events_config.topic,
|
||||
)
|
||||
try:
|
||||
_latency_test(llm, subscriber)
|
||||
_accuracy_test(llm, subscriber)
|
||||
finally:
|
||||
subscriber.close()
|
||||
del llm
|
||||
|
||||
@@ -262,6 +262,43 @@ def test_cpu_manager():
|
||||
)
|
||||
|
||||
|
||||
def test_prepare_load_preserves_key_order():
|
||||
"""block_ids[i] must correspond to keys[i] (co-indexed invariant)."""
|
||||
manager = CPUOffloadingManager(num_blocks=4, cache_policy="lru")
|
||||
|
||||
key_a, key_b, key_c = to_key(0), to_key(1), to_key(2)
|
||||
|
||||
# Store all three keys and learn their block ID assignments
|
||||
store_output = manager.prepare_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
||||
assert store_output is not None
|
||||
assert isinstance(store_output.store_spec, CPULoadStoreSpec)
|
||||
key_to_block_id = {
|
||||
k: int(bid)
|
||||
for k, bid in zip(store_output.keys_to_store, store_output.store_spec.block_ids)
|
||||
}
|
||||
manager.complete_store([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
||||
|
||||
# Forward order: [a, b, c]
|
||||
spec_fwd = manager.prepare_load([key_a, key_b, key_c], _EMPTY_REQ_CTX)
|
||||
assert isinstance(spec_fwd, CPULoadStoreSpec)
|
||||
assert [int(x) for x in spec_fwd.block_ids] == [
|
||||
key_to_block_id[key_a],
|
||||
key_to_block_id[key_b],
|
||||
key_to_block_id[key_c],
|
||||
]
|
||||
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
|
||||
|
||||
# Arbitrary permutation: [b, c, a]
|
||||
spec_perm = manager.prepare_load([key_b, key_c, key_a], _EMPTY_REQ_CTX)
|
||||
assert isinstance(spec_perm, CPULoadStoreSpec)
|
||||
assert [int(x) for x in spec_perm.block_ids] == [
|
||||
key_to_block_id[key_b],
|
||||
key_to_block_id[key_c],
|
||||
key_to_block_id[key_a],
|
||||
]
|
||||
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
|
||||
|
||||
|
||||
class TestARCPolicy:
|
||||
"""Unit tests for CPUOffloadingManager with ARC eviction policy."""
|
||||
|
||||
|
||||
@@ -0,0 +1,565 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for TieringOffloadingManager and ExampleSecondaryTier.
|
||||
|
||||
These tests verify:
|
||||
1. Basic tiered offloading operations (store, load, lookup)
|
||||
2. Cascade behavior (blocks stored to all secondary tiers)
|
||||
3. Promotion behavior (blocks loaded from secondary to primary to GPU)
|
||||
4. ref_cnt management (blocks protected during async transfers)
|
||||
5. Eviction coordination between tiers
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import JobMetadata
|
||||
from vllm.v1.kv_offload.tiering.example import ExampleSecondaryTier
|
||||
from vllm.v1.kv_offload.tiering.manager import (
|
||||
CPUPrimaryTierOffloadingManager,
|
||||
TieringOffloadingManager,
|
||||
)
|
||||
|
||||
_CTX = ReqContext(req_id="test")
|
||||
_MOCK_VLLM_CONFIG = MagicMock()
|
||||
|
||||
|
||||
def _mock_mmap_region(num_blocks: int, row_bytes: int = 16):
|
||||
"""Create a mock SharedOffloadRegion for testing."""
|
||||
mock = MagicMock()
|
||||
view = memoryview(torch.zeros((num_blocks, row_bytes), dtype=torch.int8).numpy())
|
||||
mock.create_kv_memoryview.return_value = view
|
||||
return mock
|
||||
|
||||
|
||||
def to_keys(int_ids: Iterable[int]) -> list[OffloadKey]:
|
||||
return [make_offload_key(str(i).encode(), 0) for i in int_ids]
|
||||
|
||||
|
||||
def count_hits(manager, keys: list[OffloadKey]) -> int | None:
|
||||
"""Count consecutive lookup hits from the start of keys.
|
||||
|
||||
Returns the count of leading True results, or None if any lookup
|
||||
returns None (retry-later signal).
|
||||
"""
|
||||
count = 0
|
||||
for key in keys:
|
||||
result = manager.lookup(key, _CTX)
|
||||
if result is None:
|
||||
return None
|
||||
if not result:
|
||||
break
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
class TestExampleSecondaryTier:
|
||||
"""Tests for ExampleSecondaryTier implementation."""
|
||||
|
||||
def test_basic_store_and_lookup(self):
|
||||
"""Test basic store and lookup operations."""
|
||||
mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy())
|
||||
tier = ExampleSecondaryTier(
|
||||
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
|
||||
)
|
||||
|
||||
# Initially empty
|
||||
blocks = to_keys(range(3))
|
||||
assert tier.lookup(blocks[0], _CTX) is False
|
||||
|
||||
# Store blocks (simulate with direct insertion for testing)
|
||||
tier.blocks[blocks[0]] = True
|
||||
tier.blocks[blocks[1]] = True
|
||||
|
||||
# Lookup should find first two blocks
|
||||
assert tier.lookup(blocks[0], _CTX) is True
|
||||
assert tier.lookup(blocks[1], _CTX) is True
|
||||
|
||||
# Third block not present
|
||||
assert tier.lookup(blocks[2], _CTX) is False
|
||||
|
||||
def test_lru_eviction(self):
|
||||
"""Test LRU eviction policy."""
|
||||
mock_view = memoryview(torch.zeros((4, 16), dtype=torch.int8).numpy())
|
||||
tier = ExampleSecondaryTier(
|
||||
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=3
|
||||
)
|
||||
|
||||
# Fill tier to capacity
|
||||
blocks = to_keys(range(3))
|
||||
for block in blocks:
|
||||
tier.blocks[block] = True
|
||||
|
||||
assert tier.get_num_blocks() == 3
|
||||
|
||||
# Touch first block (make it most recently used)
|
||||
tier.touch([blocks[0]], _CTX)
|
||||
|
||||
# Store new block should evict blocks[1] (least recently used)
|
||||
new_block = to_keys([3])[0]
|
||||
|
||||
tier.submit_store(
|
||||
JobMetadata(
|
||||
job_id=1,
|
||||
keys=[new_block],
|
||||
block_ids=np.array([0], dtype=np.int64),
|
||||
is_promotion=False,
|
||||
req_context=_CTX,
|
||||
)
|
||||
)
|
||||
|
||||
# Complete the job
|
||||
tier.get_finished()
|
||||
|
||||
# Verify new block is stored and blocks[1] was evicted (LRU)
|
||||
assert new_block in tier.blocks
|
||||
assert blocks[1] not in tier.blocks
|
||||
# blocks[0] and blocks[2] should still be present
|
||||
assert blocks[0] in tier.blocks
|
||||
assert blocks[2] in tier.blocks
|
||||
|
||||
def test_async_simulation(self):
|
||||
"""Test simulated async behavior."""
|
||||
mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy())
|
||||
tier = ExampleSecondaryTier(
|
||||
vllm_config=_MOCK_VLLM_CONFIG,
|
||||
primary_kv_view=mock_view,
|
||||
max_blocks=10,
|
||||
simulate_async=True,
|
||||
)
|
||||
|
||||
blocks = to_keys(range(2))
|
||||
|
||||
# Submit store job
|
||||
tier.submit_store(
|
||||
JobMetadata(
|
||||
job_id=1,
|
||||
keys=blocks,
|
||||
block_ids=np.array([0, 1], dtype=np.int64),
|
||||
is_promotion=False,
|
||||
req_context=_CTX,
|
||||
)
|
||||
)
|
||||
|
||||
# Blocks should not yet be stored (pending async completion)
|
||||
assert tier.get_num_blocks() == 0
|
||||
|
||||
# First get_finished() should complete the job
|
||||
completed = list(tier.get_finished())
|
||||
assert len(completed) == 1
|
||||
assert completed[0].job_id == 1
|
||||
assert completed[0].success is True
|
||||
|
||||
# Blocks should now be stored
|
||||
assert tier.get_num_blocks() == 2
|
||||
|
||||
|
||||
class TestTieringOffloadingManager:
|
||||
"""Tests for TieringOffloadingManager."""
|
||||
|
||||
@pytest.fixture
|
||||
def manager_setup(self):
|
||||
# Create primary tier (CPU-based)
|
||||
mock_region = _mock_mmap_region(5)
|
||||
self.primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=5, mmap_region=mock_region
|
||||
)
|
||||
|
||||
mock_view = mock_region.create_kv_memoryview()
|
||||
|
||||
# Create secondary tiers with the primary view
|
||||
self.secondary_tier1 = ExampleSecondaryTier(
|
||||
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
|
||||
)
|
||||
self.secondary_tier2 = ExampleSecondaryTier(
|
||||
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
|
||||
)
|
||||
|
||||
# Create tiered manager
|
||||
self.manager = TieringOffloadingManager(
|
||||
primary_tier=self.primary_tier,
|
||||
secondary_tiers=[self.secondary_tier1, self.secondary_tier2],
|
||||
)
|
||||
|
||||
def test_basic_store_to_primary(self, manager_setup):
|
||||
"""Test basic store operation to primary tier."""
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Prepare store
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
assert len(result.keys_to_store) == 3
|
||||
|
||||
# Complete store
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
|
||||
# Blocks should be in primary tier
|
||||
assert count_hits(self.primary_tier, blocks) == 3
|
||||
|
||||
def test_cascade_to_all_secondary_tiers(self, manager_setup):
|
||||
"""Test that blocks are cascaded to ALL secondary tiers."""
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_store
|
||||
)
|
||||
self.secondary_tier2.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier2.submit_store
|
||||
)
|
||||
|
||||
# Store to primary
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
|
||||
# Complete store (triggers cascade via submit_store on each tier)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
|
||||
# submit_store was called once per secondary tier
|
||||
self.secondary_tier1.submit_store.assert_called_once()
|
||||
self.secondary_tier2.submit_store.assert_called_once()
|
||||
|
||||
# Blocks should be in both secondary tiers
|
||||
assert self.secondary_tier1.get_num_blocks() == 3
|
||||
assert self.secondary_tier2.get_num_blocks() == 3
|
||||
|
||||
# Verify blocks are present
|
||||
assert all(self.secondary_tier1.lookup(b, _CTX) for b in blocks)
|
||||
assert all(self.secondary_tier2.lookup(b, _CTX) for b in blocks)
|
||||
|
||||
def test_ref_cnt_protection_during_cascade(self, manager_setup):
|
||||
"""Test that ref_cnt protects blocks during cascade."""
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Store to primary
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
|
||||
# After complete_store, blocks should have ref_cnt > 0
|
||||
# (one for each secondary tier)
|
||||
for block_hash in blocks:
|
||||
block = self.primary_tier._policy.get(block_hash)
|
||||
# ref_cnt should be 2 (one for each secondary tier)
|
||||
assert block.ref_cnt == 2
|
||||
|
||||
# End of step 1: _maybe_process_finished_jobs() was already called by
|
||||
# prepare_store() above (setting the per-step flag), so take_events()
|
||||
# does NOT poll get_finished() again — cascade completions remain
|
||||
# unprocessed until the next step.
|
||||
list(self.manager.take_events())
|
||||
|
||||
# ref_cnt still held: cascade jobs finished (sync tier) but haven't
|
||||
# been polled yet because the per-step guard skipped the second call.
|
||||
for block_hash in blocks:
|
||||
block = self.primary_tier._policy.get(block_hash)
|
||||
assert block.ref_cnt == 2
|
||||
|
||||
# Secondary tiers have completed jobs waiting to be drained
|
||||
assert len(self.secondary_tier1.completed_jobs) > 0
|
||||
assert len(self.secondary_tier2.completed_jobs) > 0
|
||||
|
||||
# End of step 2: flag was reset, so _maybe_process_finished_jobs()
|
||||
# runs and processes the cascade completions (complete_read → ref_cnt--)
|
||||
list(self.manager.take_events())
|
||||
|
||||
# After cascade completes, ref_cnt should be 0
|
||||
for block_hash in blocks:
|
||||
block = self.primary_tier._policy.get(block_hash)
|
||||
assert block.ref_cnt == 0
|
||||
|
||||
# All completed jobs have been drained
|
||||
assert len(self.secondary_tier1.completed_jobs) == 0
|
||||
assert len(self.secondary_tier2.completed_jobs) == 0
|
||||
|
||||
def test_lookup_from_primary(self, manager_setup):
|
||||
"""Test lookup when blocks are in primary tier."""
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Store blocks
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
|
||||
# Lookup should find all blocks in primary
|
||||
assert count_hits(self.manager, blocks) == 3
|
||||
|
||||
def test_promotion_from_secondary(self, manager_setup):
|
||||
"""Test promotion of blocks from secondary to primary tier."""
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Manually add blocks to secondary tier (simulate previous cascade)
|
||||
for block in blocks:
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
|
||||
# Lookup each block to initiate promotion for all of them
|
||||
for block in blocks:
|
||||
result = self.manager.lookup(block, _CTX)
|
||||
assert result is None # Retry later (promotion initiated)
|
||||
|
||||
# End of step 1: flushes deferred submit_load() calls
|
||||
list(self.manager.take_events())
|
||||
|
||||
# End of step 2: processes the completed promotion jobs
|
||||
list(self.manager.take_events())
|
||||
|
||||
# Now blocks should be in primary tier
|
||||
assert count_hits(self.primary_tier, blocks) == 3
|
||||
|
||||
# Next lookup should succeed
|
||||
assert count_hits(self.manager, blocks) == 3
|
||||
|
||||
def test_partial_lookup(self, manager_setup):
|
||||
"""Test lookup with partial hits."""
|
||||
blocks = to_keys(range(5))
|
||||
|
||||
# Store first 3 blocks to primary
|
||||
self.manager.prepare_store(blocks[:3], _CTX)
|
||||
self.manager.complete_store(blocks[:3], _CTX, success=True)
|
||||
|
||||
# Lookup all 5 blocks should return 3 (first 3 found)
|
||||
assert count_hits(self.manager, blocks) == 3
|
||||
|
||||
def test_eviction_in_primary_tier(self, manager_setup):
|
||||
"""Test eviction in primary tier when capacity is exceeded."""
|
||||
# Primary tier has capacity of 5 blocks
|
||||
# First, fill the primary tier
|
||||
blocks = to_keys(range(5))
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
assert len(result.keys_to_store) == 5
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
|
||||
# End of step: release ref_cnt from cascade
|
||||
list(self.manager.take_events())
|
||||
|
||||
# Now try to store 2 more blocks (should trigger eviction)
|
||||
more_blocks = to_keys(range(5, 7))
|
||||
result = self.manager.prepare_store(more_blocks, _CTX)
|
||||
|
||||
# Should evict 2 blocks from primary tier
|
||||
assert result is not None
|
||||
assert len(result.evicted_keys) == 2
|
||||
assert len(result.keys_to_store) == 2
|
||||
|
||||
def test_touch_propagates_to_all_tiers(self, manager_setup):
|
||||
"""Test that touch() propagates to all tiers."""
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Store blocks
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
list(self.manager.take_events())
|
||||
|
||||
# Touch blocks
|
||||
self.manager.touch(blocks, _CTX)
|
||||
|
||||
# Verify touch was called on primary tier (check LRU order)
|
||||
# In LRU, touched blocks should be at the end
|
||||
primary_keys = list(self.primary_tier._policy.blocks.keys())
|
||||
assert primary_keys[-3:] == list(reversed(blocks))
|
||||
|
||||
# Verify touch was called on all secondary tiers
|
||||
secondary1_keys = list(self.secondary_tier1.blocks.keys())
|
||||
assert secondary1_keys[-3:] == list(reversed(blocks))
|
||||
|
||||
secondary2_keys = list(self.secondary_tier2.blocks.keys())
|
||||
assert secondary2_keys[-3:] == list(reversed(blocks))
|
||||
|
||||
def test_failed_store_no_cascade(self, manager_setup):
|
||||
"""Test that failed GPU→primary store doesn't cascade."""
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_store
|
||||
)
|
||||
self.secondary_tier2.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier2.submit_store
|
||||
)
|
||||
|
||||
# Prepare store
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
|
||||
# Complete store with failure — cascade must not happen
|
||||
self.manager.complete_store(blocks, _CTX, success=False)
|
||||
|
||||
# submit_store was never called on either secondary tier
|
||||
self.secondary_tier1.submit_store.assert_not_called()
|
||||
self.secondary_tier2.submit_store.assert_not_called()
|
||||
|
||||
def test_multiple_secondary_tiers_independent_eviction(self):
|
||||
"""Test that secondary tiers manage their own evictions."""
|
||||
mock_region = _mock_mmap_region(10)
|
||||
mock_view = mock_region.create_kv_memoryview()
|
||||
|
||||
# Create tier with small capacity
|
||||
small_tier = ExampleSecondaryTier(
|
||||
vllm_config=_MOCK_VLLM_CONFIG,
|
||||
primary_kv_view=mock_view,
|
||||
max_blocks=5,
|
||||
simulate_async=False,
|
||||
)
|
||||
large_tier = ExampleSecondaryTier(
|
||||
vllm_config=_MOCK_VLLM_CONFIG,
|
||||
primary_kv_view=mock_view,
|
||||
max_blocks=10,
|
||||
simulate_async=False,
|
||||
)
|
||||
|
||||
# Create a fresh primary tier for this test
|
||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=10, mmap_region=mock_region
|
||||
)
|
||||
|
||||
manager = TieringOffloadingManager(
|
||||
primary_tier=primary_tier,
|
||||
secondary_tiers=[small_tier, large_tier],
|
||||
)
|
||||
|
||||
# First, store 5 blocks to fill the small tier
|
||||
blocks1 = to_keys(range(5))
|
||||
result = manager.prepare_store(blocks1, _CTX)
|
||||
assert result is not None
|
||||
manager.complete_store(blocks1, _CTX, success=True)
|
||||
list(manager.take_events())
|
||||
|
||||
# Both tiers should have 5 blocks
|
||||
assert small_tier.get_num_blocks() == 5
|
||||
assert large_tier.get_num_blocks() == 5
|
||||
|
||||
# Now store 3 more blocks - small tier should evict 3 blocks
|
||||
blocks2 = to_keys(range(5, 8))
|
||||
result = manager.prepare_store(blocks2, _CTX)
|
||||
assert result is not None
|
||||
manager.complete_store(blocks2, _CTX, success=True)
|
||||
list(manager.take_events())
|
||||
|
||||
# Small tier should still have 5 blocks (evicted 3, added 3)
|
||||
assert small_tier.get_num_blocks() == 5
|
||||
|
||||
# Large tier should have all 8 blocks
|
||||
assert large_tier.get_num_blocks() == 8
|
||||
|
||||
def test_lookup_batches_submit_load_per_request(self, manager_setup):
|
||||
"""lookup() defers submit_load until take_events(), one call per request.
|
||||
|
||||
Blocks from different requests each get their own submit_load call, each
|
||||
carrying the correct req_context.
|
||||
"""
|
||||
blocks = to_keys(range(4))
|
||||
for block in blocks:
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
|
||||
self.secondary_tier1.submit_load = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_load
|
||||
)
|
||||
|
||||
ctx_a = ReqContext(req_id="req_a")
|
||||
ctx_b = ReqContext(req_id="req_b")
|
||||
|
||||
# All lookups return None: secondary hit triggers promotion (in-flight)
|
||||
assert self.manager.lookup(blocks[0], ctx_a) is None
|
||||
assert self.manager.lookup(blocks[1], ctx_a) is None
|
||||
assert self.manager.lookup(blocks[2], ctx_b) is None
|
||||
assert self.manager.lookup(blocks[3], ctx_b) is None
|
||||
|
||||
# submit_load must not fire during lookup - only at end of step
|
||||
self.secondary_tier1.submit_load.assert_not_called()
|
||||
|
||||
# simulate end of step
|
||||
list(self.manager.take_events())
|
||||
|
||||
assert self.secondary_tier1.submit_load.call_count == 2
|
||||
calls = self.secondary_tier1.submit_load.call_args_list
|
||||
jm_a = calls[0].args[0]
|
||||
jm_b = calls[1].args[0]
|
||||
assert set(jm_a.keys) == {blocks[0], blocks[1]}
|
||||
assert jm_a.req_context is ctx_a
|
||||
assert set(jm_b.keys) == {blocks[2], blocks[3]}
|
||||
assert jm_b.req_context is ctx_b
|
||||
|
||||
def test_lookup_shared_block_no_duplicate_promotion(self, manager_setup):
|
||||
"""A block looked up by two requests in the same step is promoted once.
|
||||
|
||||
The first lookup initiates promotion (returns None via secondary hit).
|
||||
The second lookup sees ref_cnt=-1 on the primary slot and returns None
|
||||
via the primary in-flight path — without triggering a second promotion.
|
||||
"""
|
||||
shared_block = to_keys([0])[0]
|
||||
self.secondary_tier1.blocks[shared_block] = True
|
||||
|
||||
self.secondary_tier1.submit_load = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_load
|
||||
)
|
||||
|
||||
ctx_a = ReqContext(req_id="req_a")
|
||||
ctx_b = ReqContext(req_id="req_b")
|
||||
|
||||
result_a = self.manager.lookup(shared_block, ctx_a)
|
||||
result_b = self.manager.lookup(shared_block, ctx_b)
|
||||
|
||||
# Both see None (in-flight), but promotion is only queued once
|
||||
assert result_a is None
|
||||
assert result_b is None
|
||||
|
||||
list(self.manager.take_events())
|
||||
|
||||
# Only one submit_load call despite two lookups
|
||||
self.secondary_tier1.submit_load.assert_called_once()
|
||||
job_metadata = self.secondary_tier1.submit_load.call_args.args[0]
|
||||
assert list(job_metadata.keys) == [shared_block]
|
||||
assert job_metadata.req_context is ctx_a
|
||||
|
||||
def test_complete_store_forwards_req_context_to_submit_store(self, manager_setup):
|
||||
"""complete_store cascades to secondary tiers with the correct req_context."""
|
||||
blocks = to_keys(range(2))
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_store
|
||||
)
|
||||
|
||||
ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"})
|
||||
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.complete_store(blocks, ctx, success=True)
|
||||
|
||||
assert self.secondary_tier1.submit_store.call_count == 1
|
||||
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
||||
assert job_metadata.req_context is ctx
|
||||
|
||||
|
||||
class TestTieringOffloadingWithoutSecondaryTiers:
|
||||
"""Test TieringOffloadingManager with no secondary tiers (backward compat)."""
|
||||
|
||||
def test_works_without_secondary_tiers(self):
|
||||
"""Test that manager works with empty secondary_tiers list."""
|
||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=5, mmap_region=_mock_mmap_region(5)
|
||||
)
|
||||
|
||||
# Create manager with no secondary tiers
|
||||
manager = TieringOffloadingManager(
|
||||
primary_tier=primary_tier, secondary_tiers=[]
|
||||
)
|
||||
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Should work like a regular OffloadingManager
|
||||
result = manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
manager.complete_store(blocks, _CTX, success=True)
|
||||
|
||||
assert count_hits(manager, blocks) == 3
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -4,8 +4,6 @@
|
||||
Core abstractions for KV cache offloading in vLLM v1.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Collection, Iterable, Iterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
@@ -201,7 +199,7 @@ class OffloadingManager(ABC):
|
||||
"""
|
||||
Marks blocks which were previously prepared to be stored, as stored.
|
||||
Following this call, the blocks become loadable.
|
||||
If if_success is False, blocks that were not marked as stored will be
|
||||
If success is False, blocks that were not marked as stored will be
|
||||
removed.
|
||||
|
||||
Args:
|
||||
@@ -328,7 +326,7 @@ class CanonicalKVCaches:
|
||||
class OffloadingSpec(ABC):
|
||||
"""Spec for an offloading connector"""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig):
|
||||
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
|
||||
logger.warning(
|
||||
"Initializing OffloadingSpec. This API is experimental and "
|
||||
"subject to change in the future as we iterate the design."
|
||||
@@ -394,7 +392,7 @@ class OffloadingSpec(ABC):
|
||||
@abstractmethod
|
||||
def get_handlers(
|
||||
self, kv_caches: CanonicalKVCaches
|
||||
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
|
||||
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], "OffloadingHandler"]]:
|
||||
"""
|
||||
Get offloading handlers along with their respective src and dst types.
|
||||
|
||||
|
||||
@@ -124,6 +124,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
block_size_factor: int,
|
||||
kv_cache_groups_data_refs: list[list[CanonicalKVCacheRef]],
|
||||
gpu_to_cpu: bool,
|
||||
mmap_region: SharedOffloadRegion | None = None,
|
||||
):
|
||||
"""
|
||||
Initialize a SingleDirectionOffloadingHandler.
|
||||
@@ -167,6 +168,8 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
self.dst_block_size_factor = block_size_factor if self.gpu_to_cpu else 1
|
||||
|
||||
self.transfer_type = ("GPU", "CPU") if self.gpu_to_cpu else ("CPU", "GPU")
|
||||
# mmap_region to clean up on shutdown (gpu_to_cpu handler owns it)
|
||||
self._mmap_region = mmap_region
|
||||
# job_id -> event
|
||||
self._transfer_events: dict[int, torch.Event] = {}
|
||||
# queue of transfers (job_id, stream, event)
|
||||
@@ -382,6 +385,9 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
self._event_pool.clear()
|
||||
self.src_tensors.clear()
|
||||
self.dst_tensors.clear()
|
||||
if self._mmap_region is not None:
|
||||
self._mmap_region.cleanup()
|
||||
self._mmap_region = None
|
||||
|
||||
|
||||
class CpuGpuOffloadingHandlers:
|
||||
@@ -434,6 +440,7 @@ class CpuGpuOffloadingHandlers:
|
||||
block_size_factor=block_size_factor,
|
||||
kv_cache_groups_data_refs=kv_caches.group_data_refs,
|
||||
gpu_to_cpu=True,
|
||||
mmap_region=mmap_region,
|
||||
)
|
||||
|
||||
self.cpu_to_gpu_handler = SingleDirectionOffloadingHandler(
|
||||
|
||||
@@ -153,6 +153,20 @@ class SharedOffloadRegion:
|
||||
self._views.append(worker_layer_view)
|
||||
return worker_layer_view
|
||||
|
||||
def create_kv_memoryview(self) -> memoryview:
|
||||
"""Return a zero-copy memoryview over the entire KV buffer.
|
||||
|
||||
Shape: (num_blocks, row_stride_bytes). Secondary tiers address
|
||||
block *b* as ``view[b]``.
|
||||
"""
|
||||
kv_tensor = self._base.view(self.num_blocks, self._row_stride)
|
||||
np_arr = kv_tensor.numpy()
|
||||
assert np_arr.ctypes.data == self._base.data_ptr(), (
|
||||
"view()/numpy() created a copy instead of sharing the mmap buffer; "
|
||||
"secondary tiers require zero-copy access to primary KV data"
|
||||
)
|
||||
return memoryview(np_arr)
|
||||
|
||||
def cleanup(self) -> None:
|
||||
if self.is_pinned and self._base is not None:
|
||||
base_ptr = self._base.data_ptr()
|
||||
|
||||
@@ -44,6 +44,10 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
if kv_bytes_per_offloaded_block > 0
|
||||
else 0
|
||||
)
|
||||
world_size = vllm_config.parallel_config.world_size
|
||||
self.cpu_page_size_per_worker: int = (
|
||||
kv_bytes_per_offloaded_block // world_size if world_size > 0 else 0
|
||||
)
|
||||
|
||||
# scheduler-side
|
||||
self._manager: OffloadingManager | None = None
|
||||
@@ -77,6 +81,15 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
)
|
||||
return self._manager
|
||||
|
||||
def _create_handlers(
|
||||
self, kv_caches: CanonicalKVCaches
|
||||
) -> CpuGpuOffloadingHandlers:
|
||||
return CpuGpuOffloadingHandlers(
|
||||
kv_caches=kv_caches,
|
||||
block_size_factor=self.block_size_factor,
|
||||
num_cpu_blocks=self.num_blocks,
|
||||
)
|
||||
|
||||
def get_handlers(
|
||||
self, kv_caches: CanonicalKVCaches
|
||||
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
|
||||
@@ -85,12 +98,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
raise Exception(
|
||||
"CPU Offloading is currently only supported on CUDA-alike GPUs"
|
||||
)
|
||||
|
||||
self._handlers = CpuGpuOffloadingHandlers(
|
||||
kv_caches=kv_caches,
|
||||
block_size_factor=self.block_size_factor,
|
||||
num_cpu_blocks=self.num_blocks,
|
||||
)
|
||||
self._handlers = self._create_handlers(kv_caches)
|
||||
|
||||
assert self._handlers is not None
|
||||
yield GPULoadStoreSpec, CPULoadStoreSpec, self._handlers.gpu_to_cpu_handler
|
||||
|
||||
@@ -56,3 +56,8 @@ class OffloadingSpecFactory:
|
||||
OffloadingSpecFactory.register_spec(
|
||||
"CPUOffloadingSpec", "vllm.v1.kv_offload.cpu.spec", "CPUOffloadingSpec"
|
||||
)
|
||||
OffloadingSpecFactory.register_spec(
|
||||
"TieringOffloadingSpec",
|
||||
"vllm.v1.kv_offload.tiering.spec",
|
||||
"TieringOffloadingSpec",
|
||||
)
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Abstract interfaces and data types for the secondary tiering layer.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Collection, Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
# Type alias for job IDs used in async transfer tracking
|
||||
JobId = int
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobMetadata:
|
||||
"""Metadata for an in-flight async transfer job."""
|
||||
|
||||
job_id: JobId
|
||||
keys: Collection[OffloadKey]
|
||||
block_ids: np.ndarray
|
||||
is_promotion: bool
|
||||
req_context: ReqContext
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobResult:
|
||||
"""Result of an async transfer job (successful or failed)."""
|
||||
|
||||
job_id: JobId
|
||||
success: bool
|
||||
|
||||
|
||||
class SecondaryTierManager(ABC):
|
||||
"""
|
||||
Abstract interface for managing a single non-primary offloading tier.
|
||||
|
||||
Secondary tiers cannot directly access GPU memory. All data transfers
|
||||
must go through the CPU (primary) tier:
|
||||
- Store: GPU → CPU (primary) → secondary (cascade)
|
||||
- Load: secondary → CPU (primary) → GPU (promotion)
|
||||
|
||||
IMPORTANT: All methods run in the Scheduler process and must be
|
||||
lightweight and non-blocking. submit_load() and submit_store() submit
|
||||
async jobs; get_finished() polls for completion.
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig", primary_kv_view: memoryview) -> None:
|
||||
self._vllm_config = vllm_config
|
||||
self._primary_kv_view: memoryview = primary_kv_view
|
||||
|
||||
@abstractmethod
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
"""
|
||||
Check whether a block exists in this secondary tier.
|
||||
|
||||
Args:
|
||||
key: Offload key to look up.
|
||||
req_context: per-request context (e.g. kv_transfer_params).
|
||||
|
||||
Returns:
|
||||
True if the block is present and ready,
|
||||
False if not found,
|
||||
or None if the block is being transferred (retry later).
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
||||
"""
|
||||
Submit an async job to store blocks from the primary tier to this
|
||||
secondary tier.
|
||||
|
||||
This method must be lightweight and non-blocking: allocate metadata
|
||||
and submit the transfer, but do NOT perform the data copy on the
|
||||
calling thread.
|
||||
|
||||
Preconditions (guaranteed by the framework):
|
||||
- ``job_metadata.block_ids`` are valid primary-tier slots, pinned
|
||||
(ref-counted) for the duration of the transfer.
|
||||
|
||||
The implementation is responsible for:
|
||||
1. Filtering out blocks already present in this tier
|
||||
2. Evicting blocks if capacity is needed
|
||||
3. Allocating space in this tier
|
||||
4. Submitting the async transfer (read from primary via block_ids)
|
||||
|
||||
Report completion via ``get_finished()``.
|
||||
|
||||
Args:
|
||||
job_metadata: Job metadata including job_id, keys, and block_ids
|
||||
identifying the primary-tier slots to read from.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def submit_load(self, job_metadata: JobMetadata) -> None:
|
||||
"""
|
||||
Submit an async job to load blocks from this secondary tier to the
|
||||
primary tier.
|
||||
|
||||
This method must be lightweight and non-blocking: mark blocks as
|
||||
in-flight and submit the transfer, but do NOT perform the data copy
|
||||
on the calling thread.
|
||||
|
||||
Preconditions (guaranteed by the framework):
|
||||
- ``job_metadata.block_ids`` are allocated primary-tier slots
|
||||
ready to receive data.
|
||||
|
||||
The implementation must copy data from this tier into the
|
||||
primary-tier slots identified by ``block_ids``.
|
||||
|
||||
Report completion via ``get_finished()``.
|
||||
|
||||
Args:
|
||||
job_metadata: Job metadata including job_id, keys, and block_ids
|
||||
identifying the primary-tier slots to write into.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_finished(self) -> Iterable[JobResult]:
|
||||
"""
|
||||
Return all jobs (loads and stores) that completed since the last call.
|
||||
|
||||
The framework uses these results to release resources and finalize
|
||||
transfers.
|
||||
|
||||
Returns:
|
||||
Iterable of JobResult objects for jobs finished since the
|
||||
last call.
|
||||
"""
|
||||
pass
|
||||
|
||||
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
||||
"""
|
||||
Mark blocks as recently used for eviction policy.
|
||||
|
||||
Args:
|
||||
keys: Offload keys to mark as recently used.
|
||||
req_context: Per-request context.
|
||||
"""
|
||||
return
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Release resources held by this tier (threads, connections, etc.)."""
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_tier_type() -> str:
|
||||
"""
|
||||
Get the type identifier of this tier (e.g., "example", "storage").
|
||||
|
||||
Must match the "type" field in the tier config dict.
|
||||
|
||||
Returns:
|
||||
Tier type string.
|
||||
"""
|
||||
pass
|
||||
@@ -0,0 +1,235 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
ExampleSecondaryTier: A simple in-memory secondary tier for testing.
|
||||
|
||||
This implementation provides a minimal secondary tier that stores blocks
|
||||
in memory (using a dictionary) and simulates async transfers with immediate
|
||||
completion. It's useful for testing the TieringOffloadingManager without
|
||||
requiring actual storage or network backends.
|
||||
"""
|
||||
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Collection, Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobId,
|
||||
JobMetadata,
|
||||
JobResult,
|
||||
SecondaryTierManager,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class _JobMetadata:
|
||||
"""Internal metadata for tracking job details."""
|
||||
|
||||
job_id: JobId
|
||||
keys: Collection[OffloadKey]
|
||||
is_store: bool # True for store jobs, False for load jobs
|
||||
|
||||
|
||||
class ExampleSecondaryTier(SecondaryTierManager):
|
||||
"""
|
||||
A simple in-memory secondary tier for testing.
|
||||
|
||||
This implementation:
|
||||
- Stores blocks in a dictionary (key -> True)
|
||||
- Simulates async transfers with immediate completion
|
||||
- Uses LRU eviction policy
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
primary_kv_view: memoryview,
|
||||
max_blocks: int = 1000,
|
||||
simulate_async: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the example secondary tier.
|
||||
|
||||
Args:
|
||||
vllm_config: Global vLLM configuration.
|
||||
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
|
||||
max_blocks: Maximum number of blocks this tier can store
|
||||
simulate_async: If True, jobs complete on next get_finished() call.
|
||||
If False, jobs complete immediately.
|
||||
"""
|
||||
super().__init__(vllm_config, primary_kv_view)
|
||||
self.max_blocks = max_blocks
|
||||
self.simulate_async = simulate_async
|
||||
|
||||
# key -> True (only care about presence)
|
||||
self.blocks: OrderedDict[OffloadKey, bool] = OrderedDict()
|
||||
|
||||
# Completed jobs waiting to be retrieved by get_finished()
|
||||
self.completed_jobs: list[JobResult] = []
|
||||
|
||||
# Pending jobs (for simulated async mode)
|
||||
self.pending_jobs: list[_JobMetadata] = []
|
||||
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
"""
|
||||
Check whether a block exists in this secondary tier.
|
||||
|
||||
Args:
|
||||
key: Offload key to look up.
|
||||
req_context: Per-request context.
|
||||
|
||||
Returns:
|
||||
True if the block is present, False if not found.
|
||||
"""
|
||||
return key in self.blocks
|
||||
|
||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
||||
"""
|
||||
Submit an async job to store blocks from primary tier to this tier.
|
||||
|
||||
Args:
|
||||
job_metadata: Job metadata including job_id, keys, and
|
||||
spec for reading blocks from the primary tier.
|
||||
"""
|
||||
job_id = job_metadata.job_id
|
||||
keys = job_metadata.keys
|
||||
block_ids = job_metadata.block_ids
|
||||
|
||||
assert len(keys) == len(block_ids), (
|
||||
f"Length mismatch: {len(keys)} keys but {len(block_ids)} block_ids"
|
||||
)
|
||||
|
||||
# Filter out blocks already present
|
||||
blocks_to_store = [bh for bh in keys if bh not in self.blocks]
|
||||
|
||||
if not blocks_to_store:
|
||||
# All blocks already present
|
||||
return
|
||||
|
||||
# Evict blocks if needed (LRU policy)
|
||||
num_blocks_to_evict = len(blocks_to_store) - (
|
||||
self.max_blocks - len(self.blocks)
|
||||
)
|
||||
|
||||
evicted = []
|
||||
if num_blocks_to_evict > 0:
|
||||
# Collect eviction candidates first (LRU order), then delete atomically
|
||||
protected = set(keys)
|
||||
for key in self.blocks:
|
||||
if key not in protected:
|
||||
evicted.append(key)
|
||||
if len(evicted) == num_blocks_to_evict:
|
||||
break
|
||||
else:
|
||||
# Could not collect enough eviction candidates
|
||||
return
|
||||
for key in evicted:
|
||||
del self.blocks[key]
|
||||
|
||||
# Create internal job metadata
|
||||
internal_job_metadata = _JobMetadata(
|
||||
job_id=job_id, keys=blocks_to_store, is_store=True
|
||||
)
|
||||
|
||||
if self.simulate_async:
|
||||
# Job will complete on next get_finished() call
|
||||
self.pending_jobs.append(internal_job_metadata)
|
||||
else:
|
||||
# Job completes immediately
|
||||
self._complete_store_job(internal_job_metadata)
|
||||
|
||||
def submit_load(self, job_metadata: JobMetadata) -> None:
|
||||
"""
|
||||
Submit an async job to load blocks from this tier to primary tier.
|
||||
|
||||
Args:
|
||||
job_metadata: Job metadata including job_id, keys, and
|
||||
spec for writing blocks into the primary tier.
|
||||
"""
|
||||
job_id = job_metadata.job_id
|
||||
keys = job_metadata.keys
|
||||
block_ids = job_metadata.block_ids
|
||||
|
||||
assert len(keys) == len(block_ids), (
|
||||
f"Length mismatch: {len(keys)} keys but {len(block_ids)} block_ids"
|
||||
)
|
||||
|
||||
# Verify all blocks exist
|
||||
for key in keys:
|
||||
if key not in self.blocks:
|
||||
return
|
||||
|
||||
# Create internal job metadata
|
||||
internal_job_metadata = _JobMetadata(job_id=job_id, keys=keys, is_store=False)
|
||||
|
||||
if self.simulate_async:
|
||||
# Job will complete on next get_finished() call
|
||||
self.pending_jobs.append(internal_job_metadata)
|
||||
else:
|
||||
# Job completes immediately
|
||||
self._complete_load_job(internal_job_metadata)
|
||||
|
||||
def get_finished(self) -> Iterable[JobResult]:
|
||||
"""
|
||||
Poll for finished async jobs.
|
||||
|
||||
Returns:
|
||||
Iterable of JobResult objects for all jobs that have
|
||||
finished since the last call.
|
||||
"""
|
||||
# Move pending jobs to completed
|
||||
if self.simulate_async and self.pending_jobs:
|
||||
for job_metadata in self.pending_jobs:
|
||||
if job_metadata.is_store:
|
||||
self._complete_store_job(job_metadata)
|
||||
else:
|
||||
self._complete_load_job(job_metadata)
|
||||
self.pending_jobs.clear()
|
||||
|
||||
# Return completed jobs
|
||||
result = self.completed_jobs
|
||||
self.completed_jobs = []
|
||||
return result
|
||||
|
||||
def _complete_store_job(self, job_metadata: _JobMetadata):
|
||||
"""Complete a store job by adding blocks to storage."""
|
||||
for key in job_metadata.keys:
|
||||
self.blocks[key] = True
|
||||
# Return simplified JobResult (only job_id and success)
|
||||
self.completed_jobs.append(JobResult(job_id=job_metadata.job_id, success=True))
|
||||
|
||||
def _complete_load_job(self, job_metadata: _JobMetadata):
|
||||
"""Complete a load job."""
|
||||
# Return simplified JobResult (only job_id and success)
|
||||
self.completed_jobs.append(JobResult(job_id=job_metadata.job_id, success=True))
|
||||
|
||||
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
||||
"""
|
||||
Mark blocks as recently used (move to end of LRU list).
|
||||
|
||||
Args:
|
||||
keys: Blocks to mark as recently used.
|
||||
req_context: Per-request context.
|
||||
"""
|
||||
for key in reversed(list(keys)):
|
||||
if key in self.blocks:
|
||||
self.blocks.move_to_end(key)
|
||||
|
||||
@staticmethod
|
||||
def get_tier_type() -> str:
|
||||
return "example"
|
||||
|
||||
def get_num_blocks(self) -> int:
|
||||
"""Get the number of blocks currently stored in this tier."""
|
||||
return len(self.blocks)
|
||||
|
||||
def clear(self):
|
||||
"""Clear all blocks and pending jobs (for testing)."""
|
||||
self.blocks.clear()
|
||||
self.completed_jobs.clear()
|
||||
self.pending_jobs.clear()
|
||||
@@ -0,0 +1,56 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Factory for creating secondary tier implementations.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
|
||||
from vllm.v1.kv_offload.tiering.example import ExampleSecondaryTier
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
SUPPORTED_TIERS: tuple[type[SecondaryTierManager], ...] = (ExampleSecondaryTier,)
|
||||
|
||||
_TIER_REGISTRY: dict[str, type[SecondaryTierManager]] = {
|
||||
cls.get_tier_type(): cls for cls in SUPPORTED_TIERS
|
||||
}
|
||||
|
||||
|
||||
def create_secondary_tier(
|
||||
tier_config: dict,
|
||||
primary_kv_view: memoryview,
|
||||
vllm_config: "VllmConfig",
|
||||
) -> SecondaryTierManager:
|
||||
"""
|
||||
Create a secondary tier from configuration.
|
||||
|
||||
Args:
|
||||
tier_config: Dictionary with tier configuration containing:
|
||||
- type (required): Type of secondary tier (e.g., "example")
|
||||
- Additional tier-specific parameters are passed directly
|
||||
to the tier constructor
|
||||
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
|
||||
vllm_config: Global vLLM configuration.
|
||||
|
||||
Returns:
|
||||
SecondaryTierManager instance
|
||||
|
||||
Raises:
|
||||
ValueError: If tier type is unknown or configuration is invalid
|
||||
"""
|
||||
config = tier_config.copy()
|
||||
|
||||
tier_type = config.pop("type", None)
|
||||
if not tier_type:
|
||||
raise ValueError("Secondary tier configuration must include 'type'")
|
||||
|
||||
cls = _TIER_REGISTRY.get(tier_type)
|
||||
if cls is None:
|
||||
raise ValueError(
|
||||
f"Unknown secondary tier type: {tier_type!r}. "
|
||||
f"Supported types: {list(_TIER_REGISTRY)}"
|
||||
)
|
||||
return cls(vllm_config=vllm_config, primary_kv_view=primary_kv_view, **config)
|
||||
@@ -0,0 +1,505 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
TieringOffloadingManager: Multi-tier KV cache offloading orchestrator.
|
||||
|
||||
This manager coordinates between a CPU primary tier (with direct GPU access)
|
||||
and zero or more secondary tiers (Storage, Network, etc.) to provide
|
||||
hierarchical KV cache offloading.
|
||||
|
||||
Key Design Principles:
|
||||
1. Always offload to all tiers — When a block is stored to the primary tier,
|
||||
it is cascaded to ALL secondary tiers
|
||||
2. Primary tier is the gateway — Secondary tiers cannot access GPU memory
|
||||
directly; all data flows through the CPU primary tier
|
||||
3. Staged promotion — Blocks in secondary tiers must be promoted to the
|
||||
primary tier before GPU can access them
|
||||
4. Transparent retry mechanism — Return None from lookup() to signal
|
||||
"data is being promoted, try later"
|
||||
5. ref_cnt as eviction protection — primary.prepare_read() increments ref_cnt,
|
||||
protecting blocks from eviction until complete_read() is called
|
||||
"""
|
||||
|
||||
from collections.abc import Collection, Iterable
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LoadStoreSpec,
|
||||
OffloadingEvent,
|
||||
OffloadingManager,
|
||||
OffloadKey,
|
||||
PrepareStoreOutput,
|
||||
ReqContext,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobId,
|
||||
JobMetadata,
|
||||
SecondaryTierManager,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PendingPromotion:
|
||||
"""Accumulator for blocks awaiting submit_load() for one (tier, request)."""
|
||||
|
||||
req_context: ReqContext
|
||||
keys: list[OffloadKey] = field(default_factory=list)
|
||||
block_ids: list[int] = field(default_factory=list)
|
||||
|
||||
|
||||
class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
|
||||
"""CPUOffloadingManager with a primary/secondary transfer interface.
|
||||
|
||||
The inherited prepare_store/complete_store/prepare_load/complete_load are the
|
||||
GPU-facing OffloadingManager interface. These aliases expose the same operations
|
||||
from the secondary tier perspective, where read/write refers to secondary
|
||||
accessing primary. This avoids confusion when reading TieringOffloadingManager
|
||||
code (e.g. calling prepare_load inside a cascade/store path would be misleading).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_blocks: int,
|
||||
mmap_region: SharedOffloadRegion,
|
||||
cache_policy: str = "lru",
|
||||
enable_events: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
num_blocks=num_blocks,
|
||||
cache_policy=cache_policy, # type: ignore[arg-type]
|
||||
enable_events=enable_events,
|
||||
)
|
||||
self._mmap_region = mmap_region
|
||||
# read/write is for CPU<->secondary transfers,
|
||||
# load/store is for CPU<->GPU transfers.
|
||||
# These aliases avoid calling prepare_load inside a store path.
|
||||
self.prepare_read = self.prepare_load
|
||||
self.complete_read = self.complete_load
|
||||
self.prepare_write = self.prepare_store
|
||||
self.complete_write = self.complete_store
|
||||
|
||||
self._kv_memoryview = mmap_region.create_kv_memoryview()
|
||||
|
||||
def get_kv_memoryview(self) -> memoryview:
|
||||
"""Return the memoryview over the primary tier's KV cache buffer.
|
||||
|
||||
The view has shape (num_blocks, row_stride_bytes) and is backed by the
|
||||
SharedOffloadRegion mmap. Secondary tiers address block *b* as
|
||||
``view[b]``.
|
||||
"""
|
||||
return self._kv_memoryview
|
||||
|
||||
def shutdown(self) -> None:
|
||||
super().shutdown()
|
||||
self._kv_memoryview.release()
|
||||
self._mmap_region.cleanup()
|
||||
|
||||
|
||||
class TieringOffloadingManager(OffloadingManager):
|
||||
"""
|
||||
Orchestrates multi-tier KV cache offloading.
|
||||
|
||||
This manager coordinates between a CPU primary tier (with direct GPU access)
|
||||
and zero or more secondary tiers (Storage, Network, etc.) to provide
|
||||
hierarchical KV cache offloading.
|
||||
|
||||
Key internal state:
|
||||
- Minimal state tracking; relies on secondary tiers to report completion
|
||||
via get_finished()
|
||||
- Secondary tiers return JobResult objects containing all necessary
|
||||
information
|
||||
- job_id_counter: monotonically increasing counter for job IDs
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
primary_tier: CPUPrimaryTierOffloadingManager,
|
||||
secondary_tiers: list[SecondaryTierManager] | None = None,
|
||||
enable_events: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the TieringOffloadingManager.
|
||||
|
||||
Args:
|
||||
primary_tier: The primary tier manager (CPU-based).
|
||||
secondary_tiers: List of secondary tier managers (e.g., Storage,
|
||||
Network). Can be None or empty list.
|
||||
enable_events: Whether to track offloading events
|
||||
"""
|
||||
self.primary_tier: CPUPrimaryTierOffloadingManager = primary_tier
|
||||
self.secondary_tiers = secondary_tiers or []
|
||||
|
||||
self._job_id_counter: int = 0
|
||||
self.events: list[OffloadingEvent] | None = [] if enable_events else None
|
||||
|
||||
# Job tracking: maps job_id to metadata for all in-flight transfers.
|
||||
# JobMetadata.is_promotion distinguishes direction:
|
||||
# True: secondary → primary (promotion)
|
||||
# False: primary → secondary (cascade)
|
||||
self._transfer_jobs: dict[JobId, JobMetadata] = {}
|
||||
|
||||
# Pending promotion requests accumulated during lookup() calls; flushed
|
||||
# as one batched submit_load() per (tier, request) in take_events().
|
||||
# Outer key: tier. Inner key: req_context.req_id — the same ReqContext
|
||||
# object is reused for all block lookups of a given request per engine step.
|
||||
self._pending_load_submissions: dict[
|
||||
SecondaryTierManager, dict[str, PendingPromotion]
|
||||
] = {}
|
||||
|
||||
# Gate for once-per-step execution of _maybe_process_finished_jobs().
|
||||
# Reset at the end of each step in take_events().
|
||||
self._processed_jobs_this_step: bool = False
|
||||
|
||||
def _next_job_id(self) -> JobId:
|
||||
"""Generate a unique job ID for async transfer tracking."""
|
||||
job_id = self._job_id_counter
|
||||
self._job_id_counter += 1
|
||||
return job_id
|
||||
|
||||
def _maybe_process_finished_jobs(self):
|
||||
"""
|
||||
Poll secondary tiers for completed jobs (at most once per step).
|
||||
|
||||
Guarded by _processed_jobs_this_step: the first call in an engine step
|
||||
does the actual polling; subsequent calls are no-ops. The flag is reset
|
||||
in take_events() at the end of each step.
|
||||
"""
|
||||
if self._processed_jobs_this_step:
|
||||
return
|
||||
self._processed_jobs_this_step = True
|
||||
self._process_finished_jobs()
|
||||
|
||||
def _process_finished_jobs(self):
|
||||
"""
|
||||
Unconditionally poll all secondary tiers for completed jobs.
|
||||
|
||||
This method:
|
||||
1. Calls get_finished() on each secondary tier
|
||||
2. For completed stores (primary→secondary): calls primary.complete_read()
|
||||
to decrement ref_cnt
|
||||
3. For completed loads (secondary→primary): calls primary.complete_write()
|
||||
to make blocks available
|
||||
"""
|
||||
for i, tier in enumerate(self.secondary_tiers):
|
||||
for completed_job in tier.get_finished():
|
||||
job_id = completed_job.job_id
|
||||
job_metadata = self._transfer_jobs.pop(job_id, None)
|
||||
assert job_metadata is not None, (
|
||||
f"Finished job_id {job_id} from tier #{i}"
|
||||
f" ({tier.get_tier_type()}) not in _transfer_jobs"
|
||||
)
|
||||
|
||||
if job_metadata.is_promotion:
|
||||
# secondary→primary transfer (promotion) completed.
|
||||
# Make blocks available in primary tier.
|
||||
self.primary_tier.complete_write(
|
||||
job_metadata.keys,
|
||||
job_metadata.req_context,
|
||||
completed_job.success,
|
||||
)
|
||||
else:
|
||||
# primary→secondary transfer completed.
|
||||
# Decrement ref_cnt on primary blocks.
|
||||
self.primary_tier.complete_read(
|
||||
job_metadata.keys, job_metadata.req_context
|
||||
)
|
||||
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
"""
|
||||
Check whether a single block is offloaded and ready.
|
||||
|
||||
Algorithm:
|
||||
1. Process any completed async jobs first.
|
||||
2. Query primary tier — short-circuit on hit or in-flight.
|
||||
3. On primary miss, query secondary tiers — stop on first
|
||||
hit and initiate promotion.
|
||||
|
||||
Args:
|
||||
key: Block hash to look up.
|
||||
req_context: Per-request context.
|
||||
|
||||
Returns:
|
||||
True — block is ready in the primary tier.
|
||||
None — block found but not yet ready (primary in-flight,
|
||||
promotion started, or a secondary tier is busy).
|
||||
False — block not found in any tier, or primary is full
|
||||
and cannot accept a promotion.
|
||||
"""
|
||||
self._maybe_process_finished_jobs()
|
||||
|
||||
primary_hit = self.primary_tier.lookup(key, req_context)
|
||||
if primary_hit is True:
|
||||
return True
|
||||
if primary_hit is None:
|
||||
return None
|
||||
|
||||
any_none = False
|
||||
for tier in self.secondary_tiers:
|
||||
result = tier.lookup(key, req_context)
|
||||
if result is True:
|
||||
if not self._initiate_promotion(tier, key, req_context):
|
||||
return False # primary full, block unavailable
|
||||
return None # promotion started, retry later
|
||||
if result is None:
|
||||
any_none = True
|
||||
|
||||
if any_none:
|
||||
return None
|
||||
return False
|
||||
|
||||
def _initiate_promotion(
|
||||
self,
|
||||
tier: SecondaryTierManager,
|
||||
key: OffloadKey,
|
||||
req_context: ReqContext,
|
||||
) -> bool:
|
||||
"""
|
||||
Queue a block for promotion from a secondary tier to the primary tier.
|
||||
|
||||
Allocates space in the primary tier immediately (sets ref_cnt=-1 so
|
||||
subsequent lookups within the same step see the slot as in-flight),
|
||||
then defers the actual submit_load() call to _flush_pending_promotions()
|
||||
so all blocks queued during one engine step are submitted as a single
|
||||
batched job.
|
||||
|
||||
Args:
|
||||
tier: The secondary tier to promote from
|
||||
key: Block to promote
|
||||
req_context: Per-request context forwarded to primary.prepare_write().
|
||||
|
||||
Returns:
|
||||
True if promotion was initiated, False if primary tier is full.
|
||||
"""
|
||||
# Allocate space in primary tier for promoted block.
|
||||
# Must happen immediately so primary.lookup() returns None (in-flight)
|
||||
# for this key on any subsequent lookup() call within the same step,
|
||||
# preventing duplicate promotion attempts.
|
||||
primary_write_result = self.primary_tier.prepare_write([key], req_context)
|
||||
|
||||
if primary_write_result is None:
|
||||
# Primary tier is full; caller should treat the block as unavailable
|
||||
# rather than retrying indefinitely.
|
||||
return False
|
||||
|
||||
store_spec = primary_write_result.store_spec
|
||||
assert isinstance(store_spec, CPULoadStoreSpec)
|
||||
# Defer submit_load to take_events(). Group by (tier, request) so each
|
||||
# request's blocks are submitted as one batched job per tier.
|
||||
tier_pending = self._pending_load_submissions.setdefault(tier, {})
|
||||
ctx_id = req_context.req_id
|
||||
if ctx_id not in tier_pending:
|
||||
tier_pending[ctx_id] = PendingPromotion(
|
||||
keys=[], block_ids=[], req_context=req_context
|
||||
)
|
||||
entry = tier_pending[ctx_id]
|
||||
entry.keys.extend(primary_write_result.keys_to_store)
|
||||
entry.block_ids.extend(store_spec.block_ids)
|
||||
return True
|
||||
|
||||
def _flush_pending_promotions(self) -> None:
|
||||
"""Submit one batched submit_load() per (tier, request).
|
||||
|
||||
Called from take_events() at the end of each engine step, flushing
|
||||
all promotion requests deferred during lookup().
|
||||
"""
|
||||
if not self._pending_load_submissions:
|
||||
return
|
||||
|
||||
for tier, pending_by_ctx in self._pending_load_submissions.items():
|
||||
for entry in pending_by_ctx.values():
|
||||
job_id = self._next_job_id()
|
||||
job_metadata = JobMetadata(
|
||||
job_id=job_id,
|
||||
keys=entry.keys,
|
||||
block_ids=np.array(entry.block_ids, dtype=np.int64),
|
||||
is_promotion=True,
|
||||
req_context=entry.req_context,
|
||||
)
|
||||
self._transfer_jobs[job_id] = job_metadata
|
||||
tier.submit_load(job_metadata)
|
||||
|
||||
self._pending_load_submissions.clear()
|
||||
|
||||
def prepare_load(
|
||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||
) -> LoadStoreSpec:
|
||||
"""
|
||||
Prepare blocks to be loaded from primary tier to GPU.
|
||||
|
||||
CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure
|
||||
that any completed promotions have been finalized and blocks are ready.
|
||||
|
||||
This increments ref_cnt on the blocks in the primary tier, protecting
|
||||
them from eviction during the transfer.
|
||||
|
||||
Args:
|
||||
keys: Blocks to prepare for loading.
|
||||
req_context: Per-request context.
|
||||
|
||||
Returns:
|
||||
LoadStoreSpec for reading from primary tier.
|
||||
"""
|
||||
# Process completed promotions to ensure blocks are ready
|
||||
self._maybe_process_finished_jobs()
|
||||
|
||||
return self.primary_tier.prepare_load(keys, req_context)
|
||||
|
||||
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
||||
"""
|
||||
Mark blocks as recently used in all tiers.
|
||||
|
||||
Args:
|
||||
keys: Blocks to mark as recently used.
|
||||
req_context: Per-request context.
|
||||
"""
|
||||
self.primary_tier.touch(keys, req_context)
|
||||
for tier in self.secondary_tiers:
|
||||
tier.touch(keys, req_context)
|
||||
|
||||
def complete_load(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
||||
"""
|
||||
Mark blocks as done loading from primary tier to GPU.
|
||||
|
||||
This decrements ref_cnt on the blocks in the primary tier, allowing
|
||||
them to be evicted again.
|
||||
|
||||
Args:
|
||||
keys: Blocks that finished loading.
|
||||
req_context: Per-request context.
|
||||
"""
|
||||
self.primary_tier.complete_load(keys, req_context)
|
||||
|
||||
def prepare_store(
|
||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||
) -> PrepareStoreOutput | None:
|
||||
"""
|
||||
Prepare blocks to be stored from GPU to primary tier.
|
||||
|
||||
CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure
|
||||
that any completed async transfers have their ref_cnt decremented
|
||||
before the primary tier makes eviction decisions.
|
||||
|
||||
Args:
|
||||
keys: Blocks to prepare for storing.
|
||||
req_context: Per-request context.
|
||||
|
||||
Returns:
|
||||
PrepareStoreOutput describing where to store blocks and what was
|
||||
evicted, or None if store cannot proceed.
|
||||
"""
|
||||
# Step 1: Poll for completed async jobs FIRST
|
||||
# This decrements ref_cnt on primary blocks that have been
|
||||
# successfully transferred to secondary tiers.
|
||||
self._maybe_process_finished_jobs()
|
||||
|
||||
# Step 2: Store to primary tier
|
||||
primary_result = self.primary_tier.prepare_store(keys, req_context)
|
||||
|
||||
# Note: Secondary tier cascading will happen in complete_store()
|
||||
# after the GPU→Primary transfer completes and blocks are ready.
|
||||
|
||||
return primary_result
|
||||
|
||||
def complete_store(
|
||||
self,
|
||||
keys: Collection[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
success: bool = True,
|
||||
):
|
||||
"""
|
||||
Mark blocks as done storing from GPU to primary tier.
|
||||
|
||||
This is where secondary tier cascading happens — after blocks are
|
||||
confirmed to be in the primary tier, they are cascaded to ALL
|
||||
secondary tiers.
|
||||
|
||||
For each secondary tier:
|
||||
1. Call primary.prepare_read() to get LoadStoreSpec AND increment
|
||||
ref_cnt (protecting blocks during async transfer)
|
||||
2. Call tier.submit_store() to start async transfer: primary→secondary
|
||||
3. Track the job in _store_jobs dictionary
|
||||
|
||||
Args:
|
||||
keys: Blocks that finished storing.
|
||||
success: Whether the GPU→primary transfer succeeded.
|
||||
req_context: Per-request context forwarded to primary.prepare_read().
|
||||
"""
|
||||
# Step 1: Complete store in primary tier (makes blocks loadable)
|
||||
self.primary_tier.complete_store(keys, req_context, success)
|
||||
|
||||
if not success:
|
||||
# If GPU→Primary transfer failed, don't cascade to secondary tiers
|
||||
return
|
||||
|
||||
# Step 2: Cascade to ALL secondary tiers
|
||||
# For each secondary tier, call primary.prepare_read() to get the
|
||||
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from
|
||||
# eviction during the async transfer). One prepare_read() call per
|
||||
# secondary tier.
|
||||
for tier in self.secondary_tiers:
|
||||
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
|
||||
|
||||
# Submit async store job: primary→secondary
|
||||
job_id = self._next_job_id()
|
||||
|
||||
# Track this store job
|
||||
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
|
||||
job_metadata = JobMetadata(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=primary_blocks_spec.block_ids,
|
||||
is_promotion=False,
|
||||
req_context=req_context,
|
||||
)
|
||||
self._transfer_jobs[job_id] = job_metadata
|
||||
|
||||
tier.submit_store(job_metadata)
|
||||
|
||||
# Note: The async transfers are now in flight. Their completion is
|
||||
# tracked via get_finished() / _maybe_process_finished_jobs().
|
||||
|
||||
def take_events(self) -> Iterable[OffloadingEvent]:
|
||||
"""
|
||||
End-of-step hook: flush deferred work, yield events, reset per-step state.
|
||||
|
||||
Called once per engine step from Scheduler.update_from_output() →
|
||||
connector.take_events(). Ensures _maybe_process_finished_jobs() has run
|
||||
at least once this step, flushes pending promotions, yields collected
|
||||
events, and resets the per-step flag.
|
||||
|
||||
Yields:
|
||||
New OffloadingEvents collected since the last call.
|
||||
"""
|
||||
# TODO: Move _flush_pending_promotions() to a dedicated end_of_batch()
|
||||
# hook once one exists. For now, take_events() serves as the flush
|
||||
# point under the assumption that it is called at the end of each
|
||||
# engine step (Scheduler.update_from_output() → connector.take_events()).
|
||||
# When the dedicated hook is added, update tests that rely on
|
||||
# take_events() to signal end of step.
|
||||
|
||||
self._maybe_process_finished_jobs()
|
||||
|
||||
self._flush_pending_promotions()
|
||||
|
||||
# Reset the per-step gate so next step's first call does real work.
|
||||
self._processed_jobs_this_step = False
|
||||
|
||||
if self.events is not None:
|
||||
yield from self.events
|
||||
self.events.clear()
|
||||
|
||||
yield from self.primary_tier.take_events()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Shutdown all tiers and release resources."""
|
||||
for tier in self.secondary_tiers:
|
||||
tier.shutdown()
|
||||
self.primary_tier.shutdown()
|
||||
@@ -0,0 +1,187 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
TieringOffloadingSpec: Spec for multi-tier KV cache offloading.
|
||||
|
||||
This spec creates a TieringOffloadingManager with a CPU primary tier
|
||||
and configurable secondary tiers (e.g., Storage, Network).
|
||||
|
||||
Configuration via kv_connector_extra_config:
|
||||
- cpu_bytes_to_use: (required) Bytes to allocate for CPU primary tier
|
||||
- block_size: (optional) Block size for offloaded blocks (default: GPU block size)
|
||||
- eviction_policy: (optional) Primary tier eviction policy: "lru" or
|
||||
"arc" (default: "lru")
|
||||
- secondary_tiers: (optional) List of secondary tier configurations
|
||||
Each secondary tier config is a dict with:
|
||||
- type: (required) Type of secondary tier (e.g., "example", "storage", "network")
|
||||
- Additional tier-specific parameters are passed directly to the tier
|
||||
constructor. See each tier's documentation for supported parameters.
|
||||
|
||||
Example configuration:
|
||||
{
|
||||
"cpu_bytes_to_use": 10737418240, # 10 GB
|
||||
"block_size": 16,
|
||||
"eviction_policy": "lru",
|
||||
"secondary_tiers": [
|
||||
{
|
||||
"type": "example",
|
||||
# Tier-specific parameters (for ExampleSecondaryTier):
|
||||
"max_blocks": 10000,
|
||||
"simulate_async": False
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_offload.base import CanonicalKVCaches, OffloadingManager
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
from vllm.v1.kv_offload.tiering.factory import create_secondary_tier
|
||||
from vllm.v1.kv_offload.tiering.manager import (
|
||||
CPUPrimaryTierOffloadingManager,
|
||||
TieringOffloadingManager,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
"""
|
||||
Spec for multi-tier KV cache offloading.
|
||||
|
||||
Creates a TieringOffloadingManager with:
|
||||
- Primary tier: CPU (LRU or ARC eviction policy)
|
||||
- Secondary tiers: Configurable via extra_config
|
||||
|
||||
The CPU primary tier has direct GPU access and serves as the gateway for
|
||||
all GPU↔offload operations. Secondary tiers cannot directly access GPU
|
||||
memory and must transfer data through the primary tier.
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig):
|
||||
super().__init__(vllm_config, kv_cache_config)
|
||||
# Redeclare for mypy: parent sets this but `--follow-imports skip` hides it
|
||||
self._manager: OffloadingManager | None = None
|
||||
|
||||
# Parse secondary tier configurations
|
||||
self.secondary_tier_configs = self.extra_config.get("secondary_tiers", [])
|
||||
if not isinstance(self.secondary_tier_configs, list):
|
||||
raise ValueError("secondary_tiers must be a list of tier configurations")
|
||||
|
||||
# Scheduler-side mmap (rank=None); kept for cleanup
|
||||
self._scheduler_mmap: SharedOffloadRegion | None = None
|
||||
|
||||
def get_manager(self) -> OffloadingManager:
|
||||
"""
|
||||
Get the TieringOffloadingManager.
|
||||
|
||||
Creates a TieringOffloadingManager with:
|
||||
- Primary tier: CPU (LRU or ARC)
|
||||
- Secondary tiers: As configured in extra_config
|
||||
|
||||
Returns:
|
||||
TieringOffloadingManager instance
|
||||
"""
|
||||
if not self._manager:
|
||||
kv_events_config = self.vllm_config.kv_events_config
|
||||
enable_events = (
|
||||
kv_events_config is not None and kv_events_config.enable_kv_cache_events
|
||||
)
|
||||
|
||||
# Create scheduler-side SharedOffloadRegion (rank=None) so the
|
||||
# primary tier can eagerly create a memoryview over _base.
|
||||
world_size = self.vllm_config.parallel_config.world_size
|
||||
scheduler_mmap = SharedOffloadRegion(
|
||||
instance_id=self.vllm_config.instance_id,
|
||||
total_size_bytes=self.cpu_page_size_per_worker
|
||||
* world_size
|
||||
* self.num_blocks,
|
||||
num_blocks=self.num_blocks,
|
||||
rank=None,
|
||||
num_workers=world_size,
|
||||
cpu_page_size=self.cpu_page_size_per_worker,
|
||||
)
|
||||
self._scheduler_mmap = scheduler_mmap
|
||||
|
||||
# Create primary tier (CPU-based)
|
||||
assert len(self.gpu_block_size) == 1
|
||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=self.num_blocks,
|
||||
cache_policy=self.eviction_policy, # type: ignore[arg-type]
|
||||
enable_events=enable_events,
|
||||
mmap_region=scheduler_mmap,
|
||||
)
|
||||
|
||||
# Create secondary tiers
|
||||
primary_kv_view = primary_tier.get_kv_memoryview()
|
||||
secondary_tiers = []
|
||||
for i, tier_config in enumerate(self.secondary_tier_configs):
|
||||
try:
|
||||
tier = create_secondary_tier(
|
||||
tier_config, primary_kv_view, self.vllm_config
|
||||
)
|
||||
secondary_tiers.append(tier)
|
||||
logger.info(
|
||||
"Created secondary tier #%d (%s)",
|
||||
i,
|
||||
tier.get_tier_type(),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Failed to create secondary tier from config %s: %s",
|
||||
tier_config,
|
||||
e,
|
||||
)
|
||||
raise
|
||||
|
||||
# Create TieringOffloadingManager. GPU↔CPU transfers use the inherited
|
||||
# get_handlers(); secondary tier transfers are handled by the
|
||||
# secondary tier managers and need no additional handlers here.
|
||||
tiering_manager = TieringOffloadingManager(
|
||||
primary_tier=primary_tier,
|
||||
secondary_tiers=secondary_tiers,
|
||||
enable_events=enable_events,
|
||||
)
|
||||
if int(self.extra_config.get("store_threshold", 0)) >= 2:
|
||||
raise ValueError(
|
||||
"store_threshold is not supported for TieringOffloadingSpec"
|
||||
)
|
||||
self._manager = tiering_manager
|
||||
|
||||
logger.info(
|
||||
"Created TieringOffloadingManager with primary tier "
|
||||
"(%s, %s blocks) and %s secondary tier(s)",
|
||||
self.eviction_policy,
|
||||
self.num_blocks,
|
||||
len(secondary_tiers),
|
||||
)
|
||||
|
||||
return self._manager
|
||||
|
||||
def _create_handlers(
|
||||
self, kv_caches: CanonicalKVCaches
|
||||
) -> CpuGpuOffloadingHandlers:
|
||||
world_size = self.vllm_config.parallel_config.world_size
|
||||
rank = torch.accelerator.current_device_index()
|
||||
worker_mmap = SharedOffloadRegion(
|
||||
instance_id=self.vllm_config.instance_id,
|
||||
total_size_bytes=self.cpu_page_size_per_worker
|
||||
* world_size
|
||||
* self.num_blocks,
|
||||
num_blocks=self.num_blocks,
|
||||
rank=rank,
|
||||
num_workers=world_size,
|
||||
cpu_page_size=self.cpu_page_size_per_worker,
|
||||
)
|
||||
return CpuGpuOffloadingHandlers(
|
||||
kv_caches=kv_caches,
|
||||
block_size_factor=self.block_size_factor,
|
||||
num_cpu_blocks=self.num_blocks,
|
||||
mmap_region=worker_mmap,
|
||||
)
|
||||
Reference in New Issue
Block a user