From 11f6b545d41a42422410bb529466463f466cafd9 Mon Sep 17 00:00:00 2001 From: Ronen Schaffer Date: Wed, 13 May 2026 17:21:43 +0300 Subject: [PATCH] [kv_offload] Add multi-tier KV cache offloading framework (#40020) Signed-off-by: Ronen Schaffer --- .../unit/test_offloading_connector.py | 45 ++ tests/v1/kv_offload/cpu/test_manager.py | 37 ++ .../v1/kv_offload/test_tiering_offloading.py | 565 ++++++++++++++++++ vllm/v1/kv_offload/base.py | 8 +- vllm/v1/kv_offload/cpu/gpu_worker.py | 7 + .../kv_offload/cpu/shared_offload_region.py | 14 + vllm/v1/kv_offload/cpu/spec.py | 20 +- vllm/v1/kv_offload/factory.py | 5 + vllm/v1/kv_offload/tiering/__init__.py | 0 vllm/v1/kv_offload/tiering/base.py | 168 ++++++ .../v1/kv_offload/tiering/example/__init__.py | 235 ++++++++ vllm/v1/kv_offload/tiering/factory.py | 56 ++ vllm/v1/kv_offload/tiering/manager.py | 505 ++++++++++++++++ vllm/v1/kv_offload/tiering/spec.py | 187 ++++++ 14 files changed, 1841 insertions(+), 11 deletions(-) create mode 100644 tests/v1/kv_offload/test_tiering_offloading.py create mode 100644 vllm/v1/kv_offload/tiering/__init__.py create mode 100644 vllm/v1/kv_offload/tiering/base.py create mode 100644 vllm/v1/kv_offload/tiering/example/__init__.py create mode 100644 vllm/v1/kv_offload/tiering/factory.py create mode 100644 vllm/v1/kv_offload/tiering/manager.py create mode 100644 vllm/v1/kv_offload/tiering/spec.py diff --git a/tests/v1/kv_connector/unit/test_offloading_connector.py b/tests/v1/kv_connector/unit/test_offloading_connector.py index 555daea50e..35ebfa1e73 100644 --- a/tests/v1/kv_connector/unit/test_offloading_connector.py +++ b/tests/v1/kv_connector/unit/test_offloading_connector.py @@ -292,3 +292,48 @@ def test_cpu_offloading( if subscriber is not None: subscriber.close() del llm + + +def test_tiering_offloading() -> None: + """Tests OffloadingConnector with TieringOffloadingSpec.""" + extra_config: dict = { + "cpu_bytes_to_use": 500 << 20, + "block_size": 48, + "spec_name": "TieringOffloadingSpec", + "secondary_tiers": [{"type": "example"}], + } + kv_transfer_config = KVTransferConfig( + kv_connector="OffloadingConnector", + kv_role="kv_both", + kv_connector_extra_config=extra_config, + ) + + port: int + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("0.0.0.0", 0)) + port = s.getsockname()[1] + events_endpoint = f"tcp://*:{port}" + kv_events_config = KVEventsConfig( + enable_kv_cache_events=True, + publisher="zmq", + endpoint=events_endpoint, + topic="test", + ) + + llm = LLM( + model="meta-llama/Llama-3.2-1B-Instruct", + max_model_len=4096, + gpu_memory_utilization=0.5, + kv_events_config=kv_events_config, + kv_transfer_config=kv_transfer_config, + ) + subscriber = MockSubscriber( + events_endpoint.replace("*", "127.0.0.1"), + topic=kv_events_config.topic, + ) + try: + _latency_test(llm, subscriber) + _accuracy_test(llm, subscriber) + finally: + subscriber.close() + del llm diff --git a/tests/v1/kv_offload/cpu/test_manager.py b/tests/v1/kv_offload/cpu/test_manager.py index 8bb77bf763..3957294f8b 100644 --- a/tests/v1/kv_offload/cpu/test_manager.py +++ b/tests/v1/kv_offload/cpu/test_manager.py @@ -262,6 +262,43 @@ def test_cpu_manager(): ) +def test_prepare_load_preserves_key_order(): + """block_ids[i] must correspond to keys[i] (co-indexed invariant).""" + manager = CPUOffloadingManager(num_blocks=4, cache_policy="lru") + + key_a, key_b, key_c = to_key(0), to_key(1), to_key(2) + + # Store all three keys and learn their block ID assignments + store_output = manager.prepare_store([key_a, key_b, key_c], _EMPTY_REQ_CTX) + assert store_output is not None + assert isinstance(store_output.store_spec, CPULoadStoreSpec) + key_to_block_id = { + k: int(bid) + for k, bid in zip(store_output.keys_to_store, store_output.store_spec.block_ids) + } + manager.complete_store([key_a, key_b, key_c], _EMPTY_REQ_CTX) + + # Forward order: [a, b, c] + spec_fwd = manager.prepare_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) + assert isinstance(spec_fwd, CPULoadStoreSpec) + assert [int(x) for x in spec_fwd.block_ids] == [ + key_to_block_id[key_a], + key_to_block_id[key_b], + key_to_block_id[key_c], + ] + manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant + + # Arbitrary permutation: [b, c, a] + spec_perm = manager.prepare_load([key_b, key_c, key_a], _EMPTY_REQ_CTX) + assert isinstance(spec_perm, CPULoadStoreSpec) + assert [int(x) for x in spec_perm.block_ids] == [ + key_to_block_id[key_b], + key_to_block_id[key_c], + key_to_block_id[key_a], + ] + manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant + + class TestARCPolicy: """Unit tests for CPUOffloadingManager with ARC eviction policy.""" diff --git a/tests/v1/kv_offload/test_tiering_offloading.py b/tests/v1/kv_offload/test_tiering_offloading.py new file mode 100644 index 0000000000..a5c8d8673d --- /dev/null +++ b/tests/v1/kv_offload/test_tiering_offloading.py @@ -0,0 +1,565 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Unit tests for TieringOffloadingManager and ExampleSecondaryTier. + +These tests verify: +1. Basic tiered offloading operations (store, load, lookup) +2. Cascade behavior (blocks stored to all secondary tiers) +3. Promotion behavior (blocks loaded from secondary to primary to GPU) +4. ref_cnt management (blocks protected during async transfers) +5. Eviction coordination between tiers +""" + +from collections.abc import Iterable +from unittest.mock import MagicMock + +import numpy as np +import pytest +import torch + +from vllm.v1.kv_offload.base import ( + OffloadKey, + ReqContext, + make_offload_key, +) +from vllm.v1.kv_offload.tiering.base import JobMetadata +from vllm.v1.kv_offload.tiering.example import ExampleSecondaryTier +from vllm.v1.kv_offload.tiering.manager import ( + CPUPrimaryTierOffloadingManager, + TieringOffloadingManager, +) + +_CTX = ReqContext(req_id="test") +_MOCK_VLLM_CONFIG = MagicMock() + + +def _mock_mmap_region(num_blocks: int, row_bytes: int = 16): + """Create a mock SharedOffloadRegion for testing.""" + mock = MagicMock() + view = memoryview(torch.zeros((num_blocks, row_bytes), dtype=torch.int8).numpy()) + mock.create_kv_memoryview.return_value = view + return mock + + +def to_keys(int_ids: Iterable[int]) -> list[OffloadKey]: + return [make_offload_key(str(i).encode(), 0) for i in int_ids] + + +def count_hits(manager, keys: list[OffloadKey]) -> int | None: + """Count consecutive lookup hits from the start of keys. + + Returns the count of leading True results, or None if any lookup + returns None (retry-later signal). + """ + count = 0 + for key in keys: + result = manager.lookup(key, _CTX) + if result is None: + return None + if not result: + break + count += 1 + return count + + +class TestExampleSecondaryTier: + """Tests for ExampleSecondaryTier implementation.""" + + def test_basic_store_and_lookup(self): + """Test basic store and lookup operations.""" + mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy()) + tier = ExampleSecondaryTier( + vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10 + ) + + # Initially empty + blocks = to_keys(range(3)) + assert tier.lookup(blocks[0], _CTX) is False + + # Store blocks (simulate with direct insertion for testing) + tier.blocks[blocks[0]] = True + tier.blocks[blocks[1]] = True + + # Lookup should find first two blocks + assert tier.lookup(blocks[0], _CTX) is True + assert tier.lookup(blocks[1], _CTX) is True + + # Third block not present + assert tier.lookup(blocks[2], _CTX) is False + + def test_lru_eviction(self): + """Test LRU eviction policy.""" + mock_view = memoryview(torch.zeros((4, 16), dtype=torch.int8).numpy()) + tier = ExampleSecondaryTier( + vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=3 + ) + + # Fill tier to capacity + blocks = to_keys(range(3)) + for block in blocks: + tier.blocks[block] = True + + assert tier.get_num_blocks() == 3 + + # Touch first block (make it most recently used) + tier.touch([blocks[0]], _CTX) + + # Store new block should evict blocks[1] (least recently used) + new_block = to_keys([3])[0] + + tier.submit_store( + JobMetadata( + job_id=1, + keys=[new_block], + block_ids=np.array([0], dtype=np.int64), + is_promotion=False, + req_context=_CTX, + ) + ) + + # Complete the job + tier.get_finished() + + # Verify new block is stored and blocks[1] was evicted (LRU) + assert new_block in tier.blocks + assert blocks[1] not in tier.blocks + # blocks[0] and blocks[2] should still be present + assert blocks[0] in tier.blocks + assert blocks[2] in tier.blocks + + def test_async_simulation(self): + """Test simulated async behavior.""" + mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy()) + tier = ExampleSecondaryTier( + vllm_config=_MOCK_VLLM_CONFIG, + primary_kv_view=mock_view, + max_blocks=10, + simulate_async=True, + ) + + blocks = to_keys(range(2)) + + # Submit store job + tier.submit_store( + JobMetadata( + job_id=1, + keys=blocks, + block_ids=np.array([0, 1], dtype=np.int64), + is_promotion=False, + req_context=_CTX, + ) + ) + + # Blocks should not yet be stored (pending async completion) + assert tier.get_num_blocks() == 0 + + # First get_finished() should complete the job + completed = list(tier.get_finished()) + assert len(completed) == 1 + assert completed[0].job_id == 1 + assert completed[0].success is True + + # Blocks should now be stored + assert tier.get_num_blocks() == 2 + + +class TestTieringOffloadingManager: + """Tests for TieringOffloadingManager.""" + + @pytest.fixture + def manager_setup(self): + # Create primary tier (CPU-based) + mock_region = _mock_mmap_region(5) + self.primary_tier = CPUPrimaryTierOffloadingManager( + num_blocks=5, mmap_region=mock_region + ) + + mock_view = mock_region.create_kv_memoryview() + + # Create secondary tiers with the primary view + self.secondary_tier1 = ExampleSecondaryTier( + vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10 + ) + self.secondary_tier2 = ExampleSecondaryTier( + vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10 + ) + + # Create tiered manager + self.manager = TieringOffloadingManager( + primary_tier=self.primary_tier, + secondary_tiers=[self.secondary_tier1, self.secondary_tier2], + ) + + def test_basic_store_to_primary(self, manager_setup): + """Test basic store operation to primary tier.""" + blocks = to_keys(range(3)) + + # Prepare store + result = self.manager.prepare_store(blocks, _CTX) + assert result is not None + assert len(result.keys_to_store) == 3 + + # Complete store + self.manager.complete_store(blocks, _CTX, success=True) + + # Blocks should be in primary tier + assert count_hits(self.primary_tier, blocks) == 3 + + def test_cascade_to_all_secondary_tiers(self, manager_setup): + """Test that blocks are cascaded to ALL secondary tiers.""" + blocks = to_keys(range(3)) + + self.secondary_tier1.submit_store = MagicMock( + wraps=self.secondary_tier1.submit_store + ) + self.secondary_tier2.submit_store = MagicMock( + wraps=self.secondary_tier2.submit_store + ) + + # Store to primary + result = self.manager.prepare_store(blocks, _CTX) + assert result is not None + + # Complete store (triggers cascade via submit_store on each tier) + self.manager.complete_store(blocks, _CTX, success=True) + + # submit_store was called once per secondary tier + self.secondary_tier1.submit_store.assert_called_once() + self.secondary_tier2.submit_store.assert_called_once() + + # Blocks should be in both secondary tiers + assert self.secondary_tier1.get_num_blocks() == 3 + assert self.secondary_tier2.get_num_blocks() == 3 + + # Verify blocks are present + assert all(self.secondary_tier1.lookup(b, _CTX) for b in blocks) + assert all(self.secondary_tier2.lookup(b, _CTX) for b in blocks) + + def test_ref_cnt_protection_during_cascade(self, manager_setup): + """Test that ref_cnt protects blocks during cascade.""" + blocks = to_keys(range(3)) + + # Store to primary + result = self.manager.prepare_store(blocks, _CTX) + assert result is not None + self.manager.complete_store(blocks, _CTX, success=True) + + # After complete_store, blocks should have ref_cnt > 0 + # (one for each secondary tier) + for block_hash in blocks: + block = self.primary_tier._policy.get(block_hash) + # ref_cnt should be 2 (one for each secondary tier) + assert block.ref_cnt == 2 + + # End of step 1: _maybe_process_finished_jobs() was already called by + # prepare_store() above (setting the per-step flag), so take_events() + # does NOT poll get_finished() again — cascade completions remain + # unprocessed until the next step. + list(self.manager.take_events()) + + # ref_cnt still held: cascade jobs finished (sync tier) but haven't + # been polled yet because the per-step guard skipped the second call. + for block_hash in blocks: + block = self.primary_tier._policy.get(block_hash) + assert block.ref_cnt == 2 + + # Secondary tiers have completed jobs waiting to be drained + assert len(self.secondary_tier1.completed_jobs) > 0 + assert len(self.secondary_tier2.completed_jobs) > 0 + + # End of step 2: flag was reset, so _maybe_process_finished_jobs() + # runs and processes the cascade completions (complete_read → ref_cnt--) + list(self.manager.take_events()) + + # After cascade completes, ref_cnt should be 0 + for block_hash in blocks: + block = self.primary_tier._policy.get(block_hash) + assert block.ref_cnt == 0 + + # All completed jobs have been drained + assert len(self.secondary_tier1.completed_jobs) == 0 + assert len(self.secondary_tier2.completed_jobs) == 0 + + def test_lookup_from_primary(self, manager_setup): + """Test lookup when blocks are in primary tier.""" + blocks = to_keys(range(3)) + + # Store blocks + self.manager.prepare_store(blocks, _CTX) + self.manager.complete_store(blocks, _CTX, success=True) + + # Lookup should find all blocks in primary + assert count_hits(self.manager, blocks) == 3 + + def test_promotion_from_secondary(self, manager_setup): + """Test promotion of blocks from secondary to primary tier.""" + blocks = to_keys(range(3)) + + # Manually add blocks to secondary tier (simulate previous cascade) + for block in blocks: + self.secondary_tier1.blocks[block] = True + + # Lookup each block to initiate promotion for all of them + for block in blocks: + result = self.manager.lookup(block, _CTX) + assert result is None # Retry later (promotion initiated) + + # End of step 1: flushes deferred submit_load() calls + list(self.manager.take_events()) + + # End of step 2: processes the completed promotion jobs + list(self.manager.take_events()) + + # Now blocks should be in primary tier + assert count_hits(self.primary_tier, blocks) == 3 + + # Next lookup should succeed + assert count_hits(self.manager, blocks) == 3 + + def test_partial_lookup(self, manager_setup): + """Test lookup with partial hits.""" + blocks = to_keys(range(5)) + + # Store first 3 blocks to primary + self.manager.prepare_store(blocks[:3], _CTX) + self.manager.complete_store(blocks[:3], _CTX, success=True) + + # Lookup all 5 blocks should return 3 (first 3 found) + assert count_hits(self.manager, blocks) == 3 + + def test_eviction_in_primary_tier(self, manager_setup): + """Test eviction in primary tier when capacity is exceeded.""" + # Primary tier has capacity of 5 blocks + # First, fill the primary tier + blocks = to_keys(range(5)) + result = self.manager.prepare_store(blocks, _CTX) + assert result is not None + assert len(result.keys_to_store) == 5 + self.manager.complete_store(blocks, _CTX, success=True) + + # End of step: release ref_cnt from cascade + list(self.manager.take_events()) + + # Now try to store 2 more blocks (should trigger eviction) + more_blocks = to_keys(range(5, 7)) + result = self.manager.prepare_store(more_blocks, _CTX) + + # Should evict 2 blocks from primary tier + assert result is not None + assert len(result.evicted_keys) == 2 + assert len(result.keys_to_store) == 2 + + def test_touch_propagates_to_all_tiers(self, manager_setup): + """Test that touch() propagates to all tiers.""" + blocks = to_keys(range(3)) + + # Store blocks + self.manager.prepare_store(blocks, _CTX) + self.manager.complete_store(blocks, _CTX, success=True) + list(self.manager.take_events()) + + # Touch blocks + self.manager.touch(blocks, _CTX) + + # Verify touch was called on primary tier (check LRU order) + # In LRU, touched blocks should be at the end + primary_keys = list(self.primary_tier._policy.blocks.keys()) + assert primary_keys[-3:] == list(reversed(blocks)) + + # Verify touch was called on all secondary tiers + secondary1_keys = list(self.secondary_tier1.blocks.keys()) + assert secondary1_keys[-3:] == list(reversed(blocks)) + + secondary2_keys = list(self.secondary_tier2.blocks.keys()) + assert secondary2_keys[-3:] == list(reversed(blocks)) + + def test_failed_store_no_cascade(self, manager_setup): + """Test that failed GPU→primary store doesn't cascade.""" + blocks = to_keys(range(3)) + + self.secondary_tier1.submit_store = MagicMock( + wraps=self.secondary_tier1.submit_store + ) + self.secondary_tier2.submit_store = MagicMock( + wraps=self.secondary_tier2.submit_store + ) + + # Prepare store + result = self.manager.prepare_store(blocks, _CTX) + assert result is not None + + # Complete store with failure — cascade must not happen + self.manager.complete_store(blocks, _CTX, success=False) + + # submit_store was never called on either secondary tier + self.secondary_tier1.submit_store.assert_not_called() + self.secondary_tier2.submit_store.assert_not_called() + + def test_multiple_secondary_tiers_independent_eviction(self): + """Test that secondary tiers manage their own evictions.""" + mock_region = _mock_mmap_region(10) + mock_view = mock_region.create_kv_memoryview() + + # Create tier with small capacity + small_tier = ExampleSecondaryTier( + vllm_config=_MOCK_VLLM_CONFIG, + primary_kv_view=mock_view, + max_blocks=5, + simulate_async=False, + ) + large_tier = ExampleSecondaryTier( + vllm_config=_MOCK_VLLM_CONFIG, + primary_kv_view=mock_view, + max_blocks=10, + simulate_async=False, + ) + + # Create a fresh primary tier for this test + primary_tier = CPUPrimaryTierOffloadingManager( + num_blocks=10, mmap_region=mock_region + ) + + manager = TieringOffloadingManager( + primary_tier=primary_tier, + secondary_tiers=[small_tier, large_tier], + ) + + # First, store 5 blocks to fill the small tier + blocks1 = to_keys(range(5)) + result = manager.prepare_store(blocks1, _CTX) + assert result is not None + manager.complete_store(blocks1, _CTX, success=True) + list(manager.take_events()) + + # Both tiers should have 5 blocks + assert small_tier.get_num_blocks() == 5 + assert large_tier.get_num_blocks() == 5 + + # Now store 3 more blocks - small tier should evict 3 blocks + blocks2 = to_keys(range(5, 8)) + result = manager.prepare_store(blocks2, _CTX) + assert result is not None + manager.complete_store(blocks2, _CTX, success=True) + list(manager.take_events()) + + # Small tier should still have 5 blocks (evicted 3, added 3) + assert small_tier.get_num_blocks() == 5 + + # Large tier should have all 8 blocks + assert large_tier.get_num_blocks() == 8 + + def test_lookup_batches_submit_load_per_request(self, manager_setup): + """lookup() defers submit_load until take_events(), one call per request. + + Blocks from different requests each get their own submit_load call, each + carrying the correct req_context. + """ + blocks = to_keys(range(4)) + for block in blocks: + self.secondary_tier1.blocks[block] = True + + self.secondary_tier1.submit_load = MagicMock( + wraps=self.secondary_tier1.submit_load + ) + + ctx_a = ReqContext(req_id="req_a") + ctx_b = ReqContext(req_id="req_b") + + # All lookups return None: secondary hit triggers promotion (in-flight) + assert self.manager.lookup(blocks[0], ctx_a) is None + assert self.manager.lookup(blocks[1], ctx_a) is None + assert self.manager.lookup(blocks[2], ctx_b) is None + assert self.manager.lookup(blocks[3], ctx_b) is None + + # submit_load must not fire during lookup - only at end of step + self.secondary_tier1.submit_load.assert_not_called() + + # simulate end of step + list(self.manager.take_events()) + + assert self.secondary_tier1.submit_load.call_count == 2 + calls = self.secondary_tier1.submit_load.call_args_list + jm_a = calls[0].args[0] + jm_b = calls[1].args[0] + assert set(jm_a.keys) == {blocks[0], blocks[1]} + assert jm_a.req_context is ctx_a + assert set(jm_b.keys) == {blocks[2], blocks[3]} + assert jm_b.req_context is ctx_b + + def test_lookup_shared_block_no_duplicate_promotion(self, manager_setup): + """A block looked up by two requests in the same step is promoted once. + + The first lookup initiates promotion (returns None via secondary hit). + The second lookup sees ref_cnt=-1 on the primary slot and returns None + via the primary in-flight path — without triggering a second promotion. + """ + shared_block = to_keys([0])[0] + self.secondary_tier1.blocks[shared_block] = True + + self.secondary_tier1.submit_load = MagicMock( + wraps=self.secondary_tier1.submit_load + ) + + ctx_a = ReqContext(req_id="req_a") + ctx_b = ReqContext(req_id="req_b") + + result_a = self.manager.lookup(shared_block, ctx_a) + result_b = self.manager.lookup(shared_block, ctx_b) + + # Both see None (in-flight), but promotion is only queued once + assert result_a is None + assert result_b is None + + list(self.manager.take_events()) + + # Only one submit_load call despite two lookups + self.secondary_tier1.submit_load.assert_called_once() + job_metadata = self.secondary_tier1.submit_load.call_args.args[0] + assert list(job_metadata.keys) == [shared_block] + assert job_metadata.req_context is ctx_a + + def test_complete_store_forwards_req_context_to_submit_store(self, manager_setup): + """complete_store cascades to secondary tiers with the correct req_context.""" + blocks = to_keys(range(2)) + + self.secondary_tier1.submit_store = MagicMock( + wraps=self.secondary_tier1.submit_store + ) + + ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"}) + + self.manager.prepare_store(blocks, ctx) + self.manager.complete_store(blocks, ctx, success=True) + + assert self.secondary_tier1.submit_store.call_count == 1 + job_metadata = self.secondary_tier1.submit_store.call_args.args[0] + assert job_metadata.req_context is ctx + + +class TestTieringOffloadingWithoutSecondaryTiers: + """Test TieringOffloadingManager with no secondary tiers (backward compat).""" + + def test_works_without_secondary_tiers(self): + """Test that manager works with empty secondary_tiers list.""" + primary_tier = CPUPrimaryTierOffloadingManager( + num_blocks=5, mmap_region=_mock_mmap_region(5) + ) + + # Create manager with no secondary tiers + manager = TieringOffloadingManager( + primary_tier=primary_tier, secondary_tiers=[] + ) + + blocks = to_keys(range(3)) + + # Should work like a regular OffloadingManager + result = manager.prepare_store(blocks, _CTX) + assert result is not None + manager.complete_store(blocks, _CTX, success=True) + + assert count_hits(manager, blocks) == 3 + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index 482865cb7c..6f80ba6252 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -4,8 +4,6 @@ Core abstractions for KV cache offloading in vLLM v1. """ -from __future__ import annotations - from abc import ABC, abstractmethod from collections.abc import Collection, Iterable, Iterator, Sequence from dataclasses import dataclass @@ -201,7 +199,7 @@ class OffloadingManager(ABC): """ Marks blocks which were previously prepared to be stored, as stored. Following this call, the blocks become loadable. - If if_success is False, blocks that were not marked as stored will be + If success is False, blocks that were not marked as stored will be removed. Args: @@ -328,7 +326,7 @@ class CanonicalKVCaches: class OffloadingSpec(ABC): """Spec for an offloading connector""" - def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig): + def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"): logger.warning( "Initializing OffloadingSpec. This API is experimental and " "subject to change in the future as we iterate the design." @@ -394,7 +392,7 @@ class OffloadingSpec(ABC): @abstractmethod def get_handlers( self, kv_caches: CanonicalKVCaches - ) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]: + ) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], "OffloadingHandler"]]: """ Get offloading handlers along with their respective src and dst types. diff --git a/vllm/v1/kv_offload/cpu/gpu_worker.py b/vllm/v1/kv_offload/cpu/gpu_worker.py index 2da3f1038e..119778368c 100644 --- a/vllm/v1/kv_offload/cpu/gpu_worker.py +++ b/vllm/v1/kv_offload/cpu/gpu_worker.py @@ -124,6 +124,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): block_size_factor: int, kv_cache_groups_data_refs: list[list[CanonicalKVCacheRef]], gpu_to_cpu: bool, + mmap_region: SharedOffloadRegion | None = None, ): """ Initialize a SingleDirectionOffloadingHandler. @@ -167,6 +168,8 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): self.dst_block_size_factor = block_size_factor if self.gpu_to_cpu else 1 self.transfer_type = ("GPU", "CPU") if self.gpu_to_cpu else ("CPU", "GPU") + # mmap_region to clean up on shutdown (gpu_to_cpu handler owns it) + self._mmap_region = mmap_region # job_id -> event self._transfer_events: dict[int, torch.Event] = {} # queue of transfers (job_id, stream, event) @@ -382,6 +385,9 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): self._event_pool.clear() self.src_tensors.clear() self.dst_tensors.clear() + if self._mmap_region is not None: + self._mmap_region.cleanup() + self._mmap_region = None class CpuGpuOffloadingHandlers: @@ -434,6 +440,7 @@ class CpuGpuOffloadingHandlers: block_size_factor=block_size_factor, kv_cache_groups_data_refs=kv_caches.group_data_refs, gpu_to_cpu=True, + mmap_region=mmap_region, ) self.cpu_to_gpu_handler = SingleDirectionOffloadingHandler( diff --git a/vllm/v1/kv_offload/cpu/shared_offload_region.py b/vllm/v1/kv_offload/cpu/shared_offload_region.py index b2e21d06c9..1166b44fc7 100644 --- a/vllm/v1/kv_offload/cpu/shared_offload_region.py +++ b/vllm/v1/kv_offload/cpu/shared_offload_region.py @@ -153,6 +153,20 @@ class SharedOffloadRegion: self._views.append(worker_layer_view) return worker_layer_view + def create_kv_memoryview(self) -> memoryview: + """Return a zero-copy memoryview over the entire KV buffer. + + Shape: (num_blocks, row_stride_bytes). Secondary tiers address + block *b* as ``view[b]``. + """ + kv_tensor = self._base.view(self.num_blocks, self._row_stride) + np_arr = kv_tensor.numpy() + assert np_arr.ctypes.data == self._base.data_ptr(), ( + "view()/numpy() created a copy instead of sharing the mmap buffer; " + "secondary tiers require zero-copy access to primary KV data" + ) + return memoryview(np_arr) + def cleanup(self) -> None: if self.is_pinned and self._base is not None: base_ptr = self._base.data_ptr() diff --git a/vllm/v1/kv_offload/cpu/spec.py b/vllm/v1/kv_offload/cpu/spec.py index 754c60f132..99265e09c0 100644 --- a/vllm/v1/kv_offload/cpu/spec.py +++ b/vllm/v1/kv_offload/cpu/spec.py @@ -44,6 +44,10 @@ class CPUOffloadingSpec(OffloadingSpec): if kv_bytes_per_offloaded_block > 0 else 0 ) + world_size = vllm_config.parallel_config.world_size + self.cpu_page_size_per_worker: int = ( + kv_bytes_per_offloaded_block // world_size if world_size > 0 else 0 + ) # scheduler-side self._manager: OffloadingManager | None = None @@ -77,6 +81,15 @@ class CPUOffloadingSpec(OffloadingSpec): ) return self._manager + def _create_handlers( + self, kv_caches: CanonicalKVCaches + ) -> CpuGpuOffloadingHandlers: + return CpuGpuOffloadingHandlers( + kv_caches=kv_caches, + block_size_factor=self.block_size_factor, + num_cpu_blocks=self.num_blocks, + ) + def get_handlers( self, kv_caches: CanonicalKVCaches ) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]: @@ -85,12 +98,7 @@ class CPUOffloadingSpec(OffloadingSpec): raise Exception( "CPU Offloading is currently only supported on CUDA-alike GPUs" ) - - self._handlers = CpuGpuOffloadingHandlers( - kv_caches=kv_caches, - block_size_factor=self.block_size_factor, - num_cpu_blocks=self.num_blocks, - ) + self._handlers = self._create_handlers(kv_caches) assert self._handlers is not None yield GPULoadStoreSpec, CPULoadStoreSpec, self._handlers.gpu_to_cpu_handler diff --git a/vllm/v1/kv_offload/factory.py b/vllm/v1/kv_offload/factory.py index ecfc270b64..8b967f771b 100644 --- a/vllm/v1/kv_offload/factory.py +++ b/vllm/v1/kv_offload/factory.py @@ -56,3 +56,8 @@ class OffloadingSpecFactory: OffloadingSpecFactory.register_spec( "CPUOffloadingSpec", "vllm.v1.kv_offload.cpu.spec", "CPUOffloadingSpec" ) +OffloadingSpecFactory.register_spec( + "TieringOffloadingSpec", + "vllm.v1.kv_offload.tiering.spec", + "TieringOffloadingSpec", +) diff --git a/vllm/v1/kv_offload/tiering/__init__.py b/vllm/v1/kv_offload/tiering/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/v1/kv_offload/tiering/base.py b/vllm/v1/kv_offload/tiering/base.py new file mode 100644 index 0000000000..ace325185a --- /dev/null +++ b/vllm/v1/kv_offload/tiering/base.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Abstract interfaces and data types for the secondary tiering layer. +""" + +from abc import ABC, abstractmethod +from collections.abc import Collection, Iterable +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import numpy as np + +from vllm.v1.kv_offload.base import OffloadKey, ReqContext + +if TYPE_CHECKING: + from vllm.config import VllmConfig + +# Type alias for job IDs used in async transfer tracking +JobId = int + + +@dataclass +class JobMetadata: + """Metadata for an in-flight async transfer job.""" + + job_id: JobId + keys: Collection[OffloadKey] + block_ids: np.ndarray + is_promotion: bool + req_context: ReqContext + + +@dataclass +class JobResult: + """Result of an async transfer job (successful or failed).""" + + job_id: JobId + success: bool + + +class SecondaryTierManager(ABC): + """ + Abstract interface for managing a single non-primary offloading tier. + + Secondary tiers cannot directly access GPU memory. All data transfers + must go through the CPU (primary) tier: + - Store: GPU → CPU (primary) → secondary (cascade) + - Load: secondary → CPU (primary) → GPU (promotion) + + IMPORTANT: All methods run in the Scheduler process and must be + lightweight and non-blocking. submit_load() and submit_store() submit + async jobs; get_finished() polls for completion. + """ + + def __init__(self, vllm_config: "VllmConfig", primary_kv_view: memoryview) -> None: + self._vllm_config = vllm_config + self._primary_kv_view: memoryview = primary_kv_view + + @abstractmethod + def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None: + """ + Check whether a block exists in this secondary tier. + + Args: + key: Offload key to look up. + req_context: per-request context (e.g. kv_transfer_params). + + Returns: + True if the block is present and ready, + False if not found, + or None if the block is being transferred (retry later). + """ + pass + + @abstractmethod + def submit_store(self, job_metadata: JobMetadata) -> None: + """ + Submit an async job to store blocks from the primary tier to this + secondary tier. + + This method must be lightweight and non-blocking: allocate metadata + and submit the transfer, but do NOT perform the data copy on the + calling thread. + + Preconditions (guaranteed by the framework): + - ``job_metadata.block_ids`` are valid primary-tier slots, pinned + (ref-counted) for the duration of the transfer. + + The implementation is responsible for: + 1. Filtering out blocks already present in this tier + 2. Evicting blocks if capacity is needed + 3. Allocating space in this tier + 4. Submitting the async transfer (read from primary via block_ids) + + Report completion via ``get_finished()``. + + Args: + job_metadata: Job metadata including job_id, keys, and block_ids + identifying the primary-tier slots to read from. + """ + pass + + @abstractmethod + def submit_load(self, job_metadata: JobMetadata) -> None: + """ + Submit an async job to load blocks from this secondary tier to the + primary tier. + + This method must be lightweight and non-blocking: mark blocks as + in-flight and submit the transfer, but do NOT perform the data copy + on the calling thread. + + Preconditions (guaranteed by the framework): + - ``job_metadata.block_ids`` are allocated primary-tier slots + ready to receive data. + + The implementation must copy data from this tier into the + primary-tier slots identified by ``block_ids``. + + Report completion via ``get_finished()``. + + Args: + job_metadata: Job metadata including job_id, keys, and block_ids + identifying the primary-tier slots to write into. + """ + pass + + @abstractmethod + def get_finished(self) -> Iterable[JobResult]: + """ + Return all jobs (loads and stores) that completed since the last call. + + The framework uses these results to release resources and finalize + transfers. + + Returns: + Iterable of JobResult objects for jobs finished since the + last call. + """ + pass + + def touch(self, keys: Collection[OffloadKey], req_context: ReqContext): + """ + Mark blocks as recently used for eviction policy. + + Args: + keys: Offload keys to mark as recently used. + req_context: Per-request context. + """ + return + + def shutdown(self) -> None: + """Release resources held by this tier (threads, connections, etc.).""" + return + + @staticmethod + @abstractmethod + def get_tier_type() -> str: + """ + Get the type identifier of this tier (e.g., "example", "storage"). + + Must match the "type" field in the tier config dict. + + Returns: + Tier type string. + """ + pass diff --git a/vllm/v1/kv_offload/tiering/example/__init__.py b/vllm/v1/kv_offload/tiering/example/__init__.py new file mode 100644 index 0000000000..8c3edb693f --- /dev/null +++ b/vllm/v1/kv_offload/tiering/example/__init__.py @@ -0,0 +1,235 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +ExampleSecondaryTier: A simple in-memory secondary tier for testing. + +This implementation provides a minimal secondary tier that stores blocks +in memory (using a dictionary) and simulates async transfers with immediate +completion. It's useful for testing the TieringOffloadingManager without +requiring actual storage or network backends. +""" + +from collections import OrderedDict +from collections.abc import Collection, Iterable +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from vllm.v1.kv_offload.base import OffloadKey, ReqContext +from vllm.v1.kv_offload.tiering.base import ( + JobId, + JobMetadata, + JobResult, + SecondaryTierManager, +) + +if TYPE_CHECKING: + from vllm.config import VllmConfig + + +@dataclass +class _JobMetadata: + """Internal metadata for tracking job details.""" + + job_id: JobId + keys: Collection[OffloadKey] + is_store: bool # True for store jobs, False for load jobs + + +class ExampleSecondaryTier(SecondaryTierManager): + """ + A simple in-memory secondary tier for testing. + + This implementation: + - Stores blocks in a dictionary (key -> True) + - Simulates async transfers with immediate completion + - Uses LRU eviction policy + """ + + def __init__( + self, + vllm_config: "VllmConfig", + primary_kv_view: memoryview, + max_blocks: int = 1000, + simulate_async: bool = False, + ): + """ + Initialize the example secondary tier. + + Args: + vllm_config: Global vLLM configuration. + primary_kv_view: Memoryview of the primary tier's CPU KV cache. + max_blocks: Maximum number of blocks this tier can store + simulate_async: If True, jobs complete on next get_finished() call. + If False, jobs complete immediately. + """ + super().__init__(vllm_config, primary_kv_view) + self.max_blocks = max_blocks + self.simulate_async = simulate_async + + # key -> True (only care about presence) + self.blocks: OrderedDict[OffloadKey, bool] = OrderedDict() + + # Completed jobs waiting to be retrieved by get_finished() + self.completed_jobs: list[JobResult] = [] + + # Pending jobs (for simulated async mode) + self.pending_jobs: list[_JobMetadata] = [] + + def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None: + """ + Check whether a block exists in this secondary tier. + + Args: + key: Offload key to look up. + req_context: Per-request context. + + Returns: + True if the block is present, False if not found. + """ + return key in self.blocks + + def submit_store(self, job_metadata: JobMetadata) -> None: + """ + Submit an async job to store blocks from primary tier to this tier. + + Args: + job_metadata: Job metadata including job_id, keys, and + spec for reading blocks from the primary tier. + """ + job_id = job_metadata.job_id + keys = job_metadata.keys + block_ids = job_metadata.block_ids + + assert len(keys) == len(block_ids), ( + f"Length mismatch: {len(keys)} keys but {len(block_ids)} block_ids" + ) + + # Filter out blocks already present + blocks_to_store = [bh for bh in keys if bh not in self.blocks] + + if not blocks_to_store: + # All blocks already present + return + + # Evict blocks if needed (LRU policy) + num_blocks_to_evict = len(blocks_to_store) - ( + self.max_blocks - len(self.blocks) + ) + + evicted = [] + if num_blocks_to_evict > 0: + # Collect eviction candidates first (LRU order), then delete atomically + protected = set(keys) + for key in self.blocks: + if key not in protected: + evicted.append(key) + if len(evicted) == num_blocks_to_evict: + break + else: + # Could not collect enough eviction candidates + return + for key in evicted: + del self.blocks[key] + + # Create internal job metadata + internal_job_metadata = _JobMetadata( + job_id=job_id, keys=blocks_to_store, is_store=True + ) + + if self.simulate_async: + # Job will complete on next get_finished() call + self.pending_jobs.append(internal_job_metadata) + else: + # Job completes immediately + self._complete_store_job(internal_job_metadata) + + def submit_load(self, job_metadata: JobMetadata) -> None: + """ + Submit an async job to load blocks from this tier to primary tier. + + Args: + job_metadata: Job metadata including job_id, keys, and + spec for writing blocks into the primary tier. + """ + job_id = job_metadata.job_id + keys = job_metadata.keys + block_ids = job_metadata.block_ids + + assert len(keys) == len(block_ids), ( + f"Length mismatch: {len(keys)} keys but {len(block_ids)} block_ids" + ) + + # Verify all blocks exist + for key in keys: + if key not in self.blocks: + return + + # Create internal job metadata + internal_job_metadata = _JobMetadata(job_id=job_id, keys=keys, is_store=False) + + if self.simulate_async: + # Job will complete on next get_finished() call + self.pending_jobs.append(internal_job_metadata) + else: + # Job completes immediately + self._complete_load_job(internal_job_metadata) + + def get_finished(self) -> Iterable[JobResult]: + """ + Poll for finished async jobs. + + Returns: + Iterable of JobResult objects for all jobs that have + finished since the last call. + """ + # Move pending jobs to completed + if self.simulate_async and self.pending_jobs: + for job_metadata in self.pending_jobs: + if job_metadata.is_store: + self._complete_store_job(job_metadata) + else: + self._complete_load_job(job_metadata) + self.pending_jobs.clear() + + # Return completed jobs + result = self.completed_jobs + self.completed_jobs = [] + return result + + def _complete_store_job(self, job_metadata: _JobMetadata): + """Complete a store job by adding blocks to storage.""" + for key in job_metadata.keys: + self.blocks[key] = True + # Return simplified JobResult (only job_id and success) + self.completed_jobs.append(JobResult(job_id=job_metadata.job_id, success=True)) + + def _complete_load_job(self, job_metadata: _JobMetadata): + """Complete a load job.""" + # Return simplified JobResult (only job_id and success) + self.completed_jobs.append(JobResult(job_id=job_metadata.job_id, success=True)) + + def touch(self, keys: Collection[OffloadKey], req_context: ReqContext): + """ + Mark blocks as recently used (move to end of LRU list). + + Args: + keys: Blocks to mark as recently used. + req_context: Per-request context. + """ + for key in reversed(list(keys)): + if key in self.blocks: + self.blocks.move_to_end(key) + + @staticmethod + def get_tier_type() -> str: + return "example" + + def get_num_blocks(self) -> int: + """Get the number of blocks currently stored in this tier.""" + return len(self.blocks) + + def clear(self): + """Clear all blocks and pending jobs (for testing).""" + self.blocks.clear() + self.completed_jobs.clear() + self.pending_jobs.clear() diff --git a/vllm/v1/kv_offload/tiering/factory.py b/vllm/v1/kv_offload/tiering/factory.py new file mode 100644 index 0000000000..e8f4a9b32d --- /dev/null +++ b/vllm/v1/kv_offload/tiering/factory.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Factory for creating secondary tier implementations. +""" + +from typing import TYPE_CHECKING + +from vllm.v1.kv_offload.tiering.base import SecondaryTierManager +from vllm.v1.kv_offload.tiering.example import ExampleSecondaryTier + +if TYPE_CHECKING: + from vllm.config import VllmConfig + +SUPPORTED_TIERS: tuple[type[SecondaryTierManager], ...] = (ExampleSecondaryTier,) + +_TIER_REGISTRY: dict[str, type[SecondaryTierManager]] = { + cls.get_tier_type(): cls for cls in SUPPORTED_TIERS +} + + +def create_secondary_tier( + tier_config: dict, + primary_kv_view: memoryview, + vllm_config: "VllmConfig", +) -> SecondaryTierManager: + """ + Create a secondary tier from configuration. + + Args: + tier_config: Dictionary with tier configuration containing: + - type (required): Type of secondary tier (e.g., "example") + - Additional tier-specific parameters are passed directly + to the tier constructor + primary_kv_view: Memoryview of the primary tier's CPU KV cache. + vllm_config: Global vLLM configuration. + + Returns: + SecondaryTierManager instance + + Raises: + ValueError: If tier type is unknown or configuration is invalid + """ + config = tier_config.copy() + + tier_type = config.pop("type", None) + if not tier_type: + raise ValueError("Secondary tier configuration must include 'type'") + + cls = _TIER_REGISTRY.get(tier_type) + if cls is None: + raise ValueError( + f"Unknown secondary tier type: {tier_type!r}. " + f"Supported types: {list(_TIER_REGISTRY)}" + ) + return cls(vllm_config=vllm_config, primary_kv_view=primary_kv_view, **config) diff --git a/vllm/v1/kv_offload/tiering/manager.py b/vllm/v1/kv_offload/tiering/manager.py new file mode 100644 index 0000000000..578992fdd7 --- /dev/null +++ b/vllm/v1/kv_offload/tiering/manager.py @@ -0,0 +1,505 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +TieringOffloadingManager: Multi-tier KV cache offloading orchestrator. + +This manager coordinates between a CPU primary tier (with direct GPU access) +and zero or more secondary tiers (Storage, Network, etc.) to provide +hierarchical KV cache offloading. + +Key Design Principles: +1. Always offload to all tiers — When a block is stored to the primary tier, + it is cascaded to ALL secondary tiers +2. Primary tier is the gateway — Secondary tiers cannot access GPU memory + directly; all data flows through the CPU primary tier +3. Staged promotion — Blocks in secondary tiers must be promoted to the + primary tier before GPU can access them +4. Transparent retry mechanism — Return None from lookup() to signal + "data is being promoted, try later" +5. ref_cnt as eviction protection — primary.prepare_read() increments ref_cnt, + protecting blocks from eviction until complete_read() is called +""" + +from collections.abc import Collection, Iterable +from dataclasses import dataclass, field + +import numpy as np + +from vllm.logger import init_logger +from vllm.v1.kv_offload.base import ( + LoadStoreSpec, + OffloadingEvent, + OffloadingManager, + OffloadKey, + PrepareStoreOutput, + ReqContext, +) +from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec +from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager +from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion +from vllm.v1.kv_offload.tiering.base import ( + JobId, + JobMetadata, + SecondaryTierManager, +) + +logger = init_logger(__name__) + + +@dataclass +class PendingPromotion: + """Accumulator for blocks awaiting submit_load() for one (tier, request).""" + + req_context: ReqContext + keys: list[OffloadKey] = field(default_factory=list) + block_ids: list[int] = field(default_factory=list) + + +class CPUPrimaryTierOffloadingManager(CPUOffloadingManager): + """CPUOffloadingManager with a primary/secondary transfer interface. + + The inherited prepare_store/complete_store/prepare_load/complete_load are the + GPU-facing OffloadingManager interface. These aliases expose the same operations + from the secondary tier perspective, where read/write refers to secondary + accessing primary. This avoids confusion when reading TieringOffloadingManager + code (e.g. calling prepare_load inside a cascade/store path would be misleading). + """ + + def __init__( + self, + num_blocks: int, + mmap_region: SharedOffloadRegion, + cache_policy: str = "lru", + enable_events: bool = False, + ): + super().__init__( + num_blocks=num_blocks, + cache_policy=cache_policy, # type: ignore[arg-type] + enable_events=enable_events, + ) + self._mmap_region = mmap_region + # read/write is for CPU<->secondary transfers, + # load/store is for CPU<->GPU transfers. + # These aliases avoid calling prepare_load inside a store path. + self.prepare_read = self.prepare_load + self.complete_read = self.complete_load + self.prepare_write = self.prepare_store + self.complete_write = self.complete_store + + self._kv_memoryview = mmap_region.create_kv_memoryview() + + def get_kv_memoryview(self) -> memoryview: + """Return the memoryview over the primary tier's KV cache buffer. + + The view has shape (num_blocks, row_stride_bytes) and is backed by the + SharedOffloadRegion mmap. Secondary tiers address block *b* as + ``view[b]``. + """ + return self._kv_memoryview + + def shutdown(self) -> None: + super().shutdown() + self._kv_memoryview.release() + self._mmap_region.cleanup() + + +class TieringOffloadingManager(OffloadingManager): + """ + Orchestrates multi-tier KV cache offloading. + + This manager coordinates between a CPU primary tier (with direct GPU access) + and zero or more secondary tiers (Storage, Network, etc.) to provide + hierarchical KV cache offloading. + + Key internal state: + - Minimal state tracking; relies on secondary tiers to report completion + via get_finished() + - Secondary tiers return JobResult objects containing all necessary + information + - job_id_counter: monotonically increasing counter for job IDs + """ + + def __init__( + self, + primary_tier: CPUPrimaryTierOffloadingManager, + secondary_tiers: list[SecondaryTierManager] | None = None, + enable_events: bool = False, + ): + """ + Initialize the TieringOffloadingManager. + + Args: + primary_tier: The primary tier manager (CPU-based). + secondary_tiers: List of secondary tier managers (e.g., Storage, + Network). Can be None or empty list. + enable_events: Whether to track offloading events + """ + self.primary_tier: CPUPrimaryTierOffloadingManager = primary_tier + self.secondary_tiers = secondary_tiers or [] + + self._job_id_counter: int = 0 + self.events: list[OffloadingEvent] | None = [] if enable_events else None + + # Job tracking: maps job_id to metadata for all in-flight transfers. + # JobMetadata.is_promotion distinguishes direction: + # True: secondary → primary (promotion) + # False: primary → secondary (cascade) + self._transfer_jobs: dict[JobId, JobMetadata] = {} + + # Pending promotion requests accumulated during lookup() calls; flushed + # as one batched submit_load() per (tier, request) in take_events(). + # Outer key: tier. Inner key: req_context.req_id — the same ReqContext + # object is reused for all block lookups of a given request per engine step. + self._pending_load_submissions: dict[ + SecondaryTierManager, dict[str, PendingPromotion] + ] = {} + + # Gate for once-per-step execution of _maybe_process_finished_jobs(). + # Reset at the end of each step in take_events(). + self._processed_jobs_this_step: bool = False + + def _next_job_id(self) -> JobId: + """Generate a unique job ID for async transfer tracking.""" + job_id = self._job_id_counter + self._job_id_counter += 1 + return job_id + + def _maybe_process_finished_jobs(self): + """ + Poll secondary tiers for completed jobs (at most once per step). + + Guarded by _processed_jobs_this_step: the first call in an engine step + does the actual polling; subsequent calls are no-ops. The flag is reset + in take_events() at the end of each step. + """ + if self._processed_jobs_this_step: + return + self._processed_jobs_this_step = True + self._process_finished_jobs() + + def _process_finished_jobs(self): + """ + Unconditionally poll all secondary tiers for completed jobs. + + This method: + 1. Calls get_finished() on each secondary tier + 2. For completed stores (primary→secondary): calls primary.complete_read() + to decrement ref_cnt + 3. For completed loads (secondary→primary): calls primary.complete_write() + to make blocks available + """ + for i, tier in enumerate(self.secondary_tiers): + for completed_job in tier.get_finished(): + job_id = completed_job.job_id + job_metadata = self._transfer_jobs.pop(job_id, None) + assert job_metadata is not None, ( + f"Finished job_id {job_id} from tier #{i}" + f" ({tier.get_tier_type()}) not in _transfer_jobs" + ) + + if job_metadata.is_promotion: + # secondary→primary transfer (promotion) completed. + # Make blocks available in primary tier. + self.primary_tier.complete_write( + job_metadata.keys, + job_metadata.req_context, + completed_job.success, + ) + else: + # primary→secondary transfer completed. + # Decrement ref_cnt on primary blocks. + self.primary_tier.complete_read( + job_metadata.keys, job_metadata.req_context + ) + + def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None: + """ + Check whether a single block is offloaded and ready. + + Algorithm: + 1. Process any completed async jobs first. + 2. Query primary tier — short-circuit on hit or in-flight. + 3. On primary miss, query secondary tiers — stop on first + hit and initiate promotion. + + Args: + key: Block hash to look up. + req_context: Per-request context. + + Returns: + True — block is ready in the primary tier. + None — block found but not yet ready (primary in-flight, + promotion started, or a secondary tier is busy). + False — block not found in any tier, or primary is full + and cannot accept a promotion. + """ + self._maybe_process_finished_jobs() + + primary_hit = self.primary_tier.lookup(key, req_context) + if primary_hit is True: + return True + if primary_hit is None: + return None + + any_none = False + for tier in self.secondary_tiers: + result = tier.lookup(key, req_context) + if result is True: + if not self._initiate_promotion(tier, key, req_context): + return False # primary full, block unavailable + return None # promotion started, retry later + if result is None: + any_none = True + + if any_none: + return None + return False + + def _initiate_promotion( + self, + tier: SecondaryTierManager, + key: OffloadKey, + req_context: ReqContext, + ) -> bool: + """ + Queue a block for promotion from a secondary tier to the primary tier. + + Allocates space in the primary tier immediately (sets ref_cnt=-1 so + subsequent lookups within the same step see the slot as in-flight), + then defers the actual submit_load() call to _flush_pending_promotions() + so all blocks queued during one engine step are submitted as a single + batched job. + + Args: + tier: The secondary tier to promote from + key: Block to promote + req_context: Per-request context forwarded to primary.prepare_write(). + + Returns: + True if promotion was initiated, False if primary tier is full. + """ + # Allocate space in primary tier for promoted block. + # Must happen immediately so primary.lookup() returns None (in-flight) + # for this key on any subsequent lookup() call within the same step, + # preventing duplicate promotion attempts. + primary_write_result = self.primary_tier.prepare_write([key], req_context) + + if primary_write_result is None: + # Primary tier is full; caller should treat the block as unavailable + # rather than retrying indefinitely. + return False + + store_spec = primary_write_result.store_spec + assert isinstance(store_spec, CPULoadStoreSpec) + # Defer submit_load to take_events(). Group by (tier, request) so each + # request's blocks are submitted as one batched job per tier. + tier_pending = self._pending_load_submissions.setdefault(tier, {}) + ctx_id = req_context.req_id + if ctx_id not in tier_pending: + tier_pending[ctx_id] = PendingPromotion( + keys=[], block_ids=[], req_context=req_context + ) + entry = tier_pending[ctx_id] + entry.keys.extend(primary_write_result.keys_to_store) + entry.block_ids.extend(store_spec.block_ids) + return True + + def _flush_pending_promotions(self) -> None: + """Submit one batched submit_load() per (tier, request). + + Called from take_events() at the end of each engine step, flushing + all promotion requests deferred during lookup(). + """ + if not self._pending_load_submissions: + return + + for tier, pending_by_ctx in self._pending_load_submissions.items(): + for entry in pending_by_ctx.values(): + job_id = self._next_job_id() + job_metadata = JobMetadata( + job_id=job_id, + keys=entry.keys, + block_ids=np.array(entry.block_ids, dtype=np.int64), + is_promotion=True, + req_context=entry.req_context, + ) + self._transfer_jobs[job_id] = job_metadata + tier.submit_load(job_metadata) + + self._pending_load_submissions.clear() + + def prepare_load( + self, keys: Collection[OffloadKey], req_context: ReqContext + ) -> LoadStoreSpec: + """ + Prepare blocks to be loaded from primary tier to GPU. + + CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure + that any completed promotions have been finalized and blocks are ready. + + This increments ref_cnt on the blocks in the primary tier, protecting + them from eviction during the transfer. + + Args: + keys: Blocks to prepare for loading. + req_context: Per-request context. + + Returns: + LoadStoreSpec for reading from primary tier. + """ + # Process completed promotions to ensure blocks are ready + self._maybe_process_finished_jobs() + + return self.primary_tier.prepare_load(keys, req_context) + + def touch(self, keys: Collection[OffloadKey], req_context: ReqContext): + """ + Mark blocks as recently used in all tiers. + + Args: + keys: Blocks to mark as recently used. + req_context: Per-request context. + """ + self.primary_tier.touch(keys, req_context) + for tier in self.secondary_tiers: + tier.touch(keys, req_context) + + def complete_load(self, keys: Collection[OffloadKey], req_context: ReqContext): + """ + Mark blocks as done loading from primary tier to GPU. + + This decrements ref_cnt on the blocks in the primary tier, allowing + them to be evicted again. + + Args: + keys: Blocks that finished loading. + req_context: Per-request context. + """ + self.primary_tier.complete_load(keys, req_context) + + def prepare_store( + self, keys: Collection[OffloadKey], req_context: ReqContext + ) -> PrepareStoreOutput | None: + """ + Prepare blocks to be stored from GPU to primary tier. + + CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure + that any completed async transfers have their ref_cnt decremented + before the primary tier makes eviction decisions. + + Args: + keys: Blocks to prepare for storing. + req_context: Per-request context. + + Returns: + PrepareStoreOutput describing where to store blocks and what was + evicted, or None if store cannot proceed. + """ + # Step 1: Poll for completed async jobs FIRST + # This decrements ref_cnt on primary blocks that have been + # successfully transferred to secondary tiers. + self._maybe_process_finished_jobs() + + # Step 2: Store to primary tier + primary_result = self.primary_tier.prepare_store(keys, req_context) + + # Note: Secondary tier cascading will happen in complete_store() + # after the GPU→Primary transfer completes and blocks are ready. + + return primary_result + + def complete_store( + self, + keys: Collection[OffloadKey], + req_context: ReqContext, + success: bool = True, + ): + """ + Mark blocks as done storing from GPU to primary tier. + + This is where secondary tier cascading happens — after blocks are + confirmed to be in the primary tier, they are cascaded to ALL + secondary tiers. + + For each secondary tier: + 1. Call primary.prepare_read() to get LoadStoreSpec AND increment + ref_cnt (protecting blocks during async transfer) + 2. Call tier.submit_store() to start async transfer: primary→secondary + 3. Track the job in _store_jobs dictionary + + Args: + keys: Blocks that finished storing. + success: Whether the GPU→primary transfer succeeded. + req_context: Per-request context forwarded to primary.prepare_read(). + """ + # Step 1: Complete store in primary tier (makes blocks loadable) + self.primary_tier.complete_store(keys, req_context, success) + + if not success: + # If GPU→Primary transfer failed, don't cascade to secondary tiers + return + + # Step 2: Cascade to ALL secondary tiers + # For each secondary tier, call primary.prepare_read() to get the + # LoadStoreSpec AND to increment ref_cnt (protecting blocks from + # eviction during the async transfer). One prepare_read() call per + # secondary tier. + for tier in self.secondary_tiers: + primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context) + + # Submit async store job: primary→secondary + job_id = self._next_job_id() + + # Track this store job + assert isinstance(primary_blocks_spec, CPULoadStoreSpec) + job_metadata = JobMetadata( + job_id=job_id, + keys=keys, + block_ids=primary_blocks_spec.block_ids, + is_promotion=False, + req_context=req_context, + ) + self._transfer_jobs[job_id] = job_metadata + + tier.submit_store(job_metadata) + + # Note: The async transfers are now in flight. Their completion is + # tracked via get_finished() / _maybe_process_finished_jobs(). + + def take_events(self) -> Iterable[OffloadingEvent]: + """ + End-of-step hook: flush deferred work, yield events, reset per-step state. + + Called once per engine step from Scheduler.update_from_output() → + connector.take_events(). Ensures _maybe_process_finished_jobs() has run + at least once this step, flushes pending promotions, yields collected + events, and resets the per-step flag. + + Yields: + New OffloadingEvents collected since the last call. + """ + # TODO: Move _flush_pending_promotions() to a dedicated end_of_batch() + # hook once one exists. For now, take_events() serves as the flush + # point under the assumption that it is called at the end of each + # engine step (Scheduler.update_from_output() → connector.take_events()). + # When the dedicated hook is added, update tests that rely on + # take_events() to signal end of step. + + self._maybe_process_finished_jobs() + + self._flush_pending_promotions() + + # Reset the per-step gate so next step's first call does real work. + self._processed_jobs_this_step = False + + if self.events is not None: + yield from self.events + self.events.clear() + + yield from self.primary_tier.take_events() + + def shutdown(self) -> None: + """Shutdown all tiers and release resources.""" + for tier in self.secondary_tiers: + tier.shutdown() + self.primary_tier.shutdown() diff --git a/vllm/v1/kv_offload/tiering/spec.py b/vllm/v1/kv_offload/tiering/spec.py new file mode 100644 index 0000000000..0a372ae8ef --- /dev/null +++ b/vllm/v1/kv_offload/tiering/spec.py @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +TieringOffloadingSpec: Spec for multi-tier KV cache offloading. + +This spec creates a TieringOffloadingManager with a CPU primary tier +and configurable secondary tiers (e.g., Storage, Network). + +Configuration via kv_connector_extra_config: + - cpu_bytes_to_use: (required) Bytes to allocate for CPU primary tier + - block_size: (optional) Block size for offloaded blocks (default: GPU block size) + - eviction_policy: (optional) Primary tier eviction policy: "lru" or + "arc" (default: "lru") + - secondary_tiers: (optional) List of secondary tier configurations + Each secondary tier config is a dict with: + - type: (required) Type of secondary tier (e.g., "example", "storage", "network") + - Additional tier-specific parameters are passed directly to the tier + constructor. See each tier's documentation for supported parameters. + +Example configuration: +{ + "cpu_bytes_to_use": 10737418240, # 10 GB + "block_size": 16, + "eviction_policy": "lru", + "secondary_tiers": [ + { + "type": "example", + # Tier-specific parameters (for ExampleSecondaryTier): + "max_blocks": 10000, + "simulate_async": False + } + ] +} +""" + +import torch + +from vllm.config import VllmConfig +from vllm.logger import init_logger +from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.kv_offload.base import CanonicalKVCaches, OffloadingManager +from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers +from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion +from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec +from vllm.v1.kv_offload.tiering.factory import create_secondary_tier +from vllm.v1.kv_offload.tiering.manager import ( + CPUPrimaryTierOffloadingManager, + TieringOffloadingManager, +) + +logger = init_logger(__name__) + + +class TieringOffloadingSpec(CPUOffloadingSpec): + """ + Spec for multi-tier KV cache offloading. + + Creates a TieringOffloadingManager with: + - Primary tier: CPU (LRU or ARC eviction policy) + - Secondary tiers: Configurable via extra_config + + The CPU primary tier has direct GPU access and serves as the gateway for + all GPU↔offload operations. Secondary tiers cannot directly access GPU + memory and must transfer data through the primary tier. + """ + + def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig): + super().__init__(vllm_config, kv_cache_config) + # Redeclare for mypy: parent sets this but `--follow-imports skip` hides it + self._manager: OffloadingManager | None = None + + # Parse secondary tier configurations + self.secondary_tier_configs = self.extra_config.get("secondary_tiers", []) + if not isinstance(self.secondary_tier_configs, list): + raise ValueError("secondary_tiers must be a list of tier configurations") + + # Scheduler-side mmap (rank=None); kept for cleanup + self._scheduler_mmap: SharedOffloadRegion | None = None + + def get_manager(self) -> OffloadingManager: + """ + Get the TieringOffloadingManager. + + Creates a TieringOffloadingManager with: + - Primary tier: CPU (LRU or ARC) + - Secondary tiers: As configured in extra_config + + Returns: + TieringOffloadingManager instance + """ + if not self._manager: + kv_events_config = self.vllm_config.kv_events_config + enable_events = ( + kv_events_config is not None and kv_events_config.enable_kv_cache_events + ) + + # Create scheduler-side SharedOffloadRegion (rank=None) so the + # primary tier can eagerly create a memoryview over _base. + world_size = self.vllm_config.parallel_config.world_size + scheduler_mmap = SharedOffloadRegion( + instance_id=self.vllm_config.instance_id, + total_size_bytes=self.cpu_page_size_per_worker + * world_size + * self.num_blocks, + num_blocks=self.num_blocks, + rank=None, + num_workers=world_size, + cpu_page_size=self.cpu_page_size_per_worker, + ) + self._scheduler_mmap = scheduler_mmap + + # Create primary tier (CPU-based) + assert len(self.gpu_block_size) == 1 + primary_tier = CPUPrimaryTierOffloadingManager( + num_blocks=self.num_blocks, + cache_policy=self.eviction_policy, # type: ignore[arg-type] + enable_events=enable_events, + mmap_region=scheduler_mmap, + ) + + # Create secondary tiers + primary_kv_view = primary_tier.get_kv_memoryview() + secondary_tiers = [] + for i, tier_config in enumerate(self.secondary_tier_configs): + try: + tier = create_secondary_tier( + tier_config, primary_kv_view, self.vllm_config + ) + secondary_tiers.append(tier) + logger.info( + "Created secondary tier #%d (%s)", + i, + tier.get_tier_type(), + ) + except Exception as e: + logger.error( + "Failed to create secondary tier from config %s: %s", + tier_config, + e, + ) + raise + + # Create TieringOffloadingManager. GPU↔CPU transfers use the inherited + # get_handlers(); secondary tier transfers are handled by the + # secondary tier managers and need no additional handlers here. + tiering_manager = TieringOffloadingManager( + primary_tier=primary_tier, + secondary_tiers=secondary_tiers, + enable_events=enable_events, + ) + if int(self.extra_config.get("store_threshold", 0)) >= 2: + raise ValueError( + "store_threshold is not supported for TieringOffloadingSpec" + ) + self._manager = tiering_manager + + logger.info( + "Created TieringOffloadingManager with primary tier " + "(%s, %s blocks) and %s secondary tier(s)", + self.eviction_policy, + self.num_blocks, + len(secondary_tiers), + ) + + return self._manager + + def _create_handlers( + self, kv_caches: CanonicalKVCaches + ) -> CpuGpuOffloadingHandlers: + world_size = self.vllm_config.parallel_config.world_size + rank = torch.accelerator.current_device_index() + worker_mmap = SharedOffloadRegion( + instance_id=self.vllm_config.instance_id, + total_size_bytes=self.cpu_page_size_per_worker + * world_size + * self.num_blocks, + num_blocks=self.num_blocks, + rank=rank, + num_workers=world_size, + cpu_page_size=self.cpu_page_size_per_worker, + ) + return CpuGpuOffloadingHandlers( + kv_caches=kv_caches, + block_size_factor=self.block_size_factor, + num_cpu_blocks=self.num_blocks, + mmap_region=worker_mmap, + )