From dc1be79031d948d7a18c37600881e45ca708d913 Mon Sep 17 00:00:00 2001 From: Philip Pesic <116986036+philippesic@users.noreply.github.com> Date: Wed, 29 Jul 2026 04:34:55 +0000 Subject: [PATCH] Add CachePolicyFactory for pluggable/external eviction policies (#49114) Signed-off-by: Philip Pesic --- docs/features/kv_offloading_usage.md | 33 +++++- tests/v1/kv_offload/cpu/__init__.py | 0 tests/v1/kv_offload/cpu/policies/__init__.py | 0 .../kv_offload/cpu/policies/test_factory.py | 111 ++++++++++++++++++ tests/v1/kv_offload/cpu/test_manager.py | 2 + vllm/v1/kv_offload/base.py | 8 -- vllm/v1/kv_offload/cpu/manager.py | 25 ++-- vllm/v1/kv_offload/cpu/policies/arc.py | 2 +- vllm/v1/kv_offload/cpu/policies/base.py | 4 +- vllm/v1/kv_offload/cpu/policies/factory.py | 87 ++++++++++++++ vllm/v1/kv_offload/cpu/policies/lru.py | 1 + vllm/v1/kv_offload/cpu/spec.py | 6 +- vllm/v1/kv_offload/factory.py | 7 ++ vllm/v1/kv_offload/tiering/manager.py | 4 +- vllm/v1/kv_offload/tiering/spec.py | 12 +- 15 files changed, 269 insertions(+), 33 deletions(-) create mode 100644 tests/v1/kv_offload/cpu/__init__.py create mode 100644 tests/v1/kv_offload/cpu/policies/__init__.py create mode 100644 tests/v1/kv_offload/cpu/policies/test_factory.py create mode 100644 vllm/v1/kv_offload/cpu/policies/factory.py diff --git a/docs/features/kv_offloading_usage.md b/docs/features/kv_offloading_usage.md index 13dbd5299d..72f838c7d2 100644 --- a/docs/features/kv_offloading_usage.md +++ b/docs/features/kv_offloading_usage.md @@ -70,7 +70,8 @@ vllm serve \ | `cpu_bytes_to_use` | yes | — | both | Total bytes of host memory reserved for the CPU tier across all workers (not per-worker). | | `block_size` | no | GPU block size | both | Offloaded block size in tokens; must be a multiple of the GPU block size. Mutually exclusive with `blocks_per_chunk`. | | `blocks_per_chunk` | no | `1` | both | Offloaded chunk size in GPU blocks; must be > 0. Alternative to `block_size` for models whose KV cache groups have different block sizes. | -| `eviction_policy` | no | `lru` | both | Primary tier policy: `lru` or `arc`. | +| `eviction_policy` | no | `lru` | both | Primary tier policy: built-in `lru`/`arc`, or a custom `CachePolicy` name (see [Custom Eviction Policies](#custom-eviction-policies)). | +| `cache_policy_module_path` | no | — | both | Python import path for a custom `CachePolicy` not in the built-in registry. Required only when `eviction_policy` is not built-in and wasn't pre-registered via `CachePolicyFactory` (advanced). | | `store_threshold` | no | `0` | single-tier | Min lookups before a block is offloaded. Values ≥ 2 are rejected by `TieringOffloadingSpec`. | | `max_tracker_size` | no | `64000` | single-tier | Max entries in the lookup tracker. | | `secondary_tiers` | no | `[]` | multi-tier | List of secondary tier configs (see below). | @@ -78,6 +79,36 @@ vllm serve \ | `self_describing_kv_events` | no | `false` | both | Opt-in. When `true` *and* KV cache events are enabled (`--kv-events-config` with `enable_kv_cache_events`), the connector emits self-describing block-granular `BlockStored`/`BlockRemoved` payloads (constituent block hashes, whole-chunk `token_ids`, per-block `block_size`, parent hash, LoRA + group/cache-spec metadata) instead of the placeholder fallback, so external KV-event consumers can index offloaded blocks. Inert unless events are enabled. With `TieringOffloadingSpec`, a CPU promotion is self-describing when a local request observes its primary-tier `HIT` before event translation; otherwise its stored event may retain the placeholder, while a later `HIT` can backfill metadata for removal. Pending-removal/re-promotion races and externally initiated promotions may also produce placeholders, and consumers must ignore removals for unknown hashes. Full-attention groups only; sliding-window/SSM groups keep the placeholder fallback. In chunk mode (`block_size` > GPU block size, or `blocks_per_chunk` > 1), overlapping chunks re-announce shared per-block hashes, so consumers must reference-count (deduplicate) repeated store/remove announcements. | | `spec_module_path` | no | — | both | Python import path for a custom `OffloadingSpec` not in the built-in registry. Required only when `spec_name` is not built-in (advanced). | +## Custom Eviction Policies + +`eviction_policy` resolves through `CachePolicyFactory` (`vllm/v1/kv_offload/cpu/policies/factory.py`), which pre-registers the built-in `lru` and `arc` policies. + +### Out-of-tree (recommended) + +Implement `CachePolicy` (`vllm/v1/kv_offload/cpu/policies/base.py`) in your own package — no vLLM fork or patch required — and point `kv_connector_extra_config` at it directly: + +```json +{ + "cpu_bytes_to_use": 10737418240, + "eviction_policy": "MyCachePolicy", + "cache_policy_module_path": "my_package.my_module" +} +``` + +`eviction_policy` is checked against the built-in registry first; if it isn't a registered name, vLLM imports `cache_policy_module_path` and looks up `eviction_policy` as a class name in that module — the same fallback `spec_module_path` provides for a custom `OffloadingSpec`. No import or registration call needs to run before the server starts. + +### Registering a friendly short name (in-process only) + +If you control the process that constructs the vLLM engine (e.g. an embedding application), you can register a short name once at startup instead of repeating the module path in every config: + +```python +from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory + +CachePolicyFactory.register_cache_policy("my_policy", "my_package.my_module", "MyCachePolicy") +``` + +Then set `"eviction_policy": "my_policy"` in `kv_connector_extra_config`, the same as `"lru"`/`"arc"`. This only takes effect within the process that ran the `register_cache_policy` call — it does not help when the server is launched as a separate process (e.g. via the `vllm serve` CLI), where the out-of-tree `cache_policy_module_path` config above is the only option. + ## Secondary Tiers Each entry in `secondary_tiers` is a dict with a required `type` field plus tier-specific fields. diff --git a/tests/v1/kv_offload/cpu/__init__.py b/tests/v1/kv_offload/cpu/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/v1/kv_offload/cpu/policies/__init__.py b/tests/v1/kv_offload/cpu/policies/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/v1/kv_offload/cpu/policies/test_factory.py b/tests/v1/kv_offload/cpu/policies/test_factory.py new file mode 100644 index 0000000000..14ccf8b67e --- /dev/null +++ b/tests/v1/kv_offload/cpu/policies/test_factory.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Iterable + +import pytest + +from vllm.v1.kv_offload.base import OffloadKey, ReqContext +from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager +from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy +from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy +from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory +from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy + + +class _DummyCachePolicy(CachePolicy): + """Minimal CachePolicy for CachePolicyFactory registration tests. Loaded + by module path, so it must be importable at module scope (mirrors + tests/v1/kv_offload/test_factory.py's SingleArgExternalOffloadingSpec).""" + + def __init__(self, cache_capacity: int) -> None: + self.cache_capacity = cache_capacity + + def get(self, key: OffloadKey) -> BlockStatus | None: + return None + + def insert(self, key: OffloadKey, block: BlockStatus) -> None: + pass + + def remove(self, key: OffloadKey) -> None: + pass + + def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None: + pass + + def evict( + self, n: int, protected: set[OffloadKey] + ) -> list[tuple[OffloadKey, BlockStatus]] | None: + return None + + def clear(self) -> None: + pass + + +@pytest.fixture(autouse=True) +def restore_cache_policy_registry(): + """Save and restore CachePolicyFactory._registry between tests.""" + original = dict(CachePolicyFactory._registry) + yield + CachePolicyFactory._registry = original + + +class TestCachePolicyFactory: + """Unit tests for CachePolicyFactory (registration/resolution by name).""" + + def test_pre_registered_policies_can_be_imported(self): + """If someone moves a policy module but forgets to update + factory.py, CI fails.""" + for name in CachePolicyFactory._registry: + cls = CachePolicyFactory._registry[name]() + assert issubclass(cls, CachePolicy) + + def test_lru_and_arc_registered(self): + assert CachePolicyFactory.get_cache_policy_cls("lru") is LRUCachePolicy + assert CachePolicyFactory.get_cache_policy_cls("arc") is ARCCachePolicy + + def test_register_and_resolve_custom_policy(self): + CachePolicyFactory.register_cache_policy( + "dummy", + "tests.v1.kv_offload.cpu.policies.test_factory", + "_DummyCachePolicy", + ) + policy_cls = CachePolicyFactory.get_cache_policy_cls("dummy") + assert policy_cls is _DummyCachePolicy + + manager = CPUOffloadingManager(num_blocks=4, cache_policy="dummy") + assert isinstance(manager._policy, _DummyCachePolicy) + + def test_unregistered_policy_raises(self): + with pytest.raises(ValueError, match="Unknown cache policy"): + CachePolicyFactory.get_cache_policy_cls("nonexistent") + + def test_duplicate_registration_raises(self): + with pytest.raises(ValueError, match="is already registered"): + CachePolicyFactory.register_cache_policy("lru", "some.module", "SomeClass") + + def test_dynamic_load_via_cache_policy_module_path(self): + """Out-of-tree policy loaded via cache_policy_module_path, no + register_cache_policy() call -- this is how external projects + integrate a custom CachePolicy without forking/patching vLLM. + Mirrors tests/v1/kv_offload/test_factory.py's + test_dynamic_load_via_spec_module_path.""" + policy_cls = CachePolicyFactory.get_cache_policy_cls( + "_DummyCachePolicy", "tests.v1.kv_offload.cpu.policies.test_factory" + ) + assert policy_cls is _DummyCachePolicy + + def test_manager_resolves_policy_via_module_path(self): + """End-to-end: CPUOffloadingManager resolves an unregistered policy + purely from cache_policy_module_path.""" + manager = CPUOffloadingManager( + num_blocks=4, + cache_policy="_DummyCachePolicy", + cache_policy_module_path="tests.v1.kv_offload.cpu.policies.test_factory", + ) + assert isinstance(manager._policy, _DummyCachePolicy) + + def test_unregistered_policy_without_module_path_raises(self): + """eviction_policy not in registry + no cache_policy_module_path -> + ValueError, same shape as the OffloadingSpecFactory error path.""" + with pytest.raises(ValueError, match="Unknown cache policy"): + CachePolicyFactory.get_cache_policy_cls("nonexistent", None) diff --git a/tests/v1/kv_offload/cpu/test_manager.py b/tests/v1/kv_offload/cpu/test_manager.py index 6520a93fd1..5a36be16bf 100644 --- a/tests/v1/kv_offload/cpu/test_manager.py +++ b/tests/v1/kv_offload/cpu/test_manager.py @@ -37,6 +37,7 @@ _EMPTY_REQ_CTX = make_req_context() def make_cpu_manager( num_blocks: int = 4, cache_policy: str = "lru", + cache_policy_module_path: str | None = None, enable_events: bool = False, store_threshold: int = 0, max_tracker_size: int = 64_000, @@ -44,6 +45,7 @@ def make_cpu_manager( return CPUOffloadingManager( num_blocks=num_blocks, cache_policy=cache_policy, + cache_policy_module_path=cache_policy_module_path, enable_events=enable_events, store_threshold=store_threshold, max_tracker_size=max_tracker_size, diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index 6ff09f23d3..b0db86cf9b 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -13,8 +13,6 @@ from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, NewType, TypeVar import numpy as np import torch -from vllm.logger import init_logger - if TYPE_CHECKING: from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( OffloadingConnectorStats, @@ -27,8 +25,6 @@ from vllm.v1.kv_offload.config import OffloadingConfig # Use the helper functions below to construct / decompose keys. OffloadKey = NewType("OffloadKey", bytes) -logger = init_logger(__name__) - def make_offload_key(block_hash: bytes, group_idx: int) -> OffloadKey: """Pack a block hash and group index into an `OffloadKey`.""" @@ -540,10 +536,6 @@ class OffloadingSpec(ABC): return {} def __init__(self, config: OffloadingConfig): - logger.warning( - "Initializing OffloadingSpec. This API is experimental and " - "subject to change in the future as we iterate the design." - ) self.config = config self.extra_config = config.extra_config self.replicated_layout: bool = False diff --git a/vllm/v1/kv_offload/cpu/manager.py b/vllm/v1/kv_offload/cpu/manager.py index c2ec4170b8..0eef5cf7e7 100644 --- a/vllm/v1/kv_offload/cpu/manager.py +++ b/vllm/v1/kv_offload/cpu/manager.py @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections import OrderedDict from collections.abc import Collection, Iterable -from typing import Literal from typing_extensions import override @@ -24,19 +23,15 @@ from vllm.v1.kv_offload.cpu.common import ( CPULoadStoreSpec, CPUOffloadingMetrics, ) -from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy -from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy - -_CACHE_POLICIES: dict[str, type[CachePolicy]] = { - "lru": LRUCachePolicy, - "arc": ARCCachePolicy, -} +from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory class CPUOffloadingManager(OffloadingManager): """ - An OffloadingManager with a pluggable CachePolicy (LRU or ARC). + An OffloadingManager with a pluggable CachePolicy, resolved by name via + CachePolicyFactory (built in: "lru", "arc"; external policies can either + register their own or be loaded out-of-tree via cache_policy_module_path). The manager owns all shared logic: ref-counting, event emission, block pool management, and the prepare_store/complete_store skeletons. @@ -47,7 +42,8 @@ class CPUOffloadingManager(OffloadingManager): def __init__( self, num_blocks: int, - cache_policy: Literal["lru", "arc"] = "lru", + cache_policy: str = "lru", + cache_policy_module_path: str | None = None, enable_events: bool = False, store_threshold: int = 1, max_tracker_size: int = 64_000, @@ -57,12 +53,9 @@ class CPUOffloadingManager(OffloadingManager): self._num_allocated_blocks: int = 0 self._free_list: list[int] = [] self.events: list[OffloadingEvent] | None = [] if enable_events else None - policy_cls = _CACHE_POLICIES.get(cache_policy) - if policy_cls is None: - raise ValueError( - f"Unknown cache policy: {cache_policy!r}. " - f"Supported: {list(_CACHE_POLICIES)}" - ) + policy_cls = CachePolicyFactory.get_cache_policy_cls( + cache_policy, cache_policy_module_path + ) self._policy: CachePolicy = policy_cls(cache_capacity=num_blocks) # Track the number of blocks in the cache that are evictable. i.e. ref_cnt 0. self._num_evictable_cache_blocks: int = 0 diff --git a/vllm/v1/kv_offload/cpu/policies/arc.py b/vllm/v1/kv_offload/cpu/policies/arc.py index f682a47e45..d6569cbcfd 100644 --- a/vllm/v1/kv_offload/cpu/policies/arc.py +++ b/vllm/v1/kv_offload/cpu/policies/arc.py @@ -48,7 +48,7 @@ class ARCCachePolicy(CachePolicy): """ def __init__(self, cache_capacity: int): - self.cache_capacity: int = cache_capacity + super().__init__(cache_capacity) self.target_t1_size: float = 0.0 self.t1: OrderedDict[OffloadKey, BlockStatus] = OrderedDict() self.t2: OrderedDict[OffloadKey, BlockStatus] = OrderedDict() diff --git a/vllm/v1/kv_offload/cpu/policies/base.py b/vllm/v1/kv_offload/cpu/policies/base.py index 2b6681e499..908907d326 100644 --- a/vllm/v1/kv_offload/cpu/policies/base.py +++ b/vllm/v1/kv_offload/cpu/policies/base.py @@ -41,8 +41,8 @@ class CachePolicy(ABC): and eviction, so they cannot be separated cleanly. """ - @abstractmethod - def __init__(self, cache_capacity: int) -> None: ... + def __init__(self, cache_capacity: int) -> None: + self.cache_capacity = cache_capacity @abstractmethod def get(self, key: OffloadKey) -> BlockStatus | None: diff --git a/vllm/v1/kv_offload/cpu/policies/factory.py b/vllm/v1/kv_offload/cpu/policies/factory.py new file mode 100644 index 0000000000..88d11b270c --- /dev/null +++ b/vllm/v1/kv_offload/cpu/policies/factory.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import importlib +from collections.abc import Callable + +from vllm.logger import init_logger +from vllm.v1.kv_offload.cpu.policies.base import CachePolicy + +logger = init_logger(__name__) + + +class CachePolicyFactory: + """Registry for CachePolicy implementations, resolved by name. + + Mirrors OffloadingSpecFactory (vllm/v1/kv_offload/factory.py): built-in + policies are pre-registered below. External policies can either + register_cache_policy() a friendly short name up front, or skip + registration entirely and pass a module path at lookup time (out-of-tree, + no vLLM fork/patch required) -- see get_cache_policy_cls. + """ + + _registry: dict[str, Callable[[], type[CachePolicy]]] = {} + + @classmethod + def register_cache_policy( + cls, name: str, module_path: str, class_name: str + ) -> None: + """Register a cache policy with a lazy-loading module and class name.""" + if name in cls._registry: + raise ValueError(f"Cache policy '{name}' is already registered.") + + def loader() -> type[CachePolicy]: + module = importlib.import_module(module_path) + return getattr(module, class_name) + + cls._registry[name] = loader + + @classmethod + def get_cache_policy_cls( + cls, name: str, module_path: str | None = None + ) -> type[CachePolicy]: + """Get a cache policy class by name. + + Args: + name: Name of the cache policy. Checked against the registry + first; if it's not registered and `module_path` is given, + `name` is imported from there instead -- an out-of-tree + policy needs no register_cache_policy() call at all, just + this module path passed through config (mirrors + OffloadingSpecFactory.get_spec_cls's spec_module_path + fallback). + module_path: Python import path to load `name` from when it is + not a registered policy. + + Returns: + The cache policy class. + + Raises ValueError if the cache policy is neither registered nor + resolvable via `module_path`. + """ + if name in cls._registry: + return cls._registry[name]() + if module_path is None: + raise ValueError( + f"Unknown cache policy: {name!r}. Supported: {list(cls._registry)}. " + "For an out-of-tree policy, also set cache_policy_module_path." + ) + logger.warning( + "Loading out-of-tree cache policy '%s' from '%s'. This API is " + "experimental and subject to change in the future as we " + "iterate the design.", + name, + module_path, + ) + module = importlib.import_module(module_path) + policy_cls = getattr(module, name) + assert issubclass(policy_cls, CachePolicy) + return policy_cls + + +# Register built-in policies here. +CachePolicyFactory.register_cache_policy( + "lru", "vllm.v1.kv_offload.cpu.policies.lru", "LRUCachePolicy" +) +CachePolicyFactory.register_cache_policy( + "arc", "vllm.v1.kv_offload.cpu.policies.arc", "ARCCachePolicy" +) diff --git a/vllm/v1/kv_offload/cpu/policies/lru.py b/vllm/v1/kv_offload/cpu/policies/lru.py index efa24fe903..e8ccf0bdef 100644 --- a/vllm/v1/kv_offload/cpu/policies/lru.py +++ b/vllm/v1/kv_offload/cpu/policies/lru.py @@ -19,6 +19,7 @@ class LRUCachePolicy(CachePolicy): """ def __init__(self, cache_capacity: int): + super().__init__(cache_capacity) # Blocks with ref_cnt 0 (not participating in any loads/stores) ordered in LRU self.evictable_blocks: OrderedDict[OffloadKey, None] = OrderedDict() self.blocks: dict[OffloadKey, BlockStatus] = {} diff --git a/vllm/v1/kv_offload/cpu/spec.py b/vllm/v1/kv_offload/cpu/spec.py index d755bfecbc..9162c7b18b 100644 --- a/vllm/v1/kv_offload/cpu/spec.py +++ b/vllm/v1/kv_offload/cpu/spec.py @@ -117,6 +117,9 @@ class CPUOffloadingSpec(OffloadingSpec): self._worker: CPUOffloadingWorker | None = None self.eviction_policy: str = self.extra_config.get("eviction_policy", "lru") + self.cache_policy_module_path: str | None = self.extra_config.get( + "cache_policy_module_path" + ) @override def get_manager(self) -> OffloadingManager: @@ -131,7 +134,8 @@ class CPUOffloadingSpec(OffloadingSpec): self._manager = CPUOffloadingManager( num_blocks=self.num_blocks, - cache_policy=self.eviction_policy, # type: ignore[arg-type] + cache_policy=self.eviction_policy, + cache_policy_module_path=self.cache_policy_module_path, enable_events=self.kv_events_config.enable_kv_cache_events, store_threshold=store_threshold, max_tracker_size=max_tracker_size, diff --git a/vllm/v1/kv_offload/factory.py b/vllm/v1/kv_offload/factory.py index 931fda8308..19bc401277 100644 --- a/vllm/v1/kv_offload/factory.py +++ b/vllm/v1/kv_offload/factory.py @@ -35,6 +35,13 @@ class OffloadingSpecFactory: spec_module_path = extra_config.get("spec_module_path") if spec_module_path is None: raise ValueError(f"Unsupported spec type: {spec_name}") + logger.warning( + "Loading out-of-tree offloading spec '%s' from '%s'. This " + "API is experimental and subject to change in the future " + "as we iterate the design.", + spec_name, + spec_module_path, + ) spec_module = importlib.import_module(spec_module_path) spec_cls = getattr(spec_module, spec_name) assert issubclass(spec_cls, OffloadingSpec) diff --git a/vllm/v1/kv_offload/tiering/manager.py b/vllm/v1/kv_offload/tiering/manager.py index c673809613..d1d3c42115 100644 --- a/vllm/v1/kv_offload/tiering/manager.py +++ b/vllm/v1/kv_offload/tiering/manager.py @@ -93,11 +93,13 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager): num_blocks: int, mmap_region: SharedOffloadRegion, cache_policy: str = "lru", + cache_policy_module_path: str | None = None, enable_events: bool = False, ): super().__init__( num_blocks=num_blocks, - cache_policy=cache_policy, # type: ignore[arg-type] + cache_policy=cache_policy, + cache_policy_module_path=cache_policy_module_path, enable_events=enable_events, ) self._mmap_region = mmap_region diff --git a/vllm/v1/kv_offload/tiering/spec.py b/vllm/v1/kv_offload/tiering/spec.py index 0d200a4239..e6ec192415 100644 --- a/vllm/v1/kv_offload/tiering/spec.py +++ b/vllm/v1/kv_offload/tiering/spec.py @@ -9,8 +9,13 @@ and configurable secondary tiers (e.g., Storage, Network). Configuration via kv_connector_extra_config: - cpu_bytes_to_use: (required) Bytes to allocate for CPU primary tier - block_size: (optional) Block size for offloaded blocks (default: GPU block size) - - eviction_policy: (optional) Primary tier eviction policy: "lru" or - "arc" (default: "lru") + - eviction_policy: (optional) Primary tier eviction policy: built-in "lru"/ + "arc", or the name of a policy registered via CachePolicyFactory, or an + out-of-tree CachePolicy class name paired with cache_policy_module_path + (default: "lru") + - cache_policy_module_path: (optional) Python import path to load + eviction_policy from when it names an out-of-tree CachePolicy not + registered via CachePolicyFactory - secondary_tiers: (optional) List of secondary tier configurations Each secondary tier config is a dict with: - type: (required) Type of secondary tier (e.g., "example", "storage", "network") @@ -178,7 +183,8 @@ class TieringOffloadingSpec(CPUOffloadingSpec): # Create primary tier (CPU-based) primary_tier = CPUPrimaryTierOffloadingManager( num_blocks=self.num_blocks, - cache_policy=self.eviction_policy, # type: ignore[arg-type] + cache_policy=self.eviction_policy, + cache_policy_module_path=self.cache_policy_module_path, enable_events=self.kv_events_config.enable_kv_cache_events, mmap_region=scheduler_mmap, )