[KV Offload] Support out-of-tree secondary tier managers via module_path (#51007)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Ronen Schaffer
2026-08-05 20:30:53 +03:00
committed by GitHub
co-authored by Or Ozeri
parent d4da0c55af
commit 8543522ca7
6 changed files with 153 additions and 15 deletions
+7 -4
View File
@@ -65,10 +65,13 @@ class CachePolicyFactory:
f"Unknown cache policy: {name!r}. Supported: {list(cls._registry)}. "
"For an out-of-tree policy, also set cache_policy_module_path."
)
logger.warning(
"Loading out-of-tree cache policy '%s' from '%s'. This API is "
"experimental and subject to change in the future as we "
"iterate the design.",
logger.warning_once(
"Loading out-of-tree cache policy. This API is "
"experimental and subject to change in the future "
"as we iterate the design."
)
logger.info(
"Loading out-of-tree cache policy '%s' from '%s'.",
name,
module_path,
)
+7 -4
View File
@@ -35,10 +35,13 @@ class OffloadingSpecFactory:
spec_module_path = extra_config.get("spec_module_path")
if spec_module_path is None:
raise ValueError(f"Unsupported spec type: {spec_name}")
logger.warning(
"Loading out-of-tree offloading spec '%s' from '%s'. This "
"API is experimental and subject to change in the future "
"as we iterate the design.",
logger.warning_once(
"Loading out-of-tree offloading spec. This API is "
"experimental and subject to change in the future "
"as we iterate the design."
)
logger.info(
"Loading out-of-tree offloading spec '%s' from '%s'.",
spec_name,
spec_module_path,
)
+49 -3
View File
@@ -4,13 +4,25 @@ import importlib
from collections.abc import Callable
from typing import TYPE_CHECKING
from vllm.logger import init_logger
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
if TYPE_CHECKING:
from vllm.v1.kv_offload.base import OffloadingSpec
logger = init_logger(__name__)
class SecondaryTierFactory:
"""Registry for SecondaryTierManager implementations, resolved by type.
Mirrors OffloadingSpecFactory (vllm/v1/kv_offload/factory.py): built-in
tiers are pre-registered below. External tiers can either
register_tier() a friendly short name up front, or skip registration
entirely and pass a module_path at lookup time (out-of-tree, no vLLM
fork/patch required) -- see get_tier_class.
"""
_registry: dict[str, Callable[[], type[SecondaryTierManager]]] = {}
@classmethod
@@ -34,6 +46,7 @@ class SecondaryTierFactory:
tier_cls = cls.get_tier_class(tier_config)
config = tier_config.copy()
tier_type = config.pop("type")
config.pop("module_path", None)
return tier_cls(
offloading_spec=offloading_spec,
primary_kv_view=primary_kv_view,
@@ -43,15 +56,48 @@ class SecondaryTierFactory:
@classmethod
def get_tier_class(cls, tier_config: dict) -> type[SecondaryTierManager]:
"""Get a secondary tier class by type.
Args:
tier_config: Tier configuration dict. Must contain 'type' (the
tier name or out-of-tree class name). If 'type' is not a
registered tier and 'module_path' is given, the class is
imported from there -- an out-of-tree tier needs no
register_tier() call, just this module path in the config.
Returns:
The secondary tier manager class.
Raises:
ValueError: If 'type' is missing, or is neither registered nor
resolvable via module_path.
"""
tier_type = tier_config.get("type")
if not tier_type:
raise ValueError("Secondary tier configuration must include 'type'")
if tier_type not in cls._registry:
if tier_type in cls._registry:
return cls._registry[tier_type]()
module_path = tier_config.get("module_path")
if module_path is None:
raise ValueError(
f"Unknown secondary tier type: {tier_type!r}. "
f"Supported types: {list(cls._registry)}"
f"Supported types: {list(cls._registry)}. "
"For an out-of-tree tier, also set 'module_path'."
)
return cls._registry[tier_type]()
logger.warning_once(
"Loading out-of-tree secondary tier. This API is "
"experimental and subject to change in the future "
"as we iterate the design."
)
logger.info(
"Loading out-of-tree secondary tier '%s' from '%s'.",
tier_type,
module_path,
)
module = importlib.import_module(module_path)
tier_cls = getattr(module, tier_type)
assert issubclass(tier_cls, SecondaryTierManager)
return tier_cls
SecondaryTierFactory.register_tier(
+18 -1
View File
@@ -18,7 +18,12 @@ Configuration via kv_connector_extra_config:
registered via CachePolicyFactory
- secondary_tiers: (optional) List of secondary tier configurations
Each secondary tier config is a dict with:
- type: (required) Type of secondary tier (e.g., "example", "storage", "network")
- type: (required) Type of secondary tier (e.g., "example", "fs",
"p2p", "obj"), or the class name of an out-of-tree
SecondaryTierManager paired with module_path.
- module_path: (optional) Python import path to load 'type' from
when it names an out-of-tree SecondaryTierManager not registered
via SecondaryTierFactory.register_tier()
- Additional tier-specific parameters are passed directly to the tier
constructor. See each tier's documentation for supported parameters.
@@ -34,6 +39,18 @@ Example configuration:
}
]
}
Example out-of-tree tier configuration:
{
"cpu_bytes_to_use": 10737418240,
"secondary_tiers": [
{
"type": "MyCustomTier",
"module_path": "my_package.my_module",
"custom_param": "value"
}
]
}
"""
from typing import Any