[KV Offload] Support out-of-tree secondary tier managers via module_path (#51007)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Or Ozeri
parent
d4da0c55af
commit
8543522ca7
@@ -65,10 +65,13 @@ class CachePolicyFactory:
|
||||
f"Unknown cache policy: {name!r}. Supported: {list(cls._registry)}. "
|
||||
"For an out-of-tree policy, also set cache_policy_module_path."
|
||||
)
|
||||
logger.warning(
|
||||
"Loading out-of-tree cache policy '%s' from '%s'. This API is "
|
||||
"experimental and subject to change in the future as we "
|
||||
"iterate the design.",
|
||||
logger.warning_once(
|
||||
"Loading out-of-tree cache policy. This API is "
|
||||
"experimental and subject to change in the future "
|
||||
"as we iterate the design."
|
||||
)
|
||||
logger.info(
|
||||
"Loading out-of-tree cache policy '%s' from '%s'.",
|
||||
name,
|
||||
module_path,
|
||||
)
|
||||
|
||||
@@ -35,10 +35,13 @@ class OffloadingSpecFactory:
|
||||
spec_module_path = extra_config.get("spec_module_path")
|
||||
if spec_module_path is None:
|
||||
raise ValueError(f"Unsupported spec type: {spec_name}")
|
||||
logger.warning(
|
||||
"Loading out-of-tree offloading spec '%s' from '%s'. This "
|
||||
"API is experimental and subject to change in the future "
|
||||
"as we iterate the design.",
|
||||
logger.warning_once(
|
||||
"Loading out-of-tree offloading spec. This API is "
|
||||
"experimental and subject to change in the future "
|
||||
"as we iterate the design."
|
||||
)
|
||||
logger.info(
|
||||
"Loading out-of-tree offloading spec '%s' from '%s'.",
|
||||
spec_name,
|
||||
spec_module_path,
|
||||
)
|
||||
|
||||
@@ -4,13 +4,25 @@ import importlib
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SecondaryTierFactory:
|
||||
"""Registry for SecondaryTierManager implementations, resolved by type.
|
||||
|
||||
Mirrors OffloadingSpecFactory (vllm/v1/kv_offload/factory.py): built-in
|
||||
tiers are pre-registered below. External tiers can either
|
||||
register_tier() a friendly short name up front, or skip registration
|
||||
entirely and pass a module_path at lookup time (out-of-tree, no vLLM
|
||||
fork/patch required) -- see get_tier_class.
|
||||
"""
|
||||
|
||||
_registry: dict[str, Callable[[], type[SecondaryTierManager]]] = {}
|
||||
|
||||
@classmethod
|
||||
@@ -34,6 +46,7 @@ class SecondaryTierFactory:
|
||||
tier_cls = cls.get_tier_class(tier_config)
|
||||
config = tier_config.copy()
|
||||
tier_type = config.pop("type")
|
||||
config.pop("module_path", None)
|
||||
return tier_cls(
|
||||
offloading_spec=offloading_spec,
|
||||
primary_kv_view=primary_kv_view,
|
||||
@@ -43,15 +56,48 @@ class SecondaryTierFactory:
|
||||
|
||||
@classmethod
|
||||
def get_tier_class(cls, tier_config: dict) -> type[SecondaryTierManager]:
|
||||
"""Get a secondary tier class by type.
|
||||
|
||||
Args:
|
||||
tier_config: Tier configuration dict. Must contain 'type' (the
|
||||
tier name or out-of-tree class name). If 'type' is not a
|
||||
registered tier and 'module_path' is given, the class is
|
||||
imported from there -- an out-of-tree tier needs no
|
||||
register_tier() call, just this module path in the config.
|
||||
|
||||
Returns:
|
||||
The secondary tier manager class.
|
||||
|
||||
Raises:
|
||||
ValueError: If 'type' is missing, or is neither registered nor
|
||||
resolvable via module_path.
|
||||
"""
|
||||
tier_type = tier_config.get("type")
|
||||
if not tier_type:
|
||||
raise ValueError("Secondary tier configuration must include 'type'")
|
||||
if tier_type not in cls._registry:
|
||||
if tier_type in cls._registry:
|
||||
return cls._registry[tier_type]()
|
||||
module_path = tier_config.get("module_path")
|
||||
if module_path is None:
|
||||
raise ValueError(
|
||||
f"Unknown secondary tier type: {tier_type!r}. "
|
||||
f"Supported types: {list(cls._registry)}"
|
||||
f"Supported types: {list(cls._registry)}. "
|
||||
"For an out-of-tree tier, also set 'module_path'."
|
||||
)
|
||||
return cls._registry[tier_type]()
|
||||
logger.warning_once(
|
||||
"Loading out-of-tree secondary tier. This API is "
|
||||
"experimental and subject to change in the future "
|
||||
"as we iterate the design."
|
||||
)
|
||||
logger.info(
|
||||
"Loading out-of-tree secondary tier '%s' from '%s'.",
|
||||
tier_type,
|
||||
module_path,
|
||||
)
|
||||
module = importlib.import_module(module_path)
|
||||
tier_cls = getattr(module, tier_type)
|
||||
assert issubclass(tier_cls, SecondaryTierManager)
|
||||
return tier_cls
|
||||
|
||||
|
||||
SecondaryTierFactory.register_tier(
|
||||
|
||||
@@ -18,7 +18,12 @@ Configuration via kv_connector_extra_config:
|
||||
registered via CachePolicyFactory
|
||||
- secondary_tiers: (optional) List of secondary tier configurations
|
||||
Each secondary tier config is a dict with:
|
||||
- type: (required) Type of secondary tier (e.g., "example", "storage", "network")
|
||||
- type: (required) Type of secondary tier (e.g., "example", "fs",
|
||||
"p2p", "obj"), or the class name of an out-of-tree
|
||||
SecondaryTierManager paired with module_path.
|
||||
- module_path: (optional) Python import path to load 'type' from
|
||||
when it names an out-of-tree SecondaryTierManager not registered
|
||||
via SecondaryTierFactory.register_tier()
|
||||
- Additional tier-specific parameters are passed directly to the tier
|
||||
constructor. See each tier's documentation for supported parameters.
|
||||
|
||||
@@ -34,6 +39,18 @@ Example configuration:
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Example out-of-tree tier configuration:
|
||||
{
|
||||
"cpu_bytes_to_use": 10737418240,
|
||||
"secondary_tiers": [
|
||||
{
|
||||
"type": "MyCustomTier",
|
||||
"module_path": "my_package.my_module",
|
||||
"custom_param": "value"
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
Reference in New Issue
Block a user