Signed-off-by: Bill Nell <bnell@redhat.com> Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com> Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
175 lines
6.0 KiB
Python
175 lines
6.0 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
import importlib
|
|
from collections.abc import Callable
|
|
from typing import TYPE_CHECKING
|
|
|
|
from vllm.logger import init_logger
|
|
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.v1.kv_offload.backpressure import BackpressureDetector
|
|
from vllm.v1.kv_offload.base import OffloadingSpec
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
_DEFAULT_PACKAGE = "vllm.v1.kv_offload.tiering.backpressure"
|
|
|
|
|
|
def _import_class(name: str) -> type:
|
|
"""Import a class by name.
|
|
|
|
If ``name`` contains a dot it is treated as a fully qualified path
|
|
(e.g. ``'my_package.MyDetector'``). Otherwise it is looked up in
|
|
the default backpressure module
|
|
(``vllm.v1.kv_offload.tiering.backpressure``), so short names like
|
|
``'DropStorePolicy'`` or ``'EMABackpressureDetector'`` work out of
|
|
the box.
|
|
"""
|
|
if "." in name:
|
|
module_path, _, class_name = name.rpartition(".")
|
|
else:
|
|
module_path, class_name = _DEFAULT_PACKAGE, name
|
|
module = importlib.import_module(module_path)
|
|
return getattr(module, class_name)
|
|
|
|
|
|
class SecondaryTierFactory:
|
|
"""Registry for SecondaryTierManager implementations, resolved by type.
|
|
|
|
Mirrors OffloadingSpecFactory (vllm/v1/kv_offload/factory.py): built-in
|
|
tiers are pre-registered below. External tiers can either
|
|
register_tier() a friendly short name up front, or skip registration
|
|
entirely and pass a module_path at lookup time (out-of-tree, no vLLM
|
|
fork/patch required) -- see get_tier_class.
|
|
"""
|
|
|
|
_registry: dict[str, Callable[[], type[SecondaryTierManager]]] = {}
|
|
|
|
@classmethod
|
|
def register_tier(cls, tier_type: str, module_path: str, class_name: str) -> None:
|
|
if tier_type in cls._registry:
|
|
raise ValueError(f"Tier '{tier_type}' is already registered.")
|
|
|
|
def loader() -> type[SecondaryTierManager]:
|
|
module = importlib.import_module(module_path)
|
|
return getattr(module, class_name)
|
|
|
|
cls._registry[tier_type] = loader
|
|
|
|
@classmethod
|
|
def create_secondary_tier(
|
|
cls,
|
|
tier_config: dict,
|
|
primary_kv_view: memoryview,
|
|
offloading_spec: "OffloadingSpec",
|
|
) -> SecondaryTierManager:
|
|
tier_cls = cls.get_tier_class(tier_config)
|
|
config = tier_config.copy()
|
|
tier_type = config.pop("type")
|
|
config.pop("module_path", None)
|
|
locality = config.get("locality")
|
|
bp_config = config.pop("backpressure", None)
|
|
bp_detector = None
|
|
if bp_config is not None:
|
|
bp_config = bp_config.copy()
|
|
bp_cls_name = bp_config.pop("backpressure_cls", None)
|
|
if bp_cls_name is None:
|
|
raise ValueError(
|
|
"backpressure config must include 'backpressure_cls' "
|
|
"(e.g. 'EMABackpressureDetector')"
|
|
)
|
|
policy_cls_name = bp_config.pop("policy_cls", None)
|
|
policy = _import_class(policy_cls_name)() if policy_cls_name else None
|
|
detector_cls: type[BackpressureDetector] = _import_class(bp_cls_name)
|
|
defaults = detector_cls.default_config(tier_type, locality=locality)
|
|
if defaults:
|
|
for k, v in defaults.items():
|
|
bp_config.setdefault(k, v)
|
|
bp_detector = detector_cls(policy=policy, **bp_config)
|
|
return tier_cls(
|
|
offloading_spec=offloading_spec,
|
|
primary_kv_view=primary_kv_view,
|
|
tier_type=tier_type,
|
|
backpressure_detector=bp_detector,
|
|
**config,
|
|
)
|
|
|
|
@classmethod
|
|
def get_tier_class(cls, tier_config: dict) -> type[SecondaryTierManager]:
|
|
"""Get a secondary tier class by type.
|
|
|
|
Args:
|
|
tier_config: Tier configuration dict. Must contain 'type' (the
|
|
tier name or out-of-tree class name). If 'type' is not a
|
|
registered tier and 'module_path' is given, the class is
|
|
imported from there -- an out-of-tree tier needs no
|
|
register_tier() call, just this module path in the config.
|
|
|
|
Returns:
|
|
The secondary tier manager class.
|
|
|
|
Raises:
|
|
ValueError: If 'type' is missing, or is neither registered nor
|
|
resolvable via module_path.
|
|
|
|
"""
|
|
tier_type = tier_config.get("type")
|
|
if not tier_type:
|
|
raise ValueError("Secondary tier configuration must include 'type'")
|
|
if tier_type in cls._registry:
|
|
return cls._registry[tier_type]()
|
|
module_path = tier_config.get("module_path")
|
|
if module_path is None:
|
|
raise ValueError(
|
|
f"Unknown secondary tier type: {tier_type!r}. "
|
|
f"Supported types: {list(cls._registry)}. "
|
|
"For an out-of-tree tier, also set 'module_path'."
|
|
)
|
|
logger.warning_once(
|
|
"Loading out-of-tree secondary tier. This API is "
|
|
"experimental and subject to change in the future "
|
|
"as we iterate the design."
|
|
)
|
|
logger.info(
|
|
"Loading out-of-tree secondary tier '%s' from '%s'.",
|
|
tier_type,
|
|
module_path,
|
|
)
|
|
module = importlib.import_module(module_path)
|
|
tier_cls = getattr(module, tier_type)
|
|
assert issubclass(tier_cls, SecondaryTierManager)
|
|
return tier_cls
|
|
|
|
|
|
SecondaryTierFactory.register_tier(
|
|
"example",
|
|
"vllm.v1.kv_offload.tiering.example.manager",
|
|
"ExampleSecondaryTierManager",
|
|
)
|
|
|
|
SecondaryTierFactory.register_tier(
|
|
"fs",
|
|
"vllm.v1.kv_offload.tiering.fs.manager",
|
|
"FileSystemTierManager",
|
|
)
|
|
|
|
SecondaryTierFactory.register_tier(
|
|
"p2p",
|
|
"vllm.v1.kv_offload.tiering.p2p.manager",
|
|
"P2PSecondaryTierManager",
|
|
)
|
|
|
|
SecondaryTierFactory.register_tier(
|
|
"obj",
|
|
"vllm.v1.kv_offload.tiering.obj.manager",
|
|
"ObjectStoreSecondaryTierManager",
|
|
)
|
|
|
|
SecondaryTierFactory.register_tier(
|
|
"kvcr",
|
|
"vllm.v1.kv_offload.tiering.kvcr.manager",
|
|
"KVCRSecondaryTierManager",
|
|
)
|