[KV Transfer] Enable HMA by default for connectors that support it (#41847)
Signed-off-by: Ethan Feng <ethan.fengch@gmail.com>
This commit is contained in:
@@ -5,6 +5,7 @@ import importlib
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from vllm.config.kv_transfer import KVTransferConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.base import (
|
||||
KVConnectorBase,
|
||||
KVConnectorBaseType,
|
||||
@@ -18,7 +19,6 @@ from vllm.utils.func_utils import supports_kw
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kv_transfer import KVTransferConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -53,7 +53,7 @@ class KVConnectorFactory:
|
||||
|
||||
# check if the connector supports HMA
|
||||
hma_enabled = not config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
if hma_enabled and not supports_hma(connector_cls):
|
||||
if hma_enabled and not cls.supports_hma_config(kv_transfer_config):
|
||||
raise ValueError(
|
||||
f"Connector {connector_cls.__name__} does not support HMA but "
|
||||
f"HMA is enabled. Please set `--disable-hybrid-kv-cache-manager`."
|
||||
@@ -127,6 +127,23 @@ class KVConnectorFactory:
|
||||
raise ValueError(f"Unsupported connector type: {connector_name}")
|
||||
return connector_cls
|
||||
|
||||
@classmethod
|
||||
def supports_hma_config(cls, kv_transfer_config: "KVTransferConfig") -> bool:
|
||||
"""Return whether this KV transfer config supports HMA.
|
||||
|
||||
MultiConnector is a special case: the wrapper class implements
|
||||
SupportsHMA, but effective support depends on every configured child.
|
||||
"""
|
||||
connector_cls = cls.get_connector_class(kv_transfer_config)
|
||||
if kv_transfer_config.kv_connector != "MultiConnector":
|
||||
return supports_hma(connector_cls)
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import (
|
||||
MultiConnector,
|
||||
)
|
||||
|
||||
return MultiConnector.all_children_support_hma(kv_transfer_config)
|
||||
|
||||
|
||||
# Register various connectors here.
|
||||
# The registration should not be done in each individual file, as we want to
|
||||
|
||||
@@ -19,7 +19,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorRole,
|
||||
KVConnectorWorkerMetadata,
|
||||
SupportsHMA,
|
||||
supports_hma,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
@@ -151,6 +150,22 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def all_children_support_hma(cls, kv_transfer_config: "KVTransferConfig") -> bool:
|
||||
"""Return True only if every configured child connector supports HMA."""
|
||||
connectors_config = kv_transfer_config.kv_connector_extra_config.get(
|
||||
"connectors", []
|
||||
)
|
||||
if not connectors_config:
|
||||
return False
|
||||
for conn_config in connectors_config:
|
||||
child_config = KVTransferConfig(
|
||||
**{"engine_id": kv_transfer_config.engine_id, **conn_config}
|
||||
)
|
||||
if not KVConnectorFactory.supports_hma_config(child_config):
|
||||
return False
|
||||
return True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
@@ -169,7 +184,10 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
self._connectors.append(connector_cls(temp_config, role, kv_cache_config))
|
||||
self._ktc_kv_transfer_config.append(temp_config.kv_transfer_config)
|
||||
|
||||
self._all_support_hma = all(supports_hma(c) for c in self._connectors)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
self._all_support_hma = MultiConnector.all_children_support_hma(
|
||||
vllm_config.kv_transfer_config
|
||||
)
|
||||
assert (
|
||||
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
or self._all_support_hma
|
||||
|
||||
Reference in New Issue
Block a user