[KV Transfer] Enable HMA by default for connectors that support it (#41847)

Signed-off-by: Ethan Feng <ethan.fengch@gmail.com>
This commit is contained in:
Ethan Feng
2026-05-26 12:28:51 +02:00
committed by GitHub
parent 97e4022c6c
commit 755043cf3c
7 changed files with 161 additions and 41 deletions
@@ -0,0 +1,107 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for HMA auto-disable with KV transfer connectors."""
import pytest
from vllm.config import DeviceConfig, KVTransferConfig, SchedulerConfig, VllmConfig
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole
from vllm.platforms import current_platform
from vllm.v1.kv_cache_interface import KVCacheConfig
pytestmark = pytest.mark.cpu_test
@pytest.fixture(autouse=True)
def mock_hybrid_kv_cache_supported(monkeypatch):
monkeypatch.setattr(current_platform, "support_hybrid_kv_cache", lambda: True)
@pytest.mark.parametrize(
"kv_transfer_config,expect_disabled",
[
( # HMA-supporting connector → HMA stays enabled
KVTransferConfig(
kv_connector="SimpleCPUOffloadConnector",
kv_role="kv_both",
kv_connector_extra_config={"cpu_bytes_to_use": 1 << 30},
),
False,
),
( # Non-HMA connector → HMA is auto-disabled
KVTransferConfig(kv_connector="ExampleConnector", kv_role="kv_both"),
True,
),
( # MultiConnector: all HMA children → HMA stays enabled
KVTransferConfig(
kv_connector="MultiConnector",
kv_role="kv_both",
kv_connector_extra_config={
"connectors": [
{
"kv_connector": "SimpleCPUOffloadConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {"cpu_bytes_to_use": 1 << 30},
},
{
"kv_connector": "OffloadingConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {"cpu_bytes_to_use": 1 << 30},
},
]
},
),
False,
),
( # MultiConnector: mixed children → HMA is auto-disabled
KVTransferConfig(
kv_connector="MultiConnector",
kv_role="kv_both",
kv_connector_extra_config={
"connectors": [
{
"kv_connector": "SimpleCPUOffloadConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {"cpu_bytes_to_use": 1 << 30},
},
{"kv_connector": "ExampleConnector", "kv_role": "kv_both"},
]
},
),
True,
),
],
ids=["hma_connector", "non_hma_connector", "multi_all_hma", "multi_mixed"],
)
def test_hma_auto_config(kv_transfer_config, expect_disabled):
vllm_config = VllmConfig(
device_config=DeviceConfig("cpu"),
kv_transfer_config=kv_transfer_config,
)
assert (
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager is expect_disabled
)
def test_explicit_hma_with_non_hma_connector_errors_at_factory():
vllm_config = VllmConfig(
device_config=DeviceConfig("cpu"),
scheduler_config=SchedulerConfig(
max_model_len=16,
is_encoder_decoder=False,
disable_hybrid_kv_cache_manager=False,
),
kv_transfer_config=KVTransferConfig(
kv_connector="ExampleConnector",
kv_role="kv_both",
),
)
kv_cache_config = KVCacheConfig(
num_blocks=0, kv_cache_tensors=[], kv_cache_groups=[]
)
with pytest.raises(ValueError, match="does not support HMA but HMA is enabled"):
KVConnectorFactory.create_connector(
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
)
@@ -1000,11 +1000,8 @@ def _make_multi_connector(connector_names: list[str]) -> MultiConnector:
)
def test_multi_connector_hma_opt_in():
def test_multi_connector_hma_support_detection():
"""
MultiConnector currently assumes HMA is opt-in: it needs
--no-disable-hybrid-kv-cache-manager to be enabled.
At runtime, _all_support_hma is True only when every sub-connector
implements SupportsHMA. Test all combinations of HMA / non-HMA
sub-connectors.
@@ -723,8 +723,7 @@ def test_has_mamba_init(
block_size = 16
vllm_config = create_vllm_config(block_size=block_size)
# VllmConfig.__post_init__ auto-disables HMA when kv_transfer_config
# is set; override so we can test the scheduler's own derivation.
# Explicitly enable HMA so we can test the scheduler's own derivation.
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = False
kv_cache_config = make_kv_cache_config(
block_size=block_size,
@@ -280,7 +280,7 @@ def test_cpu_offloading(
kv_events_config=kv_events_config,
kv_transfer_config=kv_transfer_config,
**({"attention_config": {"backend": attn_backend}} if attn_backend else {}),
# HMA models need explicit opt-in when kv_transfer_config is set
# Keep HMA explicitly enabled for HMA model coverage.
**({"disable_hybrid_kv_cache_manager": False} if uses_hma else {}),
**({"enable_prefix_caching": True} if force_prefix_caching else {}),
# ROCm: batch size 1 to reduce variability