[mooncake] support CUSTOM_MEM_POOL in vllm (#49300)

Signed-off-by: bruce.xu <bruce.xb@alibaba-inc.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: bruce.xu <bruce.xb@alibaba-inc.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
void
2026-09-27 00:26:05 +00:00
committed by GitHub
co-authored by bruce.xu Yifan Qiao OpenAI Codex
parent cd1daf58a9
commit eb0f2ca37f
7 changed files with 193 additions and 4 deletions
@@ -290,6 +290,8 @@ class _FakeKVTransferConfig:
class _FakeModelConfig:
model = "test-model"
use_mla = False
enable_sleep_mode = False
enable_cumem_allocator = False
def get_num_layers(self, parallel_config) -> int:
return 1
@@ -432,6 +434,64 @@ def _patch_worker_runtime(
monkeypatch.setattr(worker, "LookupKeyServer", MagicMock())
def test_create_mem_pool_returns_none_when_unconfigured():
assert worker.MooncakeStoreWorker._create_mem_pool({}, _FakeModelConfig()) is None
def test_create_mem_pool_rejects_unknown_pool():
with pytest.raises(
ValueError,
match="Unsupported custom_mem_pool='UNKNOWN'",
):
worker.MooncakeStoreWorker._create_mem_pool(
{"custom_mem_pool": "unknown"}, _FakeModelConfig()
)
@pytest.mark.parametrize(
"conflicting_option", ["enable_sleep_mode", "enable_cumem_allocator"]
)
def test_create_mem_pool_rejects_cumem_conflicts(conflicting_option):
model_config = _FakeModelConfig()
setattr(model_config, conflicting_option, True)
with pytest.raises(ValueError, match="custom_mem_pool is incompatible"):
worker.MooncakeStoreWorker._create_mem_pool(
{"custom_mem_pool": "NVLINK"}, model_config
)
@pytest.mark.parametrize(
("pool_type", "allocator_name"),
[("nvlink", "NVLinkAllocator"), ("barex", "BarexAllocator")],
)
def test_create_mem_pool_uses_mooncake_allocator(
monkeypatch, pool_type, allocator_name
):
allocator_cls = MagicMock()
allocator = allocator_cls.get_allocator.return_value
allocator_module = types.ModuleType("mooncake.allocator")
setattr(allocator_module, allocator_name, allocator_cls)
mooncake_module = types.ModuleType("mooncake")
mooncake_module.allocator = allocator_module # type: ignore[attr-defined]
monkeypatch.setitem(sys.modules, "mooncake", mooncake_module)
monkeypatch.setitem(sys.modules, "mooncake.allocator", allocator_module)
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 2)
mem_pool = MagicMock()
mem_pool_ctor = MagicMock(return_value=mem_pool)
monkeypatch.setattr(torch.cuda, "MemPool", mem_pool_ctor)
result = worker.MooncakeStoreWorker._create_mem_pool(
{"custom_mem_pool": pool_type}, _FakeModelConfig()
)
assert result is mem_pool
allocator_cls.get_allocator.assert_called_once_with(torch.device("cuda", 2))
allocator.allocator.assert_called_once_with()
mem_pool_ctor.assert_called_once_with(allocator.allocator.return_value)
def test_pool_key_to_string_without_prefix_is_unchanged():
"""Default (empty) cache_prefix keeps keys byte-identical to the
historical unprefixed format so existing deployments keep their hits."""
@@ -68,6 +68,7 @@ class MockConnector(KVConnectorBase_V1):
# Override just build_kv_connector_stats
mock.build_kv_connector_stats = cls.build_kv_connector_stats
mock.get_kv_connector_stats.return_value = None
mock.get_mem_pool_context.return_value = None
return mock
@classmethod
@@ -201,6 +202,36 @@ def mc() -> MultiConnector:
return mc
def test_multi_connector_mem_pool_context_none(mc: MultiConnector):
assert mc.get_mem_pool_context() is None
def test_multi_connector_forwards_mem_pool_context(mc: MultiConnector):
context = MagicMock()
provider = MagicMock(spec_set=KVConnectorBase_V1)
provider.get_mem_pool_context.return_value = context
mc._connectors = [mc._connectors[0], provider]
assert mc.get_mem_pool_context() is context
provider.get_mem_pool_context.assert_called_once_with()
def test_multi_connector_rejects_multiple_mem_pool_contexts(mc: MultiConnector):
providers = [
MagicMock(spec_set=KVConnectorBase_V1),
MagicMock(spec_set=KVConnectorBase_V1),
]
for provider in providers:
provider.get_mem_pool_context.return_value = MagicMock()
mc._connectors = providers
with pytest.raises(
ValueError,
match="Multiple connectors provide a KV cache memory pool",
):
mc.get_mem_pool_context()
# Helper function to compare directories recursively
def _compare_directories(dir1: Path, dir2: Path) -> bool:
"""Compares two directories recursively for identical content."""
@@ -320,6 +351,7 @@ def test_multi_example_connector_consistency():
# First three events are from initialization. Layer hooks run before the
# deferred load starts after the forward pass.
expected_worker_prefix = [
"get_mem_pool_context",
"register_kv_caches",
"set_host_xfer_buffer_ops",
"get_handshake_metadata",