[mooncake] support CUSTOM_MEM_POOL in vllm (#49300)

Signed-off-by: bruce.xu <bruce.xb@alibaba-inc.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: bruce.xu <bruce.xb@alibaba-inc.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
void
2026-09-27 00:26:05 +00:00
committed by GitHub
co-authored by bruce.xu Yifan Qiao OpenAI Codex
parent cd1daf58a9
commit eb0f2ca37f
7 changed files with 193 additions and 4 deletions
@@ -42,6 +42,7 @@ The class provides the following primitives:
import enum
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable
from contextlib import AbstractContextManager
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Literal
@@ -227,6 +228,13 @@ class KVConnectorBase_V1(ABC):
# Worker-side methods
# ==============================
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Return a custom KV cache allocation context, if configured.
Returning None uses the engine's default memory pool.
"""
return None
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
"""Set the connector metadata from the scheduler.
@@ -13,6 +13,7 @@ enabling prefix caching via hash-based deduplication.
from collections import Counter
from collections.abc import Iterable, Sequence
from contextlib import AbstractContextManager
from typing import Any
import torch
@@ -322,6 +323,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
# Worker-side methods
# ============================================================
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Return a context manager for the custom MemPool, or None.
Called by the Worker before ``initialize_kv_cache`` so that KV
cache is allocated from the Mooncake-managed pool when
``custom_mem_pool`` is set in ``kv_connector_extra_config``.
"""
if self.connector_worker is None:
return None
return self.connector_worker.get_mem_pool_context()
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
@@ -19,13 +19,14 @@ import threading
import time
from collections.abc import Callable, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import AbstractContextManager
from typing import Any, TypeVar
import torch
import zmq
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config import ModelConfig, VllmConfig
from vllm.distributed import (
get_dcp_group,
get_pcp_group,
@@ -1414,6 +1415,41 @@ class KVCacheStoreRecvingThread(KVTransferThread):
class MooncakeStoreWorker:
"""Worker-side component for MooncakeStoreConnector."""
@staticmethod
def _create_mem_pool(
extra_config: dict[str, Any], model_config: ModelConfig
) -> torch.cuda.MemPool | None:
pool_type = str(extra_config.get("custom_mem_pool") or "").upper()
if not pool_type:
return None
if pool_type not in ("NVLINK", "BAREX"):
raise ValueError(
f"Unsupported custom_mem_pool={pool_type!r}, "
"expected 'NVLINK' or 'BAREX'"
)
if model_config.enable_sleep_mode or model_config.enable_cumem_allocator:
raise ValueError(
"custom_mem_pool is incompatible with enable_sleep_mode "
"or enable_cumem_allocator; CuMemAllocator cannot manage "
"allocations from the custom pool."
)
try:
if pool_type == "NVLINK":
from mooncake.allocator import NVLinkAllocator as allocator_cls
else: # pool_type == "BAREX"
from mooncake.allocator import BarexAllocator as allocator_cls
except ImportError as e:
raise ImportError(
f"custom_mem_pool={pool_type!r} requires "
"mooncake-transfer-engine>=0.3.8. Please upgrade Mooncake."
) from e
device = torch.device("cuda", torch.accelerator.current_device_index())
allocator = allocator_cls.get_allocator(device)
mem_pool = torch.cuda.MemPool(allocator.allocator())
logger.info("Using Mooncake custom memory pool: %s", pool_type)
return mem_pool
def __init__(
self,
vllm_config: VllmConfig,
@@ -1471,6 +1507,10 @@ class MooncakeStoreWorker:
# Initialize MooncakeDistributedStore with its own TransferEngine
store_config = MooncakeStoreConfig.load_from_config()
# Validate and create the custom memory pool before initializing the
# store so failures cannot leak a partially initialized store handle.
self._mem_pool = self._create_mem_pool(extra_config, model_config)
self.store = MooncakeDistributedStore()
local_ip = get_ip()
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
@@ -1741,6 +1781,13 @@ class MooncakeStoreWorker:
)
return token_dbs
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Return a context manager for the custom MemPool, or None if
no custom pool is configured."""
if self._mem_pool is None:
return None
return torch.cuda.use_mem_pool(self._mem_pool)
def _spec_tp_replication_factor(self, spec: KVCacheSpec) -> int:
if self.dcp_size > 1:
return 1
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from collections.abc import Callable, Iterable
from contextlib import AbstractContextManager
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
@@ -294,6 +295,26 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
# ==============================
# Worker-side methods
# ==============================
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Forward a custom KV cache memory pool from a child connector.
KV cache is allocated once and can only live in one pool, so at
most one child may provide a context.
"""
found: list[tuple[str, AbstractContextManager]] = []
for connector in self._connectors:
if (ctx := connector.get_mem_pool_context()) is not None:
found.append((type(connector).__name__, ctx))
if len(found) > 1:
names = [name for name, _ in found]
raise ValueError(
f"Multiple connectors provide a KV cache memory pool {names}; "
"KV cache is allocated once and can only live in one pool. "
"Configure custom_mem_pool on at most one connector."
)
return found[0][1] if found else None
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
for c in self._connectors:
c.start_load_kv(forward_context, **kwargs)