[mooncake] support CUSTOM_MEM_POOL in vllm (#49300)
Signed-off-by: bruce.xu <bruce.xb@alibaba-inc.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: bruce.xu <bruce.xb@alibaba-inc.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
bruce.xu
Yifan Qiao
OpenAI Codex
parent
cd1daf58a9
commit
eb0f2ca37f
@@ -42,6 +42,7 @@ The class provides the following primitives:
|
||||
import enum
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import AbstractContextManager
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
@@ -227,6 +228,13 @@ class KVConnectorBase_V1(ABC):
|
||||
# Worker-side methods
|
||||
# ==============================
|
||||
|
||||
def get_mem_pool_context(self) -> AbstractContextManager | None:
|
||||
"""Return a custom KV cache allocation context, if configured.
|
||||
|
||||
Returning None uses the engine's default memory pool.
|
||||
"""
|
||||
return None
|
||||
|
||||
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
|
||||
"""Set the connector metadata from the scheduler.
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ enabling prefix caching via hash-based deduplication.
|
||||
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Sequence
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -322,6 +323,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# Worker-side methods
|
||||
# ============================================================
|
||||
|
||||
def get_mem_pool_context(self) -> AbstractContextManager | None:
|
||||
"""Return a context manager for the custom MemPool, or None.
|
||||
|
||||
Called by the Worker before ``initialize_kv_cache`` so that KV
|
||||
cache is allocated from the Mooncake-managed pool when
|
||||
``custom_mem_pool`` is set in ``kv_connector_extra_config``.
|
||||
"""
|
||||
if self.connector_worker is None:
|
||||
return None
|
||||
return self.connector_worker.get_mem_pool_context()
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
@@ -19,13 +19,14 @@ import threading
|
||||
import time
|
||||
from collections.abc import Callable, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Any, TypeVar
|
||||
|
||||
import torch
|
||||
import zmq
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_dcp_group,
|
||||
get_pcp_group,
|
||||
@@ -1414,6 +1415,41 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
class MooncakeStoreWorker:
|
||||
"""Worker-side component for MooncakeStoreConnector."""
|
||||
|
||||
@staticmethod
|
||||
def _create_mem_pool(
|
||||
extra_config: dict[str, Any], model_config: ModelConfig
|
||||
) -> torch.cuda.MemPool | None:
|
||||
pool_type = str(extra_config.get("custom_mem_pool") or "").upper()
|
||||
if not pool_type:
|
||||
return None
|
||||
if pool_type not in ("NVLINK", "BAREX"):
|
||||
raise ValueError(
|
||||
f"Unsupported custom_mem_pool={pool_type!r}, "
|
||||
"expected 'NVLINK' or 'BAREX'"
|
||||
)
|
||||
if model_config.enable_sleep_mode or model_config.enable_cumem_allocator:
|
||||
raise ValueError(
|
||||
"custom_mem_pool is incompatible with enable_sleep_mode "
|
||||
"or enable_cumem_allocator; CuMemAllocator cannot manage "
|
||||
"allocations from the custom pool."
|
||||
)
|
||||
try:
|
||||
if pool_type == "NVLINK":
|
||||
from mooncake.allocator import NVLinkAllocator as allocator_cls
|
||||
else: # pool_type == "BAREX"
|
||||
from mooncake.allocator import BarexAllocator as allocator_cls
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
f"custom_mem_pool={pool_type!r} requires "
|
||||
"mooncake-transfer-engine>=0.3.8. Please upgrade Mooncake."
|
||||
) from e
|
||||
|
||||
device = torch.device("cuda", torch.accelerator.current_device_index())
|
||||
allocator = allocator_cls.get_allocator(device)
|
||||
mem_pool = torch.cuda.MemPool(allocator.allocator())
|
||||
logger.info("Using Mooncake custom memory pool: %s", pool_type)
|
||||
return mem_pool
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -1471,6 +1507,10 @@ class MooncakeStoreWorker:
|
||||
|
||||
# Initialize MooncakeDistributedStore with its own TransferEngine
|
||||
store_config = MooncakeStoreConfig.load_from_config()
|
||||
# Validate and create the custom memory pool before initializing the
|
||||
# store so failures cannot leak a partially initialized store handle.
|
||||
self._mem_pool = self._create_mem_pool(extra_config, model_config)
|
||||
|
||||
self.store = MooncakeDistributedStore()
|
||||
local_ip = get_ip()
|
||||
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
|
||||
@@ -1741,6 +1781,13 @@ class MooncakeStoreWorker:
|
||||
)
|
||||
return token_dbs
|
||||
|
||||
def get_mem_pool_context(self) -> AbstractContextManager | None:
|
||||
"""Return a context manager for the custom MemPool, or None if
|
||||
no custom pool is configured."""
|
||||
if self._mem_pool is None:
|
||||
return None
|
||||
return torch.cuda.use_mem_pool(self._mem_pool)
|
||||
|
||||
def _spec_tp_replication_factor(self, spec: KVCacheSpec) -> int:
|
||||
if self.dcp_size > 1:
|
||||
return 1
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import AbstractContextManager
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
@@ -294,6 +295,26 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# ==============================
|
||||
# Worker-side methods
|
||||
# ==============================
|
||||
def get_mem_pool_context(self) -> AbstractContextManager | None:
|
||||
"""Forward a custom KV cache memory pool from a child connector.
|
||||
|
||||
KV cache is allocated once and can only live in one pool, so at
|
||||
most one child may provide a context.
|
||||
"""
|
||||
found: list[tuple[str, AbstractContextManager]] = []
|
||||
for connector in self._connectors:
|
||||
if (ctx := connector.get_mem_pool_context()) is not None:
|
||||
found.append((type(connector).__name__, ctx))
|
||||
|
||||
if len(found) > 1:
|
||||
names = [name for name, _ in found]
|
||||
raise ValueError(
|
||||
f"Multiple connectors provide a KV cache memory pool {names}; "
|
||||
"KV cache is allocated once and can only live in one pool. "
|
||||
"Configure custom_mem_pool on at most one connector."
|
||||
)
|
||||
return found[0][1] if found else None
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
|
||||
for c in self._connectors:
|
||||
c.start_load_kv(forward_context, **kwargs)
|
||||
|
||||
Reference in New Issue
Block a user