[KV Connector] Support disk offloading in MooncakeStoreConnector (#42689)

Signed-off-by: Zhewen Li <zhewenli@inferact.ai>
Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Zhewen Li
2026-05-16 13:34:15 -07:00
committed by GitHub
co-authored by Zhewen Li Claude Opus 4.7
parent 787bc0d031
commit 36e74c9ea4
5 changed files with 1385 additions and 55 deletions
@@ -0,0 +1,115 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mooncake requester config helpers."""
from collections.abc import Mapping
from typing import Any
import torch
import vllm.envs as envs
from vllm.logger import init_logger
logger = init_logger(__name__)
def normalize_string_override(value: Any) -> str | None:
if not isinstance(value, str):
return None
normalized = value.strip()
return normalized or None
def get_current_physical_gpu_index() -> int | None:
try:
from vllm.platforms import current_platform
except ImportError:
return None
try:
device_index = torch.accelerator.current_device_index()
physical_device_id = current_platform.device_id_to_physical_device_id(
device_index
)
return int(physical_device_id)
except Exception:
return None
def get_requester_local_hostname(local_ip: str) -> str:
override = normalize_string_override(envs.MOONCAKE_REQUESTER_LOCAL_HOSTNAME)
if override is not None:
return override
return local_ip
def get_configured_preferred_segment(
extra_config: Mapping[str, Any],
) -> str | None:
preferred_segment = normalize_string_override(extra_config.get("preferred_segment"))
if preferred_segment is not None:
return preferred_segment
if extra_config.get("preferred_segment") is not None:
raise ValueError(
"Mooncake preferred_segment override must be a non-empty string"
)
env_value = normalize_string_override(envs.MOONCAKE_PREFERRED_SEGMENT)
if env_value is not None:
logger.info(
"Mooncake preferred_segment from MOONCAKE_PREFERRED_SEGMENT: %s",
env_value,
)
return env_value
return None
def _get_explicit_worker_rnic(device_list: str) -> str:
entries = [entry.strip() for entry in device_list.split(",")]
if any(not entry for entry in entries):
raise ValueError(
"Mooncake worker device_name contains an empty RDMA device entry"
)
if len(entries) == 1:
return entries[0]
gpu_index = get_current_physical_gpu_index()
if gpu_index is None:
raise RuntimeError(
"Mooncake RDMA requester could not determine the local physical GPU index"
)
if gpu_index >= len(entries):
raise ValueError(
"Mooncake worker device list does not cover local GPU "
f"{gpu_index}: {device_list}"
)
device_name = entries[gpu_index]
logger.info(
"Mooncake selected worker RNIC %s from explicit device list for local GPU %s",
device_name,
gpu_index,
)
return device_name
def get_configured_worker_rnic(
*,
protocol: str,
configured_device: str,
) -> str:
normalized_device = normalize_string_override(configured_device)
if normalized_device is not None:
return _get_explicit_worker_rnic(normalized_device)
if protocol not in {"rdma", "efa"}:
return ""
logger.warning(
"No RDMA devices specified for Mooncake backend (protocol=%s). "
"Set 'device_name' in mooncake_config.json to a single RNIC name "
"or a comma-separated CSV indexed by physical GPU; falling back to "
"Mooncake's built-in auto-selection, which may converge on the same "
"NIC across all DP ranks and saturate bandwidth.",
protocol,
)
return ""
@@ -13,10 +13,11 @@ and MooncakeDistributedStore integration.
import json
import os
import queue
import socket
import threading
from collections import defaultdict
from dataclasses import dataclass
from typing import Any
from typing import Any, Literal
import regex as re
import torch
@@ -31,10 +32,11 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.distributed.kv_events import BlockStored
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
get_mooncake_dp_engine_index,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
ChunkedTokenDatabase,
KeyMetadata,
MooncakeStoreConnectorMetadata,
@@ -49,19 +51,48 @@ logger = init_logger(__name__)
DEFAULT_GLOBAL_SEGMENT_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
DEFAULT_LOCAL_BUFFER_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
MOONCAKE_NO_AVAILABLE_HANDLE = -200
# Mirrors FileStorageConfig::local_buffer_size in Mooncake C++.
DEFAULT_MOONCAKE_DISK_STAGING_BUFFER_BYTES = 1280 * 1024 * 1024
# Mirrors DirectIO alignment in Mooncake's AllocateBatch.
_DIRECT_IO_ALIGNMENT = 4096
_DIRECT_IO_PADDING_BYTES = 2 * _DIRECT_IO_ALIGNMENT
MooncakeMode = Literal["embedded", "standalone-store"]
@dataclass
class MooncakeStoreConfig:
"""Configuration for MooncakeDistributedStore."""
"""Configuration for MooncakeDistributedStore.
``mode`` selects the topology: ``embedded`` (each rank contributes
``global_segment_size`` in-process) or ``standalone-store`` (rank
contributes 0; an external ``mooncake_client`` process owns the pool
and the SSD tier).
"""
metadata_server: str
global_segment_size: int
local_buffer_size: int
master_server_address: str
protocol: str
device_name: str
master_server_address: str
mode: MooncakeMode = "embedded"
global_segment_size: int = DEFAULT_GLOBAL_SEGMENT_SIZE
local_buffer_size: int = DEFAULT_LOCAL_BUFFER_SIZE
enable_offload: bool = False
def __post_init__(self) -> None:
if self.mode not in ("embedded", "standalone-store"):
raise ValueError(f"unknown Mooncake mode: {self.mode!r}")
if self.local_buffer_size <= 0:
raise ValueError("local_buffer_size must be > 0")
if self.mode == "embedded" and self.global_segment_size == 0:
raise ValueError("embedded mode requires global_segment_size > 0")
if self.mode == "standalone-store" and self.global_segment_size != 0:
raise ValueError("standalone-store mode requires global_segment_size == 0")
@staticmethod
def from_file(file_path: str) -> "MooncakeStoreConfig":
@@ -69,15 +100,17 @@ class MooncakeStoreConfig:
config = json.load(file)
return MooncakeStoreConfig(
metadata_server=config.get("metadata_server", ""),
master_server_address=config.get("master_server_address", ""),
protocol=config.get("protocol", "rdma"),
device_name=config.get("device_name", ""),
mode=config.get("mode", "embedded"),
global_segment_size=_parse_size(
config.get("global_segment_size", DEFAULT_GLOBAL_SEGMENT_SIZE)
),
local_buffer_size=_parse_size(
config.get("local_buffer_size", DEFAULT_LOCAL_BUFFER_SIZE)
),
protocol=config.get("protocol", "rdma"),
device_name=config.get("device_name", ""),
master_server_address=config.get("master_server_address", ""),
enable_offload=bool(config.get("enable_offload", False)),
)
@staticmethod
@@ -125,6 +158,158 @@ def _parse_size(value: Any) -> int:
return int(numeric_value * multiplier)
def _align_up(value: int, alignment: int) -> int:
return ((value + alignment - 1) // alignment) * alignment
def _estimate_disk_offload_staging_bytes(size_list: list[int]) -> int:
data_size = sum(size_list)
return _align_up(data_size, _DIRECT_IO_ALIGNMENT) + _DIRECT_IO_PADDING_BYTES
def _get_usable_disk_offload_buffer_budget_bytes(raw_budget_bytes: int) -> int:
return max(1, int(raw_budget_bytes * envs.VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO))
def _split_disk_offload_load_batches(
keys: list[str],
addrs: list[list[int]],
sizes: list[list[int]],
usable_budget_bytes: int,
raw_budget_bytes: int,
) -> tuple[list[tuple[list[str], list[list[int]], list[list[int]]]], str | None]:
"""Split a GET into sub-batches that fit the owner's staging buffer.
``addrs[i]`` / ``sizes[i]`` are scatter-gather lists (K/V or multi-layer
segments) for key ``i``. ``usable_budget_bytes`` caps a multi-key batch;
``raw_budget_bytes`` is the hard per-key cap.
Returns ``(batches, oversize_key)``. Aborts with ``([], key)`` if any
single key exceeds ``raw_budget_bytes``; otherwise ``oversize_key`` is
``None``.
"""
batches: list[tuple[list[str], list[list[int]], list[list[int]]]] = []
batch_keys: list[str] = []
batch_addrs: list[list[int]] = []
batch_sizes: list[list[int]] = []
batch_bytes = 0
for key, addr, size in zip(keys, addrs, sizes, strict=True):
key_bytes = _estimate_disk_offload_staging_bytes(size)
if key_bytes > raw_budget_bytes:
return [], key
if key_bytes > usable_budget_bytes:
if batch_keys:
batches.append((batch_keys, batch_addrs, batch_sizes))
batch_keys, batch_addrs, batch_sizes = [], [], []
batch_bytes = 0
batches.append(([key], [addr], [size]))
continue
if batch_keys and batch_bytes + key_bytes > usable_budget_bytes:
batches.append((batch_keys, batch_addrs, batch_sizes))
batch_keys, batch_addrs, batch_sizes = [], [], []
batch_bytes = 0
batch_keys.append(key)
batch_addrs.append(addr)
batch_sizes.append(size)
batch_bytes += key_bytes
if batch_keys:
batches.append((batch_keys, batch_addrs, batch_sizes))
return batches, None
def _call_replica_predicate(replica_desc: Any, method_name: str) -> bool:
method = getattr(replica_desc, method_name, None)
if method is None:
return False
try:
return bool(method())
except Exception:
return False
def _classify_replica_tier(replica_descs: Any) -> str:
if not replica_descs:
return "unknown"
try:
replica_desc = replica_descs[0]
except (IndexError, KeyError, TypeError):
return "unknown"
if _call_replica_predicate(replica_desc, "is_memory_replica"):
return "memory"
if _call_replica_predicate(
replica_desc, "is_disk_replica"
) or _call_replica_predicate(replica_desc, "is_local_disk_replica"):
return "disk"
return "unknown"
def _get_replica_tiers_by_key(store: Any, keys: list[str]) -> dict[str, str]:
tiers_by_key = {key: "unknown" for key in keys}
try:
replica_descs_by_key = store.batch_get_replica_desc(keys)
except Exception as e:
logger.warning(
"Failed to get Mooncake replica descriptors for tier logging "
"(batch_keys=%d, error=%s); marking tiers unknown",
len(keys),
e,
)
return tiers_by_key
for key in keys:
if hasattr(replica_descs_by_key, "get"):
replica_descs = replica_descs_by_key.get(key)
else:
try:
replica_descs = replica_descs_by_key[key]
except (KeyError, TypeError):
replica_descs = None
tiers_by_key[key] = _classify_replica_tier(replica_descs)
return tiers_by_key
def _log_mooncake_load_tier_summary(
req_id: str,
batch_keys: list[str],
load_results: list[int],
tiers_by_key: dict[str, str],
) -> None:
tier_counts = {"memory": 0, "disk": 0, "unknown": 0}
bytes_by_tier = {"memory": 0, "disk": 0, "unknown": 0}
success_keys = 0
failed_keys = 0
for index, key in enumerate(batch_keys):
tier = tiers_by_key.get(key, "unknown")
if tier not in tier_counts:
tier = "unknown"
tier_counts[tier] += 1
value = load_results[index] if index < len(load_results) else -1
if value >= 0:
success_keys += 1
bytes_by_tier[tier] += int(value)
else:
failed_keys += 1
logger.info(
"Mooncake load tier summary: req_id=%s batch_keys=%d "
"memory_keys=%d disk_keys=%d unknown_keys=%d "
"success_keys=%d failed_keys=%d bytes_by_tier=%s",
req_id,
len(batch_keys),
tier_counts["memory"],
tier_counts["disk"],
tier_counts["unknown"],
success_keys,
failed_keys,
bytes_by_tier,
)
# ============================================================
# Transfer Threads
# ============================================================
@@ -207,6 +392,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
kv_role: str,
ready_event: threading.Event,
enable_kv_event: bool = False,
replicate_config: Any = None,
):
super().__init__(
store,
@@ -220,8 +406,11 @@ class KVCacheStoreSendingThread(KVTransferThread):
self.kv_role = kv_role
self.stored_requests: defaultdict[str, int] = defaultdict(int)
self.enable_kv_event = enable_kv_event
# Caller always passes a non-None ReplicateConfig — see
# MooncakeStoreWorker.__init__ where store_replicate_config is built.
self.replicate_config = replicate_config
# Pause store requests when CPU offloading is under pressure.
# Pause store requests when CPU/disk offloading is under pressure.
self._store_pressure_active = False
self._skip_store_requests: set[str] = set()
@@ -270,7 +459,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
return
if self._should_skip_request(req_id):
logger.debug(
"Skipping Mooncake store for request %s while CPU offloading "
"Skipping Mooncake store for request %s while CPU/disk offloading "
"is under pressure",
req_id,
)
@@ -357,7 +546,12 @@ class KVCacheStoreSendingThread(KVTransferThread):
current_event.synchronize()
try:
res = self.store.batch_put_from_multi_buffers(keys, addrs, sizes)
res = self.store.batch_put_from_multi_buffers(
keys,
addrs,
sizes,
self.replicate_config,
)
failed = [i for i, v in enumerate(res) if v < 0]
if failed:
# Compute total bytes attempted for this batch
@@ -379,7 +573,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
and not self._mark_request_skipped_for_pressure(req_id)
):
logger.warning(
"Detected Mooncake CPU offloading pressure "
"Detected Mooncake CPU/disk offloading pressure "
"(NO_AVAILABLE_HANDLE); skipping future store "
"batches for request %s until a later store "
"batch succeeds",
@@ -387,7 +581,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
)
elif self._clear_store_pressure():
logger.info(
"Mooncake CPU offloading pressure cleared after a "
"Mooncake CPU/disk offloading pressure cleared after a "
"successful store batch"
)
except Exception as e:
@@ -410,6 +604,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
block_size: int,
tp_rank: int,
ready_event: threading.Event,
disk_offload_buffer_budget_bytes: int | None = None,
):
super().__init__(
store,
@@ -419,6 +614,14 @@ class KVCacheStoreRecvingThread(KVTransferThread):
ready_event,
name="KVCacheStoreRecvingThread",
)
self.disk_offload_buffer_budget_bytes = disk_offload_buffer_budget_bytes
self.usable_disk_offload_buffer_budget_bytes = (
None
if disk_offload_buffer_budget_bytes is None
else _get_usable_disk_offload_buffer_budget_bytes(
disk_offload_buffer_budget_bytes
)
)
def _handle_request(self, req_meta: ReqMeta):
token_len = req_meta.load_spec.token_len # type: ignore[union-attr]
@@ -456,26 +659,69 @@ class KVCacheStoreRecvingThread(KVTransferThread):
+ size_list[: self.tp_rank % len(size_list)]
)
try:
res = self.store.batch_get_into_multi_buffers(
key_list_c, addr_list_c, size_list_c
load_batches = [(key_list_c, addr_list_c, size_list_c)]
if self.usable_disk_offload_buffer_budget_bytes is not None:
total_staging_bytes = sum(
_estimate_disk_offload_staging_bytes(size) for size in size_list_c
)
failed = [
(key, value)
for key, value in zip(key_list_c, res, strict=True)
if value < 0
]
if failed:
logger.warning(
"Failed to get %d Mooncake keys (batch_keys=%d, first_failures=%s)",
len(failed),
len(key_list_c),
failed[:3],
if total_staging_bytes > self.usable_disk_offload_buffer_budget_bytes:
assert self.disk_offload_buffer_budget_bytes is not None
load_batches, oversized_key = _split_disk_offload_load_batches(
key_list_c,
addr_list_c,
size_list_c,
self.usable_disk_offload_buffer_budget_bytes,
self.disk_offload_buffer_budget_bytes,
)
if oversized_key is not None:
oversized_key_index = key_list_c.index(oversized_key)
oversized_key_bytes = _estimate_disk_offload_staging_bytes(
size_list_c[oversized_key_index]
)
logger.warning(
"Skipping Mooncake load for request %s because key %s "
"requires %d staging bytes, exceeding budget %d",
req_id,
oversized_key,
oversized_key_bytes,
self.disk_offload_buffer_budget_bytes,
)
self.set_finished_request(req_id)
self.request_queue.task_done()
return
current_batch_keys: list[str] = key_list_c
try:
for batch_keys, batch_addrs, batch_sizes in load_batches:
current_batch_keys = batch_keys
tiers_by_key: dict[str, str] | None = None
if envs.VLLM_MOONCAKE_STORE_TIER_LOG:
tiers_by_key = _get_replica_tiers_by_key(self.store, batch_keys)
res = self.store.batch_get_into_multi_buffers(
batch_keys, batch_addrs, batch_sizes
)
if tiers_by_key is not None:
_log_mooncake_load_tier_summary(
req_id, batch_keys, res, tiers_by_key
)
failed = [
(key, value)
for key, value in zip(batch_keys, res, strict=True)
if value < 0
]
if failed:
logger.warning(
"Failed to get %d Mooncake keys from sub-batch "
"(batch_keys=%d, first_failures=%s)",
len(failed),
len(batch_keys),
failed[:3],
)
break
except Exception as e:
logger.warning(
"Failed to get Mooncake batch %s, error: %s",
key_list_c[:3],
"Failed to get Mooncake sub-batch %s, error: %s",
current_batch_keys[:3],
e,
)
@@ -493,7 +739,10 @@ class MooncakeStoreWorker:
def __init__(self, vllm_config: VllmConfig):
try:
from mooncake.store import MooncakeDistributedStore # type: ignore
from mooncake.store import ( # type: ignore
MooncakeDistributedStore,
ReplicateConfig,
)
except ImportError as e:
raise ImportError(
"Please install mooncake by following the instructions at "
@@ -561,24 +810,79 @@ class MooncakeStoreWorker:
# Initialize MooncakeDistributedStore with its own TransferEngine
store_config = MooncakeStoreConfig.load_from_env()
extra_config = (
vllm_config.kv_transfer_config.kv_connector_extra_config
if vllm_config.kv_transfer_config
else {}
)
store_config.device_name = rdma_utils.get_configured_worker_rnic(
protocol=store_config.protocol,
configured_device=store_config.device_name,
)
self.store = MooncakeDistributedStore()
local_seg = get_ip()
config_dict = {
"local_hostname": local_seg,
"metadata_server": store_config.metadata_server,
"global_segment_size": str(store_config.global_segment_size),
"local_buffer_size": str(store_config.local_buffer_size),
"protocol": store_config.protocol,
"rdma_devices": store_config.device_name,
"master_server_addr": store_config.master_server_address,
}
ret = self.store.setup(config_dict)
local_ip = get_ip()
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
ret = self.store.setup(
local_hostname,
store_config.metadata_server,
store_config.global_segment_size,
store_config.local_buffer_size,
store_config.protocol,
store_config.device_name,
store_config.master_server_address,
)
if ret != 0:
msg = "Initialize MooncakeDistributedStore failed."
logger.error(msg)
raise RuntimeError(msg)
preferred_segment = rdma_utils.get_configured_preferred_segment(extra_config)
self.preferred_segment = preferred_segment
self.store_replicate_config = ReplicateConfig()
if preferred_segment is not None:
self.store_replicate_config.preferred_segment = preferred_segment
logger.info(
"Mooncake mode=%s (global_segment_size=%d, local_buffer_size=%d, "
"preferred_segment=%s, enable_offload=%s)",
store_config.mode,
store_config.global_segment_size,
store_config.local_buffer_size,
preferred_segment or "<none>",
store_config.enable_offload,
)
if store_config.mode == "embedded":
if store_config.enable_offload and preferred_segment is None:
logger.warning(
"enable_offload is set in embedded mode without "
"preferred_segment; SSD tier will only see puts that "
"happen to land on the owner segment."
)
if preferred_segment is not None:
logger.warning(
"preferred_segment=%s with mode=embedded: rank-"
"contributed segments will be idle.",
preferred_segment,
)
elif (
store_config.mode == "standalone-store" and not store_config.enable_offload
):
logger.warning(
"standalone-store mode without enable_offload: large prefills "
"may exceed the owner DirectIO budget."
)
self.disk_offload_buffer_budget_bytes = (
DEFAULT_MOONCAKE_DISK_STAGING_BUFFER_BYTES
if store_config.enable_offload
else None
)
# Start lookup server on rank 0 for scheduler-side prefix queries
self.lookup_server: LookupKeyServer | None = None
if vllm_config.parallel_config.rank == 0:
self.lookup_server = LookupKeyServer(self, vllm_config)
kv_event_config = vllm_config.kv_events_config
self.enable_kv_events = False
if kv_event_config and kv_event_config.enable_kv_cache_events:
@@ -588,11 +892,6 @@ class MooncakeStoreWorker:
self.kv_recv_thread: KVCacheStoreRecvingThread | None = None
self.finished_store_req: set[str] = set()
# Start lookup server on rank 0 for scheduler-side prefix queries
self.lookup_server: LookupKeyServer | None = None
if vllm_config.parallel_config.rank == 0:
self.lookup_server = LookupKeyServer(self, vllm_config)
def register_cross_layers_kv_caches(self, kv_cache: torch.Tensor) -> None:
"""Register a cross-layers KV cache tensor.
@@ -695,6 +994,7 @@ class MooncakeStoreWorker:
self.kv_role,
ready_event_sending,
self.enable_kv_events,
self.store_replicate_config,
)
self.kv_send_thread.start()
@@ -705,6 +1005,7 @@ class MooncakeStoreWorker:
self.block_size,
self.tp_rank,
ready_event_recving,
disk_offload_buffer_budget_bytes=self.disk_offload_buffer_budget_bytes,
)
self.kv_recv_thread.start()
ready_event_recving.wait()
@@ -967,13 +1268,15 @@ class LookupKeyClient:
def get_zmq_rpc_path_lookup(vllm_config: VllmConfig) -> str:
"""Construct IPC path for ZMQ lookup socket."""
assert vllm_config.kv_transfer_config is not None
dp_rank = get_mooncake_dp_engine_index(vllm_config.parallel_config)
base_url = envs.VLLM_RPC_BASE_PATH
rpc_port = 0
assert vllm_config.kv_transfer_config is not None
hostname = socket.gethostname()
extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
if "lookup_rpc_port" in extra_config:
rpc_port = extra_config["lookup_rpc_port"]
uid = os.getuid()
logger.debug("Base URL: %s, RPC Port: %s, UID: %s", base_url, rpc_port, uid)
return f"ipc://{base_url}/lookup_rpc_port_{rpc_port}_uid{uid}_dp_rank{dp_rank}"
logger.debug("Base URL: %s, RPC Port: %s", base_url, rpc_port)
return (
f"ipc://{base_url}/lookup_rpc_port_{rpc_port}_host_{hostname}_dp_rank{dp_rank}"
)