[KV Connector] Support sleep mode with MooncakeConnector over RDMA (#59625)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: Dakai An <77474977+andakai@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-10-14 04:18:47 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 Dakai An Dakai An
parent f4b8d43a68
commit 3ccc0a90b6
14 changed files with 205 additions and 8 deletions
@@ -56,6 +56,7 @@ from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.config.kv_transfer import KVTransferConfig
from vllm.distributed.kv_events import KVCacheEvent, KVConnectorKVEvents
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
@@ -205,6 +206,11 @@ class KVConnectorBase_V1(ABC):
"""
return self._kv_transfer_config.is_kv_producer
@classmethod
def supports_sleep_mode(cls, kv_transfer_config: "KVTransferConfig") -> bool:
"""Whether the connector can follow the KV cache remap of sleep mode."""
return True
def __init__(
self,
vllm_config: "VllmConfig",
@@ -293,6 +299,15 @@ class KVConnectorBase_V1(ABC):
"""
return
def unregister_kv_caches(self) -> None:
"""Drop KV cache registrations before sleep. Transfers are already
drained. Idempotent."""
return
def restore_kv_caches(self) -> None:
"""Redo `unregister_kv_caches` after wake-up. Idempotent."""
return
def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp):
"""Set the xPU-specific ops for copying KV between host and device.
Needed when host buffer is used for kv transfer (e.g., in NixlConnector)
@@ -3,6 +3,7 @@
import asyncio
import itertools
import logging
import os
import queue
import threading
import time
@@ -21,7 +22,7 @@ import zmq.asyncio
from huggingface_hub.utils import httpx
from vllm import envs
from vllm.config import VllmConfig
from vllm.config import KVTransferConfig, VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import (
EngineId,
TransferTopology,
@@ -800,10 +801,34 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
############################################################
# Worker Side Methods
############################################################
@classmethod
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
# Sleep mode is only validated over RDMA, not NVLink or TENT.
if any(
v in os.environ
for v in (
"MC_FORCE_MNNVL",
"MC_INTRANODE_NVLINK",
"MC_USE_TENT",
"MC_USE_TEV1",
)
):
return False
extra_config = kv_transfer_config.kv_connector_extra_config
return extra_config.get("mooncake_protocol", "rdma") == "rdma"
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def unregister_kv_caches(self) -> None:
assert self.connector_worker is not None
self.connector_worker.unregister_kv_caches()
def restore_kv_caches(self) -> None:
assert self.connector_worker is not None
self.connector_worker.restore_kv_caches()
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
@@ -1239,6 +1264,8 @@ class MooncakeConnectorWorker:
self.region_row_offsets: list[int] = []
self.opaque_packed_storages: set[int] = set()
self.seen_base_addresses: list[int] = []
self._kv_data_lens: list[int] = []
self._kv_unregistered = False
# Aligned regions depend only on the peer's registered layout.
# The third item is an error string when alignment cannot proceed.
self._prepared_transfer_regions: dict[
@@ -2156,6 +2183,7 @@ class MooncakeConnectorWorker:
self.kv_caches_base_addr = region_base_addresses
self.seen_base_addresses = kv_data_ptrs
self._kv_data_lens = kv_data_lens
if not kv_data_ptrs:
raise RuntimeError("No KV cache tensors were registered with Mooncake.")
@@ -2211,6 +2239,24 @@ class MooncakeConnectorWorker:
f"{ready_timeout:.0f}s."
)
def unregister_kv_caches(self) -> None:
if self._kv_unregistered or not self.seen_base_addresses:
return
ret_value = self.engine.batch_unregister_memory(self.seen_base_addresses)
if ret_value != 0:
raise RuntimeError("Mooncake batch memory unregistration failed.")
self._kv_unregistered = True
def restore_kv_caches(self) -> None:
if not self._kv_unregistered:
return
ret_value = self.engine.batch_register_memory(
self.seen_base_addresses, self._kv_data_lens
)
if ret_value != 0:
raise RuntimeError("Mooncake batch memory registration failed.")
self._kv_unregistered = False
async def fetch_finished_recving_reqs(self) -> set[ReqId]:
finished_recving_reqs = self.finished_recving_reqs
self.finished_recving_reqs = set()
@@ -18,7 +18,7 @@ from typing import Any
import torch
from vllm.config import VllmConfig
from vllm.config import KVTransferConfig, VllmConfig
from vllm.distributed.kv_events import (
BlockStored,
KVCacheEvent,
@@ -127,6 +127,10 @@ class MooncakeStoreKVEvents(KVConnectorKVEvents):
class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
"""KV connector using MooncakeDistributedStore as shared KV pool."""
@classmethod
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
return False
@staticmethod
def _validate_kv_cache_config(
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
@@ -16,7 +16,7 @@ import numpy as np
import torch
import zmq
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.config import CUDAGraphMode, KVTransferConfig, VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import (
BlockIds,
clip_ssm_state_blocks,
@@ -229,6 +229,10 @@ def resolve_moriio_transfer_ack(
class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
_cache_hit_source = CacheHitSource.P2P
@classmethod
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
return False
@property
def supports_divergent_local_hybrid_hits(self) -> bool:
# The READ path always transfers the recurrent-state slot, including
@@ -222,6 +222,17 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
def requires_kv_delivery(self) -> bool:
return any(c.requires_kv_delivery for c in self._connectors)
@classmethod
def supports_sleep_mode(cls, kv_transfer_config: "KVTransferConfig") -> bool:
children = [
KVTransferConfig(**{"engine_id": kv_transfer_config.engine_id, **c})
for c in kv_transfer_config.kv_connector_extra_config.get("connectors", [])
]
return bool(children) and all(
KVConnectorFactory.get_connector_class(c).supports_sleep_mode(c)
for c in children
)
@classmethod
def _get_connector_classes_and_configs(
cls, vllm_config: "VllmConfig"
@@ -252,6 +263,14 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
for c in self._connectors:
c.register_kv_caches(kv_caches)
def unregister_kv_caches(self) -> None:
for c in self._connectors:
c.unregister_kv_caches()
def restore_kv_caches(self) -> None:
for c in self._connectors:
c.restore_kv_caches()
def bind_kv_cache_manager(self, kv_cache_manager: "KVCacheManager") -> None:
for connector in self._connectors:
connector.bind_kv_cache_manager(kv_cache_manager)
@@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.config import KVTransferConfig, VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import (
EngineId,
)
@@ -140,6 +140,10 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
############################################################
# Class Methods
############################################################
@classmethod
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
return False
@classmethod
def get_required_kvcache_layout(cls, vllm_config: VllmConfig):
if vllm_config.model_config is None: