[KV Connector] Support sleep mode with MooncakeConnector over RDMA (#59625)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Dakai An <dakaian108@gmail.com> Co-authored-by: Dakai An <77474977+andakai@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
Dakai An
Dakai An
parent
f4b8d43a68
commit
3ccc0a90b6
@@ -56,6 +56,7 @@ from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kv_transfer import KVTransferConfig
|
||||
from vllm.distributed.kv_events import KVCacheEvent, KVConnectorKVEvents
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
@@ -205,6 +206,11 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return self._kv_transfer_config.is_kv_producer
|
||||
|
||||
@classmethod
|
||||
def supports_sleep_mode(cls, kv_transfer_config: "KVTransferConfig") -> bool:
|
||||
"""Whether the connector can follow the KV cache remap of sleep mode."""
|
||||
return True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
@@ -293,6 +299,15 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return
|
||||
|
||||
def unregister_kv_caches(self) -> None:
|
||||
"""Drop KV cache registrations before sleep. Transfers are already
|
||||
drained. Idempotent."""
|
||||
return
|
||||
|
||||
def restore_kv_caches(self) -> None:
|
||||
"""Redo `unregister_kv_caches` after wake-up. Idempotent."""
|
||||
return
|
||||
|
||||
def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp):
|
||||
"""Set the xPU-specific ops for copying KV between host and device.
|
||||
Needed when host buffer is used for kv transfer (e.g., in NixlConnector)
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import asyncio
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
@@ -21,7 +22,7 @@ import zmq.asyncio
|
||||
from huggingface_hub.utils import httpx
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import KVTransferConfig, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
EngineId,
|
||||
TransferTopology,
|
||||
@@ -800,10 +801,34 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
############################################################
|
||||
# Worker Side Methods
|
||||
############################################################
|
||||
@classmethod
|
||||
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
|
||||
# Sleep mode is only validated over RDMA, not NVLink or TENT.
|
||||
if any(
|
||||
v in os.environ
|
||||
for v in (
|
||||
"MC_FORCE_MNNVL",
|
||||
"MC_INTRANODE_NVLINK",
|
||||
"MC_USE_TENT",
|
||||
"MC_USE_TEV1",
|
||||
)
|
||||
):
|
||||
return False
|
||||
extra_config = kv_transfer_config.kv_connector_extra_config
|
||||
return extra_config.get("mooncake_protocol", "rdma") == "rdma"
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
def unregister_kv_caches(self) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.unregister_kv_caches()
|
||||
|
||||
def restore_kv_caches(self) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.restore_kv_caches()
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
@@ -1239,6 +1264,8 @@ class MooncakeConnectorWorker:
|
||||
self.region_row_offsets: list[int] = []
|
||||
self.opaque_packed_storages: set[int] = set()
|
||||
self.seen_base_addresses: list[int] = []
|
||||
self._kv_data_lens: list[int] = []
|
||||
self._kv_unregistered = False
|
||||
# Aligned regions depend only on the peer's registered layout.
|
||||
# The third item is an error string when alignment cannot proceed.
|
||||
self._prepared_transfer_regions: dict[
|
||||
@@ -2156,6 +2183,7 @@ class MooncakeConnectorWorker:
|
||||
|
||||
self.kv_caches_base_addr = region_base_addresses
|
||||
self.seen_base_addresses = kv_data_ptrs
|
||||
self._kv_data_lens = kv_data_lens
|
||||
|
||||
if not kv_data_ptrs:
|
||||
raise RuntimeError("No KV cache tensors were registered with Mooncake.")
|
||||
@@ -2211,6 +2239,24 @@ class MooncakeConnectorWorker:
|
||||
f"{ready_timeout:.0f}s."
|
||||
)
|
||||
|
||||
def unregister_kv_caches(self) -> None:
|
||||
if self._kv_unregistered or not self.seen_base_addresses:
|
||||
return
|
||||
ret_value = self.engine.batch_unregister_memory(self.seen_base_addresses)
|
||||
if ret_value != 0:
|
||||
raise RuntimeError("Mooncake batch memory unregistration failed.")
|
||||
self._kv_unregistered = True
|
||||
|
||||
def restore_kv_caches(self) -> None:
|
||||
if not self._kv_unregistered:
|
||||
return
|
||||
ret_value = self.engine.batch_register_memory(
|
||||
self.seen_base_addresses, self._kv_data_lens
|
||||
)
|
||||
if ret_value != 0:
|
||||
raise RuntimeError("Mooncake batch memory registration failed.")
|
||||
self._kv_unregistered = False
|
||||
|
||||
async def fetch_finished_recving_reqs(self) -> set[ReqId]:
|
||||
finished_recving_reqs = self.finished_recving_reqs
|
||||
self.finished_recving_reqs = set()
|
||||
|
||||
@@ -18,7 +18,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import KVTransferConfig, VllmConfig
|
||||
from vllm.distributed.kv_events import (
|
||||
BlockStored,
|
||||
KVCacheEvent,
|
||||
@@ -127,6 +127,10 @@ class MooncakeStoreKVEvents(KVConnectorKVEvents):
|
||||
class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""KV connector using MooncakeDistributedStore as shared KV pool."""
|
||||
|
||||
@classmethod
|
||||
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _validate_kv_cache_config(
|
||||
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
|
||||
|
||||
@@ -16,7 +16,7 @@ import numpy as np
|
||||
import torch
|
||||
import zmq
|
||||
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.config import CUDAGraphMode, KVTransferConfig, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
BlockIds,
|
||||
clip_ssm_state_blocks,
|
||||
@@ -229,6 +229,10 @@ def resolve_moriio_transfer_ack(
|
||||
class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
_cache_hit_source = CacheHitSource.P2P
|
||||
|
||||
@classmethod
|
||||
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
# The READ path always transfers the recurrent-state slot, including
|
||||
|
||||
@@ -222,6 +222,17 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
return any(c.requires_kv_delivery for c in self._connectors)
|
||||
|
||||
@classmethod
|
||||
def supports_sleep_mode(cls, kv_transfer_config: "KVTransferConfig") -> bool:
|
||||
children = [
|
||||
KVTransferConfig(**{"engine_id": kv_transfer_config.engine_id, **c})
|
||||
for c in kv_transfer_config.kv_connector_extra_config.get("connectors", [])
|
||||
]
|
||||
return bool(children) and all(
|
||||
KVConnectorFactory.get_connector_class(c).supports_sleep_mode(c)
|
||||
for c in children
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _get_connector_classes_and_configs(
|
||||
cls, vllm_config: "VllmConfig"
|
||||
@@ -252,6 +263,14 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
for c in self._connectors:
|
||||
c.register_kv_caches(kv_caches)
|
||||
|
||||
def unregister_kv_caches(self) -> None:
|
||||
for c in self._connectors:
|
||||
c.unregister_kv_caches()
|
||||
|
||||
def restore_kv_caches(self) -> None:
|
||||
for c in self._connectors:
|
||||
c.restore_kv_caches()
|
||||
|
||||
def bind_kv_cache_manager(self, kv_cache_manager: "KVCacheManager") -> None:
|
||||
for connector in self._connectors:
|
||||
connector.bind_kv_cache_manager(kv_cache_manager)
|
||||
|
||||
@@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import KVTransferConfig, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
EngineId,
|
||||
)
|
||||
@@ -140,6 +140,10 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
############################################################
|
||||
# Class Methods
|
||||
############################################################
|
||||
@classmethod
|
||||
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: VllmConfig):
|
||||
if vllm_config.model_config is None:
|
||||
|
||||
Reference in New Issue
Block a user