[KV Connector] Support sleep mode with MooncakeConnector over RDMA (#59625)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: Dakai An <77474977+andakai@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-10-14 04:18:47 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 Dakai An Dakai An
parent f4b8d43a68
commit 3ccc0a90b6
14 changed files with 205 additions and 8 deletions
+6
View File
@@ -308,6 +308,8 @@ class Worker(WorkerBase):
name: buffer.cpu().clone() for name, buffer in draft.named_buffers()
}
if has_kv_transfer_group():
get_kv_transfer_group().unregister_kv_caches()
self.sleep_mode_backend.suspend(level)
if self.vllm_config.model_config.enable_nccl_comm_suspend:
suspend_device_comms()
@@ -331,6 +333,8 @@ class Worker(WorkerBase):
def wake_up(self, tags: list[str] | None = None) -> None:
self.sleep_mode_backend.resume(tags)
if has_kv_transfer_group() and (tags is None or "kv_cache" in tags):
get_kv_transfer_group().restore_kv_caches()
if self.vllm_config.model_config.enable_nccl_comm_suspend:
resume_device_comms()
@@ -356,6 +360,8 @@ class Worker(WorkerBase):
self.synchronize_device()
def discard(self, tags: tuple[str, ...]) -> None:
if has_kv_transfer_group() and "kv_cache" in tags:
get_kv_transfer_group().unregister_kv_caches()
self.sleep_mode_backend.discard(tags)
def checkpoint_prepare(self) -> None: