[KV Connector] Support sleep mode with MooncakeConnector over RDMA (#59625)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Dakai An <dakaian108@gmail.com> Co-authored-by: Dakai An <77474977+andakai@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
Dakai An
Dakai An
parent
f4b8d43a68
commit
3ccc0a90b6
@@ -308,6 +308,8 @@ class Worker(WorkerBase):
|
||||
name: buffer.cpu().clone() for name, buffer in draft.named_buffers()
|
||||
}
|
||||
|
||||
if has_kv_transfer_group():
|
||||
get_kv_transfer_group().unregister_kv_caches()
|
||||
self.sleep_mode_backend.suspend(level)
|
||||
if self.vllm_config.model_config.enable_nccl_comm_suspend:
|
||||
suspend_device_comms()
|
||||
@@ -331,6 +333,8 @@ class Worker(WorkerBase):
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
self.sleep_mode_backend.resume(tags)
|
||||
if has_kv_transfer_group() and (tags is None or "kv_cache" in tags):
|
||||
get_kv_transfer_group().restore_kv_caches()
|
||||
if self.vllm_config.model_config.enable_nccl_comm_suspend:
|
||||
resume_device_comms()
|
||||
|
||||
@@ -356,6 +360,8 @@ class Worker(WorkerBase):
|
||||
self.synchronize_device()
|
||||
|
||||
def discard(self, tags: tuple[str, ...]) -> None:
|
||||
if has_kv_transfer_group() and "kv_cache" in tags:
|
||||
get_kv_transfer_group().unregister_kv_caches()
|
||||
self.sleep_mode_backend.discard(tags)
|
||||
|
||||
def checkpoint_prepare(self) -> None:
|
||||
|
||||
Reference in New Issue
Block a user