[Feature] Offload KV-init runtime state on sleep (#59158)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-02 09:49:21 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent c15672cfb4
commit 8e58ac22af
4 changed files with 45 additions and 4 deletions
+3 -1
View File
@@ -129,7 +129,9 @@ class CuMemBackend(SleepModeBackend):
self._state = "SUSPENDED"
allocator = get_mem_allocator_instance()
allocator.sleep(offload_tags=("weights",) if level == 1 else tuple())
# Runtime state is kept at every level; weights only at level 1.
offload_tags = ("weights", "runtime") if level == 1 else ("runtime",)
allocator.sleep(offload_tags=offload_tags)
def resume(self, tags: list[str] | None = None) -> None:
from vllm.device_allocator import get_mem_allocator_instance