[Feature] Offload KV-init runtime state on sleep (#59158)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c15672cfb4
commit
8e58ac22af
@@ -129,7 +129,9 @@ class CuMemBackend(SleepModeBackend):
|
||||
|
||||
self._state = "SUSPENDED"
|
||||
allocator = get_mem_allocator_instance()
|
||||
allocator.sleep(offload_tags=("weights",) if level == 1 else tuple())
|
||||
# Runtime state is kept at every level; weights only at level 1.
|
||||
offload_tags = ("weights", "runtime") if level == 1 else ("runtime",)
|
||||
allocator.sleep(offload_tags=offload_tags)
|
||||
|
||||
def resume(self, tags: list[str] | None = None) -> None:
|
||||
from vllm.device_allocator import get_mem_allocator_instance
|
||||
|
||||
Reference in New Issue
Block a user