[Feature] Offload KV-init runtime state on sleep (#59158)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-02 09:49:21 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent c15672cfb4
commit 8e58ac22af
4 changed files with 45 additions and 4 deletions
@@ -327,6 +327,35 @@ def test_reentered_tag(free_x_early):
allocator.release_pools()
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_runtime_state_survives_sleep(level):
"""Runtime state is offloaded at both levels and restored by the first
selective wake; a nested kv_cache pool keeps its own tag."""
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool("runtime"):
state = torch.arange(1 << 20, device=DEVICE_TYPE)
with allocator.use_memory_pool("kv_cache"):
kv = torch.zeros(1 << 20, device=DEVICE_TYPE)
assert {d.tag for d in allocator.pointer_to_data.values()} == {
"runtime",
"kv_cache",
}
backend = CuMemBackend()
backend.suspend(level=level)
assert mapped_usage(allocator) == 0
backend.resume(tags=["weights"])
assert torch.equal(state, torch.arange(1 << 20, device=DEVICE_TYPE))
backend.resume(tags=["kv_cache"])
kv.fill_(1)
assert int(kv.sum()) == kv.numel()
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_level2_discards_ordinary_tensor_with_weights_tag():