[Feature] Offload KV-init runtime state on sleep (#59158)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c15672cfb4
commit
8e58ac22af
@@ -327,6 +327,35 @@ def test_reentered_tag(free_x_early):
|
||||
allocator.release_pools()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_runtime_state_survives_sleep(level):
|
||||
"""Runtime state is offloaded at both levels and restored by the first
|
||||
selective wake; a nested kv_cache pool keeps its own tag."""
|
||||
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
|
||||
|
||||
allocator = get_mem_allocator_instance()
|
||||
with allocator.use_memory_pool("runtime"):
|
||||
state = torch.arange(1 << 20, device=DEVICE_TYPE)
|
||||
with allocator.use_memory_pool("kv_cache"):
|
||||
kv = torch.zeros(1 << 20, device=DEVICE_TYPE)
|
||||
assert {d.tag for d in allocator.pointer_to_data.values()} == {
|
||||
"runtime",
|
||||
"kv_cache",
|
||||
}
|
||||
|
||||
backend = CuMemBackend()
|
||||
backend.suspend(level=level)
|
||||
assert mapped_usage(allocator) == 0
|
||||
|
||||
backend.resume(tags=["weights"])
|
||||
assert torch.equal(state, torch.arange(1 << 20, device=DEVICE_TYPE))
|
||||
backend.resume(tags=["kv_cache"])
|
||||
kv.fill_(1)
|
||||
assert int(kv.sum()) == kv.numel()
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_level2_discards_ordinary_tensor_with_weights_tag():
|
||||
|
||||
Reference in New Issue
Block a user