[Feature] Offload KV-init runtime state on sleep (#59158)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-02 09:49:21 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent c15672cfb4
commit 8e58ac22af
4 changed files with 45 additions and 4 deletions
@@ -327,6 +327,35 @@ def test_reentered_tag(free_x_early):
allocator.release_pools()
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_runtime_state_survives_sleep(level):
"""Runtime state is offloaded at both levels and restored by the first
selective wake; a nested kv_cache pool keeps its own tag."""
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool("runtime"):
state = torch.arange(1 << 20, device=DEVICE_TYPE)
with allocator.use_memory_pool("kv_cache"):
kv = torch.zeros(1 << 20, device=DEVICE_TYPE)
assert {d.tag for d in allocator.pointer_to_data.values()} == {
"runtime",
"kv_cache",
}
backend = CuMemBackend()
backend.suspend(level=level)
assert mapped_usage(allocator) == 0
backend.resume(tags=["weights"])
assert torch.equal(state, torch.arange(1 << 20, device=DEVICE_TYPE))
backend.resume(tags=["kv_cache"])
kv.fill_(1)
assert int(kv.sum()) == kv.numel()
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_level2_discards_ordinary_tensor_with_weights_tag():
+3 -1
View File
@@ -129,7 +129,9 @@ class CuMemBackend(SleepModeBackend):
self._state = "SUSPENDED"
allocator = get_mem_allocator_instance()
allocator.sleep(offload_tags=("weights",) if level == 1 else tuple())
# Runtime state is kept at every level; weights only at level 1.
offload_tags = ("weights", "runtime") if level == 1 else ("runtime",)
allocator.sleep(offload_tags=offload_tags)
def resume(self, tags: list[str] | None = None) -> None:
from vllm.device_allocator import get_mem_allocator_instance
+3
View File
@@ -6531,6 +6531,9 @@ class GPUModelRunner(
if hasattr(self, "kv_cache_config"):
delattr(self, "kv_cache_config")
self.cache_config.num_gpu_blocks = None
# Profiling may have rebuilt the InputBatch outside the worker's
# "runtime" pool; the real initialize_kv_cache rebuilds it.
self._init_block_sizes = []
for layer in self.compilation_config.static_forward_context.values():
if hasattr(layer, "kv_cache"):
+10 -3
View File
@@ -841,10 +841,17 @@ class Worker(WorkerBase):
if mem_pool_context is None:
mem_pool_context = self._maybe_get_memory_pool_context(tag="kv_cache")
self.model_runner.initialize_kv_cache(
kv_cache_config,
kv_cache_allocation_context=mem_pool_context,
# Offload KV-init state, except on XPU, whose outermost pool would win.
runtime_pool = (
nullcontext()
if current_platform.is_xpu()
else self._maybe_get_memory_pool_context(tag="runtime")
)
with runtime_pool:
self.model_runner.initialize_kv_cache(
kv_cache_config,
kv_cache_allocation_context=mem_pool_context,
)
# Build KV-zero metadata outside the CuMem pool so the bookkeeping
# GPU tensors (seg_addrs, block-id buffers) use the standard PyTorch