[Feature] Offload KV-init runtime state on sleep (#59158)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c15672cfb4
commit
8e58ac22af
@@ -327,6 +327,35 @@ def test_reentered_tag(free_x_early):
|
||||
allocator.release_pools()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_runtime_state_survives_sleep(level):
|
||||
"""Runtime state is offloaded at both levels and restored by the first
|
||||
selective wake; a nested kv_cache pool keeps its own tag."""
|
||||
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
|
||||
|
||||
allocator = get_mem_allocator_instance()
|
||||
with allocator.use_memory_pool("runtime"):
|
||||
state = torch.arange(1 << 20, device=DEVICE_TYPE)
|
||||
with allocator.use_memory_pool("kv_cache"):
|
||||
kv = torch.zeros(1 << 20, device=DEVICE_TYPE)
|
||||
assert {d.tag for d in allocator.pointer_to_data.values()} == {
|
||||
"runtime",
|
||||
"kv_cache",
|
||||
}
|
||||
|
||||
backend = CuMemBackend()
|
||||
backend.suspend(level=level)
|
||||
assert mapped_usage(allocator) == 0
|
||||
|
||||
backend.resume(tags=["weights"])
|
||||
assert torch.equal(state, torch.arange(1 << 20, device=DEVICE_TYPE))
|
||||
backend.resume(tags=["kv_cache"])
|
||||
kv.fill_(1)
|
||||
assert int(kv.sum()) == kv.numel()
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_level2_discards_ordinary_tensor_with_weights_tag():
|
||||
|
||||
@@ -129,7 +129,9 @@ class CuMemBackend(SleepModeBackend):
|
||||
|
||||
self._state = "SUSPENDED"
|
||||
allocator = get_mem_allocator_instance()
|
||||
allocator.sleep(offload_tags=("weights",) if level == 1 else tuple())
|
||||
# Runtime state is kept at every level; weights only at level 1.
|
||||
offload_tags = ("weights", "runtime") if level == 1 else ("runtime",)
|
||||
allocator.sleep(offload_tags=offload_tags)
|
||||
|
||||
def resume(self, tags: list[str] | None = None) -> None:
|
||||
from vllm.device_allocator import get_mem_allocator_instance
|
||||
|
||||
@@ -6531,6 +6531,9 @@ class GPUModelRunner(
|
||||
if hasattr(self, "kv_cache_config"):
|
||||
delattr(self, "kv_cache_config")
|
||||
self.cache_config.num_gpu_blocks = None
|
||||
# Profiling may have rebuilt the InputBatch outside the worker's
|
||||
# "runtime" pool; the real initialize_kv_cache rebuilds it.
|
||||
self._init_block_sizes = []
|
||||
|
||||
for layer in self.compilation_config.static_forward_context.values():
|
||||
if hasattr(layer, "kv_cache"):
|
||||
|
||||
@@ -841,10 +841,17 @@ class Worker(WorkerBase):
|
||||
if mem_pool_context is None:
|
||||
mem_pool_context = self._maybe_get_memory_pool_context(tag="kv_cache")
|
||||
|
||||
self.model_runner.initialize_kv_cache(
|
||||
kv_cache_config,
|
||||
kv_cache_allocation_context=mem_pool_context,
|
||||
# Offload KV-init state, except on XPU, whose outermost pool would win.
|
||||
runtime_pool = (
|
||||
nullcontext()
|
||||
if current_platform.is_xpu()
|
||||
else self._maybe_get_memory_pool_context(tag="runtime")
|
||||
)
|
||||
with runtime_pool:
|
||||
self.model_runner.initialize_kv_cache(
|
||||
kv_cache_config,
|
||||
kv_cache_allocation_context=mem_pool_context,
|
||||
)
|
||||
|
||||
# Build KV-zero metadata outside the CuMem pool so the bookkeeping
|
||||
# GPU tensors (seg_addrs, block-id buffers) use the standard PyTorch
|
||||
|
||||
Reference in New Issue
Block a user