diff --git a/tests/basic_correctness/memory/cumem/test_cumem.py b/tests/basic_correctness/memory/cumem/test_cumem.py index 36a54ae219..fa06facf29 100644 --- a/tests/basic_correctness/memory/cumem/test_cumem.py +++ b/tests/basic_correctness/memory/cumem/test_cumem.py @@ -327,6 +327,35 @@ def test_reentered_tag(free_x_early): allocator.release_pools() +@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"]) +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator") +def test_runtime_state_survives_sleep(level): + """Runtime state is offloaded at both levels and restored by the first + selective wake; a nested kv_cache pool keeps its own tag.""" + from vllm.device_allocator.sleep_mode_backend import CuMemBackend + + allocator = get_mem_allocator_instance() + with allocator.use_memory_pool("runtime"): + state = torch.arange(1 << 20, device=DEVICE_TYPE) + with allocator.use_memory_pool("kv_cache"): + kv = torch.zeros(1 << 20, device=DEVICE_TYPE) + assert {d.tag for d in allocator.pointer_to_data.values()} == { + "runtime", + "kv_cache", + } + + backend = CuMemBackend() + backend.suspend(level=level) + assert mapped_usage(allocator) == 0 + + backend.resume(tags=["weights"]) + assert torch.equal(state, torch.arange(1 << 20, device=DEVICE_TYPE)) + backend.resume(tags=["kv_cache"]) + kv.fill_(1) + assert int(kv.sum()) == kv.numel() + + @create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") @pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator") def test_level2_discards_ordinary_tensor_with_weights_tag(): diff --git a/vllm/device_allocator/sleep_mode_backend.py b/vllm/device_allocator/sleep_mode_backend.py index 023af04275..8ec9986ffc 100644 --- a/vllm/device_allocator/sleep_mode_backend.py +++ b/vllm/device_allocator/sleep_mode_backend.py @@ -129,7 +129,9 @@ class CuMemBackend(SleepModeBackend): self._state = "SUSPENDED" allocator = get_mem_allocator_instance() - allocator.sleep(offload_tags=("weights",) if level == 1 else tuple()) + # Runtime state is kept at every level; weights only at level 1. + offload_tags = ("weights", "runtime") if level == 1 else ("runtime",) + allocator.sleep(offload_tags=offload_tags) def resume(self, tags: list[str] | None = None) -> None: from vllm.device_allocator import get_mem_allocator_instance diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 9a0a4eeda9..9ba7d89140 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -6531,6 +6531,9 @@ class GPUModelRunner( if hasattr(self, "kv_cache_config"): delattr(self, "kv_cache_config") self.cache_config.num_gpu_blocks = None + # Profiling may have rebuilt the InputBatch outside the worker's + # "runtime" pool; the real initialize_kv_cache rebuilds it. + self._init_block_sizes = [] for layer in self.compilation_config.static_forward_context.values(): if hasattr(layer, "kv_cache"): diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 15e2ee0611..55d56919a4 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -841,10 +841,17 @@ class Worker(WorkerBase): if mem_pool_context is None: mem_pool_context = self._maybe_get_memory_pool_context(tag="kv_cache") - self.model_runner.initialize_kv_cache( - kv_cache_config, - kv_cache_allocation_context=mem_pool_context, + # Offload KV-init state, except on XPU, whose outermost pool would win. + runtime_pool = ( + nullcontext() + if current_platform.is_xpu() + else self._maybe_get_memory_pool_context(tag="runtime") ) + with runtime_pool: + self.model_runner.initialize_kv_cache( + kv_cache_config, + kv_cache_allocation_context=mem_pool_context, + ) # Build KV-zero metadata outside the CuMem pool so the bookkeeping # GPU tensors (seg_addrs, block-id buffers) use the standard PyTorch