diff --git a/docs/features/sleep_mode.md b/docs/features/sleep_mode.md index dcb6ad8c94..2616f374ea 100644 --- a/docs/features/sleep_mode.md +++ b/docs/features/sleep_mode.md @@ -74,6 +74,18 @@ llm.wake_up(tags=["weights"]) llm.wake_up(tags=["kv_cache"]) ``` +#### Release only KV cache memory + +`LLM.release_kv_cache_memory()` discards KV cache physical memory while keeping model weights resident. It requires a completed pause and all executor memory to be resident: full sleep, partial wake-up, and repeated release without restoring memory are rejected. Requests retained with `mode="keep"` are recomputed after wake-up. + +Use the default `cumem` backend with managed allocations: `enable_cumem_allocator=True` on CUDA/ROCm (also enabled by `enable_sleep_mode=True`), or `enable_sleep_mode=True` on XPU. Other backends must implement selective discard; CPU and unmanaged allocations are unsupported. + +```python +llm.sleep(level=0, mode="keep") # Wait for the pause to complete. +llm.release_kv_cache_memory() +llm.wake_up(tags=["kv_cache"]) # Reallocate KV cache and resume scheduling. +``` + ### Online Serving To enable sleep mode in a vLLM server you need to initialize it with the flag `VLLM_SERVER_DEV_MODE=1` and pass `--enable-sleep-mode` to the vLLM server. @@ -107,9 +119,18 @@ curl -X POST 'http://localhost:8000/collective_rpc' -H 'Content-Type: applicatio curl -X POST 'http://localhost:8000/wake_up?tags=kv_cache' ``` +To release only KV cache memory, wait for the level 0 pause to complete before calling the release endpoint. The same resident-memory and backend requirements as the Python API apply. + +```bash +curl -X POST 'http://localhost:8000/sleep?level=0&mode=keep' +curl -X POST 'http://localhost:8000/release_kv_cache_memory' +curl -X POST 'http://localhost:8000/wake_up?tags=kv_cache' +``` + #### HTTP endpoints - `POST /sleep?level=1` — Put the model to sleep (`level=1`). +- `POST /release_kv_cache_memory` — Discard KV cache memory after a completed pause, while all executor memory is resident. - `POST /wake_up` — Wake up the model. Supports optional `tags` query parameters for partial wake-up (e.g., `?tags=weights`). - `POST /collective_rpc` — Perform a collective remote procedure call (RPC). - `GET /is_sleeping` — Check if the model is sleeping. diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index 4cd7420e56..aa6aacf0e0 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -911,13 +911,22 @@ impl EngineCoreClient { Ok(()) } - /// Wake the engine from sleep, optionally limiting the wake-up to specific - /// tags. - pub async fn wake_up(&self, tags: Option>) -> Result<()> { - self.call_utility::<(), _>("wake_up", (tags,)).await?; + /// Release KV cache memory while keeping model weights resident. + pub async fn release_kv_cache_memory(&self) -> Result<()> { + self.call_utility::<(), _>("release_kv_cache_memory", ()).await?; Ok(()) } + /// Wake the engine from sleep, optionally limiting the wake-up to specific + /// tags, and return whether every engine is fully awake. + pub async fn wake_up(&self, tags: Option>) -> Result { + Ok(self + .call_utility::("wake_up", (tags,)) + .await? + .into_iter() + .all(|fully_awake| fully_awake)) + } + /// Pause the scheduler so generation can be halted pub async fn pause_scheduler(&self, mode: PauseMode, clear_cache: bool) -> Result<()> { self.call_utility::<(), _>("pause_scheduler", (mode, clear_cache)).await?; diff --git a/rust/src/mock-engine/src/engine.rs b/rust/src/mock-engine/src/engine.rs index 38f9b21be1..4844aba120 100644 --- a/rust/src/mock-engine/src/engine.rs +++ b/rust/src/mock-engine/src/engine.rs @@ -95,12 +95,10 @@ fn utility_response( "get_supported_tasks" => utility_envelope(vec!["generate"]), "is_sleeping" => utility_envelope(false), "reset_prefix_cache" => utility_envelope(true), - "reset_mm_cache" - | "reset_encoder_cache" - | "profile" - | "sleep" - | "wake_up" - | "execute_dummy_batch" => utility_envelope(()), + "wake_up" => utility_envelope(true), + "reset_mm_cache" | "reset_encoder_cache" | "profile" | "sleep" | "execute_dummy_batch" => { + utility_envelope(()) + } _ => utility_envelope(Value::Nil), }?; diff --git a/rust/src/mock-engine/src/tests.rs b/rust/src/mock-engine/src/tests.rs index bee710fd0c..c1cfbb49cb 100644 --- a/rust/src/mock-engine/src/tests.rs +++ b/rust/src/mock-engine/src/tests.rs @@ -195,6 +195,7 @@ async fn utility_requests_return_minimal_success_responses() { assert!(client.reset_prefix_cache(false, false).await.expect("reset prefix cache")); client.reset_mm_cache().await.expect("reset mm cache"); client.reset_encoder_cache().await.expect("reset encoder cache"); + assert!(client.wake_up(Some(vec!["kv_cache".to_string()])).await.expect("wake up")); shutdown_mock(client, shutdown, task).await; } diff --git a/rust/src/server/src/middleware/metrics.rs b/rust/src/server/src/middleware/metrics.rs index 1e6a79a556..de07620cf7 100644 --- a/rust/src/server/src/middleware/metrics.rs +++ b/rust/src/server/src/middleware/metrics.rs @@ -26,6 +26,7 @@ const EXCLUDED_HANDLERS: &[&str] = &[ "/reset_encoder_cache", "/collective_rpc", "/sleep", + "/release_kv_cache_memory", "/wake_up", "/is_sleeping", ]; diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index 41e4987882..1cd6af3d4f 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -142,6 +142,10 @@ fn build_router_with_options( .route("/weight_info", get(weight_transfer::weight_info)) .route("/abort_requests", post(abort_requests::abort_requests)) .route("/sleep", post(sleep::sleep)) + .route( + "/release_kv_cache_memory", + post(sleep::release_kv_cache_memory), + ) .route("/wake_up", post(sleep::wake_up)) .route("/is_sleeping", get(sleep::is_sleeping)) .route("/pause", post(pause::pause)) diff --git a/rust/src/server/src/routes/sleep.rs b/rust/src/server/src/routes/sleep.rs index 4bf9981402..d64a0850b6 100644 --- a/rust/src/server/src/routes/sleep.rs +++ b/rust/src/server/src/routes/sleep.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use axum::Json; use axum::extract::rejection::QueryRejection; -use axum::extract::{Query, State}; +use axum::extract::{Query, RawQuery, State}; use axum::http::StatusCode; use serde::{Deserialize, Serialize}; use vllm_engine_core_client::protocol::utility::PauseMode; @@ -27,12 +27,6 @@ pub(crate) struct SleepParams { mode: PauseMode, } -#[derive(Debug, Default, Deserialize)] -pub(crate) struct WakeUpParams { - #[serde(default)] - tags: Option>, -} - const fn default_sleep_level() -> u32 { 1 } @@ -57,14 +51,35 @@ pub async fn sleep( Ok(StatusCode::OK) } -/// Wake the engine from sleep mode. -pub async fn wake_up( +/// Release KV cache memory while keeping model weights resident. +pub async fn release_kv_cache_memory( State(state): State>, - Query(params): Query, ) -> Result { state .engine_core_client() - .wake_up(params.tags) + .release_kv_cache_memory() + .await + .map_err(|error| utility_call_error("release_kv_cache_memory", error))?; + + Ok(StatusCode::OK) +} + +/// Wake the engine from sleep mode. +pub async fn wake_up( + State(state): State>, + RawQuery(query): RawQuery, +) -> Result { + let tags = query.and_then(|query| { + let tags = url::form_urlencoded::parse(query.as_bytes()) + .filter(|(key, _)| key == "tags") + .map(|(_, value)| value.into_owned()) + .collect::>(); + (!tags.is_empty()).then_some(tags) + }); + + state + .engine_core_client() + .wake_up(tags) .await .map_err(|error| utility_call_error("wake_up", error))?; diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 4731958e2f..c27ac20fad 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -5805,7 +5805,46 @@ async fn sleep_route_uses_python_compatible_default_query_values() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] -async fn wake_up_route_without_tags_sends_none() { +async fn release_kv_cache_memory_route_sends_expected_utility_call() { + let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let array = payload.as_array().expect("utility payload array"); + let call_id = array[1].as_u64().expect("call id"); + + assert_eq!(array[2], Value::from("release_kv_cache_memory")); + assert_eq!(array[3], Value::Array(Vec::new())); + + send_outputs(push, utility_outputs(call_id, utility_none_result())).await; + }) + }) + .await; + + let response = app + .clone() + .call( + Request::builder() + .method("POST") + .uri("/release_kv_cache_memory") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + let status = response.status(); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); + assert!(body.is_empty()); + engine_task.await.expect("mock engine task"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn wake_up_route_without_tags_accepts_bool_result() { let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { boxed_test_future(async move { let utility = recv_engine_message(dealer).await; @@ -5818,7 +5857,7 @@ async fn wake_up_route_without_tags_sends_none() { assert_eq!(array[2], Value::from("wake_up")); assert_eq!(array[3], Value::Array(vec![Value::Nil])); - send_outputs(push, utility_outputs(call_id, utility_none_result())).await; + send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await; }) }) .await; @@ -5842,6 +5881,51 @@ async fn wake_up_route_without_tags_sends_none() { engine_task.await.expect("mock engine task"); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn wake_up_route_accepts_repeated_tags() { + let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let array = payload.as_array().expect("utility payload array"); + let call_id = array[1].as_u64().expect("call id"); + + assert_eq!(array[2], Value::from("wake_up")); + assert_eq!( + array[3], + Value::Array(vec![Value::Array(vec![ + Value::from("weights"), + Value::from("kv_cache"), + ])]) + ); + + send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await; + }) + }) + .await; + + let response = app + .clone() + .call( + Request::builder() + .method("POST") + .uri("/wake_up?tags=weights&tags%5B%5D=ignored&tags=kv_cache") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + let status = response.status(); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); + assert!(body.is_empty()); + engine_task.await.expect("mock engine task"); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn is_sleeping_route_returns_json_payload() { @@ -6160,6 +6244,7 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() { for (method, uri) in [ ("GET", "/is_sleeping"), ("POST", "/sleep"), + ("POST", "/release_kv_cache_memory"), ("POST", "/wake_up"), ("GET", "/is_paused"), ("POST", "/pause"), diff --git a/tests/basic_correctness/test_mem.py b/tests/basic_correctness/test_mem.py index 6b4895c643..746838dc40 100644 --- a/tests/basic_correctness/test_mem.py +++ b/tests/basic_correctness/test_mem.py @@ -109,6 +109,91 @@ def test_basic_cumem(): assert torch.allclose(output, torch.ones_like(output) * 3) +@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"]) +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch): + """Rejected release and KV-only/full sleep cycles preserve greedy tokens.""" + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + kv_cache_memory_bytes = 256 * 1024 * 1024 + llm = LLM( + "Qwen/Qwen3-0.6B", + enable_sleep_mode=True, + enforce_eager=True, + kv_cache_memory_bytes=kv_cache_memory_bytes, + max_model_len=1024, + max_num_seqs=4, + gpu_memory_utilization=0.05, + ) + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + + def get_mapped_bytes(worker): + return mapped_usage(get_mem_allocator_instance()) + + mapped_before = llm.collective_rpc(get_mapped_bytes)[0] + # Utility RPC transports the engine error as a plain Exception. + with pytest.raises(Exception, match="requires a completed pause"): + llm.release_kv_cache_memory() + assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before + assert not llm.llm_engine.is_sleeping() + assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected + + llm.sleep(level=0) + llm.release_kv_cache_memory() + mapped_after = llm.collective_rpc(get_mapped_bytes)[0] + assert mapped_before - mapped_after >= kv_cache_memory_bytes * 0.99 + assert mapped_after > 0 + assert llm.llm_engine.is_sleeping() + + if full_sleep: + llm.sleep(level=full_sleep) + assert llm.collective_rpc(get_mapped_bytes)[0] == 0 + llm.wake_up(tags=None if full_sleep else ["kv_cache"]) + if full_sleep == 2: + llm.collective_rpc("reload_weights") + assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before + assert not llm.llm_engine.is_sleeping() + actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + assert actual == expected + + +@pytest.mark.parametrize("first_level", [1, 2]) +@pytest.mark.parametrize("second_level", [1, 2]) +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch): + """Repeated sleep after KV-only wake preserves mappings and recoverability.""" + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + llm = LLM( + "Qwen/Qwen3-0.6B", + enable_sleep_mode=True, + enforce_eager=True, + kv_cache_memory_bytes=256 * 1024 * 1024, + max_model_len=1024, + max_num_seqs=4, + gpu_memory_utilization=0.05, + ) + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + + def get_mapped_bytes(worker): + return mapped_usage(get_mem_allocator_instance()) + + llm.sleep(level=first_level) + llm.wake_up(tags=["kv_cache"]) + mapped_before = llm.collective_rpc(get_mapped_bytes)[0] + assert mapped_before > 0 + llm.sleep(level=second_level) + assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before + llm.wake_up(tags=["weights"]) + if first_level == 2: + llm.collective_rpc("reload_weights") + assert not llm.llm_engine.is_sleeping() + actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + assert actual == expected + + @create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") def test_discard_tags(): """Test that discard(tags) selectively frees GPU memory for specific diff --git a/tests/entrypoints/serve/dev/test_sleep.py b/tests/entrypoints/serve/dev/test_sleep.py index 5ebee50600..34e69c9ece 100644 --- a/tests/entrypoints/serve/dev/test_sleep.py +++ b/tests/entrypoints/serve/dev/test_sleep.py @@ -1,14 +1,37 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from unittest.mock import AsyncMock + +import pytest import requests +from fastapi import FastAPI +from fastapi.testclient import TestClient from prometheus_client.parser import text_string_to_metric_families from tests.utils import RemoteOpenAIServer +from vllm.entrypoints.serve.dev.sleep.api_router import attach_router MODEL_NAME = "meta-llama/Llama-3.2-1B" +@pytest.mark.cpu_test +@pytest.mark.parametrize("fails", [False, True]) +def test_release_kv_cache_memory_route(fails): + app = FastAPI() + app.state.engine_client = AsyncMock() + release = app.state.engine_client.release_kv_cache_memory + if fails: + release.side_effect = RuntimeError("requires a completed pause first") + attach_router(app) + + with TestClient(app, raise_server_exceptions=False) as client: + response = client.post("/release_kv_cache_memory") + + assert response.status_code == (500 if fails else 200) + release.assert_awaited_once_with() + + def test_sleep_mode(): # dtype, max-len etc set so that this can run in CI args = [ @@ -63,7 +86,7 @@ def test_sleep_mode(): ) assert response.status_code == 200 - # is sleeping should be false after waking up any part of the engine + # Partial wake keeps the engine sleeping. response = requests.get(remote_server.url_for("is_sleeping")) assert response.status_code == 200 assert response.json().get("is_sleeping") is True diff --git a/tests/multimodal/test_cache.py b/tests/multimodal/test_cache.py index f77dacd341..0c68d2bac0 100644 --- a/tests/multimodal/test_cache.py +++ b/tests/multimodal/test_cache.py @@ -1,7 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import multiprocessing as mp +from concurrent.futures import ThreadPoolExecutor +from functools import partial from types import SimpleNamespace +from unittest.mock import Mock import numpy as np import pytest @@ -34,6 +37,8 @@ from vllm.multimodal.inputs import ( PlaceholderRange, ) from vllm.multimodal.processing import PromptInsertion +from vllm.renderers.base import BaseRenderer +from vllm.utils.async_utils import make_async from vllm.utils.mem_constants import GiB_bytes, MiB_bytes from ..utils import create_new_process_for_each_test @@ -755,6 +760,76 @@ def test_processor_cache_shared_across_loras(): assert feature_lora_b.data == item_data +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize( + "release_error", + [ + None, + "requires a completed pause first", + "requires all executor memory to be resident", + ], + ids=["released", "not-paused", "nonresident-memory"], +) +@pytest.mark.asyncio +async def test_release_kv_cache_resends_mm_payload(use_async, release_error): + """Release must not leave a sender hit pointing at a cleared receiver.""" + from vllm.v1.engine.async_llm import AsyncLLM + from vllm.v1.engine.llm_engine import LLMEngine + + model_config = SimpleNamespace( + get_multimodal_config=lambda: MultiModalConfig(mm_processor_cache_gb=1) + ) + sender = MultiModalProcessorSenderCache(model_config) + receiver = MultiModalReceiverCache(model_config) + item = _dummy_item({"pixel_values": 16}) + mm_hash = "image_A" + payload, _ = sender.get_and_update_item((item, []), mm_hash) + assert receiver.get_and_update_item(payload, mm_hash) is item + assert sender.get_and_update_item(None, mm_hash)[0] is None + + renderer = SimpleNamespace(mm_processor_cache=sender, _mm_cache_stats=None) + renderer.clear_mm_cache = partial(BaseRenderer.clear_mm_cache, renderer) + + def release(): + if release_error: + raise RuntimeError(release_error) + receiver.clear_cache() + + with ThreadPoolExecutor(max_workers=1) as executor: + renderer._clear_mm_cache_async = make_async( + renderer.clear_mm_cache, executor=executor + ) + renderer.clear_mm_cache_async = partial( + BaseRenderer.clear_mm_cache_async, renderer + ) + engine = SimpleNamespace( + renderer=renderer, + engine_core=SimpleNamespace( + release_kv_cache_memory=release, + release_kv_cache_memory_async=make_async(release, executor=executor), + ), + logger_manager=Mock(), + ) + + async def call_release(): + if use_async: + await AsyncLLM.release_kv_cache_memory(engine) + else: + LLMEngine.release_kv_cache_memory(engine) + + if release_error: + with pytest.raises(RuntimeError, match=release_error): + await call_release() + engine.logger_manager.record_sleep_state.assert_not_called() + else: + await call_release() + engine.logger_manager.record_sleep_state.assert_called_once_with(1, 0) + + payload, _ = sender.get_and_update_item((item, []), mm_hash) + assert payload is item + assert receiver.get_and_update_item(payload, mm_hash) is item + + _SLEEP_VISION_PROMPT = ( "<|im_start|>system\nYou are a helpful assistant.<|im_end|>" "\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>" diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 298c8c6185..4951014d1b 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -36,10 +36,13 @@ from vllm.utils.hashing import sha256 from vllm.v1.core.encoder_cache_manager import EncoderCacheManager from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash +from vllm.v1.core.sched.interface import PauseState from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput from vllm.v1.core.sched.scheduler import Scheduler from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs from vllm.v1.engine import FinishReason +from vllm.v1.engine.core import EngineCore +from vllm.v1.executor.uniproc_executor import UniProcExecutor from vllm.v1.hisparse.coordinator import get_hisparse_coordinator from vllm.v1.kv_cache_interface import ( FullAttentionSpec, @@ -1487,6 +1490,79 @@ def test_scheduler_reset_prefix_cache(): assert scheduler.waiting[i] == request +def test_kv_cache_release_after_keep_pause_preserves_requests(): + """A completed keep pause permits release and recomputation of live requests.""" + scheduler = create_scheduler(max_num_seqs=1, enable_prefix_caching=True) + running, waiting = create_requests(num_requests=2) + for request in (running, waiting): + scheduler.add_request(request) + scheduler_output = scheduler.schedule() + scheduler.update_from_output( + scheduler_output, + ModelRunnerOutput( + req_ids=[running.request_id], + req_id_to_index={running.request_id: 0}, + sampled_token_ids=[[1]], + ), + ) + assert scheduler.kv_cache_manager.get_block_ids(running.request_id)[0] + + core = object.__new__(EngineCore) + core.scheduler = scheduler + core.model_executor = Mock(is_sleeping=False) + core.mm_receiver_cache = None + core.batch_queue = None + + assert core.pause_scheduler(mode="keep", clear_cache=False) is None + assert scheduler.running == [running] + assert list(scheduler.waiting) == [waiting] + assert not scheduler.has_requests() + + core.release_kv_cache_memory() + + core.model_executor.discard.assert_called_once_with(("kv_cache",)) + assert scheduler.requests == { + request.request_id: request for request in (running, waiting) + } + assert not scheduler.running + assert list(scheduler.waiting) == [running, waiting] + assert running.status == RequestStatus.PREEMPTED + assert running.num_computed_tokens == 0 + assert list(running.output_token_ids) == [1] + assert not scheduler.kv_cache_manager.get_block_ids(running.request_id)[0] + assert scheduler.schedule().total_num_scheduled_tokens == 0 + + assert core.wake_up(tags=["kv_cache"]) + resumed = scheduler.schedule() + assert resumed.num_scheduled_tokens == {running.request_id: running.num_tokens} + + +@pytest.mark.parametrize( + "sleeping_tags", + [{"weights", "kv_cache"}, {"weights"}, {"kv_cache"}], + ids=["full-sleep", "weights-only", "kv-only-or-repeated-release"], +) +def test_kv_cache_release_rejects_nonresident_memory(sleeping_tags): + """Reject release without resetting caches or discarding more memory.""" + core = object.__new__(EngineCore) + core.scheduler = create_scheduler() + core.scheduler.set_pause_state(PauseState.PAUSED_ALL) + core.batch_queue = None + core._reset_caches = Mock() + core.model_executor = object.__new__(UniProcExecutor) + core.model_executor.sleeping_tags = sleeping_tags.copy() + core.model_executor.collective_rpc = Mock() + + with pytest.raises( + RuntimeError, match="requires all executor memory to be resident" + ): + core.release_kv_cache_memory() + + core._reset_caches.assert_not_called() + core.model_executor.collective_rpc.assert_not_called() + assert core.model_executor.sleeping_tags == sleeping_tags + + def test_reset_connector_cache_no_connector_is_no_op_success(): """``reset_connector_cache`` must return True when no connector is configured. diff --git a/tests/v1/engine/test_engine_core.py b/tests/v1/engine/test_engine_core.py index ba1e9a4d20..e16e0550c2 100644 --- a/tests/v1/engine/test_engine_core.py +++ b/tests/v1/engine/test_engine_core.py @@ -23,6 +23,7 @@ from vllm.config import ( from vllm.engine.arg_utils import EngineArgs from vllm.platforms import current_platform from vllm.utils.torch_utils import set_default_torch_num_threads +from vllm.v1.core.sched.interface import PauseState from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc from vllm.v1.executor.abstract import Executor @@ -667,6 +668,29 @@ def _pausable_engine_core_proc() -> EngineCoreProc: return core +@pytest.mark.parametrize( + "pause_state,has_requests,has_batches", + [ + pytest.param(PauseState.UNPAUSED, False, False, id="not-paused"), + pytest.param(PauseState.PAUSED_ALL, True, False, id="pending-requests"), + pytest.param(PauseState.PAUSED_ALL, False, True, id="pending-batches"), + ], +) +def test_kv_cache_release_rejects_unsafe_state(pause_state, has_requests, has_batches): + """Reject release before touching caches or memory if work can still use KV.""" + core = _pausable_engine_core_proc() + core.scheduler.pause_state = pause_state + core.scheduler.has_requests.return_value = has_requests + core.batch_queue = [object()] if has_batches else None + core._reset_caches = MagicMock() + + with pytest.raises(RuntimeError, match="requires a completed pause"): + core.release_kv_cache_memory() + + core._reset_caches.assert_not_called() + core.model_executor.discard.assert_not_called() + + @pytest.mark.parametrize("deferred", [False, True]) def test_pause_synchronizes_device_before_cache_reset(deferred: bool): """A resolved pause promises an idle device: the barrier must run before diff --git a/vllm/device_allocator/sleep_mode_backend.py b/vllm/device_allocator/sleep_mode_backend.py index 45a635d613..023af04275 100644 --- a/vllm/device_allocator/sleep_mode_backend.py +++ b/vllm/device_allocator/sleep_mode_backend.py @@ -73,6 +73,13 @@ class SleepModeBackend(ABC): (suspended) engine from a healthy-serving one (see RFC #34303).""" return self._state + def discard(self, tags: tuple[str, ...]) -> None: + """Discard selected tagged allocations without preserving contents.""" + raise RuntimeError( + f"Sleep-mode backend {type(self).__name__!r} does not support " + "selective discard." + ) + # -- Capability introspection (no instance required) -- @classmethod @@ -132,6 +139,13 @@ class CuMemBackend(SleepModeBackend): allocator.wake_up(tags) self._state = "RUNNING" + def discard(self, tags: tuple[str, ...]) -> None: + from vllm.device_allocator import get_mem_allocator_instance + + allocator = get_mem_allocator_instance() + allocator.discard(tags) + self._state = "SUSPENDED" + @classmethod def preserves_communicators(cls) -> bool: # Comm identity/topology survive memory suspension; no reinit needed. diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index c44fd44d0f..602e2570a2 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -186,6 +186,11 @@ class EngineClient(ABC): """Sleep the engine.""" ... + @abstractmethod + async def release_kv_cache_memory(self) -> None: + """Discard KV cache physical GPU memory. Requires a completed pause.""" + ... + @abstractmethod async def wake_up(self, tags: list[str] | None = None) -> None: """Wake up the engine.""" diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index d9a2a46701..999310aa61 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -830,6 +830,14 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): """ self.llm_engine.sleep(level=level, mode=mode) + def release_kv_cache_memory(self) -> None: + """Release the GPU physical memory backing the KV cache. + + Requires a completed ``sleep(level=0)`` and resident executor memory. + Restore with ``wake_up(tags=["kv_cache"])``; kept requests are recomputed. + """ + self.llm_engine.release_kv_cache_memory() + def wake_up(self, tags: list[str] | None = None): """Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep] method for more details. diff --git a/vllm/entrypoints/serve/dev/sleep/api_router.py b/vllm/entrypoints/serve/dev/sleep/api_router.py index fedff8da74..846a487706 100644 --- a/vllm/entrypoints/serve/dev/sleep/api_router.py +++ b/vllm/entrypoints/serve/dev/sleep/api_router.py @@ -27,6 +27,12 @@ async def sleep(raw_request: Request): return Response(status_code=200) +@router.post("/release_kv_cache_memory") +async def release_kv_cache_memory(raw_request: Request): + await engine_client(raw_request).release_kv_cache_memory() + return Response(status_code=200) + + @router.post("/wake_up") async def wake_up(raw_request: Request): tags = raw_request.query_params.getlist("tags") diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 3e399a1a81..ad9d9bf6c7 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -1079,10 +1079,17 @@ class AsyncLLM(EngineClient): if self.logger_manager is not None: self.logger_manager.record_sleep_state(1, level) - async def wake_up(self, tags: list[str] | None = None) -> None: - await self.engine_core.wake_up_async(tags) + async def release_kv_cache_memory(self) -> None: + await self.renderer.clear_mm_cache_async() + await self.engine_core.release_kv_cache_memory_async() if self.logger_manager is not None: + self.logger_manager.record_sleep_state(1, 0) + + async def wake_up(self, tags: list[str] | None = None) -> None: + fully_awake = await self.engine_core.wake_up_async(tags) + + if self.logger_manager is not None and fully_awake: self.logger_manager.record_sleep_state(0, 0) async def checkpoint_prepare(self) -> None: diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 14224f95e0..5ee2c6f8c0 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -957,12 +957,15 @@ class EngineCore: pause_future.add_done_callback(pause_complete) return future - def wake_up(self, tags: list[str] | None = None): + def wake_up(self, tags: list[str] | None = None) -> bool: """Wake up the engine from sleep. Args: tags: Tags to wake up. Use ["scheduling"] for level 0 wake up. + Returns: + Whether all executor memory is resident again (fully awake). + """ if tags is not None and "scheduling" in tags: # Remove "scheduling" from tags if there are other tags to process. @@ -973,8 +976,30 @@ class EngineCore: # Partial wakes intentionally keep the remaining allocations asleep. # Resume scheduling only once all executor memory is resident again. - if not self.model_executor.is_sleeping: + fully_awake = not self.model_executor.is_sleeping + if fully_awake: self.resume_scheduler() + return fully_awake + + def release_kv_cache_memory(self) -> None: + """Discard KV cache physical memory. Requires a completed pause + and all executor memory to be resident. Kept requests are recomputed + after wake-up. + """ + if not ( + self.is_scheduler_paused() + and not self.scheduler.has_requests() + and not self.batch_queue + ): + raise RuntimeError( + "release_kv_cache_memory() requires a completed pause first" + ) + if self.model_executor.is_sleeping: + raise RuntimeError( + "release_kv_cache_memory() requires all executor memory to be resident" + ) + self._reset_caches() + self.model_executor.discard(("kv_cache",)) def is_sleeping(self) -> bool: """Check if engine is sleeping at any level.""" diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index 4ca565d56c..399af7c0b9 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -193,7 +193,10 @@ class EngineCoreClient(ABC): def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None: raise NotImplementedError - def wake_up(self, tags: list[str] | None = None) -> None: + def release_kv_cache_memory(self) -> None: + raise NotImplementedError + + def wake_up(self, tags: list[str] | None = None) -> bool: raise NotImplementedError def is_sleeping(self) -> bool: @@ -285,7 +288,10 @@ class EngineCoreClient(ABC): async def sleep_async(self, level: int = 1, mode: PauseMode = "abort") -> None: raise NotImplementedError - async def wake_up_async(self, tags: list[str] | None = None) -> None: + async def release_kv_cache_memory_async(self) -> None: + raise NotImplementedError + + async def wake_up_async(self, tags: list[str] | None = None) -> bool: raise NotImplementedError async def is_sleeping_async(self) -> bool: @@ -397,8 +403,11 @@ class InprocClient(EngineCoreClient): result = self.engine_core.sleep(level, mode) assert result is None - def wake_up(self, tags: list[str] | None = None) -> None: - self.engine_core.wake_up(tags) + def release_kv_cache_memory(self) -> None: + self.engine_core.release_kv_cache_memory() + + def wake_up(self, tags: list[str] | None = None) -> bool: + return self.engine_core.wake_up(tags) def is_sleeping(self) -> bool: return self.engine_core.is_sleeping() @@ -1021,8 +1030,11 @@ class SyncMPClient(MPClient): def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None: self.call_utility("sleep", level, mode) - def wake_up(self, tags: list[str] | None = None) -> None: - self.call_utility("wake_up", tags) + def release_kv_cache_memory(self) -> None: + self.call_utility("release_kv_cache_memory") + + def wake_up(self, tags: list[str] | None = None) -> bool: + return self.call_utility("wake_up", tags) def is_sleeping(self) -> bool: return self.call_utility("is_sleeping") @@ -1265,8 +1277,11 @@ class AsyncMPClient(MPClient): async def sleep_async(self, level: int = 1, mode: PauseMode = "abort") -> None: await self.call_utility_async("sleep", level, mode) - async def wake_up_async(self, tags: list[str] | None = None) -> None: - await self.call_utility_async("wake_up", tags) + async def release_kv_cache_memory_async(self) -> None: + await self.call_utility_async("release_kv_cache_memory") + + async def wake_up_async(self, tags: list[str] | None = None) -> bool: + return await self.call_utility_async("wake_up", tags) async def is_sleeping_async(self) -> bool: return await self.call_utility_async("is_sleeping") diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index c6c77f72e7..a2d97b5544 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -375,10 +375,17 @@ class LLMEngine: if self.logger_manager is not None: self.logger_manager.record_sleep_state(1, level) - def wake_up(self, tags: list[str] | None = None): - self.engine_core.wake_up(tags) + def release_kv_cache_memory(self) -> None: + self.renderer.clear_mm_cache() + self.engine_core.release_kv_cache_memory() if self.logger_manager is not None: + self.logger_manager.record_sleep_state(1, 0) + + def wake_up(self, tags: list[str] | None = None): + fully_awake = self.engine_core.wake_up(tags) + + if self.logger_manager is not None and fully_awake: self.logger_manager.record_sleep_state(0, 0) def is_sleeping(self) -> bool: diff --git a/vllm/v1/executor/abstract.py b/vllm/v1/executor/abstract.py index 6b07bad17c..f234e943b4 100644 --- a/vllm/v1/executor/abstract.py +++ b/vllm/v1/executor/abstract.py @@ -32,6 +32,9 @@ logger = init_logger(__name__) _R = TypeVar("_R") +# Memory-pool tags managed by sleep/wake_up/discard. +SLEEP_TAGS = frozenset(("weights", "kv_cache")) + FailureCallback = Callable[[], None] @@ -108,7 +111,6 @@ class Executor(ABC): self.speculative_config = vllm_config.speculative_config self.observability_config = vllm_config.observability_config self._init_executor() - self.is_sleeping = False self.sleeping_tags: set[str] = set() self.kv_output_aggregator: KVOutputAggregator | None = None self.ec_output_aggregator: ECOutputAggregator | None = None @@ -338,15 +340,18 @@ class Executor(ABC): """Reset the encoder cache in each worker to clear cached encoder outputs.""" self.collective_rpc("reset_encoder_cache") + @property + def is_sleeping(self) -> bool: + return bool(self.sleeping_tags) + def sleep(self, level: int = 1): - if self.is_sleeping: + if "weights" in self.sleeping_tags: logger.warning("Executor is already sleeping.") return time_before_sleep = time.perf_counter() self.collective_rpc("sleep", kwargs=dict(level=level)) time_after_sleep = time.perf_counter() - self.sleeping_tags = {"weights", "kv_cache"} - self.is_sleeping = True + self.sleeping_tags |= SLEEP_TAGS logger.info( "It took %.6f seconds to fall asleep.", time_after_sleep - time_before_sleep ) @@ -375,8 +380,23 @@ class Executor(ABC): self.sleeping_tags.remove(tag) else: self.sleeping_tags.clear() - if not self.sleeping_tags: - self.is_sleeping = False + + def discard(self, tags: tuple[str, ...]) -> None: + tags_to_discard = set(tags) - self.sleeping_tags + if not tags_to_discard: + logger.warning("Tags %s are already sleeping.", tags) + return + time_before_discard = time.perf_counter() + try: + self.collective_rpc("discard", args=(tuple(tags_to_discard),)) + finally: + self.sleeping_tags |= tags_to_discard + time_after_discard = time.perf_counter() + logger.info( + "It took %.6f seconds to discard tags %s.", + time_after_discard - time_before_discard, + tags_to_discard, + ) def reinitialize_distributed( self, reconfig_request: ReconfigureDistributedRequest diff --git a/vllm/v1/worker/cpu_worker.py b/vllm/v1/worker/cpu_worker.py index 2ee2164dc3..1badcfe6e8 100644 --- a/vllm/v1/worker/cpu_worker.py +++ b/vllm/v1/worker/cpu_worker.py @@ -198,6 +198,9 @@ class CPUWorker(Worker): logger.warning("sleep mode is not supported on CPU, ignore it.") pass + def discard(self, tags: tuple[str, ...]) -> None: + raise RuntimeError("Selective discard is not supported on CPU.") + def _should_warm_up_model(self) -> bool: # VLLM_CPU_CI_ENV always skips warmup to save CI time. if envs.VLLM_CPU_CI_ENV: diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 02f599e57e..8ced3bdd43 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -304,6 +304,9 @@ class Worker(WorkerBase): self.synchronize_device() + def discard(self, tags: tuple[str, ...]) -> None: + self.sleep_mode_backend.discard(tags) + def checkpoint_prepare(self) -> None: checkpoint_prepare_distributed_state()