[Frontend][Core] Add release_kv_cache_memory() API (#44890)

Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: AlanFokCo <alanfok2868@gmail.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com>
Co-authored-by: AlanFokCo <alanfok2868@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Dakai An
2026-09-18 07:12:12 +08:00
committed by GitHub
co-authored by AlanFokCo aoshen02 OpenAI Codex
parent a524f80b1f
commit 092bdd6d57
24 changed files with 574 additions and 44 deletions
+21
View File
@@ -74,6 +74,18 @@ llm.wake_up(tags=["weights"])
llm.wake_up(tags=["kv_cache"])
```
#### Release only KV cache memory
`LLM.release_kv_cache_memory()` discards KV cache physical memory while keeping model weights resident. It requires a completed pause and all executor memory to be resident: full sleep, partial wake-up, and repeated release without restoring memory are rejected. Requests retained with `mode="keep"` are recomputed after wake-up.
Use the default `cumem` backend with managed allocations: `enable_cumem_allocator=True` on CUDA/ROCm (also enabled by `enable_sleep_mode=True`), or `enable_sleep_mode=True` on XPU. Other backends must implement selective discard; CPU and unmanaged allocations are unsupported.
```python
llm.sleep(level=0, mode="keep") # Wait for the pause to complete.
llm.release_kv_cache_memory()
llm.wake_up(tags=["kv_cache"]) # Reallocate KV cache and resume scheduling.
```
### Online Serving
To enable sleep mode in a vLLM server you need to initialize it with the flag `VLLM_SERVER_DEV_MODE=1` and pass `--enable-sleep-mode` to the vLLM server.
@@ -107,9 +119,18 @@ curl -X POST 'http://localhost:8000/collective_rpc' -H 'Content-Type: applicatio
curl -X POST 'http://localhost:8000/wake_up?tags=kv_cache'
```
To release only KV cache memory, wait for the level 0 pause to complete before calling the release endpoint. The same resident-memory and backend requirements as the Python API apply.
```bash
curl -X POST 'http://localhost:8000/sleep?level=0&mode=keep'
curl -X POST 'http://localhost:8000/release_kv_cache_memory'
curl -X POST 'http://localhost:8000/wake_up?tags=kv_cache'
```
#### HTTP endpoints
- `POST /sleep?level=1` — Put the model to sleep (`level=1`).
- `POST /release_kv_cache_memory` — Discard KV cache memory after a completed pause, while all executor memory is resident.
- `POST /wake_up` — Wake up the model. Supports optional `tags` query parameters for partial wake-up (e.g., `?tags=weights`).
- `POST /collective_rpc` — Perform a collective remote procedure call (RPC).
- `GET /is_sleeping` — Check if the model is sleeping.
+13 -4
View File
@@ -911,13 +911,22 @@ impl EngineCoreClient {
Ok(())
}
/// Wake the engine from sleep, optionally limiting the wake-up to specific
/// tags.
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<()> {
self.call_utility::<(), _>("wake_up", (tags,)).await?;
/// Release KV cache memory while keeping model weights resident.
pub async fn release_kv_cache_memory(&self) -> Result<()> {
self.call_utility::<(), _>("release_kv_cache_memory", ()).await?;
Ok(())
}
/// Wake the engine from sleep, optionally limiting the wake-up to specific
/// tags, and return whether every engine is fully awake.
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<bool> {
Ok(self
.call_utility::<bool, _>("wake_up", (tags,))
.await?
.into_iter()
.all(|fully_awake| fully_awake))
}
/// Pause the scheduler so generation can be halted
pub async fn pause_scheduler(&self, mode: PauseMode, clear_cache: bool) -> Result<()> {
self.call_utility::<(), _>("pause_scheduler", (mode, clear_cache)).await?;
+4 -6
View File
@@ -95,12 +95,10 @@ fn utility_response(
"get_supported_tasks" => utility_envelope(vec!["generate"]),
"is_sleeping" => utility_envelope(false),
"reset_prefix_cache" => utility_envelope(true),
"reset_mm_cache"
| "reset_encoder_cache"
| "profile"
| "sleep"
| "wake_up"
| "execute_dummy_batch" => utility_envelope(()),
"wake_up" => utility_envelope(true),
"reset_mm_cache" | "reset_encoder_cache" | "profile" | "sleep" | "execute_dummy_batch" => {
utility_envelope(())
}
_ => utility_envelope(Value::Nil),
}?;
+1
View File
@@ -195,6 +195,7 @@ async fn utility_requests_return_minimal_success_responses() {
assert!(client.reset_prefix_cache(false, false).await.expect("reset prefix cache"));
client.reset_mm_cache().await.expect("reset mm cache");
client.reset_encoder_cache().await.expect("reset encoder cache");
assert!(client.wake_up(Some(vec!["kv_cache".to_string()])).await.expect("wake up"));
shutdown_mock(client, shutdown, task).await;
}
@@ -26,6 +26,7 @@ const EXCLUDED_HANDLERS: &[&str] = &[
"/reset_encoder_cache",
"/collective_rpc",
"/sleep",
"/release_kv_cache_memory",
"/wake_up",
"/is_sleeping",
];
+4
View File
@@ -142,6 +142,10 @@ fn build_router_with_options(
.route("/weight_info", get(weight_transfer::weight_info))
.route("/abort_requests", post(abort_requests::abort_requests))
.route("/sleep", post(sleep::sleep))
.route(
"/release_kv_cache_memory",
post(sleep::release_kv_cache_memory),
)
.route("/wake_up", post(sleep::wake_up))
.route("/is_sleeping", get(sleep::is_sleeping))
.route("/pause", post(pause::pause))
+26 -11
View File
@@ -5,7 +5,7 @@ use std::sync::Arc;
use axum::Json;
use axum::extract::rejection::QueryRejection;
use axum::extract::{Query, State};
use axum::extract::{Query, RawQuery, State};
use axum::http::StatusCode;
use serde::{Deserialize, Serialize};
use vllm_engine_core_client::protocol::utility::PauseMode;
@@ -27,12 +27,6 @@ pub(crate) struct SleepParams {
mode: PauseMode,
}
#[derive(Debug, Default, Deserialize)]
pub(crate) struct WakeUpParams {
#[serde(default)]
tags: Option<Vec<String>>,
}
const fn default_sleep_level() -> u32 {
1
}
@@ -57,14 +51,35 @@ pub async fn sleep(
Ok(StatusCode::OK)
}
/// Wake the engine from sleep mode.
pub async fn wake_up(
/// Release KV cache memory while keeping model weights resident.
pub async fn release_kv_cache_memory(
State(state): State<Arc<AppState>>,
Query(params): Query<WakeUpParams>,
) -> Result<StatusCode, ApiError> {
state
.engine_core_client()
.wake_up(params.tags)
.release_kv_cache_memory()
.await
.map_err(|error| utility_call_error("release_kv_cache_memory", error))?;
Ok(StatusCode::OK)
}
/// Wake the engine from sleep mode.
pub async fn wake_up(
State(state): State<Arc<AppState>>,
RawQuery(query): RawQuery,
) -> Result<StatusCode, ApiError> {
let tags = query.and_then(|query| {
let tags = url::form_urlencoded::parse(query.as_bytes())
.filter(|(key, _)| key == "tags")
.map(|(_, value)| value.into_owned())
.collect::<Vec<_>>();
(!tags.is_empty()).then_some(tags)
});
state
.engine_core_client()
.wake_up(tags)
.await
.map_err(|error| utility_call_error("wake_up", error))?;
+87 -2
View File
@@ -5805,7 +5805,46 @@ async fn sleep_route_uses_python_compatible_default_query_values() {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn wake_up_route_without_tags_sends_none() {
async fn release_kv_cache_memory_route_sends_expected_utility_call() {
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("release_kv_cache_memory"));
assert_eq!(array[3], Value::Array(Vec::new()));
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
})
})
.await;
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/release_kv_cache_memory")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn wake_up_route_without_tags_accepts_bool_result() {
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
@@ -5818,7 +5857,7 @@ async fn wake_up_route_without_tags_sends_none() {
assert_eq!(array[2], Value::from("wake_up"));
assert_eq!(array[3], Value::Array(vec![Value::Nil]));
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
})
})
.await;
@@ -5842,6 +5881,51 @@ async fn wake_up_route_without_tags_sends_none() {
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn wake_up_route_accepts_repeated_tags() {
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("wake_up"));
assert_eq!(
array[3],
Value::Array(vec![Value::Array(vec![
Value::from("weights"),
Value::from("kv_cache"),
])])
);
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
})
})
.await;
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/wake_up?tags=weights&tags%5B%5D=ignored&tags=kv_cache")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn is_sleeping_route_returns_json_payload() {
@@ -6160,6 +6244,7 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() {
for (method, uri) in [
("GET", "/is_sleeping"),
("POST", "/sleep"),
("POST", "/release_kv_cache_memory"),
("POST", "/wake_up"),
("GET", "/is_paused"),
("POST", "/pause"),
+85
View File
@@ -109,6 +109,91 @@ def test_basic_cumem():
assert torch.allclose(output, torch.ones_like(output) * 3)
@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch):
"""Rejected release and KV-only/full sleep cycles preserve greedy tokens."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
kv_cache_memory_bytes = 256 * 1024 * 1024
llm = LLM(
"Qwen/Qwen3-0.6B",
enable_sleep_mode=True,
enforce_eager=True,
kv_cache_memory_bytes=kv_cache_memory_bytes,
max_model_len=1024,
max_num_seqs=4,
gpu_memory_utilization=0.05,
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
def get_mapped_bytes(worker):
return mapped_usage(get_mem_allocator_instance())
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
# Utility RPC transports the engine error as a plain Exception.
with pytest.raises(Exception, match="requires a completed pause"):
llm.release_kv_cache_memory()
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
assert not llm.llm_engine.is_sleeping()
assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected
llm.sleep(level=0)
llm.release_kv_cache_memory()
mapped_after = llm.collective_rpc(get_mapped_bytes)[0]
assert mapped_before - mapped_after >= kv_cache_memory_bytes * 0.99
assert mapped_after > 0
assert llm.llm_engine.is_sleeping()
if full_sleep:
llm.sleep(level=full_sleep)
assert llm.collective_rpc(get_mapped_bytes)[0] == 0
llm.wake_up(tags=None if full_sleep else ["kv_cache"])
if full_sleep == 2:
llm.collective_rpc("reload_weights")
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
assert not llm.llm_engine.is_sleeping()
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
assert actual == expected
@pytest.mark.parametrize("first_level", [1, 2])
@pytest.mark.parametrize("second_level", [1, 2])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch):
"""Repeated sleep after KV-only wake preserves mappings and recoverability."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
llm = LLM(
"Qwen/Qwen3-0.6B",
enable_sleep_mode=True,
enforce_eager=True,
kv_cache_memory_bytes=256 * 1024 * 1024,
max_model_len=1024,
max_num_seqs=4,
gpu_memory_utilization=0.05,
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
def get_mapped_bytes(worker):
return mapped_usage(get_mem_allocator_instance())
llm.sleep(level=first_level)
llm.wake_up(tags=["kv_cache"])
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
assert mapped_before > 0
llm.sleep(level=second_level)
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
llm.wake_up(tags=["weights"])
if first_level == 2:
llm.collective_rpc("reload_weights")
assert not llm.llm_engine.is_sleeping()
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
assert actual == expected
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_discard_tags():
"""Test that discard(tags) selectively frees GPU memory for specific
+24 -1
View File
@@ -1,14 +1,37 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import AsyncMock
import pytest
import requests
from fastapi import FastAPI
from fastapi.testclient import TestClient
from prometheus_client.parser import text_string_to_metric_families
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.serve.dev.sleep.api_router import attach_router
MODEL_NAME = "meta-llama/Llama-3.2-1B"
@pytest.mark.cpu_test
@pytest.mark.parametrize("fails", [False, True])
def test_release_kv_cache_memory_route(fails):
app = FastAPI()
app.state.engine_client = AsyncMock()
release = app.state.engine_client.release_kv_cache_memory
if fails:
release.side_effect = RuntimeError("requires a completed pause first")
attach_router(app)
with TestClient(app, raise_server_exceptions=False) as client:
response = client.post("/release_kv_cache_memory")
assert response.status_code == (500 if fails else 200)
release.assert_awaited_once_with()
def test_sleep_mode():
# dtype, max-len etc set so that this can run in CI
args = [
@@ -63,7 +86,7 @@ def test_sleep_mode():
)
assert response.status_code == 200
# is sleeping should be false after waking up any part of the engine
# Partial wake keeps the engine sleeping.
response = requests.get(remote_server.url_for("is_sleeping"))
assert response.status_code == 200
assert response.json().get("is_sleeping") is True
+75
View File
@@ -1,7 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import multiprocessing as mp
from concurrent.futures import ThreadPoolExecutor
from functools import partial
from types import SimpleNamespace
from unittest.mock import Mock
import numpy as np
import pytest
@@ -34,6 +37,8 @@ from vllm.multimodal.inputs import (
PlaceholderRange,
)
from vllm.multimodal.processing import PromptInsertion
from vllm.renderers.base import BaseRenderer
from vllm.utils.async_utils import make_async
from vllm.utils.mem_constants import GiB_bytes, MiB_bytes
from ..utils import create_new_process_for_each_test
@@ -755,6 +760,76 @@ def test_processor_cache_shared_across_loras():
assert feature_lora_b.data == item_data
@pytest.mark.parametrize("use_async", [False, True])
@pytest.mark.parametrize(
"release_error",
[
None,
"requires a completed pause first",
"requires all executor memory to be resident",
],
ids=["released", "not-paused", "nonresident-memory"],
)
@pytest.mark.asyncio
async def test_release_kv_cache_resends_mm_payload(use_async, release_error):
"""Release must not leave a sender hit pointing at a cleared receiver."""
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.engine.llm_engine import LLMEngine
model_config = SimpleNamespace(
get_multimodal_config=lambda: MultiModalConfig(mm_processor_cache_gb=1)
)
sender = MultiModalProcessorSenderCache(model_config)
receiver = MultiModalReceiverCache(model_config)
item = _dummy_item({"pixel_values": 16})
mm_hash = "image_A"
payload, _ = sender.get_and_update_item((item, []), mm_hash)
assert receiver.get_and_update_item(payload, mm_hash) is item
assert sender.get_and_update_item(None, mm_hash)[0] is None
renderer = SimpleNamespace(mm_processor_cache=sender, _mm_cache_stats=None)
renderer.clear_mm_cache = partial(BaseRenderer.clear_mm_cache, renderer)
def release():
if release_error:
raise RuntimeError(release_error)
receiver.clear_cache()
with ThreadPoolExecutor(max_workers=1) as executor:
renderer._clear_mm_cache_async = make_async(
renderer.clear_mm_cache, executor=executor
)
renderer.clear_mm_cache_async = partial(
BaseRenderer.clear_mm_cache_async, renderer
)
engine = SimpleNamespace(
renderer=renderer,
engine_core=SimpleNamespace(
release_kv_cache_memory=release,
release_kv_cache_memory_async=make_async(release, executor=executor),
),
logger_manager=Mock(),
)
async def call_release():
if use_async:
await AsyncLLM.release_kv_cache_memory(engine)
else:
LLMEngine.release_kv_cache_memory(engine)
if release_error:
with pytest.raises(RuntimeError, match=release_error):
await call_release()
engine.logger_manager.record_sleep_state.assert_not_called()
else:
await call_release()
engine.logger_manager.record_sleep_state.assert_called_once_with(1, 0)
payload, _ = sender.get_and_update_item((item, []), mm_hash)
assert payload is item
assert receiver.get_and_update_item(payload, mm_hash) is item
_SLEEP_VISION_PROMPT = (
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
"\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
+76
View File
@@ -36,10 +36,13 @@ from vllm.utils.hashing import sha256
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
from vllm.v1.core.sched.interface import PauseState
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
from vllm.v1.engine import FinishReason
from vllm.v1.engine.core import EngineCore
from vllm.v1.executor.uniproc_executor import UniProcExecutor
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
@@ -1487,6 +1490,79 @@ def test_scheduler_reset_prefix_cache():
assert scheduler.waiting[i] == request
def test_kv_cache_release_after_keep_pause_preserves_requests():
"""A completed keep pause permits release and recomputation of live requests."""
scheduler = create_scheduler(max_num_seqs=1, enable_prefix_caching=True)
running, waiting = create_requests(num_requests=2)
for request in (running, waiting):
scheduler.add_request(request)
scheduler_output = scheduler.schedule()
scheduler.update_from_output(
scheduler_output,
ModelRunnerOutput(
req_ids=[running.request_id],
req_id_to_index={running.request_id: 0},
sampled_token_ids=[[1]],
),
)
assert scheduler.kv_cache_manager.get_block_ids(running.request_id)[0]
core = object.__new__(EngineCore)
core.scheduler = scheduler
core.model_executor = Mock(is_sleeping=False)
core.mm_receiver_cache = None
core.batch_queue = None
assert core.pause_scheduler(mode="keep", clear_cache=False) is None
assert scheduler.running == [running]
assert list(scheduler.waiting) == [waiting]
assert not scheduler.has_requests()
core.release_kv_cache_memory()
core.model_executor.discard.assert_called_once_with(("kv_cache",))
assert scheduler.requests == {
request.request_id: request for request in (running, waiting)
}
assert not scheduler.running
assert list(scheduler.waiting) == [running, waiting]
assert running.status == RequestStatus.PREEMPTED
assert running.num_computed_tokens == 0
assert list(running.output_token_ids) == [1]
assert not scheduler.kv_cache_manager.get_block_ids(running.request_id)[0]
assert scheduler.schedule().total_num_scheduled_tokens == 0
assert core.wake_up(tags=["kv_cache"])
resumed = scheduler.schedule()
assert resumed.num_scheduled_tokens == {running.request_id: running.num_tokens}
@pytest.mark.parametrize(
"sleeping_tags",
[{"weights", "kv_cache"}, {"weights"}, {"kv_cache"}],
ids=["full-sleep", "weights-only", "kv-only-or-repeated-release"],
)
def test_kv_cache_release_rejects_nonresident_memory(sleeping_tags):
"""Reject release without resetting caches or discarding more memory."""
core = object.__new__(EngineCore)
core.scheduler = create_scheduler()
core.scheduler.set_pause_state(PauseState.PAUSED_ALL)
core.batch_queue = None
core._reset_caches = Mock()
core.model_executor = object.__new__(UniProcExecutor)
core.model_executor.sleeping_tags = sleeping_tags.copy()
core.model_executor.collective_rpc = Mock()
with pytest.raises(
RuntimeError, match="requires all executor memory to be resident"
):
core.release_kv_cache_memory()
core._reset_caches.assert_not_called()
core.model_executor.collective_rpc.assert_not_called()
assert core.model_executor.sleeping_tags == sleeping_tags
def test_reset_connector_cache_no_connector_is_no_op_success():
"""``reset_connector_cache`` must return True when no connector is
configured.
+24
View File
@@ -23,6 +23,7 @@ from vllm.config import (
from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_default_torch_num_threads
from vllm.v1.core.sched.interface import PauseState
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc
from vllm.v1.executor.abstract import Executor
@@ -667,6 +668,29 @@ def _pausable_engine_core_proc() -> EngineCoreProc:
return core
@pytest.mark.parametrize(
"pause_state,has_requests,has_batches",
[
pytest.param(PauseState.UNPAUSED, False, False, id="not-paused"),
pytest.param(PauseState.PAUSED_ALL, True, False, id="pending-requests"),
pytest.param(PauseState.PAUSED_ALL, False, True, id="pending-batches"),
],
)
def test_kv_cache_release_rejects_unsafe_state(pause_state, has_requests, has_batches):
"""Reject release before touching caches or memory if work can still use KV."""
core = _pausable_engine_core_proc()
core.scheduler.pause_state = pause_state
core.scheduler.has_requests.return_value = has_requests
core.batch_queue = [object()] if has_batches else None
core._reset_caches = MagicMock()
with pytest.raises(RuntimeError, match="requires a completed pause"):
core.release_kv_cache_memory()
core._reset_caches.assert_not_called()
core.model_executor.discard.assert_not_called()
@pytest.mark.parametrize("deferred", [False, True])
def test_pause_synchronizes_device_before_cache_reset(deferred: bool):
"""A resolved pause promises an idle device: the barrier must run before
@@ -73,6 +73,13 @@ class SleepModeBackend(ABC):
(suspended) engine from a healthy-serving one (see RFC #34303)."""
return self._state
def discard(self, tags: tuple[str, ...]) -> None:
"""Discard selected tagged allocations without preserving contents."""
raise RuntimeError(
f"Sleep-mode backend {type(self).__name__!r} does not support "
"selective discard."
)
# -- Capability introspection (no instance required) --
@classmethod
@@ -132,6 +139,13 @@ class CuMemBackend(SleepModeBackend):
allocator.wake_up(tags)
self._state = "RUNNING"
def discard(self, tags: tuple[str, ...]) -> None:
from vllm.device_allocator import get_mem_allocator_instance
allocator = get_mem_allocator_instance()
allocator.discard(tags)
self._state = "SUSPENDED"
@classmethod
def preserves_communicators(cls) -> bool:
# Comm identity/topology survive memory suspension; no reinit needed.
+5
View File
@@ -186,6 +186,11 @@ class EngineClient(ABC):
"""Sleep the engine."""
...
@abstractmethod
async def release_kv_cache_memory(self) -> None:
"""Discard KV cache physical GPU memory. Requires a completed pause."""
...
@abstractmethod
async def wake_up(self, tags: list[str] | None = None) -> None:
"""Wake up the engine."""
+8
View File
@@ -830,6 +830,14 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
"""
self.llm_engine.sleep(level=level, mode=mode)
def release_kv_cache_memory(self) -> None:
"""Release the GPU physical memory backing the KV cache.
Requires a completed ``sleep(level=0)`` and resident executor memory.
Restore with ``wake_up(tags=["kv_cache"])``; kept requests are recomputed.
"""
self.llm_engine.release_kv_cache_memory()
def wake_up(self, tags: list[str] | None = None):
"""Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep]
method for more details.
@@ -27,6 +27,12 @@ async def sleep(raw_request: Request):
return Response(status_code=200)
@router.post("/release_kv_cache_memory")
async def release_kv_cache_memory(raw_request: Request):
await engine_client(raw_request).release_kv_cache_memory()
return Response(status_code=200)
@router.post("/wake_up")
async def wake_up(raw_request: Request):
tags = raw_request.query_params.getlist("tags")
+9 -2
View File
@@ -1079,10 +1079,17 @@ class AsyncLLM(EngineClient):
if self.logger_manager is not None:
self.logger_manager.record_sleep_state(1, level)
async def wake_up(self, tags: list[str] | None = None) -> None:
await self.engine_core.wake_up_async(tags)
async def release_kv_cache_memory(self) -> None:
await self.renderer.clear_mm_cache_async()
await self.engine_core.release_kv_cache_memory_async()
if self.logger_manager is not None:
self.logger_manager.record_sleep_state(1, 0)
async def wake_up(self, tags: list[str] | None = None) -> None:
fully_awake = await self.engine_core.wake_up_async(tags)
if self.logger_manager is not None and fully_awake:
self.logger_manager.record_sleep_state(0, 0)
async def checkpoint_prepare(self) -> None:
+27 -2
View File
@@ -957,12 +957,15 @@ class EngineCore:
pause_future.add_done_callback(pause_complete)
return future
def wake_up(self, tags: list[str] | None = None):
def wake_up(self, tags: list[str] | None = None) -> bool:
"""Wake up the engine from sleep.
Args:
tags: Tags to wake up. Use ["scheduling"] for level 0 wake up.
Returns:
Whether all executor memory is resident again (fully awake).
"""
if tags is not None and "scheduling" in tags:
# Remove "scheduling" from tags if there are other tags to process.
@@ -973,8 +976,30 @@ class EngineCore:
# Partial wakes intentionally keep the remaining allocations asleep.
# Resume scheduling only once all executor memory is resident again.
if not self.model_executor.is_sleeping:
fully_awake = not self.model_executor.is_sleeping
if fully_awake:
self.resume_scheduler()
return fully_awake
def release_kv_cache_memory(self) -> None:
"""Discard KV cache physical memory. Requires a completed pause
and all executor memory to be resident. Kept requests are recomputed
after wake-up.
"""
if not (
self.is_scheduler_paused()
and not self.scheduler.has_requests()
and not self.batch_queue
):
raise RuntimeError(
"release_kv_cache_memory() requires a completed pause first"
)
if self.model_executor.is_sleeping:
raise RuntimeError(
"release_kv_cache_memory() requires all executor memory to be resident"
)
self._reset_caches()
self.model_executor.discard(("kv_cache",))
def is_sleeping(self) -> bool:
"""Check if engine is sleeping at any level."""
+23 -8
View File
@@ -193,7 +193,10 @@ class EngineCoreClient(ABC):
def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None:
raise NotImplementedError
def wake_up(self, tags: list[str] | None = None) -> None:
def release_kv_cache_memory(self) -> None:
raise NotImplementedError
def wake_up(self, tags: list[str] | None = None) -> bool:
raise NotImplementedError
def is_sleeping(self) -> bool:
@@ -285,7 +288,10 @@ class EngineCoreClient(ABC):
async def sleep_async(self, level: int = 1, mode: PauseMode = "abort") -> None:
raise NotImplementedError
async def wake_up_async(self, tags: list[str] | None = None) -> None:
async def release_kv_cache_memory_async(self) -> None:
raise NotImplementedError
async def wake_up_async(self, tags: list[str] | None = None) -> bool:
raise NotImplementedError
async def is_sleeping_async(self) -> bool:
@@ -397,8 +403,11 @@ class InprocClient(EngineCoreClient):
result = self.engine_core.sleep(level, mode)
assert result is None
def wake_up(self, tags: list[str] | None = None) -> None:
self.engine_core.wake_up(tags)
def release_kv_cache_memory(self) -> None:
self.engine_core.release_kv_cache_memory()
def wake_up(self, tags: list[str] | None = None) -> bool:
return self.engine_core.wake_up(tags)
def is_sleeping(self) -> bool:
return self.engine_core.is_sleeping()
@@ -1021,8 +1030,11 @@ class SyncMPClient(MPClient):
def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None:
self.call_utility("sleep", level, mode)
def wake_up(self, tags: list[str] | None = None) -> None:
self.call_utility("wake_up", tags)
def release_kv_cache_memory(self) -> None:
self.call_utility("release_kv_cache_memory")
def wake_up(self, tags: list[str] | None = None) -> bool:
return self.call_utility("wake_up", tags)
def is_sleeping(self) -> bool:
return self.call_utility("is_sleeping")
@@ -1265,8 +1277,11 @@ class AsyncMPClient(MPClient):
async def sleep_async(self, level: int = 1, mode: PauseMode = "abort") -> None:
await self.call_utility_async("sleep", level, mode)
async def wake_up_async(self, tags: list[str] | None = None) -> None:
await self.call_utility_async("wake_up", tags)
async def release_kv_cache_memory_async(self) -> None:
await self.call_utility_async("release_kv_cache_memory")
async def wake_up_async(self, tags: list[str] | None = None) -> bool:
return await self.call_utility_async("wake_up", tags)
async def is_sleeping_async(self) -> bool:
return await self.call_utility_async("is_sleeping")
+9 -2
View File
@@ -375,10 +375,17 @@ class LLMEngine:
if self.logger_manager is not None:
self.logger_manager.record_sleep_state(1, level)
def wake_up(self, tags: list[str] | None = None):
self.engine_core.wake_up(tags)
def release_kv_cache_memory(self) -> None:
self.renderer.clear_mm_cache()
self.engine_core.release_kv_cache_memory()
if self.logger_manager is not None:
self.logger_manager.record_sleep_state(1, 0)
def wake_up(self, tags: list[str] | None = None):
fully_awake = self.engine_core.wake_up(tags)
if self.logger_manager is not None and fully_awake:
self.logger_manager.record_sleep_state(0, 0)
def is_sleeping(self) -> bool:
+26 -6
View File
@@ -32,6 +32,9 @@ logger = init_logger(__name__)
_R = TypeVar("_R")
# Memory-pool tags managed by sleep/wake_up/discard.
SLEEP_TAGS = frozenset(("weights", "kv_cache"))
FailureCallback = Callable[[], None]
@@ -108,7 +111,6 @@ class Executor(ABC):
self.speculative_config = vllm_config.speculative_config
self.observability_config = vllm_config.observability_config
self._init_executor()
self.is_sleeping = False
self.sleeping_tags: set[str] = set()
self.kv_output_aggregator: KVOutputAggregator | None = None
self.ec_output_aggregator: ECOutputAggregator | None = None
@@ -338,15 +340,18 @@ class Executor(ABC):
"""Reset the encoder cache in each worker to clear cached encoder outputs."""
self.collective_rpc("reset_encoder_cache")
@property
def is_sleeping(self) -> bool:
return bool(self.sleeping_tags)
def sleep(self, level: int = 1):
if self.is_sleeping:
if "weights" in self.sleeping_tags:
logger.warning("Executor is already sleeping.")
return
time_before_sleep = time.perf_counter()
self.collective_rpc("sleep", kwargs=dict(level=level))
time_after_sleep = time.perf_counter()
self.sleeping_tags = {"weights", "kv_cache"}
self.is_sleeping = True
self.sleeping_tags |= SLEEP_TAGS
logger.info(
"It took %.6f seconds to fall asleep.", time_after_sleep - time_before_sleep
)
@@ -375,8 +380,23 @@ class Executor(ABC):
self.sleeping_tags.remove(tag)
else:
self.sleeping_tags.clear()
if not self.sleeping_tags:
self.is_sleeping = False
def discard(self, tags: tuple[str, ...]) -> None:
tags_to_discard = set(tags) - self.sleeping_tags
if not tags_to_discard:
logger.warning("Tags %s are already sleeping.", tags)
return
time_before_discard = time.perf_counter()
try:
self.collective_rpc("discard", args=(tuple(tags_to_discard),))
finally:
self.sleeping_tags |= tags_to_discard
time_after_discard = time.perf_counter()
logger.info(
"It took %.6f seconds to discard tags %s.",
time_after_discard - time_before_discard,
tags_to_discard,
)
def reinitialize_distributed(
self, reconfig_request: ReconfigureDistributedRequest
+3
View File
@@ -198,6 +198,9 @@ class CPUWorker(Worker):
logger.warning("sleep mode is not supported on CPU, ignore it.")
pass
def discard(self, tags: tuple[str, ...]) -> None:
raise RuntimeError("Selective discard is not supported on CPU.")
def _should_warm_up_model(self) -> bool:
# VLLM_CPU_CI_ENV always skips warmup to save CI time.
if envs.VLLM_CPU_CI_ENV:
+3
View File
@@ -304,6 +304,9 @@ class Worker(WorkerBase):
self.synchronize_device()
def discard(self, tags: tuple[str, ...]) -> None:
self.sleep_mode_backend.discard(tags)
def checkpoint_prepare(self) -> None:
checkpoint_prepare_distributed_state()