[Frontend][Core] Add release_kv_cache_memory() API (#44890)
Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: AlanFokCo <alanfok2868@gmail.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com> Co-authored-by: AlanFokCo <alanfok2868@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
AlanFokCo
aoshen02
OpenAI Codex
parent
a524f80b1f
commit
092bdd6d57
@@ -74,6 +74,18 @@ llm.wake_up(tags=["weights"])
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
```
|
||||
|
||||
#### Release only KV cache memory
|
||||
|
||||
`LLM.release_kv_cache_memory()` discards KV cache physical memory while keeping model weights resident. It requires a completed pause and all executor memory to be resident: full sleep, partial wake-up, and repeated release without restoring memory are rejected. Requests retained with `mode="keep"` are recomputed after wake-up.
|
||||
|
||||
Use the default `cumem` backend with managed allocations: `enable_cumem_allocator=True` on CUDA/ROCm (also enabled by `enable_sleep_mode=True`), or `enable_sleep_mode=True` on XPU. Other backends must implement selective discard; CPU and unmanaged allocations are unsupported.
|
||||
|
||||
```python
|
||||
llm.sleep(level=0, mode="keep") # Wait for the pause to complete.
|
||||
llm.release_kv_cache_memory()
|
||||
llm.wake_up(tags=["kv_cache"]) # Reallocate KV cache and resume scheduling.
|
||||
```
|
||||
|
||||
### Online Serving
|
||||
|
||||
To enable sleep mode in a vLLM server you need to initialize it with the flag `VLLM_SERVER_DEV_MODE=1` and pass `--enable-sleep-mode` to the vLLM server.
|
||||
@@ -107,9 +119,18 @@ curl -X POST 'http://localhost:8000/collective_rpc' -H 'Content-Type: applicatio
|
||||
curl -X POST 'http://localhost:8000/wake_up?tags=kv_cache'
|
||||
```
|
||||
|
||||
To release only KV cache memory, wait for the level 0 pause to complete before calling the release endpoint. The same resident-memory and backend requirements as the Python API apply.
|
||||
|
||||
```bash
|
||||
curl -X POST 'http://localhost:8000/sleep?level=0&mode=keep'
|
||||
curl -X POST 'http://localhost:8000/release_kv_cache_memory'
|
||||
curl -X POST 'http://localhost:8000/wake_up?tags=kv_cache'
|
||||
```
|
||||
|
||||
#### HTTP endpoints
|
||||
|
||||
- `POST /sleep?level=1` — Put the model to sleep (`level=1`).
|
||||
- `POST /release_kv_cache_memory` — Discard KV cache memory after a completed pause, while all executor memory is resident.
|
||||
- `POST /wake_up` — Wake up the model. Supports optional `tags` query parameters for partial wake-up (e.g., `?tags=weights`).
|
||||
- `POST /collective_rpc` — Perform a collective remote procedure call (RPC).
|
||||
- `GET /is_sleeping` — Check if the model is sleeping.
|
||||
|
||||
@@ -911,13 +911,22 @@ impl EngineCoreClient {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep, optionally limiting the wake-up to specific
|
||||
/// tags.
|
||||
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<()> {
|
||||
self.call_utility::<(), _>("wake_up", (tags,)).await?;
|
||||
/// Release KV cache memory while keeping model weights resident.
|
||||
pub async fn release_kv_cache_memory(&self) -> Result<()> {
|
||||
self.call_utility::<(), _>("release_kv_cache_memory", ()).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep, optionally limiting the wake-up to specific
|
||||
/// tags, and return whether every engine is fully awake.
|
||||
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<bool> {
|
||||
Ok(self
|
||||
.call_utility::<bool, _>("wake_up", (tags,))
|
||||
.await?
|
||||
.into_iter()
|
||||
.all(|fully_awake| fully_awake))
|
||||
}
|
||||
|
||||
/// Pause the scheduler so generation can be halted
|
||||
pub async fn pause_scheduler(&self, mode: PauseMode, clear_cache: bool) -> Result<()> {
|
||||
self.call_utility::<(), _>("pause_scheduler", (mode, clear_cache)).await?;
|
||||
|
||||
@@ -95,12 +95,10 @@ fn utility_response(
|
||||
"get_supported_tasks" => utility_envelope(vec!["generate"]),
|
||||
"is_sleeping" => utility_envelope(false),
|
||||
"reset_prefix_cache" => utility_envelope(true),
|
||||
"reset_mm_cache"
|
||||
| "reset_encoder_cache"
|
||||
| "profile"
|
||||
| "sleep"
|
||||
| "wake_up"
|
||||
| "execute_dummy_batch" => utility_envelope(()),
|
||||
"wake_up" => utility_envelope(true),
|
||||
"reset_mm_cache" | "reset_encoder_cache" | "profile" | "sleep" | "execute_dummy_batch" => {
|
||||
utility_envelope(())
|
||||
}
|
||||
_ => utility_envelope(Value::Nil),
|
||||
}?;
|
||||
|
||||
|
||||
@@ -195,6 +195,7 @@ async fn utility_requests_return_minimal_success_responses() {
|
||||
assert!(client.reset_prefix_cache(false, false).await.expect("reset prefix cache"));
|
||||
client.reset_mm_cache().await.expect("reset mm cache");
|
||||
client.reset_encoder_cache().await.expect("reset encoder cache");
|
||||
assert!(client.wake_up(Some(vec!["kv_cache".to_string()])).await.expect("wake up"));
|
||||
|
||||
shutdown_mock(client, shutdown, task).await;
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ const EXCLUDED_HANDLERS: &[&str] = &[
|
||||
"/reset_encoder_cache",
|
||||
"/collective_rpc",
|
||||
"/sleep",
|
||||
"/release_kv_cache_memory",
|
||||
"/wake_up",
|
||||
"/is_sleeping",
|
||||
];
|
||||
|
||||
@@ -142,6 +142,10 @@ fn build_router_with_options(
|
||||
.route("/weight_info", get(weight_transfer::weight_info))
|
||||
.route("/abort_requests", post(abort_requests::abort_requests))
|
||||
.route("/sleep", post(sleep::sleep))
|
||||
.route(
|
||||
"/release_kv_cache_memory",
|
||||
post(sleep::release_kv_cache_memory),
|
||||
)
|
||||
.route("/wake_up", post(sleep::wake_up))
|
||||
.route("/is_sleeping", get(sleep::is_sleeping))
|
||||
.route("/pause", post(pause::pause))
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::rejection::QueryRejection;
|
||||
use axum::extract::{Query, State};
|
||||
use axum::extract::{Query, RawQuery, State};
|
||||
use axum::http::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use vllm_engine_core_client::protocol::utility::PauseMode;
|
||||
@@ -27,12 +27,6 @@ pub(crate) struct SleepParams {
|
||||
mode: PauseMode,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
pub(crate) struct WakeUpParams {
|
||||
#[serde(default)]
|
||||
tags: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
const fn default_sleep_level() -> u32 {
|
||||
1
|
||||
}
|
||||
@@ -57,14 +51,35 @@ pub async fn sleep(
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep mode.
|
||||
pub async fn wake_up(
|
||||
/// Release KV cache memory while keeping model weights resident.
|
||||
pub async fn release_kv_cache_memory(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Query(params): Query<WakeUpParams>,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
state
|
||||
.engine_core_client()
|
||||
.wake_up(params.tags)
|
||||
.release_kv_cache_memory()
|
||||
.await
|
||||
.map_err(|error| utility_call_error("release_kv_cache_memory", error))?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep mode.
|
||||
pub async fn wake_up(
|
||||
State(state): State<Arc<AppState>>,
|
||||
RawQuery(query): RawQuery,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
let tags = query.and_then(|query| {
|
||||
let tags = url::form_urlencoded::parse(query.as_bytes())
|
||||
.filter(|(key, _)| key == "tags")
|
||||
.map(|(_, value)| value.into_owned())
|
||||
.collect::<Vec<_>>();
|
||||
(!tags.is_empty()).then_some(tags)
|
||||
});
|
||||
|
||||
state
|
||||
.engine_core_client()
|
||||
.wake_up(tags)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("wake_up", error))?;
|
||||
|
||||
|
||||
@@ -5805,7 +5805,46 @@ async fn sleep_route_uses_python_compatible_default_query_values() {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn wake_up_route_without_tags_sends_none() {
|
||||
async fn release_kv_cache_memory_route_sends_expected_utility_call() {
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
|
||||
assert_eq!(array[2], Value::from("release_kv_cache_memory"));
|
||||
assert_eq!(array[3], Value::Array(Vec::new()));
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/release_kv_cache_memory")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn wake_up_route_without_tags_accepts_bool_result() {
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
@@ -5818,7 +5857,7 @@ async fn wake_up_route_without_tags_sends_none() {
|
||||
assert_eq!(array[2], Value::from("wake_up"));
|
||||
assert_eq!(array[3], Value::Array(vec![Value::Nil]));
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
@@ -5842,6 +5881,51 @@ async fn wake_up_route_without_tags_sends_none() {
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn wake_up_route_accepts_repeated_tags() {
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
|
||||
assert_eq!(array[2], Value::from("wake_up"));
|
||||
assert_eq!(
|
||||
array[3],
|
||||
Value::Array(vec![Value::Array(vec![
|
||||
Value::from("weights"),
|
||||
Value::from("kv_cache"),
|
||||
])])
|
||||
);
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/wake_up?tags=weights&tags%5B%5D=ignored&tags=kv_cache")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn is_sleeping_route_returns_json_payload() {
|
||||
@@ -6160,6 +6244,7 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() {
|
||||
for (method, uri) in [
|
||||
("GET", "/is_sleeping"),
|
||||
("POST", "/sleep"),
|
||||
("POST", "/release_kv_cache_memory"),
|
||||
("POST", "/wake_up"),
|
||||
("GET", "/is_paused"),
|
||||
("POST", "/pause"),
|
||||
|
||||
@@ -109,6 +109,91 @@ def test_basic_cumem():
|
||||
assert torch.allclose(output, torch.ones_like(output) * 3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch):
|
||||
"""Rejected release and KV-only/full sleep cycles preserve greedy tokens."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
kv_cache_memory_bytes = 256 * 1024 * 1024
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
enable_sleep_mode=True,
|
||||
enforce_eager=True,
|
||||
kv_cache_memory_bytes=kv_cache_memory_bytes,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=4,
|
||||
gpu_memory_utilization=0.05,
|
||||
)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
|
||||
def get_mapped_bytes(worker):
|
||||
return mapped_usage(get_mem_allocator_instance())
|
||||
|
||||
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
|
||||
# Utility RPC transports the engine error as a plain Exception.
|
||||
with pytest.raises(Exception, match="requires a completed pause"):
|
||||
llm.release_kv_cache_memory()
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected
|
||||
|
||||
llm.sleep(level=0)
|
||||
llm.release_kv_cache_memory()
|
||||
mapped_after = llm.collective_rpc(get_mapped_bytes)[0]
|
||||
assert mapped_before - mapped_after >= kv_cache_memory_bytes * 0.99
|
||||
assert mapped_after > 0
|
||||
assert llm.llm_engine.is_sleeping()
|
||||
|
||||
if full_sleep:
|
||||
llm.sleep(level=full_sleep)
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == 0
|
||||
llm.wake_up(tags=None if full_sleep else ["kv_cache"])
|
||||
if full_sleep == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_level", [1, 2])
|
||||
@pytest.mark.parametrize("second_level", [1, 2])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch):
|
||||
"""Repeated sleep after KV-only wake preserves mappings and recoverability."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
enable_sleep_mode=True,
|
||||
enforce_eager=True,
|
||||
kv_cache_memory_bytes=256 * 1024 * 1024,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=4,
|
||||
gpu_memory_utilization=0.05,
|
||||
)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
|
||||
def get_mapped_bytes(worker):
|
||||
return mapped_usage(get_mem_allocator_instance())
|
||||
|
||||
llm.sleep(level=first_level)
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
|
||||
assert mapped_before > 0
|
||||
llm.sleep(level=second_level)
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
|
||||
llm.wake_up(tags=["weights"])
|
||||
if first_level == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_discard_tags():
|
||||
"""Test that discard(tags) selectively frees GPU memory for specific
|
||||
|
||||
@@ -1,14 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.serve.dev.sleep.api_router import attach_router
|
||||
|
||||
MODEL_NAME = "meta-llama/Llama-3.2-1B"
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("fails", [False, True])
|
||||
def test_release_kv_cache_memory_route(fails):
|
||||
app = FastAPI()
|
||||
app.state.engine_client = AsyncMock()
|
||||
release = app.state.engine_client.release_kv_cache_memory
|
||||
if fails:
|
||||
release.side_effect = RuntimeError("requires a completed pause first")
|
||||
attach_router(app)
|
||||
|
||||
with TestClient(app, raise_server_exceptions=False) as client:
|
||||
response = client.post("/release_kv_cache_memory")
|
||||
|
||||
assert response.status_code == (500 if fails else 200)
|
||||
release.assert_awaited_once_with()
|
||||
|
||||
|
||||
def test_sleep_mode():
|
||||
# dtype, max-len etc set so that this can run in CI
|
||||
args = [
|
||||
@@ -63,7 +86,7 @@ def test_sleep_mode():
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# is sleeping should be false after waking up any part of the engine
|
||||
# Partial wake keeps the engine sleeping.
|
||||
response = requests.get(remote_server.url_for("is_sleeping"))
|
||||
assert response.status_code == 200
|
||||
assert response.json().get("is_sleeping") is True
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import multiprocessing as mp
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -34,6 +37,8 @@ from vllm.multimodal.inputs import (
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.multimodal.processing import PromptInsertion
|
||||
from vllm.renderers.base import BaseRenderer
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.mem_constants import GiB_bytes, MiB_bytes
|
||||
|
||||
from ..utils import create_new_process_for_each_test
|
||||
@@ -755,6 +760,76 @@ def test_processor_cache_shared_across_loras():
|
||||
assert feature_lora_b.data == item_data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_async", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"release_error",
|
||||
[
|
||||
None,
|
||||
"requires a completed pause first",
|
||||
"requires all executor memory to be resident",
|
||||
],
|
||||
ids=["released", "not-paused", "nonresident-memory"],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_kv_cache_resends_mm_payload(use_async, release_error):
|
||||
"""Release must not leave a sender hit pointing at a cleared receiver."""
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
model_config = SimpleNamespace(
|
||||
get_multimodal_config=lambda: MultiModalConfig(mm_processor_cache_gb=1)
|
||||
)
|
||||
sender = MultiModalProcessorSenderCache(model_config)
|
||||
receiver = MultiModalReceiverCache(model_config)
|
||||
item = _dummy_item({"pixel_values": 16})
|
||||
mm_hash = "image_A"
|
||||
payload, _ = sender.get_and_update_item((item, []), mm_hash)
|
||||
assert receiver.get_and_update_item(payload, mm_hash) is item
|
||||
assert sender.get_and_update_item(None, mm_hash)[0] is None
|
||||
|
||||
renderer = SimpleNamespace(mm_processor_cache=sender, _mm_cache_stats=None)
|
||||
renderer.clear_mm_cache = partial(BaseRenderer.clear_mm_cache, renderer)
|
||||
|
||||
def release():
|
||||
if release_error:
|
||||
raise RuntimeError(release_error)
|
||||
receiver.clear_cache()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
renderer._clear_mm_cache_async = make_async(
|
||||
renderer.clear_mm_cache, executor=executor
|
||||
)
|
||||
renderer.clear_mm_cache_async = partial(
|
||||
BaseRenderer.clear_mm_cache_async, renderer
|
||||
)
|
||||
engine = SimpleNamespace(
|
||||
renderer=renderer,
|
||||
engine_core=SimpleNamespace(
|
||||
release_kv_cache_memory=release,
|
||||
release_kv_cache_memory_async=make_async(release, executor=executor),
|
||||
),
|
||||
logger_manager=Mock(),
|
||||
)
|
||||
|
||||
async def call_release():
|
||||
if use_async:
|
||||
await AsyncLLM.release_kv_cache_memory(engine)
|
||||
else:
|
||||
LLMEngine.release_kv_cache_memory(engine)
|
||||
|
||||
if release_error:
|
||||
with pytest.raises(RuntimeError, match=release_error):
|
||||
await call_release()
|
||||
engine.logger_manager.record_sleep_state.assert_not_called()
|
||||
else:
|
||||
await call_release()
|
||||
engine.logger_manager.record_sleep_state.assert_called_once_with(1, 0)
|
||||
|
||||
payload, _ = sender.get_and_update_item((item, []), mm_hash)
|
||||
assert payload is item
|
||||
assert receiver.get_and_update_item(payload, mm_hash) is item
|
||||
|
||||
|
||||
_SLEEP_VISION_PROMPT = (
|
||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
|
||||
"\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
|
||||
|
||||
@@ -36,10 +36,13 @@ from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
|
||||
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
|
||||
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
|
||||
from vllm.v1.core.sched.interface import PauseState
|
||||
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.engine.core import EngineCore
|
||||
from vllm.v1.executor.uniproc_executor import UniProcExecutor
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
@@ -1487,6 +1490,79 @@ def test_scheduler_reset_prefix_cache():
|
||||
assert scheduler.waiting[i] == request
|
||||
|
||||
|
||||
def test_kv_cache_release_after_keep_pause_preserves_requests():
|
||||
"""A completed keep pause permits release and recomputation of live requests."""
|
||||
scheduler = create_scheduler(max_num_seqs=1, enable_prefix_caching=True)
|
||||
running, waiting = create_requests(num_requests=2)
|
||||
for request in (running, waiting):
|
||||
scheduler.add_request(request)
|
||||
scheduler_output = scheduler.schedule()
|
||||
scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[running.request_id],
|
||||
req_id_to_index={running.request_id: 0},
|
||||
sampled_token_ids=[[1]],
|
||||
),
|
||||
)
|
||||
assert scheduler.kv_cache_manager.get_block_ids(running.request_id)[0]
|
||||
|
||||
core = object.__new__(EngineCore)
|
||||
core.scheduler = scheduler
|
||||
core.model_executor = Mock(is_sleeping=False)
|
||||
core.mm_receiver_cache = None
|
||||
core.batch_queue = None
|
||||
|
||||
assert core.pause_scheduler(mode="keep", clear_cache=False) is None
|
||||
assert scheduler.running == [running]
|
||||
assert list(scheduler.waiting) == [waiting]
|
||||
assert not scheduler.has_requests()
|
||||
|
||||
core.release_kv_cache_memory()
|
||||
|
||||
core.model_executor.discard.assert_called_once_with(("kv_cache",))
|
||||
assert scheduler.requests == {
|
||||
request.request_id: request for request in (running, waiting)
|
||||
}
|
||||
assert not scheduler.running
|
||||
assert list(scheduler.waiting) == [running, waiting]
|
||||
assert running.status == RequestStatus.PREEMPTED
|
||||
assert running.num_computed_tokens == 0
|
||||
assert list(running.output_token_ids) == [1]
|
||||
assert not scheduler.kv_cache_manager.get_block_ids(running.request_id)[0]
|
||||
assert scheduler.schedule().total_num_scheduled_tokens == 0
|
||||
|
||||
assert core.wake_up(tags=["kv_cache"])
|
||||
resumed = scheduler.schedule()
|
||||
assert resumed.num_scheduled_tokens == {running.request_id: running.num_tokens}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sleeping_tags",
|
||||
[{"weights", "kv_cache"}, {"weights"}, {"kv_cache"}],
|
||||
ids=["full-sleep", "weights-only", "kv-only-or-repeated-release"],
|
||||
)
|
||||
def test_kv_cache_release_rejects_nonresident_memory(sleeping_tags):
|
||||
"""Reject release without resetting caches or discarding more memory."""
|
||||
core = object.__new__(EngineCore)
|
||||
core.scheduler = create_scheduler()
|
||||
core.scheduler.set_pause_state(PauseState.PAUSED_ALL)
|
||||
core.batch_queue = None
|
||||
core._reset_caches = Mock()
|
||||
core.model_executor = object.__new__(UniProcExecutor)
|
||||
core.model_executor.sleeping_tags = sleeping_tags.copy()
|
||||
core.model_executor.collective_rpc = Mock()
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError, match="requires all executor memory to be resident"
|
||||
):
|
||||
core.release_kv_cache_memory()
|
||||
|
||||
core._reset_caches.assert_not_called()
|
||||
core.model_executor.collective_rpc.assert_not_called()
|
||||
assert core.model_executor.sleeping_tags == sleeping_tags
|
||||
|
||||
|
||||
def test_reset_connector_cache_no_connector_is_no_op_success():
|
||||
"""``reset_connector_cache`` must return True when no connector is
|
||||
configured.
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.config import (
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_default_torch_num_threads
|
||||
from vllm.v1.core.sched.interface import PauseState
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
@@ -667,6 +668,29 @@ def _pausable_engine_core_proc() -> EngineCoreProc:
|
||||
return core
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pause_state,has_requests,has_batches",
|
||||
[
|
||||
pytest.param(PauseState.UNPAUSED, False, False, id="not-paused"),
|
||||
pytest.param(PauseState.PAUSED_ALL, True, False, id="pending-requests"),
|
||||
pytest.param(PauseState.PAUSED_ALL, False, True, id="pending-batches"),
|
||||
],
|
||||
)
|
||||
def test_kv_cache_release_rejects_unsafe_state(pause_state, has_requests, has_batches):
|
||||
"""Reject release before touching caches or memory if work can still use KV."""
|
||||
core = _pausable_engine_core_proc()
|
||||
core.scheduler.pause_state = pause_state
|
||||
core.scheduler.has_requests.return_value = has_requests
|
||||
core.batch_queue = [object()] if has_batches else None
|
||||
core._reset_caches = MagicMock()
|
||||
|
||||
with pytest.raises(RuntimeError, match="requires a completed pause"):
|
||||
core.release_kv_cache_memory()
|
||||
|
||||
core._reset_caches.assert_not_called()
|
||||
core.model_executor.discard.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deferred", [False, True])
|
||||
def test_pause_synchronizes_device_before_cache_reset(deferred: bool):
|
||||
"""A resolved pause promises an idle device: the barrier must run before
|
||||
|
||||
@@ -73,6 +73,13 @@ class SleepModeBackend(ABC):
|
||||
(suspended) engine from a healthy-serving one (see RFC #34303)."""
|
||||
return self._state
|
||||
|
||||
def discard(self, tags: tuple[str, ...]) -> None:
|
||||
"""Discard selected tagged allocations without preserving contents."""
|
||||
raise RuntimeError(
|
||||
f"Sleep-mode backend {type(self).__name__!r} does not support "
|
||||
"selective discard."
|
||||
)
|
||||
|
||||
# -- Capability introspection (no instance required) --
|
||||
|
||||
@classmethod
|
||||
@@ -132,6 +139,13 @@ class CuMemBackend(SleepModeBackend):
|
||||
allocator.wake_up(tags)
|
||||
self._state = "RUNNING"
|
||||
|
||||
def discard(self, tags: tuple[str, ...]) -> None:
|
||||
from vllm.device_allocator import get_mem_allocator_instance
|
||||
|
||||
allocator = get_mem_allocator_instance()
|
||||
allocator.discard(tags)
|
||||
self._state = "SUSPENDED"
|
||||
|
||||
@classmethod
|
||||
def preserves_communicators(cls) -> bool:
|
||||
# Comm identity/topology survive memory suspension; no reinit needed.
|
||||
|
||||
@@ -186,6 +186,11 @@ class EngineClient(ABC):
|
||||
"""Sleep the engine."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def release_kv_cache_memory(self) -> None:
|
||||
"""Discard KV cache physical GPU memory. Requires a completed pause."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
"""Wake up the engine."""
|
||||
|
||||
@@ -830,6 +830,14 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
"""
|
||||
self.llm_engine.sleep(level=level, mode=mode)
|
||||
|
||||
def release_kv_cache_memory(self) -> None:
|
||||
"""Release the GPU physical memory backing the KV cache.
|
||||
|
||||
Requires a completed ``sleep(level=0)`` and resident executor memory.
|
||||
Restore with ``wake_up(tags=["kv_cache"])``; kept requests are recomputed.
|
||||
"""
|
||||
self.llm_engine.release_kv_cache_memory()
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None):
|
||||
"""Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep]
|
||||
method for more details.
|
||||
|
||||
@@ -27,6 +27,12 @@ async def sleep(raw_request: Request):
|
||||
return Response(status_code=200)
|
||||
|
||||
|
||||
@router.post("/release_kv_cache_memory")
|
||||
async def release_kv_cache_memory(raw_request: Request):
|
||||
await engine_client(raw_request).release_kv_cache_memory()
|
||||
return Response(status_code=200)
|
||||
|
||||
|
||||
@router.post("/wake_up")
|
||||
async def wake_up(raw_request: Request):
|
||||
tags = raw_request.query_params.getlist("tags")
|
||||
|
||||
@@ -1079,10 +1079,17 @@ class AsyncLLM(EngineClient):
|
||||
if self.logger_manager is not None:
|
||||
self.logger_manager.record_sleep_state(1, level)
|
||||
|
||||
async def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
await self.engine_core.wake_up_async(tags)
|
||||
async def release_kv_cache_memory(self) -> None:
|
||||
await self.renderer.clear_mm_cache_async()
|
||||
await self.engine_core.release_kv_cache_memory_async()
|
||||
|
||||
if self.logger_manager is not None:
|
||||
self.logger_manager.record_sleep_state(1, 0)
|
||||
|
||||
async def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
fully_awake = await self.engine_core.wake_up_async(tags)
|
||||
|
||||
if self.logger_manager is not None and fully_awake:
|
||||
self.logger_manager.record_sleep_state(0, 0)
|
||||
|
||||
async def checkpoint_prepare(self) -> None:
|
||||
|
||||
+27
-2
@@ -957,12 +957,15 @@ class EngineCore:
|
||||
pause_future.add_done_callback(pause_complete)
|
||||
return future
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None):
|
||||
def wake_up(self, tags: list[str] | None = None) -> bool:
|
||||
"""Wake up the engine from sleep.
|
||||
|
||||
Args:
|
||||
tags: Tags to wake up. Use ["scheduling"] for level 0 wake up.
|
||||
|
||||
Returns:
|
||||
Whether all executor memory is resident again (fully awake).
|
||||
|
||||
"""
|
||||
if tags is not None and "scheduling" in tags:
|
||||
# Remove "scheduling" from tags if there are other tags to process.
|
||||
@@ -973,8 +976,30 @@ class EngineCore:
|
||||
|
||||
# Partial wakes intentionally keep the remaining allocations asleep.
|
||||
# Resume scheduling only once all executor memory is resident again.
|
||||
if not self.model_executor.is_sleeping:
|
||||
fully_awake = not self.model_executor.is_sleeping
|
||||
if fully_awake:
|
||||
self.resume_scheduler()
|
||||
return fully_awake
|
||||
|
||||
def release_kv_cache_memory(self) -> None:
|
||||
"""Discard KV cache physical memory. Requires a completed pause
|
||||
and all executor memory to be resident. Kept requests are recomputed
|
||||
after wake-up.
|
||||
"""
|
||||
if not (
|
||||
self.is_scheduler_paused()
|
||||
and not self.scheduler.has_requests()
|
||||
and not self.batch_queue
|
||||
):
|
||||
raise RuntimeError(
|
||||
"release_kv_cache_memory() requires a completed pause first"
|
||||
)
|
||||
if self.model_executor.is_sleeping:
|
||||
raise RuntimeError(
|
||||
"release_kv_cache_memory() requires all executor memory to be resident"
|
||||
)
|
||||
self._reset_caches()
|
||||
self.model_executor.discard(("kv_cache",))
|
||||
|
||||
def is_sleeping(self) -> bool:
|
||||
"""Check if engine is sleeping at any level."""
|
||||
|
||||
@@ -193,7 +193,10 @@ class EngineCoreClient(ABC):
|
||||
def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
def release_kv_cache_memory(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
def is_sleeping(self) -> bool:
|
||||
@@ -285,7 +288,10 @@ class EngineCoreClient(ABC):
|
||||
async def sleep_async(self, level: int = 1, mode: PauseMode = "abort") -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def wake_up_async(self, tags: list[str] | None = None) -> None:
|
||||
async def release_kv_cache_memory_async(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def wake_up_async(self, tags: list[str] | None = None) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
async def is_sleeping_async(self) -> bool:
|
||||
@@ -397,8 +403,11 @@ class InprocClient(EngineCoreClient):
|
||||
result = self.engine_core.sleep(level, mode)
|
||||
assert result is None
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
self.engine_core.wake_up(tags)
|
||||
def release_kv_cache_memory(self) -> None:
|
||||
self.engine_core.release_kv_cache_memory()
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> bool:
|
||||
return self.engine_core.wake_up(tags)
|
||||
|
||||
def is_sleeping(self) -> bool:
|
||||
return self.engine_core.is_sleeping()
|
||||
@@ -1021,8 +1030,11 @@ class SyncMPClient(MPClient):
|
||||
def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None:
|
||||
self.call_utility("sleep", level, mode)
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
self.call_utility("wake_up", tags)
|
||||
def release_kv_cache_memory(self) -> None:
|
||||
self.call_utility("release_kv_cache_memory")
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> bool:
|
||||
return self.call_utility("wake_up", tags)
|
||||
|
||||
def is_sleeping(self) -> bool:
|
||||
return self.call_utility("is_sleeping")
|
||||
@@ -1265,8 +1277,11 @@ class AsyncMPClient(MPClient):
|
||||
async def sleep_async(self, level: int = 1, mode: PauseMode = "abort") -> None:
|
||||
await self.call_utility_async("sleep", level, mode)
|
||||
|
||||
async def wake_up_async(self, tags: list[str] | None = None) -> None:
|
||||
await self.call_utility_async("wake_up", tags)
|
||||
async def release_kv_cache_memory_async(self) -> None:
|
||||
await self.call_utility_async("release_kv_cache_memory")
|
||||
|
||||
async def wake_up_async(self, tags: list[str] | None = None) -> bool:
|
||||
return await self.call_utility_async("wake_up", tags)
|
||||
|
||||
async def is_sleeping_async(self) -> bool:
|
||||
return await self.call_utility_async("is_sleeping")
|
||||
|
||||
@@ -375,10 +375,17 @@ class LLMEngine:
|
||||
if self.logger_manager is not None:
|
||||
self.logger_manager.record_sleep_state(1, level)
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None):
|
||||
self.engine_core.wake_up(tags)
|
||||
def release_kv_cache_memory(self) -> None:
|
||||
self.renderer.clear_mm_cache()
|
||||
self.engine_core.release_kv_cache_memory()
|
||||
|
||||
if self.logger_manager is not None:
|
||||
self.logger_manager.record_sleep_state(1, 0)
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None):
|
||||
fully_awake = self.engine_core.wake_up(tags)
|
||||
|
||||
if self.logger_manager is not None and fully_awake:
|
||||
self.logger_manager.record_sleep_state(0, 0)
|
||||
|
||||
def is_sleeping(self) -> bool:
|
||||
|
||||
@@ -32,6 +32,9 @@ logger = init_logger(__name__)
|
||||
|
||||
_R = TypeVar("_R")
|
||||
|
||||
# Memory-pool tags managed by sleep/wake_up/discard.
|
||||
SLEEP_TAGS = frozenset(("weights", "kv_cache"))
|
||||
|
||||
FailureCallback = Callable[[], None]
|
||||
|
||||
|
||||
@@ -108,7 +111,6 @@ class Executor(ABC):
|
||||
self.speculative_config = vllm_config.speculative_config
|
||||
self.observability_config = vllm_config.observability_config
|
||||
self._init_executor()
|
||||
self.is_sleeping = False
|
||||
self.sleeping_tags: set[str] = set()
|
||||
self.kv_output_aggregator: KVOutputAggregator | None = None
|
||||
self.ec_output_aggregator: ECOutputAggregator | None = None
|
||||
@@ -338,15 +340,18 @@ class Executor(ABC):
|
||||
"""Reset the encoder cache in each worker to clear cached encoder outputs."""
|
||||
self.collective_rpc("reset_encoder_cache")
|
||||
|
||||
@property
|
||||
def is_sleeping(self) -> bool:
|
||||
return bool(self.sleeping_tags)
|
||||
|
||||
def sleep(self, level: int = 1):
|
||||
if self.is_sleeping:
|
||||
if "weights" in self.sleeping_tags:
|
||||
logger.warning("Executor is already sleeping.")
|
||||
return
|
||||
time_before_sleep = time.perf_counter()
|
||||
self.collective_rpc("sleep", kwargs=dict(level=level))
|
||||
time_after_sleep = time.perf_counter()
|
||||
self.sleeping_tags = {"weights", "kv_cache"}
|
||||
self.is_sleeping = True
|
||||
self.sleeping_tags |= SLEEP_TAGS
|
||||
logger.info(
|
||||
"It took %.6f seconds to fall asleep.", time_after_sleep - time_before_sleep
|
||||
)
|
||||
@@ -375,8 +380,23 @@ class Executor(ABC):
|
||||
self.sleeping_tags.remove(tag)
|
||||
else:
|
||||
self.sleeping_tags.clear()
|
||||
if not self.sleeping_tags:
|
||||
self.is_sleeping = False
|
||||
|
||||
def discard(self, tags: tuple[str, ...]) -> None:
|
||||
tags_to_discard = set(tags) - self.sleeping_tags
|
||||
if not tags_to_discard:
|
||||
logger.warning("Tags %s are already sleeping.", tags)
|
||||
return
|
||||
time_before_discard = time.perf_counter()
|
||||
try:
|
||||
self.collective_rpc("discard", args=(tuple(tags_to_discard),))
|
||||
finally:
|
||||
self.sleeping_tags |= tags_to_discard
|
||||
time_after_discard = time.perf_counter()
|
||||
logger.info(
|
||||
"It took %.6f seconds to discard tags %s.",
|
||||
time_after_discard - time_before_discard,
|
||||
tags_to_discard,
|
||||
)
|
||||
|
||||
def reinitialize_distributed(
|
||||
self, reconfig_request: ReconfigureDistributedRequest
|
||||
|
||||
@@ -198,6 +198,9 @@ class CPUWorker(Worker):
|
||||
logger.warning("sleep mode is not supported on CPU, ignore it.")
|
||||
pass
|
||||
|
||||
def discard(self, tags: tuple[str, ...]) -> None:
|
||||
raise RuntimeError("Selective discard is not supported on CPU.")
|
||||
|
||||
def _should_warm_up_model(self) -> bool:
|
||||
# VLLM_CPU_CI_ENV always skips warmup to save CI time.
|
||||
if envs.VLLM_CPU_CI_ENV:
|
||||
|
||||
@@ -304,6 +304,9 @@ class Worker(WorkerBase):
|
||||
|
||||
self.synchronize_device()
|
||||
|
||||
def discard(self, tags: tuple[str, ...]) -> None:
|
||||
self.sleep_mode_backend.discard(tags)
|
||||
|
||||
def checkpoint_prepare(self) -> None:
|
||||
checkpoint_prepare_distributed_state()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user