[Frontend][Core] Add release_kv_cache_memory() API (#44890)

Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: AlanFokCo <alanfok2868@gmail.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com>
Co-authored-by: AlanFokCo <alanfok2868@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Dakai An
2026-09-18 07:12:12 +08:00
committed by GitHub
co-authored by AlanFokCo aoshen02 OpenAI Codex
parent a524f80b1f
commit 092bdd6d57
24 changed files with 574 additions and 44 deletions
+13 -4
View File
@@ -911,13 +911,22 @@ impl EngineCoreClient {
Ok(())
}
/// Wake the engine from sleep, optionally limiting the wake-up to specific
/// tags.
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<()> {
self.call_utility::<(), _>("wake_up", (tags,)).await?;
/// Release KV cache memory while keeping model weights resident.
pub async fn release_kv_cache_memory(&self) -> Result<()> {
self.call_utility::<(), _>("release_kv_cache_memory", ()).await?;
Ok(())
}
/// Wake the engine from sleep, optionally limiting the wake-up to specific
/// tags, and return whether every engine is fully awake.
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<bool> {
Ok(self
.call_utility::<bool, _>("wake_up", (tags,))
.await?
.into_iter()
.all(|fully_awake| fully_awake))
}
/// Pause the scheduler so generation can be halted
pub async fn pause_scheduler(&self, mode: PauseMode, clear_cache: bool) -> Result<()> {
self.call_utility::<(), _>("pause_scheduler", (mode, clear_cache)).await?;
+4 -6
View File
@@ -95,12 +95,10 @@ fn utility_response(
"get_supported_tasks" => utility_envelope(vec!["generate"]),
"is_sleeping" => utility_envelope(false),
"reset_prefix_cache" => utility_envelope(true),
"reset_mm_cache"
| "reset_encoder_cache"
| "profile"
| "sleep"
| "wake_up"
| "execute_dummy_batch" => utility_envelope(()),
"wake_up" => utility_envelope(true),
"reset_mm_cache" | "reset_encoder_cache" | "profile" | "sleep" | "execute_dummy_batch" => {
utility_envelope(())
}
_ => utility_envelope(Value::Nil),
}?;
+1
View File
@@ -195,6 +195,7 @@ async fn utility_requests_return_minimal_success_responses() {
assert!(client.reset_prefix_cache(false, false).await.expect("reset prefix cache"));
client.reset_mm_cache().await.expect("reset mm cache");
client.reset_encoder_cache().await.expect("reset encoder cache");
assert!(client.wake_up(Some(vec!["kv_cache".to_string()])).await.expect("wake up"));
shutdown_mock(client, shutdown, task).await;
}
@@ -26,6 +26,7 @@ const EXCLUDED_HANDLERS: &[&str] = &[
"/reset_encoder_cache",
"/collective_rpc",
"/sleep",
"/release_kv_cache_memory",
"/wake_up",
"/is_sleeping",
];
+4
View File
@@ -142,6 +142,10 @@ fn build_router_with_options(
.route("/weight_info", get(weight_transfer::weight_info))
.route("/abort_requests", post(abort_requests::abort_requests))
.route("/sleep", post(sleep::sleep))
.route(
"/release_kv_cache_memory",
post(sleep::release_kv_cache_memory),
)
.route("/wake_up", post(sleep::wake_up))
.route("/is_sleeping", get(sleep::is_sleeping))
.route("/pause", post(pause::pause))
+26 -11
View File
@@ -5,7 +5,7 @@ use std::sync::Arc;
use axum::Json;
use axum::extract::rejection::QueryRejection;
use axum::extract::{Query, State};
use axum::extract::{Query, RawQuery, State};
use axum::http::StatusCode;
use serde::{Deserialize, Serialize};
use vllm_engine_core_client::protocol::utility::PauseMode;
@@ -27,12 +27,6 @@ pub(crate) struct SleepParams {
mode: PauseMode,
}
#[derive(Debug, Default, Deserialize)]
pub(crate) struct WakeUpParams {
#[serde(default)]
tags: Option<Vec<String>>,
}
const fn default_sleep_level() -> u32 {
1
}
@@ -57,14 +51,35 @@ pub async fn sleep(
Ok(StatusCode::OK)
}
/// Wake the engine from sleep mode.
pub async fn wake_up(
/// Release KV cache memory while keeping model weights resident.
pub async fn release_kv_cache_memory(
State(state): State<Arc<AppState>>,
Query(params): Query<WakeUpParams>,
) -> Result<StatusCode, ApiError> {
state
.engine_core_client()
.wake_up(params.tags)
.release_kv_cache_memory()
.await
.map_err(|error| utility_call_error("release_kv_cache_memory", error))?;
Ok(StatusCode::OK)
}
/// Wake the engine from sleep mode.
pub async fn wake_up(
State(state): State<Arc<AppState>>,
RawQuery(query): RawQuery,
) -> Result<StatusCode, ApiError> {
let tags = query.and_then(|query| {
let tags = url::form_urlencoded::parse(query.as_bytes())
.filter(|(key, _)| key == "tags")
.map(|(_, value)| value.into_owned())
.collect::<Vec<_>>();
(!tags.is_empty()).then_some(tags)
});
state
.engine_core_client()
.wake_up(tags)
.await
.map_err(|error| utility_call_error("wake_up", error))?;
+87 -2
View File
@@ -5805,7 +5805,46 @@ async fn sleep_route_uses_python_compatible_default_query_values() {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn wake_up_route_without_tags_sends_none() {
async fn release_kv_cache_memory_route_sends_expected_utility_call() {
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("release_kv_cache_memory"));
assert_eq!(array[3], Value::Array(Vec::new()));
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
})
})
.await;
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/release_kv_cache_memory")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn wake_up_route_without_tags_accepts_bool_result() {
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
@@ -5818,7 +5857,7 @@ async fn wake_up_route_without_tags_sends_none() {
assert_eq!(array[2], Value::from("wake_up"));
assert_eq!(array[3], Value::Array(vec![Value::Nil]));
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
})
})
.await;
@@ -5842,6 +5881,51 @@ async fn wake_up_route_without_tags_sends_none() {
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn wake_up_route_accepts_repeated_tags() {
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("wake_up"));
assert_eq!(
array[3],
Value::Array(vec![Value::Array(vec![
Value::from("weights"),
Value::from("kv_cache"),
])])
);
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
})
})
.await;
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/wake_up?tags=weights&tags%5B%5D=ignored&tags=kv_cache")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn is_sleeping_route_returns_json_payload() {
@@ -6160,6 +6244,7 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() {
for (method, uri) in [
("GET", "/is_sleeping"),
("POST", "/sleep"),
("POST", "/release_kv_cache_memory"),
("POST", "/wake_up"),
("GET", "/is_paused"),
("POST", "/pause"),