[Frontend][Core] Add release_kv_cache_memory() API (#44890)

Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: AlanFokCo <alanfok2868@gmail.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com>
Co-authored-by: AlanFokCo <alanfok2868@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Dakai An
2026-09-18 07:12:12 +08:00
committed by GitHub
co-authored by AlanFokCo aoshen02 OpenAI Codex
parent a524f80b1f
commit 092bdd6d57
24 changed files with 574 additions and 44 deletions
@@ -26,6 +26,7 @@ const EXCLUDED_HANDLERS: &[&str] = &[
"/reset_encoder_cache",
"/collective_rpc",
"/sleep",
"/release_kv_cache_memory",
"/wake_up",
"/is_sleeping",
];
+4
View File
@@ -142,6 +142,10 @@ fn build_router_with_options(
.route("/weight_info", get(weight_transfer::weight_info))
.route("/abort_requests", post(abort_requests::abort_requests))
.route("/sleep", post(sleep::sleep))
.route(
"/release_kv_cache_memory",
post(sleep::release_kv_cache_memory),
)
.route("/wake_up", post(sleep::wake_up))
.route("/is_sleeping", get(sleep::is_sleeping))
.route("/pause", post(pause::pause))
+26 -11
View File
@@ -5,7 +5,7 @@ use std::sync::Arc;
use axum::Json;
use axum::extract::rejection::QueryRejection;
use axum::extract::{Query, State};
use axum::extract::{Query, RawQuery, State};
use axum::http::StatusCode;
use serde::{Deserialize, Serialize};
use vllm_engine_core_client::protocol::utility::PauseMode;
@@ -27,12 +27,6 @@ pub(crate) struct SleepParams {
mode: PauseMode,
}
#[derive(Debug, Default, Deserialize)]
pub(crate) struct WakeUpParams {
#[serde(default)]
tags: Option<Vec<String>>,
}
const fn default_sleep_level() -> u32 {
1
}
@@ -57,14 +51,35 @@ pub async fn sleep(
Ok(StatusCode::OK)
}
/// Wake the engine from sleep mode.
pub async fn wake_up(
/// Release KV cache memory while keeping model weights resident.
pub async fn release_kv_cache_memory(
State(state): State<Arc<AppState>>,
Query(params): Query<WakeUpParams>,
) -> Result<StatusCode, ApiError> {
state
.engine_core_client()
.wake_up(params.tags)
.release_kv_cache_memory()
.await
.map_err(|error| utility_call_error("release_kv_cache_memory", error))?;
Ok(StatusCode::OK)
}
/// Wake the engine from sleep mode.
pub async fn wake_up(
State(state): State<Arc<AppState>>,
RawQuery(query): RawQuery,
) -> Result<StatusCode, ApiError> {
let tags = query.and_then(|query| {
let tags = url::form_urlencoded::parse(query.as_bytes())
.filter(|(key, _)| key == "tags")
.map(|(_, value)| value.into_owned())
.collect::<Vec<_>>();
(!tags.is_empty()).then_some(tags)
});
state
.engine_core_client()
.wake_up(tags)
.await
.map_err(|error| utility_call_error("wake_up", error))?;
+87 -2
View File
@@ -5805,7 +5805,46 @@ async fn sleep_route_uses_python_compatible_default_query_values() {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn wake_up_route_without_tags_sends_none() {
async fn release_kv_cache_memory_route_sends_expected_utility_call() {
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("release_kv_cache_memory"));
assert_eq!(array[3], Value::Array(Vec::new()));
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
})
})
.await;
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/release_kv_cache_memory")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn wake_up_route_without_tags_accepts_bool_result() {
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
@@ -5818,7 +5857,7 @@ async fn wake_up_route_without_tags_sends_none() {
assert_eq!(array[2], Value::from("wake_up"));
assert_eq!(array[3], Value::Array(vec![Value::Nil]));
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
})
})
.await;
@@ -5842,6 +5881,51 @@ async fn wake_up_route_without_tags_sends_none() {
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn wake_up_route_accepts_repeated_tags() {
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("wake_up"));
assert_eq!(
array[3],
Value::Array(vec![Value::Array(vec![
Value::from("weights"),
Value::from("kv_cache"),
])])
);
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
})
})
.await;
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/wake_up?tags=weights&tags%5B%5D=ignored&tags=kv_cache")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
assert!(body.is_empty());
engine_task.await.expect("mock engine task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn is_sleeping_route_returns_json_payload() {
@@ -6160,6 +6244,7 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() {
for (method, uri) in [
("GET", "/is_sleeping"),
("POST", "/sleep"),
("POST", "/release_kv_cache_memory"),
("POST", "/wake_up"),
("GET", "/is_paused"),
("POST", "/pause"),