[Frontend][Core] Add release_kv_cache_memory() API (#44890)
Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: AlanFokCo <alanfok2868@gmail.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com> Co-authored-by: AlanFokCo <alanfok2868@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
AlanFokCo
aoshen02
OpenAI Codex
parent
a524f80b1f
commit
092bdd6d57
@@ -26,6 +26,7 @@ const EXCLUDED_HANDLERS: &[&str] = &[
|
||||
"/reset_encoder_cache",
|
||||
"/collective_rpc",
|
||||
"/sleep",
|
||||
"/release_kv_cache_memory",
|
||||
"/wake_up",
|
||||
"/is_sleeping",
|
||||
];
|
||||
|
||||
@@ -142,6 +142,10 @@ fn build_router_with_options(
|
||||
.route("/weight_info", get(weight_transfer::weight_info))
|
||||
.route("/abort_requests", post(abort_requests::abort_requests))
|
||||
.route("/sleep", post(sleep::sleep))
|
||||
.route(
|
||||
"/release_kv_cache_memory",
|
||||
post(sleep::release_kv_cache_memory),
|
||||
)
|
||||
.route("/wake_up", post(sleep::wake_up))
|
||||
.route("/is_sleeping", get(sleep::is_sleeping))
|
||||
.route("/pause", post(pause::pause))
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::rejection::QueryRejection;
|
||||
use axum::extract::{Query, State};
|
||||
use axum::extract::{Query, RawQuery, State};
|
||||
use axum::http::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use vllm_engine_core_client::protocol::utility::PauseMode;
|
||||
@@ -27,12 +27,6 @@ pub(crate) struct SleepParams {
|
||||
mode: PauseMode,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
pub(crate) struct WakeUpParams {
|
||||
#[serde(default)]
|
||||
tags: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
const fn default_sleep_level() -> u32 {
|
||||
1
|
||||
}
|
||||
@@ -57,14 +51,35 @@ pub async fn sleep(
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep mode.
|
||||
pub async fn wake_up(
|
||||
/// Release KV cache memory while keeping model weights resident.
|
||||
pub async fn release_kv_cache_memory(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Query(params): Query<WakeUpParams>,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
state
|
||||
.engine_core_client()
|
||||
.wake_up(params.tags)
|
||||
.release_kv_cache_memory()
|
||||
.await
|
||||
.map_err(|error| utility_call_error("release_kv_cache_memory", error))?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep mode.
|
||||
pub async fn wake_up(
|
||||
State(state): State<Arc<AppState>>,
|
||||
RawQuery(query): RawQuery,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
let tags = query.and_then(|query| {
|
||||
let tags = url::form_urlencoded::parse(query.as_bytes())
|
||||
.filter(|(key, _)| key == "tags")
|
||||
.map(|(_, value)| value.into_owned())
|
||||
.collect::<Vec<_>>();
|
||||
(!tags.is_empty()).then_some(tags)
|
||||
});
|
||||
|
||||
state
|
||||
.engine_core_client()
|
||||
.wake_up(tags)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("wake_up", error))?;
|
||||
|
||||
|
||||
@@ -5805,7 +5805,46 @@ async fn sleep_route_uses_python_compatible_default_query_values() {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn wake_up_route_without_tags_sends_none() {
|
||||
async fn release_kv_cache_memory_route_sends_expected_utility_call() {
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
|
||||
assert_eq!(array[2], Value::from("release_kv_cache_memory"));
|
||||
assert_eq!(array[3], Value::Array(Vec::new()));
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/release_kv_cache_memory")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn wake_up_route_without_tags_accepts_bool_result() {
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
@@ -5818,7 +5857,7 @@ async fn wake_up_route_without_tags_sends_none() {
|
||||
assert_eq!(array[2], Value::from("wake_up"));
|
||||
assert_eq!(array[3], Value::Array(vec![Value::Nil]));
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
@@ -5842,6 +5881,51 @@ async fn wake_up_route_without_tags_sends_none() {
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn wake_up_route_accepts_repeated_tags() {
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
|
||||
assert_eq!(array[2], Value::from("wake_up"));
|
||||
assert_eq!(
|
||||
array[3],
|
||||
Value::Array(vec![Value::Array(vec![
|
||||
Value::from("weights"),
|
||||
Value::from("kv_cache"),
|
||||
])])
|
||||
);
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/wake_up?tags=weights&tags%5B%5D=ignored&tags=kv_cache")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn is_sleeping_route_returns_json_payload() {
|
||||
@@ -6160,6 +6244,7 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() {
|
||||
for (method, uri) in [
|
||||
("GET", "/is_sleeping"),
|
||||
("POST", "/sleep"),
|
||||
("POST", "/release_kv_cache_memory"),
|
||||
("POST", "/wake_up"),
|
||||
("GET", "/is_paused"),
|
||||
("POST", "/pause"),
|
||||
|
||||
Reference in New Issue
Block a user