[Frontend][Core] Add release_kv_cache_memory() API (#44890)
Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: AlanFokCo <alanfok2868@gmail.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com> Co-authored-by: AlanFokCo <alanfok2868@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
AlanFokCo
aoshen02
OpenAI Codex
parent
a524f80b1f
commit
092bdd6d57
@@ -911,13 +911,22 @@ impl EngineCoreClient {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep, optionally limiting the wake-up to specific
|
||||
/// tags.
|
||||
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<()> {
|
||||
self.call_utility::<(), _>("wake_up", (tags,)).await?;
|
||||
/// Release KV cache memory while keeping model weights resident.
|
||||
pub async fn release_kv_cache_memory(&self) -> Result<()> {
|
||||
self.call_utility::<(), _>("release_kv_cache_memory", ()).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep, optionally limiting the wake-up to specific
|
||||
/// tags, and return whether every engine is fully awake.
|
||||
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<bool> {
|
||||
Ok(self
|
||||
.call_utility::<bool, _>("wake_up", (tags,))
|
||||
.await?
|
||||
.into_iter()
|
||||
.all(|fully_awake| fully_awake))
|
||||
}
|
||||
|
||||
/// Pause the scheduler so generation can be halted
|
||||
pub async fn pause_scheduler(&self, mode: PauseMode, clear_cache: bool) -> Result<()> {
|
||||
self.call_utility::<(), _>("pause_scheduler", (mode, clear_cache)).await?;
|
||||
|
||||
@@ -95,12 +95,10 @@ fn utility_response(
|
||||
"get_supported_tasks" => utility_envelope(vec!["generate"]),
|
||||
"is_sleeping" => utility_envelope(false),
|
||||
"reset_prefix_cache" => utility_envelope(true),
|
||||
"reset_mm_cache"
|
||||
| "reset_encoder_cache"
|
||||
| "profile"
|
||||
| "sleep"
|
||||
| "wake_up"
|
||||
| "execute_dummy_batch" => utility_envelope(()),
|
||||
"wake_up" => utility_envelope(true),
|
||||
"reset_mm_cache" | "reset_encoder_cache" | "profile" | "sleep" | "execute_dummy_batch" => {
|
||||
utility_envelope(())
|
||||
}
|
||||
_ => utility_envelope(Value::Nil),
|
||||
}?;
|
||||
|
||||
|
||||
@@ -195,6 +195,7 @@ async fn utility_requests_return_minimal_success_responses() {
|
||||
assert!(client.reset_prefix_cache(false, false).await.expect("reset prefix cache"));
|
||||
client.reset_mm_cache().await.expect("reset mm cache");
|
||||
client.reset_encoder_cache().await.expect("reset encoder cache");
|
||||
assert!(client.wake_up(Some(vec!["kv_cache".to_string()])).await.expect("wake up"));
|
||||
|
||||
shutdown_mock(client, shutdown, task).await;
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ const EXCLUDED_HANDLERS: &[&str] = &[
|
||||
"/reset_encoder_cache",
|
||||
"/collective_rpc",
|
||||
"/sleep",
|
||||
"/release_kv_cache_memory",
|
||||
"/wake_up",
|
||||
"/is_sleeping",
|
||||
];
|
||||
|
||||
@@ -142,6 +142,10 @@ fn build_router_with_options(
|
||||
.route("/weight_info", get(weight_transfer::weight_info))
|
||||
.route("/abort_requests", post(abort_requests::abort_requests))
|
||||
.route("/sleep", post(sleep::sleep))
|
||||
.route(
|
||||
"/release_kv_cache_memory",
|
||||
post(sleep::release_kv_cache_memory),
|
||||
)
|
||||
.route("/wake_up", post(sleep::wake_up))
|
||||
.route("/is_sleeping", get(sleep::is_sleeping))
|
||||
.route("/pause", post(pause::pause))
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::rejection::QueryRejection;
|
||||
use axum::extract::{Query, State};
|
||||
use axum::extract::{Query, RawQuery, State};
|
||||
use axum::http::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use vllm_engine_core_client::protocol::utility::PauseMode;
|
||||
@@ -27,12 +27,6 @@ pub(crate) struct SleepParams {
|
||||
mode: PauseMode,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
pub(crate) struct WakeUpParams {
|
||||
#[serde(default)]
|
||||
tags: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
const fn default_sleep_level() -> u32 {
|
||||
1
|
||||
}
|
||||
@@ -57,14 +51,35 @@ pub async fn sleep(
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep mode.
|
||||
pub async fn wake_up(
|
||||
/// Release KV cache memory while keeping model weights resident.
|
||||
pub async fn release_kv_cache_memory(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Query(params): Query<WakeUpParams>,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
state
|
||||
.engine_core_client()
|
||||
.wake_up(params.tags)
|
||||
.release_kv_cache_memory()
|
||||
.await
|
||||
.map_err(|error| utility_call_error("release_kv_cache_memory", error))?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep mode.
|
||||
pub async fn wake_up(
|
||||
State(state): State<Arc<AppState>>,
|
||||
RawQuery(query): RawQuery,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
let tags = query.and_then(|query| {
|
||||
let tags = url::form_urlencoded::parse(query.as_bytes())
|
||||
.filter(|(key, _)| key == "tags")
|
||||
.map(|(_, value)| value.into_owned())
|
||||
.collect::<Vec<_>>();
|
||||
(!tags.is_empty()).then_some(tags)
|
||||
});
|
||||
|
||||
state
|
||||
.engine_core_client()
|
||||
.wake_up(tags)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("wake_up", error))?;
|
||||
|
||||
|
||||
@@ -5805,7 +5805,46 @@ async fn sleep_route_uses_python_compatible_default_query_values() {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn wake_up_route_without_tags_sends_none() {
|
||||
async fn release_kv_cache_memory_route_sends_expected_utility_call() {
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
|
||||
assert_eq!(array[2], Value::from("release_kv_cache_memory"));
|
||||
assert_eq!(array[3], Value::Array(Vec::new()));
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/release_kv_cache_memory")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn wake_up_route_without_tags_accepts_bool_result() {
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
@@ -5818,7 +5857,7 @@ async fn wake_up_route_without_tags_sends_none() {
|
||||
assert_eq!(array[2], Value::from("wake_up"));
|
||||
assert_eq!(array[3], Value::Array(vec![Value::Nil]));
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_none_result())).await;
|
||||
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
@@ -5842,6 +5881,51 @@ async fn wake_up_route_without_tags_sends_none() {
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn wake_up_route_accepts_repeated_tags() {
|
||||
let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
|
||||
assert_eq!(array[2], Value::from("wake_up"));
|
||||
assert_eq!(
|
||||
array[3],
|
||||
Value::Array(vec![Value::Array(vec![
|
||||
Value::from("weights"),
|
||||
Value::from("kv_cache"),
|
||||
])])
|
||||
);
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/wake_up?tags=weights&tags%5B%5D=ignored&tags=kv_cache")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert!(body.is_empty());
|
||||
engine_task.await.expect("mock engine task");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn is_sleeping_route_returns_json_payload() {
|
||||
@@ -6160,6 +6244,7 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() {
|
||||
for (method, uri) in [
|
||||
("GET", "/is_sleeping"),
|
||||
("POST", "/sleep"),
|
||||
("POST", "/release_kv_cache_memory"),
|
||||
("POST", "/wake_up"),
|
||||
("GET", "/is_paused"),
|
||||
("POST", "/pause"),
|
||||
|
||||
Reference in New Issue
Block a user