[Frontend][Core] Add release_kv_cache_memory() API (#44890)
Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: AlanFokCo <alanfok2868@gmail.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com> Co-authored-by: AlanFokCo <alanfok2868@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
AlanFokCo
aoshen02
OpenAI Codex
parent
a524f80b1f
commit
092bdd6d57
@@ -95,12 +95,10 @@ fn utility_response(
|
||||
"get_supported_tasks" => utility_envelope(vec!["generate"]),
|
||||
"is_sleeping" => utility_envelope(false),
|
||||
"reset_prefix_cache" => utility_envelope(true),
|
||||
"reset_mm_cache"
|
||||
| "reset_encoder_cache"
|
||||
| "profile"
|
||||
| "sleep"
|
||||
| "wake_up"
|
||||
| "execute_dummy_batch" => utility_envelope(()),
|
||||
"wake_up" => utility_envelope(true),
|
||||
"reset_mm_cache" | "reset_encoder_cache" | "profile" | "sleep" | "execute_dummy_batch" => {
|
||||
utility_envelope(())
|
||||
}
|
||||
_ => utility_envelope(Value::Nil),
|
||||
}?;
|
||||
|
||||
|
||||
@@ -195,6 +195,7 @@ async fn utility_requests_return_minimal_success_responses() {
|
||||
assert!(client.reset_prefix_cache(false, false).await.expect("reset prefix cache"));
|
||||
client.reset_mm_cache().await.expect("reset mm cache");
|
||||
client.reset_encoder_cache().await.expect("reset encoder cache");
|
||||
assert!(client.wake_up(Some(vec!["kv_cache".to_string()])).await.expect("wake up"));
|
||||
|
||||
shutdown_mock(client, shutdown, task).await;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user