[Frontend][Core] Add release_kv_cache_memory() API (#44890)
Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: AlanFokCo <alanfok2868@gmail.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com> Co-authored-by: AlanFokCo <alanfok2868@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
AlanFokCo
aoshen02
OpenAI Codex
parent
a524f80b1f
commit
092bdd6d57
@@ -911,13 +911,22 @@ impl EngineCoreClient {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep, optionally limiting the wake-up to specific
|
||||
/// tags.
|
||||
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<()> {
|
||||
self.call_utility::<(), _>("wake_up", (tags,)).await?;
|
||||
/// Release KV cache memory while keeping model weights resident.
|
||||
pub async fn release_kv_cache_memory(&self) -> Result<()> {
|
||||
self.call_utility::<(), _>("release_kv_cache_memory", ()).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep, optionally limiting the wake-up to specific
|
||||
/// tags, and return whether every engine is fully awake.
|
||||
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<bool> {
|
||||
Ok(self
|
||||
.call_utility::<bool, _>("wake_up", (tags,))
|
||||
.await?
|
||||
.into_iter()
|
||||
.all(|fully_awake| fully_awake))
|
||||
}
|
||||
|
||||
/// Pause the scheduler so generation can be halted
|
||||
pub async fn pause_scheduler(&self, mode: PauseMode, clear_cache: bool) -> Result<()> {
|
||||
self.call_utility::<(), _>("pause_scheduler", (mode, clear_cache)).await?;
|
||||
|
||||
Reference in New Issue
Block a user