[Frontend][Core] Add release_kv_cache_memory() API (#44890)

Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: AlanFokCo <alanfok2868@gmail.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com>
Co-authored-by: AlanFokCo <alanfok2868@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Dakai An
2026-09-18 07:12:12 +08:00
committed by GitHub
co-authored by AlanFokCo aoshen02 OpenAI Codex
parent a524f80b1f
commit 092bdd6d57
24 changed files with 574 additions and 44 deletions
+13 -4
View File
@@ -911,13 +911,22 @@ impl EngineCoreClient {
Ok(())
}
/// Wake the engine from sleep, optionally limiting the wake-up to specific
/// tags.
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<()> {
self.call_utility::<(), _>("wake_up", (tags,)).await?;
/// Release KV cache memory while keeping model weights resident.
pub async fn release_kv_cache_memory(&self) -> Result<()> {
self.call_utility::<(), _>("release_kv_cache_memory", ()).await?;
Ok(())
}
/// Wake the engine from sleep, optionally limiting the wake-up to specific
/// tags, and return whether every engine is fully awake.
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<bool> {
Ok(self
.call_utility::<bool, _>("wake_up", (tags,))
.await?
.into_iter()
.all(|fully_awake| fully_awake))
}
/// Pause the scheduler so generation can be halted
pub async fn pause_scheduler(&self, mode: PauseMode, clear_cache: bool) -> Result<()> {
self.call_utility::<(), _>("pause_scheduler", (mode, clear_cache)).await?;