[Frontend][Core] Add release_kv_cache_memory() API (#44890)

Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: AlanFokCo <alanfok2868@gmail.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com>
Co-authored-by: AlanFokCo <alanfok2868@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Dakai An
2026-09-18 07:12:12 +08:00
committed by GitHub
co-authored by AlanFokCo aoshen02 OpenAI Codex
parent a524f80b1f
commit 092bdd6d57
24 changed files with 574 additions and 44 deletions
@@ -73,6 +73,13 @@ class SleepModeBackend(ABC):
(suspended) engine from a healthy-serving one (see RFC #34303)."""
return self._state
def discard(self, tags: tuple[str, ...]) -> None:
"""Discard selected tagged allocations without preserving contents."""
raise RuntimeError(
f"Sleep-mode backend {type(self).__name__!r} does not support "
"selective discard."
)
# -- Capability introspection (no instance required) --
@classmethod
@@ -132,6 +139,13 @@ class CuMemBackend(SleepModeBackend):
allocator.wake_up(tags)
self._state = "RUNNING"
def discard(self, tags: tuple[str, ...]) -> None:
from vllm.device_allocator import get_mem_allocator_instance
allocator = get_mem_allocator_instance()
allocator.discard(tags)
self._state = "SUSPENDED"
@classmethod
def preserves_communicators(cls) -> bool:
# Comm identity/topology survive memory suspension; no reinit needed.