[Frontend][Core] Add release_kv_cache_memory() API (#44890)
Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: AlanFokCo <alanfok2868@gmail.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com> Co-authored-by: AlanFokCo <alanfok2868@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
AlanFokCo
aoshen02
OpenAI Codex
parent
a524f80b1f
commit
092bdd6d57
@@ -73,6 +73,13 @@ class SleepModeBackend(ABC):
|
||||
(suspended) engine from a healthy-serving one (see RFC #34303)."""
|
||||
return self._state
|
||||
|
||||
def discard(self, tags: tuple[str, ...]) -> None:
|
||||
"""Discard selected tagged allocations without preserving contents."""
|
||||
raise RuntimeError(
|
||||
f"Sleep-mode backend {type(self).__name__!r} does not support "
|
||||
"selective discard."
|
||||
)
|
||||
|
||||
# -- Capability introspection (no instance required) --
|
||||
|
||||
@classmethod
|
||||
@@ -132,6 +139,13 @@ class CuMemBackend(SleepModeBackend):
|
||||
allocator.wake_up(tags)
|
||||
self._state = "RUNNING"
|
||||
|
||||
def discard(self, tags: tuple[str, ...]) -> None:
|
||||
from vllm.device_allocator import get_mem_allocator_instance
|
||||
|
||||
allocator = get_mem_allocator_instance()
|
||||
allocator.discard(tags)
|
||||
self._state = "SUSPENDED"
|
||||
|
||||
@classmethod
|
||||
def preserves_communicators(cls) -> bool:
|
||||
# Comm identity/topology survive memory suspension; no reinit needed.
|
||||
|
||||
Reference in New Issue
Block a user