[Frontend][Core] Add release_kv_cache_memory() API (#44890)

Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: AlanFokCo <alanfok2868@gmail.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com>
Co-authored-by: AlanFokCo <alanfok2868@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Dakai An
2026-09-18 07:12:12 +08:00
committed by GitHub
co-authored by AlanFokCo aoshen02 OpenAI Codex
parent a524f80b1f
commit 092bdd6d57
24 changed files with 574 additions and 44 deletions
+24
View File
@@ -23,6 +23,7 @@ from vllm.config import (
from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_default_torch_num_threads
from vllm.v1.core.sched.interface import PauseState
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc
from vllm.v1.executor.abstract import Executor
@@ -667,6 +668,29 @@ def _pausable_engine_core_proc() -> EngineCoreProc:
return core
@pytest.mark.parametrize(
"pause_state,has_requests,has_batches",
[
pytest.param(PauseState.UNPAUSED, False, False, id="not-paused"),
pytest.param(PauseState.PAUSED_ALL, True, False, id="pending-requests"),
pytest.param(PauseState.PAUSED_ALL, False, True, id="pending-batches"),
],
)
def test_kv_cache_release_rejects_unsafe_state(pause_state, has_requests, has_batches):
"""Reject release before touching caches or memory if work can still use KV."""
core = _pausable_engine_core_proc()
core.scheduler.pause_state = pause_state
core.scheduler.has_requests.return_value = has_requests
core.batch_queue = [object()] if has_batches else None
core._reset_caches = MagicMock()
with pytest.raises(RuntimeError, match="requires a completed pause"):
core.release_kv_cache_memory()
core._reset_caches.assert_not_called()
core.model_executor.discard.assert_not_called()
@pytest.mark.parametrize("deferred", [False, True])
def test_pause_synchronizes_device_before_cache_reset(deferred: bool):
"""A resolved pause promises an idle device: the barrier must run before