[Frontend][Core] Add release_kv_cache_memory() API (#44890)

Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: AlanFokCo <alanfok2868@gmail.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com>
Co-authored-by: AlanFokCo <alanfok2868@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Dakai An
2026-09-18 07:12:12 +08:00
committed by GitHub
co-authored by AlanFokCo aoshen02 OpenAI Codex
parent a524f80b1f
commit 092bdd6d57
24 changed files with 574 additions and 44 deletions
+85
View File
@@ -109,6 +109,91 @@ def test_basic_cumem():
assert torch.allclose(output, torch.ones_like(output) * 3)
@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch):
"""Rejected release and KV-only/full sleep cycles preserve greedy tokens."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
kv_cache_memory_bytes = 256 * 1024 * 1024
llm = LLM(
"Qwen/Qwen3-0.6B",
enable_sleep_mode=True,
enforce_eager=True,
kv_cache_memory_bytes=kv_cache_memory_bytes,
max_model_len=1024,
max_num_seqs=4,
gpu_memory_utilization=0.05,
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
def get_mapped_bytes(worker):
return mapped_usage(get_mem_allocator_instance())
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
# Utility RPC transports the engine error as a plain Exception.
with pytest.raises(Exception, match="requires a completed pause"):
llm.release_kv_cache_memory()
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
assert not llm.llm_engine.is_sleeping()
assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected
llm.sleep(level=0)
llm.release_kv_cache_memory()
mapped_after = llm.collective_rpc(get_mapped_bytes)[0]
assert mapped_before - mapped_after >= kv_cache_memory_bytes * 0.99
assert mapped_after > 0
assert llm.llm_engine.is_sleeping()
if full_sleep:
llm.sleep(level=full_sleep)
assert llm.collective_rpc(get_mapped_bytes)[0] == 0
llm.wake_up(tags=None if full_sleep else ["kv_cache"])
if full_sleep == 2:
llm.collective_rpc("reload_weights")
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
assert not llm.llm_engine.is_sleeping()
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
assert actual == expected
@pytest.mark.parametrize("first_level", [1, 2])
@pytest.mark.parametrize("second_level", [1, 2])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch):
"""Repeated sleep after KV-only wake preserves mappings and recoverability."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
llm = LLM(
"Qwen/Qwen3-0.6B",
enable_sleep_mode=True,
enforce_eager=True,
kv_cache_memory_bytes=256 * 1024 * 1024,
max_model_len=1024,
max_num_seqs=4,
gpu_memory_utilization=0.05,
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
def get_mapped_bytes(worker):
return mapped_usage(get_mem_allocator_instance())
llm.sleep(level=first_level)
llm.wake_up(tags=["kv_cache"])
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
assert mapped_before > 0
llm.sleep(level=second_level)
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
llm.wake_up(tags=["weights"])
if first_level == 2:
llm.collective_rpc("reload_weights")
assert not llm.llm_engine.is_sleeping()
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
assert actual == expected
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_discard_tags():
"""Test that discard(tags) selectively frees GPU memory for specific
+24 -1
View File
@@ -1,14 +1,37 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import AsyncMock
import pytest
import requests
from fastapi import FastAPI
from fastapi.testclient import TestClient
from prometheus_client.parser import text_string_to_metric_families
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.serve.dev.sleep.api_router import attach_router
MODEL_NAME = "meta-llama/Llama-3.2-1B"
@pytest.mark.cpu_test
@pytest.mark.parametrize("fails", [False, True])
def test_release_kv_cache_memory_route(fails):
app = FastAPI()
app.state.engine_client = AsyncMock()
release = app.state.engine_client.release_kv_cache_memory
if fails:
release.side_effect = RuntimeError("requires a completed pause first")
attach_router(app)
with TestClient(app, raise_server_exceptions=False) as client:
response = client.post("/release_kv_cache_memory")
assert response.status_code == (500 if fails else 200)
release.assert_awaited_once_with()
def test_sleep_mode():
# dtype, max-len etc set so that this can run in CI
args = [
@@ -63,7 +86,7 @@ def test_sleep_mode():
)
assert response.status_code == 200
# is sleeping should be false after waking up any part of the engine
# Partial wake keeps the engine sleeping.
response = requests.get(remote_server.url_for("is_sleeping"))
assert response.status_code == 200
assert response.json().get("is_sleeping") is True
+75
View File
@@ -1,7 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import multiprocessing as mp
from concurrent.futures import ThreadPoolExecutor
from functools import partial
from types import SimpleNamespace
from unittest.mock import Mock
import numpy as np
import pytest
@@ -34,6 +37,8 @@ from vllm.multimodal.inputs import (
PlaceholderRange,
)
from vllm.multimodal.processing import PromptInsertion
from vllm.renderers.base import BaseRenderer
from vllm.utils.async_utils import make_async
from vllm.utils.mem_constants import GiB_bytes, MiB_bytes
from ..utils import create_new_process_for_each_test
@@ -755,6 +760,76 @@ def test_processor_cache_shared_across_loras():
assert feature_lora_b.data == item_data
@pytest.mark.parametrize("use_async", [False, True])
@pytest.mark.parametrize(
"release_error",
[
None,
"requires a completed pause first",
"requires all executor memory to be resident",
],
ids=["released", "not-paused", "nonresident-memory"],
)
@pytest.mark.asyncio
async def test_release_kv_cache_resends_mm_payload(use_async, release_error):
"""Release must not leave a sender hit pointing at a cleared receiver."""
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.engine.llm_engine import LLMEngine
model_config = SimpleNamespace(
get_multimodal_config=lambda: MultiModalConfig(mm_processor_cache_gb=1)
)
sender = MultiModalProcessorSenderCache(model_config)
receiver = MultiModalReceiverCache(model_config)
item = _dummy_item({"pixel_values": 16})
mm_hash = "image_A"
payload, _ = sender.get_and_update_item((item, []), mm_hash)
assert receiver.get_and_update_item(payload, mm_hash) is item
assert sender.get_and_update_item(None, mm_hash)[0] is None
renderer = SimpleNamespace(mm_processor_cache=sender, _mm_cache_stats=None)
renderer.clear_mm_cache = partial(BaseRenderer.clear_mm_cache, renderer)
def release():
if release_error:
raise RuntimeError(release_error)
receiver.clear_cache()
with ThreadPoolExecutor(max_workers=1) as executor:
renderer._clear_mm_cache_async = make_async(
renderer.clear_mm_cache, executor=executor
)
renderer.clear_mm_cache_async = partial(
BaseRenderer.clear_mm_cache_async, renderer
)
engine = SimpleNamespace(
renderer=renderer,
engine_core=SimpleNamespace(
release_kv_cache_memory=release,
release_kv_cache_memory_async=make_async(release, executor=executor),
),
logger_manager=Mock(),
)
async def call_release():
if use_async:
await AsyncLLM.release_kv_cache_memory(engine)
else:
LLMEngine.release_kv_cache_memory(engine)
if release_error:
with pytest.raises(RuntimeError, match=release_error):
await call_release()
engine.logger_manager.record_sleep_state.assert_not_called()
else:
await call_release()
engine.logger_manager.record_sleep_state.assert_called_once_with(1, 0)
payload, _ = sender.get_and_update_item((item, []), mm_hash)
assert payload is item
assert receiver.get_and_update_item(payload, mm_hash) is item
_SLEEP_VISION_PROMPT = (
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
"\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
+76
View File
@@ -36,10 +36,13 @@ from vllm.utils.hashing import sha256
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
from vllm.v1.core.sched.interface import PauseState
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
from vllm.v1.engine import FinishReason
from vllm.v1.engine.core import EngineCore
from vllm.v1.executor.uniproc_executor import UniProcExecutor
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
@@ -1487,6 +1490,79 @@ def test_scheduler_reset_prefix_cache():
assert scheduler.waiting[i] == request
def test_kv_cache_release_after_keep_pause_preserves_requests():
"""A completed keep pause permits release and recomputation of live requests."""
scheduler = create_scheduler(max_num_seqs=1, enable_prefix_caching=True)
running, waiting = create_requests(num_requests=2)
for request in (running, waiting):
scheduler.add_request(request)
scheduler_output = scheduler.schedule()
scheduler.update_from_output(
scheduler_output,
ModelRunnerOutput(
req_ids=[running.request_id],
req_id_to_index={running.request_id: 0},
sampled_token_ids=[[1]],
),
)
assert scheduler.kv_cache_manager.get_block_ids(running.request_id)[0]
core = object.__new__(EngineCore)
core.scheduler = scheduler
core.model_executor = Mock(is_sleeping=False)
core.mm_receiver_cache = None
core.batch_queue = None
assert core.pause_scheduler(mode="keep", clear_cache=False) is None
assert scheduler.running == [running]
assert list(scheduler.waiting) == [waiting]
assert not scheduler.has_requests()
core.release_kv_cache_memory()
core.model_executor.discard.assert_called_once_with(("kv_cache",))
assert scheduler.requests == {
request.request_id: request for request in (running, waiting)
}
assert not scheduler.running
assert list(scheduler.waiting) == [running, waiting]
assert running.status == RequestStatus.PREEMPTED
assert running.num_computed_tokens == 0
assert list(running.output_token_ids) == [1]
assert not scheduler.kv_cache_manager.get_block_ids(running.request_id)[0]
assert scheduler.schedule().total_num_scheduled_tokens == 0
assert core.wake_up(tags=["kv_cache"])
resumed = scheduler.schedule()
assert resumed.num_scheduled_tokens == {running.request_id: running.num_tokens}
@pytest.mark.parametrize(
"sleeping_tags",
[{"weights", "kv_cache"}, {"weights"}, {"kv_cache"}],
ids=["full-sleep", "weights-only", "kv-only-or-repeated-release"],
)
def test_kv_cache_release_rejects_nonresident_memory(sleeping_tags):
"""Reject release without resetting caches or discarding more memory."""
core = object.__new__(EngineCore)
core.scheduler = create_scheduler()
core.scheduler.set_pause_state(PauseState.PAUSED_ALL)
core.batch_queue = None
core._reset_caches = Mock()
core.model_executor = object.__new__(UniProcExecutor)
core.model_executor.sleeping_tags = sleeping_tags.copy()
core.model_executor.collective_rpc = Mock()
with pytest.raises(
RuntimeError, match="requires all executor memory to be resident"
):
core.release_kv_cache_memory()
core._reset_caches.assert_not_called()
core.model_executor.collective_rpc.assert_not_called()
assert core.model_executor.sleeping_tags == sleeping_tags
def test_reset_connector_cache_no_connector_is_no_op_success():
"""``reset_connector_cache`` must return True when no connector is
configured.
+24
View File
@@ -23,6 +23,7 @@ from vllm.config import (
from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_default_torch_num_threads
from vllm.v1.core.sched.interface import PauseState
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc
from vllm.v1.executor.abstract import Executor
@@ -667,6 +668,29 @@ def _pausable_engine_core_proc() -> EngineCoreProc:
return core
@pytest.mark.parametrize(
"pause_state,has_requests,has_batches",
[
pytest.param(PauseState.UNPAUSED, False, False, id="not-paused"),
pytest.param(PauseState.PAUSED_ALL, True, False, id="pending-requests"),
pytest.param(PauseState.PAUSED_ALL, False, True, id="pending-batches"),
],
)
def test_kv_cache_release_rejects_unsafe_state(pause_state, has_requests, has_batches):
"""Reject release before touching caches or memory if work can still use KV."""
core = _pausable_engine_core_proc()
core.scheduler.pause_state = pause_state
core.scheduler.has_requests.return_value = has_requests
core.batch_queue = [object()] if has_batches else None
core._reset_caches = MagicMock()
with pytest.raises(RuntimeError, match="requires a completed pause"):
core.release_kv_cache_memory()
core._reset_caches.assert_not_called()
core.model_executor.discard.assert_not_called()
@pytest.mark.parametrize("deferred", [False, True])
def test_pause_synchronizes_device_before_cache_reset(deferred: bool):
"""A resolved pause promises an idle device: the barrier must run before