[Frontend][Core] Add release_kv_cache_memory() API (#44890)

Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: AlanFokCo <alanfok2868@gmail.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com>
Co-authored-by: AlanFokCo <alanfok2868@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Dakai An
2026-09-18 07:12:12 +08:00
committed by GitHub
co-authored by AlanFokCo aoshen02 OpenAI Codex
parent a524f80b1f
commit 092bdd6d57
24 changed files with 574 additions and 44 deletions
+75
View File
@@ -1,7 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import multiprocessing as mp
from concurrent.futures import ThreadPoolExecutor
from functools import partial
from types import SimpleNamespace
from unittest.mock import Mock
import numpy as np
import pytest
@@ -34,6 +37,8 @@ from vllm.multimodal.inputs import (
PlaceholderRange,
)
from vllm.multimodal.processing import PromptInsertion
from vllm.renderers.base import BaseRenderer
from vllm.utils.async_utils import make_async
from vllm.utils.mem_constants import GiB_bytes, MiB_bytes
from ..utils import create_new_process_for_each_test
@@ -755,6 +760,76 @@ def test_processor_cache_shared_across_loras():
assert feature_lora_b.data == item_data
@pytest.mark.parametrize("use_async", [False, True])
@pytest.mark.parametrize(
"release_error",
[
None,
"requires a completed pause first",
"requires all executor memory to be resident",
],
ids=["released", "not-paused", "nonresident-memory"],
)
@pytest.mark.asyncio
async def test_release_kv_cache_resends_mm_payload(use_async, release_error):
"""Release must not leave a sender hit pointing at a cleared receiver."""
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.engine.llm_engine import LLMEngine
model_config = SimpleNamespace(
get_multimodal_config=lambda: MultiModalConfig(mm_processor_cache_gb=1)
)
sender = MultiModalProcessorSenderCache(model_config)
receiver = MultiModalReceiverCache(model_config)
item = _dummy_item({"pixel_values": 16})
mm_hash = "image_A"
payload, _ = sender.get_and_update_item((item, []), mm_hash)
assert receiver.get_and_update_item(payload, mm_hash) is item
assert sender.get_and_update_item(None, mm_hash)[0] is None
renderer = SimpleNamespace(mm_processor_cache=sender, _mm_cache_stats=None)
renderer.clear_mm_cache = partial(BaseRenderer.clear_mm_cache, renderer)
def release():
if release_error:
raise RuntimeError(release_error)
receiver.clear_cache()
with ThreadPoolExecutor(max_workers=1) as executor:
renderer._clear_mm_cache_async = make_async(
renderer.clear_mm_cache, executor=executor
)
renderer.clear_mm_cache_async = partial(
BaseRenderer.clear_mm_cache_async, renderer
)
engine = SimpleNamespace(
renderer=renderer,
engine_core=SimpleNamespace(
release_kv_cache_memory=release,
release_kv_cache_memory_async=make_async(release, executor=executor),
),
logger_manager=Mock(),
)
async def call_release():
if use_async:
await AsyncLLM.release_kv_cache_memory(engine)
else:
LLMEngine.release_kv_cache_memory(engine)
if release_error:
with pytest.raises(RuntimeError, match=release_error):
await call_release()
engine.logger_manager.record_sleep_state.assert_not_called()
else:
await call_release()
engine.logger_manager.record_sleep_state.assert_called_once_with(1, 0)
payload, _ = sender.get_and_update_item((item, []), mm_hash)
assert payload is item
assert receiver.get_and_update_item(payload, mm_hash) is item
_SLEEP_VISION_PROMPT = (
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
"\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"