[Frontend][Core] Add release_kv_cache_memory() API (#44890)
Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: AlanFokCo <alanfok2868@gmail.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com> Co-authored-by: AlanFokCo <alanfok2868@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
AlanFokCo
aoshen02
OpenAI Codex
parent
a524f80b1f
commit
092bdd6d57
@@ -1,7 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import multiprocessing as mp
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -34,6 +37,8 @@ from vllm.multimodal.inputs import (
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.multimodal.processing import PromptInsertion
|
||||
from vllm.renderers.base import BaseRenderer
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.mem_constants import GiB_bytes, MiB_bytes
|
||||
|
||||
from ..utils import create_new_process_for_each_test
|
||||
@@ -755,6 +760,76 @@ def test_processor_cache_shared_across_loras():
|
||||
assert feature_lora_b.data == item_data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_async", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"release_error",
|
||||
[
|
||||
None,
|
||||
"requires a completed pause first",
|
||||
"requires all executor memory to be resident",
|
||||
],
|
||||
ids=["released", "not-paused", "nonresident-memory"],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_kv_cache_resends_mm_payload(use_async, release_error):
|
||||
"""Release must not leave a sender hit pointing at a cleared receiver."""
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
model_config = SimpleNamespace(
|
||||
get_multimodal_config=lambda: MultiModalConfig(mm_processor_cache_gb=1)
|
||||
)
|
||||
sender = MultiModalProcessorSenderCache(model_config)
|
||||
receiver = MultiModalReceiverCache(model_config)
|
||||
item = _dummy_item({"pixel_values": 16})
|
||||
mm_hash = "image_A"
|
||||
payload, _ = sender.get_and_update_item((item, []), mm_hash)
|
||||
assert receiver.get_and_update_item(payload, mm_hash) is item
|
||||
assert sender.get_and_update_item(None, mm_hash)[0] is None
|
||||
|
||||
renderer = SimpleNamespace(mm_processor_cache=sender, _mm_cache_stats=None)
|
||||
renderer.clear_mm_cache = partial(BaseRenderer.clear_mm_cache, renderer)
|
||||
|
||||
def release():
|
||||
if release_error:
|
||||
raise RuntimeError(release_error)
|
||||
receiver.clear_cache()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
renderer._clear_mm_cache_async = make_async(
|
||||
renderer.clear_mm_cache, executor=executor
|
||||
)
|
||||
renderer.clear_mm_cache_async = partial(
|
||||
BaseRenderer.clear_mm_cache_async, renderer
|
||||
)
|
||||
engine = SimpleNamespace(
|
||||
renderer=renderer,
|
||||
engine_core=SimpleNamespace(
|
||||
release_kv_cache_memory=release,
|
||||
release_kv_cache_memory_async=make_async(release, executor=executor),
|
||||
),
|
||||
logger_manager=Mock(),
|
||||
)
|
||||
|
||||
async def call_release():
|
||||
if use_async:
|
||||
await AsyncLLM.release_kv_cache_memory(engine)
|
||||
else:
|
||||
LLMEngine.release_kv_cache_memory(engine)
|
||||
|
||||
if release_error:
|
||||
with pytest.raises(RuntimeError, match=release_error):
|
||||
await call_release()
|
||||
engine.logger_manager.record_sleep_state.assert_not_called()
|
||||
else:
|
||||
await call_release()
|
||||
engine.logger_manager.record_sleep_state.assert_called_once_with(1, 0)
|
||||
|
||||
payload, _ = sender.get_and_update_item((item, []), mm_hash)
|
||||
assert payload is item
|
||||
assert receiver.get_and_update_item(payload, mm_hash) is item
|
||||
|
||||
|
||||
_SLEEP_VISION_PROMPT = (
|
||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
|
||||
"\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
|
||||
|
||||
Reference in New Issue
Block a user