[Frontend][Core] Add release_kv_cache_memory() API (#44890)
Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: AlanFokCo <alanfok2868@gmail.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com> Co-authored-by: AlanFokCo <alanfok2868@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
AlanFokCo
aoshen02
OpenAI Codex
parent
a524f80b1f
commit
092bdd6d57
@@ -109,6 +109,91 @@ def test_basic_cumem():
|
||||
assert torch.allclose(output, torch.ones_like(output) * 3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch):
|
||||
"""Rejected release and KV-only/full sleep cycles preserve greedy tokens."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
kv_cache_memory_bytes = 256 * 1024 * 1024
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
enable_sleep_mode=True,
|
||||
enforce_eager=True,
|
||||
kv_cache_memory_bytes=kv_cache_memory_bytes,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=4,
|
||||
gpu_memory_utilization=0.05,
|
||||
)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
|
||||
def get_mapped_bytes(worker):
|
||||
return mapped_usage(get_mem_allocator_instance())
|
||||
|
||||
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
|
||||
# Utility RPC transports the engine error as a plain Exception.
|
||||
with pytest.raises(Exception, match="requires a completed pause"):
|
||||
llm.release_kv_cache_memory()
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected
|
||||
|
||||
llm.sleep(level=0)
|
||||
llm.release_kv_cache_memory()
|
||||
mapped_after = llm.collective_rpc(get_mapped_bytes)[0]
|
||||
assert mapped_before - mapped_after >= kv_cache_memory_bytes * 0.99
|
||||
assert mapped_after > 0
|
||||
assert llm.llm_engine.is_sleeping()
|
||||
|
||||
if full_sleep:
|
||||
llm.sleep(level=full_sleep)
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == 0
|
||||
llm.wake_up(tags=None if full_sleep else ["kv_cache"])
|
||||
if full_sleep == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_level", [1, 2])
|
||||
@pytest.mark.parametrize("second_level", [1, 2])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch):
|
||||
"""Repeated sleep after KV-only wake preserves mappings and recoverability."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
enable_sleep_mode=True,
|
||||
enforce_eager=True,
|
||||
kv_cache_memory_bytes=256 * 1024 * 1024,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=4,
|
||||
gpu_memory_utilization=0.05,
|
||||
)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
|
||||
def get_mapped_bytes(worker):
|
||||
return mapped_usage(get_mem_allocator_instance())
|
||||
|
||||
llm.sleep(level=first_level)
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
|
||||
assert mapped_before > 0
|
||||
llm.sleep(level=second_level)
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
|
||||
llm.wake_up(tags=["weights"])
|
||||
if first_level == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_discard_tags():
|
||||
"""Test that discard(tags) selectively frees GPU memory for specific
|
||||
|
||||
@@ -1,14 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.serve.dev.sleep.api_router import attach_router
|
||||
|
||||
MODEL_NAME = "meta-llama/Llama-3.2-1B"
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("fails", [False, True])
|
||||
def test_release_kv_cache_memory_route(fails):
|
||||
app = FastAPI()
|
||||
app.state.engine_client = AsyncMock()
|
||||
release = app.state.engine_client.release_kv_cache_memory
|
||||
if fails:
|
||||
release.side_effect = RuntimeError("requires a completed pause first")
|
||||
attach_router(app)
|
||||
|
||||
with TestClient(app, raise_server_exceptions=False) as client:
|
||||
response = client.post("/release_kv_cache_memory")
|
||||
|
||||
assert response.status_code == (500 if fails else 200)
|
||||
release.assert_awaited_once_with()
|
||||
|
||||
|
||||
def test_sleep_mode():
|
||||
# dtype, max-len etc set so that this can run in CI
|
||||
args = [
|
||||
@@ -63,7 +86,7 @@ def test_sleep_mode():
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# is sleeping should be false after waking up any part of the engine
|
||||
# Partial wake keeps the engine sleeping.
|
||||
response = requests.get(remote_server.url_for("is_sleeping"))
|
||||
assert response.status_code == 200
|
||||
assert response.json().get("is_sleeping") is True
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import multiprocessing as mp
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -34,6 +37,8 @@ from vllm.multimodal.inputs import (
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.multimodal.processing import PromptInsertion
|
||||
from vllm.renderers.base import BaseRenderer
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.mem_constants import GiB_bytes, MiB_bytes
|
||||
|
||||
from ..utils import create_new_process_for_each_test
|
||||
@@ -755,6 +760,76 @@ def test_processor_cache_shared_across_loras():
|
||||
assert feature_lora_b.data == item_data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_async", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"release_error",
|
||||
[
|
||||
None,
|
||||
"requires a completed pause first",
|
||||
"requires all executor memory to be resident",
|
||||
],
|
||||
ids=["released", "not-paused", "nonresident-memory"],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_kv_cache_resends_mm_payload(use_async, release_error):
|
||||
"""Release must not leave a sender hit pointing at a cleared receiver."""
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
model_config = SimpleNamespace(
|
||||
get_multimodal_config=lambda: MultiModalConfig(mm_processor_cache_gb=1)
|
||||
)
|
||||
sender = MultiModalProcessorSenderCache(model_config)
|
||||
receiver = MultiModalReceiverCache(model_config)
|
||||
item = _dummy_item({"pixel_values": 16})
|
||||
mm_hash = "image_A"
|
||||
payload, _ = sender.get_and_update_item((item, []), mm_hash)
|
||||
assert receiver.get_and_update_item(payload, mm_hash) is item
|
||||
assert sender.get_and_update_item(None, mm_hash)[0] is None
|
||||
|
||||
renderer = SimpleNamespace(mm_processor_cache=sender, _mm_cache_stats=None)
|
||||
renderer.clear_mm_cache = partial(BaseRenderer.clear_mm_cache, renderer)
|
||||
|
||||
def release():
|
||||
if release_error:
|
||||
raise RuntimeError(release_error)
|
||||
receiver.clear_cache()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
renderer._clear_mm_cache_async = make_async(
|
||||
renderer.clear_mm_cache, executor=executor
|
||||
)
|
||||
renderer.clear_mm_cache_async = partial(
|
||||
BaseRenderer.clear_mm_cache_async, renderer
|
||||
)
|
||||
engine = SimpleNamespace(
|
||||
renderer=renderer,
|
||||
engine_core=SimpleNamespace(
|
||||
release_kv_cache_memory=release,
|
||||
release_kv_cache_memory_async=make_async(release, executor=executor),
|
||||
),
|
||||
logger_manager=Mock(),
|
||||
)
|
||||
|
||||
async def call_release():
|
||||
if use_async:
|
||||
await AsyncLLM.release_kv_cache_memory(engine)
|
||||
else:
|
||||
LLMEngine.release_kv_cache_memory(engine)
|
||||
|
||||
if release_error:
|
||||
with pytest.raises(RuntimeError, match=release_error):
|
||||
await call_release()
|
||||
engine.logger_manager.record_sleep_state.assert_not_called()
|
||||
else:
|
||||
await call_release()
|
||||
engine.logger_manager.record_sleep_state.assert_called_once_with(1, 0)
|
||||
|
||||
payload, _ = sender.get_and_update_item((item, []), mm_hash)
|
||||
assert payload is item
|
||||
assert receiver.get_and_update_item(payload, mm_hash) is item
|
||||
|
||||
|
||||
_SLEEP_VISION_PROMPT = (
|
||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
|
||||
"\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
|
||||
|
||||
@@ -36,10 +36,13 @@ from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
|
||||
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
|
||||
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
|
||||
from vllm.v1.core.sched.interface import PauseState
|
||||
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.engine.core import EngineCore
|
||||
from vllm.v1.executor.uniproc_executor import UniProcExecutor
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
@@ -1487,6 +1490,79 @@ def test_scheduler_reset_prefix_cache():
|
||||
assert scheduler.waiting[i] == request
|
||||
|
||||
|
||||
def test_kv_cache_release_after_keep_pause_preserves_requests():
|
||||
"""A completed keep pause permits release and recomputation of live requests."""
|
||||
scheduler = create_scheduler(max_num_seqs=1, enable_prefix_caching=True)
|
||||
running, waiting = create_requests(num_requests=2)
|
||||
for request in (running, waiting):
|
||||
scheduler.add_request(request)
|
||||
scheduler_output = scheduler.schedule()
|
||||
scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[running.request_id],
|
||||
req_id_to_index={running.request_id: 0},
|
||||
sampled_token_ids=[[1]],
|
||||
),
|
||||
)
|
||||
assert scheduler.kv_cache_manager.get_block_ids(running.request_id)[0]
|
||||
|
||||
core = object.__new__(EngineCore)
|
||||
core.scheduler = scheduler
|
||||
core.model_executor = Mock(is_sleeping=False)
|
||||
core.mm_receiver_cache = None
|
||||
core.batch_queue = None
|
||||
|
||||
assert core.pause_scheduler(mode="keep", clear_cache=False) is None
|
||||
assert scheduler.running == [running]
|
||||
assert list(scheduler.waiting) == [waiting]
|
||||
assert not scheduler.has_requests()
|
||||
|
||||
core.release_kv_cache_memory()
|
||||
|
||||
core.model_executor.discard.assert_called_once_with(("kv_cache",))
|
||||
assert scheduler.requests == {
|
||||
request.request_id: request for request in (running, waiting)
|
||||
}
|
||||
assert not scheduler.running
|
||||
assert list(scheduler.waiting) == [running, waiting]
|
||||
assert running.status == RequestStatus.PREEMPTED
|
||||
assert running.num_computed_tokens == 0
|
||||
assert list(running.output_token_ids) == [1]
|
||||
assert not scheduler.kv_cache_manager.get_block_ids(running.request_id)[0]
|
||||
assert scheduler.schedule().total_num_scheduled_tokens == 0
|
||||
|
||||
assert core.wake_up(tags=["kv_cache"])
|
||||
resumed = scheduler.schedule()
|
||||
assert resumed.num_scheduled_tokens == {running.request_id: running.num_tokens}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sleeping_tags",
|
||||
[{"weights", "kv_cache"}, {"weights"}, {"kv_cache"}],
|
||||
ids=["full-sleep", "weights-only", "kv-only-or-repeated-release"],
|
||||
)
|
||||
def test_kv_cache_release_rejects_nonresident_memory(sleeping_tags):
|
||||
"""Reject release without resetting caches or discarding more memory."""
|
||||
core = object.__new__(EngineCore)
|
||||
core.scheduler = create_scheduler()
|
||||
core.scheduler.set_pause_state(PauseState.PAUSED_ALL)
|
||||
core.batch_queue = None
|
||||
core._reset_caches = Mock()
|
||||
core.model_executor = object.__new__(UniProcExecutor)
|
||||
core.model_executor.sleeping_tags = sleeping_tags.copy()
|
||||
core.model_executor.collective_rpc = Mock()
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError, match="requires all executor memory to be resident"
|
||||
):
|
||||
core.release_kv_cache_memory()
|
||||
|
||||
core._reset_caches.assert_not_called()
|
||||
core.model_executor.collective_rpc.assert_not_called()
|
||||
assert core.model_executor.sleeping_tags == sleeping_tags
|
||||
|
||||
|
||||
def test_reset_connector_cache_no_connector_is_no_op_success():
|
||||
"""``reset_connector_cache`` must return True when no connector is
|
||||
configured.
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.config import (
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_default_torch_num_threads
|
||||
from vllm.v1.core.sched.interface import PauseState
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
@@ -667,6 +668,29 @@ def _pausable_engine_core_proc() -> EngineCoreProc:
|
||||
return core
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pause_state,has_requests,has_batches",
|
||||
[
|
||||
pytest.param(PauseState.UNPAUSED, False, False, id="not-paused"),
|
||||
pytest.param(PauseState.PAUSED_ALL, True, False, id="pending-requests"),
|
||||
pytest.param(PauseState.PAUSED_ALL, False, True, id="pending-batches"),
|
||||
],
|
||||
)
|
||||
def test_kv_cache_release_rejects_unsafe_state(pause_state, has_requests, has_batches):
|
||||
"""Reject release before touching caches or memory if work can still use KV."""
|
||||
core = _pausable_engine_core_proc()
|
||||
core.scheduler.pause_state = pause_state
|
||||
core.scheduler.has_requests.return_value = has_requests
|
||||
core.batch_queue = [object()] if has_batches else None
|
||||
core._reset_caches = MagicMock()
|
||||
|
||||
with pytest.raises(RuntimeError, match="requires a completed pause"):
|
||||
core.release_kv_cache_memory()
|
||||
|
||||
core._reset_caches.assert_not_called()
|
||||
core.model_executor.discard.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deferred", [False, True])
|
||||
def test_pause_synchronizes_device_before_cache_reset(deferred: bool):
|
||||
"""A resolved pause promises an idle device: the barrier must run before
|
||||
|
||||
Reference in New Issue
Block a user