[Frontend][Core] Add release_kv_cache_memory() API (#44890)
Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: AlanFokCo <alanfok2868@gmail.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com> Co-authored-by: AlanFokCo <alanfok2868@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
AlanFokCo
aoshen02
OpenAI Codex
parent
a524f80b1f
commit
092bdd6d57
@@ -1,14 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.serve.dev.sleep.api_router import attach_router
|
||||
|
||||
MODEL_NAME = "meta-llama/Llama-3.2-1B"
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("fails", [False, True])
|
||||
def test_release_kv_cache_memory_route(fails):
|
||||
app = FastAPI()
|
||||
app.state.engine_client = AsyncMock()
|
||||
release = app.state.engine_client.release_kv_cache_memory
|
||||
if fails:
|
||||
release.side_effect = RuntimeError("requires a completed pause first")
|
||||
attach_router(app)
|
||||
|
||||
with TestClient(app, raise_server_exceptions=False) as client:
|
||||
response = client.post("/release_kv_cache_memory")
|
||||
|
||||
assert response.status_code == (500 if fails else 200)
|
||||
release.assert_awaited_once_with()
|
||||
|
||||
|
||||
def test_sleep_mode():
|
||||
# dtype, max-len etc set so that this can run in CI
|
||||
args = [
|
||||
@@ -63,7 +86,7 @@ def test_sleep_mode():
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# is sleeping should be false after waking up any part of the engine
|
||||
# Partial wake keeps the engine sleeping.
|
||||
response = requests.get(remote_server.url_for("is_sleeping"))
|
||||
assert response.status_code == 200
|
||||
assert response.json().get("is_sleeping") is True
|
||||
|
||||
Reference in New Issue
Block a user