[Frontend][RL] Align sleep-mode API responses and operation metrics (#52864)

Signed-off-by: Ronald1995 <ronaldautomobile@163.com>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ronald
2026-09-29 23:16:38 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent 29773c722b
commit e5f3d08d72
14 changed files with 834 additions and 48 deletions
+179 -5
View File
@@ -1,35 +1,209 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
import requests
from fastapi import FastAPI
from fastapi.testclient import TestClient
from prometheus_client import CollectorRegistry
from prometheus_client.parser import text_string_to_metric_families
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.serve.dev.sleep import metrics as sleep_metrics
from vllm.entrypoints.serve.dev.sleep.api_router import attach_router
from vllm.entrypoints.serve.dev.sleep.metrics import SleepModeOperationMetrics
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
from vllm.entrypoints.serve.instrumentator.metrics import (
attach_router as attach_metrics_router,
)
from vllm.v1.metrics import prometheus as prometheus_metrics
MODEL_NAME = "meta-llama/Llama-3.2-1B"
@pytest.fixture
def sleep_route_app(monkeypatch):
app = FastAPI()
app.state.args = SimpleNamespace(log_error_stack=False)
app.state.engine_client = AsyncMock()
metrics = SleepModeOperationMetrics(CollectorRegistry())
monkeypatch.setattr(
"vllm.entrypoints.serve.dev.sleep.api_router.sleep_mode_operation_metrics",
lambda: metrics,
)
attach_router(app)
init_exception_handler(app)
return app, metrics
@pytest.mark.cpu_test
@pytest.mark.parametrize("level", [0, 1, 2])
def test_sleep_route_response_and_engine_arguments(sleep_route_app, level):
app, _ = sleep_route_app
with TestClient(app) as client:
response = client.post("/sleep", params={"level": level})
assert response.status_code == 200
assert response.json() == {"status": "sleeping", "level": level}
app.state.engine_client.sleep.assert_awaited_once_with(level, "abort")
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("query", "expected_param"),
[
("level=invalid", "query.level"),
("level=-1", "query.level"),
("level=3", "query.level"),
("mode=invalid", "query.mode"),
],
)
def test_sleep_route_rejects_invalid_query_before_dispatch(
sleep_route_app, query, expected_param
):
app, metrics = sleep_route_app
with TestClient(app) as client:
response = client.post(f"/sleep?{query}")
assert response.status_code == 400
assert response.json()["error"]["param"] == expected_param
app.state.engine_client.sleep.assert_not_awaited()
assert list(metrics.duration.collect()[0].samples) == []
assert list(metrics.in_flight.collect()[0].samples) == []
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("fully_awake", "tags", "expected"),
[
(True, "", {"status": "awake", "tags": None}),
(False, "?tags=weights", {"status": "sleeping", "tags": ["weights"]}),
],
)
def test_wake_route_maps_engine_result_without_state_query(
sleep_route_app, fully_awake, tags, expected
):
app, _ = sleep_route_app
app.state.engine_client.wake_up.return_value = fully_awake
with TestClient(app) as client:
response = client.post(f"/wake_up{tags}")
assert response.status_code == 200
assert response.json() == expected
app.state.engine_client.wake_up.assert_awaited_once_with(expected["tags"])
app.state.engine_client.is_sleeping.assert_not_awaited()
@pytest.mark.cpu_test
@pytest.mark.parametrize("fails", [False, True])
def test_release_kv_cache_memory_route(fails):
app = FastAPI()
app.state.engine_client = AsyncMock()
def test_release_kv_cache_memory_route(sleep_route_app, fails):
app, metrics = sleep_route_app
release = app.state.engine_client.release_kv_cache_memory
if fails:
release.side_effect = RuntimeError("requires a completed pause first")
attach_router(app)
with TestClient(app, raise_server_exceptions=False) as client:
response = client.post("/release_kv_cache_memory")
assert response.status_code == (500 if fails else 200)
if not fails:
assert response.json() == {"status": "kv_cache_released"}
release.assert_awaited_once_with()
assert metrics.in_flight.labels("release_kv_cache_memory")._value.get() == 0
assert _duration_count(metrics, "release_kv_cache_memory") == 1
@pytest.mark.cpu_test
@pytest.mark.parametrize("operation", ["sleep", "release_kv_cache_memory", "wake"])
@pytest.mark.parametrize("exception", [RuntimeError, asyncio.CancelledError])
def test_sleep_mode_recorder_tracks_duration_and_cleans_up(operation, exception):
metrics = SleepModeOperationMetrics(CollectorRegistry())
with metrics.record(operation):
assert metrics.in_flight.labels(operation)._value.get() == 1
with pytest.raises(exception), metrics.record(operation):
raise exception()
assert metrics.in_flight.labels(operation)._value.get() == 0
assert metrics.duration.labels(operation)._sum.get() >= 0
assert _duration_count(metrics, operation) == 2
def _duration_count(metrics, operation):
return next(
sample.value
for sample in metrics.duration.collect()[0].samples
if sample.name.endswith("_count") and sample.labels == {"operation": operation}
)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("path", "method", "operation", "status", "fails"),
[
("/sleep", "POST", "sleep", 200, False),
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 200, False),
("/wake_up", "POST", "wake", 200, False),
("/is_sleeping", "GET", None, 200, False),
("/sleep?level=3", "POST", None, 400, False),
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 500, True),
],
)
def test_sleep_routes_visible_on_production_metrics_endpoint(
monkeypatch, path, method, operation, status, fails
):
registry = CollectorRegistry()
monkeypatch.delenv("PROMETHEUS_MULTIPROC_DIR", raising=False)
monkeypatch.setattr(sleep_metrics, "REGISTRY", registry)
monkeypatch.setattr(sleep_metrics, "_metrics", None)
monkeypatch.setattr(prometheus_metrics, "REGISTRY", registry)
app = FastAPI()
app.state.args = SimpleNamespace(log_error_stack=False)
app.state.engine_client = AsyncMock()
app.state.engine_client.wake_up.return_value = True
app.state.engine_client.is_sleeping.return_value = False
if fails:
app.state.engine_client.release_kv_cache_memory.side_effect = RuntimeError(
"engine failed"
)
attach_router(app)
init_exception_handler(app)
attach_metrics_router(app)
with TestClient(app, raise_server_exceptions=False) as client:
assert client.request(method, path).status_code == status
response = client.get("/metrics")
assert response.status_code == 200
samples = [
sample
for family in text_string_to_metric_families(response.text)
for sample in family.samples
]
assert any(
sample.name == "http_requests_total"
and sample.labels
== {
"handler": path.split("?")[0],
"method": method,
"status": f"{status // 100}xx",
}
and sample.value == 1
for sample in samples
)
assert not any("sleep_mode_operations_total" in sample.name for sample in samples)
assert any(
sample.name == "vllm:sleep_mode_operations_in_flight"
and sample.labels == {"operation": operation}
and sample.value == 0
for sample in samples
) == (operation is not None)
assert any(
sample.name == "vllm:sleep_mode_operation_duration_seconds_count"
and sample.labels == {"operation": operation}
and sample.value == 1
for sample in samples
) == (operation is not None)
def test_sleep_mode():