[Frontend][RL] Align sleep-mode API responses and operation metrics (#52864)

Signed-off-by: Ronald1995 <ronaldautomobile@163.com>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ronald
2026-09-29 23:16:38 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent 29773c722b
commit e5f3d08d72
14 changed files with 834 additions and 48 deletions
+5 -2
View File
@@ -839,7 +839,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
"""
self.llm_engine.release_kv_cache_memory()
def wake_up(self, tags: list[str] | None = None):
def wake_up(self, tags: list[str] | None = None) -> bool:
"""Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep]
method for more details.
@@ -851,8 +851,11 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
(or None) before the engine is used again.
Use tags=["scheduling"] to resume from level 0 sleep.
Returns:
Whether the engine is fully awake.
"""
self.llm_engine.wake_up(tags)
return self.llm_engine.wake_up(tags)
def get_metrics(self) -> list["Metric"]:
"""Return a snapshot of aggregated metrics from Prometheus.
+25 -14
View File
@@ -2,11 +2,16 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fastapi import APIRouter, FastAPI, Request
from fastapi.responses import JSONResponse, Response
from typing import Annotated
from fastapi import APIRouter, FastAPI, Query, Request
from fastapi.responses import JSONResponse
from vllm.engine.protocol import EngineClient
from vllm.logger import init_logger
from vllm.v1.engine import PauseMode
from .metrics import sleep_mode_operation_metrics
logger = init_logger(__name__)
@@ -19,29 +24,35 @@ router = APIRouter()
@router.post("/sleep")
async def sleep(raw_request: Request):
# get POST params
level = raw_request.query_params.get("level", "1")
mode = raw_request.query_params.get("mode", "abort")
await engine_client(raw_request).sleep(int(level), mode)
return Response(status_code=200)
async def sleep(
raw_request: Request,
level: Annotated[int, Query(ge=0, le=2)] = 1,
mode: Annotated[PauseMode, Query()] = "abort",
) -> JSONResponse:
with sleep_mode_operation_metrics().record("sleep"):
await engine_client(raw_request).sleep(level, mode)
return JSONResponse(content={"status": "sleeping", "level": level})
@router.post("/release_kv_cache_memory")
async def release_kv_cache_memory(raw_request: Request):
await engine_client(raw_request).release_kv_cache_memory()
return Response(status_code=200)
async def release_kv_cache_memory(raw_request: Request) -> JSONResponse:
with sleep_mode_operation_metrics().record("release_kv_cache_memory"):
await engine_client(raw_request).release_kv_cache_memory()
return JSONResponse(content={"status": "kv_cache_released"})
@router.post("/wake_up")
async def wake_up(raw_request: Request):
async def wake_up(raw_request: Request) -> JSONResponse:
tags = raw_request.query_params.getlist("tags")
if tags == []:
# set to None to wake up all tags if no tags are provided
tags = None
logger.info("wake up the engine with tags: %s", tags)
await engine_client(raw_request).wake_up(tags)
return Response(status_code=200)
with sleep_mode_operation_metrics().record("wake"):
fully_awake = await engine_client(raw_request).wake_up(tags)
return JSONResponse(
content={"status": "awake" if fully_awake else "sleeping", "tags": tags}
)
@router.get("/is_sleeping")
@@ -0,0 +1,57 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Metrics for dispatched sleep-mode frontend operations."""
from collections.abc import Iterator
from contextlib import contextmanager
from threading import Lock
from time import perf_counter
from typing import Literal
from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram
Operation = Literal["sleep", "release_kv_cache_memory", "wake"]
BUCKETS = (0.01, 0.1, 1, 10, 30, 60, 120, 300, 600)
class SleepModeOperationMetrics:
def __init__(self, registry: CollectorRegistry | None = None):
registry = REGISTRY if registry is None else registry
self.duration = Histogram(
"vllm:sleep_mode_operation_duration_seconds",
"Duration of one sleep-mode operation.",
["operation"],
buckets=BUCKETS,
registry=registry,
)
self.in_flight = Gauge(
"vllm:sleep_mode_operations_in_flight",
"Sleep-mode operations currently awaited.",
["operation"],
multiprocess_mode="livesum",
registry=registry,
)
@contextmanager
def record(self, operation: Operation) -> Iterator[None]:
active = self.in_flight.labels(operation)
started = perf_counter()
active.inc()
try:
yield
finally:
active.dec()
self.duration.labels(operation).observe(perf_counter() - started)
_metrics: SleepModeOperationMetrics | None = None
_metrics_lock = Lock()
def sleep_mode_operation_metrics() -> SleepModeOperationMetrics:
global _metrics
if _metrics is None:
with _metrics_lock:
if _metrics is None:
_metrics = SleepModeOperationMetrics()
return _metrics