[Frontend][RL] Align sleep-mode API responses and operation metrics (#52864)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5.5
parent
29773c722b
commit
e5f3d08d72
@@ -839,7 +839,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
"""
|
||||
self.llm_engine.release_kv_cache_memory()
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None):
|
||||
def wake_up(self, tags: list[str] | None = None) -> bool:
|
||||
"""Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep]
|
||||
method for more details.
|
||||
|
||||
@@ -851,8 +851,11 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
(or None) before the engine is used again.
|
||||
Use tags=["scheduling"] to resume from level 0 sleep.
|
||||
|
||||
Returns:
|
||||
Whether the engine is fully awake.
|
||||
|
||||
"""
|
||||
self.llm_engine.wake_up(tags)
|
||||
return self.llm_engine.wake_up(tags)
|
||||
|
||||
def get_metrics(self) -> list["Metric"]:
|
||||
"""Return a snapshot of aggregated metrics from Prometheus.
|
||||
|
||||
@@ -2,11 +2,16 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from fastapi import APIRouter, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, FastAPI, Query, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.engine import PauseMode
|
||||
|
||||
from .metrics import sleep_mode_operation_metrics
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -19,29 +24,35 @@ router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/sleep")
|
||||
async def sleep(raw_request: Request):
|
||||
# get POST params
|
||||
level = raw_request.query_params.get("level", "1")
|
||||
mode = raw_request.query_params.get("mode", "abort")
|
||||
await engine_client(raw_request).sleep(int(level), mode)
|
||||
return Response(status_code=200)
|
||||
async def sleep(
|
||||
raw_request: Request,
|
||||
level: Annotated[int, Query(ge=0, le=2)] = 1,
|
||||
mode: Annotated[PauseMode, Query()] = "abort",
|
||||
) -> JSONResponse:
|
||||
with sleep_mode_operation_metrics().record("sleep"):
|
||||
await engine_client(raw_request).sleep(level, mode)
|
||||
return JSONResponse(content={"status": "sleeping", "level": level})
|
||||
|
||||
|
||||
@router.post("/release_kv_cache_memory")
|
||||
async def release_kv_cache_memory(raw_request: Request):
|
||||
await engine_client(raw_request).release_kv_cache_memory()
|
||||
return Response(status_code=200)
|
||||
async def release_kv_cache_memory(raw_request: Request) -> JSONResponse:
|
||||
with sleep_mode_operation_metrics().record("release_kv_cache_memory"):
|
||||
await engine_client(raw_request).release_kv_cache_memory()
|
||||
return JSONResponse(content={"status": "kv_cache_released"})
|
||||
|
||||
|
||||
@router.post("/wake_up")
|
||||
async def wake_up(raw_request: Request):
|
||||
async def wake_up(raw_request: Request) -> JSONResponse:
|
||||
tags = raw_request.query_params.getlist("tags")
|
||||
if tags == []:
|
||||
# set to None to wake up all tags if no tags are provided
|
||||
tags = None
|
||||
logger.info("wake up the engine with tags: %s", tags)
|
||||
await engine_client(raw_request).wake_up(tags)
|
||||
return Response(status_code=200)
|
||||
with sleep_mode_operation_metrics().record("wake"):
|
||||
fully_awake = await engine_client(raw_request).wake_up(tags)
|
||||
return JSONResponse(
|
||||
content={"status": "awake" if fully_awake else "sleeping", "tags": tags}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/is_sleeping")
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Metrics for dispatched sleep-mode frontend operations."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from threading import Lock
|
||||
from time import perf_counter
|
||||
from typing import Literal
|
||||
|
||||
from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram
|
||||
|
||||
Operation = Literal["sleep", "release_kv_cache_memory", "wake"]
|
||||
BUCKETS = (0.01, 0.1, 1, 10, 30, 60, 120, 300, 600)
|
||||
|
||||
|
||||
class SleepModeOperationMetrics:
|
||||
def __init__(self, registry: CollectorRegistry | None = None):
|
||||
registry = REGISTRY if registry is None else registry
|
||||
self.duration = Histogram(
|
||||
"vllm:sleep_mode_operation_duration_seconds",
|
||||
"Duration of one sleep-mode operation.",
|
||||
["operation"],
|
||||
buckets=BUCKETS,
|
||||
registry=registry,
|
||||
)
|
||||
self.in_flight = Gauge(
|
||||
"vllm:sleep_mode_operations_in_flight",
|
||||
"Sleep-mode operations currently awaited.",
|
||||
["operation"],
|
||||
multiprocess_mode="livesum",
|
||||
registry=registry,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def record(self, operation: Operation) -> Iterator[None]:
|
||||
active = self.in_flight.labels(operation)
|
||||
started = perf_counter()
|
||||
active.inc()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
active.dec()
|
||||
self.duration.labels(operation).observe(perf_counter() - started)
|
||||
|
||||
|
||||
_metrics: SleepModeOperationMetrics | None = None
|
||||
_metrics_lock = Lock()
|
||||
|
||||
|
||||
def sleep_mode_operation_metrics() -> SleepModeOperationMetrics:
|
||||
global _metrics
|
||||
if _metrics is None:
|
||||
with _metrics_lock:
|
||||
if _metrics is None:
|
||||
_metrics = SleepModeOperationMetrics()
|
||||
return _metrics
|
||||
Reference in New Issue
Block a user