e5f3d08d72
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
58 lines
1.8 KiB
Python
58 lines
1.8 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Metrics for dispatched sleep-mode frontend operations."""
|
|
|
|
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from threading import Lock
|
|
from time import perf_counter
|
|
from typing import Literal
|
|
|
|
from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram
|
|
|
|
Operation = Literal["sleep", "release_kv_cache_memory", "wake"]
|
|
BUCKETS = (0.01, 0.1, 1, 10, 30, 60, 120, 300, 600)
|
|
|
|
|
|
class SleepModeOperationMetrics:
|
|
def __init__(self, registry: CollectorRegistry | None = None):
|
|
registry = REGISTRY if registry is None else registry
|
|
self.duration = Histogram(
|
|
"vllm:sleep_mode_operation_duration_seconds",
|
|
"Duration of one sleep-mode operation.",
|
|
["operation"],
|
|
buckets=BUCKETS,
|
|
registry=registry,
|
|
)
|
|
self.in_flight = Gauge(
|
|
"vllm:sleep_mode_operations_in_flight",
|
|
"Sleep-mode operations currently awaited.",
|
|
["operation"],
|
|
multiprocess_mode="livesum",
|
|
registry=registry,
|
|
)
|
|
|
|
@contextmanager
|
|
def record(self, operation: Operation) -> Iterator[None]:
|
|
active = self.in_flight.labels(operation)
|
|
started = perf_counter()
|
|
active.inc()
|
|
try:
|
|
yield
|
|
finally:
|
|
active.dec()
|
|
self.duration.labels(operation).observe(perf_counter() - started)
|
|
|
|
|
|
_metrics: SleepModeOperationMetrics | None = None
|
|
_metrics_lock = Lock()
|
|
|
|
|
|
def sleep_mode_operation_metrics() -> SleepModeOperationMetrics:
|
|
global _metrics
|
|
if _metrics is None:
|
|
with _metrics_lock:
|
|
if _metrics is None:
|
|
_metrics = SleepModeOperationMetrics()
|
|
return _metrics
|