285 lines
11 KiB
Python
285 lines
11 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Prometheus metrics for SimpleCPUOffloadConnector.
|
|
|
|
The stats payload uses the same self-describing wire format as the
|
|
offloading connector, so it survives IPC serialization without the
|
|
metric classes on the receiving side.
|
|
"""
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
from vllm.config import VllmConfig
|
|
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
|
KVConnectorPromMetrics,
|
|
KVConnectorStats,
|
|
PromMetric,
|
|
PromMetricT,
|
|
)
|
|
|
|
|
|
class MetricName:
|
|
"""Metric names for SimpleCPUOffloadConnector."""
|
|
|
|
SAVE_OUTCOMES = "vllm:simple_kv_offload_save_outcomes_total"
|
|
LOAD_BLOCKS = "vllm:simple_kv_offload_load_blocks_total"
|
|
USED_BLOCKS = "vllm:simple_kv_offload_used_blocks"
|
|
PENDING_STORE_BLOCKS = "vllm:simple_kv_offload_pending_store_blocks"
|
|
INFO = "vllm:simple_kv_offload_info"
|
|
|
|
|
|
# outcome label value -> BoundaryStoreStats field name.
|
|
# ``published`` is the sum of all outcomes and is not labeled.
|
|
OUTCOME_TO_FIELD = {
|
|
"stored": "stored",
|
|
"dropped_cpu_full": "dropped_cpu_full",
|
|
"dropped_request_gone": "dropped_request_gone",
|
|
"dropped_null_block": "dropped_null_block",
|
|
"dropped_not_hashed": "dropped_not_hashed",
|
|
"skipped_already_cached": "skipped_already_cached",
|
|
"skipped_in_flight": "skipped_in_flight",
|
|
}
|
|
|
|
INFO_LABELS = ["backend", "page_cache", "lazy_offload", "capacity_blocks"]
|
|
|
|
|
|
class _MetricType:
|
|
"""Type tags embedded in the serialized stats payload."""
|
|
|
|
COUNTER = "counter"
|
|
GAUGE = "gauge"
|
|
|
|
|
|
class _StatsKey:
|
|
"""Top-level keys in the serialized stats dict."""
|
|
|
|
# Maps metric name -> _MetricType value
|
|
TYPES = "types"
|
|
# Maps metric name -> {label values tuple -> observed value}
|
|
DATA = "data"
|
|
|
|
|
|
@dataclass
|
|
class SimpleCPUOffloadStats(KVConnectorStats):
|
|
"""Stats payload for SimpleCPUOffloadConnector.
|
|
|
|
The ``data`` dict is structured as::
|
|
|
|
{
|
|
_StatsKey.TYPES: {name: _MetricType.*, ...},
|
|
_StatsKey.DATA: {name: {labelvalues: value, ...}, ...},
|
|
}
|
|
|
|
Counter values are summed when aggregating, gauge values keep the
|
|
latest snapshot, and unlabeled metrics use ``()`` as their
|
|
labelvalues tuple.
|
|
"""
|
|
|
|
def __post_init__(self):
|
|
if _StatsKey.DATA not in self.data:
|
|
self.reset()
|
|
|
|
def reset(self):
|
|
self.data: dict[str, Any] = {
|
|
_StatsKey.TYPES: {},
|
|
_StatsKey.DATA: {},
|
|
}
|
|
|
|
@property
|
|
def _types(self) -> dict[str, str]:
|
|
return self.data[_StatsKey.TYPES]
|
|
|
|
@property
|
|
def _values(self) -> dict[str, Any]:
|
|
return self.data[_StatsKey.DATA]
|
|
|
|
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
|
|
if other.is_empty():
|
|
return self
|
|
assert isinstance(other, SimpleCPUOffloadStats)
|
|
for key, other_label_values in other._values.items():
|
|
type_str = other._types.get(key)
|
|
if type_str is None:
|
|
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
|
|
self._types.setdefault(key, type_str)
|
|
current_label_values = self._values.setdefault(key, {})
|
|
for labelvalues, value in other_label_values.items():
|
|
if type_str == _MetricType.COUNTER:
|
|
assert isinstance(value, int | float)
|
|
current_label_values[labelvalues] = (
|
|
current_label_values.get(labelvalues, 0) + value
|
|
)
|
|
elif type_str == _MetricType.GAUGE:
|
|
assert isinstance(value, int | float)
|
|
current_label_values[labelvalues] = value
|
|
else:
|
|
raise AssertionError(
|
|
f"Unknown metric type '{type_str}' for key: {key}"
|
|
)
|
|
return self
|
|
|
|
def reduce(self) -> dict[str, int | float]:
|
|
"""Reduce the interval observations to a flat summary for logging."""
|
|
return_dict: dict[str, int | float] = {}
|
|
for key, label_value_map in self._values.items():
|
|
if key == MetricName.INFO:
|
|
# Constant config fingerprint; exclude from the log summary.
|
|
continue
|
|
type_str = self._types.get(key)
|
|
if type_str is None:
|
|
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
|
|
for labelvalues, value in label_value_map.items():
|
|
key_with_labels = f"{key}:{labelvalues}" if labelvalues else key
|
|
if type_str in (_MetricType.COUNTER, _MetricType.GAUGE):
|
|
assert isinstance(value, int | float)
|
|
return_dict[key_with_labels] = value
|
|
else:
|
|
raise AssertionError(
|
|
f"Unknown metric type '{type_str}' for key: {key}"
|
|
)
|
|
return return_dict
|
|
|
|
def is_empty(self) -> bool:
|
|
return not self.data.get(_StatsKey.DATA)
|
|
|
|
def increase_counter(
|
|
self,
|
|
counter_name: str,
|
|
counter_increase_value: int | float = 1,
|
|
labelvalues: tuple[str, ...] = (),
|
|
) -> None:
|
|
"""Increase a counter on the stats payload."""
|
|
self._types.setdefault(counter_name, _MetricType.COUNTER)
|
|
counter_values = self._values.setdefault(counter_name, {})
|
|
counter_values[labelvalues] = (
|
|
counter_values.get(labelvalues, 0) + counter_increase_value
|
|
)
|
|
|
|
def set_gauge(
|
|
self,
|
|
gauge_name: str,
|
|
gauge_value: int | float,
|
|
labelvalues: tuple[str, ...] = (),
|
|
) -> None:
|
|
"""Set a gauge snapshot on the stats payload."""
|
|
self._types.setdefault(gauge_name, _MetricType.GAUGE)
|
|
gauge_values = self._values.setdefault(gauge_name, {})
|
|
gauge_values[labelvalues] = gauge_value
|
|
|
|
|
|
class SimpleCPUOffloadPromMetrics(KVConnectorPromMetrics):
|
|
"""Prometheus registration and recording for SimpleCPUOffloadConnector."""
|
|
|
|
def __init__(
|
|
self,
|
|
vllm_config: VllmConfig,
|
|
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
|
labelnames: list[str],
|
|
per_engine_labelvalues: dict[int, list[object]],
|
|
):
|
|
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
|
|
# (engine_idx, metric_name, labelvalues) -> bound child metric
|
|
self._metrics: dict[tuple[int, str, tuple[str, ...]], PromMetricT] = {}
|
|
|
|
self._counter_save_outcomes = self._counter_cls(
|
|
name="vllm:simple_kv_offload_save_outcomes_total",
|
|
documentation=(
|
|
"Store admission decisions in SimpleCPUOffloadConnector, by "
|
|
"outcome. Outcomes classify eager-mode boundary hand-off "
|
|
"stores; lazy-mode stores are not classified."
|
|
),
|
|
labelnames=self._labelnames + ["outcome"],
|
|
)
|
|
self._counter_load_blocks = self._counter_cls(
|
|
name="vllm:simple_kv_offload_load_blocks_total",
|
|
documentation=(
|
|
"KV blocks restored to GPU from the offload pool, counted "
|
|
"when a load finishes. This is prefill work avoided "
|
|
"through offload cache hits."
|
|
),
|
|
labelnames=self._labelnames,
|
|
)
|
|
self._gauge_used_blocks = self._gauge_cls(
|
|
name="vllm:simple_kv_offload_used_blocks",
|
|
documentation=(
|
|
"Offload-pool blocks currently pinned by in-flight "
|
|
"transfers or cache hits (capacity minus free). Evictable "
|
|
"cached blocks are not counted; capacity_blocks is on "
|
|
"vllm:simple_kv_offload_info."
|
|
),
|
|
labelnames=self._labelnames,
|
|
multiprocess_mode="mostrecent",
|
|
)
|
|
self._gauge_pending_store_blocks = self._gauge_cls(
|
|
name="vllm:simple_kv_offload_pending_store_blocks",
|
|
documentation=(
|
|
"Offload blocks in pending stores: queued for dispatch, "
|
|
"issued to workers, or awaiting release after a cache "
|
|
"reset. A persistently growing value indicates a stuck "
|
|
"transfer."
|
|
),
|
|
labelnames=self._labelnames,
|
|
multiprocess_mode="mostrecent",
|
|
)
|
|
self._gauge_info = self._gauge_cls(
|
|
name="vllm:simple_kv_offload_info",
|
|
documentation=(
|
|
"SimpleCPUOffloadConnector deployment facts. Value is "
|
|
"always 1. backend is cpu or disk; page_cache and "
|
|
"lazy_offload are true/false; capacity_blocks is the "
|
|
"offload pool size."
|
|
),
|
|
labelnames=self._labelnames + INFO_LABELS,
|
|
multiprocess_mode="mostrecent",
|
|
)
|
|
|
|
self._defs: dict[str, PromMetricT] = {
|
|
MetricName.SAVE_OUTCOMES: self._counter_save_outcomes,
|
|
MetricName.LOAD_BLOCKS: self._counter_load_blocks,
|
|
MetricName.USED_BLOCKS: self._gauge_used_blocks,
|
|
MetricName.PENDING_STORE_BLOCKS: self._gauge_pending_store_blocks,
|
|
MetricName.INFO: self._gauge_info,
|
|
}
|
|
self._num_label_values: dict[str, int] = {
|
|
MetricName.SAVE_OUTCOMES: 1,
|
|
MetricName.LOAD_BLOCKS: 0,
|
|
MetricName.USED_BLOCKS: 0,
|
|
MetricName.PENDING_STORE_BLOCKS: 0,
|
|
MetricName.INFO: len(INFO_LABELS),
|
|
}
|
|
|
|
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
|
"""Observe transfer statistics."""
|
|
metric_types = transfer_stats_data.get(_StatsKey.TYPES, {})
|
|
metric_data = transfer_stats_data.get(_StatsKey.DATA, {})
|
|
for key, label_value_map in metric_data.items():
|
|
prom_metric = self._defs.get(key)
|
|
if prom_metric is None:
|
|
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
|
|
type_str = metric_types.get(key)
|
|
for labelvalues, value in label_value_map.items():
|
|
if len(labelvalues) != self._num_label_values[key]:
|
|
raise AssertionError(
|
|
f"Metric {key} expects "
|
|
f"{self._num_label_values[key]} labels, "
|
|
f"got {len(labelvalues)}"
|
|
)
|
|
child = self._metrics.get((engine_idx, key, labelvalues))
|
|
if child is None:
|
|
engine_labelvalues = self.per_engine_labelvalues[engine_idx]
|
|
child = prom_metric.labels(
|
|
*(engine_labelvalues + list(labelvalues))
|
|
)
|
|
self._metrics[(engine_idx, key, labelvalues)] = child
|
|
if type_str == _MetricType.COUNTER:
|
|
assert isinstance(value, int | float)
|
|
child.inc(value)
|
|
elif type_str == _MetricType.GAUGE:
|
|
assert isinstance(value, int | float)
|
|
child.set(value)
|
|
else:
|
|
raise AssertionError(
|
|
f"Unknown metric type '{type_str}' for key: {key}"
|
|
)
|