[Metrics][KV Offload] Add vllm:kv_offload_config_info - generic info metrics for offloading the offloading connector (#56867)
Signed-off-by: Amir Friedman <Amir.Friedman1@ibm.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Claude Opus 5
Or Ozeri
parent
3a5a6b11a3
commit
775fefa804
@@ -99,8 +99,13 @@ class KVConnectorLogging:
|
||||
# Produce a single cumulative stats object for the last time
|
||||
# interval from the recorded observations.
|
||||
xfer_metrics = self.transfer_stats_accumulator.reduce()
|
||||
xfer_metrics_str = ", ".join(f"{k}={v}" for k, v in xfer_metrics.items())
|
||||
log_fn("KV Transfer metrics: %s", xfer_metrics_str)
|
||||
# A payload can hold static config facts alone (info metric), which reduce()
|
||||
# drops, because they belong to the Prometheus metrics only.
|
||||
if xfer_metrics:
|
||||
xfer_metrics_str = ", ".join(
|
||||
f"{k}={v}" for k, v in xfer_metrics.items()
|
||||
)
|
||||
log_fn("KV Transfer metrics: %s", xfer_metrics_str)
|
||||
|
||||
# Reset metrics for next interval
|
||||
self.reset()
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
@@ -10,14 +11,30 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import (
|
||||
ConfigInfoMapping,
|
||||
OffloadingCounterMetadata,
|
||||
OffloadingGaugeMetadata,
|
||||
OffloadingHistogramMetadata,
|
||||
OffloadingMetricMetadata,
|
||||
OffloadingSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
KV_OFFLOAD_CONFIG_INFO = "vllm:kv_offload_config_info"
|
||||
|
||||
_INFO_METRIC_HELP = (
|
||||
"Static configuration of the KV offload managers of this engine instance. "
|
||||
"The configured offloading spec declares the label names, and each manager "
|
||||
"fills the values, so a series appears from the first scheduler step of its "
|
||||
"engine. A manager reports one series for each configuration it holds, and "
|
||||
"a label of its own tells the series apart. Each engine reports its own "
|
||||
"configuration, not the instance total."
|
||||
)
|
||||
|
||||
|
||||
class _TransferMetricName:
|
||||
"""Flat metric names for GPU↔offload-medium transfer operations."""
|
||||
@@ -166,6 +183,9 @@ class _StatsKey:
|
||||
TYPES = "types"
|
||||
# Maps metric name -> {label values tuple -> observed value (number or list)}
|
||||
DATA = "data"
|
||||
# One mapping of info metric label name -> label value for each series.
|
||||
# None until a manager reports it, and one empty mapping for no fact.
|
||||
INFO = "info"
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -177,6 +197,7 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
{
|
||||
_StatsKey.TYPES: {name: _MetricType.*, ...},
|
||||
_StatsKey.DATA: {name: {labelvalues: value, ...}, ...},
|
||||
_StatsKey.INFO: [{labelname: labelvalue, ...}, ...] | None,
|
||||
}
|
||||
|
||||
This structure is self-describing: it survives IPC serialization
|
||||
@@ -187,6 +208,12 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
use the latest snapshot per-label-tuple, and histogram values are lists of
|
||||
observed samples per-label-tuple. Unlabeled metrics use ``()`` as their
|
||||
labelvalues tuple.
|
||||
|
||||
``INFO`` holds the static config facts of one series each, which the
|
||||
scheduler sends once per process. It stays out of ``DATA``, because its
|
||||
label names are known only when the payload arrives. It holds label names,
|
||||
not metric names, so a manager that accesses its own metrics walks
|
||||
``metric_sections()``.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
@@ -197,6 +224,7 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
self.data: dict[str, Any] = {
|
||||
_StatsKey.TYPES: {},
|
||||
_StatsKey.DATA: {},
|
||||
_StatsKey.INFO: None,
|
||||
}
|
||||
|
||||
@property
|
||||
@@ -207,10 +235,28 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
def _values(self) -> dict[str, Any]:
|
||||
return self.data[_StatsKey.DATA]
|
||||
|
||||
def metric_sections(self) -> tuple[dict[str, Any], ...]:
|
||||
"""Return the type section and the value section, in that order.
|
||||
|
||||
A manager that accesses its own metrics walks these sections. Each one
|
||||
is a mutable reference.
|
||||
"""
|
||||
return (self._types, self._values)
|
||||
|
||||
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
|
||||
if other.is_empty():
|
||||
return self
|
||||
assert isinstance(other, OffloadingConnectorStats)
|
||||
# The scheduler payload merges into the worker payload
|
||||
# (v1/core/sched/scheduler.py), so the info must survive the merge in
|
||||
# this direction to reach the Prometheus frontend. reduce() drops it, so
|
||||
# it never reaches a log line. The scheduler sends it once, so a later
|
||||
# payload with no info must not clear it.
|
||||
other_info = other.data.get(_StatsKey.INFO)
|
||||
if other_info is not None:
|
||||
self.data[_StatsKey.INFO] = [
|
||||
dict(series_info) for series_info in other_info
|
||||
]
|
||||
other_types = other._types
|
||||
other_values = other._values
|
||||
for key, other_label_values in other_values.items():
|
||||
@@ -268,7 +314,11 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
return return_dict
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return not self.data.get(_StatsKey.DATA)
|
||||
# An info-only payload is not empty. It carries no observation, but the
|
||||
# caller drops an empty payload, and the info must reach the frontend.
|
||||
return (
|
||||
not self.data.get(_StatsKey.DATA) and self.data.get(_StatsKey.INFO) is None
|
||||
)
|
||||
|
||||
def increase_counter(
|
||||
self,
|
||||
@@ -294,6 +344,18 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
gauge_values = self._values.setdefault(gauge_name, {})
|
||||
gauge_values[labelvalues] = gauge_value
|
||||
|
||||
def set_info(self, info: Sequence[ConfigInfoMapping]) -> None:
|
||||
"""Put the static config facts of this engine on the stats payload.
|
||||
|
||||
Args:
|
||||
info: One mapping of info metric label name to label value for each
|
||||
series, as OffloadingManager.config_info() returns it. An empty
|
||||
mapping still gives one series, with an empty value on every
|
||||
label the spec declared.
|
||||
|
||||
"""
|
||||
self.data[_StatsKey.INFO] = [dict(series_info) for series_info in info]
|
||||
|
||||
def observe_histogram(
|
||||
self,
|
||||
histogram_name: str,
|
||||
@@ -322,14 +384,22 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
|
||||
kv_transfer_config = vllm_config.kv_transfer_config
|
||||
assert kv_transfer_config is not None
|
||||
extra_config = kv_transfer_config.kv_connector_extra_config
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(extra_config)
|
||||
self._spec_cls: type[OffloadingSpec] = OffloadingSpecFactory.get_spec_cls(
|
||||
extra_config
|
||||
)
|
||||
# The spec declares the info label names here; a manager fills the
|
||||
# values later, and _observe_info() aligns the two.
|
||||
self._info_keys: tuple[str, ...] = self._spec_cls.config_info_keys(extra_config)
|
||||
self._offloading_metric_metadata: dict[str, OffloadingMetricMetadata] = {
|
||||
**spec_cls.build_metric_definitions(extra_config),
|
||||
**self._spec_cls.build_metric_definitions(extra_config),
|
||||
**get_connector_metric_definitions(),
|
||||
KV_OFFLOAD_CONFIG_INFO: OffloadingGaugeMetadata(
|
||||
documentation=_INFO_METRIC_HELP, labelnames=self._info_keys
|
||||
),
|
||||
}
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
|
||||
self._observe_deprecated_metrics = issubclass(spec_cls, CPUOffloadingSpec)
|
||||
self._observe_deprecated_metrics = issubclass(self._spec_cls, CPUOffloadingSpec)
|
||||
self._offloading_metric_defs: dict[str, PromMetricT] = {}
|
||||
# (engine_idx, metric_name, labelvalues) -> metric with bound labels
|
||||
self.offloading_metrics: dict[
|
||||
@@ -393,6 +463,7 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
|
||||
metric_cls = self._counter_cls
|
||||
elif isinstance(metadata, OffloadingGaugeMetadata):
|
||||
metric_cls = self._gauge_cls
|
||||
kwargs["multiprocess_mode"] = metadata.multiprocess_mode
|
||||
elif isinstance(metadata, OffloadingHistogramMetadata):
|
||||
metric_cls = self._histogram_cls
|
||||
if metadata.buckets is not None:
|
||||
@@ -476,8 +547,52 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
|
||||
observation
|
||||
)
|
||||
|
||||
def _observe_info(self, info: Sequence[dict[str, Any]], engine_idx: int) -> None:
|
||||
"""Publish the static config facts of one engine, one series per mapping.
|
||||
|
||||
The spec declares the label names, so a payload only fills them. A
|
||||
declared name that no series fills reads empty on every series. A payload
|
||||
name that the spec did not declare is dropped. Either gap logs once.
|
||||
|
||||
The check covers the payload as a whole, and not one series, because the
|
||||
declaration holds the names of every series. A name that one series owns
|
||||
reads empty on the other series by design. A series that leaves a name of
|
||||
its own unfilled therefore stays hidden here.
|
||||
|
||||
Args:
|
||||
info: One mapping of label name to label value for each series, as
|
||||
OffloadingManager.config_info() returns it.
|
||||
engine_idx: Index of the reporting engine.
|
||||
|
||||
"""
|
||||
filled: dict[str, None] = {}
|
||||
for series_info in info:
|
||||
filled.update(dict.fromkeys(series_info))
|
||||
labelvalues = tuple(
|
||||
str(series_info.get(key, "")) for key in self._info_keys
|
||||
)
|
||||
self._set_gauge(KV_OFFLOAD_CONFIG_INFO, 1, labelvalues, engine_idx)
|
||||
|
||||
never_filled = tuple(key for key in self._info_keys if key not in filled)
|
||||
dropped = tuple(key for key in filled if key not in self._info_keys)
|
||||
if never_filled or dropped:
|
||||
logger.warning_once(
|
||||
"%s: spec %s and the manager of engine %d disagree on the KV "
|
||||
"offload config labels. Declared, and no series filled them, so "
|
||||
"empty on every series: %s. Filled, and the spec did not "
|
||||
"declare them, so dropped: %s.",
|
||||
KV_OFFLOAD_CONFIG_INFO,
|
||||
self._spec_cls.__name__,
|
||||
engine_idx,
|
||||
never_filled,
|
||||
dropped,
|
||||
)
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
"""Observe transfer statistics."""
|
||||
info = transfer_stats_data.get(_StatsKey.INFO)
|
||||
if info is not None:
|
||||
self._observe_info(info, engine_idx)
|
||||
metric_types = transfer_stats_data.get(_StatsKey.TYPES, {})
|
||||
metric_data = transfer_stats_data.get(_StatsKey.DATA, {})
|
||||
for key, label_value_map in metric_data.items():
|
||||
|
||||
@@ -574,6 +574,10 @@ class OffloadingConnectorScheduler:
|
||||
)
|
||||
self.manager: OffloadingManager = spec.get_manager()
|
||||
self._connector_stats = OffloadingConnectorStats()
|
||||
# The static config facts ride on the first stats payload of this
|
||||
# process. A Prometheus child holds the values it received, so one
|
||||
# payload is enough.
|
||||
self._info_sent = False
|
||||
|
||||
full_attention_groups: list[int] = []
|
||||
sliding_window_groups: list[int] = []
|
||||
@@ -2004,6 +2008,14 @@ class OffloadingConnectorScheduler:
|
||||
else:
|
||||
stats.aggregate(manager_stats)
|
||||
|
||||
if not self._info_sent:
|
||||
if stats is None:
|
||||
stats = OffloadingConnectorStats()
|
||||
# Sent even when the manager reports no facts, so that the metric
|
||||
# exists whenever offloading runs.
|
||||
stats.set_info(self.manager.config_info())
|
||||
self._info_sent = True
|
||||
|
||||
return stats
|
||||
|
||||
def request_finished(
|
||||
|
||||
Reference in New Issue
Block a user