[Metrics][KV Offload] Add vllm:kv_offload_config_info - generic info metrics for offloading the offloading connector (#56867)

Signed-off-by: Amir Friedman <Amir.Friedman1@ibm.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Amir Friedman
2026-10-14 12:34:37 +00:00
committed by GitHub
co-authored by Claude Opus 5 Or Ozeri
parent 3a5a6b11a3
commit 775fefa804
17 changed files with 926 additions and 42 deletions
@@ -99,8 +99,13 @@ class KVConnectorLogging:
# Produce a single cumulative stats object for the last time
# interval from the recorded observations.
xfer_metrics = self.transfer_stats_accumulator.reduce()
xfer_metrics_str = ", ".join(f"{k}={v}" for k, v in xfer_metrics.items())
log_fn("KV Transfer metrics: %s", xfer_metrics_str)
# A payload can hold static config facts alone (info metric), which reduce()
# drops, because they belong to the Prometheus metrics only.
if xfer_metrics:
xfer_metrics_str = ", ".join(
f"{k}={v}" for k, v in xfer_metrics.items()
)
log_fn("KV Transfer metrics: %s", xfer_metrics_str)
# Reset metrics for next interval
self.reset()
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any
@@ -10,14 +11,30 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
PromMetric,
PromMetricT,
)
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import (
ConfigInfoMapping,
OffloadingCounterMetadata,
OffloadingGaugeMetadata,
OffloadingHistogramMetadata,
OffloadingMetricMetadata,
OffloadingSpec,
)
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
logger = init_logger(__name__)
KV_OFFLOAD_CONFIG_INFO = "vllm:kv_offload_config_info"
_INFO_METRIC_HELP = (
"Static configuration of the KV offload managers of this engine instance. "
"The configured offloading spec declares the label names, and each manager "
"fills the values, so a series appears from the first scheduler step of its "
"engine. A manager reports one series for each configuration it holds, and "
"a label of its own tells the series apart. Each engine reports its own "
"configuration, not the instance total."
)
class _TransferMetricName:
"""Flat metric names for GPU↔offload-medium transfer operations."""
@@ -166,6 +183,9 @@ class _StatsKey:
TYPES = "types"
# Maps metric name -> {label values tuple -> observed value (number or list)}
DATA = "data"
# One mapping of info metric label name -> label value for each series.
# None until a manager reports it, and one empty mapping for no fact.
INFO = "info"
@dataclass
@@ -177,6 +197,7 @@ class OffloadingConnectorStats(KVConnectorStats):
{
_StatsKey.TYPES: {name: _MetricType.*, ...},
_StatsKey.DATA: {name: {labelvalues: value, ...}, ...},
_StatsKey.INFO: [{labelname: labelvalue, ...}, ...] | None,
}
This structure is self-describing: it survives IPC serialization
@@ -187,6 +208,12 @@ class OffloadingConnectorStats(KVConnectorStats):
use the latest snapshot per-label-tuple, and histogram values are lists of
observed samples per-label-tuple. Unlabeled metrics use ``()`` as their
labelvalues tuple.
``INFO`` holds the static config facts of one series each, which the
scheduler sends once per process. It stays out of ``DATA``, because its
label names are known only when the payload arrives. It holds label names,
not metric names, so a manager that accesses its own metrics walks
``metric_sections()``.
"""
def __post_init__(self):
@@ -197,6 +224,7 @@ class OffloadingConnectorStats(KVConnectorStats):
self.data: dict[str, Any] = {
_StatsKey.TYPES: {},
_StatsKey.DATA: {},
_StatsKey.INFO: None,
}
@property
@@ -207,10 +235,28 @@ class OffloadingConnectorStats(KVConnectorStats):
def _values(self) -> dict[str, Any]:
return self.data[_StatsKey.DATA]
def metric_sections(self) -> tuple[dict[str, Any], ...]:
"""Return the type section and the value section, in that order.
A manager that accesses its own metrics walks these sections. Each one
is a mutable reference.
"""
return (self._types, self._values)
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
if other.is_empty():
return self
assert isinstance(other, OffloadingConnectorStats)
# The scheduler payload merges into the worker payload
# (v1/core/sched/scheduler.py), so the info must survive the merge in
# this direction to reach the Prometheus frontend. reduce() drops it, so
# it never reaches a log line. The scheduler sends it once, so a later
# payload with no info must not clear it.
other_info = other.data.get(_StatsKey.INFO)
if other_info is not None:
self.data[_StatsKey.INFO] = [
dict(series_info) for series_info in other_info
]
other_types = other._types
other_values = other._values
for key, other_label_values in other_values.items():
@@ -268,7 +314,11 @@ class OffloadingConnectorStats(KVConnectorStats):
return return_dict
def is_empty(self) -> bool:
return not self.data.get(_StatsKey.DATA)
# An info-only payload is not empty. It carries no observation, but the
# caller drops an empty payload, and the info must reach the frontend.
return (
not self.data.get(_StatsKey.DATA) and self.data.get(_StatsKey.INFO) is None
)
def increase_counter(
self,
@@ -294,6 +344,18 @@ class OffloadingConnectorStats(KVConnectorStats):
gauge_values = self._values.setdefault(gauge_name, {})
gauge_values[labelvalues] = gauge_value
def set_info(self, info: Sequence[ConfigInfoMapping]) -> None:
"""Put the static config facts of this engine on the stats payload.
Args:
info: One mapping of info metric label name to label value for each
series, as OffloadingManager.config_info() returns it. An empty
mapping still gives one series, with an empty value on every
label the spec declared.
"""
self.data[_StatsKey.INFO] = [dict(series_info) for series_info in info]
def observe_histogram(
self,
histogram_name: str,
@@ -322,14 +384,22 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
kv_transfer_config = vllm_config.kv_transfer_config
assert kv_transfer_config is not None
extra_config = kv_transfer_config.kv_connector_extra_config
spec_cls = OffloadingSpecFactory.get_spec_cls(extra_config)
self._spec_cls: type[OffloadingSpec] = OffloadingSpecFactory.get_spec_cls(
extra_config
)
# The spec declares the info label names here; a manager fills the
# values later, and _observe_info() aligns the two.
self._info_keys: tuple[str, ...] = self._spec_cls.config_info_keys(extra_config)
self._offloading_metric_metadata: dict[str, OffloadingMetricMetadata] = {
**spec_cls.build_metric_definitions(extra_config),
**self._spec_cls.build_metric_definitions(extra_config),
**get_connector_metric_definitions(),
KV_OFFLOAD_CONFIG_INFO: OffloadingGaugeMetadata(
documentation=_INFO_METRIC_HELP, labelnames=self._info_keys
),
}
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
self._observe_deprecated_metrics = issubclass(spec_cls, CPUOffloadingSpec)
self._observe_deprecated_metrics = issubclass(self._spec_cls, CPUOffloadingSpec)
self._offloading_metric_defs: dict[str, PromMetricT] = {}
# (engine_idx, metric_name, labelvalues) -> metric with bound labels
self.offloading_metrics: dict[
@@ -393,6 +463,7 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
metric_cls = self._counter_cls
elif isinstance(metadata, OffloadingGaugeMetadata):
metric_cls = self._gauge_cls
kwargs["multiprocess_mode"] = metadata.multiprocess_mode
elif isinstance(metadata, OffloadingHistogramMetadata):
metric_cls = self._histogram_cls
if metadata.buckets is not None:
@@ -476,8 +547,52 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
observation
)
def _observe_info(self, info: Sequence[dict[str, Any]], engine_idx: int) -> None:
"""Publish the static config facts of one engine, one series per mapping.
The spec declares the label names, so a payload only fills them. A
declared name that no series fills reads empty on every series. A payload
name that the spec did not declare is dropped. Either gap logs once.
The check covers the payload as a whole, and not one series, because the
declaration holds the names of every series. A name that one series owns
reads empty on the other series by design. A series that leaves a name of
its own unfilled therefore stays hidden here.
Args:
info: One mapping of label name to label value for each series, as
OffloadingManager.config_info() returns it.
engine_idx: Index of the reporting engine.
"""
filled: dict[str, None] = {}
for series_info in info:
filled.update(dict.fromkeys(series_info))
labelvalues = tuple(
str(series_info.get(key, "")) for key in self._info_keys
)
self._set_gauge(KV_OFFLOAD_CONFIG_INFO, 1, labelvalues, engine_idx)
never_filled = tuple(key for key in self._info_keys if key not in filled)
dropped = tuple(key for key in filled if key not in self._info_keys)
if never_filled or dropped:
logger.warning_once(
"%s: spec %s and the manager of engine %d disagree on the KV "
"offload config labels. Declared, and no series filled them, so "
"empty on every series: %s. Filled, and the spec did not "
"declare them, so dropped: %s.",
KV_OFFLOAD_CONFIG_INFO,
self._spec_cls.__name__,
engine_idx,
never_filled,
dropped,
)
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
"""Observe transfer statistics."""
info = transfer_stats_data.get(_StatsKey.INFO)
if info is not None:
self._observe_info(info, engine_idx)
metric_types = transfer_stats_data.get(_StatsKey.TYPES, {})
metric_data = transfer_stats_data.get(_StatsKey.DATA, {})
for key, label_value_map in metric_data.items():
@@ -574,6 +574,10 @@ class OffloadingConnectorScheduler:
)
self.manager: OffloadingManager = spec.get_manager()
self._connector_stats = OffloadingConnectorStats()
# The static config facts ride on the first stats payload of this
# process. A Prometheus child holds the values it received, so one
# payload is enough.
self._info_sent = False
full_attention_groups: list[int] = []
sliding_window_groups: list[int] = []
@@ -2004,6 +2008,14 @@ class OffloadingConnectorScheduler:
else:
stats.aggregate(manager_stats)
if not self._info_sent:
if stats is None:
stats = OffloadingConnectorStats()
# Sent even when the manager reports no facts, so that the metric
# exists whenever offloading runs.
stats.set_info(self.manager.config_info())
self._info_sent = True
return stats
def request_finished(