[Metrics][KV Offload] Add vllm:kv_offload_config_info - generic info metrics for offloading the offloading connector (#56867)

Signed-off-by: Amir Friedman <Amir.Friedman1@ibm.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Amir Friedman
2026-10-14 12:34:37 +00:00
committed by GitHub
co-authored by Claude Opus 5 Or Ozeri
parent 3a5a6b11a3
commit 775fefa804
17 changed files with 926 additions and 42 deletions
@@ -2,12 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import Any
from unittest.mock import patch
from unittest.mock import MagicMock, patch
import pytest
from prometheus_client import Counter, Gauge, Histogram
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorLogging
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
KV_OFFLOAD_CONFIG_INFO,
OffloadingConnectorStats,
OffloadPromMetrics,
_ConnectorMetricName,
@@ -115,15 +117,21 @@ class _FakeVllmConfig:
def _spec_cls_with_metric_definitions(
metric_definitions: dict[str, Any],
info_keys: tuple[str, ...] = (),
) -> type:
"""Build a fake offloading spec class reporting the given metric
definitions, so tests don't need to patch the real CPU spec."""
definitions and info label names, so tests don't need to patch the real
CPU spec."""
class _FakeOffloadingSpec:
@staticmethod
def build_metric_definitions(extra_config):
return metric_definitions
@staticmethod
def config_info_keys(extra_config):
return info_keys
return _FakeOffloadingSpec
@@ -784,3 +792,333 @@ def test_prom_metrics_rejects_undeclared_metric():
_StatsKey.DATA: {"unknown:metric": {(): 1}},
}
)
def _prom_metrics(
metric_definitions: dict[str, Any] | None = None,
info_keys: tuple[str, ...] = (),
):
"""Build OffloadPromMetrics over fake metric classes, with one engine."""
spec_cls = _spec_cls_with_metric_definitions(metric_definitions or {}, info_keys)
with patch.object(OffloadingSpecFactory, "get_spec_cls", return_value=spec_cls):
return OffloadPromMetrics(
vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"], 1: ["model", "1"]},
)
def test_prom_metrics_keeps_a_gauge_declared_multiprocess_mode():
"""A gauge that declares its own multiprocess_mode keeps it.
The value on OffloadingGaugeMetadata must stay a default. A hardcoded
"mostrecent" in _create_metric passes the info-gauge test below and
silently overrides a gauge that needs another merge, for example a real
cross-frontend sum.
"""
prom_metrics = _prom_metrics(
{
PENDING_STORES: OffloadingGaugeMetadata(
documentation="gauge declaring a non-default multiprocess mode.",
multiprocess_mode="sum",
),
}
)
gauge_def = prom_metrics._offloading_metric_defs[PENDING_STORES]
assert gauge_def.kwargs["multiprocess_mode"] == "sum"
def test_prom_metrics_omits_multiprocess_mode_outside_a_gauge():
"""multiprocess_mode reaches a gauge only.
prometheus_client accepts the argument on Gauge alone, so hoisting the
kwarg out of the gauge branch breaks construction of a real Counter or
Histogram.
"""
prom_metrics = _prom_metrics(
{
MY_COUNTER: OffloadingCounterMetadata(documentation="a counter."),
LOOKUP_LATENCY: OffloadingHistogramMetadata(
documentation="a histogram.",
buckets=(0.1, 1),
),
}
)
counter_def = prom_metrics._offloading_metric_defs[MY_COUNTER]
histogram_def = prom_metrics._offloading_metric_defs[LOOKUP_LATENCY]
assert "multiprocess_mode" not in counter_def.kwargs
assert "multiprocess_mode" not in histogram_def.kwargs
def test_prom_metrics_declares_the_info_gauge_from_the_spec_keys():
"""The API-server process holds no manager, but it does hold the spec, so
it declares the gauge at startup, before any payload."""
prom_metrics = _prom_metrics(info_keys=("cpu_num_chunks", "root_dir_name"))
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
assert gauge_def.kwargs["name"] == KV_OFFLOAD_CONFIG_INFO
assert gauge_def.kwargs["labelnames"] == [
"model_name",
"engine",
"cpu_num_chunks",
"root_dir_name",
]
# The frontend declares the metric and binds no label values, so an engine
# that never reports exposes no series.
assert gauge_def.children == []
prom_metrics.observe(
{_StatsKey.INFO: [{"cpu_num_chunks": 512, "root_dir_name": "a"}]}
)
# Every value renders as a label value, and the metric itself is pinned
# to 1.
(gauge,) = gauge_def.children
assert gauge.labelvalues == ("model", "0", "512", "a")
assert gauge.set_values == [1]
def test_prom_metrics_reads_the_info_payload_by_label_name():
"""A manager orders config_info() as it likes, and msgpack keeps that
order, so the frontend must key on the name rather than on the position."""
prom_metrics = _prom_metrics(info_keys=("first", "second"))
prom_metrics.observe({_StatsKey.INFO: [{"second": 2, "first": 1}]})
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
(gauge,) = gauge_def.children
assert gauge.labelvalues == ("model", "0", "1", "2")
def test_prom_metrics_reports_one_info_series_for_each_mapping():
"""A manager reports one mapping for each series, and a label of its own
tells the series apart. Two mappings of one kind agree on every other value,
so without that label they collapse into one series."""
prom_metrics = _prom_metrics(info_keys=("source", "root_dir_name"))
prom_metrics.observe(
{
_StatsKey.INFO: [
{"source": "1", "root_dir_name": "a"},
{"source": "2", "root_dir_name": "b"},
]
}
)
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
assert [child.labelvalues for child in gauge_def.children] == [
("model", "0", "1", "a"),
("model", "0", "2", "b"),
]
assert [child.set_values for child in gauge_def.children] == [[1], [1]]
def test_prom_metrics_declares_the_info_gauge_as_most_recent():
"""The info gauge must merge across frontends by freshest write.
Offloading stats reach one frontend per step as complete per-engine
snapshots, so summing would report the number of participating API-server
processes instead of the 1 an info gauge is pinned to -- wrong only under
multiprocess deployment, and silently, since the facts ride the labels.
"""
prom_metrics = _prom_metrics(info_keys=("cpu_num_chunks",))
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
assert gauge_def.kwargs["multiprocess_mode"] == "mostrecent"
def test_prom_metrics_reports_the_info_gauge_without_manager_labels():
"""A spec that declares no fact still yields the metric, so its presence
alone answers whether offloading runs."""
prom_metrics = _prom_metrics()
prom_metrics.observe({_StatsKey.INFO: [{}]})
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
assert gauge_def.kwargs["labelnames"] == ["model_name", "engine"]
(gauge,) = gauge_def.children
assert gauge.labelvalues == ("model", "0")
def test_prom_metrics_empties_a_declared_info_label_no_manager_fills():
"""A name the spec declares and the manager skips must publish empty.
PromQL reads an empty label value as an absent label, so the series still
matches both shapes. This code runs on the engine output path of the API
server, where an exception ends the output handler, so it warns and goes
on rather than drop the whole series.
"""
prom_metrics = _prom_metrics(info_keys=("filled", "skipped"))
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics.logger"
) as mock_logger:
prom_metrics.observe({_StatsKey.INFO: [{"filled": 512}]})
assert mock_logger.warning_once.call_count == 1
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
(gauge,) = gauge_def.children
assert gauge.labelvalues == ("model", "0", "512", "")
def test_prom_metrics_keeps_quiet_when_a_name_belongs_to_another_series():
"""A name that one series owns reads empty on the other series by design.
The declaration holds the names of every series, so the frontend checks the
payload as a whole. A per-series check would read this normal shape as a
disagreement, and warn at every start of a server that reports several
series.
"""
prom_metrics = _prom_metrics(info_keys=("source", "root_dir_name", "bucket"))
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics.logger"
) as mock_logger:
prom_metrics.observe(
{
_StatsKey.INFO: [
{"source": "1", "root_dir_name": "kv_a"},
{"source": "2", "bucket": "kv_bucket"},
]
}
)
mock_logger.warning_once.assert_not_called()
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
assert [child.labelvalues for child in gauge_def.children] == [
("model", "0", "1", "kv_a", ""),
("model", "0", "2", "", "kv_bucket"),
]
def test_prom_metrics_drops_an_info_label_the_spec_did_not_declare():
"""A Prometheus metric binds its label names once, so a name that the spec
did not declare cannot join the gauge. It warns and publishes the rest."""
prom_metrics = _prom_metrics(info_keys=("declared",))
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics.logger"
) as mock_logger:
prom_metrics.observe({_StatsKey.INFO: [{"declared": 512, "undeclared": 1}]})
assert mock_logger.warning_once.call_count == 1
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
assert gauge_def.kwargs["labelnames"] == ["model_name", "engine", "declared"]
(gauge,) = gauge_def.children
assert gauge.labelvalues == ("model", "0", "512")
def test_prom_metrics_reports_the_info_of_every_engine():
"""Every engine of one instance runs one offloading config, so they share
the gauge and differ by the engine label alone."""
prom_metrics = _prom_metrics(info_keys=("cpu_num_chunks",))
prom_metrics.observe({_StatsKey.INFO: [{"cpu_num_chunks": 512}]}, engine_idx=0)
prom_metrics.observe({_StatsKey.INFO: [{"cpu_num_chunks": 512}]}, engine_idx=1)
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
assert [child.labelvalues for child in gauge_def.children] == [
("model", "0", "512"),
("model", "1", "512"),
]
def test_stats_hold_the_config_info_without_an_observation():
"""An info-only payload must survive the scheduler gate, which drops an
empty payload, and must add no line to the interval log."""
stats = OffloadingConnectorStats()
assert stats.is_empty()
stats.set_info([{"cpu_num_chunks": 512}])
assert not stats.is_empty()
assert stats.reduce() == {}
def test_aggregate_keeps_the_config_info_of_the_other_payload():
"""The scheduler payload holds the info and merges into the worker payload
(v1/core/sched/scheduler.py), so a dropped info would hide the metric on
every step that also transfers, which is every step that matters."""
worker_stats = OffloadingConnectorStats()
worker_stats.increase_counter(LOAD_BYTES, 100)
scheduler_stats = OffloadingConnectorStats()
scheduler_stats.set_info([{"cpu_num_chunks": 512}])
worker_stats.aggregate(scheduler_stats)
assert worker_stats.data[_StatsKey.INFO] == [{"cpu_num_chunks": 512}]
assert worker_stats.data[_StatsKey.DATA][LOAD_BYTES] == {(): 100}
def test_aggregate_keeps_the_config_info_of_a_later_payload():
"""The scheduler sends the info once, so a later payload must not clear
it."""
stats = OffloadingConnectorStats()
stats.set_info([{"cpu_num_chunks": 512}])
later_stats = OffloadingConnectorStats()
later_stats.increase_counter(LOAD_BYTES, 100)
stats.aggregate(later_stats)
assert stats.data[_StatsKey.INFO] == [{"cpu_num_chunks": 512}]
def test_metric_sections_leave_out_the_config_info():
"""A manager accesses its own metrics by a walk over the metric sections, as
KVCRSecondaryTierManager does in v1/kv_offload/tiering/kvcr/manager.py. The
info section holds label names, not metric names, so it stays out of that
walk."""
stats = OffloadingConnectorStats()
stats.increase_counter(MY_COUNTER, 2, ("remote_deliver",))
stats.set_info([{"info_label": "local"}])
info = stats.data[_StatsKey.INFO]
sections = stats.metric_sections()
assert not any(section is info for section in sections)
assert sections == (stats.data[_StatsKey.TYPES], stats.data[_StatsKey.DATA])
def test_logging_writes_no_line_for_an_info_only_payload():
"""An info-only payload reaches the logging path, and reduce() drops the info.
The info belongs to the Prometheus frontend only. Without the guard in
KVConnectorLogging.log(), such a payload writes "KV Transfer metrics: "
with an empty body.
"""
stats = OffloadingConnectorStats()
stats.set_info([{"info_label": "local"}])
assert not stats.is_empty()
assert stats.reduce() == {}
kv_logging = KVConnectorLogging(None)
kv_logging.transfer_stats_accumulator = stats
log_fn = MagicMock()
kv_logging.log(log_fn=log_fn)
log_fn.assert_not_called()
assert kv_logging.transfer_stats_accumulator is None
def test_scheduler_sends_the_config_info_once(request_runner):
"""MockOffloadingSpec publishes no fact, so the payload holds one empty
mapping, which still yields the metric."""
runner = request_runner(
block_size=4,
num_gpu_blocks=10,
async_scheduling=False,
)
stats = runner.connector_scheduler.get_stats()
assert stats is not None
assert stats.data[_StatsKey.INFO] == [{}]
assert runner.connector_scheduler.get_stats() is None
@@ -132,6 +132,7 @@ class MockOffloadingSpec(OffloadingSpec):
self.manager.lookup.return_value = LookupResult.MISS
self.manager.get_load_source.return_value = CacheHitSource.EXTERNAL_UNSPECIFIED
self.manager.get_stats.return_value = None
self.manager.config_info.return_value = [{}]
self.manager.on_new_request.return_value = RequestOffloadingContext()
self.handler = MockOffloadingWorker()
+10
View File
@@ -24,6 +24,7 @@ from vllm.v1.kv_offload.cpu.common import (
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
def make_req_context(
@@ -238,6 +239,15 @@ def test_filter_reused_manager_reports_stores_skipped_counter():
assert stats.reduce()[CPUOffloadingMetrics.STORES_SKIPPED] == 0
def test_cpu_manager_fills_the_config_info_keys_of_the_spec():
"""A name the declaration misses becomes a dropped label at runtime."""
manager = make_cpu_manager(num_chunks=4)
(info,) = manager.config_info()
assert tuple(info) == CPUOffloadingSpec.config_info_keys({})
assert info["cpu_num_chunks"] == 4
def test_cpu_manager_reports_cache_usage_gauge():
def check_usage_stats(manager: CPUOffloadingManager, value: float):
stats = manager.get_stats()
@@ -11,7 +11,7 @@ These tests verify:
"""
from collections.abc import Iterable
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
import numpy as np
import pytest
@@ -99,7 +99,8 @@ def has_histogram_count(reduced: dict[str, int | float], metric_name: str) -> bo
class MetricsSecondaryTierManager(SecondaryTierManager):
"""Test-only secondary tier that declares and emits one labeled metric."""
"""Test-only secondary tier that declares and emits one labeled metric,
and publishes one config fact."""
MY_TIER_METRIC = "my_tier_metric"
@@ -112,6 +113,13 @@ class MetricsSecondaryTierManager(SecondaryTierManager):
)
}
@classmethod
def config_info_keys(cls, extra_config):
return ("path",)
def config_info(self):
return {"path": f"/mnt/{self.tier_type}"}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.stats: OffloadingConnectorStats | None = None
@@ -248,6 +256,19 @@ class TestExampleSecondaryTierManager:
# Third chunk not present
assert tier.lookup(chunks[2], _CTX) is LookupResult.MISS
def test_config_info_keys_match_the_filled_values(self):
"""A name the declaration misses becomes a dropped label at runtime."""
mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy())
tier = ExampleSecondaryTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=mock_view,
tier_type="example",
custom_param=67,
)
info = tier.config_info()
assert tuple(info) == ExampleSecondaryTierManager.config_info_keys({})
assert info["example_info"] == 67
# What a request-level cascade does with a key already present in the primary
# tier, keyed by what primary lookup says about it. SUPPLY reaches the peer now,
@@ -1545,3 +1566,175 @@ def test_parse_tier_filter_skips_bad_entries():
if __name__ == "__main__":
pytest.main([__file__, "-v"])
def test_tiering_manager_reports_one_config_info_series_for_each_tier():
"""The info metric holds one series for each tier, and the tier label tells
the series apart. A tier cannot know its own index, so the manager adds the
label. A tier fills the names it owns only, so the CPU primary tier fills
its cpu_ labels on its own series alone."""
mock_region = _mock_mmap_region(5)
primary_tier = CPUPrimaryTierOffloadingManager(
num_chunks=5, mmap_region=mock_region
)
secondary_tiers = [
MetricsSecondaryTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=mock_region.create_kv_memoryview(),
tier_type="test_metrics",
)
for _ in range(2)
]
manager = TieringOffloadingManager(
primary_tier=primary_tier,
secondary_tiers=secondary_tiers,
)
assert manager.config_info() == [
{"tier": "0:primary", "cpu_num_chunks": 5},
{"tier": "1:test_metrics", "path": "/mnt/test_metrics"},
{"tier": "2:test_metrics", "path": "/mnt/test_metrics"},
]
def test_tiering_manager_reports_the_primary_series_with_no_secondary_tier():
"""A tiering manager runs with no secondary tier, and the info metric then
holds the primary series alone. The series still carries the tier label, so
the frontend publishes one series and not a bare metric."""
mock_region = _mock_mmap_region(5)
manager = TieringOffloadingManager(
primary_tier=CPUPrimaryTierOffloadingManager(
num_chunks=5, mmap_region=mock_region
),
secondary_tiers=[],
)
assert manager.config_info() == [{"tier": "0:primary", "cpu_num_chunks": 5}]
def test_tiering_manager_rejects_a_primary_tier_with_two_info_mappings():
"""This manager holds one primary tier, so two mappings would give two
series with one tier label. If the two mappings agree on every declared
name, the frontend cache then collapses them into one series and warns
about nothing. The unpack fails at engine start instead."""
class _TwoMappingPrimary(CPUPrimaryTierOffloadingManager):
def config_info(self):
return [{"path": "/a"}, {"path": "/b"}]
mock_region = _mock_mmap_region(5)
manager = TieringOffloadingManager(
primary_tier=_TwoMappingPrimary(num_chunks=5, mmap_region=mock_region),
secondary_tiers=[],
)
with pytest.raises(ValueError):
manager.config_info()
def test_tiering_manager_keeps_the_tier_label_out_of_the_tier_mapping():
"""The manager adds the tier label to a copy. A tier that returns a stored
mapping must not gain a foreign key that outlives the call."""
class _StoredInfoTier(MetricsSecondaryTierManager):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.info: dict[str, str] = {"path": "/mnt/a"}
def config_info(self):
return self.info
mock_region = _mock_mmap_region(5)
tier = _StoredInfoTier(
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=mock_region.create_kv_memoryview(),
tier_type="test_metrics",
)
manager = TieringOffloadingManager(
primary_tier=CPUPrimaryTierOffloadingManager(
num_chunks=5, mmap_region=mock_region
),
secondary_tiers=[tier],
)
manager.config_info()
assert tier.info == {"path": "/mnt/a"}
def test_tiering_spec_declares_the_config_info_keys_the_manager_fills():
"""The spec declares the info label names in the API-server process, and
the manager fills the values in the engine process. A drift between the two
empties every declared label and drops every filled value, so this pins
both sides to the same literal names. The frontend reads a value by name,
so the order of the names does not have to agree."""
tier_configs = [{"type": "test_metrics"}, {"type": "test_metrics"}]
mock_region = _mock_mmap_region(5)
manager = TieringOffloadingManager(
primary_tier=CPUPrimaryTierOffloadingManager(
num_chunks=5, mmap_region=mock_region
),
secondary_tiers=[
MetricsSecondaryTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=mock_region.create_kv_memoryview(),
tier_type=tier_config["type"],
)
for tier_config in tier_configs
],
)
with patch.object(
SecondaryTierFactory,
"get_tier_class",
return_value=MetricsSecondaryTierManager,
):
keys = TieringOffloadingSpec.config_info_keys({"secondary_tiers": tier_configs})
assert keys == ("cpu_num_chunks", "tier", "path")
filled: set[str] = set()
for info in manager.config_info():
assert set(info) <= set(keys)
filled |= set(info)
assert filled == set(keys)
def test_tiering_spec_passes_each_tier_its_own_config():
"""A tier reads its own parameters out of its own config dict, not out of
the instance-wide kv_connector_extra_config. A tier that names a label
after one of its own keys declares nothing from the wrong dict."""
seen: list[dict] = []
class _RecordingTier(MetricsSecondaryTierManager):
@classmethod
def config_info_keys(cls, extra_config):
seen.append(extra_config)
return ()
tier_configs = [{"type": "test_metrics", "root_dir": "/mnt/a"}]
with patch.object(
SecondaryTierFactory, "get_tier_class", return_value=_RecordingTier
):
TieringOffloadingSpec.config_info_keys({"secondary_tiers": tier_configs})
assert seen == tier_configs
def test_tiering_spec_rejects_a_tier_that_declares_the_reserved_tier_label():
"""TieringOffloadingManager.config_info() writes the tier label over the
value of a tier, so a tier that declares the name loses its own value with
no message. The declaration must fail at start-up instead."""
class _ReservedLabelTier(MetricsSecondaryTierManager):
@classmethod
def config_info_keys(cls, extra_config):
return ("tier",)
tier_configs = [{"type": "test_metrics"}]
with (
patch.object(
SecondaryTierFactory, "get_tier_class", return_value=_ReservedLabelTier
),
pytest.raises(AssertionError, match="reserved label 'tier'"),
):
TieringOffloadingSpec.config_info_keys({"secondary_tiers": tier_configs})