[Metrics][KV Offload] Add Prometheus metrics for SimpleCPUOffloadConnector (#57251)

Signed-off-by: Vincent <vincexxchan@gmail.com>
This commit is contained in:
mevince
2026-09-26 17:17:11 -07:00
committed by GitHub
parent 4be061c5af
commit cd1daf58a9
6 changed files with 903 additions and 6 deletions
@@ -23,6 +23,10 @@ METRIC_SOURCE_FILES = [
"path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py",
"key": "metrics-nixl",
},
{
"path": "vllm/v1/simple_kv_offload/metrics.py",
"key": "metrics-simple-kv-offload",
},
{"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"},
]
+24
View File
@@ -45,6 +45,30 @@ The following metrics are exposed:
--8<-- "gen:metrics-nixl"
## Simple CPU Offload Connector Metrics
These metrics are exposed when the `SimpleCPUOffloadConnector` KV connector
is configured (e.g. `--kv-transfer-config='{"kv_connector":
"SimpleCPUOffloadConnector", "kv_role": "kv_both", "kv_connector_extra_config":
{"kv_offload_backend": "disk", "disk_path": "/mnt/nvme/kv"}}'`). They are
updated once per engine step.
Caveats to keep in mind when interpreting them:
- A "completed" store means the write syscalls returned; it is **not**
fsync-durable, and the disk backend's file is process-lifetime scratch,
unlinked at startup and shutdown.
- `save_outcomes_total` classifies eager-mode boundary hand-off stores;
lazy-mode stores are not classified.
- `used_blocks` counts blocks pinned by in-flight transfers or cache hits;
warm cached blocks that are evictable are not counted. Use the
`capacity_blocks` label of `simple_kv_offload_info` as the denominator.
- Counters and gauges are quantized to engine steps. They are reported by the
scheduler process and reflect engine-wide logical block counts, not values
pooled from individual tensor-parallel workers.
--8<-- "gen:metrics-simple-kv-offload"
## Model Flops Utilization (MFU) Performance Metrics
These metrics are available via `--enable-mfu-metrics`:
@@ -0,0 +1,466 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for SimpleCPUOffloadConnector metrics."""
from __future__ import annotations
import dataclasses
from types import SimpleNamespace
from typing import Any
import pytest
from prometheus_client import Counter, Gauge, Histogram
from tests.v1.kv_connector.unit.test_simple_cpu_offload_connector import (
_make_connector,
)
from tests.v1.simple_kv_offload.test_scheduler import (
make_request,
make_scheduler,
make_scheduler_output,
)
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
SimpleCPUOffloadConnector,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.simple_kv_offload.manager import (
BoundaryStoreStats,
LoadRequestState,
TransferMeta,
)
from vllm.v1.simple_kv_offload.metrics import (
INFO_LABELS,
OUTCOME_TO_FIELD,
MetricName,
SimpleCPUOffloadStats,
_MetricType,
_StatsKey,
)
SAVE_OUTCOMES = MetricName.SAVE_OUTCOMES
LOAD_BLOCKS = MetricName.LOAD_BLOCKS
USED_BLOCKS = MetricName.USED_BLOCKS
PENDING_STORE_BLOCKS = MetricName.PENDING_STORE_BLOCKS
INFO = MetricName.INFO
class _FakeMetric:
def __init__(self, **kwargs: Any):
self.kwargs = kwargs
self.children: list[_FakeMetric] = []
self.observed: list[int | float] = []
self.increments: list[int | float] = []
self.set_values: list[int | float] = []
self.labelvalues: tuple[object, ...] = ()
def labels(self, *labelvalues: object) -> _FakeMetric:
child = _FakeMetric(**self.kwargs)
child.labelvalues = labelvalues
self.children.append(child)
return child
def observe(self, value: int | float) -> None:
self.observed.append(value)
def inc(self, value: int | float) -> None:
self.increments.append(value)
def set(self, value: int | float) -> None:
self.set_values.append(value)
def _fake_metric_types() -> dict[type, type]:
return {Gauge: _FakeMetric, Counter: _FakeMetric, Histogram: _FakeMetric}
def _make_prom_metrics(per_engine: dict[int, list[object]] | None = None) -> Any:
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
return SimpleCPUOffloadPromMetrics(
vllm_config=SimpleNamespace(kv_transfer_config=None),
metric_types=_fake_metric_types(),
labelnames=["model_name", "engine"],
per_engine_labelvalues=per_engine or {0: ["model", "0"]},
)
def _stats_payload(types: dict, data: dict) -> dict[str, Any]:
return {_StatsKey.TYPES: types, _StatsKey.DATA: data}
def _values(stats: SimpleCPUOffloadStats, name: str) -> dict:
return stats.data[_StatsKey.DATA][name]
# ---------------------------------------------------------------------------
# Stats class
# ---------------------------------------------------------------------------
def test_build_kv_connector_stats_none_and_empty() -> None:
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data=None)
assert isinstance(stats, SimpleCPUOffloadStats)
assert stats.is_empty()
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data={})
assert isinstance(stats, SimpleCPUOffloadStats)
assert stats.is_empty()
def test_stats_round_trip_through_serialized_dict() -> None:
stats = SimpleCPUOffloadStats()
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
stats.increase_counter(LOAD_BLOCKS, 4)
stats.set_gauge(USED_BLOCKS, 7)
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
rebuilt = SimpleCPUOffloadConnector.build_kv_connector_stats(data=stats.to_dict())
assert isinstance(rebuilt, SimpleCPUOffloadStats)
assert _values(rebuilt, SAVE_OUTCOMES) == {("stored",): 3}
assert _values(rebuilt, LOAD_BLOCKS) == {(): 4}
assert _values(rebuilt, USED_BLOCKS) == {(): 7}
assert _values(rebuilt, INFO) == {("cpu", "false", "false", "64"): 1}
def test_stats_aggregate_sums_counters_and_keeps_latest_gauge() -> None:
stats1 = SimpleCPUOffloadStats()
stats1.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
stats1.increase_counter(SAVE_OUTCOMES, 1, ("dropped_cpu_full",))
stats1.set_gauge(USED_BLOCKS, 7)
stats2 = SimpleCPUOffloadStats()
stats2.increase_counter(SAVE_OUTCOMES, 4, ("stored",))
stats2.set_gauge(USED_BLOCKS, 2)
result = stats1.aggregate(stats2)
assert result is stats1
outcomes = _values(stats1, SAVE_OUTCOMES)
assert outcomes == {("stored",): 7, ("dropped_cpu_full",): 1}
assert _values(stats1, USED_BLOCKS) == {(): 2}
def test_stats_aggregate_merges_types_from_other() -> None:
stats1 = SimpleCPUOffloadStats()
stats1.set_gauge(USED_BLOCKS, 1)
stats2 = SimpleCPUOffloadStats()
stats2.increase_counter(LOAD_BLOCKS, 2)
stats1.aggregate(stats2)
assert stats1.data[_StatsKey.TYPES][LOAD_BLOCKS] == _MetricType.COUNTER
assert _values(stats1, LOAD_BLOCKS) == {(): 2}
def test_stats_reduce_and_reset() -> None:
stats = SimpleCPUOffloadStats()
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
stats.set_gauge(PENDING_STORE_BLOCKS, 4)
stats.increase_counter(LOAD_BLOCKS, 9)
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
reduced = stats.reduce()
assert reduced[f"{SAVE_OUTCOMES}:{('stored',)}"] == 3
assert reduced[PENDING_STORE_BLOCKS] == 4
assert reduced[LOAD_BLOCKS] == 9
assert not any(key.startswith(INFO) for key in reduced)
stats.reset()
assert stats.is_empty()
def test_outcome_labels_match_boundary_store_stats_fields() -> None:
"""Every outcome label maps 1:1 to a BoundaryStoreStats counter field."""
boundary_fields = {
f.name for f in dataclasses.fields(BoundaryStoreStats) if f.name != "published"
}
assert set(OUTCOME_TO_FIELD.values()) == boundary_fields
assert len(set(OUTCOME_TO_FIELD.values())) == len(OUTCOME_TO_FIELD)
# ---------------------------------------------------------------------------
# Prom metrics class
# ---------------------------------------------------------------------------
def test_prom_metrics_registers_tier1_metrics() -> None:
prom = _make_prom_metrics()
defs = prom._defs
assert set(defs) == {
SAVE_OUTCOMES,
LOAD_BLOCKS,
USED_BLOCKS,
PENDING_STORE_BLOCKS,
INFO,
}
# The prom class registers literal names (required by the docs
# generator); they must stay in sync with the emission-side constants.
assert {m.kwargs["name"] for m in defs.values()} == {
SAVE_OUTCOMES,
LOAD_BLOCKS,
USED_BLOCKS,
PENDING_STORE_BLOCKS,
INFO,
}
for metric in defs.values():
assert metric.kwargs["documentation"]
assert metric.kwargs["labelnames"][:2] == ["model_name", "engine"]
assert defs[SAVE_OUTCOMES].kwargs["labelnames"] == [
"model_name",
"engine",
"outcome",
]
assert defs[LOAD_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
assert defs[INFO].kwargs["labelnames"] == [
"model_name",
"engine",
*INFO_LABELS,
]
assert defs[USED_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
# mostrecent: multiprocess scrapes expose one series per label set, not
# one pid-tagged series per writer process.
for gauge in (USED_BLOCKS, PENDING_STORE_BLOCKS, INFO):
assert defs[gauge].kwargs["multiprocess_mode"] == "mostrecent"
def test_prom_metrics_observe_routes_each_series() -> None:
prom = _make_prom_metrics()
prom.observe(
_stats_payload(
types={
SAVE_OUTCOMES: _MetricType.COUNTER,
LOAD_BLOCKS: _MetricType.COUNTER,
USED_BLOCKS: _MetricType.GAUGE,
PENDING_STORE_BLOCKS: _MetricType.GAUGE,
INFO: _MetricType.GAUGE,
},
data={
SAVE_OUTCOMES: {("stored",): 3},
LOAD_BLOCKS: {(): 4},
USED_BLOCKS: {(): 5},
PENDING_STORE_BLOCKS: {(): 1},
INFO: {("cpu", "false", "false", "64"): 1},
},
)
)
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].increments == [3]
assert prom._metrics[(0, LOAD_BLOCKS, ())].increments == [4]
assert prom._metrics[(0, USED_BLOCKS, ())].set_values == [5]
assert prom._metrics[(0, PENDING_STORE_BLOCKS, ())].set_values == [1]
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].set_values == [1]
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].labelvalues == (
"model",
"0",
"stored",
)
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].labelvalues == (
"model",
"0",
"cpu",
"false",
"false",
"64",
)
def test_prom_metrics_reuses_bound_children() -> None:
prom = _make_prom_metrics()
payload = _stats_payload(
types={SAVE_OUTCOMES: _MetricType.COUNTER},
data={SAVE_OUTCOMES: {("stored",): 1}},
)
prom.observe(payload)
prom.observe(payload)
child = prom._metrics[(0, SAVE_OUTCOMES, ("stored",))]
assert child.increments == [1, 1]
assert len(prom._defs[SAVE_OUTCOMES].children) == 1
def test_prom_metrics_rejects_unknown_metric() -> None:
prom = _make_prom_metrics()
with pytest.raises(AssertionError, match="Unknown"):
prom.observe(
_stats_payload(
types={"vllm:not_a_metric": _MetricType.COUNTER},
data={"vllm:not_a_metric": {(): 1}},
)
)
def test_prom_metrics_rejects_wrong_label_count() -> None:
prom = _make_prom_metrics()
with pytest.raises(AssertionError, match="labels"):
prom.observe(
_stats_payload(
types={SAVE_OUTCOMES: _MetricType.COUNTER},
data={SAVE_OUTCOMES: {("stored", "extra"): 1}},
)
)
def test_prom_metrics_routes_per_engine() -> None:
prom = _make_prom_metrics(per_engine={0: ["model", "0"], 1: ["model", "1"]})
prom.observe(
_stats_payload(
types={USED_BLOCKS: _MetricType.GAUGE}, data={USED_BLOCKS: {(): 3}}
),
engine_idx=1,
)
assert (0, USED_BLOCKS, ()) not in prom._metrics
assert prom._metrics[(1, USED_BLOCKS, ())].set_values == [3]
def test_connector_build_prom_metrics() -> None:
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
prom = SimpleCPUOffloadConnector.build_prom_metrics(
vllm_config=SimpleNamespace(kv_transfer_config=None),
metric_types=_fake_metric_types(),
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
assert isinstance(prom, SimpleCPUOffloadPromMetrics)
# ---------------------------------------------------------------------------
# Manager: interval stats
# ---------------------------------------------------------------------------
def test_manager_get_stats_reports_outcome_deltas_per_interval() -> None:
fixture = make_scheduler()
sched = fixture.scheduler
sched.boundary_store_stats.stored += 3
sched.boundary_store_stats.dropped_cpu_full += 2
sched.boundary_store_stats.skipped_in_flight += 1
stats = sched.get_stats()
assert _values(stats, SAVE_OUTCOMES) == {
("stored",): 3,
("dropped_cpu_full",): 2,
("skipped_in_flight",): 1,
}
# Deltas are per-interval: a second drain reports no new outcomes.
stats2 = sched.get_stats()
assert SAVE_OUTCOMES not in stats2.data[_StatsKey.DATA]
def test_manager_reset_preserves_undrained_outcome_deltas() -> None:
fixture = make_scheduler()
sched = fixture.scheduler
sched.boundary_store_stats.stored += 3
sched.reset()
stats = sched.get_stats()
assert _values(stats, SAVE_OUTCOMES) == {("stored",): 3}
def test_manager_get_stats_gauges_and_info_labels() -> None:
fixture = make_scheduler(num_cpu_blocks=8)
sched = fixture.scheduler
sched._store_event_to_blocks[0] = TransferMeta([10], [11])
sched._pending_finished_stores.append(TransferMeta([12], [13]))
sched._abandoned_store_event_to_blocks[1] = TransferMeta([14], [15])
stats = sched.get_stats()
assert _values(stats, PENDING_STORE_BLOCKS) == {(): 3}
used = _values(stats, USED_BLOCKS)[()]
assert used == sched.num_cpu_blocks - sched.cpu_block_pool.get_num_free_blocks()
info = _values(stats, INFO)
assert info == {("cpu", "false", "false", str(sched.num_cpu_blocks)): 1}
# Gauges are re-reported every drain.
stats2 = sched.get_stats()
assert _values(stats2, PENDING_STORE_BLOCKS) == {(): 3}
def test_manager_info_labels_reflect_disk_mode() -> None:
connector = _make_connector(
extra_config={
"kv_offload_backend": "disk",
"disk_path": "/tmp/fake-disk-metrics",
"disk_capacity_bytes": 1024**3,
"use_page_cache": True,
}
)
sched = connector.scheduler_manager
assert sched is not None
assert sched._info_labelvalues[0] == "disk"
assert sched._info_labelvalues[1] == "true"
stats = sched.get_stats()
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
def test_manager_info_labels_page_cache_false_for_cpu_backend() -> None:
"""CPU backend ignores use_page_cache; the info gauge must not claim it."""
connector = _make_connector(extra_config={"use_page_cache": True})
sched = connector.scheduler_manager
assert sched is not None
assert sched._info_labelvalues[:2] == ("cpu", "false")
stats = sched.get_stats()
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
def test_manager_counts_load_blocks_completed() -> None:
fixture = make_scheduler(num_cpu_blocks=8)
sched = fixture.scheduler
request = make_request(num_blocks=2)
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
sched._reqs_to_load[request.request_id] = LoadRequestState(
request=request,
transfer_meta=TransferMeta(
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
),
)
sched.build_connector_meta(make_scheduler_output({}))
stats = sched.get_stats()
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
sched.update_connector_output(
KVConnectorOutput(finished_recving={request.request_id})
)
stats = sched.get_stats()
assert _values(stats, LOAD_BLOCKS) == {(): 2}
def test_manager_abandoned_load_counts_as_completed_after_reset() -> None:
"""Loads abandoned by reset() that still finish on the worker are
counted as loaded blocks.
"""
fixture = make_scheduler(num_cpu_blocks=8)
sched = fixture.scheduler
request = make_request(num_blocks=2)
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
sched._reqs_to_load[request.request_id] = LoadRequestState(
request=request,
transfer_meta=TransferMeta(
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
),
)
sched.build_connector_meta(make_scheduler_output({}))
stats = sched.get_stats()
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
assert sched.reset() is False
assert set(sched._abandoned_reqs_to_load) == {request.request_id}
sched.update_connector_output(
KVConnectorOutput(finished_recving={request.request_id})
)
assert sched._abandoned_reqs_to_load == {}
stats = sched.get_stats()
assert _values(stats, LOAD_BLOCKS) == {(): 2}
@@ -15,6 +15,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorRole,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import KVConnectorOutput
@@ -25,6 +31,10 @@ from vllm.v1.simple_kv_offload.manager import (
from vllm.v1.simple_kv_offload.metadata import (
SimpleCPUOffloadMetadata,
)
from vllm.v1.simple_kv_offload.metrics import (
SimpleCPUOffloadPromMetrics,
SimpleCPUOffloadStats,
)
from vllm.v1.simple_kv_offload.worker import (
SimpleCPUOffloadWorker,
)
@@ -162,6 +172,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
hash_block_size=hash_block_size,
lazy_offload=lazy_offload,
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
use_page_cache=use_page_cache if disk_mode else False,
)
elif role == KVConnectorRole.WORKER:
self.worker_handler = SimpleCPUOffloadWorker(
@@ -307,6 +318,38 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
return self.scheduler_manager.get_boundary_store_stats()
return None
def get_kv_connector_stats(self) -> KVConnectorStats | None:
# Worker-side callers may invoke this hook on connectors constructed
# via __new__ (as in test_worker.py), which lack scheduler_manager.
scheduler_manager: SimpleCPUOffloadScheduler | None = getattr(
self, "scheduler_manager", None
)
if scheduler_manager is not None:
return scheduler_manager.get_stats()
return None
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
SimpleCPUOffloadStats(data=data)
if data is not None
else SimpleCPUOffloadStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return SimpleCPUOffloadPromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
def take_events(self) -> Iterable[KVCacheEvent]:
if self.scheduler_manager is not None:
return self.scheduler_manager.take_events()
+82 -6
View File
@@ -4,7 +4,7 @@
import contextlib
from collections.abc import Iterable
from dataclasses import dataclass, field, replace
from dataclasses import dataclass, field, fields, replace
from enum import Enum, auto
from typing import TYPE_CHECKING, Any
@@ -48,6 +48,11 @@ from vllm.v1.simple_kv_offload.metadata import (
SimpleCPUOffloadMetadata,
SimpleCPUOffloadWorkerMetadata,
)
from vllm.v1.simple_kv_offload.metrics import (
OUTCOME_TO_FIELD,
MetricName,
SimpleCPUOffloadStats,
)
if TYPE_CHECKING:
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
@@ -118,7 +123,8 @@ class BoundaryStoreStats:
symptom is a lower cache hit rate with nothing in the logs, so each decline
reason is counted and exposed through
``SimpleCPUOffloadConnector.get_boundary_store_stats()``. Reset by
``reset()``; not otherwise cleared.
``reset()``, which carries any not-yet-drained deltas; not otherwise
cleared.
"""
published: int = 0
@@ -143,6 +149,7 @@ class SimpleCPUOffloadScheduler:
hash_block_size: int,
lazy_offload: bool = False,
disk_capacity_bytes: int = 0,
use_page_cache: bool = False,
):
self.vllm_config = vllm_config
self.kv_cache_config = kv_cache_config
@@ -169,6 +176,13 @@ class SimpleCPUOffloadScheduler:
self.cpu_kv_cache_config.prefix_cacheable_group_ids
)
self.kv_event_medium = MEDIUM_STORAGE if disk_capacity_bytes > 0 else MEDIUM_CPU
# Must track kv_event_medium.
self._info_labelvalues: tuple[str, ...] = (
"disk" if disk_capacity_bytes > 0 else "cpu",
str(use_page_cache).lower(),
str(lazy_offload).lower(),
str(self.num_cpu_blocks),
)
# Find the full attention kv group for prefix cache matching.
self.fa_gidx = -1
for g_idx, g in enumerate(self.cpu_kv_cache_config.kv_cache_groups):
@@ -251,6 +265,9 @@ class SimpleCPUOffloadScheduler:
self._load_event_counter: int = 0
self._store_event_counter: int = 0
self.boundary_store_stats = BoundaryStoreStats()
# Interval stats state drained by get_stats()
self._boundary_stats_snapshot = BoundaryStoreStats()
self._interval_load_blocks_completed = 0
# For TP/PP: track partial store completions across steps.
# Events must be reported by all world_size workers before considered complete.
@@ -951,6 +968,42 @@ class SimpleCPUOffloadScheduler:
def get_boundary_store_stats(self) -> BoundaryStoreStats:
return replace(self.boundary_store_stats)
def get_stats(self) -> SimpleCPUOffloadStats:
"""Drain per-step stats for the connector's stats hooks."""
stats = SimpleCPUOffloadStats()
current = self.boundary_store_stats
for outcome, field_name in OUTCOME_TO_FIELD.items():
delta = getattr(current, field_name) - getattr(
self._boundary_stats_snapshot, field_name
)
if delta > 0:
stats.increase_counter(MetricName.SAVE_OUTCOMES, delta, (outcome,))
self._boundary_stats_snapshot = replace(current)
if self._interval_load_blocks_completed:
stats.increase_counter(
MetricName.LOAD_BLOCKS,
self._interval_load_blocks_completed,
)
self._interval_load_blocks_completed = 0
stats.set_gauge(
MetricName.USED_BLOCKS,
self.num_cpu_blocks - self.cpu_block_pool.get_num_free_blocks(),
)
pending = sum(
len(t.cpu_block_ids)
for t in (
*self._store_event_to_blocks.values(),
*self._pending_finished_stores,
*self._abandoned_store_event_to_blocks.values(),
)
)
stats.set_gauge(MetricName.PENDING_STORE_BLOCKS, pending)
stats.set_gauge(MetricName.INFO, 1, self._info_labelvalues)
return stats
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
"""Handle async transfer completions from worker.
@@ -961,7 +1014,9 @@ class SimpleCPUOffloadScheduler:
"""
# --- Load completions ---
for req_id in list(connector_output.finished_recving or []):
self._cleanup_load_request(req_id)
completed_blocks = self._cleanup_load_request(req_id)
if completed_blocks:
self._interval_load_blocks_completed += completed_blocks
# --- Store completions ---
meta = connector_output.kv_connector_worker_meta
@@ -1266,18 +1321,22 @@ class SimpleCPUOffloadScheduler:
if blocks_to_free:
self.cpu_block_pool.free_blocks(blocks_to_free)
def _cleanup_load_request(self, req_id: str) -> None:
def _cleanup_load_request(self, req_id: str) -> int:
"""Release all load resources for a request.
Shared between request_finished() and update_connector_output() paths.
Removes the request from _reqs_to_load, cleans up event mappings,
and frees CPU/GPU touch refs.
Returns the number of blocks in the load if it had been issued to
the worker, else 0 (never-issued loads are not counted as completed
by the caller).
"""
state = self._reqs_to_load.pop(req_id, None)
if state is None:
state = self._abandoned_reqs_to_load.pop(req_id, None)
if state is None:
return
return 0
# Remove from load event mapping (only this req, not whole event)
if state.load_event is not None:
reqs = self._load_event_to_reqs.get(state.load_event)
@@ -1300,6 +1359,10 @@ class SimpleCPUOffloadScheduler:
for bid in state.transfer_meta.gpu_block_ids
)
if state.load_event is None:
return 0
return len(state.transfer_meta.gpu_block_ids)
def _cleanup_store_request(self, req_id: str) -> None:
"""Release store metadata for a request.
@@ -1359,7 +1422,20 @@ class SimpleCPUOffloadScheduler:
if event_idx in self._abandoned_store_event_to_blocks
}
self._cursor = None
self.boundary_store_stats = BoundaryStoreStats()
# Seed the fresh counters with outcomes since the last drain so a
# mid-interval reset() does not drop them from the next report.
undrained = BoundaryStoreStats(
**{
f.name: max(
0,
getattr(self.boundary_store_stats, f.name)
- getattr(self._boundary_stats_snapshot, f.name),
)
for f in fields(BoundaryStoreStats)
}
)
self.boundary_store_stats = undrained
self._boundary_stats_snapshot = BoundaryStoreStats()
# NOTE: _load_event_counter / _store_event_counter are not
# reset as they are monotonic and must stay ahead of the workers
# high-water marks to avoid event index collisions
+284
View File
@@ -0,0 +1,284 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Prometheus metrics for SimpleCPUOffloadConnector.
The stats payload uses the same self-describing wire format as the
offloading connector, so it survives IPC serialization without the
metric classes on the receiving side.
"""
from dataclasses import dataclass
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
class MetricName:
"""Metric names for SimpleCPUOffloadConnector."""
SAVE_OUTCOMES = "vllm:simple_kv_offload_save_outcomes_total"
LOAD_BLOCKS = "vllm:simple_kv_offload_load_blocks_total"
USED_BLOCKS = "vllm:simple_kv_offload_used_blocks"
PENDING_STORE_BLOCKS = "vllm:simple_kv_offload_pending_store_blocks"
INFO = "vllm:simple_kv_offload_info"
# outcome label value -> BoundaryStoreStats field name.
# ``published`` is the sum of all outcomes and is not labeled.
OUTCOME_TO_FIELD = {
"stored": "stored",
"dropped_cpu_full": "dropped_cpu_full",
"dropped_request_gone": "dropped_request_gone",
"dropped_null_block": "dropped_null_block",
"dropped_not_hashed": "dropped_not_hashed",
"skipped_already_cached": "skipped_already_cached",
"skipped_in_flight": "skipped_in_flight",
}
INFO_LABELS = ["backend", "page_cache", "lazy_offload", "capacity_blocks"]
class _MetricType:
"""Type tags embedded in the serialized stats payload."""
COUNTER = "counter"
GAUGE = "gauge"
class _StatsKey:
"""Top-level keys in the serialized stats dict."""
# Maps metric name -> _MetricType value
TYPES = "types"
# Maps metric name -> {label values tuple -> observed value}
DATA = "data"
@dataclass
class SimpleCPUOffloadStats(KVConnectorStats):
"""Stats payload for SimpleCPUOffloadConnector.
The ``data`` dict is structured as::
{
_StatsKey.TYPES: {name: _MetricType.*, ...},
_StatsKey.DATA: {name: {labelvalues: value, ...}, ...},
}
Counter values are summed when aggregating, gauge values keep the
latest snapshot, and unlabeled metrics use ``()`` as their
labelvalues tuple.
"""
def __post_init__(self):
if _StatsKey.DATA not in self.data:
self.reset()
def reset(self):
self.data: dict[str, Any] = {
_StatsKey.TYPES: {},
_StatsKey.DATA: {},
}
@property
def _types(self) -> dict[str, str]:
return self.data[_StatsKey.TYPES]
@property
def _values(self) -> dict[str, Any]:
return self.data[_StatsKey.DATA]
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
if other.is_empty():
return self
assert isinstance(other, SimpleCPUOffloadStats)
for key, other_label_values in other._values.items():
type_str = other._types.get(key)
if type_str is None:
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
self._types.setdefault(key, type_str)
current_label_values = self._values.setdefault(key, {})
for labelvalues, value in other_label_values.items():
if type_str == _MetricType.COUNTER:
assert isinstance(value, int | float)
current_label_values[labelvalues] = (
current_label_values.get(labelvalues, 0) + value
)
elif type_str == _MetricType.GAUGE:
assert isinstance(value, int | float)
current_label_values[labelvalues] = value
else:
raise AssertionError(
f"Unknown metric type '{type_str}' for key: {key}"
)
return self
def reduce(self) -> dict[str, int | float]:
"""Reduce the interval observations to a flat summary for logging."""
return_dict: dict[str, int | float] = {}
for key, label_value_map in self._values.items():
if key == MetricName.INFO:
# Constant config fingerprint; exclude from the log summary.
continue
type_str = self._types.get(key)
if type_str is None:
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
for labelvalues, value in label_value_map.items():
key_with_labels = f"{key}:{labelvalues}" if labelvalues else key
if type_str in (_MetricType.COUNTER, _MetricType.GAUGE):
assert isinstance(value, int | float)
return_dict[key_with_labels] = value
else:
raise AssertionError(
f"Unknown metric type '{type_str}' for key: {key}"
)
return return_dict
def is_empty(self) -> bool:
return not self.data.get(_StatsKey.DATA)
def increase_counter(
self,
counter_name: str,
counter_increase_value: int | float = 1,
labelvalues: tuple[str, ...] = (),
) -> None:
"""Increase a counter on the stats payload."""
self._types.setdefault(counter_name, _MetricType.COUNTER)
counter_values = self._values.setdefault(counter_name, {})
counter_values[labelvalues] = (
counter_values.get(labelvalues, 0) + counter_increase_value
)
def set_gauge(
self,
gauge_name: str,
gauge_value: int | float,
labelvalues: tuple[str, ...] = (),
) -> None:
"""Set a gauge snapshot on the stats payload."""
self._types.setdefault(gauge_name, _MetricType.GAUGE)
gauge_values = self._values.setdefault(gauge_name, {})
gauge_values[labelvalues] = gauge_value
class SimpleCPUOffloadPromMetrics(KVConnectorPromMetrics):
"""Prometheus registration and recording for SimpleCPUOffloadConnector."""
def __init__(
self,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
# (engine_idx, metric_name, labelvalues) -> bound child metric
self._metrics: dict[tuple[int, str, tuple[str, ...]], PromMetricT] = {}
self._counter_save_outcomes = self._counter_cls(
name="vllm:simple_kv_offload_save_outcomes_total",
documentation=(
"Store admission decisions in SimpleCPUOffloadConnector, by "
"outcome. Outcomes classify eager-mode boundary hand-off "
"stores; lazy-mode stores are not classified."
),
labelnames=self._labelnames + ["outcome"],
)
self._counter_load_blocks = self._counter_cls(
name="vllm:simple_kv_offload_load_blocks_total",
documentation=(
"KV blocks restored to GPU from the offload pool, counted "
"when a load finishes. This is prefill work avoided "
"through offload cache hits."
),
labelnames=self._labelnames,
)
self._gauge_used_blocks = self._gauge_cls(
name="vllm:simple_kv_offload_used_blocks",
documentation=(
"Offload-pool blocks currently pinned by in-flight "
"transfers or cache hits (capacity minus free). Evictable "
"cached blocks are not counted; capacity_blocks is on "
"vllm:simple_kv_offload_info."
),
labelnames=self._labelnames,
multiprocess_mode="mostrecent",
)
self._gauge_pending_store_blocks = self._gauge_cls(
name="vllm:simple_kv_offload_pending_store_blocks",
documentation=(
"Offload blocks in pending stores: queued for dispatch, "
"issued to workers, or awaiting release after a cache "
"reset. A persistently growing value indicates a stuck "
"transfer."
),
labelnames=self._labelnames,
multiprocess_mode="mostrecent",
)
self._gauge_info = self._gauge_cls(
name="vllm:simple_kv_offload_info",
documentation=(
"SimpleCPUOffloadConnector deployment facts. Value is "
"always 1. backend is cpu or disk; page_cache and "
"lazy_offload are true/false; capacity_blocks is the "
"offload pool size."
),
labelnames=self._labelnames + INFO_LABELS,
multiprocess_mode="mostrecent",
)
self._defs: dict[str, PromMetricT] = {
MetricName.SAVE_OUTCOMES: self._counter_save_outcomes,
MetricName.LOAD_BLOCKS: self._counter_load_blocks,
MetricName.USED_BLOCKS: self._gauge_used_blocks,
MetricName.PENDING_STORE_BLOCKS: self._gauge_pending_store_blocks,
MetricName.INFO: self._gauge_info,
}
self._num_label_values: dict[str, int] = {
MetricName.SAVE_OUTCOMES: 1,
MetricName.LOAD_BLOCKS: 0,
MetricName.USED_BLOCKS: 0,
MetricName.PENDING_STORE_BLOCKS: 0,
MetricName.INFO: len(INFO_LABELS),
}
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
"""Observe transfer statistics."""
metric_types = transfer_stats_data.get(_StatsKey.TYPES, {})
metric_data = transfer_stats_data.get(_StatsKey.DATA, {})
for key, label_value_map in metric_data.items():
prom_metric = self._defs.get(key)
if prom_metric is None:
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
type_str = metric_types.get(key)
for labelvalues, value in label_value_map.items():
if len(labelvalues) != self._num_label_values[key]:
raise AssertionError(
f"Metric {key} expects "
f"{self._num_label_values[key]} labels, "
f"got {len(labelvalues)}"
)
child = self._metrics.get((engine_idx, key, labelvalues))
if child is None:
engine_labelvalues = self.per_engine_labelvalues[engine_idx]
child = prom_metric.labels(
*(engine_labelvalues + list(labelvalues))
)
self._metrics[(engine_idx, key, labelvalues)] = child
if type_str == _MetricType.COUNTER:
assert isinstance(value, int | float)
child.inc(value)
elif type_str == _MetricType.GAUGE:
assert isinstance(value, int | float)
child.set(value)
else:
raise AssertionError(
f"Unknown metric type '{type_str}' for key: {key}"
)