[Metrics][KV Offload] Add Prometheus metrics for SimpleCPUOffloadConnector (#57251)
Signed-off-by: Vincent <vincexxchan@gmail.com>
This commit is contained in:
@@ -23,6 +23,10 @@ METRIC_SOURCE_FILES = [
|
||||
"path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py",
|
||||
"key": "metrics-nixl",
|
||||
},
|
||||
{
|
||||
"path": "vllm/v1/simple_kv_offload/metrics.py",
|
||||
"key": "metrics-simple-kv-offload",
|
||||
},
|
||||
{"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"},
|
||||
]
|
||||
|
||||
|
||||
@@ -45,6 +45,30 @@ The following metrics are exposed:
|
||||
|
||||
--8<-- "gen:metrics-nixl"
|
||||
|
||||
## Simple CPU Offload Connector Metrics
|
||||
|
||||
These metrics are exposed when the `SimpleCPUOffloadConnector` KV connector
|
||||
is configured (e.g. `--kv-transfer-config='{"kv_connector":
|
||||
"SimpleCPUOffloadConnector", "kv_role": "kv_both", "kv_connector_extra_config":
|
||||
{"kv_offload_backend": "disk", "disk_path": "/mnt/nvme/kv"}}'`). They are
|
||||
updated once per engine step.
|
||||
|
||||
Caveats to keep in mind when interpreting them:
|
||||
|
||||
- A "completed" store means the write syscalls returned; it is **not**
|
||||
fsync-durable, and the disk backend's file is process-lifetime scratch,
|
||||
unlinked at startup and shutdown.
|
||||
- `save_outcomes_total` classifies eager-mode boundary hand-off stores;
|
||||
lazy-mode stores are not classified.
|
||||
- `used_blocks` counts blocks pinned by in-flight transfers or cache hits;
|
||||
warm cached blocks that are evictable are not counted. Use the
|
||||
`capacity_blocks` label of `simple_kv_offload_info` as the denominator.
|
||||
- Counters and gauges are quantized to engine steps. They are reported by the
|
||||
scheduler process and reflect engine-wide logical block counts, not values
|
||||
pooled from individual tensor-parallel workers.
|
||||
|
||||
--8<-- "gen:metrics-simple-kv-offload"
|
||||
|
||||
## Model Flops Utilization (MFU) Performance Metrics
|
||||
|
||||
These metrics are available via `--enable-mfu-metrics`:
|
||||
|
||||
@@ -0,0 +1,466 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for SimpleCPUOffloadConnector metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from prometheus_client import Counter, Gauge, Histogram
|
||||
|
||||
from tests.v1.kv_connector.unit.test_simple_cpu_offload_connector import (
|
||||
_make_connector,
|
||||
)
|
||||
from tests.v1.simple_kv_offload.test_scheduler import (
|
||||
make_request,
|
||||
make_scheduler,
|
||||
make_scheduler_output,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
|
||||
SimpleCPUOffloadConnector,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.simple_kv_offload.manager import (
|
||||
BoundaryStoreStats,
|
||||
LoadRequestState,
|
||||
TransferMeta,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metrics import (
|
||||
INFO_LABELS,
|
||||
OUTCOME_TO_FIELD,
|
||||
MetricName,
|
||||
SimpleCPUOffloadStats,
|
||||
_MetricType,
|
||||
_StatsKey,
|
||||
)
|
||||
|
||||
SAVE_OUTCOMES = MetricName.SAVE_OUTCOMES
|
||||
LOAD_BLOCKS = MetricName.LOAD_BLOCKS
|
||||
USED_BLOCKS = MetricName.USED_BLOCKS
|
||||
PENDING_STORE_BLOCKS = MetricName.PENDING_STORE_BLOCKS
|
||||
INFO = MetricName.INFO
|
||||
|
||||
|
||||
class _FakeMetric:
|
||||
def __init__(self, **kwargs: Any):
|
||||
self.kwargs = kwargs
|
||||
self.children: list[_FakeMetric] = []
|
||||
self.observed: list[int | float] = []
|
||||
self.increments: list[int | float] = []
|
||||
self.set_values: list[int | float] = []
|
||||
self.labelvalues: tuple[object, ...] = ()
|
||||
|
||||
def labels(self, *labelvalues: object) -> _FakeMetric:
|
||||
child = _FakeMetric(**self.kwargs)
|
||||
child.labelvalues = labelvalues
|
||||
self.children.append(child)
|
||||
return child
|
||||
|
||||
def observe(self, value: int | float) -> None:
|
||||
self.observed.append(value)
|
||||
|
||||
def inc(self, value: int | float) -> None:
|
||||
self.increments.append(value)
|
||||
|
||||
def set(self, value: int | float) -> None:
|
||||
self.set_values.append(value)
|
||||
|
||||
|
||||
def _fake_metric_types() -> dict[type, type]:
|
||||
return {Gauge: _FakeMetric, Counter: _FakeMetric, Histogram: _FakeMetric}
|
||||
|
||||
|
||||
def _make_prom_metrics(per_engine: dict[int, list[object]] | None = None) -> Any:
|
||||
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
|
||||
|
||||
return SimpleCPUOffloadPromMetrics(
|
||||
vllm_config=SimpleNamespace(kv_transfer_config=None),
|
||||
metric_types=_fake_metric_types(),
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues=per_engine or {0: ["model", "0"]},
|
||||
)
|
||||
|
||||
|
||||
def _stats_payload(types: dict, data: dict) -> dict[str, Any]:
|
||||
return {_StatsKey.TYPES: types, _StatsKey.DATA: data}
|
||||
|
||||
|
||||
def _values(stats: SimpleCPUOffloadStats, name: str) -> dict:
|
||||
return stats.data[_StatsKey.DATA][name]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stats class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_kv_connector_stats_none_and_empty() -> None:
|
||||
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data=None)
|
||||
assert isinstance(stats, SimpleCPUOffloadStats)
|
||||
assert stats.is_empty()
|
||||
|
||||
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data={})
|
||||
assert isinstance(stats, SimpleCPUOffloadStats)
|
||||
assert stats.is_empty()
|
||||
|
||||
|
||||
def test_stats_round_trip_through_serialized_dict() -> None:
|
||||
stats = SimpleCPUOffloadStats()
|
||||
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
|
||||
stats.increase_counter(LOAD_BLOCKS, 4)
|
||||
stats.set_gauge(USED_BLOCKS, 7)
|
||||
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
|
||||
|
||||
rebuilt = SimpleCPUOffloadConnector.build_kv_connector_stats(data=stats.to_dict())
|
||||
assert isinstance(rebuilt, SimpleCPUOffloadStats)
|
||||
assert _values(rebuilt, SAVE_OUTCOMES) == {("stored",): 3}
|
||||
assert _values(rebuilt, LOAD_BLOCKS) == {(): 4}
|
||||
assert _values(rebuilt, USED_BLOCKS) == {(): 7}
|
||||
assert _values(rebuilt, INFO) == {("cpu", "false", "false", "64"): 1}
|
||||
|
||||
|
||||
def test_stats_aggregate_sums_counters_and_keeps_latest_gauge() -> None:
|
||||
stats1 = SimpleCPUOffloadStats()
|
||||
stats1.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
|
||||
stats1.increase_counter(SAVE_OUTCOMES, 1, ("dropped_cpu_full",))
|
||||
stats1.set_gauge(USED_BLOCKS, 7)
|
||||
|
||||
stats2 = SimpleCPUOffloadStats()
|
||||
stats2.increase_counter(SAVE_OUTCOMES, 4, ("stored",))
|
||||
stats2.set_gauge(USED_BLOCKS, 2)
|
||||
|
||||
result = stats1.aggregate(stats2)
|
||||
assert result is stats1
|
||||
outcomes = _values(stats1, SAVE_OUTCOMES)
|
||||
assert outcomes == {("stored",): 7, ("dropped_cpu_full",): 1}
|
||||
assert _values(stats1, USED_BLOCKS) == {(): 2}
|
||||
|
||||
|
||||
def test_stats_aggregate_merges_types_from_other() -> None:
|
||||
stats1 = SimpleCPUOffloadStats()
|
||||
stats1.set_gauge(USED_BLOCKS, 1)
|
||||
|
||||
stats2 = SimpleCPUOffloadStats()
|
||||
stats2.increase_counter(LOAD_BLOCKS, 2)
|
||||
|
||||
stats1.aggregate(stats2)
|
||||
assert stats1.data[_StatsKey.TYPES][LOAD_BLOCKS] == _MetricType.COUNTER
|
||||
assert _values(stats1, LOAD_BLOCKS) == {(): 2}
|
||||
|
||||
|
||||
def test_stats_reduce_and_reset() -> None:
|
||||
stats = SimpleCPUOffloadStats()
|
||||
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
|
||||
stats.set_gauge(PENDING_STORE_BLOCKS, 4)
|
||||
stats.increase_counter(LOAD_BLOCKS, 9)
|
||||
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
|
||||
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{SAVE_OUTCOMES}:{('stored',)}"] == 3
|
||||
assert reduced[PENDING_STORE_BLOCKS] == 4
|
||||
assert reduced[LOAD_BLOCKS] == 9
|
||||
assert not any(key.startswith(INFO) for key in reduced)
|
||||
|
||||
stats.reset()
|
||||
assert stats.is_empty()
|
||||
|
||||
|
||||
def test_outcome_labels_match_boundary_store_stats_fields() -> None:
|
||||
"""Every outcome label maps 1:1 to a BoundaryStoreStats counter field."""
|
||||
boundary_fields = {
|
||||
f.name for f in dataclasses.fields(BoundaryStoreStats) if f.name != "published"
|
||||
}
|
||||
assert set(OUTCOME_TO_FIELD.values()) == boundary_fields
|
||||
assert len(set(OUTCOME_TO_FIELD.values())) == len(OUTCOME_TO_FIELD)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prom metrics class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_prom_metrics_registers_tier1_metrics() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
defs = prom._defs
|
||||
assert set(defs) == {
|
||||
SAVE_OUTCOMES,
|
||||
LOAD_BLOCKS,
|
||||
USED_BLOCKS,
|
||||
PENDING_STORE_BLOCKS,
|
||||
INFO,
|
||||
}
|
||||
# The prom class registers literal names (required by the docs
|
||||
# generator); they must stay in sync with the emission-side constants.
|
||||
assert {m.kwargs["name"] for m in defs.values()} == {
|
||||
SAVE_OUTCOMES,
|
||||
LOAD_BLOCKS,
|
||||
USED_BLOCKS,
|
||||
PENDING_STORE_BLOCKS,
|
||||
INFO,
|
||||
}
|
||||
for metric in defs.values():
|
||||
assert metric.kwargs["documentation"]
|
||||
assert metric.kwargs["labelnames"][:2] == ["model_name", "engine"]
|
||||
|
||||
assert defs[SAVE_OUTCOMES].kwargs["labelnames"] == [
|
||||
"model_name",
|
||||
"engine",
|
||||
"outcome",
|
||||
]
|
||||
assert defs[LOAD_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
|
||||
assert defs[INFO].kwargs["labelnames"] == [
|
||||
"model_name",
|
||||
"engine",
|
||||
*INFO_LABELS,
|
||||
]
|
||||
assert defs[USED_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
|
||||
|
||||
# mostrecent: multiprocess scrapes expose one series per label set, not
|
||||
# one pid-tagged series per writer process.
|
||||
for gauge in (USED_BLOCKS, PENDING_STORE_BLOCKS, INFO):
|
||||
assert defs[gauge].kwargs["multiprocess_mode"] == "mostrecent"
|
||||
|
||||
|
||||
def test_prom_metrics_observe_routes_each_series() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={
|
||||
SAVE_OUTCOMES: _MetricType.COUNTER,
|
||||
LOAD_BLOCKS: _MetricType.COUNTER,
|
||||
USED_BLOCKS: _MetricType.GAUGE,
|
||||
PENDING_STORE_BLOCKS: _MetricType.GAUGE,
|
||||
INFO: _MetricType.GAUGE,
|
||||
},
|
||||
data={
|
||||
SAVE_OUTCOMES: {("stored",): 3},
|
||||
LOAD_BLOCKS: {(): 4},
|
||||
USED_BLOCKS: {(): 5},
|
||||
PENDING_STORE_BLOCKS: {(): 1},
|
||||
INFO: {("cpu", "false", "false", "64"): 1},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].increments == [3]
|
||||
assert prom._metrics[(0, LOAD_BLOCKS, ())].increments == [4]
|
||||
assert prom._metrics[(0, USED_BLOCKS, ())].set_values == [5]
|
||||
assert prom._metrics[(0, PENDING_STORE_BLOCKS, ())].set_values == [1]
|
||||
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].set_values == [1]
|
||||
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].labelvalues == (
|
||||
"model",
|
||||
"0",
|
||||
"stored",
|
||||
)
|
||||
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].labelvalues == (
|
||||
"model",
|
||||
"0",
|
||||
"cpu",
|
||||
"false",
|
||||
"false",
|
||||
"64",
|
||||
)
|
||||
|
||||
|
||||
def test_prom_metrics_reuses_bound_children() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
payload = _stats_payload(
|
||||
types={SAVE_OUTCOMES: _MetricType.COUNTER},
|
||||
data={SAVE_OUTCOMES: {("stored",): 1}},
|
||||
)
|
||||
prom.observe(payload)
|
||||
prom.observe(payload)
|
||||
|
||||
child = prom._metrics[(0, SAVE_OUTCOMES, ("stored",))]
|
||||
assert child.increments == [1, 1]
|
||||
assert len(prom._defs[SAVE_OUTCOMES].children) == 1
|
||||
|
||||
|
||||
def test_prom_metrics_rejects_unknown_metric() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
with pytest.raises(AssertionError, match="Unknown"):
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={"vllm:not_a_metric": _MetricType.COUNTER},
|
||||
data={"vllm:not_a_metric": {(): 1}},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_prom_metrics_rejects_wrong_label_count() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
with pytest.raises(AssertionError, match="labels"):
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={SAVE_OUTCOMES: _MetricType.COUNTER},
|
||||
data={SAVE_OUTCOMES: {("stored", "extra"): 1}},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_prom_metrics_routes_per_engine() -> None:
|
||||
prom = _make_prom_metrics(per_engine={0: ["model", "0"], 1: ["model", "1"]})
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={USED_BLOCKS: _MetricType.GAUGE}, data={USED_BLOCKS: {(): 3}}
|
||||
),
|
||||
engine_idx=1,
|
||||
)
|
||||
assert (0, USED_BLOCKS, ()) not in prom._metrics
|
||||
assert prom._metrics[(1, USED_BLOCKS, ())].set_values == [3]
|
||||
|
||||
|
||||
def test_connector_build_prom_metrics() -> None:
|
||||
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
|
||||
|
||||
prom = SimpleCPUOffloadConnector.build_prom_metrics(
|
||||
vllm_config=SimpleNamespace(kv_transfer_config=None),
|
||||
metric_types=_fake_metric_types(),
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues={0: ["model", "0"]},
|
||||
)
|
||||
assert isinstance(prom, SimpleCPUOffloadPromMetrics)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Manager: interval stats
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_manager_get_stats_reports_outcome_deltas_per_interval() -> None:
|
||||
fixture = make_scheduler()
|
||||
sched = fixture.scheduler
|
||||
|
||||
sched.boundary_store_stats.stored += 3
|
||||
sched.boundary_store_stats.dropped_cpu_full += 2
|
||||
sched.boundary_store_stats.skipped_in_flight += 1
|
||||
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, SAVE_OUTCOMES) == {
|
||||
("stored",): 3,
|
||||
("dropped_cpu_full",): 2,
|
||||
("skipped_in_flight",): 1,
|
||||
}
|
||||
|
||||
# Deltas are per-interval: a second drain reports no new outcomes.
|
||||
stats2 = sched.get_stats()
|
||||
assert SAVE_OUTCOMES not in stats2.data[_StatsKey.DATA]
|
||||
|
||||
|
||||
def test_manager_reset_preserves_undrained_outcome_deltas() -> None:
|
||||
fixture = make_scheduler()
|
||||
sched = fixture.scheduler
|
||||
|
||||
sched.boundary_store_stats.stored += 3
|
||||
sched.reset()
|
||||
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, SAVE_OUTCOMES) == {("stored",): 3}
|
||||
|
||||
|
||||
def test_manager_get_stats_gauges_and_info_labels() -> None:
|
||||
fixture = make_scheduler(num_cpu_blocks=8)
|
||||
sched = fixture.scheduler
|
||||
|
||||
sched._store_event_to_blocks[0] = TransferMeta([10], [11])
|
||||
sched._pending_finished_stores.append(TransferMeta([12], [13]))
|
||||
sched._abandoned_store_event_to_blocks[1] = TransferMeta([14], [15])
|
||||
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, PENDING_STORE_BLOCKS) == {(): 3}
|
||||
used = _values(stats, USED_BLOCKS)[()]
|
||||
assert used == sched.num_cpu_blocks - sched.cpu_block_pool.get_num_free_blocks()
|
||||
|
||||
info = _values(stats, INFO)
|
||||
assert info == {("cpu", "false", "false", str(sched.num_cpu_blocks)): 1}
|
||||
|
||||
# Gauges are re-reported every drain.
|
||||
stats2 = sched.get_stats()
|
||||
assert _values(stats2, PENDING_STORE_BLOCKS) == {(): 3}
|
||||
|
||||
|
||||
def test_manager_info_labels_reflect_disk_mode() -> None:
|
||||
connector = _make_connector(
|
||||
extra_config={
|
||||
"kv_offload_backend": "disk",
|
||||
"disk_path": "/tmp/fake-disk-metrics",
|
||||
"disk_capacity_bytes": 1024**3,
|
||||
"use_page_cache": True,
|
||||
}
|
||||
)
|
||||
sched = connector.scheduler_manager
|
||||
assert sched is not None
|
||||
assert sched._info_labelvalues[0] == "disk"
|
||||
assert sched._info_labelvalues[1] == "true"
|
||||
stats = sched.get_stats()
|
||||
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
|
||||
|
||||
|
||||
def test_manager_info_labels_page_cache_false_for_cpu_backend() -> None:
|
||||
"""CPU backend ignores use_page_cache; the info gauge must not claim it."""
|
||||
connector = _make_connector(extra_config={"use_page_cache": True})
|
||||
sched = connector.scheduler_manager
|
||||
assert sched is not None
|
||||
assert sched._info_labelvalues[:2] == ("cpu", "false")
|
||||
stats = sched.get_stats()
|
||||
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
|
||||
|
||||
|
||||
def test_manager_counts_load_blocks_completed() -> None:
|
||||
fixture = make_scheduler(num_cpu_blocks=8)
|
||||
sched = fixture.scheduler
|
||||
|
||||
request = make_request(num_blocks=2)
|
||||
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
|
||||
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
|
||||
sched._reqs_to_load[request.request_id] = LoadRequestState(
|
||||
request=request,
|
||||
transfer_meta=TransferMeta(
|
||||
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
|
||||
),
|
||||
)
|
||||
|
||||
sched.build_connector_meta(make_scheduler_output({}))
|
||||
stats = sched.get_stats()
|
||||
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
|
||||
|
||||
sched.update_connector_output(
|
||||
KVConnectorOutput(finished_recving={request.request_id})
|
||||
)
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, LOAD_BLOCKS) == {(): 2}
|
||||
|
||||
|
||||
def test_manager_abandoned_load_counts_as_completed_after_reset() -> None:
|
||||
"""Loads abandoned by reset() that still finish on the worker are
|
||||
counted as loaded blocks.
|
||||
"""
|
||||
fixture = make_scheduler(num_cpu_blocks=8)
|
||||
sched = fixture.scheduler
|
||||
|
||||
request = make_request(num_blocks=2)
|
||||
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
|
||||
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
|
||||
sched._reqs_to_load[request.request_id] = LoadRequestState(
|
||||
request=request,
|
||||
transfer_meta=TransferMeta(
|
||||
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
|
||||
),
|
||||
)
|
||||
|
||||
sched.build_connector_meta(make_scheduler_output({}))
|
||||
stats = sched.get_stats()
|
||||
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
|
||||
|
||||
assert sched.reset() is False
|
||||
assert set(sched._abandoned_reqs_to_load) == {request.request_id}
|
||||
|
||||
sched.update_connector_output(
|
||||
KVConnectorOutput(finished_recving={request.request_id})
|
||||
)
|
||||
assert sched._abandoned_reqs_to_load == {}
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, LOAD_BLOCKS) == {(): 2}
|
||||
@@ -15,6 +15,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
@@ -25,6 +31,10 @@ from vllm.v1.simple_kv_offload.manager import (
|
||||
from vllm.v1.simple_kv_offload.metadata import (
|
||||
SimpleCPUOffloadMetadata,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metrics import (
|
||||
SimpleCPUOffloadPromMetrics,
|
||||
SimpleCPUOffloadStats,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.worker import (
|
||||
SimpleCPUOffloadWorker,
|
||||
)
|
||||
@@ -162,6 +172,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
hash_block_size=hash_block_size,
|
||||
lazy_offload=lazy_offload,
|
||||
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
|
||||
use_page_cache=use_page_cache if disk_mode else False,
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.worker_handler = SimpleCPUOffloadWorker(
|
||||
@@ -307,6 +318,38 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return self.scheduler_manager.get_boundary_store_stats()
|
||||
return None
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
# Worker-side callers may invoke this hook on connectors constructed
|
||||
# via __new__ (as in test_worker.py), which lack scheduler_manager.
|
||||
scheduler_manager: SimpleCPUOffloadScheduler | None = getattr(
|
||||
self, "scheduler_manager", None
|
||||
)
|
||||
if scheduler_manager is not None:
|
||||
return scheduler_manager.get_stats()
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
SimpleCPUOffloadStats(data=data)
|
||||
if data is not None
|
||||
else SimpleCPUOffloadStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return SimpleCPUOffloadPromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.take_events()
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import contextlib
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass, field, replace
|
||||
from dataclasses import dataclass, field, fields, replace
|
||||
from enum import Enum, auto
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
@@ -48,6 +48,11 @@ from vllm.v1.simple_kv_offload.metadata import (
|
||||
SimpleCPUOffloadMetadata,
|
||||
SimpleCPUOffloadWorkerMetadata,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metrics import (
|
||||
OUTCOME_TO_FIELD,
|
||||
MetricName,
|
||||
SimpleCPUOffloadStats,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
@@ -118,7 +123,8 @@ class BoundaryStoreStats:
|
||||
symptom is a lower cache hit rate with nothing in the logs, so each decline
|
||||
reason is counted and exposed through
|
||||
``SimpleCPUOffloadConnector.get_boundary_store_stats()``. Reset by
|
||||
``reset()``; not otherwise cleared.
|
||||
``reset()``, which carries any not-yet-drained deltas; not otherwise
|
||||
cleared.
|
||||
"""
|
||||
|
||||
published: int = 0
|
||||
@@ -143,6 +149,7 @@ class SimpleCPUOffloadScheduler:
|
||||
hash_block_size: int,
|
||||
lazy_offload: bool = False,
|
||||
disk_capacity_bytes: int = 0,
|
||||
use_page_cache: bool = False,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.kv_cache_config = kv_cache_config
|
||||
@@ -169,6 +176,13 @@ class SimpleCPUOffloadScheduler:
|
||||
self.cpu_kv_cache_config.prefix_cacheable_group_ids
|
||||
)
|
||||
self.kv_event_medium = MEDIUM_STORAGE if disk_capacity_bytes > 0 else MEDIUM_CPU
|
||||
# Must track kv_event_medium.
|
||||
self._info_labelvalues: tuple[str, ...] = (
|
||||
"disk" if disk_capacity_bytes > 0 else "cpu",
|
||||
str(use_page_cache).lower(),
|
||||
str(lazy_offload).lower(),
|
||||
str(self.num_cpu_blocks),
|
||||
)
|
||||
# Find the full attention kv group for prefix cache matching.
|
||||
self.fa_gidx = -1
|
||||
for g_idx, g in enumerate(self.cpu_kv_cache_config.kv_cache_groups):
|
||||
@@ -251,6 +265,9 @@ class SimpleCPUOffloadScheduler:
|
||||
self._load_event_counter: int = 0
|
||||
self._store_event_counter: int = 0
|
||||
self.boundary_store_stats = BoundaryStoreStats()
|
||||
# Interval stats state drained by get_stats()
|
||||
self._boundary_stats_snapshot = BoundaryStoreStats()
|
||||
self._interval_load_blocks_completed = 0
|
||||
|
||||
# For TP/PP: track partial store completions across steps.
|
||||
# Events must be reported by all world_size workers before considered complete.
|
||||
@@ -951,6 +968,42 @@ class SimpleCPUOffloadScheduler:
|
||||
def get_boundary_store_stats(self) -> BoundaryStoreStats:
|
||||
return replace(self.boundary_store_stats)
|
||||
|
||||
def get_stats(self) -> SimpleCPUOffloadStats:
|
||||
"""Drain per-step stats for the connector's stats hooks."""
|
||||
stats = SimpleCPUOffloadStats()
|
||||
|
||||
current = self.boundary_store_stats
|
||||
for outcome, field_name in OUTCOME_TO_FIELD.items():
|
||||
delta = getattr(current, field_name) - getattr(
|
||||
self._boundary_stats_snapshot, field_name
|
||||
)
|
||||
if delta > 0:
|
||||
stats.increase_counter(MetricName.SAVE_OUTCOMES, delta, (outcome,))
|
||||
self._boundary_stats_snapshot = replace(current)
|
||||
|
||||
if self._interval_load_blocks_completed:
|
||||
stats.increase_counter(
|
||||
MetricName.LOAD_BLOCKS,
|
||||
self._interval_load_blocks_completed,
|
||||
)
|
||||
self._interval_load_blocks_completed = 0
|
||||
|
||||
stats.set_gauge(
|
||||
MetricName.USED_BLOCKS,
|
||||
self.num_cpu_blocks - self.cpu_block_pool.get_num_free_blocks(),
|
||||
)
|
||||
pending = sum(
|
||||
len(t.cpu_block_ids)
|
||||
for t in (
|
||||
*self._store_event_to_blocks.values(),
|
||||
*self._pending_finished_stores,
|
||||
*self._abandoned_store_event_to_blocks.values(),
|
||||
)
|
||||
)
|
||||
stats.set_gauge(MetricName.PENDING_STORE_BLOCKS, pending)
|
||||
stats.set_gauge(MetricName.INFO, 1, self._info_labelvalues)
|
||||
return stats
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
"""Handle async transfer completions from worker.
|
||||
|
||||
@@ -961,7 +1014,9 @@ class SimpleCPUOffloadScheduler:
|
||||
"""
|
||||
# --- Load completions ---
|
||||
for req_id in list(connector_output.finished_recving or []):
|
||||
self._cleanup_load_request(req_id)
|
||||
completed_blocks = self._cleanup_load_request(req_id)
|
||||
if completed_blocks:
|
||||
self._interval_load_blocks_completed += completed_blocks
|
||||
|
||||
# --- Store completions ---
|
||||
meta = connector_output.kv_connector_worker_meta
|
||||
@@ -1266,18 +1321,22 @@ class SimpleCPUOffloadScheduler:
|
||||
if blocks_to_free:
|
||||
self.cpu_block_pool.free_blocks(blocks_to_free)
|
||||
|
||||
def _cleanup_load_request(self, req_id: str) -> None:
|
||||
def _cleanup_load_request(self, req_id: str) -> int:
|
||||
"""Release all load resources for a request.
|
||||
|
||||
Shared between request_finished() and update_connector_output() paths.
|
||||
Removes the request from _reqs_to_load, cleans up event mappings,
|
||||
and frees CPU/GPU touch refs.
|
||||
|
||||
Returns the number of blocks in the load if it had been issued to
|
||||
the worker, else 0 (never-issued loads are not counted as completed
|
||||
by the caller).
|
||||
"""
|
||||
state = self._reqs_to_load.pop(req_id, None)
|
||||
if state is None:
|
||||
state = self._abandoned_reqs_to_load.pop(req_id, None)
|
||||
if state is None:
|
||||
return
|
||||
return 0
|
||||
# Remove from load event mapping (only this req, not whole event)
|
||||
if state.load_event is not None:
|
||||
reqs = self._load_event_to_reqs.get(state.load_event)
|
||||
@@ -1300,6 +1359,10 @@ class SimpleCPUOffloadScheduler:
|
||||
for bid in state.transfer_meta.gpu_block_ids
|
||||
)
|
||||
|
||||
if state.load_event is None:
|
||||
return 0
|
||||
return len(state.transfer_meta.gpu_block_ids)
|
||||
|
||||
def _cleanup_store_request(self, req_id: str) -> None:
|
||||
"""Release store metadata for a request.
|
||||
|
||||
@@ -1359,7 +1422,20 @@ class SimpleCPUOffloadScheduler:
|
||||
if event_idx in self._abandoned_store_event_to_blocks
|
||||
}
|
||||
self._cursor = None
|
||||
self.boundary_store_stats = BoundaryStoreStats()
|
||||
# Seed the fresh counters with outcomes since the last drain so a
|
||||
# mid-interval reset() does not drop them from the next report.
|
||||
undrained = BoundaryStoreStats(
|
||||
**{
|
||||
f.name: max(
|
||||
0,
|
||||
getattr(self.boundary_store_stats, f.name)
|
||||
- getattr(self._boundary_stats_snapshot, f.name),
|
||||
)
|
||||
for f in fields(BoundaryStoreStats)
|
||||
}
|
||||
)
|
||||
self.boundary_store_stats = undrained
|
||||
self._boundary_stats_snapshot = BoundaryStoreStats()
|
||||
# NOTE: _load_event_counter / _store_event_counter are not
|
||||
# reset as they are monotonic and must stay ahead of the workers
|
||||
# high-water marks to avoid event index collisions
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Prometheus metrics for SimpleCPUOffloadConnector.
|
||||
|
||||
The stats payload uses the same self-describing wire format as the
|
||||
offloading connector, so it survives IPC serialization without the
|
||||
metric classes on the receiving side.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
|
||||
|
||||
class MetricName:
|
||||
"""Metric names for SimpleCPUOffloadConnector."""
|
||||
|
||||
SAVE_OUTCOMES = "vllm:simple_kv_offload_save_outcomes_total"
|
||||
LOAD_BLOCKS = "vllm:simple_kv_offload_load_blocks_total"
|
||||
USED_BLOCKS = "vllm:simple_kv_offload_used_blocks"
|
||||
PENDING_STORE_BLOCKS = "vllm:simple_kv_offload_pending_store_blocks"
|
||||
INFO = "vllm:simple_kv_offload_info"
|
||||
|
||||
|
||||
# outcome label value -> BoundaryStoreStats field name.
|
||||
# ``published`` is the sum of all outcomes and is not labeled.
|
||||
OUTCOME_TO_FIELD = {
|
||||
"stored": "stored",
|
||||
"dropped_cpu_full": "dropped_cpu_full",
|
||||
"dropped_request_gone": "dropped_request_gone",
|
||||
"dropped_null_block": "dropped_null_block",
|
||||
"dropped_not_hashed": "dropped_not_hashed",
|
||||
"skipped_already_cached": "skipped_already_cached",
|
||||
"skipped_in_flight": "skipped_in_flight",
|
||||
}
|
||||
|
||||
INFO_LABELS = ["backend", "page_cache", "lazy_offload", "capacity_blocks"]
|
||||
|
||||
|
||||
class _MetricType:
|
||||
"""Type tags embedded in the serialized stats payload."""
|
||||
|
||||
COUNTER = "counter"
|
||||
GAUGE = "gauge"
|
||||
|
||||
|
||||
class _StatsKey:
|
||||
"""Top-level keys in the serialized stats dict."""
|
||||
|
||||
# Maps metric name -> _MetricType value
|
||||
TYPES = "types"
|
||||
# Maps metric name -> {label values tuple -> observed value}
|
||||
DATA = "data"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SimpleCPUOffloadStats(KVConnectorStats):
|
||||
"""Stats payload for SimpleCPUOffloadConnector.
|
||||
|
||||
The ``data`` dict is structured as::
|
||||
|
||||
{
|
||||
_StatsKey.TYPES: {name: _MetricType.*, ...},
|
||||
_StatsKey.DATA: {name: {labelvalues: value, ...}, ...},
|
||||
}
|
||||
|
||||
Counter values are summed when aggregating, gauge values keep the
|
||||
latest snapshot, and unlabeled metrics use ``()`` as their
|
||||
labelvalues tuple.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
if _StatsKey.DATA not in self.data:
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.data: dict[str, Any] = {
|
||||
_StatsKey.TYPES: {},
|
||||
_StatsKey.DATA: {},
|
||||
}
|
||||
|
||||
@property
|
||||
def _types(self) -> dict[str, str]:
|
||||
return self.data[_StatsKey.TYPES]
|
||||
|
||||
@property
|
||||
def _values(self) -> dict[str, Any]:
|
||||
return self.data[_StatsKey.DATA]
|
||||
|
||||
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
|
||||
if other.is_empty():
|
||||
return self
|
||||
assert isinstance(other, SimpleCPUOffloadStats)
|
||||
for key, other_label_values in other._values.items():
|
||||
type_str = other._types.get(key)
|
||||
if type_str is None:
|
||||
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
|
||||
self._types.setdefault(key, type_str)
|
||||
current_label_values = self._values.setdefault(key, {})
|
||||
for labelvalues, value in other_label_values.items():
|
||||
if type_str == _MetricType.COUNTER:
|
||||
assert isinstance(value, int | float)
|
||||
current_label_values[labelvalues] = (
|
||||
current_label_values.get(labelvalues, 0) + value
|
||||
)
|
||||
elif type_str == _MetricType.GAUGE:
|
||||
assert isinstance(value, int | float)
|
||||
current_label_values[labelvalues] = value
|
||||
else:
|
||||
raise AssertionError(
|
||||
f"Unknown metric type '{type_str}' for key: {key}"
|
||||
)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
"""Reduce the interval observations to a flat summary for logging."""
|
||||
return_dict: dict[str, int | float] = {}
|
||||
for key, label_value_map in self._values.items():
|
||||
if key == MetricName.INFO:
|
||||
# Constant config fingerprint; exclude from the log summary.
|
||||
continue
|
||||
type_str = self._types.get(key)
|
||||
if type_str is None:
|
||||
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
|
||||
for labelvalues, value in label_value_map.items():
|
||||
key_with_labels = f"{key}:{labelvalues}" if labelvalues else key
|
||||
if type_str in (_MetricType.COUNTER, _MetricType.GAUGE):
|
||||
assert isinstance(value, int | float)
|
||||
return_dict[key_with_labels] = value
|
||||
else:
|
||||
raise AssertionError(
|
||||
f"Unknown metric type '{type_str}' for key: {key}"
|
||||
)
|
||||
return return_dict
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return not self.data.get(_StatsKey.DATA)
|
||||
|
||||
def increase_counter(
|
||||
self,
|
||||
counter_name: str,
|
||||
counter_increase_value: int | float = 1,
|
||||
labelvalues: tuple[str, ...] = (),
|
||||
) -> None:
|
||||
"""Increase a counter on the stats payload."""
|
||||
self._types.setdefault(counter_name, _MetricType.COUNTER)
|
||||
counter_values = self._values.setdefault(counter_name, {})
|
||||
counter_values[labelvalues] = (
|
||||
counter_values.get(labelvalues, 0) + counter_increase_value
|
||||
)
|
||||
|
||||
def set_gauge(
|
||||
self,
|
||||
gauge_name: str,
|
||||
gauge_value: int | float,
|
||||
labelvalues: tuple[str, ...] = (),
|
||||
) -> None:
|
||||
"""Set a gauge snapshot on the stats payload."""
|
||||
self._types.setdefault(gauge_name, _MetricType.GAUGE)
|
||||
gauge_values = self._values.setdefault(gauge_name, {})
|
||||
gauge_values[labelvalues] = gauge_value
|
||||
|
||||
|
||||
class SimpleCPUOffloadPromMetrics(KVConnectorPromMetrics):
|
||||
"""Prometheus registration and recording for SimpleCPUOffloadConnector."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
):
|
||||
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
|
||||
# (engine_idx, metric_name, labelvalues) -> bound child metric
|
||||
self._metrics: dict[tuple[int, str, tuple[str, ...]], PromMetricT] = {}
|
||||
|
||||
self._counter_save_outcomes = self._counter_cls(
|
||||
name="vllm:simple_kv_offload_save_outcomes_total",
|
||||
documentation=(
|
||||
"Store admission decisions in SimpleCPUOffloadConnector, by "
|
||||
"outcome. Outcomes classify eager-mode boundary hand-off "
|
||||
"stores; lazy-mode stores are not classified."
|
||||
),
|
||||
labelnames=self._labelnames + ["outcome"],
|
||||
)
|
||||
self._counter_load_blocks = self._counter_cls(
|
||||
name="vllm:simple_kv_offload_load_blocks_total",
|
||||
documentation=(
|
||||
"KV blocks restored to GPU from the offload pool, counted "
|
||||
"when a load finishes. This is prefill work avoided "
|
||||
"through offload cache hits."
|
||||
),
|
||||
labelnames=self._labelnames,
|
||||
)
|
||||
self._gauge_used_blocks = self._gauge_cls(
|
||||
name="vllm:simple_kv_offload_used_blocks",
|
||||
documentation=(
|
||||
"Offload-pool blocks currently pinned by in-flight "
|
||||
"transfers or cache hits (capacity minus free). Evictable "
|
||||
"cached blocks are not counted; capacity_blocks is on "
|
||||
"vllm:simple_kv_offload_info."
|
||||
),
|
||||
labelnames=self._labelnames,
|
||||
multiprocess_mode="mostrecent",
|
||||
)
|
||||
self._gauge_pending_store_blocks = self._gauge_cls(
|
||||
name="vllm:simple_kv_offload_pending_store_blocks",
|
||||
documentation=(
|
||||
"Offload blocks in pending stores: queued for dispatch, "
|
||||
"issued to workers, or awaiting release after a cache "
|
||||
"reset. A persistently growing value indicates a stuck "
|
||||
"transfer."
|
||||
),
|
||||
labelnames=self._labelnames,
|
||||
multiprocess_mode="mostrecent",
|
||||
)
|
||||
self._gauge_info = self._gauge_cls(
|
||||
name="vllm:simple_kv_offload_info",
|
||||
documentation=(
|
||||
"SimpleCPUOffloadConnector deployment facts. Value is "
|
||||
"always 1. backend is cpu or disk; page_cache and "
|
||||
"lazy_offload are true/false; capacity_blocks is the "
|
||||
"offload pool size."
|
||||
),
|
||||
labelnames=self._labelnames + INFO_LABELS,
|
||||
multiprocess_mode="mostrecent",
|
||||
)
|
||||
|
||||
self._defs: dict[str, PromMetricT] = {
|
||||
MetricName.SAVE_OUTCOMES: self._counter_save_outcomes,
|
||||
MetricName.LOAD_BLOCKS: self._counter_load_blocks,
|
||||
MetricName.USED_BLOCKS: self._gauge_used_blocks,
|
||||
MetricName.PENDING_STORE_BLOCKS: self._gauge_pending_store_blocks,
|
||||
MetricName.INFO: self._gauge_info,
|
||||
}
|
||||
self._num_label_values: dict[str, int] = {
|
||||
MetricName.SAVE_OUTCOMES: 1,
|
||||
MetricName.LOAD_BLOCKS: 0,
|
||||
MetricName.USED_BLOCKS: 0,
|
||||
MetricName.PENDING_STORE_BLOCKS: 0,
|
||||
MetricName.INFO: len(INFO_LABELS),
|
||||
}
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
"""Observe transfer statistics."""
|
||||
metric_types = transfer_stats_data.get(_StatsKey.TYPES, {})
|
||||
metric_data = transfer_stats_data.get(_StatsKey.DATA, {})
|
||||
for key, label_value_map in metric_data.items():
|
||||
prom_metric = self._defs.get(key)
|
||||
if prom_metric is None:
|
||||
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
|
||||
type_str = metric_types.get(key)
|
||||
for labelvalues, value in label_value_map.items():
|
||||
if len(labelvalues) != self._num_label_values[key]:
|
||||
raise AssertionError(
|
||||
f"Metric {key} expects "
|
||||
f"{self._num_label_values[key]} labels, "
|
||||
f"got {len(labelvalues)}"
|
||||
)
|
||||
child = self._metrics.get((engine_idx, key, labelvalues))
|
||||
if child is None:
|
||||
engine_labelvalues = self.per_engine_labelvalues[engine_idx]
|
||||
child = prom_metric.labels(
|
||||
*(engine_labelvalues + list(labelvalues))
|
||||
)
|
||||
self._metrics[(engine_idx, key, labelvalues)] = child
|
||||
if type_str == _MetricType.COUNTER:
|
||||
assert isinstance(value, int | float)
|
||||
child.inc(value)
|
||||
elif type_str == _MetricType.GAUGE:
|
||||
assert isinstance(value, int | float)
|
||||
child.set(value)
|
||||
else:
|
||||
raise AssertionError(
|
||||
f"Unknown metric type '{type_str}' for key: {key}"
|
||||
)
|
||||
Reference in New Issue
Block a user