diff --git a/docs/mkdocs/gen_files/generate_metrics.py b/docs/mkdocs/gen_files/generate_metrics.py index b952f53977..4d4fa93cc5 100644 --- a/docs/mkdocs/gen_files/generate_metrics.py +++ b/docs/mkdocs/gen_files/generate_metrics.py @@ -23,6 +23,10 @@ METRIC_SOURCE_FILES = [ "path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py", "key": "metrics-nixl", }, + { + "path": "vllm/v1/simple_kv_offload/metrics.py", + "key": "metrics-simple-kv-offload", + }, {"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"}, ] diff --git a/docs/usage/metrics.md b/docs/usage/metrics.md index 305c21478f..8254a7f420 100644 --- a/docs/usage/metrics.md +++ b/docs/usage/metrics.md @@ -45,6 +45,30 @@ The following metrics are exposed: --8<-- "gen:metrics-nixl" +## Simple CPU Offload Connector Metrics + +These metrics are exposed when the `SimpleCPUOffloadConnector` KV connector +is configured (e.g. `--kv-transfer-config='{"kv_connector": +"SimpleCPUOffloadConnector", "kv_role": "kv_both", "kv_connector_extra_config": +{"kv_offload_backend": "disk", "disk_path": "/mnt/nvme/kv"}}'`). They are +updated once per engine step. + +Caveats to keep in mind when interpreting them: + +- A "completed" store means the write syscalls returned; it is **not** + fsync-durable, and the disk backend's file is process-lifetime scratch, + unlinked at startup and shutdown. +- `save_outcomes_total` classifies eager-mode boundary hand-off stores; + lazy-mode stores are not classified. +- `used_blocks` counts blocks pinned by in-flight transfers or cache hits; + warm cached blocks that are evictable are not counted. Use the + `capacity_blocks` label of `simple_kv_offload_info` as the denominator. +- Counters and gauges are quantized to engine steps. They are reported by the + scheduler process and reflect engine-wide logical block counts, not values + pooled from individual tensor-parallel workers. + +--8<-- "gen:metrics-simple-kv-offload" + ## Model Flops Utilization (MFU) Performance Metrics These metrics are available via `--enable-mfu-metrics`: diff --git a/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py b/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py new file mode 100644 index 0000000000..16358dec7e --- /dev/null +++ b/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py @@ -0,0 +1,466 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for SimpleCPUOffloadConnector metrics.""" + +from __future__ import annotations + +import dataclasses +from types import SimpleNamespace +from typing import Any + +import pytest +from prometheus_client import Counter, Gauge, Histogram + +from tests.v1.kv_connector.unit.test_simple_cpu_offload_connector import ( + _make_connector, +) +from tests.v1.simple_kv_offload.test_scheduler import ( + make_request, + make_scheduler, + make_scheduler_output, +) +from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import ( + SimpleCPUOffloadConnector, +) +from vllm.v1.outputs import KVConnectorOutput +from vllm.v1.simple_kv_offload.manager import ( + BoundaryStoreStats, + LoadRequestState, + TransferMeta, +) +from vllm.v1.simple_kv_offload.metrics import ( + INFO_LABELS, + OUTCOME_TO_FIELD, + MetricName, + SimpleCPUOffloadStats, + _MetricType, + _StatsKey, +) + +SAVE_OUTCOMES = MetricName.SAVE_OUTCOMES +LOAD_BLOCKS = MetricName.LOAD_BLOCKS +USED_BLOCKS = MetricName.USED_BLOCKS +PENDING_STORE_BLOCKS = MetricName.PENDING_STORE_BLOCKS +INFO = MetricName.INFO + + +class _FakeMetric: + def __init__(self, **kwargs: Any): + self.kwargs = kwargs + self.children: list[_FakeMetric] = [] + self.observed: list[int | float] = [] + self.increments: list[int | float] = [] + self.set_values: list[int | float] = [] + self.labelvalues: tuple[object, ...] = () + + def labels(self, *labelvalues: object) -> _FakeMetric: + child = _FakeMetric(**self.kwargs) + child.labelvalues = labelvalues + self.children.append(child) + return child + + def observe(self, value: int | float) -> None: + self.observed.append(value) + + def inc(self, value: int | float) -> None: + self.increments.append(value) + + def set(self, value: int | float) -> None: + self.set_values.append(value) + + +def _fake_metric_types() -> dict[type, type]: + return {Gauge: _FakeMetric, Counter: _FakeMetric, Histogram: _FakeMetric} + + +def _make_prom_metrics(per_engine: dict[int, list[object]] | None = None) -> Any: + from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics + + return SimpleCPUOffloadPromMetrics( + vllm_config=SimpleNamespace(kv_transfer_config=None), + metric_types=_fake_metric_types(), + labelnames=["model_name", "engine"], + per_engine_labelvalues=per_engine or {0: ["model", "0"]}, + ) + + +def _stats_payload(types: dict, data: dict) -> dict[str, Any]: + return {_StatsKey.TYPES: types, _StatsKey.DATA: data} + + +def _values(stats: SimpleCPUOffloadStats, name: str) -> dict: + return stats.data[_StatsKey.DATA][name] + + +# --------------------------------------------------------------------------- +# Stats class +# --------------------------------------------------------------------------- + + +def test_build_kv_connector_stats_none_and_empty() -> None: + stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data=None) + assert isinstance(stats, SimpleCPUOffloadStats) + assert stats.is_empty() + + stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data={}) + assert isinstance(stats, SimpleCPUOffloadStats) + assert stats.is_empty() + + +def test_stats_round_trip_through_serialized_dict() -> None: + stats = SimpleCPUOffloadStats() + stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) + stats.increase_counter(LOAD_BLOCKS, 4) + stats.set_gauge(USED_BLOCKS, 7) + stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64")) + + rebuilt = SimpleCPUOffloadConnector.build_kv_connector_stats(data=stats.to_dict()) + assert isinstance(rebuilt, SimpleCPUOffloadStats) + assert _values(rebuilt, SAVE_OUTCOMES) == {("stored",): 3} + assert _values(rebuilt, LOAD_BLOCKS) == {(): 4} + assert _values(rebuilt, USED_BLOCKS) == {(): 7} + assert _values(rebuilt, INFO) == {("cpu", "false", "false", "64"): 1} + + +def test_stats_aggregate_sums_counters_and_keeps_latest_gauge() -> None: + stats1 = SimpleCPUOffloadStats() + stats1.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) + stats1.increase_counter(SAVE_OUTCOMES, 1, ("dropped_cpu_full",)) + stats1.set_gauge(USED_BLOCKS, 7) + + stats2 = SimpleCPUOffloadStats() + stats2.increase_counter(SAVE_OUTCOMES, 4, ("stored",)) + stats2.set_gauge(USED_BLOCKS, 2) + + result = stats1.aggregate(stats2) + assert result is stats1 + outcomes = _values(stats1, SAVE_OUTCOMES) + assert outcomes == {("stored",): 7, ("dropped_cpu_full",): 1} + assert _values(stats1, USED_BLOCKS) == {(): 2} + + +def test_stats_aggregate_merges_types_from_other() -> None: + stats1 = SimpleCPUOffloadStats() + stats1.set_gauge(USED_BLOCKS, 1) + + stats2 = SimpleCPUOffloadStats() + stats2.increase_counter(LOAD_BLOCKS, 2) + + stats1.aggregate(stats2) + assert stats1.data[_StatsKey.TYPES][LOAD_BLOCKS] == _MetricType.COUNTER + assert _values(stats1, LOAD_BLOCKS) == {(): 2} + + +def test_stats_reduce_and_reset() -> None: + stats = SimpleCPUOffloadStats() + stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) + stats.set_gauge(PENDING_STORE_BLOCKS, 4) + stats.increase_counter(LOAD_BLOCKS, 9) + stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64")) + + reduced = stats.reduce() + assert reduced[f"{SAVE_OUTCOMES}:{('stored',)}"] == 3 + assert reduced[PENDING_STORE_BLOCKS] == 4 + assert reduced[LOAD_BLOCKS] == 9 + assert not any(key.startswith(INFO) for key in reduced) + + stats.reset() + assert stats.is_empty() + + +def test_outcome_labels_match_boundary_store_stats_fields() -> None: + """Every outcome label maps 1:1 to a BoundaryStoreStats counter field.""" + boundary_fields = { + f.name for f in dataclasses.fields(BoundaryStoreStats) if f.name != "published" + } + assert set(OUTCOME_TO_FIELD.values()) == boundary_fields + assert len(set(OUTCOME_TO_FIELD.values())) == len(OUTCOME_TO_FIELD) + + +# --------------------------------------------------------------------------- +# Prom metrics class +# --------------------------------------------------------------------------- + + +def test_prom_metrics_registers_tier1_metrics() -> None: + prom = _make_prom_metrics() + defs = prom._defs + assert set(defs) == { + SAVE_OUTCOMES, + LOAD_BLOCKS, + USED_BLOCKS, + PENDING_STORE_BLOCKS, + INFO, + } + # The prom class registers literal names (required by the docs + # generator); they must stay in sync with the emission-side constants. + assert {m.kwargs["name"] for m in defs.values()} == { + SAVE_OUTCOMES, + LOAD_BLOCKS, + USED_BLOCKS, + PENDING_STORE_BLOCKS, + INFO, + } + for metric in defs.values(): + assert metric.kwargs["documentation"] + assert metric.kwargs["labelnames"][:2] == ["model_name", "engine"] + + assert defs[SAVE_OUTCOMES].kwargs["labelnames"] == [ + "model_name", + "engine", + "outcome", + ] + assert defs[LOAD_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"] + assert defs[INFO].kwargs["labelnames"] == [ + "model_name", + "engine", + *INFO_LABELS, + ] + assert defs[USED_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"] + + # mostrecent: multiprocess scrapes expose one series per label set, not + # one pid-tagged series per writer process. + for gauge in (USED_BLOCKS, PENDING_STORE_BLOCKS, INFO): + assert defs[gauge].kwargs["multiprocess_mode"] == "mostrecent" + + +def test_prom_metrics_observe_routes_each_series() -> None: + prom = _make_prom_metrics() + prom.observe( + _stats_payload( + types={ + SAVE_OUTCOMES: _MetricType.COUNTER, + LOAD_BLOCKS: _MetricType.COUNTER, + USED_BLOCKS: _MetricType.GAUGE, + PENDING_STORE_BLOCKS: _MetricType.GAUGE, + INFO: _MetricType.GAUGE, + }, + data={ + SAVE_OUTCOMES: {("stored",): 3}, + LOAD_BLOCKS: {(): 4}, + USED_BLOCKS: {(): 5}, + PENDING_STORE_BLOCKS: {(): 1}, + INFO: {("cpu", "false", "false", "64"): 1}, + }, + ) + ) + + assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].increments == [3] + assert prom._metrics[(0, LOAD_BLOCKS, ())].increments == [4] + assert prom._metrics[(0, USED_BLOCKS, ())].set_values == [5] + assert prom._metrics[(0, PENDING_STORE_BLOCKS, ())].set_values == [1] + assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].set_values == [1] + assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].labelvalues == ( + "model", + "0", + "stored", + ) + assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].labelvalues == ( + "model", + "0", + "cpu", + "false", + "false", + "64", + ) + + +def test_prom_metrics_reuses_bound_children() -> None: + prom = _make_prom_metrics() + payload = _stats_payload( + types={SAVE_OUTCOMES: _MetricType.COUNTER}, + data={SAVE_OUTCOMES: {("stored",): 1}}, + ) + prom.observe(payload) + prom.observe(payload) + + child = prom._metrics[(0, SAVE_OUTCOMES, ("stored",))] + assert child.increments == [1, 1] + assert len(prom._defs[SAVE_OUTCOMES].children) == 1 + + +def test_prom_metrics_rejects_unknown_metric() -> None: + prom = _make_prom_metrics() + with pytest.raises(AssertionError, match="Unknown"): + prom.observe( + _stats_payload( + types={"vllm:not_a_metric": _MetricType.COUNTER}, + data={"vllm:not_a_metric": {(): 1}}, + ) + ) + + +def test_prom_metrics_rejects_wrong_label_count() -> None: + prom = _make_prom_metrics() + with pytest.raises(AssertionError, match="labels"): + prom.observe( + _stats_payload( + types={SAVE_OUTCOMES: _MetricType.COUNTER}, + data={SAVE_OUTCOMES: {("stored", "extra"): 1}}, + ) + ) + + +def test_prom_metrics_routes_per_engine() -> None: + prom = _make_prom_metrics(per_engine={0: ["model", "0"], 1: ["model", "1"]}) + prom.observe( + _stats_payload( + types={USED_BLOCKS: _MetricType.GAUGE}, data={USED_BLOCKS: {(): 3}} + ), + engine_idx=1, + ) + assert (0, USED_BLOCKS, ()) not in prom._metrics + assert prom._metrics[(1, USED_BLOCKS, ())].set_values == [3] + + +def test_connector_build_prom_metrics() -> None: + from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics + + prom = SimpleCPUOffloadConnector.build_prom_metrics( + vllm_config=SimpleNamespace(kv_transfer_config=None), + metric_types=_fake_metric_types(), + labelnames=["model_name", "engine"], + per_engine_labelvalues={0: ["model", "0"]}, + ) + assert isinstance(prom, SimpleCPUOffloadPromMetrics) + + +# --------------------------------------------------------------------------- +# Manager: interval stats +# --------------------------------------------------------------------------- + + +def test_manager_get_stats_reports_outcome_deltas_per_interval() -> None: + fixture = make_scheduler() + sched = fixture.scheduler + + sched.boundary_store_stats.stored += 3 + sched.boundary_store_stats.dropped_cpu_full += 2 + sched.boundary_store_stats.skipped_in_flight += 1 + + stats = sched.get_stats() + assert _values(stats, SAVE_OUTCOMES) == { + ("stored",): 3, + ("dropped_cpu_full",): 2, + ("skipped_in_flight",): 1, + } + + # Deltas are per-interval: a second drain reports no new outcomes. + stats2 = sched.get_stats() + assert SAVE_OUTCOMES not in stats2.data[_StatsKey.DATA] + + +def test_manager_reset_preserves_undrained_outcome_deltas() -> None: + fixture = make_scheduler() + sched = fixture.scheduler + + sched.boundary_store_stats.stored += 3 + sched.reset() + + stats = sched.get_stats() + assert _values(stats, SAVE_OUTCOMES) == {("stored",): 3} + + +def test_manager_get_stats_gauges_and_info_labels() -> None: + fixture = make_scheduler(num_cpu_blocks=8) + sched = fixture.scheduler + + sched._store_event_to_blocks[0] = TransferMeta([10], [11]) + sched._pending_finished_stores.append(TransferMeta([12], [13])) + sched._abandoned_store_event_to_blocks[1] = TransferMeta([14], [15]) + + stats = sched.get_stats() + assert _values(stats, PENDING_STORE_BLOCKS) == {(): 3} + used = _values(stats, USED_BLOCKS)[()] + assert used == sched.num_cpu_blocks - sched.cpu_block_pool.get_num_free_blocks() + + info = _values(stats, INFO) + assert info == {("cpu", "false", "false", str(sched.num_cpu_blocks)): 1} + + # Gauges are re-reported every drain. + stats2 = sched.get_stats() + assert _values(stats2, PENDING_STORE_BLOCKS) == {(): 3} + + +def test_manager_info_labels_reflect_disk_mode() -> None: + connector = _make_connector( + extra_config={ + "kv_offload_backend": "disk", + "disk_path": "/tmp/fake-disk-metrics", + "disk_capacity_bytes": 1024**3, + "use_page_cache": True, + } + ) + sched = connector.scheduler_manager + assert sched is not None + assert sched._info_labelvalues[0] == "disk" + assert sched._info_labelvalues[1] == "true" + stats = sched.get_stats() + assert set(_values(stats, INFO)) == {sched._info_labelvalues} + + +def test_manager_info_labels_page_cache_false_for_cpu_backend() -> None: + """CPU backend ignores use_page_cache; the info gauge must not claim it.""" + connector = _make_connector(extra_config={"use_page_cache": True}) + sched = connector.scheduler_manager + assert sched is not None + assert sched._info_labelvalues[:2] == ("cpu", "false") + stats = sched.get_stats() + assert set(_values(stats, INFO)) == {sched._info_labelvalues} + + +def test_manager_counts_load_blocks_completed() -> None: + fixture = make_scheduler(num_cpu_blocks=8) + sched = fixture.scheduler + + request = make_request(num_blocks=2) + gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2) + cpu_blocks = sched.cpu_block_pool.get_new_blocks(2) + sched._reqs_to_load[request.request_id] = LoadRequestState( + request=request, + transfer_meta=TransferMeta( + [b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks] + ), + ) + + sched.build_connector_meta(make_scheduler_output({})) + stats = sched.get_stats() + assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA] + + sched.update_connector_output( + KVConnectorOutput(finished_recving={request.request_id}) + ) + stats = sched.get_stats() + assert _values(stats, LOAD_BLOCKS) == {(): 2} + + +def test_manager_abandoned_load_counts_as_completed_after_reset() -> None: + """Loads abandoned by reset() that still finish on the worker are + counted as loaded blocks. + """ + fixture = make_scheduler(num_cpu_blocks=8) + sched = fixture.scheduler + + request = make_request(num_blocks=2) + gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2) + cpu_blocks = sched.cpu_block_pool.get_new_blocks(2) + sched._reqs_to_load[request.request_id] = LoadRequestState( + request=request, + transfer_meta=TransferMeta( + [b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks] + ), + ) + + sched.build_connector_meta(make_scheduler_output({})) + stats = sched.get_stats() + assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA] + + assert sched.reset() is False + assert set(sched._abandoned_reqs_to_load) == {request.request_id} + + sched.update_connector_output( + KVConnectorOutput(finished_recving={request.request_id}) + ) + assert sched._abandoned_reqs_to_load == {} + stats = sched.get_stats() + assert _values(stats, LOAD_BLOCKS) == {(): 2} diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py index 64bf8146c9..9124d9eac1 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py @@ -15,6 +15,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorRole, SupportsHMA, ) +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, +) from vllm.logger import init_logger from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.outputs import KVConnectorOutput @@ -25,6 +31,10 @@ from vllm.v1.simple_kv_offload.manager import ( from vllm.v1.simple_kv_offload.metadata import ( SimpleCPUOffloadMetadata, ) +from vllm.v1.simple_kv_offload.metrics import ( + SimpleCPUOffloadPromMetrics, + SimpleCPUOffloadStats, +) from vllm.v1.simple_kv_offload.worker import ( SimpleCPUOffloadWorker, ) @@ -162,6 +172,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA): hash_block_size=hash_block_size, lazy_offload=lazy_offload, disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0, + use_page_cache=use_page_cache if disk_mode else False, ) elif role == KVConnectorRole.WORKER: self.worker_handler = SimpleCPUOffloadWorker( @@ -307,6 +318,38 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA): return self.scheduler_manager.get_boundary_store_stats() return None + def get_kv_connector_stats(self) -> KVConnectorStats | None: + # Worker-side callers may invoke this hook on connectors constructed + # via __new__ (as in test_worker.py), which lack scheduler_manager. + scheduler_manager: SimpleCPUOffloadScheduler | None = getattr( + self, "scheduler_manager", None + ) + if scheduler_manager is not None: + return scheduler_manager.get_stats() + return None + + @classmethod + def build_kv_connector_stats( + cls, data: dict[str, Any] | None = None + ) -> KVConnectorStats | None: + return ( + SimpleCPUOffloadStats(data=data) + if data is not None + else SimpleCPUOffloadStats() + ) + + @classmethod + def build_prom_metrics( + cls, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ) -> KVConnectorPromMetrics: + return SimpleCPUOffloadPromMetrics( + vllm_config, metric_types, labelnames, per_engine_labelvalues + ) + def take_events(self) -> Iterable[KVCacheEvent]: if self.scheduler_manager is not None: return self.scheduler_manager.take_events() diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index 344daa36b8..f70afaa343 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -4,7 +4,7 @@ import contextlib from collections.abc import Iterable -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field, fields, replace from enum import Enum, auto from typing import TYPE_CHECKING, Any @@ -48,6 +48,11 @@ from vllm.v1.simple_kv_offload.metadata import ( SimpleCPUOffloadMetadata, SimpleCPUOffloadWorkerMetadata, ) +from vllm.v1.simple_kv_offload.metrics import ( + OUTCOME_TO_FIELD, + MetricName, + SimpleCPUOffloadStats, +) if TYPE_CHECKING: from vllm.v1.core.kv_cache_manager import KVCacheBlocks @@ -118,7 +123,8 @@ class BoundaryStoreStats: symptom is a lower cache hit rate with nothing in the logs, so each decline reason is counted and exposed through ``SimpleCPUOffloadConnector.get_boundary_store_stats()``. Reset by - ``reset()``; not otherwise cleared. + ``reset()``, which carries any not-yet-drained deltas; not otherwise + cleared. """ published: int = 0 @@ -143,6 +149,7 @@ class SimpleCPUOffloadScheduler: hash_block_size: int, lazy_offload: bool = False, disk_capacity_bytes: int = 0, + use_page_cache: bool = False, ): self.vllm_config = vllm_config self.kv_cache_config = kv_cache_config @@ -169,6 +176,13 @@ class SimpleCPUOffloadScheduler: self.cpu_kv_cache_config.prefix_cacheable_group_ids ) self.kv_event_medium = MEDIUM_STORAGE if disk_capacity_bytes > 0 else MEDIUM_CPU + # Must track kv_event_medium. + self._info_labelvalues: tuple[str, ...] = ( + "disk" if disk_capacity_bytes > 0 else "cpu", + str(use_page_cache).lower(), + str(lazy_offload).lower(), + str(self.num_cpu_blocks), + ) # Find the full attention kv group for prefix cache matching. self.fa_gidx = -1 for g_idx, g in enumerate(self.cpu_kv_cache_config.kv_cache_groups): @@ -251,6 +265,9 @@ class SimpleCPUOffloadScheduler: self._load_event_counter: int = 0 self._store_event_counter: int = 0 self.boundary_store_stats = BoundaryStoreStats() + # Interval stats state drained by get_stats() + self._boundary_stats_snapshot = BoundaryStoreStats() + self._interval_load_blocks_completed = 0 # For TP/PP: track partial store completions across steps. # Events must be reported by all world_size workers before considered complete. @@ -951,6 +968,42 @@ class SimpleCPUOffloadScheduler: def get_boundary_store_stats(self) -> BoundaryStoreStats: return replace(self.boundary_store_stats) + def get_stats(self) -> SimpleCPUOffloadStats: + """Drain per-step stats for the connector's stats hooks.""" + stats = SimpleCPUOffloadStats() + + current = self.boundary_store_stats + for outcome, field_name in OUTCOME_TO_FIELD.items(): + delta = getattr(current, field_name) - getattr( + self._boundary_stats_snapshot, field_name + ) + if delta > 0: + stats.increase_counter(MetricName.SAVE_OUTCOMES, delta, (outcome,)) + self._boundary_stats_snapshot = replace(current) + + if self._interval_load_blocks_completed: + stats.increase_counter( + MetricName.LOAD_BLOCKS, + self._interval_load_blocks_completed, + ) + self._interval_load_blocks_completed = 0 + + stats.set_gauge( + MetricName.USED_BLOCKS, + self.num_cpu_blocks - self.cpu_block_pool.get_num_free_blocks(), + ) + pending = sum( + len(t.cpu_block_ids) + for t in ( + *self._store_event_to_blocks.values(), + *self._pending_finished_stores, + *self._abandoned_store_event_to_blocks.values(), + ) + ) + stats.set_gauge(MetricName.PENDING_STORE_BLOCKS, pending) + stats.set_gauge(MetricName.INFO, 1, self._info_labelvalues) + return stats + def update_connector_output(self, connector_output: KVConnectorOutput) -> None: """Handle async transfer completions from worker. @@ -961,7 +1014,9 @@ class SimpleCPUOffloadScheduler: """ # --- Load completions --- for req_id in list(connector_output.finished_recving or []): - self._cleanup_load_request(req_id) + completed_blocks = self._cleanup_load_request(req_id) + if completed_blocks: + self._interval_load_blocks_completed += completed_blocks # --- Store completions --- meta = connector_output.kv_connector_worker_meta @@ -1266,18 +1321,22 @@ class SimpleCPUOffloadScheduler: if blocks_to_free: self.cpu_block_pool.free_blocks(blocks_to_free) - def _cleanup_load_request(self, req_id: str) -> None: + def _cleanup_load_request(self, req_id: str) -> int: """Release all load resources for a request. Shared between request_finished() and update_connector_output() paths. Removes the request from _reqs_to_load, cleans up event mappings, and frees CPU/GPU touch refs. + + Returns the number of blocks in the load if it had been issued to + the worker, else 0 (never-issued loads are not counted as completed + by the caller). """ state = self._reqs_to_load.pop(req_id, None) if state is None: state = self._abandoned_reqs_to_load.pop(req_id, None) if state is None: - return + return 0 # Remove from load event mapping (only this req, not whole event) if state.load_event is not None: reqs = self._load_event_to_reqs.get(state.load_event) @@ -1300,6 +1359,10 @@ class SimpleCPUOffloadScheduler: for bid in state.transfer_meta.gpu_block_ids ) + if state.load_event is None: + return 0 + return len(state.transfer_meta.gpu_block_ids) + def _cleanup_store_request(self, req_id: str) -> None: """Release store metadata for a request. @@ -1359,7 +1422,20 @@ class SimpleCPUOffloadScheduler: if event_idx in self._abandoned_store_event_to_blocks } self._cursor = None - self.boundary_store_stats = BoundaryStoreStats() + # Seed the fresh counters with outcomes since the last drain so a + # mid-interval reset() does not drop them from the next report. + undrained = BoundaryStoreStats( + **{ + f.name: max( + 0, + getattr(self.boundary_store_stats, f.name) + - getattr(self._boundary_stats_snapshot, f.name), + ) + for f in fields(BoundaryStoreStats) + } + ) + self.boundary_store_stats = undrained + self._boundary_stats_snapshot = BoundaryStoreStats() # NOTE: _load_event_counter / _store_event_counter are not # reset as they are monotonic and must stay ahead of the workers # high-water marks to avoid event index collisions diff --git a/vllm/v1/simple_kv_offload/metrics.py b/vllm/v1/simple_kv_offload/metrics.py new file mode 100644 index 0000000000..9017d40cfd --- /dev/null +++ b/vllm/v1/simple_kv_offload/metrics.py @@ -0,0 +1,284 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Prometheus metrics for SimpleCPUOffloadConnector. + +The stats payload uses the same self-describing wire format as the +offloading connector, so it survives IPC serialization without the +metric classes on the receiving side. +""" + +from dataclasses import dataclass +from typing import Any + +from vllm.config import VllmConfig +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, +) + + +class MetricName: + """Metric names for SimpleCPUOffloadConnector.""" + + SAVE_OUTCOMES = "vllm:simple_kv_offload_save_outcomes_total" + LOAD_BLOCKS = "vllm:simple_kv_offload_load_blocks_total" + USED_BLOCKS = "vllm:simple_kv_offload_used_blocks" + PENDING_STORE_BLOCKS = "vllm:simple_kv_offload_pending_store_blocks" + INFO = "vllm:simple_kv_offload_info" + + +# outcome label value -> BoundaryStoreStats field name. +# ``published`` is the sum of all outcomes and is not labeled. +OUTCOME_TO_FIELD = { + "stored": "stored", + "dropped_cpu_full": "dropped_cpu_full", + "dropped_request_gone": "dropped_request_gone", + "dropped_null_block": "dropped_null_block", + "dropped_not_hashed": "dropped_not_hashed", + "skipped_already_cached": "skipped_already_cached", + "skipped_in_flight": "skipped_in_flight", +} + +INFO_LABELS = ["backend", "page_cache", "lazy_offload", "capacity_blocks"] + + +class _MetricType: + """Type tags embedded in the serialized stats payload.""" + + COUNTER = "counter" + GAUGE = "gauge" + + +class _StatsKey: + """Top-level keys in the serialized stats dict.""" + + # Maps metric name -> _MetricType value + TYPES = "types" + # Maps metric name -> {label values tuple -> observed value} + DATA = "data" + + +@dataclass +class SimpleCPUOffloadStats(KVConnectorStats): + """Stats payload for SimpleCPUOffloadConnector. + + The ``data`` dict is structured as:: + + { + _StatsKey.TYPES: {name: _MetricType.*, ...}, + _StatsKey.DATA: {name: {labelvalues: value, ...}, ...}, + } + + Counter values are summed when aggregating, gauge values keep the + latest snapshot, and unlabeled metrics use ``()`` as their + labelvalues tuple. + """ + + def __post_init__(self): + if _StatsKey.DATA not in self.data: + self.reset() + + def reset(self): + self.data: dict[str, Any] = { + _StatsKey.TYPES: {}, + _StatsKey.DATA: {}, + } + + @property + def _types(self) -> dict[str, str]: + return self.data[_StatsKey.TYPES] + + @property + def _values(self) -> dict[str, Any]: + return self.data[_StatsKey.DATA] + + def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats": + if other.is_empty(): + return self + assert isinstance(other, SimpleCPUOffloadStats) + for key, other_label_values in other._values.items(): + type_str = other._types.get(key) + if type_str is None: + raise AssertionError(f"Unknown simple KV offload stats key: {key}") + self._types.setdefault(key, type_str) + current_label_values = self._values.setdefault(key, {}) + for labelvalues, value in other_label_values.items(): + if type_str == _MetricType.COUNTER: + assert isinstance(value, int | float) + current_label_values[labelvalues] = ( + current_label_values.get(labelvalues, 0) + value + ) + elif type_str == _MetricType.GAUGE: + assert isinstance(value, int | float) + current_label_values[labelvalues] = value + else: + raise AssertionError( + f"Unknown metric type '{type_str}' for key: {key}" + ) + return self + + def reduce(self) -> dict[str, int | float]: + """Reduce the interval observations to a flat summary for logging.""" + return_dict: dict[str, int | float] = {} + for key, label_value_map in self._values.items(): + if key == MetricName.INFO: + # Constant config fingerprint; exclude from the log summary. + continue + type_str = self._types.get(key) + if type_str is None: + raise AssertionError(f"Unknown simple KV offload stats key: {key}") + for labelvalues, value in label_value_map.items(): + key_with_labels = f"{key}:{labelvalues}" if labelvalues else key + if type_str in (_MetricType.COUNTER, _MetricType.GAUGE): + assert isinstance(value, int | float) + return_dict[key_with_labels] = value + else: + raise AssertionError( + f"Unknown metric type '{type_str}' for key: {key}" + ) + return return_dict + + def is_empty(self) -> bool: + return not self.data.get(_StatsKey.DATA) + + def increase_counter( + self, + counter_name: str, + counter_increase_value: int | float = 1, + labelvalues: tuple[str, ...] = (), + ) -> None: + """Increase a counter on the stats payload.""" + self._types.setdefault(counter_name, _MetricType.COUNTER) + counter_values = self._values.setdefault(counter_name, {}) + counter_values[labelvalues] = ( + counter_values.get(labelvalues, 0) + counter_increase_value + ) + + def set_gauge( + self, + gauge_name: str, + gauge_value: int | float, + labelvalues: tuple[str, ...] = (), + ) -> None: + """Set a gauge snapshot on the stats payload.""" + self._types.setdefault(gauge_name, _MetricType.GAUGE) + gauge_values = self._values.setdefault(gauge_name, {}) + gauge_values[labelvalues] = gauge_value + + +class SimpleCPUOffloadPromMetrics(KVConnectorPromMetrics): + """Prometheus registration and recording for SimpleCPUOffloadConnector.""" + + def __init__( + self, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ): + super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues) + # (engine_idx, metric_name, labelvalues) -> bound child metric + self._metrics: dict[tuple[int, str, tuple[str, ...]], PromMetricT] = {} + + self._counter_save_outcomes = self._counter_cls( + name="vllm:simple_kv_offload_save_outcomes_total", + documentation=( + "Store admission decisions in SimpleCPUOffloadConnector, by " + "outcome. Outcomes classify eager-mode boundary hand-off " + "stores; lazy-mode stores are not classified." + ), + labelnames=self._labelnames + ["outcome"], + ) + self._counter_load_blocks = self._counter_cls( + name="vllm:simple_kv_offload_load_blocks_total", + documentation=( + "KV blocks restored to GPU from the offload pool, counted " + "when a load finishes. This is prefill work avoided " + "through offload cache hits." + ), + labelnames=self._labelnames, + ) + self._gauge_used_blocks = self._gauge_cls( + name="vllm:simple_kv_offload_used_blocks", + documentation=( + "Offload-pool blocks currently pinned by in-flight " + "transfers or cache hits (capacity minus free). Evictable " + "cached blocks are not counted; capacity_blocks is on " + "vllm:simple_kv_offload_info." + ), + labelnames=self._labelnames, + multiprocess_mode="mostrecent", + ) + self._gauge_pending_store_blocks = self._gauge_cls( + name="vllm:simple_kv_offload_pending_store_blocks", + documentation=( + "Offload blocks in pending stores: queued for dispatch, " + "issued to workers, or awaiting release after a cache " + "reset. A persistently growing value indicates a stuck " + "transfer." + ), + labelnames=self._labelnames, + multiprocess_mode="mostrecent", + ) + self._gauge_info = self._gauge_cls( + name="vllm:simple_kv_offload_info", + documentation=( + "SimpleCPUOffloadConnector deployment facts. Value is " + "always 1. backend is cpu or disk; page_cache and " + "lazy_offload are true/false; capacity_blocks is the " + "offload pool size." + ), + labelnames=self._labelnames + INFO_LABELS, + multiprocess_mode="mostrecent", + ) + + self._defs: dict[str, PromMetricT] = { + MetricName.SAVE_OUTCOMES: self._counter_save_outcomes, + MetricName.LOAD_BLOCKS: self._counter_load_blocks, + MetricName.USED_BLOCKS: self._gauge_used_blocks, + MetricName.PENDING_STORE_BLOCKS: self._gauge_pending_store_blocks, + MetricName.INFO: self._gauge_info, + } + self._num_label_values: dict[str, int] = { + MetricName.SAVE_OUTCOMES: 1, + MetricName.LOAD_BLOCKS: 0, + MetricName.USED_BLOCKS: 0, + MetricName.PENDING_STORE_BLOCKS: 0, + MetricName.INFO: len(INFO_LABELS), + } + + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): + """Observe transfer statistics.""" + metric_types = transfer_stats_data.get(_StatsKey.TYPES, {}) + metric_data = transfer_stats_data.get(_StatsKey.DATA, {}) + for key, label_value_map in metric_data.items(): + prom_metric = self._defs.get(key) + if prom_metric is None: + raise AssertionError(f"Unknown simple KV offload stats key: {key}") + type_str = metric_types.get(key) + for labelvalues, value in label_value_map.items(): + if len(labelvalues) != self._num_label_values[key]: + raise AssertionError( + f"Metric {key} expects " + f"{self._num_label_values[key]} labels, " + f"got {len(labelvalues)}" + ) + child = self._metrics.get((engine_idx, key, labelvalues)) + if child is None: + engine_labelvalues = self.per_engine_labelvalues[engine_idx] + child = prom_metric.labels( + *(engine_labelvalues + list(labelvalues)) + ) + self._metrics[(engine_idx, key, labelvalues)] = child + if type_str == _MetricType.COUNTER: + assert isinstance(value, int | float) + child.inc(value) + elif type_str == _MetricType.GAUGE: + assert isinstance(value, int | float) + child.set(value) + else: + raise AssertionError( + f"Unknown metric type '{type_str}' for key: {key}" + )