From e040899a006ffd07c51151b32d29649f30cb21e9 Mon Sep 17 00:00:00 2001 From: Srinivas Krovvidi <194645829+Srinivasoo7@users.noreply.github.com> Date: Tue, 7 Jul 2026 01:26:28 -0500 Subject: [PATCH] [KV Offloading] Add basic offloading metrics (#45958) Signed-off-by: srinivas_oo7 Signed-off-by: Srinivas Krovvidi <194645829+Srinivasoo7@users.noreply.github.com> Co-authored-by: srinivas_oo7 Co-authored-by: Or Ozeri Co-authored-by: Or Ozeri --- .../unit/offloading_connector/test_metrics.py | 38 ++++++++++ .../offloading_connector/test_scheduler.py | 70 +++++++++++++++++++ .../unit/offloading_connector/utils.py | 19 ++++- tests/v1/kv_offload/cpu/test_manager.py | 58 +++++++++++++++ tests/v1/kv_offload/test_factory.py | 32 ++++++++- .../kv_connector/v1/offloading/metrics.py | 54 +++++++++++++- .../kv_connector/v1/offloading/scheduler.py | 45 +++++++++--- vllm/v1/kv_offload/cpu/common.py | 1 + vllm/v1/kv_offload/cpu/manager.py | 9 +++ vllm/v1/kv_offload/cpu/spec.py | 10 ++- 10 files changed, 320 insertions(+), 16 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_metrics.py b/tests/v1/kv_connector/unit/offloading_connector/test_metrics.py index 6f36a6c814..fb6b785c88 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_metrics.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_metrics.py @@ -10,9 +10,11 @@ from prometheus_client import Counter, Gauge, Histogram from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( OffloadingConnectorStats, OffloadPromMetrics, + _ConnectorMetricName, _MetricType, _StatsKey, _TransferMetricName, + get_connector_metric_definitions, ) from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import ( OffloadingConnector, @@ -37,6 +39,42 @@ MY_COUNTER = "my_counter" MY_LABEL = "my_label" +def test_connector_metric_histogram_buckets(): + metadata = get_connector_metric_definitions() + + sync_delay = metadata[_ConnectorMetricName.LOOKUP_SYNC_DELAY] + assert isinstance(sync_delay, OffloadingHistogramMetadata) + assert sync_delay.buckets == ( + 0.00001, + 0.00005, + 0.0001, + 0.0005, + 0.001, + 0.005, + 0.01, + 0.05, + 0.1, + 0.5, + 1, + ) + + async_delay = metadata[_ConnectorMetricName.LOOKUP_ASYNC_DELAY] + assert isinstance(async_delay, OffloadingHistogramMetadata) + assert async_delay.buckets == ( + 0.0001, + 0.0005, + 0.001, + 0.005, + 0.01, + 0.05, + 0.1, + 0.5, + 1, + 5, + 10, + ) + + class _FakeMetric: def __init__(self, **kwargs): self.kwargs = kwargs diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index ad5792e6c3..836f93eb59 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -11,6 +11,10 @@ from tests.v1.kv_connector.unit.offloading_connector.utils import ( to_keys, ) from tests.v1.kv_connector.unit.utils import EOS_TOKEN_ID +from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( + OffloadingConnectorStats, + _ConnectorMetricName, +) from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import ( OffloadingConnectorScheduler, RequestOffloadState, @@ -32,6 +36,72 @@ from vllm.v1.kv_offload.base import ( from vllm.v1.request import RequestStatus +def _reduce_kv_connector_stats(runner): + reduced: dict[str, int | float] = {} + for payload in runner.kv_connector_stats: + stats = ( + payload + if hasattr(payload, "reduce") + else OffloadingConnectorStats(data=payload) + ) + for key, value in stats.reduce().items(): + reduced[key] = reduced.get(key, 0) + value + return reduced + + +def test_scheduler_reports_allocation_failure(request_runner): + runner = request_runner( + block_size=4, + num_gpu_blocks=10, + async_scheduling=False, + ) + runner.new_request(token_ids=[0] * 4) + runner.manager.prepare_store.side_effect = lambda keys, req_context: None + + runner.run(decoded_tokens=[EOS_TOKEN_ID]) + + reduced = _reduce_kv_connector_stats(runner) + assert reduced[_ConnectorMetricName.ALLOCATION_FAILURE] == 1 + + +def test_scheduler_reports_lookup_sync_delay(request_runner): + runner = request_runner( + block_size=4, + num_gpu_blocks=10, + async_scheduling=False, + ) + runner.new_request(token_ids=[1] * 4) + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output([]) + ) + + runner.run(decoded_tokens=[EOS_TOKEN_ID]) + + reduced = _reduce_kv_connector_stats(runner) + assert reduced[f"{_ConnectorMetricName.LOOKUP_SYNC_DELAY}_count"] == 1 + assert reduced[f"{_ConnectorMetricName.LOOKUP_SYNC_DELAY}_sum"] > 0 + + +def test_scheduler_reports_lookup_async_delay_on_resolve(request_runner): + """A deferred lookup reports its async delay once it resolves.""" + runner = request_runner( + block_size=4, + num_gpu_blocks=10, + async_scheduling=False, + ) + runner.manager.lookup.side_effect = [LookupResult.RETRY, LookupResult.MISS] + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output([]) + ) + + runner.new_request(token_ids=[1] * 4) + runner.run(decoded_tokens=[EOS_TOKEN_ID]) + + reduced = _reduce_kv_connector_stats(runner) + assert reduced[f"{_ConnectorMetricName.LOOKUP_ASYNC_DELAY}_count"] == 1 + assert reduced[f"{_ConnectorMetricName.LOOKUP_ASYNC_DELAY}_sum"] > 0 + + @pytest.mark.parametrize("async_scheduling", [True, False]) def test_offloading_connector(request_runner, async_scheduling: bool): block_size = 4 diff --git a/tests/v1/kv_connector/unit/offloading_connector/utils.py b/tests/v1/kv_connector/unit/offloading_connector/utils.py index c2884649bd..0d152a2038 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/utils.py +++ b/tests/v1/kv_connector/unit/offloading_connector/utils.py @@ -133,6 +133,7 @@ class MockOffloadingSpec(OffloadingSpec): self.manager = MagicMock(spec=OffloadingManager) self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys) self.manager.lookup.return_value = LookupResult.MISS + self.manager.get_stats.return_value = None self.manager.on_new_request.return_value = RequestOffloadingContext() self.handler = MockOffloadingWorker() @@ -335,6 +336,7 @@ class RequestRunner: self.completed_loads: list[TransferSummary] = [] self.completed_stores: list[TransferSummary] = [] self.flushed_gpu_blocks: set[GPUBlock] = set() + self.kv_connector_stats: list[Any] = [] # block_id -> GPUBlock self.gpu_blocks: dict[int, GPUBlock] = {} @@ -348,6 +350,12 @@ class RequestRunner: slot_mapping={}, ) + def _record_kv_connector_stats(self, engine_outputs: dict[int, Any]) -> None: + for output in engine_outputs.values(): + scheduler_stats = output.scheduler_stats + if scheduler_stats is not None and scheduler_stats.kv_connector_stats: + self.kv_connector_stats.append(scheduler_stats.kv_connector_stats) + def new_request( self, token_ids: list[int], @@ -524,13 +532,17 @@ class RequestRunner: if self.async_scheduling: # in async scheduling we update the output of the previous step if prev_model_runner_output is not None: - self.scheduler.update_from_output( + engine_outputs = self.scheduler.update_from_output( prev_scheduler_output, prev_model_runner_output ) + self._record_kv_connector_stats(engine_outputs) prev_scheduler_output = scheduler_output prev_model_runner_output = model_runner_output else: - self.scheduler.update_from_output(scheduler_output, model_runner_output) + engine_outputs = self.scheduler.update_from_output( + scheduler_output, model_runner_output + ) + self._record_kv_connector_stats(engine_outputs) if post_step_fn is not None: post_step_fn() @@ -546,9 +558,10 @@ class RequestRunner: if token_id is None: if self.async_scheduling: # sample last token - self.scheduler.update_from_output( + engine_outputs = self.scheduler.update_from_output( prev_scheduler_output, prev_model_runner_output ) + self._record_kv_connector_stats(engine_outputs) break self._parse_transfers() diff --git a/tests/v1/kv_offload/cpu/test_manager.py b/tests/v1/kv_offload/cpu/test_manager.py index 89a0374b46..15cac05158 100644 --- a/tests/v1/kv_offload/cpu/test_manager.py +++ b/tests/v1/kv_offload/cpu/test_manager.py @@ -224,6 +224,64 @@ def test_cpu_manager_reports_cache_usage_gauge(): check_usage_stats(manager, 0.0) +def test_cpu_manager_reports_allocation_size_histogram(): + manager = make_cpu_manager(num_blocks=4, cache_policy="lru") + + manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX) + manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX) + manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX) + + stats = manager.get_stats() + + assert stats is not None + reduced = stats.reduce() + assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count"] == 2 + assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_sum"] == 3 + + # The cache-usage gauge is always reported, so get_stats() never returns + # None, but the histogram has nothing new once its samples are consumed. + second_stats = manager.get_stats() + assert second_stats is not None + assert f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count" not in ( + second_stats.reduce() + ) + + +def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch): + manager = make_cpu_manager(num_blocks=4, cache_policy="lru") + + def fail_allocate_blocks(keys): + raise RuntimeError("allocation failed") + + monkeypatch.setattr(manager, "_allocate_blocks", fail_allocate_blocks) + + with pytest.raises(RuntimeError, match="allocation failed"): + manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX) + + stats = manager.get_stats() + + assert stats is not None + reduced = stats.reduce() + assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count"] == 1 + assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_sum"] == 3 + + +def test_cpu_manager_reports_allocation_size_on_eviction_failure(): + manager = make_cpu_manager(num_blocks=1, cache_policy="lru") + + manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX) + manager.get_stats() + + assert manager.prepare_store(to_keys([2]), _EMPTY_REQ_CTX) is None + + stats = manager.get_stats() + + assert stats is not None + reduced = stats.reduce() + assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count"] == 1 + assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_sum"] == 1 + + def test_cpu_manager(): """ Tests CPUOffloadingManager with lru policy. diff --git a/tests/v1/kv_offload/test_factory.py b/tests/v1/kv_offload/test_factory.py index 543ee44330..b051f44ecf 100644 --- a/tests/v1/kv_offload/test_factory.py +++ b/tests/v1/kv_offload/test_factory.py @@ -21,7 +21,7 @@ from vllm.v1.kv_cache_interface import ( KVCacheGroupSpec, KVCacheTensor, ) -from vllm.v1.kv_offload.base import OffloadingSpec +from vllm.v1.kv_offload.base import OffloadingHistogramMetadata, OffloadingSpec from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec from vllm.v1.kv_offload.factory import OffloadingSpecFactory from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec @@ -248,8 +248,8 @@ def test_duplicate_registration_raises(): # --------------------------------------------------------------------------- -def test_build_metric_definitions_empty_below_threshold(): - """store_threshold < 2 → only base metric (no stores_skipped).""" +def test_build_metric_definitions_below_threshold(): + """store_threshold < 2 keeps stores_skipped disabled.""" from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics config = _make_vllm_config(store_threshold=1) @@ -258,6 +258,32 @@ def test_build_metric_definitions_empty_below_threshold(): config.kv_transfer_config.kv_connector_extra_config ) assert CPUOffloadingMetrics.STORES_SKIPPED not in metrics + assert CPUOffloadingMetrics.CPU_ALLOCATION_SIZE in metrics + + +def test_build_metric_definitions_allocation_size_histogram(): + """CPU allocation size is always reported as a histogram.""" + from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics + + config = _make_vllm_config(store_threshold=0) + spec_cls = OffloadingSpecFactory.get_spec_cls(config) + metrics = spec_cls.build_metric_definitions( + config.kv_transfer_config.kv_connector_extra_config + ) + metadata = metrics[CPUOffloadingMetrics.CPU_ALLOCATION_SIZE] + assert isinstance(metadata, OffloadingHistogramMetadata) + assert metadata.buckets == ( + 1, + 4, + 16, + 64, + 256, + 1024, + 4096, + 16384, + 65536, + 262144, + ) def test_build_metric_definitions_returns_counter_at_threshold(): diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/metrics.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/metrics.py index a90250d285..baa168e170 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/metrics.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/metrics.py @@ -30,6 +30,14 @@ class _TransferMetricName: STORE_SIZE = "vllm:kv_offload_store_size" +class _ConnectorMetricName: + """Connector-side metrics emitted by scheduler-side offloading code.""" + + LOOKUP_SYNC_DELAY = "vllm:kv_offload_lookup_sync_delay_seconds" + LOOKUP_ASYNC_DELAY = "vllm:kv_offload_lookup_async_delay_seconds" + ALLOCATION_FAILURE = "vllm:kv_offload_allocation_failure" + + class _TransferType: """Transfer direction labels for deprecated CPU offload metrics.""" @@ -74,6 +82,50 @@ def get_connector_metric_definitions() -> dict[str, OffloadingMetricMetadata]: documentation="Histogram of KV offload store operation size, in bytes.", buckets=TRANSFER_SIZE_BUCKETS, ), + _ConnectorMetricName.LOOKUP_SYNC_DELAY: OffloadingHistogramMetadata( + documentation=( + "Histogram of the time spent in a single offload lookup call, " + "in seconds." + ), + buckets=( + 0.00001, + 0.00005, + 0.0001, + 0.0005, + 0.001, + 0.005, + 0.01, + 0.05, + 0.1, + 0.5, + 1, + ), + ), + _ConnectorMetricName.LOOKUP_ASYNC_DELAY: OffloadingHistogramMetadata( + documentation=( + "Histogram of time between a request's offload lookup first " + "deferring and the following lookup resolving, or request " + "finish, in seconds." + ), + buckets=( + 0.0001, + 0.0005, + 0.001, + 0.005, + 0.01, + 0.05, + 0.1, + 0.5, + 1, + 5, + 10, + ), + ), + _ConnectorMetricName.ALLOCATION_FAILURE: OffloadingCounterMetadata( + documentation=( + "Number of KV offload store allocation attempts that failed." + ), + ), } @@ -223,7 +275,7 @@ class OffloadingConnectorStats(KVConnectorStats): def increase_counter( self, counter_name: str, - counter_increase_value: int | float, + counter_increase_value: int | float = 1, labelvalues: tuple[str, ...] = (), ) -> None: """Increase a counter on the stats payload.""" diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 284098ce94..cf0a506a02 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import time from collections.abc import Iterable, Sequence from dataclasses import dataclass, field from itertools import islice @@ -21,6 +22,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.events import ( ) from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( OffloadingConnectorStats, + _ConnectorMetricName, _TransferMetricName, ) from vllm.logger import init_logger @@ -246,6 +248,9 @@ class RequestOffloadState: # In-flight job IDs. Per the connector's invariant, at any given time # this contains either a single load job, or one or more store jobs. transfer_jobs: set[int] = field(default_factory=set) + # time.monotonic() of this request's first deferred offload lookup; + # None once consumed (observed) or while no lookup is pending. + deferred_lookup_start_time: float | None = None def __post_init__(self) -> None: self.group_states = tuple( @@ -325,7 +330,7 @@ class OffloadingConnectorScheduler: ): self.config = SchedulerOffloadConfig.from_spec(spec) self.manager: OffloadingManager = spec.get_manager() - self._connector_stats: OffloadingConnectorStats | None = None + self._connector_stats = OffloadingConnectorStats() full_attention_groups: list[int] = [] sliding_window_groups: list[int] = [] @@ -375,6 +380,18 @@ class OffloadingConnectorScheduler: self._events_tracker = OffloadingEventsTracker(spec.kv_events_config) + def _maybe_observe_lookup_async_delay( + self, req_status: RequestOffloadState + ) -> None: + start_time = req_status.deferred_lookup_start_time + if start_time is None: + return + req_status.deferred_lookup_start_time = None + self._connector_stats.observe_histogram( + _ConnectorMetricName.LOOKUP_ASYNC_DELAY, + time.monotonic() - start_time, + ) + def _generate_job_id(self) -> int: job_id = self._job_counter self._job_counter += 1 @@ -685,7 +702,17 @@ class OffloadingConnectorScheduler: if request.skip_reading_prefix_cache: num_hit_tokens = 0 else: + lookup_start = time.monotonic() num_hit_tokens = self._lookup(req_status) + self._connector_stats.observe_histogram( + _ConnectorMetricName.LOOKUP_SYNC_DELAY, + time.monotonic() - lookup_start, + ) + if num_hit_tokens is None: + if req_status.deferred_lookup_start_time is None: + req_status.deferred_lookup_start_time = lookup_start + else: + self._maybe_observe_lookup_async_delay(req_status) req_status.update_num_hit_blocks(num_computed_tokens + (num_hit_tokens or 0)) self._touch(req_status) @@ -925,6 +952,9 @@ class OffloadingConnectorScheduler: new_offload_keys, req_status.req_context ) if store_output is None: + self._connector_stats.increase_counter( + _ConnectorMetricName.ALLOCATION_FAILURE + ) logger.warning("Request %s: cannot store blocks", req_id) continue @@ -1115,10 +1145,7 @@ class OffloadingConnectorScheduler: transfer_stats.observe_histogram( _TransferMetricName.STORE_SIZE, size ) - if self._connector_stats is None: - self._connector_stats = transfer_stats - else: - self._connector_stats.aggregate(transfer_stats) + self._connector_stats.aggregate(transfer_stats) for job_id, count in meta.completed_jobs.items(): assert count > 0 @@ -1159,8 +1186,10 @@ class OffloadingConnectorScheduler: del self._req_status[job_status.req_id] def get_stats(self) -> OffloadingConnectorStats | None: - stats = self._connector_stats - self._connector_stats = None + stats: OffloadingConnectorStats | None = None + if not self._connector_stats.is_empty(): + stats = self._connector_stats + self._connector_stats = OffloadingConnectorStats() manager_stats = self.manager.get_stats() if manager_stats is not None: @@ -1198,7 +1227,7 @@ class OffloadingConnectorScheduler: return False, None self.manager.on_request_finished(req_status.req_context) - + self._maybe_observe_lookup_async_delay(req_status) if not req_status.transfer_jobs: # No in-flight jobs: no later complete_store()/complete_load() calls # need this request's state. diff --git a/vllm/v1/kv_offload/cpu/common.py b/vllm/v1/kv_offload/cpu/common.py index 14c96680fd..b6a2c2256a 100644 --- a/vllm/v1/kv_offload/cpu/common.py +++ b/vllm/v1/kv_offload/cpu/common.py @@ -8,6 +8,7 @@ from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec class CPUOffloadingMetrics: STORES_SKIPPED = "vllm:kv_offload_stores_skipped" CPU_CACHE_USAGE_PERC = "vllm:kv_offload_cpu_cache_usage_perc" + CPU_ALLOCATION_SIZE = "vllm:kv_offload_cpu_allocation_size" class CPULoadStoreSpec(BlockIDsLoadStoreSpec): diff --git a/vllm/v1/kv_offload/cpu/manager.py b/vllm/v1/kv_offload/cpu/manager.py index 0424196c9f..fa5a9e04e6 100644 --- a/vllm/v1/kv_offload/cpu/manager.py +++ b/vllm/v1/kv_offload/cpu/manager.py @@ -69,6 +69,7 @@ class CPUOffloadingManager(OffloadingManager): self.store_threshold: int = store_threshold self.max_tracker_size: int = max_tracker_size self.stores_skipped_in_current_batch: int = 0 + self.allocation_sizes_in_current_batch: list[int] = [] # Number of block references. It is ordered so can evict the LRU entry in O(1). self.counts: OrderedDict[OffloadKey, int] | None = ( @@ -185,6 +186,7 @@ class CPUOffloadingManager(OffloadingManager): evicted_keys=[], ) + self.allocation_sizes_in_current_batch.append(len(keys_to_store)) num_blocks_to_evict = len(keys_to_store) - self._get_num_free_blocks() to_evict: list[OffloadKey] = [] @@ -300,10 +302,17 @@ class CPUOffloadingManager(OffloadingManager): usage = num_used / self._num_blocks if self._num_blocks > 0 else 0.0 stats.set_gauge(CPUOffloadingMetrics.CPU_CACHE_USAGE_PERC, usage) + for allocation_size in self.allocation_sizes_in_current_batch: + stats.observe_histogram( + CPUOffloadingMetrics.CPU_ALLOCATION_SIZE, allocation_size + ) + self.allocation_sizes_in_current_batch.clear() + if self.store_threshold >= 2: stats.increase_counter( CPUOffloadingMetrics.STORES_SKIPPED, self.stores_skipped_in_current_batch, ) self.stores_skipped_in_current_batch = 0 + return stats diff --git a/vllm/v1/kv_offload/cpu/spec.py b/vllm/v1/kv_offload/cpu/spec.py index 16729a9dbb..26ea372819 100644 --- a/vllm/v1/kv_offload/cpu/spec.py +++ b/vllm/v1/kv_offload/cpu/spec.py @@ -12,6 +12,7 @@ from vllm.v1.kv_offload.base import ( CanonicalKVCaches, OffloadingCounterMetadata, OffloadingGaugeMetadata, + OffloadingHistogramMetadata, OffloadingManager, OffloadingMetricMetadata, OffloadingSpec, @@ -37,7 +38,14 @@ class CPUOffloadingSpec(OffloadingSpec): "values indicate transfers (stores or promotions) may be " "dropped due to insufficient capacity." ), - ) + ), + CPUOffloadingMetrics.CPU_ALLOCATION_SIZE: OffloadingHistogramMetadata( + documentation=( + "Histogram of the number of CPU blocks requested by each " + "KV offload prepare_store call." + ), + buckets=(1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144), + ), } store_threshold = int(extra_config.get("store_threshold", 0)) if store_threshold >= 2: