[KV Offloading] Add basic offloading metrics (#45958)

Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Signed-off-by: Srinivas Krovvidi <194645829+Srinivasoo7@users.noreply.github.com>
Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Srinivas Krovvidi
2026-07-07 09:26:28 +03:00
committed by GitHub
co-authored by srinivas_oo7 Or Ozeri Or Ozeri
parent dd5c299fbe
commit e040899a00
10 changed files with 320 additions and 16 deletions
@@ -10,9 +10,11 @@ from prometheus_client import Counter, Gauge, Histogram
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
OffloadPromMetrics,
_ConnectorMetricName,
_MetricType,
_StatsKey,
_TransferMetricName,
get_connector_metric_definitions,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import (
OffloadingConnector,
@@ -37,6 +39,42 @@ MY_COUNTER = "my_counter"
MY_LABEL = "my_label"
def test_connector_metric_histogram_buckets():
metadata = get_connector_metric_definitions()
sync_delay = metadata[_ConnectorMetricName.LOOKUP_SYNC_DELAY]
assert isinstance(sync_delay, OffloadingHistogramMetadata)
assert sync_delay.buckets == (
0.00001,
0.00005,
0.0001,
0.0005,
0.001,
0.005,
0.01,
0.05,
0.1,
0.5,
1,
)
async_delay = metadata[_ConnectorMetricName.LOOKUP_ASYNC_DELAY]
assert isinstance(async_delay, OffloadingHistogramMetadata)
assert async_delay.buckets == (
0.0001,
0.0005,
0.001,
0.005,
0.01,
0.05,
0.1,
0.5,
1,
5,
10,
)
class _FakeMetric:
def __init__(self, **kwargs):
self.kwargs = kwargs
@@ -11,6 +11,10 @@ from tests.v1.kv_connector.unit.offloading_connector.utils import (
to_keys,
)
from tests.v1.kv_connector.unit.utils import EOS_TOKEN_ID
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
_ConnectorMetricName,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
OffloadingConnectorScheduler,
RequestOffloadState,
@@ -32,6 +36,72 @@ from vllm.v1.kv_offload.base import (
from vllm.v1.request import RequestStatus
def _reduce_kv_connector_stats(runner):
reduced: dict[str, int | float] = {}
for payload in runner.kv_connector_stats:
stats = (
payload
if hasattr(payload, "reduce")
else OffloadingConnectorStats(data=payload)
)
for key, value in stats.reduce().items():
reduced[key] = reduced.get(key, 0) + value
return reduced
def test_scheduler_reports_allocation_failure(request_runner):
runner = request_runner(
block_size=4,
num_gpu_blocks=10,
async_scheduling=False,
)
runner.new_request(token_ids=[0] * 4)
runner.manager.prepare_store.side_effect = lambda keys, req_context: None
runner.run(decoded_tokens=[EOS_TOKEN_ID])
reduced = _reduce_kv_connector_stats(runner)
assert reduced[_ConnectorMetricName.ALLOCATION_FAILURE] == 1
def test_scheduler_reports_lookup_sync_delay(request_runner):
runner = request_runner(
block_size=4,
num_gpu_blocks=10,
async_scheduling=False,
)
runner.new_request(token_ids=[1] * 4)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output([])
)
runner.run(decoded_tokens=[EOS_TOKEN_ID])
reduced = _reduce_kv_connector_stats(runner)
assert reduced[f"{_ConnectorMetricName.LOOKUP_SYNC_DELAY}_count"] == 1
assert reduced[f"{_ConnectorMetricName.LOOKUP_SYNC_DELAY}_sum"] > 0
def test_scheduler_reports_lookup_async_delay_on_resolve(request_runner):
"""A deferred lookup reports its async delay once it resolves."""
runner = request_runner(
block_size=4,
num_gpu_blocks=10,
async_scheduling=False,
)
runner.manager.lookup.side_effect = [LookupResult.RETRY, LookupResult.MISS]
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output([])
)
runner.new_request(token_ids=[1] * 4)
runner.run(decoded_tokens=[EOS_TOKEN_ID])
reduced = _reduce_kv_connector_stats(runner)
assert reduced[f"{_ConnectorMetricName.LOOKUP_ASYNC_DELAY}_count"] == 1
assert reduced[f"{_ConnectorMetricName.LOOKUP_ASYNC_DELAY}_sum"] > 0
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_offloading_connector(request_runner, async_scheduling: bool):
block_size = 4
@@ -133,6 +133,7 @@ class MockOffloadingSpec(OffloadingSpec):
self.manager = MagicMock(spec=OffloadingManager)
self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys)
self.manager.lookup.return_value = LookupResult.MISS
self.manager.get_stats.return_value = None
self.manager.on_new_request.return_value = RequestOffloadingContext()
self.handler = MockOffloadingWorker()
@@ -335,6 +336,7 @@ class RequestRunner:
self.completed_loads: list[TransferSummary] = []
self.completed_stores: list[TransferSummary] = []
self.flushed_gpu_blocks: set[GPUBlock] = set()
self.kv_connector_stats: list[Any] = []
# block_id -> GPUBlock
self.gpu_blocks: dict[int, GPUBlock] = {}
@@ -348,6 +350,12 @@ class RequestRunner:
slot_mapping={},
)
def _record_kv_connector_stats(self, engine_outputs: dict[int, Any]) -> None:
for output in engine_outputs.values():
scheduler_stats = output.scheduler_stats
if scheduler_stats is not None and scheduler_stats.kv_connector_stats:
self.kv_connector_stats.append(scheduler_stats.kv_connector_stats)
def new_request(
self,
token_ids: list[int],
@@ -524,13 +532,17 @@ class RequestRunner:
if self.async_scheduling:
# in async scheduling we update the output of the previous step
if prev_model_runner_output is not None:
self.scheduler.update_from_output(
engine_outputs = self.scheduler.update_from_output(
prev_scheduler_output, prev_model_runner_output
)
self._record_kv_connector_stats(engine_outputs)
prev_scheduler_output = scheduler_output
prev_model_runner_output = model_runner_output
else:
self.scheduler.update_from_output(scheduler_output, model_runner_output)
engine_outputs = self.scheduler.update_from_output(
scheduler_output, model_runner_output
)
self._record_kv_connector_stats(engine_outputs)
if post_step_fn is not None:
post_step_fn()
@@ -546,9 +558,10 @@ class RequestRunner:
if token_id is None:
if self.async_scheduling:
# sample last token
self.scheduler.update_from_output(
engine_outputs = self.scheduler.update_from_output(
prev_scheduler_output, prev_model_runner_output
)
self._record_kv_connector_stats(engine_outputs)
break
self._parse_transfers()
+58
View File
@@ -224,6 +224,64 @@ def test_cpu_manager_reports_cache_usage_gauge():
check_usage_stats(manager, 0.0)
def test_cpu_manager_reports_allocation_size_histogram():
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
stats = manager.get_stats()
assert stats is not None
reduced = stats.reduce()
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count"] == 2
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_sum"] == 3
# The cache-usage gauge is always reported, so get_stats() never returns
# None, but the histogram has nothing new once its samples are consumed.
second_stats = manager.get_stats()
assert second_stats is not None
assert f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count" not in (
second_stats.reduce()
)
def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
def fail_allocate_blocks(keys):
raise RuntimeError("allocation failed")
monkeypatch.setattr(manager, "_allocate_blocks", fail_allocate_blocks)
with pytest.raises(RuntimeError, match="allocation failed"):
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
stats = manager.get_stats()
assert stats is not None
reduced = stats.reduce()
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count"] == 1
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_sum"] == 3
def test_cpu_manager_reports_allocation_size_on_eviction_failure():
manager = make_cpu_manager(num_blocks=1, cache_policy="lru")
manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
manager.get_stats()
assert manager.prepare_store(to_keys([2]), _EMPTY_REQ_CTX) is None
stats = manager.get_stats()
assert stats is not None
reduced = stats.reduce()
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count"] == 1
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_sum"] == 1
def test_cpu_manager():
"""
Tests CPUOffloadingManager with lru policy.
+29 -3
View File
@@ -21,7 +21,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheGroupSpec,
KVCacheTensor,
)
from vllm.v1.kv_offload.base import OffloadingSpec
from vllm.v1.kv_offload.base import OffloadingHistogramMetadata, OffloadingSpec
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
@@ -248,8 +248,8 @@ def test_duplicate_registration_raises():
# ---------------------------------------------------------------------------
def test_build_metric_definitions_empty_below_threshold():
"""store_threshold < 2 → only base metric (no stores_skipped)."""
def test_build_metric_definitions_below_threshold():
"""store_threshold < 2 keeps stores_skipped disabled."""
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
config = _make_vllm_config(store_threshold=1)
@@ -258,6 +258,32 @@ def test_build_metric_definitions_empty_below_threshold():
config.kv_transfer_config.kv_connector_extra_config
)
assert CPUOffloadingMetrics.STORES_SKIPPED not in metrics
assert CPUOffloadingMetrics.CPU_ALLOCATION_SIZE in metrics
def test_build_metric_definitions_allocation_size_histogram():
"""CPU allocation size is always reported as a histogram."""
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
config = _make_vllm_config(store_threshold=0)
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
metrics = spec_cls.build_metric_definitions(
config.kv_transfer_config.kv_connector_extra_config
)
metadata = metrics[CPUOffloadingMetrics.CPU_ALLOCATION_SIZE]
assert isinstance(metadata, OffloadingHistogramMetadata)
assert metadata.buckets == (
1,
4,
16,
64,
256,
1024,
4096,
16384,
65536,
262144,
)
def test_build_metric_definitions_returns_counter_at_threshold():
@@ -30,6 +30,14 @@ class _TransferMetricName:
STORE_SIZE = "vllm:kv_offload_store_size"
class _ConnectorMetricName:
"""Connector-side metrics emitted by scheduler-side offloading code."""
LOOKUP_SYNC_DELAY = "vllm:kv_offload_lookup_sync_delay_seconds"
LOOKUP_ASYNC_DELAY = "vllm:kv_offload_lookup_async_delay_seconds"
ALLOCATION_FAILURE = "vllm:kv_offload_allocation_failure"
class _TransferType:
"""Transfer direction labels for deprecated CPU offload metrics."""
@@ -74,6 +82,50 @@ def get_connector_metric_definitions() -> dict[str, OffloadingMetricMetadata]:
documentation="Histogram of KV offload store operation size, in bytes.",
buckets=TRANSFER_SIZE_BUCKETS,
),
_ConnectorMetricName.LOOKUP_SYNC_DELAY: OffloadingHistogramMetadata(
documentation=(
"Histogram of the time spent in a single offload lookup call, "
"in seconds."
),
buckets=(
0.00001,
0.00005,
0.0001,
0.0005,
0.001,
0.005,
0.01,
0.05,
0.1,
0.5,
1,
),
),
_ConnectorMetricName.LOOKUP_ASYNC_DELAY: OffloadingHistogramMetadata(
documentation=(
"Histogram of time between a request's offload lookup first "
"deferring and the following lookup resolving, or request "
"finish, in seconds."
),
buckets=(
0.0001,
0.0005,
0.001,
0.005,
0.01,
0.05,
0.1,
0.5,
1,
5,
10,
),
),
_ConnectorMetricName.ALLOCATION_FAILURE: OffloadingCounterMetadata(
documentation=(
"Number of KV offload store allocation attempts that failed."
),
),
}
@@ -223,7 +275,7 @@ class OffloadingConnectorStats(KVConnectorStats):
def increase_counter(
self,
counter_name: str,
counter_increase_value: int | float,
counter_increase_value: int | float = 1,
labelvalues: tuple[str, ...] = (),
) -> None:
"""Increase a counter on the stats payload."""
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from itertools import islice
@@ -21,6 +22,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.events import (
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
_ConnectorMetricName,
_TransferMetricName,
)
from vllm.logger import init_logger
@@ -246,6 +248,9 @@ class RequestOffloadState:
# In-flight job IDs. Per the connector's invariant, at any given time
# this contains either a single load job, or one or more store jobs.
transfer_jobs: set[int] = field(default_factory=set)
# time.monotonic() of this request's first deferred offload lookup;
# None once consumed (observed) or while no lookup is pending.
deferred_lookup_start_time: float | None = None
def __post_init__(self) -> None:
self.group_states = tuple(
@@ -325,7 +330,7 @@ class OffloadingConnectorScheduler:
):
self.config = SchedulerOffloadConfig.from_spec(spec)
self.manager: OffloadingManager = spec.get_manager()
self._connector_stats: OffloadingConnectorStats | None = None
self._connector_stats = OffloadingConnectorStats()
full_attention_groups: list[int] = []
sliding_window_groups: list[int] = []
@@ -375,6 +380,18 @@ class OffloadingConnectorScheduler:
self._events_tracker = OffloadingEventsTracker(spec.kv_events_config)
def _maybe_observe_lookup_async_delay(
self, req_status: RequestOffloadState
) -> None:
start_time = req_status.deferred_lookup_start_time
if start_time is None:
return
req_status.deferred_lookup_start_time = None
self._connector_stats.observe_histogram(
_ConnectorMetricName.LOOKUP_ASYNC_DELAY,
time.monotonic() - start_time,
)
def _generate_job_id(self) -> int:
job_id = self._job_counter
self._job_counter += 1
@@ -685,7 +702,17 @@ class OffloadingConnectorScheduler:
if request.skip_reading_prefix_cache:
num_hit_tokens = 0
else:
lookup_start = time.monotonic()
num_hit_tokens = self._lookup(req_status)
self._connector_stats.observe_histogram(
_ConnectorMetricName.LOOKUP_SYNC_DELAY,
time.monotonic() - lookup_start,
)
if num_hit_tokens is None:
if req_status.deferred_lookup_start_time is None:
req_status.deferred_lookup_start_time = lookup_start
else:
self._maybe_observe_lookup_async_delay(req_status)
req_status.update_num_hit_blocks(num_computed_tokens + (num_hit_tokens or 0))
self._touch(req_status)
@@ -925,6 +952,9 @@ class OffloadingConnectorScheduler:
new_offload_keys, req_status.req_context
)
if store_output is None:
self._connector_stats.increase_counter(
_ConnectorMetricName.ALLOCATION_FAILURE
)
logger.warning("Request %s: cannot store blocks", req_id)
continue
@@ -1115,10 +1145,7 @@ class OffloadingConnectorScheduler:
transfer_stats.observe_histogram(
_TransferMetricName.STORE_SIZE, size
)
if self._connector_stats is None:
self._connector_stats = transfer_stats
else:
self._connector_stats.aggregate(transfer_stats)
self._connector_stats.aggregate(transfer_stats)
for job_id, count in meta.completed_jobs.items():
assert count > 0
@@ -1159,8 +1186,10 @@ class OffloadingConnectorScheduler:
del self._req_status[job_status.req_id]
def get_stats(self) -> OffloadingConnectorStats | None:
stats = self._connector_stats
self._connector_stats = None
stats: OffloadingConnectorStats | None = None
if not self._connector_stats.is_empty():
stats = self._connector_stats
self._connector_stats = OffloadingConnectorStats()
manager_stats = self.manager.get_stats()
if manager_stats is not None:
@@ -1198,7 +1227,7 @@ class OffloadingConnectorScheduler:
return False, None
self.manager.on_request_finished(req_status.req_context)
self._maybe_observe_lookup_async_delay(req_status)
if not req_status.transfer_jobs:
# No in-flight jobs: no later complete_store()/complete_load() calls
# need this request's state.
+1
View File
@@ -8,6 +8,7 @@ from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
class CPUOffloadingMetrics:
STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
CPU_CACHE_USAGE_PERC = "vllm:kv_offload_cpu_cache_usage_perc"
CPU_ALLOCATION_SIZE = "vllm:kv_offload_cpu_allocation_size"
class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
+9
View File
@@ -69,6 +69,7 @@ class CPUOffloadingManager(OffloadingManager):
self.store_threshold: int = store_threshold
self.max_tracker_size: int = max_tracker_size
self.stores_skipped_in_current_batch: int = 0
self.allocation_sizes_in_current_batch: list[int] = []
# Number of block references. It is ordered so can evict the LRU entry in O(1).
self.counts: OrderedDict[OffloadKey, int] | None = (
@@ -185,6 +186,7 @@ class CPUOffloadingManager(OffloadingManager):
evicted_keys=[],
)
self.allocation_sizes_in_current_batch.append(len(keys_to_store))
num_blocks_to_evict = len(keys_to_store) - self._get_num_free_blocks()
to_evict: list[OffloadKey] = []
@@ -300,10 +302,17 @@ class CPUOffloadingManager(OffloadingManager):
usage = num_used / self._num_blocks if self._num_blocks > 0 else 0.0
stats.set_gauge(CPUOffloadingMetrics.CPU_CACHE_USAGE_PERC, usage)
for allocation_size in self.allocation_sizes_in_current_batch:
stats.observe_histogram(
CPUOffloadingMetrics.CPU_ALLOCATION_SIZE, allocation_size
)
self.allocation_sizes_in_current_batch.clear()
if self.store_threshold >= 2:
stats.increase_counter(
CPUOffloadingMetrics.STORES_SKIPPED,
self.stores_skipped_in_current_batch,
)
self.stores_skipped_in_current_batch = 0
return stats
+9 -1
View File
@@ -12,6 +12,7 @@ from vllm.v1.kv_offload.base import (
CanonicalKVCaches,
OffloadingCounterMetadata,
OffloadingGaugeMetadata,
OffloadingHistogramMetadata,
OffloadingManager,
OffloadingMetricMetadata,
OffloadingSpec,
@@ -37,7 +38,14 @@ class CPUOffloadingSpec(OffloadingSpec):
"values indicate transfers (stores or promotions) may be "
"dropped due to insufficient capacity."
),
)
),
CPUOffloadingMetrics.CPU_ALLOCATION_SIZE: OffloadingHistogramMetadata(
documentation=(
"Histogram of the number of CPU blocks requested by each "
"KV offload prepare_store call."
),
buckets=(1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144),
),
}
store_threshold = int(extra_config.get("store_threshold", 0))
if store_threshold >= 2: