[KV Offloading] Add basic offloading metrics (#45958)

Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Signed-off-by: Srinivas Krovvidi <194645829+Srinivasoo7@users.noreply.github.com>
Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Srinivas Krovvidi
2026-07-07 09:26:28 +03:00
committed by GitHub
co-authored by srinivas_oo7 Or Ozeri Or Ozeri
parent dd5c299fbe
commit e040899a00
10 changed files with 320 additions and 16 deletions
@@ -30,6 +30,14 @@ class _TransferMetricName:
STORE_SIZE = "vllm:kv_offload_store_size"
class _ConnectorMetricName:
"""Connector-side metrics emitted by scheduler-side offloading code."""
LOOKUP_SYNC_DELAY = "vllm:kv_offload_lookup_sync_delay_seconds"
LOOKUP_ASYNC_DELAY = "vllm:kv_offload_lookup_async_delay_seconds"
ALLOCATION_FAILURE = "vllm:kv_offload_allocation_failure"
class _TransferType:
"""Transfer direction labels for deprecated CPU offload metrics."""
@@ -74,6 +82,50 @@ def get_connector_metric_definitions() -> dict[str, OffloadingMetricMetadata]:
documentation="Histogram of KV offload store operation size, in bytes.",
buckets=TRANSFER_SIZE_BUCKETS,
),
_ConnectorMetricName.LOOKUP_SYNC_DELAY: OffloadingHistogramMetadata(
documentation=(
"Histogram of the time spent in a single offload lookup call, "
"in seconds."
),
buckets=(
0.00001,
0.00005,
0.0001,
0.0005,
0.001,
0.005,
0.01,
0.05,
0.1,
0.5,
1,
),
),
_ConnectorMetricName.LOOKUP_ASYNC_DELAY: OffloadingHistogramMetadata(
documentation=(
"Histogram of time between a request's offload lookup first "
"deferring and the following lookup resolving, or request "
"finish, in seconds."
),
buckets=(
0.0001,
0.0005,
0.001,
0.005,
0.01,
0.05,
0.1,
0.5,
1,
5,
10,
),
),
_ConnectorMetricName.ALLOCATION_FAILURE: OffloadingCounterMetadata(
documentation=(
"Number of KV offload store allocation attempts that failed."
),
),
}
@@ -223,7 +275,7 @@ class OffloadingConnectorStats(KVConnectorStats):
def increase_counter(
self,
counter_name: str,
counter_increase_value: int | float,
counter_increase_value: int | float = 1,
labelvalues: tuple[str, ...] = (),
) -> None:
"""Increase a counter on the stats payload."""
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from itertools import islice
@@ -21,6 +22,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.events import (
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
_ConnectorMetricName,
_TransferMetricName,
)
from vllm.logger import init_logger
@@ -246,6 +248,9 @@ class RequestOffloadState:
# In-flight job IDs. Per the connector's invariant, at any given time
# this contains either a single load job, or one or more store jobs.
transfer_jobs: set[int] = field(default_factory=set)
# time.monotonic() of this request's first deferred offload lookup;
# None once consumed (observed) or while no lookup is pending.
deferred_lookup_start_time: float | None = None
def __post_init__(self) -> None:
self.group_states = tuple(
@@ -325,7 +330,7 @@ class OffloadingConnectorScheduler:
):
self.config = SchedulerOffloadConfig.from_spec(spec)
self.manager: OffloadingManager = spec.get_manager()
self._connector_stats: OffloadingConnectorStats | None = None
self._connector_stats = OffloadingConnectorStats()
full_attention_groups: list[int] = []
sliding_window_groups: list[int] = []
@@ -375,6 +380,18 @@ class OffloadingConnectorScheduler:
self._events_tracker = OffloadingEventsTracker(spec.kv_events_config)
def _maybe_observe_lookup_async_delay(
self, req_status: RequestOffloadState
) -> None:
start_time = req_status.deferred_lookup_start_time
if start_time is None:
return
req_status.deferred_lookup_start_time = None
self._connector_stats.observe_histogram(
_ConnectorMetricName.LOOKUP_ASYNC_DELAY,
time.monotonic() - start_time,
)
def _generate_job_id(self) -> int:
job_id = self._job_counter
self._job_counter += 1
@@ -685,7 +702,17 @@ class OffloadingConnectorScheduler:
if request.skip_reading_prefix_cache:
num_hit_tokens = 0
else:
lookup_start = time.monotonic()
num_hit_tokens = self._lookup(req_status)
self._connector_stats.observe_histogram(
_ConnectorMetricName.LOOKUP_SYNC_DELAY,
time.monotonic() - lookup_start,
)
if num_hit_tokens is None:
if req_status.deferred_lookup_start_time is None:
req_status.deferred_lookup_start_time = lookup_start
else:
self._maybe_observe_lookup_async_delay(req_status)
req_status.update_num_hit_blocks(num_computed_tokens + (num_hit_tokens or 0))
self._touch(req_status)
@@ -925,6 +952,9 @@ class OffloadingConnectorScheduler:
new_offload_keys, req_status.req_context
)
if store_output is None:
self._connector_stats.increase_counter(
_ConnectorMetricName.ALLOCATION_FAILURE
)
logger.warning("Request %s: cannot store blocks", req_id)
continue
@@ -1115,10 +1145,7 @@ class OffloadingConnectorScheduler:
transfer_stats.observe_histogram(
_TransferMetricName.STORE_SIZE, size
)
if self._connector_stats is None:
self._connector_stats = transfer_stats
else:
self._connector_stats.aggregate(transfer_stats)
self._connector_stats.aggregate(transfer_stats)
for job_id, count in meta.completed_jobs.items():
assert count > 0
@@ -1159,8 +1186,10 @@ class OffloadingConnectorScheduler:
del self._req_status[job_status.req_id]
def get_stats(self) -> OffloadingConnectorStats | None:
stats = self._connector_stats
self._connector_stats = None
stats: OffloadingConnectorStats | None = None
if not self._connector_stats.is_empty():
stats = self._connector_stats
self._connector_stats = OffloadingConnectorStats()
manager_stats = self.manager.get_stats()
if manager_stats is not None:
@@ -1198,7 +1227,7 @@ class OffloadingConnectorScheduler:
return False, None
self.manager.on_request_finished(req_status.req_context)
self._maybe_observe_lookup_async_delay(req_status)
if not req_status.transfer_jobs:
# No in-flight jobs: no later complete_store()/complete_load() calls
# need this request's state.