[KV Offloading] Add basic offloading metrics (#45958)
Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com> Signed-off-by: Srinivas Krovvidi <194645829+Srinivasoo7@users.noreply.github.com> Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
srinivas_oo7
Or Ozeri
Or Ozeri
parent
dd5c299fbe
commit
e040899a00
@@ -30,6 +30,14 @@ class _TransferMetricName:
|
||||
STORE_SIZE = "vllm:kv_offload_store_size"
|
||||
|
||||
|
||||
class _ConnectorMetricName:
|
||||
"""Connector-side metrics emitted by scheduler-side offloading code."""
|
||||
|
||||
LOOKUP_SYNC_DELAY = "vllm:kv_offload_lookup_sync_delay_seconds"
|
||||
LOOKUP_ASYNC_DELAY = "vllm:kv_offload_lookup_async_delay_seconds"
|
||||
ALLOCATION_FAILURE = "vllm:kv_offload_allocation_failure"
|
||||
|
||||
|
||||
class _TransferType:
|
||||
"""Transfer direction labels for deprecated CPU offload metrics."""
|
||||
|
||||
@@ -74,6 +82,50 @@ def get_connector_metric_definitions() -> dict[str, OffloadingMetricMetadata]:
|
||||
documentation="Histogram of KV offload store operation size, in bytes.",
|
||||
buckets=TRANSFER_SIZE_BUCKETS,
|
||||
),
|
||||
_ConnectorMetricName.LOOKUP_SYNC_DELAY: OffloadingHistogramMetadata(
|
||||
documentation=(
|
||||
"Histogram of the time spent in a single offload lookup call, "
|
||||
"in seconds."
|
||||
),
|
||||
buckets=(
|
||||
0.00001,
|
||||
0.00005,
|
||||
0.0001,
|
||||
0.0005,
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.05,
|
||||
0.1,
|
||||
0.5,
|
||||
1,
|
||||
),
|
||||
),
|
||||
_ConnectorMetricName.LOOKUP_ASYNC_DELAY: OffloadingHistogramMetadata(
|
||||
documentation=(
|
||||
"Histogram of time between a request's offload lookup first "
|
||||
"deferring and the following lookup resolving, or request "
|
||||
"finish, in seconds."
|
||||
),
|
||||
buckets=(
|
||||
0.0001,
|
||||
0.0005,
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.05,
|
||||
0.1,
|
||||
0.5,
|
||||
1,
|
||||
5,
|
||||
10,
|
||||
),
|
||||
),
|
||||
_ConnectorMetricName.ALLOCATION_FAILURE: OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Number of KV offload store allocation attempts that failed."
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -223,7 +275,7 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
def increase_counter(
|
||||
self,
|
||||
counter_name: str,
|
||||
counter_increase_value: int | float,
|
||||
counter_increase_value: int | float = 1,
|
||||
labelvalues: tuple[str, ...] = (),
|
||||
) -> None:
|
||||
"""Increase a counter on the stats payload."""
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from itertools import islice
|
||||
@@ -21,6 +22,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.events import (
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
_ConnectorMetricName,
|
||||
_TransferMetricName,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
@@ -246,6 +248,9 @@ class RequestOffloadState:
|
||||
# In-flight job IDs. Per the connector's invariant, at any given time
|
||||
# this contains either a single load job, or one or more store jobs.
|
||||
transfer_jobs: set[int] = field(default_factory=set)
|
||||
# time.monotonic() of this request's first deferred offload lookup;
|
||||
# None once consumed (observed) or while no lookup is pending.
|
||||
deferred_lookup_start_time: float | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.group_states = tuple(
|
||||
@@ -325,7 +330,7 @@ class OffloadingConnectorScheduler:
|
||||
):
|
||||
self.config = SchedulerOffloadConfig.from_spec(spec)
|
||||
self.manager: OffloadingManager = spec.get_manager()
|
||||
self._connector_stats: OffloadingConnectorStats | None = None
|
||||
self._connector_stats = OffloadingConnectorStats()
|
||||
|
||||
full_attention_groups: list[int] = []
|
||||
sliding_window_groups: list[int] = []
|
||||
@@ -375,6 +380,18 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
self._events_tracker = OffloadingEventsTracker(spec.kv_events_config)
|
||||
|
||||
def _maybe_observe_lookup_async_delay(
|
||||
self, req_status: RequestOffloadState
|
||||
) -> None:
|
||||
start_time = req_status.deferred_lookup_start_time
|
||||
if start_time is None:
|
||||
return
|
||||
req_status.deferred_lookup_start_time = None
|
||||
self._connector_stats.observe_histogram(
|
||||
_ConnectorMetricName.LOOKUP_ASYNC_DELAY,
|
||||
time.monotonic() - start_time,
|
||||
)
|
||||
|
||||
def _generate_job_id(self) -> int:
|
||||
job_id = self._job_counter
|
||||
self._job_counter += 1
|
||||
@@ -685,7 +702,17 @@ class OffloadingConnectorScheduler:
|
||||
if request.skip_reading_prefix_cache:
|
||||
num_hit_tokens = 0
|
||||
else:
|
||||
lookup_start = time.monotonic()
|
||||
num_hit_tokens = self._lookup(req_status)
|
||||
self._connector_stats.observe_histogram(
|
||||
_ConnectorMetricName.LOOKUP_SYNC_DELAY,
|
||||
time.monotonic() - lookup_start,
|
||||
)
|
||||
if num_hit_tokens is None:
|
||||
if req_status.deferred_lookup_start_time is None:
|
||||
req_status.deferred_lookup_start_time = lookup_start
|
||||
else:
|
||||
self._maybe_observe_lookup_async_delay(req_status)
|
||||
req_status.update_num_hit_blocks(num_computed_tokens + (num_hit_tokens or 0))
|
||||
|
||||
self._touch(req_status)
|
||||
@@ -925,6 +952,9 @@ class OffloadingConnectorScheduler:
|
||||
new_offload_keys, req_status.req_context
|
||||
)
|
||||
if store_output is None:
|
||||
self._connector_stats.increase_counter(
|
||||
_ConnectorMetricName.ALLOCATION_FAILURE
|
||||
)
|
||||
logger.warning("Request %s: cannot store blocks", req_id)
|
||||
continue
|
||||
|
||||
@@ -1115,10 +1145,7 @@ class OffloadingConnectorScheduler:
|
||||
transfer_stats.observe_histogram(
|
||||
_TransferMetricName.STORE_SIZE, size
|
||||
)
|
||||
if self._connector_stats is None:
|
||||
self._connector_stats = transfer_stats
|
||||
else:
|
||||
self._connector_stats.aggregate(transfer_stats)
|
||||
self._connector_stats.aggregate(transfer_stats)
|
||||
|
||||
for job_id, count in meta.completed_jobs.items():
|
||||
assert count > 0
|
||||
@@ -1159,8 +1186,10 @@ class OffloadingConnectorScheduler:
|
||||
del self._req_status[job_status.req_id]
|
||||
|
||||
def get_stats(self) -> OffloadingConnectorStats | None:
|
||||
stats = self._connector_stats
|
||||
self._connector_stats = None
|
||||
stats: OffloadingConnectorStats | None = None
|
||||
if not self._connector_stats.is_empty():
|
||||
stats = self._connector_stats
|
||||
self._connector_stats = OffloadingConnectorStats()
|
||||
|
||||
manager_stats = self.manager.get_stats()
|
||||
if manager_stats is not None:
|
||||
@@ -1198,7 +1227,7 @@ class OffloadingConnectorScheduler:
|
||||
return False, None
|
||||
|
||||
self.manager.on_request_finished(req_status.req_context)
|
||||
|
||||
self._maybe_observe_lookup_async_delay(req_status)
|
||||
if not req_status.transfer_jobs:
|
||||
# No in-flight jobs: no later complete_store()/complete_load() calls
|
||||
# need this request's state.
|
||||
|
||||
Reference in New Issue
Block a user