[KV Offloading] Add basic offloading metrics (#45958)
Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com> Signed-off-by: Srinivas Krovvidi <194645829+Srinivasoo7@users.noreply.github.com> Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
srinivas_oo7
Or Ozeri
Or Ozeri
parent
dd5c299fbe
commit
e040899a00
@@ -10,9 +10,11 @@ from prometheus_client import Counter, Gauge, Histogram
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
OffloadPromMetrics,
|
||||
_ConnectorMetricName,
|
||||
_MetricType,
|
||||
_StatsKey,
|
||||
_TransferMetricName,
|
||||
get_connector_metric_definitions,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import (
|
||||
OffloadingConnector,
|
||||
@@ -37,6 +39,42 @@ MY_COUNTER = "my_counter"
|
||||
MY_LABEL = "my_label"
|
||||
|
||||
|
||||
def test_connector_metric_histogram_buckets():
|
||||
metadata = get_connector_metric_definitions()
|
||||
|
||||
sync_delay = metadata[_ConnectorMetricName.LOOKUP_SYNC_DELAY]
|
||||
assert isinstance(sync_delay, OffloadingHistogramMetadata)
|
||||
assert sync_delay.buckets == (
|
||||
0.00001,
|
||||
0.00005,
|
||||
0.0001,
|
||||
0.0005,
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.05,
|
||||
0.1,
|
||||
0.5,
|
||||
1,
|
||||
)
|
||||
|
||||
async_delay = metadata[_ConnectorMetricName.LOOKUP_ASYNC_DELAY]
|
||||
assert isinstance(async_delay, OffloadingHistogramMetadata)
|
||||
assert async_delay.buckets == (
|
||||
0.0001,
|
||||
0.0005,
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.05,
|
||||
0.1,
|
||||
0.5,
|
||||
1,
|
||||
5,
|
||||
10,
|
||||
)
|
||||
|
||||
|
||||
class _FakeMetric:
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
|
||||
@@ -11,6 +11,10 @@ from tests.v1.kv_connector.unit.offloading_connector.utils import (
|
||||
to_keys,
|
||||
)
|
||||
from tests.v1.kv_connector.unit.utils import EOS_TOKEN_ID
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
_ConnectorMetricName,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
|
||||
OffloadingConnectorScheduler,
|
||||
RequestOffloadState,
|
||||
@@ -32,6 +36,72 @@ from vllm.v1.kv_offload.base import (
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
|
||||
def _reduce_kv_connector_stats(runner):
|
||||
reduced: dict[str, int | float] = {}
|
||||
for payload in runner.kv_connector_stats:
|
||||
stats = (
|
||||
payload
|
||||
if hasattr(payload, "reduce")
|
||||
else OffloadingConnectorStats(data=payload)
|
||||
)
|
||||
for key, value in stats.reduce().items():
|
||||
reduced[key] = reduced.get(key, 0) + value
|
||||
return reduced
|
||||
|
||||
|
||||
def test_scheduler_reports_allocation_failure(request_runner):
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=False,
|
||||
)
|
||||
runner.new_request(token_ids=[0] * 4)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: None
|
||||
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
|
||||
reduced = _reduce_kv_connector_stats(runner)
|
||||
assert reduced[_ConnectorMetricName.ALLOCATION_FAILURE] == 1
|
||||
|
||||
|
||||
def test_scheduler_reports_lookup_sync_delay(request_runner):
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=False,
|
||||
)
|
||||
runner.new_request(token_ids=[1] * 4)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
|
||||
reduced = _reduce_kv_connector_stats(runner)
|
||||
assert reduced[f"{_ConnectorMetricName.LOOKUP_SYNC_DELAY}_count"] == 1
|
||||
assert reduced[f"{_ConnectorMetricName.LOOKUP_SYNC_DELAY}_sum"] > 0
|
||||
|
||||
|
||||
def test_scheduler_reports_lookup_async_delay_on_resolve(request_runner):
|
||||
"""A deferred lookup reports its async delay once it resolves."""
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=False,
|
||||
)
|
||||
runner.manager.lookup.side_effect = [LookupResult.RETRY, LookupResult.MISS]
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[1] * 4)
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
|
||||
reduced = _reduce_kv_connector_stats(runner)
|
||||
assert reduced[f"{_ConnectorMetricName.LOOKUP_ASYNC_DELAY}_count"] == 1
|
||||
assert reduced[f"{_ConnectorMetricName.LOOKUP_ASYNC_DELAY}_sum"] > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
block_size = 4
|
||||
|
||||
@@ -133,6 +133,7 @@ class MockOffloadingSpec(OffloadingSpec):
|
||||
self.manager = MagicMock(spec=OffloadingManager)
|
||||
self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys)
|
||||
self.manager.lookup.return_value = LookupResult.MISS
|
||||
self.manager.get_stats.return_value = None
|
||||
self.manager.on_new_request.return_value = RequestOffloadingContext()
|
||||
self.handler = MockOffloadingWorker()
|
||||
|
||||
@@ -335,6 +336,7 @@ class RequestRunner:
|
||||
self.completed_loads: list[TransferSummary] = []
|
||||
self.completed_stores: list[TransferSummary] = []
|
||||
self.flushed_gpu_blocks: set[GPUBlock] = set()
|
||||
self.kv_connector_stats: list[Any] = []
|
||||
|
||||
# block_id -> GPUBlock
|
||||
self.gpu_blocks: dict[int, GPUBlock] = {}
|
||||
@@ -348,6 +350,12 @@ class RequestRunner:
|
||||
slot_mapping={},
|
||||
)
|
||||
|
||||
def _record_kv_connector_stats(self, engine_outputs: dict[int, Any]) -> None:
|
||||
for output in engine_outputs.values():
|
||||
scheduler_stats = output.scheduler_stats
|
||||
if scheduler_stats is not None and scheduler_stats.kv_connector_stats:
|
||||
self.kv_connector_stats.append(scheduler_stats.kv_connector_stats)
|
||||
|
||||
def new_request(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
@@ -524,13 +532,17 @@ class RequestRunner:
|
||||
if self.async_scheduling:
|
||||
# in async scheduling we update the output of the previous step
|
||||
if prev_model_runner_output is not None:
|
||||
self.scheduler.update_from_output(
|
||||
engine_outputs = self.scheduler.update_from_output(
|
||||
prev_scheduler_output, prev_model_runner_output
|
||||
)
|
||||
self._record_kv_connector_stats(engine_outputs)
|
||||
prev_scheduler_output = scheduler_output
|
||||
prev_model_runner_output = model_runner_output
|
||||
else:
|
||||
self.scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
engine_outputs = self.scheduler.update_from_output(
|
||||
scheduler_output, model_runner_output
|
||||
)
|
||||
self._record_kv_connector_stats(engine_outputs)
|
||||
|
||||
if post_step_fn is not None:
|
||||
post_step_fn()
|
||||
@@ -546,9 +558,10 @@ class RequestRunner:
|
||||
if token_id is None:
|
||||
if self.async_scheduling:
|
||||
# sample last token
|
||||
self.scheduler.update_from_output(
|
||||
engine_outputs = self.scheduler.update_from_output(
|
||||
prev_scheduler_output, prev_model_runner_output
|
||||
)
|
||||
self._record_kv_connector_stats(engine_outputs)
|
||||
break
|
||||
|
||||
self._parse_transfers()
|
||||
|
||||
@@ -224,6 +224,64 @@ def test_cpu_manager_reports_cache_usage_gauge():
|
||||
check_usage_stats(manager, 0.0)
|
||||
|
||||
|
||||
def test_cpu_manager_reports_allocation_size_histogram():
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
||||
|
||||
stats = manager.get_stats()
|
||||
|
||||
assert stats is not None
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count"] == 2
|
||||
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_sum"] == 3
|
||||
|
||||
# The cache-usage gauge is always reported, so get_stats() never returns
|
||||
# None, but the histogram has nothing new once its samples are consumed.
|
||||
second_stats = manager.get_stats()
|
||||
assert second_stats is not None
|
||||
assert f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count" not in (
|
||||
second_stats.reduce()
|
||||
)
|
||||
|
||||
|
||||
def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
|
||||
def fail_allocate_blocks(keys):
|
||||
raise RuntimeError("allocation failed")
|
||||
|
||||
monkeypatch.setattr(manager, "_allocate_blocks", fail_allocate_blocks)
|
||||
|
||||
with pytest.raises(RuntimeError, match="allocation failed"):
|
||||
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
||||
|
||||
stats = manager.get_stats()
|
||||
|
||||
assert stats is not None
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count"] == 1
|
||||
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_sum"] == 3
|
||||
|
||||
|
||||
def test_cpu_manager_reports_allocation_size_on_eviction_failure():
|
||||
manager = make_cpu_manager(num_blocks=1, cache_policy="lru")
|
||||
|
||||
manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
manager.get_stats()
|
||||
|
||||
assert manager.prepare_store(to_keys([2]), _EMPTY_REQ_CTX) is None
|
||||
|
||||
stats = manager.get_stats()
|
||||
|
||||
assert stats is not None
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count"] == 1
|
||||
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_sum"] == 1
|
||||
|
||||
|
||||
def test_cpu_manager():
|
||||
"""
|
||||
Tests CPUOffloadingManager with lru policy.
|
||||
|
||||
@@ -21,7 +21,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec
|
||||
from vllm.v1.kv_offload.base import OffloadingHistogramMetadata, OffloadingSpec
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
|
||||
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
|
||||
@@ -248,8 +248,8 @@ def test_duplicate_registration_raises():
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_metric_definitions_empty_below_threshold():
|
||||
"""store_threshold < 2 → only base metric (no stores_skipped)."""
|
||||
def test_build_metric_definitions_below_threshold():
|
||||
"""store_threshold < 2 keeps stores_skipped disabled."""
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
|
||||
config = _make_vllm_config(store_threshold=1)
|
||||
@@ -258,6 +258,32 @@ def test_build_metric_definitions_empty_below_threshold():
|
||||
config.kv_transfer_config.kv_connector_extra_config
|
||||
)
|
||||
assert CPUOffloadingMetrics.STORES_SKIPPED not in metrics
|
||||
assert CPUOffloadingMetrics.CPU_ALLOCATION_SIZE in metrics
|
||||
|
||||
|
||||
def test_build_metric_definitions_allocation_size_histogram():
|
||||
"""CPU allocation size is always reported as a histogram."""
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
|
||||
config = _make_vllm_config(store_threshold=0)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
metrics = spec_cls.build_metric_definitions(
|
||||
config.kv_transfer_config.kv_connector_extra_config
|
||||
)
|
||||
metadata = metrics[CPUOffloadingMetrics.CPU_ALLOCATION_SIZE]
|
||||
assert isinstance(metadata, OffloadingHistogramMetadata)
|
||||
assert metadata.buckets == (
|
||||
1,
|
||||
4,
|
||||
16,
|
||||
64,
|
||||
256,
|
||||
1024,
|
||||
4096,
|
||||
16384,
|
||||
65536,
|
||||
262144,
|
||||
)
|
||||
|
||||
|
||||
def test_build_metric_definitions_returns_counter_at_threshold():
|
||||
|
||||
@@ -30,6 +30,14 @@ class _TransferMetricName:
|
||||
STORE_SIZE = "vllm:kv_offload_store_size"
|
||||
|
||||
|
||||
class _ConnectorMetricName:
|
||||
"""Connector-side metrics emitted by scheduler-side offloading code."""
|
||||
|
||||
LOOKUP_SYNC_DELAY = "vllm:kv_offload_lookup_sync_delay_seconds"
|
||||
LOOKUP_ASYNC_DELAY = "vllm:kv_offload_lookup_async_delay_seconds"
|
||||
ALLOCATION_FAILURE = "vllm:kv_offload_allocation_failure"
|
||||
|
||||
|
||||
class _TransferType:
|
||||
"""Transfer direction labels for deprecated CPU offload metrics."""
|
||||
|
||||
@@ -74,6 +82,50 @@ def get_connector_metric_definitions() -> dict[str, OffloadingMetricMetadata]:
|
||||
documentation="Histogram of KV offload store operation size, in bytes.",
|
||||
buckets=TRANSFER_SIZE_BUCKETS,
|
||||
),
|
||||
_ConnectorMetricName.LOOKUP_SYNC_DELAY: OffloadingHistogramMetadata(
|
||||
documentation=(
|
||||
"Histogram of the time spent in a single offload lookup call, "
|
||||
"in seconds."
|
||||
),
|
||||
buckets=(
|
||||
0.00001,
|
||||
0.00005,
|
||||
0.0001,
|
||||
0.0005,
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.05,
|
||||
0.1,
|
||||
0.5,
|
||||
1,
|
||||
),
|
||||
),
|
||||
_ConnectorMetricName.LOOKUP_ASYNC_DELAY: OffloadingHistogramMetadata(
|
||||
documentation=(
|
||||
"Histogram of time between a request's offload lookup first "
|
||||
"deferring and the following lookup resolving, or request "
|
||||
"finish, in seconds."
|
||||
),
|
||||
buckets=(
|
||||
0.0001,
|
||||
0.0005,
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.05,
|
||||
0.1,
|
||||
0.5,
|
||||
1,
|
||||
5,
|
||||
10,
|
||||
),
|
||||
),
|
||||
_ConnectorMetricName.ALLOCATION_FAILURE: OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Number of KV offload store allocation attempts that failed."
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -223,7 +275,7 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
def increase_counter(
|
||||
self,
|
||||
counter_name: str,
|
||||
counter_increase_value: int | float,
|
||||
counter_increase_value: int | float = 1,
|
||||
labelvalues: tuple[str, ...] = (),
|
||||
) -> None:
|
||||
"""Increase a counter on the stats payload."""
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from itertools import islice
|
||||
@@ -21,6 +22,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.events import (
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
_ConnectorMetricName,
|
||||
_TransferMetricName,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
@@ -246,6 +248,9 @@ class RequestOffloadState:
|
||||
# In-flight job IDs. Per the connector's invariant, at any given time
|
||||
# this contains either a single load job, or one or more store jobs.
|
||||
transfer_jobs: set[int] = field(default_factory=set)
|
||||
# time.monotonic() of this request's first deferred offload lookup;
|
||||
# None once consumed (observed) or while no lookup is pending.
|
||||
deferred_lookup_start_time: float | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.group_states = tuple(
|
||||
@@ -325,7 +330,7 @@ class OffloadingConnectorScheduler:
|
||||
):
|
||||
self.config = SchedulerOffloadConfig.from_spec(spec)
|
||||
self.manager: OffloadingManager = spec.get_manager()
|
||||
self._connector_stats: OffloadingConnectorStats | None = None
|
||||
self._connector_stats = OffloadingConnectorStats()
|
||||
|
||||
full_attention_groups: list[int] = []
|
||||
sliding_window_groups: list[int] = []
|
||||
@@ -375,6 +380,18 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
self._events_tracker = OffloadingEventsTracker(spec.kv_events_config)
|
||||
|
||||
def _maybe_observe_lookup_async_delay(
|
||||
self, req_status: RequestOffloadState
|
||||
) -> None:
|
||||
start_time = req_status.deferred_lookup_start_time
|
||||
if start_time is None:
|
||||
return
|
||||
req_status.deferred_lookup_start_time = None
|
||||
self._connector_stats.observe_histogram(
|
||||
_ConnectorMetricName.LOOKUP_ASYNC_DELAY,
|
||||
time.monotonic() - start_time,
|
||||
)
|
||||
|
||||
def _generate_job_id(self) -> int:
|
||||
job_id = self._job_counter
|
||||
self._job_counter += 1
|
||||
@@ -685,7 +702,17 @@ class OffloadingConnectorScheduler:
|
||||
if request.skip_reading_prefix_cache:
|
||||
num_hit_tokens = 0
|
||||
else:
|
||||
lookup_start = time.monotonic()
|
||||
num_hit_tokens = self._lookup(req_status)
|
||||
self._connector_stats.observe_histogram(
|
||||
_ConnectorMetricName.LOOKUP_SYNC_DELAY,
|
||||
time.monotonic() - lookup_start,
|
||||
)
|
||||
if num_hit_tokens is None:
|
||||
if req_status.deferred_lookup_start_time is None:
|
||||
req_status.deferred_lookup_start_time = lookup_start
|
||||
else:
|
||||
self._maybe_observe_lookup_async_delay(req_status)
|
||||
req_status.update_num_hit_blocks(num_computed_tokens + (num_hit_tokens or 0))
|
||||
|
||||
self._touch(req_status)
|
||||
@@ -925,6 +952,9 @@ class OffloadingConnectorScheduler:
|
||||
new_offload_keys, req_status.req_context
|
||||
)
|
||||
if store_output is None:
|
||||
self._connector_stats.increase_counter(
|
||||
_ConnectorMetricName.ALLOCATION_FAILURE
|
||||
)
|
||||
logger.warning("Request %s: cannot store blocks", req_id)
|
||||
continue
|
||||
|
||||
@@ -1115,10 +1145,7 @@ class OffloadingConnectorScheduler:
|
||||
transfer_stats.observe_histogram(
|
||||
_TransferMetricName.STORE_SIZE, size
|
||||
)
|
||||
if self._connector_stats is None:
|
||||
self._connector_stats = transfer_stats
|
||||
else:
|
||||
self._connector_stats.aggregate(transfer_stats)
|
||||
self._connector_stats.aggregate(transfer_stats)
|
||||
|
||||
for job_id, count in meta.completed_jobs.items():
|
||||
assert count > 0
|
||||
@@ -1159,8 +1186,10 @@ class OffloadingConnectorScheduler:
|
||||
del self._req_status[job_status.req_id]
|
||||
|
||||
def get_stats(self) -> OffloadingConnectorStats | None:
|
||||
stats = self._connector_stats
|
||||
self._connector_stats = None
|
||||
stats: OffloadingConnectorStats | None = None
|
||||
if not self._connector_stats.is_empty():
|
||||
stats = self._connector_stats
|
||||
self._connector_stats = OffloadingConnectorStats()
|
||||
|
||||
manager_stats = self.manager.get_stats()
|
||||
if manager_stats is not None:
|
||||
@@ -1198,7 +1227,7 @@ class OffloadingConnectorScheduler:
|
||||
return False, None
|
||||
|
||||
self.manager.on_request_finished(req_status.req_context)
|
||||
|
||||
self._maybe_observe_lookup_async_delay(req_status)
|
||||
if not req_status.transfer_jobs:
|
||||
# No in-flight jobs: no later complete_store()/complete_load() calls
|
||||
# need this request's state.
|
||||
|
||||
@@ -8,6 +8,7 @@ from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
|
||||
class CPUOffloadingMetrics:
|
||||
STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
|
||||
CPU_CACHE_USAGE_PERC = "vllm:kv_offload_cpu_cache_usage_perc"
|
||||
CPU_ALLOCATION_SIZE = "vllm:kv_offload_cpu_allocation_size"
|
||||
|
||||
|
||||
class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
|
||||
|
||||
@@ -69,6 +69,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
self.store_threshold: int = store_threshold
|
||||
self.max_tracker_size: int = max_tracker_size
|
||||
self.stores_skipped_in_current_batch: int = 0
|
||||
self.allocation_sizes_in_current_batch: list[int] = []
|
||||
|
||||
# Number of block references. It is ordered so can evict the LRU entry in O(1).
|
||||
self.counts: OrderedDict[OffloadKey, int] | None = (
|
||||
@@ -185,6 +186,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
evicted_keys=[],
|
||||
)
|
||||
|
||||
self.allocation_sizes_in_current_batch.append(len(keys_to_store))
|
||||
num_blocks_to_evict = len(keys_to_store) - self._get_num_free_blocks()
|
||||
|
||||
to_evict: list[OffloadKey] = []
|
||||
@@ -300,10 +302,17 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
usage = num_used / self._num_blocks if self._num_blocks > 0 else 0.0
|
||||
stats.set_gauge(CPUOffloadingMetrics.CPU_CACHE_USAGE_PERC, usage)
|
||||
|
||||
for allocation_size in self.allocation_sizes_in_current_batch:
|
||||
stats.observe_histogram(
|
||||
CPUOffloadingMetrics.CPU_ALLOCATION_SIZE, allocation_size
|
||||
)
|
||||
self.allocation_sizes_in_current_batch.clear()
|
||||
|
||||
if self.store_threshold >= 2:
|
||||
stats.increase_counter(
|
||||
CPUOffloadingMetrics.STORES_SKIPPED,
|
||||
self.stores_skipped_in_current_batch,
|
||||
)
|
||||
self.stores_skipped_in_current_batch = 0
|
||||
|
||||
return stats
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCaches,
|
||||
OffloadingCounterMetadata,
|
||||
OffloadingGaugeMetadata,
|
||||
OffloadingHistogramMetadata,
|
||||
OffloadingManager,
|
||||
OffloadingMetricMetadata,
|
||||
OffloadingSpec,
|
||||
@@ -37,7 +38,14 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
"values indicate transfers (stores or promotions) may be "
|
||||
"dropped due to insufficient capacity."
|
||||
),
|
||||
)
|
||||
),
|
||||
CPUOffloadingMetrics.CPU_ALLOCATION_SIZE: OffloadingHistogramMetadata(
|
||||
documentation=(
|
||||
"Histogram of the number of CPU blocks requested by each "
|
||||
"KV offload prepare_store call."
|
||||
),
|
||||
buckets=(1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144),
|
||||
),
|
||||
}
|
||||
store_threshold = int(extra_config.get("store_threshold", 0))
|
||||
if store_threshold >= 2:
|
||||
|
||||
Reference in New Issue
Block a user