[KV Offloading] Add basic offloading metrics (#45958)
Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com> Signed-off-by: Srinivas Krovvidi <194645829+Srinivasoo7@users.noreply.github.com> Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
srinivas_oo7
Or Ozeri
Or Ozeri
parent
dd5c299fbe
commit
e040899a00
@@ -10,9 +10,11 @@ from prometheus_client import Counter, Gauge, Histogram
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
OffloadPromMetrics,
|
||||
_ConnectorMetricName,
|
||||
_MetricType,
|
||||
_StatsKey,
|
||||
_TransferMetricName,
|
||||
get_connector_metric_definitions,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import (
|
||||
OffloadingConnector,
|
||||
@@ -37,6 +39,42 @@ MY_COUNTER = "my_counter"
|
||||
MY_LABEL = "my_label"
|
||||
|
||||
|
||||
def test_connector_metric_histogram_buckets():
|
||||
metadata = get_connector_metric_definitions()
|
||||
|
||||
sync_delay = metadata[_ConnectorMetricName.LOOKUP_SYNC_DELAY]
|
||||
assert isinstance(sync_delay, OffloadingHistogramMetadata)
|
||||
assert sync_delay.buckets == (
|
||||
0.00001,
|
||||
0.00005,
|
||||
0.0001,
|
||||
0.0005,
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.05,
|
||||
0.1,
|
||||
0.5,
|
||||
1,
|
||||
)
|
||||
|
||||
async_delay = metadata[_ConnectorMetricName.LOOKUP_ASYNC_DELAY]
|
||||
assert isinstance(async_delay, OffloadingHistogramMetadata)
|
||||
assert async_delay.buckets == (
|
||||
0.0001,
|
||||
0.0005,
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.05,
|
||||
0.1,
|
||||
0.5,
|
||||
1,
|
||||
5,
|
||||
10,
|
||||
)
|
||||
|
||||
|
||||
class _FakeMetric:
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
|
||||
@@ -11,6 +11,10 @@ from tests.v1.kv_connector.unit.offloading_connector.utils import (
|
||||
to_keys,
|
||||
)
|
||||
from tests.v1.kv_connector.unit.utils import EOS_TOKEN_ID
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
_ConnectorMetricName,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
|
||||
OffloadingConnectorScheduler,
|
||||
RequestOffloadState,
|
||||
@@ -32,6 +36,72 @@ from vllm.v1.kv_offload.base import (
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
|
||||
def _reduce_kv_connector_stats(runner):
|
||||
reduced: dict[str, int | float] = {}
|
||||
for payload in runner.kv_connector_stats:
|
||||
stats = (
|
||||
payload
|
||||
if hasattr(payload, "reduce")
|
||||
else OffloadingConnectorStats(data=payload)
|
||||
)
|
||||
for key, value in stats.reduce().items():
|
||||
reduced[key] = reduced.get(key, 0) + value
|
||||
return reduced
|
||||
|
||||
|
||||
def test_scheduler_reports_allocation_failure(request_runner):
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=False,
|
||||
)
|
||||
runner.new_request(token_ids=[0] * 4)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: None
|
||||
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
|
||||
reduced = _reduce_kv_connector_stats(runner)
|
||||
assert reduced[_ConnectorMetricName.ALLOCATION_FAILURE] == 1
|
||||
|
||||
|
||||
def test_scheduler_reports_lookup_sync_delay(request_runner):
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=False,
|
||||
)
|
||||
runner.new_request(token_ids=[1] * 4)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
|
||||
reduced = _reduce_kv_connector_stats(runner)
|
||||
assert reduced[f"{_ConnectorMetricName.LOOKUP_SYNC_DELAY}_count"] == 1
|
||||
assert reduced[f"{_ConnectorMetricName.LOOKUP_SYNC_DELAY}_sum"] > 0
|
||||
|
||||
|
||||
def test_scheduler_reports_lookup_async_delay_on_resolve(request_runner):
|
||||
"""A deferred lookup reports its async delay once it resolves."""
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=False,
|
||||
)
|
||||
runner.manager.lookup.side_effect = [LookupResult.RETRY, LookupResult.MISS]
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[1] * 4)
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
|
||||
reduced = _reduce_kv_connector_stats(runner)
|
||||
assert reduced[f"{_ConnectorMetricName.LOOKUP_ASYNC_DELAY}_count"] == 1
|
||||
assert reduced[f"{_ConnectorMetricName.LOOKUP_ASYNC_DELAY}_sum"] > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
block_size = 4
|
||||
|
||||
@@ -133,6 +133,7 @@ class MockOffloadingSpec(OffloadingSpec):
|
||||
self.manager = MagicMock(spec=OffloadingManager)
|
||||
self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys)
|
||||
self.manager.lookup.return_value = LookupResult.MISS
|
||||
self.manager.get_stats.return_value = None
|
||||
self.manager.on_new_request.return_value = RequestOffloadingContext()
|
||||
self.handler = MockOffloadingWorker()
|
||||
|
||||
@@ -335,6 +336,7 @@ class RequestRunner:
|
||||
self.completed_loads: list[TransferSummary] = []
|
||||
self.completed_stores: list[TransferSummary] = []
|
||||
self.flushed_gpu_blocks: set[GPUBlock] = set()
|
||||
self.kv_connector_stats: list[Any] = []
|
||||
|
||||
# block_id -> GPUBlock
|
||||
self.gpu_blocks: dict[int, GPUBlock] = {}
|
||||
@@ -348,6 +350,12 @@ class RequestRunner:
|
||||
slot_mapping={},
|
||||
)
|
||||
|
||||
def _record_kv_connector_stats(self, engine_outputs: dict[int, Any]) -> None:
|
||||
for output in engine_outputs.values():
|
||||
scheduler_stats = output.scheduler_stats
|
||||
if scheduler_stats is not None and scheduler_stats.kv_connector_stats:
|
||||
self.kv_connector_stats.append(scheduler_stats.kv_connector_stats)
|
||||
|
||||
def new_request(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
@@ -524,13 +532,17 @@ class RequestRunner:
|
||||
if self.async_scheduling:
|
||||
# in async scheduling we update the output of the previous step
|
||||
if prev_model_runner_output is not None:
|
||||
self.scheduler.update_from_output(
|
||||
engine_outputs = self.scheduler.update_from_output(
|
||||
prev_scheduler_output, prev_model_runner_output
|
||||
)
|
||||
self._record_kv_connector_stats(engine_outputs)
|
||||
prev_scheduler_output = scheduler_output
|
||||
prev_model_runner_output = model_runner_output
|
||||
else:
|
||||
self.scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
engine_outputs = self.scheduler.update_from_output(
|
||||
scheduler_output, model_runner_output
|
||||
)
|
||||
self._record_kv_connector_stats(engine_outputs)
|
||||
|
||||
if post_step_fn is not None:
|
||||
post_step_fn()
|
||||
@@ -546,9 +558,10 @@ class RequestRunner:
|
||||
if token_id is None:
|
||||
if self.async_scheduling:
|
||||
# sample last token
|
||||
self.scheduler.update_from_output(
|
||||
engine_outputs = self.scheduler.update_from_output(
|
||||
prev_scheduler_output, prev_model_runner_output
|
||||
)
|
||||
self._record_kv_connector_stats(engine_outputs)
|
||||
break
|
||||
|
||||
self._parse_transfers()
|
||||
|
||||
@@ -224,6 +224,64 @@ def test_cpu_manager_reports_cache_usage_gauge():
|
||||
check_usage_stats(manager, 0.0)
|
||||
|
||||
|
||||
def test_cpu_manager_reports_allocation_size_histogram():
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
||||
|
||||
stats = manager.get_stats()
|
||||
|
||||
assert stats is not None
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count"] == 2
|
||||
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_sum"] == 3
|
||||
|
||||
# The cache-usage gauge is always reported, so get_stats() never returns
|
||||
# None, but the histogram has nothing new once its samples are consumed.
|
||||
second_stats = manager.get_stats()
|
||||
assert second_stats is not None
|
||||
assert f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count" not in (
|
||||
second_stats.reduce()
|
||||
)
|
||||
|
||||
|
||||
def test_cpu_manager_reports_allocation_size_on_allocation_failure(monkeypatch):
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
|
||||
def fail_allocate_blocks(keys):
|
||||
raise RuntimeError("allocation failed")
|
||||
|
||||
monkeypatch.setattr(manager, "_allocate_blocks", fail_allocate_blocks)
|
||||
|
||||
with pytest.raises(RuntimeError, match="allocation failed"):
|
||||
manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
||||
|
||||
stats = manager.get_stats()
|
||||
|
||||
assert stats is not None
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count"] == 1
|
||||
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_sum"] == 3
|
||||
|
||||
|
||||
def test_cpu_manager_reports_allocation_size_on_eviction_failure():
|
||||
manager = make_cpu_manager(num_blocks=1, cache_policy="lru")
|
||||
|
||||
manager.prepare_store(to_keys([1]), _EMPTY_REQ_CTX)
|
||||
manager.get_stats()
|
||||
|
||||
assert manager.prepare_store(to_keys([2]), _EMPTY_REQ_CTX) is None
|
||||
|
||||
stats = manager.get_stats()
|
||||
|
||||
assert stats is not None
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_count"] == 1
|
||||
assert reduced[f"{CPUOffloadingMetrics.CPU_ALLOCATION_SIZE}_sum"] == 1
|
||||
|
||||
|
||||
def test_cpu_manager():
|
||||
"""
|
||||
Tests CPUOffloadingManager with lru policy.
|
||||
|
||||
@@ -21,7 +21,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec
|
||||
from vllm.v1.kv_offload.base import OffloadingHistogramMetadata, OffloadingSpec
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
|
||||
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
|
||||
@@ -248,8 +248,8 @@ def test_duplicate_registration_raises():
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_metric_definitions_empty_below_threshold():
|
||||
"""store_threshold < 2 → only base metric (no stores_skipped)."""
|
||||
def test_build_metric_definitions_below_threshold():
|
||||
"""store_threshold < 2 keeps stores_skipped disabled."""
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
|
||||
config = _make_vllm_config(store_threshold=1)
|
||||
@@ -258,6 +258,32 @@ def test_build_metric_definitions_empty_below_threshold():
|
||||
config.kv_transfer_config.kv_connector_extra_config
|
||||
)
|
||||
assert CPUOffloadingMetrics.STORES_SKIPPED not in metrics
|
||||
assert CPUOffloadingMetrics.CPU_ALLOCATION_SIZE in metrics
|
||||
|
||||
|
||||
def test_build_metric_definitions_allocation_size_histogram():
|
||||
"""CPU allocation size is always reported as a histogram."""
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
|
||||
config = _make_vllm_config(store_threshold=0)
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(config)
|
||||
metrics = spec_cls.build_metric_definitions(
|
||||
config.kv_transfer_config.kv_connector_extra_config
|
||||
)
|
||||
metadata = metrics[CPUOffloadingMetrics.CPU_ALLOCATION_SIZE]
|
||||
assert isinstance(metadata, OffloadingHistogramMetadata)
|
||||
assert metadata.buckets == (
|
||||
1,
|
||||
4,
|
||||
16,
|
||||
64,
|
||||
256,
|
||||
1024,
|
||||
4096,
|
||||
16384,
|
||||
65536,
|
||||
262144,
|
||||
)
|
||||
|
||||
|
||||
def test_build_metric_definitions_returns_counter_at_threshold():
|
||||
|
||||
Reference in New Issue
Block a user