diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_metrics.py b/tests/v1/kv_connector/unit/offloading_connector/test_metrics.py index 88ccb0aeb6..f9a4b37795 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_metrics.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_metrics.py @@ -1,11 +1,102 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace +from typing import Any +from unittest.mock import patch + +import pytest +from prometheus_client import Counter, Gauge, Histogram + from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( OffloadingConnectorStats, + OffloadPromMetrics, + _MetricType, + _StatsKey, + _TransferMetricName, ) from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import ( OffloadingConnector, ) +from vllm.v1.kv_offload.base import ( + OffloadingCounterMetadata, + OffloadingGaugeMetadata, + OffloadingHistogramMetadata, +) +from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec + +LOAD_BYTES = _TransferMetricName.LOAD_BYTES +LOAD_TIME = _TransferMetricName.LOAD_TIME +LOAD_SIZE = _TransferMetricName.LOAD_SIZE +STORE_BYTES = _TransferMetricName.STORE_BYTES +STORE_TIME = _TransferMetricName.STORE_TIME +STORE_SIZE = _TransferMetricName.STORE_SIZE +STORES_SKIPPED = "vllm:kv_offload_stores_skipped" +PENDING_STORES = "vllm:kv_offload_pending_stores" +LOOKUP_LATENCY = "vllm:kv_offload_lookup_latency_seconds" + + +class _FakeMetric: + def __init__(self, **kwargs): + self.kwargs = kwargs + self.children: list[_FakeMetric] = [] + self.observed: list[int | float] = [] + self.increments: list[int | float] = [] + self.set_values: list[int | float] = [] + self.labelvalues: tuple[object, ...] = () + + def labels(self, *labelvalues): + child = _FakeMetric(**self.kwargs) + child.labelvalues = labelvalues + self.children.append(child) + return child + + def observe(self, value): + self.observed.append(value) + + def inc(self, value): + self.increments.append(value) + + def set(self, value): + self.set_values.append(value) + + +class _FakeVllmConfig: + def __init__(self, store_threshold: int = 2): + self.kv_transfer_config = SimpleNamespace( + kv_connector_extra_config={"store_threshold": store_threshold} + ) + + +def _metric_metadata(): + return { + LOAD_BYTES: OffloadingCounterMetadata( + documentation="load bytes", + ), + LOAD_TIME: OffloadingCounterMetadata( + documentation="load time", + ), + LOAD_SIZE: OffloadingHistogramMetadata( + documentation="load size", + ), + STORE_BYTES: OffloadingCounterMetadata( + documentation="store bytes", + ), + STORE_TIME: OffloadingCounterMetadata( + documentation="store time", + ), + STORE_SIZE: OffloadingHistogramMetadata( + documentation="store size", + ), + STORES_SKIPPED: OffloadingCounterMetadata( + documentation="stores skipped", + ), + PENDING_STORES: OffloadingGaugeMetadata( + documentation="pending stores", + ), + LOOKUP_LATENCY: OffloadingHistogramMetadata( + documentation="lookup latency", + ), + } def test_build_kv_connector_stats_with_none(): @@ -14,7 +105,6 @@ def test_build_kv_connector_stats_with_none(): assert stats is not None assert isinstance(stats, OffloadingConnectorStats) - assert len(stats.data) == 0 assert stats.is_empty() @@ -24,7 +114,6 @@ def test_build_kv_connector_stats_with_empty_dict(): assert stats is not None assert isinstance(stats, OffloadingConnectorStats) - assert len(stats.data) == 0 assert stats.is_empty() @@ -32,114 +121,186 @@ def test_build_kv_connector_stats_reconstructs_offload_stats(): """Test that OffloadingConnector stats are properly reconstructed with correct data.""" serialized_data = { - "CPU_to_GPU": [ - {"op_size": 16, "op_time": 1.0}, - {"op_size": 8, "op_time": 0.5}, - ], - "GPU_to_CPU": [ - {"op_size": 1, "op_time": 0.1}, - {"op_size": 2, "op_time": 0.2}, - ], + _StatsKey.TYPES: { + LOAD_BYTES: _MetricType.COUNTER, + LOAD_TIME: _MetricType.COUNTER, + LOAD_SIZE: _MetricType.HISTOGRAM, + STORE_BYTES: _MetricType.COUNTER, + STORE_TIME: _MetricType.COUNTER, + STORE_SIZE: _MetricType.HISTOGRAM, + STORES_SKIPPED: _MetricType.COUNTER, + }, + _StatsKey.DATA: { + LOAD_BYTES: 24, + LOAD_TIME: 1.5, + LOAD_SIZE: [16, 8], + STORE_BYTES: 3, + STORE_TIME: 0.3, + STORE_SIZE: [1, 2], + STORES_SKIPPED: 5, + }, } stats = OffloadingConnector.build_kv_connector_stats(data=serialized_data) - offload_connector_stats = stats - assert isinstance(offload_connector_stats, OffloadingConnectorStats) - assert offload_connector_stats.data["CPU_to_GPU"] == [ - {"op_size": 16, "op_time": 1.0}, - {"op_size": 8, "op_time": 0.5}, - ] - assert offload_connector_stats.data["GPU_to_CPU"] == [ - {"op_size": 1, "op_time": 0.1}, - {"op_size": 2, "op_time": 0.2}, - ] + assert isinstance(stats, OffloadingConnectorStats) + values = stats.data[_StatsKey.DATA] + assert values[LOAD_BYTES] == 24 + assert values[LOAD_TIME] == 1.5 + assert values[LOAD_SIZE] == [16, 8] + assert values[STORE_BYTES] == 3 + assert values[STORE_TIME] == 0.3 + assert values[STORE_SIZE] == [1, 2] + assert values[STORES_SKIPPED] == 5 + + +def _make_stats_data( + metric_data: dict[str, Any], + metric_metadata: dict[str, Any], +) -> dict[str, Any]: + """Build a structured data dict from flat metric data and metadata.""" + metric_types = {} + for key in metric_data: + md = metric_metadata[key] + if isinstance(md, OffloadingCounterMetadata): + metric_types[key] = _MetricType.COUNTER + elif isinstance(md, OffloadingGaugeMetadata): + metric_types[key] = _MetricType.GAUGE + elif isinstance(md, OffloadingHistogramMetadata): + metric_types[key] = _MetricType.HISTOGRAM + return { + _StatsKey.TYPES: metric_types, + _StatsKey.DATA: metric_data, + } def test_aggregate_same_connector(): """Test aggregating stats from the same connector type.""" + metadata = _metric_metadata() stats1 = OffloadingConnectorStats( - data={ - "CPU_to_GPU": [ - {"op_size": 16, "op_time": 1.0}, - {"op_size": 8, "op_time": 0.5}, - ], - "GPU_to_CPU": [ - {"op_size": 1, "op_time": 0.1}, - {"op_size": 2, "op_time": 0.2}, - ], - } + data=_make_stats_data( + { + LOAD_BYTES: 24, + LOAD_TIME: 1.5, + LOAD_SIZE: [16, 8], + STORE_BYTES: 3, + STORE_TIME: 0.3, + STORE_SIZE: [1, 2], + STORES_SKIPPED: 1, + PENDING_STORES: 3, + LOOKUP_LATENCY: [0.1], + }, + metadata, + ), ) stats2 = OffloadingConnectorStats( - data={ - "CPU_to_GPU": [ - {"op_size": 3, "op_time": 0.2}, - {"op_size": 7, "op_time": 0.9}, - ], - "GPU_to_CPU": [{"op_size": 16, "op_time": 2}], - } + data=_make_stats_data( + { + LOAD_BYTES: 10, + LOAD_TIME: 1.1, + LOAD_SIZE: [3, 7], + STORE_BYTES: 16, + STORE_TIME: 2, + STORE_SIZE: [16], + STORES_SKIPPED: 3, + PENDING_STORES: 1, + LOOKUP_LATENCY: [0.2, 0.3], + }, + metadata, + ), ) result = stats1.aggregate(stats2) assert result is stats1 # Should return self - offload_connector_stats = result - assert offload_connector_stats.data["CPU_to_GPU"] == [ - {"op_size": 16, "op_time": 1.0}, - {"op_size": 8, "op_time": 0.5}, - {"op_size": 3, "op_time": 0.2}, - {"op_size": 7, "op_time": 0.9}, - ] - assert offload_connector_stats.data["GPU_to_CPU"] == [ - {"op_size": 1, "op_time": 0.1}, - {"op_size": 2, "op_time": 0.2}, - {"op_size": 16, "op_time": 2}, - ] + values = result.data[_StatsKey.DATA] + assert values[LOAD_BYTES] == 34 + assert values[LOAD_TIME] == 2.6 + assert values[LOAD_SIZE] == [16, 8, 3, 7] + assert values[STORE_BYTES] == 19 + assert values[STORE_TIME] == 2.3 + assert values[STORE_SIZE] == [1, 2, 16] + assert values[STORES_SKIPPED] == 4 + assert values[PENDING_STORES] == 1 + assert values[LOOKUP_LATENCY] == [0.1, 0.2, 0.3] + + +def test_aggregate_merges_types(): + stats1 = OffloadingConnectorStats( + data={ + _StatsKey.TYPES: {LOAD_BYTES: _MetricType.COUNTER}, + _StatsKey.DATA: {LOAD_BYTES: 1}, + }, + ) + stats2 = OffloadingConnectorStats( + data={ + _StatsKey.TYPES: {PENDING_STORES: _MetricType.GAUGE}, + _StatsKey.DATA: {PENDING_STORES: 2}, + }, + ) + + result = stats1.aggregate(stats2) + + assert result.data[_StatsKey.DATA][PENDING_STORES] == 2 + assert result.data[_StatsKey.TYPES][PENDING_STORES] == _MetricType.GAUGE def test_reduce(): - """Test that reduce() correctly reduces all nested connector stats.""" + """Test that reduce() correctly reduces connector stats.""" + metadata = _metric_metadata() stats = OffloadingConnectorStats( - data={ - "CPU_to_GPU": [ - {"op_size": 16, "op_time": 1.0}, - {"op_size": 8, "op_time": 0.5}, - {"op_size": 3, "op_time": 0.2}, - {"op_size": 7, "op_time": 0.9}, - ], - "GPU_to_CPU": [ - {"op_size": 1, "op_time": 0.1}, - {"op_size": 2, "op_time": 0.2}, - {"op_size": 16, "op_time": 2}, - ], - } + data=_make_stats_data( + { + LOAD_BYTES: 34, + LOAD_TIME: 2.6, + LOAD_SIZE: [16, 8, 3, 7], + STORE_BYTES: 19, + STORE_TIME: 2.3, + STORE_SIZE: [1, 2, 16], + STORES_SKIPPED: 11, + PENDING_STORES: 2, + LOOKUP_LATENCY: [0.1, 0.2, 0.3], + }, + metadata, + ), ) reduced = stats.reduce() assert isinstance(reduced, dict) - # Check that the stats were reduced (should have aggregated values) - assert "CPU_to_GPU_total_bytes" in reduced - assert "CPU_to_GPU_total_time" in reduced - assert "GPU_to_CPU_total_bytes" in reduced - assert "GPU_to_CPU_total_time" in reduced - assert reduced["CPU_to_GPU_total_bytes"] == 34 - assert reduced["CPU_to_GPU_total_time"] == 2.6 - assert reduced["GPU_to_CPU_total_time"] == 2.3 - assert reduced["GPU_to_CPU_total_bytes"] == 19 + assert reduced[LOAD_BYTES] == 34 + assert reduced[LOAD_TIME] == 2.6 + assert reduced[f"{LOAD_SIZE}_count"] == 4 + assert reduced[f"{LOAD_SIZE}_sum"] == 34 + assert reduced[STORE_BYTES] == 19 + assert reduced[STORE_TIME] == 2.3 + assert reduced[f"{STORE_SIZE}_count"] == 3 + assert reduced[f"{STORE_SIZE}_sum"] == 19 + assert reduced[STORES_SKIPPED] == 11 + assert reduced[PENDING_STORES] == 2 + assert reduced[f"{LOOKUP_LATENCY}_count"] == 3 + assert reduced[f"{LOOKUP_LATENCY}_sum"] == sum([0.1, 0.2, 0.3]) def test_reset(): - """Test that reset() resets all nested connector stats.""" + """Test that reset() resets all connector stats.""" + metadata = _metric_metadata() offload_connector_stats = OffloadingConnectorStats( - data={ - "CPU_to_GPU": [ - {"op_size": 3, "op_time": 0.2}, - {"op_size": 7, "op_time": 0.9}, - ], - "GPU_to_CPU": [{"op_size": 16, "op_time": 2}], - } + data=_make_stats_data( + { + LOAD_BYTES: 10, + LOAD_TIME: 1.1, + LOAD_SIZE: [3, 7], + STORE_BYTES: 16, + STORE_TIME: 2, + STORE_SIZE: [16], + STORES_SKIPPED: 4, + PENDING_STORES: 2, + LOOKUP_LATENCY: [0.1], + }, + metadata, + ), ) assert not offload_connector_stats.is_empty() @@ -148,4 +309,215 @@ def test_reset(): # After reset, stats should be empty assert offload_connector_stats.is_empty() - assert len(offload_connector_stats.data) == 0 + + +def test_prom_metrics_observes_manager_counter(): + prom_metrics = OffloadPromMetrics( + vllm_config=_FakeVllmConfig(), # type: ignore[arg-type] + metric_types={ + Gauge: _FakeMetric, + Counter: _FakeMetric, + Histogram: _FakeMetric, + }, + labelnames=["model_name", "engine"], + per_engine_labelvalues={0: ["model", "0"]}, + ) + + prom_metrics.observe( + { + _StatsKey.TYPES: {STORES_SKIPPED: _MetricType.COUNTER}, + _StatsKey.DATA: {STORES_SKIPPED: 7}, + } + ) + + counter = prom_metrics.offloading_metrics[(0, STORES_SKIPPED)] + assert counter.increments == [7] + counter_def = prom_metrics._offloading_metric_defs[STORES_SKIPPED] + assert counter_def.kwargs["name"] == "vllm:kv_offload_stores_skipped" + assert counter.labelvalues == ("model", "0") + + +def test_prom_metrics_observes_flat_transfer_metrics_and_legacy_metrics(): + prom_metrics = OffloadPromMetrics( + vllm_config=_FakeVllmConfig(), # type: ignore[arg-type] + metric_types={ + Gauge: _FakeMetric, + Counter: _FakeMetric, + Histogram: _FakeMetric, + }, + labelnames=["model_name", "engine"], + per_engine_labelvalues={0: ["model", "0"]}, + ) + + prom_metrics.observe( + { + _StatsKey.TYPES: { + LOAD_BYTES: _MetricType.COUNTER, + LOAD_TIME: _MetricType.COUNTER, + LOAD_SIZE: _MetricType.HISTOGRAM, + STORE_BYTES: _MetricType.COUNTER, + STORE_TIME: _MetricType.COUNTER, + STORE_SIZE: _MetricType.HISTOGRAM, + }, + _StatsKey.DATA: { + LOAD_BYTES: 24, + LOAD_TIME: 1.5, + LOAD_SIZE: [16, 8], + STORE_BYTES: 3, + STORE_TIME: 0.3, + STORE_SIZE: [1, 2], + }, + } + ) + + assert prom_metrics.offloading_metrics[(0, LOAD_BYTES)].increments == [24] + assert prom_metrics.offloading_metrics[(0, LOAD_TIME)].increments == [1.5] + assert prom_metrics.offloading_metrics[(0, LOAD_SIZE)].observed == [16, 8] + assert prom_metrics.offloading_metrics[(0, STORE_BYTES)].increments == [3] + assert prom_metrics.offloading_metrics[(0, STORE_TIME)].increments == [0.3] + assert prom_metrics.offloading_metrics[(0, STORE_SIZE)].observed == [1, 2] + + assert prom_metrics.counter_kv_bytes[(0, "CPU_to_GPU")].increments == [24] + assert prom_metrics.counter_kv_transfer_time[(0, "CPU_to_GPU")].increments == [1.5] + assert prom_metrics.histogram_transfer_size[(0, "CPU_to_GPU")].observed == [16, 8] + assert prom_metrics.counter_kv_bytes[(0, "GPU_to_CPU")].increments == [3] + assert prom_metrics.counter_kv_transfer_time[(0, "GPU_to_CPU")].increments == [0.3] + assert prom_metrics.histogram_transfer_size[(0, "GPU_to_CPU")].observed == [1, 2] + + +def test_prom_metrics_observes_manager_gauge_and_histogram(): + metric_definitions = { + PENDING_STORES: OffloadingGaugeMetadata( + documentation="Number of currently pending KV offload stores.", + ), + LOOKUP_LATENCY: OffloadingHistogramMetadata( + documentation="KV offload lookup latency.", + buckets=(0.1, 1.0), + ), + } + with patch.object( + CPUOffloadingSpec, "build_metric_definitions", return_value=metric_definitions + ): + prom_metrics = OffloadPromMetrics( + vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type] + metric_types={ + Gauge: _FakeMetric, + Counter: _FakeMetric, + Histogram: _FakeMetric, + }, + labelnames=["model_name", "engine"], + per_engine_labelvalues={0: ["model", "0"]}, + ) + + prom_metrics.observe( + { + _StatsKey.TYPES: { + PENDING_STORES: _MetricType.GAUGE, + LOOKUP_LATENCY: _MetricType.HISTOGRAM, + }, + _StatsKey.DATA: { + PENDING_STORES: 5, + LOOKUP_LATENCY: [0.2, 0.4], + }, + } + ) + + gauge = prom_metrics.offloading_metrics[(0, PENDING_STORES)] + histogram = prom_metrics.offloading_metrics[(0, LOOKUP_LATENCY)] + assert gauge.set_values == [5] + assert histogram.observed == [0.2, 0.4] + histogram_def = prom_metrics._offloading_metric_defs[LOOKUP_LATENCY] + assert histogram_def.kwargs["buckets"] == (0.1, 1.0) + + +def test_prom_metrics_uses_configured_manager_metrics(): + prom_metrics = OffloadPromMetrics( + vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type] + metric_types={ + Gauge: _FakeMetric, + Counter: _FakeMetric, + Histogram: _FakeMetric, + }, + labelnames=["model_name", "engine"], + per_engine_labelvalues={0: ["model", "0"]}, + ) + + assert STORES_SKIPPED not in prom_metrics._offloading_metric_metadata + + +def test_aggregate_into_empty_stats(): + """Aggregating non-empty stats into a fresh (empty) stats object works.""" + empty = OffloadingConnectorStats() + assert empty.is_empty() + + non_empty = OffloadingConnectorStats( + data={ + _StatsKey.TYPES: { + LOAD_BYTES: _MetricType.COUNTER, + LOAD_SIZE: _MetricType.HISTOGRAM, + PENDING_STORES: _MetricType.GAUGE, + }, + _StatsKey.DATA: { + LOAD_BYTES: 42, + LOAD_SIZE: [10, 20], + PENDING_STORES: 3, + }, + }, + ) + + result = empty.aggregate(non_empty) + + assert result is empty + values = result.data[_StatsKey.DATA] + assert values[LOAD_BYTES] == 42 + assert values[LOAD_SIZE] == [10, 20] + assert values[PENDING_STORES] == 3 + + +def test_prom_metrics_multi_engine_routing(): + """Metrics are routed to the correct engine index.""" + prom_metrics = OffloadPromMetrics( + vllm_config=_FakeVllmConfig(), # type: ignore[arg-type] + metric_types={ + Gauge: _FakeMetric, + Counter: _FakeMetric, + Histogram: _FakeMetric, + }, + labelnames=["model_name", "engine"], + per_engine_labelvalues={0: ["model", "0"], 1: ["model", "1"]}, + ) + + prom_metrics.observe( + { + _StatsKey.TYPES: {LOAD_BYTES: _MetricType.COUNTER}, + _StatsKey.DATA: {LOAD_BYTES: 100}, + }, + engine_idx=1, + ) + + engine0 = prom_metrics.offloading_metrics[(0, LOAD_BYTES)] + engine1 = prom_metrics.offloading_metrics[(1, LOAD_BYTES)] + assert engine0.increments == [] + assert engine1.increments == [100] + + +def test_prom_metrics_rejects_undeclared_metric(): + """observe() asserts if a metric was never declared in metadata.""" + prom_metrics = OffloadPromMetrics( + vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type] + metric_types={ + Gauge: _FakeMetric, + Counter: _FakeMetric, + Histogram: _FakeMetric, + }, + labelnames=["model_name", "engine"], + per_engine_labelvalues={0: ["model", "0"]}, + ) + + with pytest.raises(AssertionError): + prom_metrics.observe( + { + _StatsKey.TYPES: {"unknown:metric": _MetricType.COUNTER}, + _StatsKey.DATA: {"unknown:metric": 1}, + } + ) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_worker_metadata.py b/tests/v1/kv_connector/unit/offloading_connector/test_worker_metadata.py index ab9d676cb4..2f56ede5b8 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_worker_metadata.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_worker_metadata.py @@ -4,7 +4,9 @@ import pytest from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import ( + DirectionalTransferStats, OffloadingWorkerMetadata, + TransferStats, ) pytestmark = pytest.mark.cpu_test @@ -30,3 +32,22 @@ def test_aggregate_multiple_workers(): meta3 = OffloadingWorkerMetadata(completed_jobs={42: 1, 43: 1, 8: 1}) result = meta1.aggregate(meta2).aggregate(meta3) assert result.completed_jobs == {42: 3, 43: 2, 7: 2, 8: 2} + + +def test_aggregate_transfer_stats(): + meta1 = OffloadingWorkerMetadata( + transfer_stats=TransferStats( + load=DirectionalTransferStats(bytes=10, time=0.5, sizes=[10]) + ) + ) + meta2 = OffloadingWorkerMetadata( + transfer_stats=TransferStats( + load=DirectionalTransferStats(bytes=20, time=1.0, sizes=[20, 30]) + ) + ) + + result = meta1.aggregate(meta2) + + assert result.transfer_stats.load.bytes == 30 + assert result.transfer_stats.load.time == 1.5 + assert result.transfer_stats.load.sizes == [10, 20, 30] diff --git a/tests/v1/kv_connector/unit/test_offloading_connector.py b/tests/v1/kv_connector/unit/test_offloading_connector.py index c8fa8293e7..c432b1b20e 100644 --- a/tests/v1/kv_connector/unit/test_offloading_connector.py +++ b/tests/v1/kv_connector/unit/test_offloading_connector.py @@ -5,6 +5,7 @@ import time import msgspec import msgspec.msgpack +import prometheus_client import pytest import zmq from tqdm import tqdm @@ -303,6 +304,167 @@ def test_cpu_offloading( del llm +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") +def test_cpu_offloading_metrics() -> None: + """Verify that offloading Prometheus metrics (new flat and deprecated + labeled) are emitted after stores and loads.""" + extra_config: dict = { + "cpu_bytes_to_use": 500 << 20, + "block_size": CPU_BLOCK_SIZES, + } + kv_transfer_config = KVTransferConfig( + kv_connector="OffloadingConnector", + kv_role="kv_both", + kv_connector_extra_config=extra_config, + ) + + llm = LLM( + model="meta-llama/Llama-3.2-1B-Instruct", + max_model_len=4096, + gpu_memory_utilization=0.5, + kv_transfer_config=kv_transfer_config, + disable_log_stats=False, + ) + + try: + prompt_token_ids = list(range(500)) + + # First generate: cold run, triggers a store to CPU. + # Use max_tokens>1 so the request is still producing output + # tokens when the async store completes and stats get drained. + # (The LLMEngine only records stats on steps with request outputs.) + llm.generate( + [TokensPrompt(prompt_token_ids=prompt_token_ids)], + SamplingParams(max_tokens=10), + use_tqdm=False, + ) + + # Wait for the async offload to finish, then reset GPU prefix cache + # so the next generate must load from CPU. + _wait_for_prefix_cache_reset(llm) + + # Second generate: triggers a load from CPU. + # Send a short filler alongside the load prompt so there's always + # a request producing output tokens when the load stats get drained + # (the LLMEngine only records stats on steps with request outputs). + filler = TokensPrompt(prompt_token_ids=[0]) + llm.generate( + [filler, TokensPrompt(prompt_token_ids=prompt_token_ids)], + SamplingParams(max_tokens=50), + use_tqdm=False, + ) + + # Metric helpers. + registry = prometheus_client.REGISTRY + + def _get_counter_value( + name: str, labels: dict[str, str] | None = None + ) -> float: + total = 0.0 + for metric in registry.collect(): + if metric.name == name: + for sample in metric.samples: + if sample.name != name + "_total": + continue + if labels and not all( + sample.labels.get(k) == v for k, v in labels.items() + ): + continue + total += sample.value + return total + + def _get_histogram_count( + name: str, labels: dict[str, str] | None = None + ) -> float: + total = 0.0 + for metric in registry.collect(): + if metric.name == name: + for sample in metric.samples: + if sample.name != name + "_count": + continue + if labels and not all( + sample.labels.get(k) == v for k, v in labels.items() + ): + continue + total += sample.value + return total + + # Stats are drained asynchronously — if the transfer finishes + # after the last engine step for that generate() call, the metrics + # won't appear until a subsequent step. Retry with dummy generates + # to force additional stats drains. + deadline = time.monotonic() + _RESET_CACHE_TIMEOUT + while time.monotonic() < deadline: + store_bytes = _get_counter_value("vllm:kv_offload_store_bytes") + load_bytes = _get_counter_value("vllm:kv_offload_load_bytes") + if store_bytes > 0 and load_bytes > 0: + break + llm.generate( + [TokensPrompt(prompt_token_ids=[0])], + SamplingParams(max_tokens=1), + use_tqdm=False, + ) + + # New flat counter metrics + store_bytes = _get_counter_value("vllm:kv_offload_store_bytes") + assert store_bytes > 0, f"Expected store_bytes > 0, got {store_bytes}" + load_bytes = _get_counter_value("vllm:kv_offload_load_bytes") + assert load_bytes > 0, f"Expected load_bytes > 0, got {load_bytes}" + store_time = _get_counter_value("vllm:kv_offload_store_time") + assert store_time > 0, f"Expected store_time > 0, got {store_time}" + load_time = _get_counter_value("vllm:kv_offload_load_time") + assert load_time > 0, f"Expected load_time > 0, got {load_time}" + + # New flat histogram metrics + store_size_count = _get_histogram_count("vllm:kv_offload_store_size") + assert store_size_count > 0, ( + f"Expected store_size histogram observations > 0, got {store_size_count}" + ) + load_size_count = _get_histogram_count("vllm:kv_offload_load_size") + assert load_size_count > 0, ( + f"Expected load_size histogram observations > 0, got {load_size_count}" + ) + + # Deprecated labeled metrics — verify per transfer_type label. + load_label = {"transfer_type": "CPU_to_GPU"} + store_label = {"transfer_type": "GPU_to_CPU"} + + dep_load_bytes = _get_counter_value("vllm:kv_offload_total_bytes", load_label) + assert dep_load_bytes > 0, ( + f"Expected deprecated load bytes > 0, got {dep_load_bytes}" + ) + dep_store_bytes = _get_counter_value("vllm:kv_offload_total_bytes", store_label) + assert dep_store_bytes > 0, ( + f"Expected deprecated store bytes > 0, got {dep_store_bytes}" + ) + dep_load_time = _get_counter_value("vllm:kv_offload_total_time", load_label) + assert dep_load_time > 0, ( + f"Expected deprecated load time > 0, got {dep_load_time}" + ) + dep_store_time = _get_counter_value("vllm:kv_offload_total_time", store_label) + assert dep_store_time > 0, ( + f"Expected deprecated store time > 0, got {dep_store_time}" + ) + dep_load_size = _get_histogram_count("vllm:kv_offload_size", load_label) + assert dep_load_size > 0, ( + f"Expected deprecated load size observations > 0, got {dep_load_size}" + ) + dep_store_size = _get_histogram_count("vllm:kv_offload_size", store_label) + assert dep_store_size > 0, ( + f"Expected deprecated store size observations > 0, got {dep_store_size}" + ) + + # Flat and deprecated metrics must be consistent (dual-write). + assert store_bytes == dep_store_bytes + assert load_bytes == dep_load_bytes + assert store_time == dep_store_time + assert load_time == dep_load_time + assert store_size_count == dep_store_size + assert load_size_count == dep_load_size + finally: + del llm + + def test_tiering_offloading() -> None: """Tests OffloadingConnector with TieringOffloadingSpec.""" extra_config: dict = { diff --git a/tests/v1/kv_offload/cpu/test_manager.py b/tests/v1/kv_offload/cpu/test_manager.py index 3957294f8b..8b68855def 100644 --- a/tests/v1/kv_offload/cpu/test_manager.py +++ b/tests/v1/kv_offload/cpu/test_manager.py @@ -18,6 +18,8 @@ from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy +STORES_SKIPPED = "vllm:kv_offload_stores_skipped" + def make_req_context( req_id: str = "", kv_transfer_params: dict | None = None @@ -29,6 +31,22 @@ def make_req_context( _EMPTY_REQ_CTX = make_req_context() +def make_cpu_manager( + num_blocks: int = 4, + cache_policy: str = "lru", + enable_events: bool = False, + store_threshold: int = 0, + max_tracker_size: int = 64_000, +) -> CPUOffloadingManager: + return CPUOffloadingManager( + num_blocks=num_blocks, + cache_policy=cache_policy, + enable_events=enable_events, + store_threshold=store_threshold, + max_tracker_size=max_tracker_size, + ) + + @dataclass class ExpectedPrepareStoreOutput: keys_to_store: list[int] @@ -110,7 +128,7 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy): candidate to make room for [3, 4, 5] - After complete_store([2, 3, 4, 5]), block 2 must still be present. """ - manager = CPUOffloadingManager( + manager = make_cpu_manager( num_blocks=4, cache_policy=eviction_policy, enable_events=True, @@ -144,14 +162,37 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy): assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is True +def test_filter_reused_manager_reports_stores_skipped_counter(): + manager = make_cpu_manager( + num_blocks=4, + cache_policy="lru", + store_threshold=2, + ) + + prepare_store_output = manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX) + + verify_store_output( + prepare_store_output, + ExpectedPrepareStoreOutput( + keys_to_store=[], + store_block_ids=[], + evicted_keys=[], + ), + ) + stats = manager.get_stats() + assert stats is not None + assert stats.reduce()[STORES_SKIPPED] == 3 + stats = manager.get_stats() + assert stats is not None + assert stats.reduce()[STORES_SKIPPED] == 0 + + def test_cpu_manager(): """ Tests CPUOffloadingManager with lru policy. """ # initialize a CPU manager with a capacity of 4 blocks - cpu_manager = CPUOffloadingManager( - num_blocks=4, cache_policy="lru", enable_events=True - ) + cpu_manager = make_cpu_manager(num_blocks=4, cache_policy="lru", enable_events=True) # prepare store [1, 2] prepare_store_output = cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX) @@ -264,7 +305,7 @@ def test_cpu_manager(): def test_prepare_load_preserves_key_order(): """block_ids[i] must correspond to keys[i] (co-indexed invariant).""" - manager = CPUOffloadingManager(num_blocks=4, cache_policy="lru") + manager = make_cpu_manager(num_blocks=4, cache_policy="lru") key_a, key_b, key_c = to_key(0), to_key(1), to_key(2) @@ -305,7 +346,7 @@ class TestARCPolicy: def _make_manager( self, num_blocks: int = 4, enable_events: bool = True ) -> tuple[CPUOffloadingManager, ARCCachePolicy]: - manager = CPUOffloadingManager( + manager = make_cpu_manager( num_blocks=num_blocks, cache_policy="arc", enable_events=enable_events, @@ -605,7 +646,7 @@ def test_filter_reused_manager(): """ Tests CPUOffloadingManager reuse filtering (store_threshold=2). """ - manager = CPUOffloadingManager( + manager = make_cpu_manager( num_blocks=4, cache_policy="lru", enable_events=True, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/common.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/common.py index c5a251a2a5..928fec639c 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/common.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/common.py @@ -11,6 +11,45 @@ from vllm.v1.kv_offload.worker.worker import TransferSpec ReqId = str +@dataclass(slots=True) +class DirectionalTransferStats: + bytes: int = 0 + time: float = 0.0 + sizes: list[int | float] = field(default_factory=list) + + def aggregate( + self, other: "DirectionalTransferStats" + ) -> "DirectionalTransferStats": + return DirectionalTransferStats( + bytes=self.bytes + other.bytes, + time=self.time + other.time, + sizes=[*self.sizes, *other.sizes], + ) + + def record(self, num_bytes: int, time: float) -> None: + self.bytes += num_bytes + self.time += time + self.sizes.append(num_bytes) + + def is_empty(self) -> bool: + return self.bytes == 0 and self.time == 0.0 and not self.sizes + + +@dataclass(slots=True) +class TransferStats: + load: DirectionalTransferStats = field(default_factory=DirectionalTransferStats) + store: DirectionalTransferStats = field(default_factory=DirectionalTransferStats) + + def aggregate(self, other: "TransferStats") -> "TransferStats": + return TransferStats( + load=self.load.aggregate(other.load), + store=self.store.aggregate(other.store), + ) + + def is_empty(self) -> bool: + return self.load.is_empty() and self.store.is_empty() + + @dataclass class TransferJob: """A transfer job bundling request context with transfer spec. @@ -43,6 +82,7 @@ class OffloadingWorkerMetadata(KVConnectorWorkerMetadata): """ completed_jobs: dict[int, int] = field(default_factory=dict) + transfer_stats: TransferStats = field(default_factory=TransferStats) def mark_completed(self, job_id: int) -> None: """Record a transfer job completion from this worker.""" @@ -57,4 +97,7 @@ class OffloadingWorkerMetadata(KVConnectorWorkerMetadata): for job_id, v in other.completed_jobs.items(): merged[job_id] = merged.get(job_id, 0) + v - return OffloadingWorkerMetadata(completed_jobs=merged) + return OffloadingWorkerMetadata( + completed_jobs=merged, + transfer_stats=self.transfer_stats.aggregate(other.transfer_stats), + ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/metrics.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/metrics.py index 0839b2727c..3e4463924b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/metrics.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/metrics.py @@ -10,37 +10,176 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( PromMetric, PromMetricT, ) -from vllm.logger import init_logger -from vllm.v1.kv_offload.worker.worker import TransferType - -logger = init_logger(__name__) +from vllm.v1.kv_offload.base import ( + OffloadingCounterMetadata, + OffloadingGaugeMetadata, + OffloadingHistogramMetadata, + OffloadingMetricMetadata, +) +from vllm.v1.kv_offload.factory import OffloadingSpecFactory -@dataclass -class OffloadingOperationMetrics: - op_size: int - op_time: float +class _TransferMetricName: + """Flat metric names for GPU↔offload-medium transfer operations.""" + + LOAD_BYTES = "vllm:kv_offload_load_bytes" + LOAD_TIME = "vllm:kv_offload_load_time" + LOAD_SIZE = "vllm:kv_offload_load_size" + STORE_BYTES = "vllm:kv_offload_store_bytes" + STORE_TIME = "vllm:kv_offload_store_time" + STORE_SIZE = "vllm:kv_offload_store_size" + + +class _TransferType: + """Transfer direction labels for deprecated CPU offload metrics.""" + + LOAD = "CPU_to_GPU" + STORE = "GPU_to_CPU" + ALL = (LOAD, STORE) + + +TRANSFER_SIZE_BUCKETS = ( + 1e6, + 5e6, + 10e6, + 20e6, + 40e6, + 60e6, + 80e6, + 100e6, + 150e6, + 200e6, +) + + +def get_connector_metric_definitions() -> dict[str, OffloadingMetricMetadata]: + return { + _TransferMetricName.LOAD_BYTES: OffloadingCounterMetadata( + documentation="Total bytes loaded from offload storage to GPU.", + ), + _TransferMetricName.LOAD_TIME: OffloadingCounterMetadata( + documentation="Total load time from offload storage to GPU, in seconds.", + ), + _TransferMetricName.LOAD_SIZE: OffloadingHistogramMetadata( + documentation="Histogram of KV offload load operation size, in bytes.", + buckets=TRANSFER_SIZE_BUCKETS, + ), + _TransferMetricName.STORE_BYTES: OffloadingCounterMetadata( + documentation="Total bytes stored from GPU to offload storage.", + ), + _TransferMetricName.STORE_TIME: OffloadingCounterMetadata( + documentation="Total store time from GPU to offload storage, in seconds.", + ), + _TransferMetricName.STORE_SIZE: OffloadingHistogramMetadata( + documentation="Histogram of KV offload store operation size, in bytes.", + buckets=TRANSFER_SIZE_BUCKETS, + ), + } + + +_DEPRECATED_TOTAL_BYTES = "vllm:kv_offload_total_bytes" +_DEPRECATED_TOTAL_TIME = "vllm:kv_offload_total_time" +_DEPRECATED_SIZE = "vllm:kv_offload_size" + +# Deprecated legacy transfer metrics, kept during the migration to the flat +# metric names above. These stay in a separate definition block because they +# use a transfer_type label, but are emitted from the same flat stats payload +# for compatibility. +_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS: dict[str, OffloadingMetricMetadata] = { + _DEPRECATED_TOTAL_BYTES: OffloadingCounterMetadata( + documentation="Number of bytes offloaded by KV connector", + ), + _DEPRECATED_TOTAL_TIME: OffloadingCounterMetadata( + documentation="Total time measured by all KV offloading operations", + ), + _DEPRECATED_SIZE: OffloadingHistogramMetadata( + documentation="Histogram of KV offload transfer size, in bytes.", + buckets=TRANSFER_SIZE_BUCKETS, + ), +} + + +class _MetricType: + """Type tags embedded in the serialized stats payload.""" + + COUNTER = "counter" + GAUGE = "gauge" + HISTOGRAM = "histogram" + + +class _StatsKey: + """Top-level keys in the serialized stats dict.""" + + # Maps metric name -> _MetricType value + TYPES = "types" + # Maps metric name -> observed value (number or list) + DATA = "data" @dataclass class OffloadingConnectorStats(KVConnectorStats): + """ + Offloading connector stats use flat metric names as keys. + + The ``data`` dict is structured using ``_StatsKey`` / ``_MetricType``:: + + { + _StatsKey.TYPES: {name: _MetricType.*, ...}, + _StatsKey.DATA: {name: value, ...}, + } + + This structure is self-describing: it survives IPC serialization + without needing the full ``OffloadingMetricMetadata`` objects on the + receiving side. + + Counter values are aggregated by summing, gauge values use the latest + snapshot, and histogram values are lists of observed samples. + """ + def __post_init__(self): - if not self.data: - # Empty container init, no data is passed in. + if _StatsKey.DATA not in self.data: self.reset() def reset(self): - self.data: dict[str, list[OffloadingOperationMetrics]] = {} + self.data: dict[str, Any] = { + _StatsKey.TYPES: {}, + _StatsKey.DATA: {}, + } - def aggregate(self, other: KVConnectorStats) -> KVConnectorStats: - if not other.is_empty(): - for k, v in other.data.items(): - if k not in self.data: - self.data[k] = v + @property + def _types(self) -> dict[str, str]: + return self.data[_StatsKey.TYPES] + + @property + def _values(self) -> dict[str, Any]: + return self.data[_StatsKey.DATA] + + def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats": + if other.is_empty(): + return self + assert isinstance(other, OffloadingConnectorStats) + other_types = other._types + other_values = other._values + for key, value in other_values.items(): + type_str = other_types.get(key) + if type_str is None: + raise AssertionError(f"Unknown offloading stats key: {key}") + self._types.setdefault(key, type_str) + if type_str == _MetricType.HISTOGRAM: + assert isinstance(value, list) + if key not in self._values: + self._values[key] = value else: - accumulator = self.data[k] - assert isinstance(accumulator, list) - accumulator.extend(v) + assert isinstance(self._values[key], list) + self._values[key].extend(value) + elif type_str == _MetricType.COUNTER: + assert isinstance(value, int | float) + self._values[key] = self._values.get(key, 0) + value + elif type_str == _MetricType.GAUGE: + assert isinstance(value, int | float) + self._values[key] = value + else: + raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}") return self def reduce(self) -> dict[str, int | float]: @@ -51,29 +190,44 @@ class OffloadingConnectorStats(KVConnectorStats): stats for the last time interval. """ return_dict: dict[str, int | float] = {} - for transfer_type, ops_list in self.data.items(): - assert isinstance(ops_list, list) - total_bytes = 0 - total_time = 0.0 - for op in ops_list: - assert isinstance(op, dict) - total_bytes += op["op_size"] - total_time += op["op_time"] - return_dict[f"{transfer_type}_total_bytes"] = total_bytes - return_dict[f"{transfer_type}_total_time"] = total_time + for key, value in self._values.items(): + type_str = self._types.get(key) + if type_str is None: + raise AssertionError(f"Unknown offloading stats key: {key}") + if type_str == _MetricType.HISTOGRAM: + assert isinstance(value, list) + return_dict[f"{key}_count"] = len(value) + return_dict[f"{key}_sum"] = sum(value) + elif type_str in (_MetricType.COUNTER, _MetricType.GAUGE): + assert isinstance(value, int | float) + return_dict[key] = value + else: + raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}") return return_dict def is_empty(self) -> bool: - return not self.data + return not self.data.get(_StatsKey.DATA) - def record_transfer(self, num_bytes: int, time: float, transfer_type: TransferType): - src, dst = transfer_type - transfer_type_key = src + "_to_" + dst - op = OffloadingOperationMetrics(num_bytes, time) - if transfer_type_key in self.data: - self.data[transfer_type_key].append(op) - else: - self.data[transfer_type_key] = [op] + def increase_counter( + self, counter_name: str, counter_increase_value: int | float + ) -> None: + """Increase a counter on the stats payload.""" + self._types.setdefault(counter_name, _MetricType.COUNTER) + self._values[counter_name] = ( + self._values.get(counter_name, 0) + counter_increase_value + ) + + def set_gauge(self, gauge_name: str, gauge_value: int | float) -> None: + """Set a gauge snapshot on the stats payload.""" + self._types.setdefault(gauge_name, _MetricType.GAUGE) + self._values[gauge_name] = gauge_value + + def observe_histogram( + self, histogram_name: str, histogram_value: int | float + ) -> None: + """Record a histogram observation on the stats payload.""" + self._types.setdefault(histogram_name, _MetricType.HISTOGRAM) + self._values.setdefault(histogram_name, []).append(histogram_value) class OffloadPromMetrics(KVConnectorPromMetrics): @@ -89,77 +243,145 @@ class OffloadPromMetrics(KVConnectorPromMetrics): self.histogram_transfer_size: dict[tuple[int, str], PromMetricT] = {} self.counter_kv_bytes: dict[tuple[int, str], PromMetricT] = {} self.counter_kv_transfer_time: dict[tuple[int, str], PromMetricT] = {} - buckets = [ # In bytes - 1e6, - 5e6, - 10e6, - 20e6, - 40e6, - 60e6, - 80e6, - 100e6, - 150e6, - 200e6, - ] + spec_cls = OffloadingSpecFactory.get_spec_cls(vllm_config) + kv_transfer_config = vllm_config.kv_transfer_config + assert kv_transfer_config is not None + extra_config = kv_transfer_config.kv_connector_extra_config + self._offloading_metric_metadata: dict[str, OffloadingMetricMetadata] = { + **spec_cls.build_metric_definitions(extra_config), + **get_connector_metric_definitions(), + } + from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec + + self._observe_deprecated_metrics = issubclass(spec_cls, CPUOffloadingSpec) + self._offloading_metric_defs: dict[str, PromMetricT] = {} + self.offloading_metrics: dict[tuple[int, str], PromMetricT] = {} self._counter_kv_bytes = self._counter_cls( - name="vllm:kv_offload_total_bytes", - documentation="Number of bytes offloaded by KV connector", + name=_DEPRECATED_TOTAL_BYTES, + documentation=_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[ + _DEPRECATED_TOTAL_BYTES + ].documentation, labelnames=labelnames + ["transfer_type"], ) self._counter_kv_transfer_time = self._counter_cls( - name="vllm:kv_offload_total_time", - documentation="Total time measured by all KV offloading operations", + name=_DEPRECATED_TOTAL_TIME, + documentation=_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[ + _DEPRECATED_TOTAL_TIME + ].documentation, labelnames=labelnames + ["transfer_type"], ) + deprecated_size_metadata = _DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[ + _DEPRECATED_SIZE + ] + assert isinstance(deprecated_size_metadata, OffloadingHistogramMetadata) self._histogram_transfer_size = self._histogram_cls( - name="vllm:kv_offload_size", - documentation="Histogram of KV offload transfer size, in bytes.", - buckets=buckets[:], + name=_DEPRECATED_SIZE, + documentation=deprecated_size_metadata.documentation, + buckets=deprecated_size_metadata.buckets, labelnames=labelnames + ["transfer_type"], ) - def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): - """ - Observe transfer statistics from the new data structure. - transfer_stats_data is expected to be a dict where: - - keys are transfer type strings (e.g., "cpu_to_gpu", "gpu_to_cpu") - - values are lists of OffloadingOperationMetrics objects - """ - - for transfer_type, ops in transfer_stats_data.items(): - # Cache: - if (engine_idx, transfer_type) not in self.histogram_transfer_size: + for engine_idx, labelvalues in per_engine_labelvalues.items(): + for transfer_type in _TransferType.ALL: + bounded_labelvalues = labelvalues + [transfer_type] self.histogram_transfer_size[(engine_idx, transfer_type)] = ( - self._histogram_transfer_size.labels( - *(self.per_engine_labelvalues[engine_idx] + [transfer_type]) - ) + self._histogram_transfer_size.labels(*bounded_labelvalues) ) self.counter_kv_bytes[(engine_idx, transfer_type)] = ( - self._counter_kv_bytes.labels( - *(self.per_engine_labelvalues[engine_idx] + [transfer_type]) - ) + self._counter_kv_bytes.labels(*bounded_labelvalues) ) self.counter_kv_transfer_time[(engine_idx, transfer_type)] = ( - self._counter_kv_transfer_time.labels( - *(self.per_engine_labelvalues[engine_idx] + [transfer_type]) - ) + self._counter_kv_transfer_time.labels(*bounded_labelvalues) ) - # Process ops: - assert isinstance(ops, list) - for op in ops: # ops is a list of serialized OffloadingOperationMetrics - assert isinstance(op, dict) - # Observe size histogram - self.histogram_transfer_size[(engine_idx, transfer_type)].observe( - op["op_size"] + for metric_name, metadata in self._offloading_metric_metadata.items(): + self._offloading_metric_defs[metric_name] = self._create_metric( + metric_name, metadata + ) + for engine_idx, labelvalues in per_engine_labelvalues.items(): + self.offloading_metrics[(engine_idx, metric_name)] = ( + self._offloading_metric_defs[metric_name].labels(*labelvalues) ) - # Increment byte and time counters - self.counter_kv_bytes[(engine_idx, transfer_type)].inc(op["op_size"]) + def _create_metric( + self, metric_name: str, metadata: OffloadingMetricMetadata + ) -> Any: + kwargs: dict[str, Any] = { + "name": metric_name, + "documentation": metadata.documentation, + "labelnames": self._labelnames, + } + if isinstance(metadata, OffloadingCounterMetadata): + metric_cls = self._counter_cls + elif isinstance(metadata, OffloadingGaugeMetadata): + metric_cls = self._gauge_cls + elif isinstance(metadata, OffloadingHistogramMetadata): + metric_cls = self._histogram_cls + if metadata.buckets is not None: + kwargs["buckets"] = metadata.buckets + else: + raise AssertionError(f"Unknown offloading metric metadata: {metadata}") + return metric_cls(**kwargs) - self.counter_kv_transfer_time[(engine_idx, transfer_type)].inc( - op["op_time"] + def _increase_counter( + self, metric_name: str, value: int | float, engine_idx: int + ) -> None: + self.offloading_metrics[(engine_idx, metric_name)].inc(value) + if not self._observe_deprecated_metrics: + return + # Keep deprecated CPU offload transfer metrics updated during the + # transition to flat metric names. + if metric_name == _TransferMetricName.LOAD_BYTES: + self.counter_kv_bytes[(engine_idx, _TransferType.LOAD)].inc(value) + elif metric_name == _TransferMetricName.LOAD_TIME: + self.counter_kv_transfer_time[(engine_idx, _TransferType.LOAD)].inc(value) + elif metric_name == _TransferMetricName.STORE_BYTES: + self.counter_kv_bytes[(engine_idx, _TransferType.STORE)].inc(value) + elif metric_name == _TransferMetricName.STORE_TIME: + self.counter_kv_transfer_time[(engine_idx, _TransferType.STORE)].inc(value) + + def _set_gauge(self, metric_name: str, value: int | float, engine_idx: int) -> None: + self.offloading_metrics[(engine_idx, metric_name)].set(value) + + def _observe_histogram( + self, metric_name: str, value: list[int | float], engine_idx: int + ) -> None: + for observation in value: + self.offloading_metrics[(engine_idx, metric_name)].observe(observation) + if not self._observe_deprecated_metrics: + continue + # Keep deprecated CPU offload transfer metrics updated during the + # transition to flat metric names. + if metric_name == _TransferMetricName.LOAD_SIZE: + self.histogram_transfer_size[(engine_idx, _TransferType.LOAD)].observe( + observation ) + elif metric_name == _TransferMetricName.STORE_SIZE: + self.histogram_transfer_size[(engine_idx, _TransferType.STORE)].observe( + observation + ) + + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): + """Observe transfer statistics.""" + metric_types = transfer_stats_data.get(_StatsKey.TYPES, {}) + metric_data = transfer_stats_data.get(_StatsKey.DATA, {}) + for key, value in metric_data.items(): + type_str = metric_types.get(key) + if type_str is None: + raise AssertionError(f"Unknown offloading stats key: {key}") + assert key in self._offloading_metric_defs + if type_str == _MetricType.COUNTER: + assert isinstance(value, int | float) + self._increase_counter(key, value, engine_idx) + elif type_str == _MetricType.GAUGE: + assert isinstance(value, int | float) + self._set_gauge(key, value, engine_idx) + elif type_str == _MetricType.HISTOGRAM: + assert isinstance(value, list) + assert all(isinstance(v, int | float) for v in value) + self._observe_histogram(key, value, engine_idx) + else: + raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}") diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 6ee827fa17..24e7143e63 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -14,6 +14,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import ( ReqId, TransferJob, ) +from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( + OffloadingConnectorStats, + _TransferMetricName, +) from vllm.logger import init_logger from vllm.utils.math_utils import cdiv from vllm.v1.core.kv_cache_manager import KVCacheBlocks @@ -259,9 +263,13 @@ def _create_req_context(req: Request) -> ReqContext: class OffloadingConnectorScheduler: """Implementation of Scheduler side methods""" - def __init__(self, spec: OffloadingSpec): + def __init__( + self, + spec: OffloadingSpec, + ): self.config = SchedulerOffloadConfig.from_spec(spec) self.manager: OffloadingManager = spec.get_manager() + self._connector_stats: OffloadingConnectorStats | None = None full_attention_groups: list[int] = [] sliding_window_groups: list[int] = [] @@ -952,6 +960,39 @@ class OffloadingConnectorScheduler: if not isinstance(meta, OffloadingWorkerMetadata): assert meta is None meta = OffloadingWorkerMetadata() + if not meta.transfer_stats.is_empty(): + transfer_stats = OffloadingConnectorStats() + if not meta.transfer_stats.load.is_empty(): + transfer_stats.increase_counter( + _TransferMetricName.LOAD_BYTES, + meta.transfer_stats.load.bytes, + ) + transfer_stats.increase_counter( + _TransferMetricName.LOAD_TIME, + meta.transfer_stats.load.time, + ) + for size in meta.transfer_stats.load.sizes: + transfer_stats.observe_histogram( + _TransferMetricName.LOAD_SIZE, size + ) + if not meta.transfer_stats.store.is_empty(): + transfer_stats.increase_counter( + _TransferMetricName.STORE_BYTES, + meta.transfer_stats.store.bytes, + ) + transfer_stats.increase_counter( + _TransferMetricName.STORE_TIME, + meta.transfer_stats.store.time, + ) + for size in meta.transfer_stats.store.sizes: + transfer_stats.observe_histogram( + _TransferMetricName.STORE_SIZE, size + ) + if self._connector_stats is None: + self._connector_stats = transfer_stats + else: + self._connector_stats.aggregate(transfer_stats) + for job_id, count in meta.completed_jobs.items(): assert count > 0 if job_id < self._stale_job_threshold: @@ -990,6 +1031,19 @@ class OffloadingConnectorScheduler: if not req_status.transfer_jobs and req_status.req.is_finished(): del self._req_status[job_status.req_id] + def get_stats(self) -> OffloadingConnectorStats | None: + stats = self._connector_stats + self._connector_stats = None + + manager_stats = self.manager.get_stats() + if manager_stats is not None: + if stats is None: + stats = manager_stats + else: + stats.aggregate(manager_stats) + + return stats + def request_finished( self, request: Request, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py index 8957ce3445..744a0c7429 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py @@ -5,17 +5,11 @@ from dataclasses import replace import torch -from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( - KVConnectorStats, -) from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import ( OffloadingConnectorMetadata, OffloadingWorkerMetadata, ReqId, ) -from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( - OffloadingConnectorStats, -) from vllm.logger import init_logger from vllm.v1.attention.backend import AttentionBackend from vllm.v1.kv_cache_interface import ( @@ -44,7 +38,6 @@ class OffloadingConnectorWorker: self.spec = spec self.worker = OffloadingWorker() - self.kv_connector_stats = OffloadingConnectorStats() # job_id -> req_id for in-flight loads. self._load_jobs: dict[int, ReqId] = {} self._unsubmitted_store_jobs: list[tuple[int, TransferSpec]] = [] @@ -271,15 +264,18 @@ class OffloadingConnectorWorker: # we currently do not support job failures job_id = transfer_result.job_id assert transfer_result.success + is_load = job_id in self._load_jobs if ( - transfer_result.transfer_time + transfer_result.transfer_time is not None and transfer_result.transfer_size is not None - and transfer_result.transfer_type is not None ): - self.kv_connector_stats.record_transfer( - num_bytes=transfer_result.transfer_size, - time=transfer_result.transfer_time, - transfer_type=transfer_result.transfer_type, + if is_load: + stats = self._connector_worker_meta.transfer_stats.load + else: + stats = self._connector_worker_meta.transfer_stats.store + stats.record( + transfer_result.transfer_size, + transfer_result.transfer_time, ) self._connector_worker_meta.mark_completed(job_id) @@ -297,18 +293,6 @@ class OffloadingConnectorWorker: self._connector_worker_meta = OffloadingWorkerMetadata() return meta - def get_kv_connector_stats(self) -> KVConnectorStats | None: - """ - Get the KV transfer stats for the connector. - """ - - if self.kv_connector_stats.is_empty(): - return None - # Clear stats for next iteration - kv_connector_stats = self.kv_connector_stats - self.kv_connector_stats = OffloadingConnectorStats() - return kv_connector_stats - def shutdown(self) -> None: self._unsubmitted_store_jobs.clear() self._load_jobs.clear() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py index 20888c71f8..7d567ed462 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py @@ -184,9 +184,14 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA): return True def get_kv_connector_stats(self) -> KVConnectorStats | None: - if self.connector_worker is None: - return None # We only emit stats from the worker-side - return self.connector_worker.get_kv_connector_stats() + if self.connector_scheduler is not None: + return self.connector_scheduler.get_stats() + + # TODO(orozery): Remove once PR #43877 lands + if self.connector_worker is not None: + return OffloadingConnectorStats() + + return None @classmethod def build_kv_connector_stats( diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index 5f798f41ea..16f783190f 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -19,6 +19,9 @@ from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes if TYPE_CHECKING: from vllm.config import VllmConfig + from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( + OffloadingConnectorStats, + ) from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.kv_offload.worker.worker import OffloadingHandler @@ -123,6 +126,26 @@ The class provides the following primitives: """ +@dataclass(frozen=True) +class OffloadingMetricMetadata: + documentation: str + + +@dataclass(frozen=True) +class OffloadingCounterMetadata(OffloadingMetricMetadata): + pass + + +@dataclass(frozen=True) +class OffloadingGaugeMetadata(OffloadingMetricMetadata): + pass + + +@dataclass(frozen=True) +class OffloadingHistogramMetadata(OffloadingMetricMetadata): + buckets: tuple[float, ...] | None = None + + class OffloadingManager(ABC): @abstractmethod def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None: @@ -269,6 +292,10 @@ class OffloadingManager(ABC): """Evict all tracked blocks and reset internal state.""" return + def get_stats(self) -> "OffloadingConnectorStats | None": + """Return collected metrics since last call, or None if disabled.""" + return None + def shutdown(self) -> None: """Shutdown the manager and release any resources.""" return @@ -378,6 +405,13 @@ class CanonicalKVCaches: class OffloadingSpec(ABC): """Spec for an offloading connector""" + @classmethod + def build_metric_definitions( + cls, extra_config: dict[str, Any] + ) -> dict[str, "OffloadingMetricMetadata"]: + """Return Prometheus metric definitions emitted by this spec.""" + return {} + def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"): logger.warning( "Initializing OffloadingSpec. This API is experimental and " diff --git a/vllm/v1/kv_offload/cpu/common.py b/vllm/v1/kv_offload/cpu/common.py index 42f576bb70..46bca1b906 100644 --- a/vllm/v1/kv_offload/cpu/common.py +++ b/vllm/v1/kv_offload/cpu/common.py @@ -4,6 +4,8 @@ from typing_extensions import override from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec +METRIC_STORES_SKIPPED = "vllm:kv_offload_stores_skipped" + class CPULoadStoreSpec(BlockIDsLoadStoreSpec): """ diff --git a/vllm/v1/kv_offload/cpu/manager.py b/vllm/v1/kv_offload/cpu/manager.py index a1d3a30ebb..3218e152df 100644 --- a/vllm/v1/kv_offload/cpu/manager.py +++ b/vllm/v1/kv_offload/cpu/manager.py @@ -6,6 +6,9 @@ from typing import Literal from typing_extensions import override +from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( + OffloadingConnectorStats, +) from vllm.v1.kv_offload.base import ( LoadStoreSpec, OffloadingEvent, @@ -15,7 +18,7 @@ from vllm.v1.kv_offload.base import ( ReqContext, RequestOffloadingContext, ) -from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec +from vllm.v1.kv_offload.cpu.common import METRIC_STORES_SKIPPED, CPULoadStoreSpec from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy @@ -58,6 +61,7 @@ class CPUOffloadingManager(OffloadingManager): self._policy: CachePolicy = policy_cls(cache_capacity=num_blocks) self.store_threshold: int = store_threshold self.max_tracker_size: int = max_tracker_size + self.stores_skipped_in_current_batch: int = 0 # Number of block references. It is ordered so can evict the LRU entry in O(1). self.counts: OrderedDict[OffloadKey, int] | None = ( @@ -154,7 +158,9 @@ class CPUOffloadingManager(OffloadingManager): req_context: ReqContext, ) -> PrepareStoreOutput | None: if self.counts is not None: + num_keys = len(keys) keys = [k for k in keys if self.counts.get(k, 0) >= self.store_threshold] + self.stores_skipped_in_current_batch += num_keys - len(keys) # filter out blocks that are already stored keys_to_store = [k for k in keys if self._policy.get(k) is None] @@ -253,3 +259,15 @@ class CPUOffloadingManager(OffloadingManager): if self.events is not None: yield from self.events self.events.clear() + + def get_stats(self) -> OffloadingConnectorStats | None: + if self.store_threshold < 2: + return None + + stats = OffloadingConnectorStats() + stats.increase_counter( + METRIC_STORES_SKIPPED, + self.stores_skipped_in_current_batch, + ) + self.stores_skipped_in_current_batch = 0 + return stats diff --git a/vllm/v1/kv_offload/cpu/spec.py b/vllm/v1/kv_offload/cpu/spec.py index 732891553a..d65ba9439e 100644 --- a/vllm/v1/kv_offload/cpu/spec.py +++ b/vllm/v1/kv_offload/cpu/spec.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Iterator +from typing import Any from typing_extensions import override @@ -12,10 +13,12 @@ from vllm.v1.kv_offload.base import ( CanonicalKVCaches, GPULoadStoreSpec, LoadStoreSpec, + OffloadingCounterMetadata, OffloadingManager, + OffloadingMetricMetadata, OffloadingSpec, ) -from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec +from vllm.v1.kv_offload.cpu.common import METRIC_STORES_SKIPPED, CPULoadStoreSpec from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager from vllm.v1.kv_offload.worker.worker import OffloadingHandler @@ -24,6 +27,22 @@ from vllm.v1.kv_offload.worker.worker import OffloadingHandler class CPUOffloadingSpec(OffloadingSpec): BLOCK_SIZE_ALIGNMENT = 1 + @classmethod + def build_metric_definitions( + cls, extra_config: dict[str, Any] + ) -> dict[str, OffloadingMetricMetadata]: + store_threshold = int(extra_config.get("store_threshold", 0)) + if store_threshold < 2: + return {} + return { + METRIC_STORES_SKIPPED: OffloadingCounterMetadata( + documentation=( + "Number of KV offload stores skipped because the reuse " + "threshold was not reached." + ), + ) + } + def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig): super().__init__(vllm_config, kv_cache_config) diff --git a/vllm/v1/kv_offload/factory.py b/vllm/v1/kv_offload/factory.py index 8b967f771b..abbc9c0ede 100644 --- a/vllm/v1/kv_offload/factory.py +++ b/vllm/v1/kv_offload/factory.py @@ -30,11 +30,7 @@ class OffloadingSpecFactory: cls._registry[name] = loader @classmethod - def create_spec( - cls, - config: "VllmConfig", - kv_cache_config: "KVCacheConfig", - ) -> OffloadingSpec: + def get_spec_cls(cls, config: "VllmConfig") -> type[OffloadingSpec]: kv_transfer_config = config.kv_transfer_config assert kv_transfer_config is not None extra_config = kv_transfer_config.kv_connector_extra_config @@ -48,6 +44,20 @@ class OffloadingSpecFactory: spec_module = importlib.import_module(spec_module_path) spec_cls = getattr(spec_module, spec_name) assert issubclass(spec_cls, OffloadingSpec) + return spec_cls + + @classmethod + def create_spec( + cls, + config: "VllmConfig", + kv_cache_config: "KVCacheConfig", + ) -> OffloadingSpec: + kv_transfer_config = config.kv_transfer_config + assert kv_transfer_config is not None + spec_name = kv_transfer_config.kv_connector_extra_config.get( + "spec_name", "CPUOffloadingSpec" + ) + spec_cls = cls.get_spec_cls(config) logger.info("Creating offloading spec with name: %s", spec_name) return spec_cls(config, kv_cache_config)