Feature/offloading manager stats (#35669)

Signed-off-by: Sriusa4414@gmail.com
Signed-off-by: srinivas_oo7 <Sriusa4414@gmail.com>
Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Signed-off-by: Srinivasoo7 <158864704+Srinivasoo7@users.noreply.github.com>
Signed-off-by: Or Ozeri <oro@il.ibm.com>
Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Co-authored-by: Srinivasoo7 <158864704+Srinivasoo7@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Srinivas Krovvidi
2026-06-10 12:44:55 +00:00
committed by GitHub
co-authored by srinivas_oo7 Srinivasoo7 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Or Ozeri
parent 9ad08c4d15
commit 9dfc313bdc
14 changed files with 1200 additions and 213 deletions
@@ -1,11 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import Any
from unittest.mock import patch
import pytest
from prometheus_client import Counter, Gauge, Histogram
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
OffloadPromMetrics,
_MetricType,
_StatsKey,
_TransferMetricName,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import (
OffloadingConnector,
)
from vllm.v1.kv_offload.base import (
OffloadingCounterMetadata,
OffloadingGaugeMetadata,
OffloadingHistogramMetadata,
)
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
LOAD_BYTES = _TransferMetricName.LOAD_BYTES
LOAD_TIME = _TransferMetricName.LOAD_TIME
LOAD_SIZE = _TransferMetricName.LOAD_SIZE
STORE_BYTES = _TransferMetricName.STORE_BYTES
STORE_TIME = _TransferMetricName.STORE_TIME
STORE_SIZE = _TransferMetricName.STORE_SIZE
STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
PENDING_STORES = "vllm:kv_offload_pending_stores"
LOOKUP_LATENCY = "vllm:kv_offload_lookup_latency_seconds"
class _FakeMetric:
def __init__(self, **kwargs):
self.kwargs = kwargs
self.children: list[_FakeMetric] = []
self.observed: list[int | float] = []
self.increments: list[int | float] = []
self.set_values: list[int | float] = []
self.labelvalues: tuple[object, ...] = ()
def labels(self, *labelvalues):
child = _FakeMetric(**self.kwargs)
child.labelvalues = labelvalues
self.children.append(child)
return child
def observe(self, value):
self.observed.append(value)
def inc(self, value):
self.increments.append(value)
def set(self, value):
self.set_values.append(value)
class _FakeVllmConfig:
def __init__(self, store_threshold: int = 2):
self.kv_transfer_config = SimpleNamespace(
kv_connector_extra_config={"store_threshold": store_threshold}
)
def _metric_metadata():
return {
LOAD_BYTES: OffloadingCounterMetadata(
documentation="load bytes",
),
LOAD_TIME: OffloadingCounterMetadata(
documentation="load time",
),
LOAD_SIZE: OffloadingHistogramMetadata(
documentation="load size",
),
STORE_BYTES: OffloadingCounterMetadata(
documentation="store bytes",
),
STORE_TIME: OffloadingCounterMetadata(
documentation="store time",
),
STORE_SIZE: OffloadingHistogramMetadata(
documentation="store size",
),
STORES_SKIPPED: OffloadingCounterMetadata(
documentation="stores skipped",
),
PENDING_STORES: OffloadingGaugeMetadata(
documentation="pending stores",
),
LOOKUP_LATENCY: OffloadingHistogramMetadata(
documentation="lookup latency",
),
}
def test_build_kv_connector_stats_with_none():
@@ -14,7 +105,6 @@ def test_build_kv_connector_stats_with_none():
assert stats is not None
assert isinstance(stats, OffloadingConnectorStats)
assert len(stats.data) == 0
assert stats.is_empty()
@@ -24,7 +114,6 @@ def test_build_kv_connector_stats_with_empty_dict():
assert stats is not None
assert isinstance(stats, OffloadingConnectorStats)
assert len(stats.data) == 0
assert stats.is_empty()
@@ -32,114 +121,186 @@ def test_build_kv_connector_stats_reconstructs_offload_stats():
"""Test that OffloadingConnector stats are properly reconstructed with
correct data."""
serialized_data = {
"CPU_to_GPU": [
{"op_size": 16, "op_time": 1.0},
{"op_size": 8, "op_time": 0.5},
],
"GPU_to_CPU": [
{"op_size": 1, "op_time": 0.1},
{"op_size": 2, "op_time": 0.2},
],
_StatsKey.TYPES: {
LOAD_BYTES: _MetricType.COUNTER,
LOAD_TIME: _MetricType.COUNTER,
LOAD_SIZE: _MetricType.HISTOGRAM,
STORE_BYTES: _MetricType.COUNTER,
STORE_TIME: _MetricType.COUNTER,
STORE_SIZE: _MetricType.HISTOGRAM,
STORES_SKIPPED: _MetricType.COUNTER,
},
_StatsKey.DATA: {
LOAD_BYTES: 24,
LOAD_TIME: 1.5,
LOAD_SIZE: [16, 8],
STORE_BYTES: 3,
STORE_TIME: 0.3,
STORE_SIZE: [1, 2],
STORES_SKIPPED: 5,
},
}
stats = OffloadingConnector.build_kv_connector_stats(data=serialized_data)
offload_connector_stats = stats
assert isinstance(offload_connector_stats, OffloadingConnectorStats)
assert offload_connector_stats.data["CPU_to_GPU"] == [
{"op_size": 16, "op_time": 1.0},
{"op_size": 8, "op_time": 0.5},
]
assert offload_connector_stats.data["GPU_to_CPU"] == [
{"op_size": 1, "op_time": 0.1},
{"op_size": 2, "op_time": 0.2},
]
assert isinstance(stats, OffloadingConnectorStats)
values = stats.data[_StatsKey.DATA]
assert values[LOAD_BYTES] == 24
assert values[LOAD_TIME] == 1.5
assert values[LOAD_SIZE] == [16, 8]
assert values[STORE_BYTES] == 3
assert values[STORE_TIME] == 0.3
assert values[STORE_SIZE] == [1, 2]
assert values[STORES_SKIPPED] == 5
def _make_stats_data(
metric_data: dict[str, Any],
metric_metadata: dict[str, Any],
) -> dict[str, Any]:
"""Build a structured data dict from flat metric data and metadata."""
metric_types = {}
for key in metric_data:
md = metric_metadata[key]
if isinstance(md, OffloadingCounterMetadata):
metric_types[key] = _MetricType.COUNTER
elif isinstance(md, OffloadingGaugeMetadata):
metric_types[key] = _MetricType.GAUGE
elif isinstance(md, OffloadingHistogramMetadata):
metric_types[key] = _MetricType.HISTOGRAM
return {
_StatsKey.TYPES: metric_types,
_StatsKey.DATA: metric_data,
}
def test_aggregate_same_connector():
"""Test aggregating stats from the same connector type."""
metadata = _metric_metadata()
stats1 = OffloadingConnectorStats(
data={
"CPU_to_GPU": [
{"op_size": 16, "op_time": 1.0},
{"op_size": 8, "op_time": 0.5},
],
"GPU_to_CPU": [
{"op_size": 1, "op_time": 0.1},
{"op_size": 2, "op_time": 0.2},
],
}
data=_make_stats_data(
{
LOAD_BYTES: 24,
LOAD_TIME: 1.5,
LOAD_SIZE: [16, 8],
STORE_BYTES: 3,
STORE_TIME: 0.3,
STORE_SIZE: [1, 2],
STORES_SKIPPED: 1,
PENDING_STORES: 3,
LOOKUP_LATENCY: [0.1],
},
metadata,
),
)
stats2 = OffloadingConnectorStats(
data={
"CPU_to_GPU": [
{"op_size": 3, "op_time": 0.2},
{"op_size": 7, "op_time": 0.9},
],
"GPU_to_CPU": [{"op_size": 16, "op_time": 2}],
}
data=_make_stats_data(
{
LOAD_BYTES: 10,
LOAD_TIME: 1.1,
LOAD_SIZE: [3, 7],
STORE_BYTES: 16,
STORE_TIME: 2,
STORE_SIZE: [16],
STORES_SKIPPED: 3,
PENDING_STORES: 1,
LOOKUP_LATENCY: [0.2, 0.3],
},
metadata,
),
)
result = stats1.aggregate(stats2)
assert result is stats1 # Should return self
offload_connector_stats = result
assert offload_connector_stats.data["CPU_to_GPU"] == [
{"op_size": 16, "op_time": 1.0},
{"op_size": 8, "op_time": 0.5},
{"op_size": 3, "op_time": 0.2},
{"op_size": 7, "op_time": 0.9},
]
assert offload_connector_stats.data["GPU_to_CPU"] == [
{"op_size": 1, "op_time": 0.1},
{"op_size": 2, "op_time": 0.2},
{"op_size": 16, "op_time": 2},
]
values = result.data[_StatsKey.DATA]
assert values[LOAD_BYTES] == 34
assert values[LOAD_TIME] == 2.6
assert values[LOAD_SIZE] == [16, 8, 3, 7]
assert values[STORE_BYTES] == 19
assert values[STORE_TIME] == 2.3
assert values[STORE_SIZE] == [1, 2, 16]
assert values[STORES_SKIPPED] == 4
assert values[PENDING_STORES] == 1
assert values[LOOKUP_LATENCY] == [0.1, 0.2, 0.3]
def test_aggregate_merges_types():
stats1 = OffloadingConnectorStats(
data={
_StatsKey.TYPES: {LOAD_BYTES: _MetricType.COUNTER},
_StatsKey.DATA: {LOAD_BYTES: 1},
},
)
stats2 = OffloadingConnectorStats(
data={
_StatsKey.TYPES: {PENDING_STORES: _MetricType.GAUGE},
_StatsKey.DATA: {PENDING_STORES: 2},
},
)
result = stats1.aggregate(stats2)
assert result.data[_StatsKey.DATA][PENDING_STORES] == 2
assert result.data[_StatsKey.TYPES][PENDING_STORES] == _MetricType.GAUGE
def test_reduce():
"""Test that reduce() correctly reduces all nested connector stats."""
"""Test that reduce() correctly reduces connector stats."""
metadata = _metric_metadata()
stats = OffloadingConnectorStats(
data={
"CPU_to_GPU": [
{"op_size": 16, "op_time": 1.0},
{"op_size": 8, "op_time": 0.5},
{"op_size": 3, "op_time": 0.2},
{"op_size": 7, "op_time": 0.9},
],
"GPU_to_CPU": [
{"op_size": 1, "op_time": 0.1},
{"op_size": 2, "op_time": 0.2},
{"op_size": 16, "op_time": 2},
],
}
data=_make_stats_data(
{
LOAD_BYTES: 34,
LOAD_TIME: 2.6,
LOAD_SIZE: [16, 8, 3, 7],
STORE_BYTES: 19,
STORE_TIME: 2.3,
STORE_SIZE: [1, 2, 16],
STORES_SKIPPED: 11,
PENDING_STORES: 2,
LOOKUP_LATENCY: [0.1, 0.2, 0.3],
},
metadata,
),
)
reduced = stats.reduce()
assert isinstance(reduced, dict)
# Check that the stats were reduced (should have aggregated values)
assert "CPU_to_GPU_total_bytes" in reduced
assert "CPU_to_GPU_total_time" in reduced
assert "GPU_to_CPU_total_bytes" in reduced
assert "GPU_to_CPU_total_time" in reduced
assert reduced["CPU_to_GPU_total_bytes"] == 34
assert reduced["CPU_to_GPU_total_time"] == 2.6
assert reduced["GPU_to_CPU_total_time"] == 2.3
assert reduced["GPU_to_CPU_total_bytes"] == 19
assert reduced[LOAD_BYTES] == 34
assert reduced[LOAD_TIME] == 2.6
assert reduced[f"{LOAD_SIZE}_count"] == 4
assert reduced[f"{LOAD_SIZE}_sum"] == 34
assert reduced[STORE_BYTES] == 19
assert reduced[STORE_TIME] == 2.3
assert reduced[f"{STORE_SIZE}_count"] == 3
assert reduced[f"{STORE_SIZE}_sum"] == 19
assert reduced[STORES_SKIPPED] == 11
assert reduced[PENDING_STORES] == 2
assert reduced[f"{LOOKUP_LATENCY}_count"] == 3
assert reduced[f"{LOOKUP_LATENCY}_sum"] == sum([0.1, 0.2, 0.3])
def test_reset():
"""Test that reset() resets all nested connector stats."""
"""Test that reset() resets all connector stats."""
metadata = _metric_metadata()
offload_connector_stats = OffloadingConnectorStats(
data={
"CPU_to_GPU": [
{"op_size": 3, "op_time": 0.2},
{"op_size": 7, "op_time": 0.9},
],
"GPU_to_CPU": [{"op_size": 16, "op_time": 2}],
}
data=_make_stats_data(
{
LOAD_BYTES: 10,
LOAD_TIME: 1.1,
LOAD_SIZE: [3, 7],
STORE_BYTES: 16,
STORE_TIME: 2,
STORE_SIZE: [16],
STORES_SKIPPED: 4,
PENDING_STORES: 2,
LOOKUP_LATENCY: [0.1],
},
metadata,
),
)
assert not offload_connector_stats.is_empty()
@@ -148,4 +309,215 @@ def test_reset():
# After reset, stats should be empty
assert offload_connector_stats.is_empty()
assert len(offload_connector_stats.data) == 0
def test_prom_metrics_observes_manager_counter():
prom_metrics = OffloadPromMetrics(
vllm_config=_FakeVllmConfig(), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
prom_metrics.observe(
{
_StatsKey.TYPES: {STORES_SKIPPED: _MetricType.COUNTER},
_StatsKey.DATA: {STORES_SKIPPED: 7},
}
)
counter = prom_metrics.offloading_metrics[(0, STORES_SKIPPED)]
assert counter.increments == [7]
counter_def = prom_metrics._offloading_metric_defs[STORES_SKIPPED]
assert counter_def.kwargs["name"] == "vllm:kv_offload_stores_skipped"
assert counter.labelvalues == ("model", "0")
def test_prom_metrics_observes_flat_transfer_metrics_and_legacy_metrics():
prom_metrics = OffloadPromMetrics(
vllm_config=_FakeVllmConfig(), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
prom_metrics.observe(
{
_StatsKey.TYPES: {
LOAD_BYTES: _MetricType.COUNTER,
LOAD_TIME: _MetricType.COUNTER,
LOAD_SIZE: _MetricType.HISTOGRAM,
STORE_BYTES: _MetricType.COUNTER,
STORE_TIME: _MetricType.COUNTER,
STORE_SIZE: _MetricType.HISTOGRAM,
},
_StatsKey.DATA: {
LOAD_BYTES: 24,
LOAD_TIME: 1.5,
LOAD_SIZE: [16, 8],
STORE_BYTES: 3,
STORE_TIME: 0.3,
STORE_SIZE: [1, 2],
},
}
)
assert prom_metrics.offloading_metrics[(0, LOAD_BYTES)].increments == [24]
assert prom_metrics.offloading_metrics[(0, LOAD_TIME)].increments == [1.5]
assert prom_metrics.offloading_metrics[(0, LOAD_SIZE)].observed == [16, 8]
assert prom_metrics.offloading_metrics[(0, STORE_BYTES)].increments == [3]
assert prom_metrics.offloading_metrics[(0, STORE_TIME)].increments == [0.3]
assert prom_metrics.offloading_metrics[(0, STORE_SIZE)].observed == [1, 2]
assert prom_metrics.counter_kv_bytes[(0, "CPU_to_GPU")].increments == [24]
assert prom_metrics.counter_kv_transfer_time[(0, "CPU_to_GPU")].increments == [1.5]
assert prom_metrics.histogram_transfer_size[(0, "CPU_to_GPU")].observed == [16, 8]
assert prom_metrics.counter_kv_bytes[(0, "GPU_to_CPU")].increments == [3]
assert prom_metrics.counter_kv_transfer_time[(0, "GPU_to_CPU")].increments == [0.3]
assert prom_metrics.histogram_transfer_size[(0, "GPU_to_CPU")].observed == [1, 2]
def test_prom_metrics_observes_manager_gauge_and_histogram():
metric_definitions = {
PENDING_STORES: OffloadingGaugeMetadata(
documentation="Number of currently pending KV offload stores.",
),
LOOKUP_LATENCY: OffloadingHistogramMetadata(
documentation="KV offload lookup latency.",
buckets=(0.1, 1.0),
),
}
with patch.object(
CPUOffloadingSpec, "build_metric_definitions", return_value=metric_definitions
):
prom_metrics = OffloadPromMetrics(
vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
prom_metrics.observe(
{
_StatsKey.TYPES: {
PENDING_STORES: _MetricType.GAUGE,
LOOKUP_LATENCY: _MetricType.HISTOGRAM,
},
_StatsKey.DATA: {
PENDING_STORES: 5,
LOOKUP_LATENCY: [0.2, 0.4],
},
}
)
gauge = prom_metrics.offloading_metrics[(0, PENDING_STORES)]
histogram = prom_metrics.offloading_metrics[(0, LOOKUP_LATENCY)]
assert gauge.set_values == [5]
assert histogram.observed == [0.2, 0.4]
histogram_def = prom_metrics._offloading_metric_defs[LOOKUP_LATENCY]
assert histogram_def.kwargs["buckets"] == (0.1, 1.0)
def test_prom_metrics_uses_configured_manager_metrics():
prom_metrics = OffloadPromMetrics(
vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
assert STORES_SKIPPED not in prom_metrics._offloading_metric_metadata
def test_aggregate_into_empty_stats():
"""Aggregating non-empty stats into a fresh (empty) stats object works."""
empty = OffloadingConnectorStats()
assert empty.is_empty()
non_empty = OffloadingConnectorStats(
data={
_StatsKey.TYPES: {
LOAD_BYTES: _MetricType.COUNTER,
LOAD_SIZE: _MetricType.HISTOGRAM,
PENDING_STORES: _MetricType.GAUGE,
},
_StatsKey.DATA: {
LOAD_BYTES: 42,
LOAD_SIZE: [10, 20],
PENDING_STORES: 3,
},
},
)
result = empty.aggregate(non_empty)
assert result is empty
values = result.data[_StatsKey.DATA]
assert values[LOAD_BYTES] == 42
assert values[LOAD_SIZE] == [10, 20]
assert values[PENDING_STORES] == 3
def test_prom_metrics_multi_engine_routing():
"""Metrics are routed to the correct engine index."""
prom_metrics = OffloadPromMetrics(
vllm_config=_FakeVllmConfig(), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"], 1: ["model", "1"]},
)
prom_metrics.observe(
{
_StatsKey.TYPES: {LOAD_BYTES: _MetricType.COUNTER},
_StatsKey.DATA: {LOAD_BYTES: 100},
},
engine_idx=1,
)
engine0 = prom_metrics.offloading_metrics[(0, LOAD_BYTES)]
engine1 = prom_metrics.offloading_metrics[(1, LOAD_BYTES)]
assert engine0.increments == []
assert engine1.increments == [100]
def test_prom_metrics_rejects_undeclared_metric():
"""observe() asserts if a metric was never declared in metadata."""
prom_metrics = OffloadPromMetrics(
vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
with pytest.raises(AssertionError):
prom_metrics.observe(
{
_StatsKey.TYPES: {"unknown:metric": _MetricType.COUNTER},
_StatsKey.DATA: {"unknown:metric": 1},
}
)
@@ -4,7 +4,9 @@
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
DirectionalTransferStats,
OffloadingWorkerMetadata,
TransferStats,
)
pytestmark = pytest.mark.cpu_test
@@ -30,3 +32,22 @@ def test_aggregate_multiple_workers():
meta3 = OffloadingWorkerMetadata(completed_jobs={42: 1, 43: 1, 8: 1})
result = meta1.aggregate(meta2).aggregate(meta3)
assert result.completed_jobs == {42: 3, 43: 2, 7: 2, 8: 2}
def test_aggregate_transfer_stats():
meta1 = OffloadingWorkerMetadata(
transfer_stats=TransferStats(
load=DirectionalTransferStats(bytes=10, time=0.5, sizes=[10])
)
)
meta2 = OffloadingWorkerMetadata(
transfer_stats=TransferStats(
load=DirectionalTransferStats(bytes=20, time=1.0, sizes=[20, 30])
)
)
result = meta1.aggregate(meta2)
assert result.transfer_stats.load.bytes == 30
assert result.transfer_stats.load.time == 1.5
assert result.transfer_stats.load.sizes == [10, 20, 30]
@@ -5,6 +5,7 @@ import time
import msgspec
import msgspec.msgpack
import prometheus_client
import pytest
import zmq
from tqdm import tqdm
@@ -303,6 +304,167 @@ def test_cpu_offloading(
del llm
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
def test_cpu_offloading_metrics() -> None:
"""Verify that offloading Prometheus metrics (new flat and deprecated
labeled) are emitted after stores and loads."""
extra_config: dict = {
"cpu_bytes_to_use": 500 << 20,
"block_size": CPU_BLOCK_SIZES,
}
kv_transfer_config = KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
kv_connector_extra_config=extra_config,
)
llm = LLM(
model="meta-llama/Llama-3.2-1B-Instruct",
max_model_len=4096,
gpu_memory_utilization=0.5,
kv_transfer_config=kv_transfer_config,
disable_log_stats=False,
)
try:
prompt_token_ids = list(range(500))
# First generate: cold run, triggers a store to CPU.
# Use max_tokens>1 so the request is still producing output
# tokens when the async store completes and stats get drained.
# (The LLMEngine only records stats on steps with request outputs.)
llm.generate(
[TokensPrompt(prompt_token_ids=prompt_token_ids)],
SamplingParams(max_tokens=10),
use_tqdm=False,
)
# Wait for the async offload to finish, then reset GPU prefix cache
# so the next generate must load from CPU.
_wait_for_prefix_cache_reset(llm)
# Second generate: triggers a load from CPU.
# Send a short filler alongside the load prompt so there's always
# a request producing output tokens when the load stats get drained
# (the LLMEngine only records stats on steps with request outputs).
filler = TokensPrompt(prompt_token_ids=[0])
llm.generate(
[filler, TokensPrompt(prompt_token_ids=prompt_token_ids)],
SamplingParams(max_tokens=50),
use_tqdm=False,
)
# Metric helpers.
registry = prometheus_client.REGISTRY
def _get_counter_value(
name: str, labels: dict[str, str] | None = None
) -> float:
total = 0.0
for metric in registry.collect():
if metric.name == name:
for sample in metric.samples:
if sample.name != name + "_total":
continue
if labels and not all(
sample.labels.get(k) == v for k, v in labels.items()
):
continue
total += sample.value
return total
def _get_histogram_count(
name: str, labels: dict[str, str] | None = None
) -> float:
total = 0.0
for metric in registry.collect():
if metric.name == name:
for sample in metric.samples:
if sample.name != name + "_count":
continue
if labels and not all(
sample.labels.get(k) == v for k, v in labels.items()
):
continue
total += sample.value
return total
# Stats are drained asynchronously — if the transfer finishes
# after the last engine step for that generate() call, the metrics
# won't appear until a subsequent step. Retry with dummy generates
# to force additional stats drains.
deadline = time.monotonic() + _RESET_CACHE_TIMEOUT
while time.monotonic() < deadline:
store_bytes = _get_counter_value("vllm:kv_offload_store_bytes")
load_bytes = _get_counter_value("vllm:kv_offload_load_bytes")
if store_bytes > 0 and load_bytes > 0:
break
llm.generate(
[TokensPrompt(prompt_token_ids=[0])],
SamplingParams(max_tokens=1),
use_tqdm=False,
)
# New flat counter metrics
store_bytes = _get_counter_value("vllm:kv_offload_store_bytes")
assert store_bytes > 0, f"Expected store_bytes > 0, got {store_bytes}"
load_bytes = _get_counter_value("vllm:kv_offload_load_bytes")
assert load_bytes > 0, f"Expected load_bytes > 0, got {load_bytes}"
store_time = _get_counter_value("vllm:kv_offload_store_time")
assert store_time > 0, f"Expected store_time > 0, got {store_time}"
load_time = _get_counter_value("vllm:kv_offload_load_time")
assert load_time > 0, f"Expected load_time > 0, got {load_time}"
# New flat histogram metrics
store_size_count = _get_histogram_count("vllm:kv_offload_store_size")
assert store_size_count > 0, (
f"Expected store_size histogram observations > 0, got {store_size_count}"
)
load_size_count = _get_histogram_count("vllm:kv_offload_load_size")
assert load_size_count > 0, (
f"Expected load_size histogram observations > 0, got {load_size_count}"
)
# Deprecated labeled metrics — verify per transfer_type label.
load_label = {"transfer_type": "CPU_to_GPU"}
store_label = {"transfer_type": "GPU_to_CPU"}
dep_load_bytes = _get_counter_value("vllm:kv_offload_total_bytes", load_label)
assert dep_load_bytes > 0, (
f"Expected deprecated load bytes > 0, got {dep_load_bytes}"
)
dep_store_bytes = _get_counter_value("vllm:kv_offload_total_bytes", store_label)
assert dep_store_bytes > 0, (
f"Expected deprecated store bytes > 0, got {dep_store_bytes}"
)
dep_load_time = _get_counter_value("vllm:kv_offload_total_time", load_label)
assert dep_load_time > 0, (
f"Expected deprecated load time > 0, got {dep_load_time}"
)
dep_store_time = _get_counter_value("vllm:kv_offload_total_time", store_label)
assert dep_store_time > 0, (
f"Expected deprecated store time > 0, got {dep_store_time}"
)
dep_load_size = _get_histogram_count("vllm:kv_offload_size", load_label)
assert dep_load_size > 0, (
f"Expected deprecated load size observations > 0, got {dep_load_size}"
)
dep_store_size = _get_histogram_count("vllm:kv_offload_size", store_label)
assert dep_store_size > 0, (
f"Expected deprecated store size observations > 0, got {dep_store_size}"
)
# Flat and deprecated metrics must be consistent (dual-write).
assert store_bytes == dep_store_bytes
assert load_bytes == dep_load_bytes
assert store_time == dep_store_time
assert load_time == dep_load_time
assert store_size_count == dep_store_size
assert load_size_count == dep_load_size
finally:
del llm
def test_tiering_offloading() -> None:
"""Tests OffloadingConnector with TieringOffloadingSpec."""
extra_config: dict = {
+48 -7
View File
@@ -18,6 +18,8 @@ from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
def make_req_context(
req_id: str = "", kv_transfer_params: dict | None = None
@@ -29,6 +31,22 @@ def make_req_context(
_EMPTY_REQ_CTX = make_req_context()
def make_cpu_manager(
num_blocks: int = 4,
cache_policy: str = "lru",
enable_events: bool = False,
store_threshold: int = 0,
max_tracker_size: int = 64_000,
) -> CPUOffloadingManager:
return CPUOffloadingManager(
num_blocks=num_blocks,
cache_policy=cache_policy,
enable_events=enable_events,
store_threshold=store_threshold,
max_tracker_size=max_tracker_size,
)
@dataclass
class ExpectedPrepareStoreOutput:
keys_to_store: list[int]
@@ -110,7 +128,7 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
candidate to make room for [3, 4, 5]
- After complete_store([2, 3, 4, 5]), block 2 must still be present.
"""
manager = CPUOffloadingManager(
manager = make_cpu_manager(
num_blocks=4,
cache_policy=eviction_policy,
enable_events=True,
@@ -144,14 +162,37 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is True
def test_filter_reused_manager_reports_stores_skipped_counter():
manager = make_cpu_manager(
num_blocks=4,
cache_policy="lru",
store_threshold=2,
)
prepare_store_output = manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
verify_store_output(
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[],
store_block_ids=[],
evicted_keys=[],
),
)
stats = manager.get_stats()
assert stats is not None
assert stats.reduce()[STORES_SKIPPED] == 3
stats = manager.get_stats()
assert stats is not None
assert stats.reduce()[STORES_SKIPPED] == 0
def test_cpu_manager():
"""
Tests CPUOffloadingManager with lru policy.
"""
# initialize a CPU manager with a capacity of 4 blocks
cpu_manager = CPUOffloadingManager(
num_blocks=4, cache_policy="lru", enable_events=True
)
cpu_manager = make_cpu_manager(num_blocks=4, cache_policy="lru", enable_events=True)
# prepare store [1, 2]
prepare_store_output = cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -264,7 +305,7 @@ def test_cpu_manager():
def test_prepare_load_preserves_key_order():
"""block_ids[i] must correspond to keys[i] (co-indexed invariant)."""
manager = CPUOffloadingManager(num_blocks=4, cache_policy="lru")
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
key_a, key_b, key_c = to_key(0), to_key(1), to_key(2)
@@ -305,7 +346,7 @@ class TestARCPolicy:
def _make_manager(
self, num_blocks: int = 4, enable_events: bool = True
) -> tuple[CPUOffloadingManager, ARCCachePolicy]:
manager = CPUOffloadingManager(
manager = make_cpu_manager(
num_blocks=num_blocks,
cache_policy="arc",
enable_events=enable_events,
@@ -605,7 +646,7 @@ def test_filter_reused_manager():
"""
Tests CPUOffloadingManager reuse filtering (store_threshold=2).
"""
manager = CPUOffloadingManager(
manager = make_cpu_manager(
num_blocks=4,
cache_policy="lru",
enable_events=True,
@@ -11,6 +11,45 @@ from vllm.v1.kv_offload.worker.worker import TransferSpec
ReqId = str
@dataclass(slots=True)
class DirectionalTransferStats:
bytes: int = 0
time: float = 0.0
sizes: list[int | float] = field(default_factory=list)
def aggregate(
self, other: "DirectionalTransferStats"
) -> "DirectionalTransferStats":
return DirectionalTransferStats(
bytes=self.bytes + other.bytes,
time=self.time + other.time,
sizes=[*self.sizes, *other.sizes],
)
def record(self, num_bytes: int, time: float) -> None:
self.bytes += num_bytes
self.time += time
self.sizes.append(num_bytes)
def is_empty(self) -> bool:
return self.bytes == 0 and self.time == 0.0 and not self.sizes
@dataclass(slots=True)
class TransferStats:
load: DirectionalTransferStats = field(default_factory=DirectionalTransferStats)
store: DirectionalTransferStats = field(default_factory=DirectionalTransferStats)
def aggregate(self, other: "TransferStats") -> "TransferStats":
return TransferStats(
load=self.load.aggregate(other.load),
store=self.store.aggregate(other.store),
)
def is_empty(self) -> bool:
return self.load.is_empty() and self.store.is_empty()
@dataclass
class TransferJob:
"""A transfer job bundling request context with transfer spec.
@@ -43,6 +82,7 @@ class OffloadingWorkerMetadata(KVConnectorWorkerMetadata):
"""
completed_jobs: dict[int, int] = field(default_factory=dict)
transfer_stats: TransferStats = field(default_factory=TransferStats)
def mark_completed(self, job_id: int) -> None:
"""Record a transfer job completion from this worker."""
@@ -57,4 +97,7 @@ class OffloadingWorkerMetadata(KVConnectorWorkerMetadata):
for job_id, v in other.completed_jobs.items():
merged[job_id] = merged.get(job_id, 0) + v
return OffloadingWorkerMetadata(completed_jobs=merged)
return OffloadingWorkerMetadata(
completed_jobs=merged,
transfer_stats=self.transfer_stats.aggregate(other.transfer_stats),
)
@@ -10,37 +10,176 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
PromMetric,
PromMetricT,
)
from vllm.logger import init_logger
from vllm.v1.kv_offload.worker.worker import TransferType
logger = init_logger(__name__)
from vllm.v1.kv_offload.base import (
OffloadingCounterMetadata,
OffloadingGaugeMetadata,
OffloadingHistogramMetadata,
OffloadingMetricMetadata,
)
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
@dataclass
class OffloadingOperationMetrics:
op_size: int
op_time: float
class _TransferMetricName:
"""Flat metric names for GPU↔offload-medium transfer operations."""
LOAD_BYTES = "vllm:kv_offload_load_bytes"
LOAD_TIME = "vllm:kv_offload_load_time"
LOAD_SIZE = "vllm:kv_offload_load_size"
STORE_BYTES = "vllm:kv_offload_store_bytes"
STORE_TIME = "vllm:kv_offload_store_time"
STORE_SIZE = "vllm:kv_offload_store_size"
class _TransferType:
"""Transfer direction labels for deprecated CPU offload metrics."""
LOAD = "CPU_to_GPU"
STORE = "GPU_to_CPU"
ALL = (LOAD, STORE)
TRANSFER_SIZE_BUCKETS = (
1e6,
5e6,
10e6,
20e6,
40e6,
60e6,
80e6,
100e6,
150e6,
200e6,
)
def get_connector_metric_definitions() -> dict[str, OffloadingMetricMetadata]:
return {
_TransferMetricName.LOAD_BYTES: OffloadingCounterMetadata(
documentation="Total bytes loaded from offload storage to GPU.",
),
_TransferMetricName.LOAD_TIME: OffloadingCounterMetadata(
documentation="Total load time from offload storage to GPU, in seconds.",
),
_TransferMetricName.LOAD_SIZE: OffloadingHistogramMetadata(
documentation="Histogram of KV offload load operation size, in bytes.",
buckets=TRANSFER_SIZE_BUCKETS,
),
_TransferMetricName.STORE_BYTES: OffloadingCounterMetadata(
documentation="Total bytes stored from GPU to offload storage.",
),
_TransferMetricName.STORE_TIME: OffloadingCounterMetadata(
documentation="Total store time from GPU to offload storage, in seconds.",
),
_TransferMetricName.STORE_SIZE: OffloadingHistogramMetadata(
documentation="Histogram of KV offload store operation size, in bytes.",
buckets=TRANSFER_SIZE_BUCKETS,
),
}
_DEPRECATED_TOTAL_BYTES = "vllm:kv_offload_total_bytes"
_DEPRECATED_TOTAL_TIME = "vllm:kv_offload_total_time"
_DEPRECATED_SIZE = "vllm:kv_offload_size"
# Deprecated legacy transfer metrics, kept during the migration to the flat
# metric names above. These stay in a separate definition block because they
# use a transfer_type label, but are emitted from the same flat stats payload
# for compatibility.
_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS: dict[str, OffloadingMetricMetadata] = {
_DEPRECATED_TOTAL_BYTES: OffloadingCounterMetadata(
documentation="Number of bytes offloaded by KV connector",
),
_DEPRECATED_TOTAL_TIME: OffloadingCounterMetadata(
documentation="Total time measured by all KV offloading operations",
),
_DEPRECATED_SIZE: OffloadingHistogramMetadata(
documentation="Histogram of KV offload transfer size, in bytes.",
buckets=TRANSFER_SIZE_BUCKETS,
),
}
class _MetricType:
"""Type tags embedded in the serialized stats payload."""
COUNTER = "counter"
GAUGE = "gauge"
HISTOGRAM = "histogram"
class _StatsKey:
"""Top-level keys in the serialized stats dict."""
# Maps metric name -> _MetricType value
TYPES = "types"
# Maps metric name -> observed value (number or list)
DATA = "data"
@dataclass
class OffloadingConnectorStats(KVConnectorStats):
"""
Offloading connector stats use flat metric names as keys.
The ``data`` dict is structured using ``_StatsKey`` / ``_MetricType``::
{
_StatsKey.TYPES: {name: _MetricType.*, ...},
_StatsKey.DATA: {name: value, ...},
}
This structure is self-describing: it survives IPC serialization
without needing the full ``OffloadingMetricMetadata`` objects on the
receiving side.
Counter values are aggregated by summing, gauge values use the latest
snapshot, and histogram values are lists of observed samples.
"""
def __post_init__(self):
if not self.data:
# Empty container init, no data is passed in.
if _StatsKey.DATA not in self.data:
self.reset()
def reset(self):
self.data: dict[str, list[OffloadingOperationMetrics]] = {}
self.data: dict[str, Any] = {
_StatsKey.TYPES: {},
_StatsKey.DATA: {},
}
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if not other.is_empty():
for k, v in other.data.items():
if k not in self.data:
self.data[k] = v
@property
def _types(self) -> dict[str, str]:
return self.data[_StatsKey.TYPES]
@property
def _values(self) -> dict[str, Any]:
return self.data[_StatsKey.DATA]
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
if other.is_empty():
return self
assert isinstance(other, OffloadingConnectorStats)
other_types = other._types
other_values = other._values
for key, value in other_values.items():
type_str = other_types.get(key)
if type_str is None:
raise AssertionError(f"Unknown offloading stats key: {key}")
self._types.setdefault(key, type_str)
if type_str == _MetricType.HISTOGRAM:
assert isinstance(value, list)
if key not in self._values:
self._values[key] = value
else:
accumulator = self.data[k]
assert isinstance(accumulator, list)
accumulator.extend(v)
assert isinstance(self._values[key], list)
self._values[key].extend(value)
elif type_str == _MetricType.COUNTER:
assert isinstance(value, int | float)
self._values[key] = self._values.get(key, 0) + value
elif type_str == _MetricType.GAUGE:
assert isinstance(value, int | float)
self._values[key] = value
else:
raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}")
return self
def reduce(self) -> dict[str, int | float]:
@@ -51,29 +190,44 @@ class OffloadingConnectorStats(KVConnectorStats):
stats for the last time interval.
"""
return_dict: dict[str, int | float] = {}
for transfer_type, ops_list in self.data.items():
assert isinstance(ops_list, list)
total_bytes = 0
total_time = 0.0
for op in ops_list:
assert isinstance(op, dict)
total_bytes += op["op_size"]
total_time += op["op_time"]
return_dict[f"{transfer_type}_total_bytes"] = total_bytes
return_dict[f"{transfer_type}_total_time"] = total_time
for key, value in self._values.items():
type_str = self._types.get(key)
if type_str is None:
raise AssertionError(f"Unknown offloading stats key: {key}")
if type_str == _MetricType.HISTOGRAM:
assert isinstance(value, list)
return_dict[f"{key}_count"] = len(value)
return_dict[f"{key}_sum"] = sum(value)
elif type_str in (_MetricType.COUNTER, _MetricType.GAUGE):
assert isinstance(value, int | float)
return_dict[key] = value
else:
raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}")
return return_dict
def is_empty(self) -> bool:
return not self.data
return not self.data.get(_StatsKey.DATA)
def record_transfer(self, num_bytes: int, time: float, transfer_type: TransferType):
src, dst = transfer_type
transfer_type_key = src + "_to_" + dst
op = OffloadingOperationMetrics(num_bytes, time)
if transfer_type_key in self.data:
self.data[transfer_type_key].append(op)
else:
self.data[transfer_type_key] = [op]
def increase_counter(
self, counter_name: str, counter_increase_value: int | float
) -> None:
"""Increase a counter on the stats payload."""
self._types.setdefault(counter_name, _MetricType.COUNTER)
self._values[counter_name] = (
self._values.get(counter_name, 0) + counter_increase_value
)
def set_gauge(self, gauge_name: str, gauge_value: int | float) -> None:
"""Set a gauge snapshot on the stats payload."""
self._types.setdefault(gauge_name, _MetricType.GAUGE)
self._values[gauge_name] = gauge_value
def observe_histogram(
self, histogram_name: str, histogram_value: int | float
) -> None:
"""Record a histogram observation on the stats payload."""
self._types.setdefault(histogram_name, _MetricType.HISTOGRAM)
self._values.setdefault(histogram_name, []).append(histogram_value)
class OffloadPromMetrics(KVConnectorPromMetrics):
@@ -89,77 +243,145 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
self.histogram_transfer_size: dict[tuple[int, str], PromMetricT] = {}
self.counter_kv_bytes: dict[tuple[int, str], PromMetricT] = {}
self.counter_kv_transfer_time: dict[tuple[int, str], PromMetricT] = {}
buckets = [ # In bytes
1e6,
5e6,
10e6,
20e6,
40e6,
60e6,
80e6,
100e6,
150e6,
200e6,
]
spec_cls = OffloadingSpecFactory.get_spec_cls(vllm_config)
kv_transfer_config = vllm_config.kv_transfer_config
assert kv_transfer_config is not None
extra_config = kv_transfer_config.kv_connector_extra_config
self._offloading_metric_metadata: dict[str, OffloadingMetricMetadata] = {
**spec_cls.build_metric_definitions(extra_config),
**get_connector_metric_definitions(),
}
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
self._observe_deprecated_metrics = issubclass(spec_cls, CPUOffloadingSpec)
self._offloading_metric_defs: dict[str, PromMetricT] = {}
self.offloading_metrics: dict[tuple[int, str], PromMetricT] = {}
self._counter_kv_bytes = self._counter_cls(
name="vllm:kv_offload_total_bytes",
documentation="Number of bytes offloaded by KV connector",
name=_DEPRECATED_TOTAL_BYTES,
documentation=_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[
_DEPRECATED_TOTAL_BYTES
].documentation,
labelnames=labelnames + ["transfer_type"],
)
self._counter_kv_transfer_time = self._counter_cls(
name="vllm:kv_offload_total_time",
documentation="Total time measured by all KV offloading operations",
name=_DEPRECATED_TOTAL_TIME,
documentation=_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[
_DEPRECATED_TOTAL_TIME
].documentation,
labelnames=labelnames + ["transfer_type"],
)
deprecated_size_metadata = _DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[
_DEPRECATED_SIZE
]
assert isinstance(deprecated_size_metadata, OffloadingHistogramMetadata)
self._histogram_transfer_size = self._histogram_cls(
name="vllm:kv_offload_size",
documentation="Histogram of KV offload transfer size, in bytes.",
buckets=buckets[:],
name=_DEPRECATED_SIZE,
documentation=deprecated_size_metadata.documentation,
buckets=deprecated_size_metadata.buckets,
labelnames=labelnames + ["transfer_type"],
)
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
"""
Observe transfer statistics from the new data structure.
transfer_stats_data is expected to be a dict where:
- keys are transfer type strings (e.g., "cpu_to_gpu", "gpu_to_cpu")
- values are lists of OffloadingOperationMetrics objects
"""
for transfer_type, ops in transfer_stats_data.items():
# Cache:
if (engine_idx, transfer_type) not in self.histogram_transfer_size:
for engine_idx, labelvalues in per_engine_labelvalues.items():
for transfer_type in _TransferType.ALL:
bounded_labelvalues = labelvalues + [transfer_type]
self.histogram_transfer_size[(engine_idx, transfer_type)] = (
self._histogram_transfer_size.labels(
*(self.per_engine_labelvalues[engine_idx] + [transfer_type])
)
self._histogram_transfer_size.labels(*bounded_labelvalues)
)
self.counter_kv_bytes[(engine_idx, transfer_type)] = (
self._counter_kv_bytes.labels(
*(self.per_engine_labelvalues[engine_idx] + [transfer_type])
)
self._counter_kv_bytes.labels(*bounded_labelvalues)
)
self.counter_kv_transfer_time[(engine_idx, transfer_type)] = (
self._counter_kv_transfer_time.labels(
*(self.per_engine_labelvalues[engine_idx] + [transfer_type])
)
self._counter_kv_transfer_time.labels(*bounded_labelvalues)
)
# Process ops:
assert isinstance(ops, list)
for op in ops: # ops is a list of serialized OffloadingOperationMetrics
assert isinstance(op, dict)
# Observe size histogram
self.histogram_transfer_size[(engine_idx, transfer_type)].observe(
op["op_size"]
for metric_name, metadata in self._offloading_metric_metadata.items():
self._offloading_metric_defs[metric_name] = self._create_metric(
metric_name, metadata
)
for engine_idx, labelvalues in per_engine_labelvalues.items():
self.offloading_metrics[(engine_idx, metric_name)] = (
self._offloading_metric_defs[metric_name].labels(*labelvalues)
)
# Increment byte and time counters
self.counter_kv_bytes[(engine_idx, transfer_type)].inc(op["op_size"])
def _create_metric(
self, metric_name: str, metadata: OffloadingMetricMetadata
) -> Any:
kwargs: dict[str, Any] = {
"name": metric_name,
"documentation": metadata.documentation,
"labelnames": self._labelnames,
}
if isinstance(metadata, OffloadingCounterMetadata):
metric_cls = self._counter_cls
elif isinstance(metadata, OffloadingGaugeMetadata):
metric_cls = self._gauge_cls
elif isinstance(metadata, OffloadingHistogramMetadata):
metric_cls = self._histogram_cls
if metadata.buckets is not None:
kwargs["buckets"] = metadata.buckets
else:
raise AssertionError(f"Unknown offloading metric metadata: {metadata}")
return metric_cls(**kwargs)
self.counter_kv_transfer_time[(engine_idx, transfer_type)].inc(
op["op_time"]
def _increase_counter(
self, metric_name: str, value: int | float, engine_idx: int
) -> None:
self.offloading_metrics[(engine_idx, metric_name)].inc(value)
if not self._observe_deprecated_metrics:
return
# Keep deprecated CPU offload transfer metrics updated during the
# transition to flat metric names.
if metric_name == _TransferMetricName.LOAD_BYTES:
self.counter_kv_bytes[(engine_idx, _TransferType.LOAD)].inc(value)
elif metric_name == _TransferMetricName.LOAD_TIME:
self.counter_kv_transfer_time[(engine_idx, _TransferType.LOAD)].inc(value)
elif metric_name == _TransferMetricName.STORE_BYTES:
self.counter_kv_bytes[(engine_idx, _TransferType.STORE)].inc(value)
elif metric_name == _TransferMetricName.STORE_TIME:
self.counter_kv_transfer_time[(engine_idx, _TransferType.STORE)].inc(value)
def _set_gauge(self, metric_name: str, value: int | float, engine_idx: int) -> None:
self.offloading_metrics[(engine_idx, metric_name)].set(value)
def _observe_histogram(
self, metric_name: str, value: list[int | float], engine_idx: int
) -> None:
for observation in value:
self.offloading_metrics[(engine_idx, metric_name)].observe(observation)
if not self._observe_deprecated_metrics:
continue
# Keep deprecated CPU offload transfer metrics updated during the
# transition to flat metric names.
if metric_name == _TransferMetricName.LOAD_SIZE:
self.histogram_transfer_size[(engine_idx, _TransferType.LOAD)].observe(
observation
)
elif metric_name == _TransferMetricName.STORE_SIZE:
self.histogram_transfer_size[(engine_idx, _TransferType.STORE)].observe(
observation
)
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
"""Observe transfer statistics."""
metric_types = transfer_stats_data.get(_StatsKey.TYPES, {})
metric_data = transfer_stats_data.get(_StatsKey.DATA, {})
for key, value in metric_data.items():
type_str = metric_types.get(key)
if type_str is None:
raise AssertionError(f"Unknown offloading stats key: {key}")
assert key in self._offloading_metric_defs
if type_str == _MetricType.COUNTER:
assert isinstance(value, int | float)
self._increase_counter(key, value, engine_idx)
elif type_str == _MetricType.GAUGE:
assert isinstance(value, int | float)
self._set_gauge(key, value, engine_idx)
elif type_str == _MetricType.HISTOGRAM:
assert isinstance(value, list)
assert all(isinstance(v, int | float) for v in value)
self._observe_histogram(key, value, engine_idx)
else:
raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}")
@@ -14,6 +14,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
ReqId,
TransferJob,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
_TransferMetricName,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
@@ -259,9 +263,13 @@ def _create_req_context(req: Request) -> ReqContext:
class OffloadingConnectorScheduler:
"""Implementation of Scheduler side methods"""
def __init__(self, spec: OffloadingSpec):
def __init__(
self,
spec: OffloadingSpec,
):
self.config = SchedulerOffloadConfig.from_spec(spec)
self.manager: OffloadingManager = spec.get_manager()
self._connector_stats: OffloadingConnectorStats | None = None
full_attention_groups: list[int] = []
sliding_window_groups: list[int] = []
@@ -952,6 +960,39 @@ class OffloadingConnectorScheduler:
if not isinstance(meta, OffloadingWorkerMetadata):
assert meta is None
meta = OffloadingWorkerMetadata()
if not meta.transfer_stats.is_empty():
transfer_stats = OffloadingConnectorStats()
if not meta.transfer_stats.load.is_empty():
transfer_stats.increase_counter(
_TransferMetricName.LOAD_BYTES,
meta.transfer_stats.load.bytes,
)
transfer_stats.increase_counter(
_TransferMetricName.LOAD_TIME,
meta.transfer_stats.load.time,
)
for size in meta.transfer_stats.load.sizes:
transfer_stats.observe_histogram(
_TransferMetricName.LOAD_SIZE, size
)
if not meta.transfer_stats.store.is_empty():
transfer_stats.increase_counter(
_TransferMetricName.STORE_BYTES,
meta.transfer_stats.store.bytes,
)
transfer_stats.increase_counter(
_TransferMetricName.STORE_TIME,
meta.transfer_stats.store.time,
)
for size in meta.transfer_stats.store.sizes:
transfer_stats.observe_histogram(
_TransferMetricName.STORE_SIZE, size
)
if self._connector_stats is None:
self._connector_stats = transfer_stats
else:
self._connector_stats.aggregate(transfer_stats)
for job_id, count in meta.completed_jobs.items():
assert count > 0
if job_id < self._stale_job_threshold:
@@ -990,6 +1031,19 @@ class OffloadingConnectorScheduler:
if not req_status.transfer_jobs and req_status.req.is_finished():
del self._req_status[job_status.req_id]
def get_stats(self) -> OffloadingConnectorStats | None:
stats = self._connector_stats
self._connector_stats = None
manager_stats = self.manager.get_stats()
if manager_stats is not None:
if stats is None:
stats = manager_stats
else:
stats.aggregate(manager_stats)
return stats
def request_finished(
self,
request: Request,
@@ -5,17 +5,11 @@ from dataclasses import replace
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorStats,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingConnectorMetadata,
OffloadingWorkerMetadata,
ReqId,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
)
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.kv_cache_interface import (
@@ -44,7 +38,6 @@ class OffloadingConnectorWorker:
self.spec = spec
self.worker = OffloadingWorker()
self.kv_connector_stats = OffloadingConnectorStats()
# job_id -> req_id for in-flight loads.
self._load_jobs: dict[int, ReqId] = {}
self._unsubmitted_store_jobs: list[tuple[int, TransferSpec]] = []
@@ -271,15 +264,18 @@ class OffloadingConnectorWorker:
# we currently do not support job failures
job_id = transfer_result.job_id
assert transfer_result.success
is_load = job_id in self._load_jobs
if (
transfer_result.transfer_time
transfer_result.transfer_time is not None
and transfer_result.transfer_size is not None
and transfer_result.transfer_type is not None
):
self.kv_connector_stats.record_transfer(
num_bytes=transfer_result.transfer_size,
time=transfer_result.transfer_time,
transfer_type=transfer_result.transfer_type,
if is_load:
stats = self._connector_worker_meta.transfer_stats.load
else:
stats = self._connector_worker_meta.transfer_stats.store
stats.record(
transfer_result.transfer_size,
transfer_result.transfer_time,
)
self._connector_worker_meta.mark_completed(job_id)
@@ -297,18 +293,6 @@ class OffloadingConnectorWorker:
self._connector_worker_meta = OffloadingWorkerMetadata()
return meta
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""
Get the KV transfer stats for the connector.
"""
if self.kv_connector_stats.is_empty():
return None
# Clear stats for next iteration
kv_connector_stats = self.kv_connector_stats
self.kv_connector_stats = OffloadingConnectorStats()
return kv_connector_stats
def shutdown(self) -> None:
self._unsubmitted_store_jobs.clear()
self._load_jobs.clear()
@@ -184,9 +184,14 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
return True
def get_kv_connector_stats(self) -> KVConnectorStats | None:
if self.connector_worker is None:
return None # We only emit stats from the worker-side
return self.connector_worker.get_kv_connector_stats()
if self.connector_scheduler is not None:
return self.connector_scheduler.get_stats()
# TODO(orozery): Remove once PR #43877 lands
if self.connector_worker is not None:
return OffloadingConnectorStats()
return None
@classmethod
def build_kv_connector_stats(
+34
View File
@@ -19,6 +19,9 @@ from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
)
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.kv_offload.worker.worker import OffloadingHandler
@@ -123,6 +126,26 @@ The class provides the following primitives:
"""
@dataclass(frozen=True)
class OffloadingMetricMetadata:
documentation: str
@dataclass(frozen=True)
class OffloadingCounterMetadata(OffloadingMetricMetadata):
pass
@dataclass(frozen=True)
class OffloadingGaugeMetadata(OffloadingMetricMetadata):
pass
@dataclass(frozen=True)
class OffloadingHistogramMetadata(OffloadingMetricMetadata):
buckets: tuple[float, ...] | None = None
class OffloadingManager(ABC):
@abstractmethod
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
@@ -269,6 +292,10 @@ class OffloadingManager(ABC):
"""Evict all tracked blocks and reset internal state."""
return
def get_stats(self) -> "OffloadingConnectorStats | None":
"""Return collected metrics since last call, or None if disabled."""
return None
def shutdown(self) -> None:
"""Shutdown the manager and release any resources."""
return
@@ -378,6 +405,13 @@ class CanonicalKVCaches:
class OffloadingSpec(ABC):
"""Spec for an offloading connector"""
@classmethod
def build_metric_definitions(
cls, extra_config: dict[str, Any]
) -> dict[str, "OffloadingMetricMetadata"]:
"""Return Prometheus metric definitions emitted by this spec."""
return {}
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
logger.warning(
"Initializing OffloadingSpec. This API is experimental and "
+2
View File
@@ -4,6 +4,8 @@ from typing_extensions import override
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
METRIC_STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
"""
+19 -1
View File
@@ -6,6 +6,9 @@ from typing import Literal
from typing_extensions import override
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
)
from vllm.v1.kv_offload.base import (
LoadStoreSpec,
OffloadingEvent,
@@ -15,7 +18,7 @@ from vllm.v1.kv_offload.base import (
ReqContext,
RequestOffloadingContext,
)
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
from vllm.v1.kv_offload.cpu.common import METRIC_STORES_SKIPPED, CPULoadStoreSpec
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
@@ -58,6 +61,7 @@ class CPUOffloadingManager(OffloadingManager):
self._policy: CachePolicy = policy_cls(cache_capacity=num_blocks)
self.store_threshold: int = store_threshold
self.max_tracker_size: int = max_tracker_size
self.stores_skipped_in_current_batch: int = 0
# Number of block references. It is ordered so can evict the LRU entry in O(1).
self.counts: OrderedDict[OffloadKey, int] | None = (
@@ -154,7 +158,9 @@ class CPUOffloadingManager(OffloadingManager):
req_context: ReqContext,
) -> PrepareStoreOutput | None:
if self.counts is not None:
num_keys = len(keys)
keys = [k for k in keys if self.counts.get(k, 0) >= self.store_threshold]
self.stores_skipped_in_current_batch += num_keys - len(keys)
# filter out blocks that are already stored
keys_to_store = [k for k in keys if self._policy.get(k) is None]
@@ -253,3 +259,15 @@ class CPUOffloadingManager(OffloadingManager):
if self.events is not None:
yield from self.events
self.events.clear()
def get_stats(self) -> OffloadingConnectorStats | None:
if self.store_threshold < 2:
return None
stats = OffloadingConnectorStats()
stats.increase_counter(
METRIC_STORES_SKIPPED,
self.stores_skipped_in_current_batch,
)
self.stores_skipped_in_current_batch = 0
return stats
+20 -1
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
from typing import Any
from typing_extensions import override
@@ -12,10 +13,12 @@ from vllm.v1.kv_offload.base import (
CanonicalKVCaches,
GPULoadStoreSpec,
LoadStoreSpec,
OffloadingCounterMetadata,
OffloadingManager,
OffloadingMetricMetadata,
OffloadingSpec,
)
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
from vllm.v1.kv_offload.cpu.common import METRIC_STORES_SKIPPED, CPULoadStoreSpec
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
from vllm.v1.kv_offload.worker.worker import OffloadingHandler
@@ -24,6 +27,22 @@ from vllm.v1.kv_offload.worker.worker import OffloadingHandler
class CPUOffloadingSpec(OffloadingSpec):
BLOCK_SIZE_ALIGNMENT = 1
@classmethod
def build_metric_definitions(
cls, extra_config: dict[str, Any]
) -> dict[str, OffloadingMetricMetadata]:
store_threshold = int(extra_config.get("store_threshold", 0))
if store_threshold < 2:
return {}
return {
METRIC_STORES_SKIPPED: OffloadingCounterMetadata(
documentation=(
"Number of KV offload stores skipped because the reuse "
"threshold was not reached."
),
)
}
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig):
super().__init__(vllm_config, kv_cache_config)
+15 -5
View File
@@ -30,11 +30,7 @@ class OffloadingSpecFactory:
cls._registry[name] = loader
@classmethod
def create_spec(
cls,
config: "VllmConfig",
kv_cache_config: "KVCacheConfig",
) -> OffloadingSpec:
def get_spec_cls(cls, config: "VllmConfig") -> type[OffloadingSpec]:
kv_transfer_config = config.kv_transfer_config
assert kv_transfer_config is not None
extra_config = kv_transfer_config.kv_connector_extra_config
@@ -48,6 +44,20 @@ class OffloadingSpecFactory:
spec_module = importlib.import_module(spec_module_path)
spec_cls = getattr(spec_module, spec_name)
assert issubclass(spec_cls, OffloadingSpec)
return spec_cls
@classmethod
def create_spec(
cls,
config: "VllmConfig",
kv_cache_config: "KVCacheConfig",
) -> OffloadingSpec:
kv_transfer_config = config.kv_transfer_config
assert kv_transfer_config is not None
spec_name = kv_transfer_config.kv_connector_extra_config.get(
"spec_name", "CPUOffloadingSpec"
)
spec_cls = cls.get_spec_cls(config)
logger.info("Creating offloading spec with name: %s", spec_name)
return spec_cls(config, kv_cache_config)