Feature/offloading manager stats (#35669)
Signed-off-by: Sriusa4414@gmail.com Signed-off-by: srinivas_oo7 <Sriusa4414@gmail.com> Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com> Signed-off-by: Srinivasoo7 <158864704+Srinivasoo7@users.noreply.github.com> Signed-off-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com> Co-authored-by: Srinivasoo7 <158864704+Srinivasoo7@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
srinivas_oo7
Srinivasoo7
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Or Ozeri
parent
9ad08c4d15
commit
9dfc313bdc
@@ -1,11 +1,102 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from prometheus_client import Counter, Gauge, Histogram
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
OffloadPromMetrics,
|
||||
_MetricType,
|
||||
_StatsKey,
|
||||
_TransferMetricName,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import (
|
||||
OffloadingConnector,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
OffloadingCounterMetadata,
|
||||
OffloadingGaugeMetadata,
|
||||
OffloadingHistogramMetadata,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
|
||||
LOAD_BYTES = _TransferMetricName.LOAD_BYTES
|
||||
LOAD_TIME = _TransferMetricName.LOAD_TIME
|
||||
LOAD_SIZE = _TransferMetricName.LOAD_SIZE
|
||||
STORE_BYTES = _TransferMetricName.STORE_BYTES
|
||||
STORE_TIME = _TransferMetricName.STORE_TIME
|
||||
STORE_SIZE = _TransferMetricName.STORE_SIZE
|
||||
STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
|
||||
PENDING_STORES = "vllm:kv_offload_pending_stores"
|
||||
LOOKUP_LATENCY = "vllm:kv_offload_lookup_latency_seconds"
|
||||
|
||||
|
||||
class _FakeMetric:
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
self.children: list[_FakeMetric] = []
|
||||
self.observed: list[int | float] = []
|
||||
self.increments: list[int | float] = []
|
||||
self.set_values: list[int | float] = []
|
||||
self.labelvalues: tuple[object, ...] = ()
|
||||
|
||||
def labels(self, *labelvalues):
|
||||
child = _FakeMetric(**self.kwargs)
|
||||
child.labelvalues = labelvalues
|
||||
self.children.append(child)
|
||||
return child
|
||||
|
||||
def observe(self, value):
|
||||
self.observed.append(value)
|
||||
|
||||
def inc(self, value):
|
||||
self.increments.append(value)
|
||||
|
||||
def set(self, value):
|
||||
self.set_values.append(value)
|
||||
|
||||
|
||||
class _FakeVllmConfig:
|
||||
def __init__(self, store_threshold: int = 2):
|
||||
self.kv_transfer_config = SimpleNamespace(
|
||||
kv_connector_extra_config={"store_threshold": store_threshold}
|
||||
)
|
||||
|
||||
|
||||
def _metric_metadata():
|
||||
return {
|
||||
LOAD_BYTES: OffloadingCounterMetadata(
|
||||
documentation="load bytes",
|
||||
),
|
||||
LOAD_TIME: OffloadingCounterMetadata(
|
||||
documentation="load time",
|
||||
),
|
||||
LOAD_SIZE: OffloadingHistogramMetadata(
|
||||
documentation="load size",
|
||||
),
|
||||
STORE_BYTES: OffloadingCounterMetadata(
|
||||
documentation="store bytes",
|
||||
),
|
||||
STORE_TIME: OffloadingCounterMetadata(
|
||||
documentation="store time",
|
||||
),
|
||||
STORE_SIZE: OffloadingHistogramMetadata(
|
||||
documentation="store size",
|
||||
),
|
||||
STORES_SKIPPED: OffloadingCounterMetadata(
|
||||
documentation="stores skipped",
|
||||
),
|
||||
PENDING_STORES: OffloadingGaugeMetadata(
|
||||
documentation="pending stores",
|
||||
),
|
||||
LOOKUP_LATENCY: OffloadingHistogramMetadata(
|
||||
documentation="lookup latency",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def test_build_kv_connector_stats_with_none():
|
||||
@@ -14,7 +105,6 @@ def test_build_kv_connector_stats_with_none():
|
||||
|
||||
assert stats is not None
|
||||
assert isinstance(stats, OffloadingConnectorStats)
|
||||
assert len(stats.data) == 0
|
||||
assert stats.is_empty()
|
||||
|
||||
|
||||
@@ -24,7 +114,6 @@ def test_build_kv_connector_stats_with_empty_dict():
|
||||
|
||||
assert stats is not None
|
||||
assert isinstance(stats, OffloadingConnectorStats)
|
||||
assert len(stats.data) == 0
|
||||
assert stats.is_empty()
|
||||
|
||||
|
||||
@@ -32,114 +121,186 @@ def test_build_kv_connector_stats_reconstructs_offload_stats():
|
||||
"""Test that OffloadingConnector stats are properly reconstructed with
|
||||
correct data."""
|
||||
serialized_data = {
|
||||
"CPU_to_GPU": [
|
||||
{"op_size": 16, "op_time": 1.0},
|
||||
{"op_size": 8, "op_time": 0.5},
|
||||
],
|
||||
"GPU_to_CPU": [
|
||||
{"op_size": 1, "op_time": 0.1},
|
||||
{"op_size": 2, "op_time": 0.2},
|
||||
],
|
||||
_StatsKey.TYPES: {
|
||||
LOAD_BYTES: _MetricType.COUNTER,
|
||||
LOAD_TIME: _MetricType.COUNTER,
|
||||
LOAD_SIZE: _MetricType.HISTOGRAM,
|
||||
STORE_BYTES: _MetricType.COUNTER,
|
||||
STORE_TIME: _MetricType.COUNTER,
|
||||
STORE_SIZE: _MetricType.HISTOGRAM,
|
||||
STORES_SKIPPED: _MetricType.COUNTER,
|
||||
},
|
||||
_StatsKey.DATA: {
|
||||
LOAD_BYTES: 24,
|
||||
LOAD_TIME: 1.5,
|
||||
LOAD_SIZE: [16, 8],
|
||||
STORE_BYTES: 3,
|
||||
STORE_TIME: 0.3,
|
||||
STORE_SIZE: [1, 2],
|
||||
STORES_SKIPPED: 5,
|
||||
},
|
||||
}
|
||||
|
||||
stats = OffloadingConnector.build_kv_connector_stats(data=serialized_data)
|
||||
|
||||
offload_connector_stats = stats
|
||||
assert isinstance(offload_connector_stats, OffloadingConnectorStats)
|
||||
assert offload_connector_stats.data["CPU_to_GPU"] == [
|
||||
{"op_size": 16, "op_time": 1.0},
|
||||
{"op_size": 8, "op_time": 0.5},
|
||||
]
|
||||
assert offload_connector_stats.data["GPU_to_CPU"] == [
|
||||
{"op_size": 1, "op_time": 0.1},
|
||||
{"op_size": 2, "op_time": 0.2},
|
||||
]
|
||||
assert isinstance(stats, OffloadingConnectorStats)
|
||||
values = stats.data[_StatsKey.DATA]
|
||||
assert values[LOAD_BYTES] == 24
|
||||
assert values[LOAD_TIME] == 1.5
|
||||
assert values[LOAD_SIZE] == [16, 8]
|
||||
assert values[STORE_BYTES] == 3
|
||||
assert values[STORE_TIME] == 0.3
|
||||
assert values[STORE_SIZE] == [1, 2]
|
||||
assert values[STORES_SKIPPED] == 5
|
||||
|
||||
|
||||
def _make_stats_data(
|
||||
metric_data: dict[str, Any],
|
||||
metric_metadata: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Build a structured data dict from flat metric data and metadata."""
|
||||
metric_types = {}
|
||||
for key in metric_data:
|
||||
md = metric_metadata[key]
|
||||
if isinstance(md, OffloadingCounterMetadata):
|
||||
metric_types[key] = _MetricType.COUNTER
|
||||
elif isinstance(md, OffloadingGaugeMetadata):
|
||||
metric_types[key] = _MetricType.GAUGE
|
||||
elif isinstance(md, OffloadingHistogramMetadata):
|
||||
metric_types[key] = _MetricType.HISTOGRAM
|
||||
return {
|
||||
_StatsKey.TYPES: metric_types,
|
||||
_StatsKey.DATA: metric_data,
|
||||
}
|
||||
|
||||
|
||||
def test_aggregate_same_connector():
|
||||
"""Test aggregating stats from the same connector type."""
|
||||
metadata = _metric_metadata()
|
||||
stats1 = OffloadingConnectorStats(
|
||||
data={
|
||||
"CPU_to_GPU": [
|
||||
{"op_size": 16, "op_time": 1.0},
|
||||
{"op_size": 8, "op_time": 0.5},
|
||||
],
|
||||
"GPU_to_CPU": [
|
||||
{"op_size": 1, "op_time": 0.1},
|
||||
{"op_size": 2, "op_time": 0.2},
|
||||
],
|
||||
}
|
||||
data=_make_stats_data(
|
||||
{
|
||||
LOAD_BYTES: 24,
|
||||
LOAD_TIME: 1.5,
|
||||
LOAD_SIZE: [16, 8],
|
||||
STORE_BYTES: 3,
|
||||
STORE_TIME: 0.3,
|
||||
STORE_SIZE: [1, 2],
|
||||
STORES_SKIPPED: 1,
|
||||
PENDING_STORES: 3,
|
||||
LOOKUP_LATENCY: [0.1],
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
)
|
||||
|
||||
stats2 = OffloadingConnectorStats(
|
||||
data={
|
||||
"CPU_to_GPU": [
|
||||
{"op_size": 3, "op_time": 0.2},
|
||||
{"op_size": 7, "op_time": 0.9},
|
||||
],
|
||||
"GPU_to_CPU": [{"op_size": 16, "op_time": 2}],
|
||||
}
|
||||
data=_make_stats_data(
|
||||
{
|
||||
LOAD_BYTES: 10,
|
||||
LOAD_TIME: 1.1,
|
||||
LOAD_SIZE: [3, 7],
|
||||
STORE_BYTES: 16,
|
||||
STORE_TIME: 2,
|
||||
STORE_SIZE: [16],
|
||||
STORES_SKIPPED: 3,
|
||||
PENDING_STORES: 1,
|
||||
LOOKUP_LATENCY: [0.2, 0.3],
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
)
|
||||
|
||||
result = stats1.aggregate(stats2)
|
||||
|
||||
assert result is stats1 # Should return self
|
||||
offload_connector_stats = result
|
||||
assert offload_connector_stats.data["CPU_to_GPU"] == [
|
||||
{"op_size": 16, "op_time": 1.0},
|
||||
{"op_size": 8, "op_time": 0.5},
|
||||
{"op_size": 3, "op_time": 0.2},
|
||||
{"op_size": 7, "op_time": 0.9},
|
||||
]
|
||||
assert offload_connector_stats.data["GPU_to_CPU"] == [
|
||||
{"op_size": 1, "op_time": 0.1},
|
||||
{"op_size": 2, "op_time": 0.2},
|
||||
{"op_size": 16, "op_time": 2},
|
||||
]
|
||||
values = result.data[_StatsKey.DATA]
|
||||
assert values[LOAD_BYTES] == 34
|
||||
assert values[LOAD_TIME] == 2.6
|
||||
assert values[LOAD_SIZE] == [16, 8, 3, 7]
|
||||
assert values[STORE_BYTES] == 19
|
||||
assert values[STORE_TIME] == 2.3
|
||||
assert values[STORE_SIZE] == [1, 2, 16]
|
||||
assert values[STORES_SKIPPED] == 4
|
||||
assert values[PENDING_STORES] == 1
|
||||
assert values[LOOKUP_LATENCY] == [0.1, 0.2, 0.3]
|
||||
|
||||
|
||||
def test_aggregate_merges_types():
|
||||
stats1 = OffloadingConnectorStats(
|
||||
data={
|
||||
_StatsKey.TYPES: {LOAD_BYTES: _MetricType.COUNTER},
|
||||
_StatsKey.DATA: {LOAD_BYTES: 1},
|
||||
},
|
||||
)
|
||||
stats2 = OffloadingConnectorStats(
|
||||
data={
|
||||
_StatsKey.TYPES: {PENDING_STORES: _MetricType.GAUGE},
|
||||
_StatsKey.DATA: {PENDING_STORES: 2},
|
||||
},
|
||||
)
|
||||
|
||||
result = stats1.aggregate(stats2)
|
||||
|
||||
assert result.data[_StatsKey.DATA][PENDING_STORES] == 2
|
||||
assert result.data[_StatsKey.TYPES][PENDING_STORES] == _MetricType.GAUGE
|
||||
|
||||
|
||||
def test_reduce():
|
||||
"""Test that reduce() correctly reduces all nested connector stats."""
|
||||
"""Test that reduce() correctly reduces connector stats."""
|
||||
metadata = _metric_metadata()
|
||||
stats = OffloadingConnectorStats(
|
||||
data={
|
||||
"CPU_to_GPU": [
|
||||
{"op_size": 16, "op_time": 1.0},
|
||||
{"op_size": 8, "op_time": 0.5},
|
||||
{"op_size": 3, "op_time": 0.2},
|
||||
{"op_size": 7, "op_time": 0.9},
|
||||
],
|
||||
"GPU_to_CPU": [
|
||||
{"op_size": 1, "op_time": 0.1},
|
||||
{"op_size": 2, "op_time": 0.2},
|
||||
{"op_size": 16, "op_time": 2},
|
||||
],
|
||||
}
|
||||
data=_make_stats_data(
|
||||
{
|
||||
LOAD_BYTES: 34,
|
||||
LOAD_TIME: 2.6,
|
||||
LOAD_SIZE: [16, 8, 3, 7],
|
||||
STORE_BYTES: 19,
|
||||
STORE_TIME: 2.3,
|
||||
STORE_SIZE: [1, 2, 16],
|
||||
STORES_SKIPPED: 11,
|
||||
PENDING_STORES: 2,
|
||||
LOOKUP_LATENCY: [0.1, 0.2, 0.3],
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
)
|
||||
|
||||
reduced = stats.reduce()
|
||||
|
||||
assert isinstance(reduced, dict)
|
||||
# Check that the stats were reduced (should have aggregated values)
|
||||
assert "CPU_to_GPU_total_bytes" in reduced
|
||||
assert "CPU_to_GPU_total_time" in reduced
|
||||
assert "GPU_to_CPU_total_bytes" in reduced
|
||||
assert "GPU_to_CPU_total_time" in reduced
|
||||
assert reduced["CPU_to_GPU_total_bytes"] == 34
|
||||
assert reduced["CPU_to_GPU_total_time"] == 2.6
|
||||
assert reduced["GPU_to_CPU_total_time"] == 2.3
|
||||
assert reduced["GPU_to_CPU_total_bytes"] == 19
|
||||
assert reduced[LOAD_BYTES] == 34
|
||||
assert reduced[LOAD_TIME] == 2.6
|
||||
assert reduced[f"{LOAD_SIZE}_count"] == 4
|
||||
assert reduced[f"{LOAD_SIZE}_sum"] == 34
|
||||
assert reduced[STORE_BYTES] == 19
|
||||
assert reduced[STORE_TIME] == 2.3
|
||||
assert reduced[f"{STORE_SIZE}_count"] == 3
|
||||
assert reduced[f"{STORE_SIZE}_sum"] == 19
|
||||
assert reduced[STORES_SKIPPED] == 11
|
||||
assert reduced[PENDING_STORES] == 2
|
||||
assert reduced[f"{LOOKUP_LATENCY}_count"] == 3
|
||||
assert reduced[f"{LOOKUP_LATENCY}_sum"] == sum([0.1, 0.2, 0.3])
|
||||
|
||||
|
||||
def test_reset():
|
||||
"""Test that reset() resets all nested connector stats."""
|
||||
"""Test that reset() resets all connector stats."""
|
||||
metadata = _metric_metadata()
|
||||
offload_connector_stats = OffloadingConnectorStats(
|
||||
data={
|
||||
"CPU_to_GPU": [
|
||||
{"op_size": 3, "op_time": 0.2},
|
||||
{"op_size": 7, "op_time": 0.9},
|
||||
],
|
||||
"GPU_to_CPU": [{"op_size": 16, "op_time": 2}],
|
||||
}
|
||||
data=_make_stats_data(
|
||||
{
|
||||
LOAD_BYTES: 10,
|
||||
LOAD_TIME: 1.1,
|
||||
LOAD_SIZE: [3, 7],
|
||||
STORE_BYTES: 16,
|
||||
STORE_TIME: 2,
|
||||
STORE_SIZE: [16],
|
||||
STORES_SKIPPED: 4,
|
||||
PENDING_STORES: 2,
|
||||
LOOKUP_LATENCY: [0.1],
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
)
|
||||
|
||||
assert not offload_connector_stats.is_empty()
|
||||
@@ -148,4 +309,215 @@ def test_reset():
|
||||
|
||||
# After reset, stats should be empty
|
||||
assert offload_connector_stats.is_empty()
|
||||
assert len(offload_connector_stats.data) == 0
|
||||
|
||||
|
||||
def test_prom_metrics_observes_manager_counter():
|
||||
prom_metrics = OffloadPromMetrics(
|
||||
vllm_config=_FakeVllmConfig(), # type: ignore[arg-type]
|
||||
metric_types={
|
||||
Gauge: _FakeMetric,
|
||||
Counter: _FakeMetric,
|
||||
Histogram: _FakeMetric,
|
||||
},
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues={0: ["model", "0"]},
|
||||
)
|
||||
|
||||
prom_metrics.observe(
|
||||
{
|
||||
_StatsKey.TYPES: {STORES_SKIPPED: _MetricType.COUNTER},
|
||||
_StatsKey.DATA: {STORES_SKIPPED: 7},
|
||||
}
|
||||
)
|
||||
|
||||
counter = prom_metrics.offloading_metrics[(0, STORES_SKIPPED)]
|
||||
assert counter.increments == [7]
|
||||
counter_def = prom_metrics._offloading_metric_defs[STORES_SKIPPED]
|
||||
assert counter_def.kwargs["name"] == "vllm:kv_offload_stores_skipped"
|
||||
assert counter.labelvalues == ("model", "0")
|
||||
|
||||
|
||||
def test_prom_metrics_observes_flat_transfer_metrics_and_legacy_metrics():
|
||||
prom_metrics = OffloadPromMetrics(
|
||||
vllm_config=_FakeVllmConfig(), # type: ignore[arg-type]
|
||||
metric_types={
|
||||
Gauge: _FakeMetric,
|
||||
Counter: _FakeMetric,
|
||||
Histogram: _FakeMetric,
|
||||
},
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues={0: ["model", "0"]},
|
||||
)
|
||||
|
||||
prom_metrics.observe(
|
||||
{
|
||||
_StatsKey.TYPES: {
|
||||
LOAD_BYTES: _MetricType.COUNTER,
|
||||
LOAD_TIME: _MetricType.COUNTER,
|
||||
LOAD_SIZE: _MetricType.HISTOGRAM,
|
||||
STORE_BYTES: _MetricType.COUNTER,
|
||||
STORE_TIME: _MetricType.COUNTER,
|
||||
STORE_SIZE: _MetricType.HISTOGRAM,
|
||||
},
|
||||
_StatsKey.DATA: {
|
||||
LOAD_BYTES: 24,
|
||||
LOAD_TIME: 1.5,
|
||||
LOAD_SIZE: [16, 8],
|
||||
STORE_BYTES: 3,
|
||||
STORE_TIME: 0.3,
|
||||
STORE_SIZE: [1, 2],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert prom_metrics.offloading_metrics[(0, LOAD_BYTES)].increments == [24]
|
||||
assert prom_metrics.offloading_metrics[(0, LOAD_TIME)].increments == [1.5]
|
||||
assert prom_metrics.offloading_metrics[(0, LOAD_SIZE)].observed == [16, 8]
|
||||
assert prom_metrics.offloading_metrics[(0, STORE_BYTES)].increments == [3]
|
||||
assert prom_metrics.offloading_metrics[(0, STORE_TIME)].increments == [0.3]
|
||||
assert prom_metrics.offloading_metrics[(0, STORE_SIZE)].observed == [1, 2]
|
||||
|
||||
assert prom_metrics.counter_kv_bytes[(0, "CPU_to_GPU")].increments == [24]
|
||||
assert prom_metrics.counter_kv_transfer_time[(0, "CPU_to_GPU")].increments == [1.5]
|
||||
assert prom_metrics.histogram_transfer_size[(0, "CPU_to_GPU")].observed == [16, 8]
|
||||
assert prom_metrics.counter_kv_bytes[(0, "GPU_to_CPU")].increments == [3]
|
||||
assert prom_metrics.counter_kv_transfer_time[(0, "GPU_to_CPU")].increments == [0.3]
|
||||
assert prom_metrics.histogram_transfer_size[(0, "GPU_to_CPU")].observed == [1, 2]
|
||||
|
||||
|
||||
def test_prom_metrics_observes_manager_gauge_and_histogram():
|
||||
metric_definitions = {
|
||||
PENDING_STORES: OffloadingGaugeMetadata(
|
||||
documentation="Number of currently pending KV offload stores.",
|
||||
),
|
||||
LOOKUP_LATENCY: OffloadingHistogramMetadata(
|
||||
documentation="KV offload lookup latency.",
|
||||
buckets=(0.1, 1.0),
|
||||
),
|
||||
}
|
||||
with patch.object(
|
||||
CPUOffloadingSpec, "build_metric_definitions", return_value=metric_definitions
|
||||
):
|
||||
prom_metrics = OffloadPromMetrics(
|
||||
vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type]
|
||||
metric_types={
|
||||
Gauge: _FakeMetric,
|
||||
Counter: _FakeMetric,
|
||||
Histogram: _FakeMetric,
|
||||
},
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues={0: ["model", "0"]},
|
||||
)
|
||||
|
||||
prom_metrics.observe(
|
||||
{
|
||||
_StatsKey.TYPES: {
|
||||
PENDING_STORES: _MetricType.GAUGE,
|
||||
LOOKUP_LATENCY: _MetricType.HISTOGRAM,
|
||||
},
|
||||
_StatsKey.DATA: {
|
||||
PENDING_STORES: 5,
|
||||
LOOKUP_LATENCY: [0.2, 0.4],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
gauge = prom_metrics.offloading_metrics[(0, PENDING_STORES)]
|
||||
histogram = prom_metrics.offloading_metrics[(0, LOOKUP_LATENCY)]
|
||||
assert gauge.set_values == [5]
|
||||
assert histogram.observed == [0.2, 0.4]
|
||||
histogram_def = prom_metrics._offloading_metric_defs[LOOKUP_LATENCY]
|
||||
assert histogram_def.kwargs["buckets"] == (0.1, 1.0)
|
||||
|
||||
|
||||
def test_prom_metrics_uses_configured_manager_metrics():
|
||||
prom_metrics = OffloadPromMetrics(
|
||||
vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type]
|
||||
metric_types={
|
||||
Gauge: _FakeMetric,
|
||||
Counter: _FakeMetric,
|
||||
Histogram: _FakeMetric,
|
||||
},
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues={0: ["model", "0"]},
|
||||
)
|
||||
|
||||
assert STORES_SKIPPED not in prom_metrics._offloading_metric_metadata
|
||||
|
||||
|
||||
def test_aggregate_into_empty_stats():
|
||||
"""Aggregating non-empty stats into a fresh (empty) stats object works."""
|
||||
empty = OffloadingConnectorStats()
|
||||
assert empty.is_empty()
|
||||
|
||||
non_empty = OffloadingConnectorStats(
|
||||
data={
|
||||
_StatsKey.TYPES: {
|
||||
LOAD_BYTES: _MetricType.COUNTER,
|
||||
LOAD_SIZE: _MetricType.HISTOGRAM,
|
||||
PENDING_STORES: _MetricType.GAUGE,
|
||||
},
|
||||
_StatsKey.DATA: {
|
||||
LOAD_BYTES: 42,
|
||||
LOAD_SIZE: [10, 20],
|
||||
PENDING_STORES: 3,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
result = empty.aggregate(non_empty)
|
||||
|
||||
assert result is empty
|
||||
values = result.data[_StatsKey.DATA]
|
||||
assert values[LOAD_BYTES] == 42
|
||||
assert values[LOAD_SIZE] == [10, 20]
|
||||
assert values[PENDING_STORES] == 3
|
||||
|
||||
|
||||
def test_prom_metrics_multi_engine_routing():
|
||||
"""Metrics are routed to the correct engine index."""
|
||||
prom_metrics = OffloadPromMetrics(
|
||||
vllm_config=_FakeVllmConfig(), # type: ignore[arg-type]
|
||||
metric_types={
|
||||
Gauge: _FakeMetric,
|
||||
Counter: _FakeMetric,
|
||||
Histogram: _FakeMetric,
|
||||
},
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues={0: ["model", "0"], 1: ["model", "1"]},
|
||||
)
|
||||
|
||||
prom_metrics.observe(
|
||||
{
|
||||
_StatsKey.TYPES: {LOAD_BYTES: _MetricType.COUNTER},
|
||||
_StatsKey.DATA: {LOAD_BYTES: 100},
|
||||
},
|
||||
engine_idx=1,
|
||||
)
|
||||
|
||||
engine0 = prom_metrics.offloading_metrics[(0, LOAD_BYTES)]
|
||||
engine1 = prom_metrics.offloading_metrics[(1, LOAD_BYTES)]
|
||||
assert engine0.increments == []
|
||||
assert engine1.increments == [100]
|
||||
|
||||
|
||||
def test_prom_metrics_rejects_undeclared_metric():
|
||||
"""observe() asserts if a metric was never declared in metadata."""
|
||||
prom_metrics = OffloadPromMetrics(
|
||||
vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type]
|
||||
metric_types={
|
||||
Gauge: _FakeMetric,
|
||||
Counter: _FakeMetric,
|
||||
Histogram: _FakeMetric,
|
||||
},
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues={0: ["model", "0"]},
|
||||
)
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
prom_metrics.observe(
|
||||
{
|
||||
_StatsKey.TYPES: {"unknown:metric": _MetricType.COUNTER},
|
||||
_StatsKey.DATA: {"unknown:metric": 1},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
DirectionalTransferStats,
|
||||
OffloadingWorkerMetadata,
|
||||
TransferStats,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
@@ -30,3 +32,22 @@ def test_aggregate_multiple_workers():
|
||||
meta3 = OffloadingWorkerMetadata(completed_jobs={42: 1, 43: 1, 8: 1})
|
||||
result = meta1.aggregate(meta2).aggregate(meta3)
|
||||
assert result.completed_jobs == {42: 3, 43: 2, 7: 2, 8: 2}
|
||||
|
||||
|
||||
def test_aggregate_transfer_stats():
|
||||
meta1 = OffloadingWorkerMetadata(
|
||||
transfer_stats=TransferStats(
|
||||
load=DirectionalTransferStats(bytes=10, time=0.5, sizes=[10])
|
||||
)
|
||||
)
|
||||
meta2 = OffloadingWorkerMetadata(
|
||||
transfer_stats=TransferStats(
|
||||
load=DirectionalTransferStats(bytes=20, time=1.0, sizes=[20, 30])
|
||||
)
|
||||
)
|
||||
|
||||
result = meta1.aggregate(meta2)
|
||||
|
||||
assert result.transfer_stats.load.bytes == 30
|
||||
assert result.transfer_stats.load.time == 1.5
|
||||
assert result.transfer_stats.load.sizes == [10, 20, 30]
|
||||
|
||||
@@ -5,6 +5,7 @@ import time
|
||||
|
||||
import msgspec
|
||||
import msgspec.msgpack
|
||||
import prometheus_client
|
||||
import pytest
|
||||
import zmq
|
||||
from tqdm import tqdm
|
||||
@@ -303,6 +304,167 @@ def test_cpu_offloading(
|
||||
del llm
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
def test_cpu_offloading_metrics() -> None:
|
||||
"""Verify that offloading Prometheus metrics (new flat and deprecated
|
||||
labeled) are emitted after stores and loads."""
|
||||
extra_config: dict = {
|
||||
"cpu_bytes_to_use": 500 << 20,
|
||||
"block_size": CPU_BLOCK_SIZES,
|
||||
}
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="OffloadingConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config=extra_config,
|
||||
)
|
||||
|
||||
llm = LLM(
|
||||
model="meta-llama/Llama-3.2-1B-Instruct",
|
||||
max_model_len=4096,
|
||||
gpu_memory_utilization=0.5,
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
disable_log_stats=False,
|
||||
)
|
||||
|
||||
try:
|
||||
prompt_token_ids = list(range(500))
|
||||
|
||||
# First generate: cold run, triggers a store to CPU.
|
||||
# Use max_tokens>1 so the request is still producing output
|
||||
# tokens when the async store completes and stats get drained.
|
||||
# (The LLMEngine only records stats on steps with request outputs.)
|
||||
llm.generate(
|
||||
[TokensPrompt(prompt_token_ids=prompt_token_ids)],
|
||||
SamplingParams(max_tokens=10),
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
# Wait for the async offload to finish, then reset GPU prefix cache
|
||||
# so the next generate must load from CPU.
|
||||
_wait_for_prefix_cache_reset(llm)
|
||||
|
||||
# Second generate: triggers a load from CPU.
|
||||
# Send a short filler alongside the load prompt so there's always
|
||||
# a request producing output tokens when the load stats get drained
|
||||
# (the LLMEngine only records stats on steps with request outputs).
|
||||
filler = TokensPrompt(prompt_token_ids=[0])
|
||||
llm.generate(
|
||||
[filler, TokensPrompt(prompt_token_ids=prompt_token_ids)],
|
||||
SamplingParams(max_tokens=50),
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
# Metric helpers.
|
||||
registry = prometheus_client.REGISTRY
|
||||
|
||||
def _get_counter_value(
|
||||
name: str, labels: dict[str, str] | None = None
|
||||
) -> float:
|
||||
total = 0.0
|
||||
for metric in registry.collect():
|
||||
if metric.name == name:
|
||||
for sample in metric.samples:
|
||||
if sample.name != name + "_total":
|
||||
continue
|
||||
if labels and not all(
|
||||
sample.labels.get(k) == v for k, v in labels.items()
|
||||
):
|
||||
continue
|
||||
total += sample.value
|
||||
return total
|
||||
|
||||
def _get_histogram_count(
|
||||
name: str, labels: dict[str, str] | None = None
|
||||
) -> float:
|
||||
total = 0.0
|
||||
for metric in registry.collect():
|
||||
if metric.name == name:
|
||||
for sample in metric.samples:
|
||||
if sample.name != name + "_count":
|
||||
continue
|
||||
if labels and not all(
|
||||
sample.labels.get(k) == v for k, v in labels.items()
|
||||
):
|
||||
continue
|
||||
total += sample.value
|
||||
return total
|
||||
|
||||
# Stats are drained asynchronously — if the transfer finishes
|
||||
# after the last engine step for that generate() call, the metrics
|
||||
# won't appear until a subsequent step. Retry with dummy generates
|
||||
# to force additional stats drains.
|
||||
deadline = time.monotonic() + _RESET_CACHE_TIMEOUT
|
||||
while time.monotonic() < deadline:
|
||||
store_bytes = _get_counter_value("vllm:kv_offload_store_bytes")
|
||||
load_bytes = _get_counter_value("vllm:kv_offload_load_bytes")
|
||||
if store_bytes > 0 and load_bytes > 0:
|
||||
break
|
||||
llm.generate(
|
||||
[TokensPrompt(prompt_token_ids=[0])],
|
||||
SamplingParams(max_tokens=1),
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
# New flat counter metrics
|
||||
store_bytes = _get_counter_value("vllm:kv_offload_store_bytes")
|
||||
assert store_bytes > 0, f"Expected store_bytes > 0, got {store_bytes}"
|
||||
load_bytes = _get_counter_value("vllm:kv_offload_load_bytes")
|
||||
assert load_bytes > 0, f"Expected load_bytes > 0, got {load_bytes}"
|
||||
store_time = _get_counter_value("vllm:kv_offload_store_time")
|
||||
assert store_time > 0, f"Expected store_time > 0, got {store_time}"
|
||||
load_time = _get_counter_value("vllm:kv_offload_load_time")
|
||||
assert load_time > 0, f"Expected load_time > 0, got {load_time}"
|
||||
|
||||
# New flat histogram metrics
|
||||
store_size_count = _get_histogram_count("vllm:kv_offload_store_size")
|
||||
assert store_size_count > 0, (
|
||||
f"Expected store_size histogram observations > 0, got {store_size_count}"
|
||||
)
|
||||
load_size_count = _get_histogram_count("vllm:kv_offload_load_size")
|
||||
assert load_size_count > 0, (
|
||||
f"Expected load_size histogram observations > 0, got {load_size_count}"
|
||||
)
|
||||
|
||||
# Deprecated labeled metrics — verify per transfer_type label.
|
||||
load_label = {"transfer_type": "CPU_to_GPU"}
|
||||
store_label = {"transfer_type": "GPU_to_CPU"}
|
||||
|
||||
dep_load_bytes = _get_counter_value("vllm:kv_offload_total_bytes", load_label)
|
||||
assert dep_load_bytes > 0, (
|
||||
f"Expected deprecated load bytes > 0, got {dep_load_bytes}"
|
||||
)
|
||||
dep_store_bytes = _get_counter_value("vllm:kv_offload_total_bytes", store_label)
|
||||
assert dep_store_bytes > 0, (
|
||||
f"Expected deprecated store bytes > 0, got {dep_store_bytes}"
|
||||
)
|
||||
dep_load_time = _get_counter_value("vllm:kv_offload_total_time", load_label)
|
||||
assert dep_load_time > 0, (
|
||||
f"Expected deprecated load time > 0, got {dep_load_time}"
|
||||
)
|
||||
dep_store_time = _get_counter_value("vllm:kv_offload_total_time", store_label)
|
||||
assert dep_store_time > 0, (
|
||||
f"Expected deprecated store time > 0, got {dep_store_time}"
|
||||
)
|
||||
dep_load_size = _get_histogram_count("vllm:kv_offload_size", load_label)
|
||||
assert dep_load_size > 0, (
|
||||
f"Expected deprecated load size observations > 0, got {dep_load_size}"
|
||||
)
|
||||
dep_store_size = _get_histogram_count("vllm:kv_offload_size", store_label)
|
||||
assert dep_store_size > 0, (
|
||||
f"Expected deprecated store size observations > 0, got {dep_store_size}"
|
||||
)
|
||||
|
||||
# Flat and deprecated metrics must be consistent (dual-write).
|
||||
assert store_bytes == dep_store_bytes
|
||||
assert load_bytes == dep_load_bytes
|
||||
assert store_time == dep_store_time
|
||||
assert load_time == dep_load_time
|
||||
assert store_size_count == dep_store_size
|
||||
assert load_size_count == dep_load_size
|
||||
finally:
|
||||
del llm
|
||||
|
||||
|
||||
def test_tiering_offloading() -> None:
|
||||
"""Tests OffloadingConnector with TieringOffloadingSpec."""
|
||||
extra_config: dict = {
|
||||
|
||||
@@ -18,6 +18,8 @@ from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
||||
|
||||
STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
|
||||
|
||||
|
||||
def make_req_context(
|
||||
req_id: str = "", kv_transfer_params: dict | None = None
|
||||
@@ -29,6 +31,22 @@ def make_req_context(
|
||||
_EMPTY_REQ_CTX = make_req_context()
|
||||
|
||||
|
||||
def make_cpu_manager(
|
||||
num_blocks: int = 4,
|
||||
cache_policy: str = "lru",
|
||||
enable_events: bool = False,
|
||||
store_threshold: int = 0,
|
||||
max_tracker_size: int = 64_000,
|
||||
) -> CPUOffloadingManager:
|
||||
return CPUOffloadingManager(
|
||||
num_blocks=num_blocks,
|
||||
cache_policy=cache_policy,
|
||||
enable_events=enable_events,
|
||||
store_threshold=store_threshold,
|
||||
max_tracker_size=max_tracker_size,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExpectedPrepareStoreOutput:
|
||||
keys_to_store: list[int]
|
||||
@@ -110,7 +128,7 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
|
||||
candidate to make room for [3, 4, 5]
|
||||
- After complete_store([2, 3, 4, 5]), block 2 must still be present.
|
||||
"""
|
||||
manager = CPUOffloadingManager(
|
||||
manager = make_cpu_manager(
|
||||
num_blocks=4,
|
||||
cache_policy=eviction_policy,
|
||||
enable_events=True,
|
||||
@@ -144,14 +162,37 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
|
||||
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is True
|
||||
|
||||
|
||||
def test_filter_reused_manager_reports_stores_skipped_counter():
|
||||
manager = make_cpu_manager(
|
||||
num_blocks=4,
|
||||
cache_policy="lru",
|
||||
store_threshold=2,
|
||||
)
|
||||
|
||||
prepare_store_output = manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
|
||||
|
||||
verify_store_output(
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[],
|
||||
store_block_ids=[],
|
||||
evicted_keys=[],
|
||||
),
|
||||
)
|
||||
stats = manager.get_stats()
|
||||
assert stats is not None
|
||||
assert stats.reduce()[STORES_SKIPPED] == 3
|
||||
stats = manager.get_stats()
|
||||
assert stats is not None
|
||||
assert stats.reduce()[STORES_SKIPPED] == 0
|
||||
|
||||
|
||||
def test_cpu_manager():
|
||||
"""
|
||||
Tests CPUOffloadingManager with lru policy.
|
||||
"""
|
||||
# initialize a CPU manager with a capacity of 4 blocks
|
||||
cpu_manager = CPUOffloadingManager(
|
||||
num_blocks=4, cache_policy="lru", enable_events=True
|
||||
)
|
||||
cpu_manager = make_cpu_manager(num_blocks=4, cache_policy="lru", enable_events=True)
|
||||
|
||||
# prepare store [1, 2]
|
||||
prepare_store_output = cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
@@ -264,7 +305,7 @@ def test_cpu_manager():
|
||||
|
||||
def test_prepare_load_preserves_key_order():
|
||||
"""block_ids[i] must correspond to keys[i] (co-indexed invariant)."""
|
||||
manager = CPUOffloadingManager(num_blocks=4, cache_policy="lru")
|
||||
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
|
||||
|
||||
key_a, key_b, key_c = to_key(0), to_key(1), to_key(2)
|
||||
|
||||
@@ -305,7 +346,7 @@ class TestARCPolicy:
|
||||
def _make_manager(
|
||||
self, num_blocks: int = 4, enable_events: bool = True
|
||||
) -> tuple[CPUOffloadingManager, ARCCachePolicy]:
|
||||
manager = CPUOffloadingManager(
|
||||
manager = make_cpu_manager(
|
||||
num_blocks=num_blocks,
|
||||
cache_policy="arc",
|
||||
enable_events=enable_events,
|
||||
@@ -605,7 +646,7 @@ def test_filter_reused_manager():
|
||||
"""
|
||||
Tests CPUOffloadingManager reuse filtering (store_threshold=2).
|
||||
"""
|
||||
manager = CPUOffloadingManager(
|
||||
manager = make_cpu_manager(
|
||||
num_blocks=4,
|
||||
cache_policy="lru",
|
||||
enable_events=True,
|
||||
|
||||
@@ -11,6 +11,45 @@ from vllm.v1.kv_offload.worker.worker import TransferSpec
|
||||
ReqId = str
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class DirectionalTransferStats:
|
||||
bytes: int = 0
|
||||
time: float = 0.0
|
||||
sizes: list[int | float] = field(default_factory=list)
|
||||
|
||||
def aggregate(
|
||||
self, other: "DirectionalTransferStats"
|
||||
) -> "DirectionalTransferStats":
|
||||
return DirectionalTransferStats(
|
||||
bytes=self.bytes + other.bytes,
|
||||
time=self.time + other.time,
|
||||
sizes=[*self.sizes, *other.sizes],
|
||||
)
|
||||
|
||||
def record(self, num_bytes: int, time: float) -> None:
|
||||
self.bytes += num_bytes
|
||||
self.time += time
|
||||
self.sizes.append(num_bytes)
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return self.bytes == 0 and self.time == 0.0 and not self.sizes
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class TransferStats:
|
||||
load: DirectionalTransferStats = field(default_factory=DirectionalTransferStats)
|
||||
store: DirectionalTransferStats = field(default_factory=DirectionalTransferStats)
|
||||
|
||||
def aggregate(self, other: "TransferStats") -> "TransferStats":
|
||||
return TransferStats(
|
||||
load=self.load.aggregate(other.load),
|
||||
store=self.store.aggregate(other.store),
|
||||
)
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return self.load.is_empty() and self.store.is_empty()
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransferJob:
|
||||
"""A transfer job bundling request context with transfer spec.
|
||||
@@ -43,6 +82,7 @@ class OffloadingWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
"""
|
||||
|
||||
completed_jobs: dict[int, int] = field(default_factory=dict)
|
||||
transfer_stats: TransferStats = field(default_factory=TransferStats)
|
||||
|
||||
def mark_completed(self, job_id: int) -> None:
|
||||
"""Record a transfer job completion from this worker."""
|
||||
@@ -57,4 +97,7 @@ class OffloadingWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
for job_id, v in other.completed_jobs.items():
|
||||
merged[job_id] = merged.get(job_id, 0) + v
|
||||
|
||||
return OffloadingWorkerMetadata(completed_jobs=merged)
|
||||
return OffloadingWorkerMetadata(
|
||||
completed_jobs=merged,
|
||||
transfer_stats=self.transfer_stats.aggregate(other.transfer_stats),
|
||||
)
|
||||
|
||||
@@ -10,37 +10,176 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.worker.worker import TransferType
|
||||
|
||||
logger = init_logger(__name__)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
OffloadingCounterMetadata,
|
||||
OffloadingGaugeMetadata,
|
||||
OffloadingHistogramMetadata,
|
||||
OffloadingMetricMetadata,
|
||||
)
|
||||
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
|
||||
|
||||
|
||||
@dataclass
|
||||
class OffloadingOperationMetrics:
|
||||
op_size: int
|
||||
op_time: float
|
||||
class _TransferMetricName:
|
||||
"""Flat metric names for GPU↔offload-medium transfer operations."""
|
||||
|
||||
LOAD_BYTES = "vllm:kv_offload_load_bytes"
|
||||
LOAD_TIME = "vllm:kv_offload_load_time"
|
||||
LOAD_SIZE = "vllm:kv_offload_load_size"
|
||||
STORE_BYTES = "vllm:kv_offload_store_bytes"
|
||||
STORE_TIME = "vllm:kv_offload_store_time"
|
||||
STORE_SIZE = "vllm:kv_offload_store_size"
|
||||
|
||||
|
||||
class _TransferType:
|
||||
"""Transfer direction labels for deprecated CPU offload metrics."""
|
||||
|
||||
LOAD = "CPU_to_GPU"
|
||||
STORE = "GPU_to_CPU"
|
||||
ALL = (LOAD, STORE)
|
||||
|
||||
|
||||
TRANSFER_SIZE_BUCKETS = (
|
||||
1e6,
|
||||
5e6,
|
||||
10e6,
|
||||
20e6,
|
||||
40e6,
|
||||
60e6,
|
||||
80e6,
|
||||
100e6,
|
||||
150e6,
|
||||
200e6,
|
||||
)
|
||||
|
||||
|
||||
def get_connector_metric_definitions() -> dict[str, OffloadingMetricMetadata]:
|
||||
return {
|
||||
_TransferMetricName.LOAD_BYTES: OffloadingCounterMetadata(
|
||||
documentation="Total bytes loaded from offload storage to GPU.",
|
||||
),
|
||||
_TransferMetricName.LOAD_TIME: OffloadingCounterMetadata(
|
||||
documentation="Total load time from offload storage to GPU, in seconds.",
|
||||
),
|
||||
_TransferMetricName.LOAD_SIZE: OffloadingHistogramMetadata(
|
||||
documentation="Histogram of KV offload load operation size, in bytes.",
|
||||
buckets=TRANSFER_SIZE_BUCKETS,
|
||||
),
|
||||
_TransferMetricName.STORE_BYTES: OffloadingCounterMetadata(
|
||||
documentation="Total bytes stored from GPU to offload storage.",
|
||||
),
|
||||
_TransferMetricName.STORE_TIME: OffloadingCounterMetadata(
|
||||
documentation="Total store time from GPU to offload storage, in seconds.",
|
||||
),
|
||||
_TransferMetricName.STORE_SIZE: OffloadingHistogramMetadata(
|
||||
documentation="Histogram of KV offload store operation size, in bytes.",
|
||||
buckets=TRANSFER_SIZE_BUCKETS,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
_DEPRECATED_TOTAL_BYTES = "vllm:kv_offload_total_bytes"
|
||||
_DEPRECATED_TOTAL_TIME = "vllm:kv_offload_total_time"
|
||||
_DEPRECATED_SIZE = "vllm:kv_offload_size"
|
||||
|
||||
# Deprecated legacy transfer metrics, kept during the migration to the flat
|
||||
# metric names above. These stay in a separate definition block because they
|
||||
# use a transfer_type label, but are emitted from the same flat stats payload
|
||||
# for compatibility.
|
||||
_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS: dict[str, OffloadingMetricMetadata] = {
|
||||
_DEPRECATED_TOTAL_BYTES: OffloadingCounterMetadata(
|
||||
documentation="Number of bytes offloaded by KV connector",
|
||||
),
|
||||
_DEPRECATED_TOTAL_TIME: OffloadingCounterMetadata(
|
||||
documentation="Total time measured by all KV offloading operations",
|
||||
),
|
||||
_DEPRECATED_SIZE: OffloadingHistogramMetadata(
|
||||
documentation="Histogram of KV offload transfer size, in bytes.",
|
||||
buckets=TRANSFER_SIZE_BUCKETS,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class _MetricType:
|
||||
"""Type tags embedded in the serialized stats payload."""
|
||||
|
||||
COUNTER = "counter"
|
||||
GAUGE = "gauge"
|
||||
HISTOGRAM = "histogram"
|
||||
|
||||
|
||||
class _StatsKey:
|
||||
"""Top-level keys in the serialized stats dict."""
|
||||
|
||||
# Maps metric name -> _MetricType value
|
||||
TYPES = "types"
|
||||
# Maps metric name -> observed value (number or list)
|
||||
DATA = "data"
|
||||
|
||||
|
||||
@dataclass
|
||||
class OffloadingConnectorStats(KVConnectorStats):
|
||||
"""
|
||||
Offloading connector stats use flat metric names as keys.
|
||||
|
||||
The ``data`` dict is structured using ``_StatsKey`` / ``_MetricType``::
|
||||
|
||||
{
|
||||
_StatsKey.TYPES: {name: _MetricType.*, ...},
|
||||
_StatsKey.DATA: {name: value, ...},
|
||||
}
|
||||
|
||||
This structure is self-describing: it survives IPC serialization
|
||||
without needing the full ``OffloadingMetricMetadata`` objects on the
|
||||
receiving side.
|
||||
|
||||
Counter values are aggregated by summing, gauge values use the latest
|
||||
snapshot, and histogram values are lists of observed samples.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
# Empty container init, no data is passed in.
|
||||
if _StatsKey.DATA not in self.data:
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.data: dict[str, list[OffloadingOperationMetrics]] = {}
|
||||
self.data: dict[str, Any] = {
|
||||
_StatsKey.TYPES: {},
|
||||
_StatsKey.DATA: {},
|
||||
}
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if not other.is_empty():
|
||||
for k, v in other.data.items():
|
||||
if k not in self.data:
|
||||
self.data[k] = v
|
||||
@property
|
||||
def _types(self) -> dict[str, str]:
|
||||
return self.data[_StatsKey.TYPES]
|
||||
|
||||
@property
|
||||
def _values(self) -> dict[str, Any]:
|
||||
return self.data[_StatsKey.DATA]
|
||||
|
||||
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
|
||||
if other.is_empty():
|
||||
return self
|
||||
assert isinstance(other, OffloadingConnectorStats)
|
||||
other_types = other._types
|
||||
other_values = other._values
|
||||
for key, value in other_values.items():
|
||||
type_str = other_types.get(key)
|
||||
if type_str is None:
|
||||
raise AssertionError(f"Unknown offloading stats key: {key}")
|
||||
self._types.setdefault(key, type_str)
|
||||
if type_str == _MetricType.HISTOGRAM:
|
||||
assert isinstance(value, list)
|
||||
if key not in self._values:
|
||||
self._values[key] = value
|
||||
else:
|
||||
accumulator = self.data[k]
|
||||
assert isinstance(accumulator, list)
|
||||
accumulator.extend(v)
|
||||
assert isinstance(self._values[key], list)
|
||||
self._values[key].extend(value)
|
||||
elif type_str == _MetricType.COUNTER:
|
||||
assert isinstance(value, int | float)
|
||||
self._values[key] = self._values.get(key, 0) + value
|
||||
elif type_str == _MetricType.GAUGE:
|
||||
assert isinstance(value, int | float)
|
||||
self._values[key] = value
|
||||
else:
|
||||
raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}")
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
@@ -51,29 +190,44 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
stats for the last time interval.
|
||||
"""
|
||||
return_dict: dict[str, int | float] = {}
|
||||
for transfer_type, ops_list in self.data.items():
|
||||
assert isinstance(ops_list, list)
|
||||
total_bytes = 0
|
||||
total_time = 0.0
|
||||
for op in ops_list:
|
||||
assert isinstance(op, dict)
|
||||
total_bytes += op["op_size"]
|
||||
total_time += op["op_time"]
|
||||
return_dict[f"{transfer_type}_total_bytes"] = total_bytes
|
||||
return_dict[f"{transfer_type}_total_time"] = total_time
|
||||
for key, value in self._values.items():
|
||||
type_str = self._types.get(key)
|
||||
if type_str is None:
|
||||
raise AssertionError(f"Unknown offloading stats key: {key}")
|
||||
if type_str == _MetricType.HISTOGRAM:
|
||||
assert isinstance(value, list)
|
||||
return_dict[f"{key}_count"] = len(value)
|
||||
return_dict[f"{key}_sum"] = sum(value)
|
||||
elif type_str in (_MetricType.COUNTER, _MetricType.GAUGE):
|
||||
assert isinstance(value, int | float)
|
||||
return_dict[key] = value
|
||||
else:
|
||||
raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}")
|
||||
return return_dict
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return not self.data
|
||||
return not self.data.get(_StatsKey.DATA)
|
||||
|
||||
def record_transfer(self, num_bytes: int, time: float, transfer_type: TransferType):
|
||||
src, dst = transfer_type
|
||||
transfer_type_key = src + "_to_" + dst
|
||||
op = OffloadingOperationMetrics(num_bytes, time)
|
||||
if transfer_type_key in self.data:
|
||||
self.data[transfer_type_key].append(op)
|
||||
else:
|
||||
self.data[transfer_type_key] = [op]
|
||||
def increase_counter(
|
||||
self, counter_name: str, counter_increase_value: int | float
|
||||
) -> None:
|
||||
"""Increase a counter on the stats payload."""
|
||||
self._types.setdefault(counter_name, _MetricType.COUNTER)
|
||||
self._values[counter_name] = (
|
||||
self._values.get(counter_name, 0) + counter_increase_value
|
||||
)
|
||||
|
||||
def set_gauge(self, gauge_name: str, gauge_value: int | float) -> None:
|
||||
"""Set a gauge snapshot on the stats payload."""
|
||||
self._types.setdefault(gauge_name, _MetricType.GAUGE)
|
||||
self._values[gauge_name] = gauge_value
|
||||
|
||||
def observe_histogram(
|
||||
self, histogram_name: str, histogram_value: int | float
|
||||
) -> None:
|
||||
"""Record a histogram observation on the stats payload."""
|
||||
self._types.setdefault(histogram_name, _MetricType.HISTOGRAM)
|
||||
self._values.setdefault(histogram_name, []).append(histogram_value)
|
||||
|
||||
|
||||
class OffloadPromMetrics(KVConnectorPromMetrics):
|
||||
@@ -89,77 +243,145 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
|
||||
self.histogram_transfer_size: dict[tuple[int, str], PromMetricT] = {}
|
||||
self.counter_kv_bytes: dict[tuple[int, str], PromMetricT] = {}
|
||||
self.counter_kv_transfer_time: dict[tuple[int, str], PromMetricT] = {}
|
||||
buckets = [ # In bytes
|
||||
1e6,
|
||||
5e6,
|
||||
10e6,
|
||||
20e6,
|
||||
40e6,
|
||||
60e6,
|
||||
80e6,
|
||||
100e6,
|
||||
150e6,
|
||||
200e6,
|
||||
]
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(vllm_config)
|
||||
kv_transfer_config = vllm_config.kv_transfer_config
|
||||
assert kv_transfer_config is not None
|
||||
extra_config = kv_transfer_config.kv_connector_extra_config
|
||||
self._offloading_metric_metadata: dict[str, OffloadingMetricMetadata] = {
|
||||
**spec_cls.build_metric_definitions(extra_config),
|
||||
**get_connector_metric_definitions(),
|
||||
}
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
|
||||
self._observe_deprecated_metrics = issubclass(spec_cls, CPUOffloadingSpec)
|
||||
self._offloading_metric_defs: dict[str, PromMetricT] = {}
|
||||
self.offloading_metrics: dict[tuple[int, str], PromMetricT] = {}
|
||||
|
||||
self._counter_kv_bytes = self._counter_cls(
|
||||
name="vllm:kv_offload_total_bytes",
|
||||
documentation="Number of bytes offloaded by KV connector",
|
||||
name=_DEPRECATED_TOTAL_BYTES,
|
||||
documentation=_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[
|
||||
_DEPRECATED_TOTAL_BYTES
|
||||
].documentation,
|
||||
labelnames=labelnames + ["transfer_type"],
|
||||
)
|
||||
|
||||
self._counter_kv_transfer_time = self._counter_cls(
|
||||
name="vllm:kv_offload_total_time",
|
||||
documentation="Total time measured by all KV offloading operations",
|
||||
name=_DEPRECATED_TOTAL_TIME,
|
||||
documentation=_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[
|
||||
_DEPRECATED_TOTAL_TIME
|
||||
].documentation,
|
||||
labelnames=labelnames + ["transfer_type"],
|
||||
)
|
||||
|
||||
deprecated_size_metadata = _DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[
|
||||
_DEPRECATED_SIZE
|
||||
]
|
||||
assert isinstance(deprecated_size_metadata, OffloadingHistogramMetadata)
|
||||
self._histogram_transfer_size = self._histogram_cls(
|
||||
name="vllm:kv_offload_size",
|
||||
documentation="Histogram of KV offload transfer size, in bytes.",
|
||||
buckets=buckets[:],
|
||||
name=_DEPRECATED_SIZE,
|
||||
documentation=deprecated_size_metadata.documentation,
|
||||
buckets=deprecated_size_metadata.buckets,
|
||||
labelnames=labelnames + ["transfer_type"],
|
||||
)
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
"""
|
||||
Observe transfer statistics from the new data structure.
|
||||
transfer_stats_data is expected to be a dict where:
|
||||
- keys are transfer type strings (e.g., "cpu_to_gpu", "gpu_to_cpu")
|
||||
- values are lists of OffloadingOperationMetrics objects
|
||||
"""
|
||||
|
||||
for transfer_type, ops in transfer_stats_data.items():
|
||||
# Cache:
|
||||
if (engine_idx, transfer_type) not in self.histogram_transfer_size:
|
||||
for engine_idx, labelvalues in per_engine_labelvalues.items():
|
||||
for transfer_type in _TransferType.ALL:
|
||||
bounded_labelvalues = labelvalues + [transfer_type]
|
||||
self.histogram_transfer_size[(engine_idx, transfer_type)] = (
|
||||
self._histogram_transfer_size.labels(
|
||||
*(self.per_engine_labelvalues[engine_idx] + [transfer_type])
|
||||
)
|
||||
self._histogram_transfer_size.labels(*bounded_labelvalues)
|
||||
)
|
||||
self.counter_kv_bytes[(engine_idx, transfer_type)] = (
|
||||
self._counter_kv_bytes.labels(
|
||||
*(self.per_engine_labelvalues[engine_idx] + [transfer_type])
|
||||
)
|
||||
self._counter_kv_bytes.labels(*bounded_labelvalues)
|
||||
)
|
||||
self.counter_kv_transfer_time[(engine_idx, transfer_type)] = (
|
||||
self._counter_kv_transfer_time.labels(
|
||||
*(self.per_engine_labelvalues[engine_idx] + [transfer_type])
|
||||
)
|
||||
self._counter_kv_transfer_time.labels(*bounded_labelvalues)
|
||||
)
|
||||
|
||||
# Process ops:
|
||||
assert isinstance(ops, list)
|
||||
for op in ops: # ops is a list of serialized OffloadingOperationMetrics
|
||||
assert isinstance(op, dict)
|
||||
# Observe size histogram
|
||||
self.histogram_transfer_size[(engine_idx, transfer_type)].observe(
|
||||
op["op_size"]
|
||||
for metric_name, metadata in self._offloading_metric_metadata.items():
|
||||
self._offloading_metric_defs[metric_name] = self._create_metric(
|
||||
metric_name, metadata
|
||||
)
|
||||
for engine_idx, labelvalues in per_engine_labelvalues.items():
|
||||
self.offloading_metrics[(engine_idx, metric_name)] = (
|
||||
self._offloading_metric_defs[metric_name].labels(*labelvalues)
|
||||
)
|
||||
|
||||
# Increment byte and time counters
|
||||
self.counter_kv_bytes[(engine_idx, transfer_type)].inc(op["op_size"])
|
||||
def _create_metric(
|
||||
self, metric_name: str, metadata: OffloadingMetricMetadata
|
||||
) -> Any:
|
||||
kwargs: dict[str, Any] = {
|
||||
"name": metric_name,
|
||||
"documentation": metadata.documentation,
|
||||
"labelnames": self._labelnames,
|
||||
}
|
||||
if isinstance(metadata, OffloadingCounterMetadata):
|
||||
metric_cls = self._counter_cls
|
||||
elif isinstance(metadata, OffloadingGaugeMetadata):
|
||||
metric_cls = self._gauge_cls
|
||||
elif isinstance(metadata, OffloadingHistogramMetadata):
|
||||
metric_cls = self._histogram_cls
|
||||
if metadata.buckets is not None:
|
||||
kwargs["buckets"] = metadata.buckets
|
||||
else:
|
||||
raise AssertionError(f"Unknown offloading metric metadata: {metadata}")
|
||||
return metric_cls(**kwargs)
|
||||
|
||||
self.counter_kv_transfer_time[(engine_idx, transfer_type)].inc(
|
||||
op["op_time"]
|
||||
def _increase_counter(
|
||||
self, metric_name: str, value: int | float, engine_idx: int
|
||||
) -> None:
|
||||
self.offloading_metrics[(engine_idx, metric_name)].inc(value)
|
||||
if not self._observe_deprecated_metrics:
|
||||
return
|
||||
# Keep deprecated CPU offload transfer metrics updated during the
|
||||
# transition to flat metric names.
|
||||
if metric_name == _TransferMetricName.LOAD_BYTES:
|
||||
self.counter_kv_bytes[(engine_idx, _TransferType.LOAD)].inc(value)
|
||||
elif metric_name == _TransferMetricName.LOAD_TIME:
|
||||
self.counter_kv_transfer_time[(engine_idx, _TransferType.LOAD)].inc(value)
|
||||
elif metric_name == _TransferMetricName.STORE_BYTES:
|
||||
self.counter_kv_bytes[(engine_idx, _TransferType.STORE)].inc(value)
|
||||
elif metric_name == _TransferMetricName.STORE_TIME:
|
||||
self.counter_kv_transfer_time[(engine_idx, _TransferType.STORE)].inc(value)
|
||||
|
||||
def _set_gauge(self, metric_name: str, value: int | float, engine_idx: int) -> None:
|
||||
self.offloading_metrics[(engine_idx, metric_name)].set(value)
|
||||
|
||||
def _observe_histogram(
|
||||
self, metric_name: str, value: list[int | float], engine_idx: int
|
||||
) -> None:
|
||||
for observation in value:
|
||||
self.offloading_metrics[(engine_idx, metric_name)].observe(observation)
|
||||
if not self._observe_deprecated_metrics:
|
||||
continue
|
||||
# Keep deprecated CPU offload transfer metrics updated during the
|
||||
# transition to flat metric names.
|
||||
if metric_name == _TransferMetricName.LOAD_SIZE:
|
||||
self.histogram_transfer_size[(engine_idx, _TransferType.LOAD)].observe(
|
||||
observation
|
||||
)
|
||||
elif metric_name == _TransferMetricName.STORE_SIZE:
|
||||
self.histogram_transfer_size[(engine_idx, _TransferType.STORE)].observe(
|
||||
observation
|
||||
)
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
"""Observe transfer statistics."""
|
||||
metric_types = transfer_stats_data.get(_StatsKey.TYPES, {})
|
||||
metric_data = transfer_stats_data.get(_StatsKey.DATA, {})
|
||||
for key, value in metric_data.items():
|
||||
type_str = metric_types.get(key)
|
||||
if type_str is None:
|
||||
raise AssertionError(f"Unknown offloading stats key: {key}")
|
||||
assert key in self._offloading_metric_defs
|
||||
if type_str == _MetricType.COUNTER:
|
||||
assert isinstance(value, int | float)
|
||||
self._increase_counter(key, value, engine_idx)
|
||||
elif type_str == _MetricType.GAUGE:
|
||||
assert isinstance(value, int | float)
|
||||
self._set_gauge(key, value, engine_idx)
|
||||
elif type_str == _MetricType.HISTOGRAM:
|
||||
assert isinstance(value, list)
|
||||
assert all(isinstance(v, int | float) for v in value)
|
||||
self._observe_histogram(key, value, engine_idx)
|
||||
else:
|
||||
raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}")
|
||||
|
||||
@@ -14,6 +14,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
ReqId,
|
||||
TransferJob,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
_TransferMetricName,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
@@ -259,9 +263,13 @@ def _create_req_context(req: Request) -> ReqContext:
|
||||
class OffloadingConnectorScheduler:
|
||||
"""Implementation of Scheduler side methods"""
|
||||
|
||||
def __init__(self, spec: OffloadingSpec):
|
||||
def __init__(
|
||||
self,
|
||||
spec: OffloadingSpec,
|
||||
):
|
||||
self.config = SchedulerOffloadConfig.from_spec(spec)
|
||||
self.manager: OffloadingManager = spec.get_manager()
|
||||
self._connector_stats: OffloadingConnectorStats | None = None
|
||||
|
||||
full_attention_groups: list[int] = []
|
||||
sliding_window_groups: list[int] = []
|
||||
@@ -952,6 +960,39 @@ class OffloadingConnectorScheduler:
|
||||
if not isinstance(meta, OffloadingWorkerMetadata):
|
||||
assert meta is None
|
||||
meta = OffloadingWorkerMetadata()
|
||||
if not meta.transfer_stats.is_empty():
|
||||
transfer_stats = OffloadingConnectorStats()
|
||||
if not meta.transfer_stats.load.is_empty():
|
||||
transfer_stats.increase_counter(
|
||||
_TransferMetricName.LOAD_BYTES,
|
||||
meta.transfer_stats.load.bytes,
|
||||
)
|
||||
transfer_stats.increase_counter(
|
||||
_TransferMetricName.LOAD_TIME,
|
||||
meta.transfer_stats.load.time,
|
||||
)
|
||||
for size in meta.transfer_stats.load.sizes:
|
||||
transfer_stats.observe_histogram(
|
||||
_TransferMetricName.LOAD_SIZE, size
|
||||
)
|
||||
if not meta.transfer_stats.store.is_empty():
|
||||
transfer_stats.increase_counter(
|
||||
_TransferMetricName.STORE_BYTES,
|
||||
meta.transfer_stats.store.bytes,
|
||||
)
|
||||
transfer_stats.increase_counter(
|
||||
_TransferMetricName.STORE_TIME,
|
||||
meta.transfer_stats.store.time,
|
||||
)
|
||||
for size in meta.transfer_stats.store.sizes:
|
||||
transfer_stats.observe_histogram(
|
||||
_TransferMetricName.STORE_SIZE, size
|
||||
)
|
||||
if self._connector_stats is None:
|
||||
self._connector_stats = transfer_stats
|
||||
else:
|
||||
self._connector_stats.aggregate(transfer_stats)
|
||||
|
||||
for job_id, count in meta.completed_jobs.items():
|
||||
assert count > 0
|
||||
if job_id < self._stale_job_threshold:
|
||||
@@ -990,6 +1031,19 @@ class OffloadingConnectorScheduler:
|
||||
if not req_status.transfer_jobs and req_status.req.is_finished():
|
||||
del self._req_status[job_status.req_id]
|
||||
|
||||
def get_stats(self) -> OffloadingConnectorStats | None:
|
||||
stats = self._connector_stats
|
||||
self._connector_stats = None
|
||||
|
||||
manager_stats = self.manager.get_stats()
|
||||
if manager_stats is not None:
|
||||
if stats is None:
|
||||
stats = manager_stats
|
||||
else:
|
||||
stats.aggregate(manager_stats)
|
||||
|
||||
return stats
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: Request,
|
||||
|
||||
@@ -5,17 +5,11 @@ from dataclasses import replace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingConnectorMetadata,
|
||||
OffloadingWorkerMetadata,
|
||||
ReqId,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -44,7 +38,6 @@ class OffloadingConnectorWorker:
|
||||
self.spec = spec
|
||||
self.worker = OffloadingWorker()
|
||||
|
||||
self.kv_connector_stats = OffloadingConnectorStats()
|
||||
# job_id -> req_id for in-flight loads.
|
||||
self._load_jobs: dict[int, ReqId] = {}
|
||||
self._unsubmitted_store_jobs: list[tuple[int, TransferSpec]] = []
|
||||
@@ -271,15 +264,18 @@ class OffloadingConnectorWorker:
|
||||
# we currently do not support job failures
|
||||
job_id = transfer_result.job_id
|
||||
assert transfer_result.success
|
||||
is_load = job_id in self._load_jobs
|
||||
if (
|
||||
transfer_result.transfer_time
|
||||
transfer_result.transfer_time is not None
|
||||
and transfer_result.transfer_size is not None
|
||||
and transfer_result.transfer_type is not None
|
||||
):
|
||||
self.kv_connector_stats.record_transfer(
|
||||
num_bytes=transfer_result.transfer_size,
|
||||
time=transfer_result.transfer_time,
|
||||
transfer_type=transfer_result.transfer_type,
|
||||
if is_load:
|
||||
stats = self._connector_worker_meta.transfer_stats.load
|
||||
else:
|
||||
stats = self._connector_worker_meta.transfer_stats.store
|
||||
stats.record(
|
||||
transfer_result.transfer_size,
|
||||
transfer_result.transfer_time,
|
||||
)
|
||||
|
||||
self._connector_worker_meta.mark_completed(job_id)
|
||||
@@ -297,18 +293,6 @@ class OffloadingConnectorWorker:
|
||||
self._connector_worker_meta = OffloadingWorkerMetadata()
|
||||
return meta
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""
|
||||
Get the KV transfer stats for the connector.
|
||||
"""
|
||||
|
||||
if self.kv_connector_stats.is_empty():
|
||||
return None
|
||||
# Clear stats for next iteration
|
||||
kv_connector_stats = self.kv_connector_stats
|
||||
self.kv_connector_stats = OffloadingConnectorStats()
|
||||
return kv_connector_stats
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._unsubmitted_store_jobs.clear()
|
||||
self._load_jobs.clear()
|
||||
|
||||
@@ -184,9 +184,14 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return True
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
if self.connector_worker is None:
|
||||
return None # We only emit stats from the worker-side
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
if self.connector_scheduler is not None:
|
||||
return self.connector_scheduler.get_stats()
|
||||
|
||||
# TODO(orozery): Remove once PR #43877 lands
|
||||
if self.connector_worker is not None:
|
||||
return OffloadingConnectorStats()
|
||||
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
|
||||
@@ -19,6 +19,9 @@ from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_offload.worker.worker import OffloadingHandler
|
||||
|
||||
@@ -123,6 +126,26 @@ The class provides the following primitives:
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OffloadingMetricMetadata:
|
||||
documentation: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OffloadingCounterMetadata(OffloadingMetricMetadata):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OffloadingGaugeMetadata(OffloadingMetricMetadata):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OffloadingHistogramMetadata(OffloadingMetricMetadata):
|
||||
buckets: tuple[float, ...] | None = None
|
||||
|
||||
|
||||
class OffloadingManager(ABC):
|
||||
@abstractmethod
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
@@ -269,6 +292,10 @@ class OffloadingManager(ABC):
|
||||
"""Evict all tracked blocks and reset internal state."""
|
||||
return
|
||||
|
||||
def get_stats(self) -> "OffloadingConnectorStats | None":
|
||||
"""Return collected metrics since last call, or None if disabled."""
|
||||
return None
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Shutdown the manager and release any resources."""
|
||||
return
|
||||
@@ -378,6 +405,13 @@ class CanonicalKVCaches:
|
||||
class OffloadingSpec(ABC):
|
||||
"""Spec for an offloading connector"""
|
||||
|
||||
@classmethod
|
||||
def build_metric_definitions(
|
||||
cls, extra_config: dict[str, Any]
|
||||
) -> dict[str, "OffloadingMetricMetadata"]:
|
||||
"""Return Prometheus metric definitions emitted by this spec."""
|
||||
return {}
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
|
||||
logger.warning(
|
||||
"Initializing OffloadingSpec. This API is experimental and "
|
||||
|
||||
@@ -4,6 +4,8 @@ from typing_extensions import override
|
||||
|
||||
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
|
||||
|
||||
METRIC_STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
|
||||
|
||||
|
||||
class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
|
||||
"""
|
||||
|
||||
@@ -6,6 +6,9 @@ from typing import Literal
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LoadStoreSpec,
|
||||
OffloadingEvent,
|
||||
@@ -15,7 +18,7 @@ from vllm.v1.kv_offload.base import (
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.cpu.common import METRIC_STORES_SKIPPED, CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
||||
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
|
||||
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
|
||||
@@ -58,6 +61,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
self._policy: CachePolicy = policy_cls(cache_capacity=num_blocks)
|
||||
self.store_threshold: int = store_threshold
|
||||
self.max_tracker_size: int = max_tracker_size
|
||||
self.stores_skipped_in_current_batch: int = 0
|
||||
|
||||
# Number of block references. It is ordered so can evict the LRU entry in O(1).
|
||||
self.counts: OrderedDict[OffloadKey, int] | None = (
|
||||
@@ -154,7 +158,9 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
req_context: ReqContext,
|
||||
) -> PrepareStoreOutput | None:
|
||||
if self.counts is not None:
|
||||
num_keys = len(keys)
|
||||
keys = [k for k in keys if self.counts.get(k, 0) >= self.store_threshold]
|
||||
self.stores_skipped_in_current_batch += num_keys - len(keys)
|
||||
# filter out blocks that are already stored
|
||||
keys_to_store = [k for k in keys if self._policy.get(k) is None]
|
||||
|
||||
@@ -253,3 +259,15 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
if self.events is not None:
|
||||
yield from self.events
|
||||
self.events.clear()
|
||||
|
||||
def get_stats(self) -> OffloadingConnectorStats | None:
|
||||
if self.store_threshold < 2:
|
||||
return None
|
||||
|
||||
stats = OffloadingConnectorStats()
|
||||
stats.increase_counter(
|
||||
METRIC_STORES_SKIPPED,
|
||||
self.stores_skipped_in_current_batch,
|
||||
)
|
||||
self.stores_skipped_in_current_batch = 0
|
||||
return stats
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
@@ -12,10 +13,12 @@ from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCaches,
|
||||
GPULoadStoreSpec,
|
||||
LoadStoreSpec,
|
||||
OffloadingCounterMetadata,
|
||||
OffloadingManager,
|
||||
OffloadingMetricMetadata,
|
||||
OffloadingSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.cpu.common import METRIC_STORES_SKIPPED, CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.worker.worker import OffloadingHandler
|
||||
@@ -24,6 +27,22 @@ from vllm.v1.kv_offload.worker.worker import OffloadingHandler
|
||||
class CPUOffloadingSpec(OffloadingSpec):
|
||||
BLOCK_SIZE_ALIGNMENT = 1
|
||||
|
||||
@classmethod
|
||||
def build_metric_definitions(
|
||||
cls, extra_config: dict[str, Any]
|
||||
) -> dict[str, OffloadingMetricMetadata]:
|
||||
store_threshold = int(extra_config.get("store_threshold", 0))
|
||||
if store_threshold < 2:
|
||||
return {}
|
||||
return {
|
||||
METRIC_STORES_SKIPPED: OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Number of KV offload stores skipped because the reuse "
|
||||
"threshold was not reached."
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig):
|
||||
super().__init__(vllm_config, kv_cache_config)
|
||||
|
||||
|
||||
@@ -30,11 +30,7 @@ class OffloadingSpecFactory:
|
||||
cls._registry[name] = loader
|
||||
|
||||
@classmethod
|
||||
def create_spec(
|
||||
cls,
|
||||
config: "VllmConfig",
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
) -> OffloadingSpec:
|
||||
def get_spec_cls(cls, config: "VllmConfig") -> type[OffloadingSpec]:
|
||||
kv_transfer_config = config.kv_transfer_config
|
||||
assert kv_transfer_config is not None
|
||||
extra_config = kv_transfer_config.kv_connector_extra_config
|
||||
@@ -48,6 +44,20 @@ class OffloadingSpecFactory:
|
||||
spec_module = importlib.import_module(spec_module_path)
|
||||
spec_cls = getattr(spec_module, spec_name)
|
||||
assert issubclass(spec_cls, OffloadingSpec)
|
||||
return spec_cls
|
||||
|
||||
@classmethod
|
||||
def create_spec(
|
||||
cls,
|
||||
config: "VllmConfig",
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
) -> OffloadingSpec:
|
||||
kv_transfer_config = config.kv_transfer_config
|
||||
assert kv_transfer_config is not None
|
||||
spec_name = kv_transfer_config.kv_connector_extra_config.get(
|
||||
"spec_name", "CPUOffloadingSpec"
|
||||
)
|
||||
spec_cls = cls.get_spec_cls(config)
|
||||
logger.info("Creating offloading spec with name: %s", spec_name)
|
||||
return spec_cls(config, kv_cache_config)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user