Feature/offloading manager stats (#35669)

Signed-off-by: Sriusa4414@gmail.com
Signed-off-by: srinivas_oo7 <Sriusa4414@gmail.com>
Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Signed-off-by: Srinivasoo7 <158864704+Srinivasoo7@users.noreply.github.com>
Signed-off-by: Or Ozeri <oro@il.ibm.com>
Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Co-authored-by: Srinivasoo7 <158864704+Srinivasoo7@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Srinivas Krovvidi
2026-06-10 12:44:55 +00:00
committed by GitHub
co-authored by srinivas_oo7 Srinivasoo7 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Or Ozeri
parent 9ad08c4d15
commit 9dfc313bdc
14 changed files with 1200 additions and 213 deletions
@@ -1,11 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import Any
from unittest.mock import patch
import pytest
from prometheus_client import Counter, Gauge, Histogram
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
OffloadPromMetrics,
_MetricType,
_StatsKey,
_TransferMetricName,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import (
OffloadingConnector,
)
from vllm.v1.kv_offload.base import (
OffloadingCounterMetadata,
OffloadingGaugeMetadata,
OffloadingHistogramMetadata,
)
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
LOAD_BYTES = _TransferMetricName.LOAD_BYTES
LOAD_TIME = _TransferMetricName.LOAD_TIME
LOAD_SIZE = _TransferMetricName.LOAD_SIZE
STORE_BYTES = _TransferMetricName.STORE_BYTES
STORE_TIME = _TransferMetricName.STORE_TIME
STORE_SIZE = _TransferMetricName.STORE_SIZE
STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
PENDING_STORES = "vllm:kv_offload_pending_stores"
LOOKUP_LATENCY = "vllm:kv_offload_lookup_latency_seconds"
class _FakeMetric:
def __init__(self, **kwargs):
self.kwargs = kwargs
self.children: list[_FakeMetric] = []
self.observed: list[int | float] = []
self.increments: list[int | float] = []
self.set_values: list[int | float] = []
self.labelvalues: tuple[object, ...] = ()
def labels(self, *labelvalues):
child = _FakeMetric(**self.kwargs)
child.labelvalues = labelvalues
self.children.append(child)
return child
def observe(self, value):
self.observed.append(value)
def inc(self, value):
self.increments.append(value)
def set(self, value):
self.set_values.append(value)
class _FakeVllmConfig:
def __init__(self, store_threshold: int = 2):
self.kv_transfer_config = SimpleNamespace(
kv_connector_extra_config={"store_threshold": store_threshold}
)
def _metric_metadata():
return {
LOAD_BYTES: OffloadingCounterMetadata(
documentation="load bytes",
),
LOAD_TIME: OffloadingCounterMetadata(
documentation="load time",
),
LOAD_SIZE: OffloadingHistogramMetadata(
documentation="load size",
),
STORE_BYTES: OffloadingCounterMetadata(
documentation="store bytes",
),
STORE_TIME: OffloadingCounterMetadata(
documentation="store time",
),
STORE_SIZE: OffloadingHistogramMetadata(
documentation="store size",
),
STORES_SKIPPED: OffloadingCounterMetadata(
documentation="stores skipped",
),
PENDING_STORES: OffloadingGaugeMetadata(
documentation="pending stores",
),
LOOKUP_LATENCY: OffloadingHistogramMetadata(
documentation="lookup latency",
),
}
def test_build_kv_connector_stats_with_none():
@@ -14,7 +105,6 @@ def test_build_kv_connector_stats_with_none():
assert stats is not None
assert isinstance(stats, OffloadingConnectorStats)
assert len(stats.data) == 0
assert stats.is_empty()
@@ -24,7 +114,6 @@ def test_build_kv_connector_stats_with_empty_dict():
assert stats is not None
assert isinstance(stats, OffloadingConnectorStats)
assert len(stats.data) == 0
assert stats.is_empty()
@@ -32,114 +121,186 @@ def test_build_kv_connector_stats_reconstructs_offload_stats():
"""Test that OffloadingConnector stats are properly reconstructed with
correct data."""
serialized_data = {
"CPU_to_GPU": [
{"op_size": 16, "op_time": 1.0},
{"op_size": 8, "op_time": 0.5},
],
"GPU_to_CPU": [
{"op_size": 1, "op_time": 0.1},
{"op_size": 2, "op_time": 0.2},
],
_StatsKey.TYPES: {
LOAD_BYTES: _MetricType.COUNTER,
LOAD_TIME: _MetricType.COUNTER,
LOAD_SIZE: _MetricType.HISTOGRAM,
STORE_BYTES: _MetricType.COUNTER,
STORE_TIME: _MetricType.COUNTER,
STORE_SIZE: _MetricType.HISTOGRAM,
STORES_SKIPPED: _MetricType.COUNTER,
},
_StatsKey.DATA: {
LOAD_BYTES: 24,
LOAD_TIME: 1.5,
LOAD_SIZE: [16, 8],
STORE_BYTES: 3,
STORE_TIME: 0.3,
STORE_SIZE: [1, 2],
STORES_SKIPPED: 5,
},
}
stats = OffloadingConnector.build_kv_connector_stats(data=serialized_data)
offload_connector_stats = stats
assert isinstance(offload_connector_stats, OffloadingConnectorStats)
assert offload_connector_stats.data["CPU_to_GPU"] == [
{"op_size": 16, "op_time": 1.0},
{"op_size": 8, "op_time": 0.5},
]
assert offload_connector_stats.data["GPU_to_CPU"] == [
{"op_size": 1, "op_time": 0.1},
{"op_size": 2, "op_time": 0.2},
]
assert isinstance(stats, OffloadingConnectorStats)
values = stats.data[_StatsKey.DATA]
assert values[LOAD_BYTES] == 24
assert values[LOAD_TIME] == 1.5
assert values[LOAD_SIZE] == [16, 8]
assert values[STORE_BYTES] == 3
assert values[STORE_TIME] == 0.3
assert values[STORE_SIZE] == [1, 2]
assert values[STORES_SKIPPED] == 5
def _make_stats_data(
metric_data: dict[str, Any],
metric_metadata: dict[str, Any],
) -> dict[str, Any]:
"""Build a structured data dict from flat metric data and metadata."""
metric_types = {}
for key in metric_data:
md = metric_metadata[key]
if isinstance(md, OffloadingCounterMetadata):
metric_types[key] = _MetricType.COUNTER
elif isinstance(md, OffloadingGaugeMetadata):
metric_types[key] = _MetricType.GAUGE
elif isinstance(md, OffloadingHistogramMetadata):
metric_types[key] = _MetricType.HISTOGRAM
return {
_StatsKey.TYPES: metric_types,
_StatsKey.DATA: metric_data,
}
def test_aggregate_same_connector():
"""Test aggregating stats from the same connector type."""
metadata = _metric_metadata()
stats1 = OffloadingConnectorStats(
data={
"CPU_to_GPU": [
{"op_size": 16, "op_time": 1.0},
{"op_size": 8, "op_time": 0.5},
],
"GPU_to_CPU": [
{"op_size": 1, "op_time": 0.1},
{"op_size": 2, "op_time": 0.2},
],
}
data=_make_stats_data(
{
LOAD_BYTES: 24,
LOAD_TIME: 1.5,
LOAD_SIZE: [16, 8],
STORE_BYTES: 3,
STORE_TIME: 0.3,
STORE_SIZE: [1, 2],
STORES_SKIPPED: 1,
PENDING_STORES: 3,
LOOKUP_LATENCY: [0.1],
},
metadata,
),
)
stats2 = OffloadingConnectorStats(
data={
"CPU_to_GPU": [
{"op_size": 3, "op_time": 0.2},
{"op_size": 7, "op_time": 0.9},
],
"GPU_to_CPU": [{"op_size": 16, "op_time": 2}],
}
data=_make_stats_data(
{
LOAD_BYTES: 10,
LOAD_TIME: 1.1,
LOAD_SIZE: [3, 7],
STORE_BYTES: 16,
STORE_TIME: 2,
STORE_SIZE: [16],
STORES_SKIPPED: 3,
PENDING_STORES: 1,
LOOKUP_LATENCY: [0.2, 0.3],
},
metadata,
),
)
result = stats1.aggregate(stats2)
assert result is stats1 # Should return self
offload_connector_stats = result
assert offload_connector_stats.data["CPU_to_GPU"] == [
{"op_size": 16, "op_time": 1.0},
{"op_size": 8, "op_time": 0.5},
{"op_size": 3, "op_time": 0.2},
{"op_size": 7, "op_time": 0.9},
]
assert offload_connector_stats.data["GPU_to_CPU"] == [
{"op_size": 1, "op_time": 0.1},
{"op_size": 2, "op_time": 0.2},
{"op_size": 16, "op_time": 2},
]
values = result.data[_StatsKey.DATA]
assert values[LOAD_BYTES] == 34
assert values[LOAD_TIME] == 2.6
assert values[LOAD_SIZE] == [16, 8, 3, 7]
assert values[STORE_BYTES] == 19
assert values[STORE_TIME] == 2.3
assert values[STORE_SIZE] == [1, 2, 16]
assert values[STORES_SKIPPED] == 4
assert values[PENDING_STORES] == 1
assert values[LOOKUP_LATENCY] == [0.1, 0.2, 0.3]
def test_aggregate_merges_types():
stats1 = OffloadingConnectorStats(
data={
_StatsKey.TYPES: {LOAD_BYTES: _MetricType.COUNTER},
_StatsKey.DATA: {LOAD_BYTES: 1},
},
)
stats2 = OffloadingConnectorStats(
data={
_StatsKey.TYPES: {PENDING_STORES: _MetricType.GAUGE},
_StatsKey.DATA: {PENDING_STORES: 2},
},
)
result = stats1.aggregate(stats2)
assert result.data[_StatsKey.DATA][PENDING_STORES] == 2
assert result.data[_StatsKey.TYPES][PENDING_STORES] == _MetricType.GAUGE
def test_reduce():
"""Test that reduce() correctly reduces all nested connector stats."""
"""Test that reduce() correctly reduces connector stats."""
metadata = _metric_metadata()
stats = OffloadingConnectorStats(
data={
"CPU_to_GPU": [
{"op_size": 16, "op_time": 1.0},
{"op_size": 8, "op_time": 0.5},
{"op_size": 3, "op_time": 0.2},
{"op_size": 7, "op_time": 0.9},
],
"GPU_to_CPU": [
{"op_size": 1, "op_time": 0.1},
{"op_size": 2, "op_time": 0.2},
{"op_size": 16, "op_time": 2},
],
}
data=_make_stats_data(
{
LOAD_BYTES: 34,
LOAD_TIME: 2.6,
LOAD_SIZE: [16, 8, 3, 7],
STORE_BYTES: 19,
STORE_TIME: 2.3,
STORE_SIZE: [1, 2, 16],
STORES_SKIPPED: 11,
PENDING_STORES: 2,
LOOKUP_LATENCY: [0.1, 0.2, 0.3],
},
metadata,
),
)
reduced = stats.reduce()
assert isinstance(reduced, dict)
# Check that the stats were reduced (should have aggregated values)
assert "CPU_to_GPU_total_bytes" in reduced
assert "CPU_to_GPU_total_time" in reduced
assert "GPU_to_CPU_total_bytes" in reduced
assert "GPU_to_CPU_total_time" in reduced
assert reduced["CPU_to_GPU_total_bytes"] == 34
assert reduced["CPU_to_GPU_total_time"] == 2.6
assert reduced["GPU_to_CPU_total_time"] == 2.3
assert reduced["GPU_to_CPU_total_bytes"] == 19
assert reduced[LOAD_BYTES] == 34
assert reduced[LOAD_TIME] == 2.6
assert reduced[f"{LOAD_SIZE}_count"] == 4
assert reduced[f"{LOAD_SIZE}_sum"] == 34
assert reduced[STORE_BYTES] == 19
assert reduced[STORE_TIME] == 2.3
assert reduced[f"{STORE_SIZE}_count"] == 3
assert reduced[f"{STORE_SIZE}_sum"] == 19
assert reduced[STORES_SKIPPED] == 11
assert reduced[PENDING_STORES] == 2
assert reduced[f"{LOOKUP_LATENCY}_count"] == 3
assert reduced[f"{LOOKUP_LATENCY}_sum"] == sum([0.1, 0.2, 0.3])
def test_reset():
"""Test that reset() resets all nested connector stats."""
"""Test that reset() resets all connector stats."""
metadata = _metric_metadata()
offload_connector_stats = OffloadingConnectorStats(
data={
"CPU_to_GPU": [
{"op_size": 3, "op_time": 0.2},
{"op_size": 7, "op_time": 0.9},
],
"GPU_to_CPU": [{"op_size": 16, "op_time": 2}],
}
data=_make_stats_data(
{
LOAD_BYTES: 10,
LOAD_TIME: 1.1,
LOAD_SIZE: [3, 7],
STORE_BYTES: 16,
STORE_TIME: 2,
STORE_SIZE: [16],
STORES_SKIPPED: 4,
PENDING_STORES: 2,
LOOKUP_LATENCY: [0.1],
},
metadata,
),
)
assert not offload_connector_stats.is_empty()
@@ -148,4 +309,215 @@ def test_reset():
# After reset, stats should be empty
assert offload_connector_stats.is_empty()
assert len(offload_connector_stats.data) == 0
def test_prom_metrics_observes_manager_counter():
prom_metrics = OffloadPromMetrics(
vllm_config=_FakeVllmConfig(), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
prom_metrics.observe(
{
_StatsKey.TYPES: {STORES_SKIPPED: _MetricType.COUNTER},
_StatsKey.DATA: {STORES_SKIPPED: 7},
}
)
counter = prom_metrics.offloading_metrics[(0, STORES_SKIPPED)]
assert counter.increments == [7]
counter_def = prom_metrics._offloading_metric_defs[STORES_SKIPPED]
assert counter_def.kwargs["name"] == "vllm:kv_offload_stores_skipped"
assert counter.labelvalues == ("model", "0")
def test_prom_metrics_observes_flat_transfer_metrics_and_legacy_metrics():
prom_metrics = OffloadPromMetrics(
vllm_config=_FakeVllmConfig(), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
prom_metrics.observe(
{
_StatsKey.TYPES: {
LOAD_BYTES: _MetricType.COUNTER,
LOAD_TIME: _MetricType.COUNTER,
LOAD_SIZE: _MetricType.HISTOGRAM,
STORE_BYTES: _MetricType.COUNTER,
STORE_TIME: _MetricType.COUNTER,
STORE_SIZE: _MetricType.HISTOGRAM,
},
_StatsKey.DATA: {
LOAD_BYTES: 24,
LOAD_TIME: 1.5,
LOAD_SIZE: [16, 8],
STORE_BYTES: 3,
STORE_TIME: 0.3,
STORE_SIZE: [1, 2],
},
}
)
assert prom_metrics.offloading_metrics[(0, LOAD_BYTES)].increments == [24]
assert prom_metrics.offloading_metrics[(0, LOAD_TIME)].increments == [1.5]
assert prom_metrics.offloading_metrics[(0, LOAD_SIZE)].observed == [16, 8]
assert prom_metrics.offloading_metrics[(0, STORE_BYTES)].increments == [3]
assert prom_metrics.offloading_metrics[(0, STORE_TIME)].increments == [0.3]
assert prom_metrics.offloading_metrics[(0, STORE_SIZE)].observed == [1, 2]
assert prom_metrics.counter_kv_bytes[(0, "CPU_to_GPU")].increments == [24]
assert prom_metrics.counter_kv_transfer_time[(0, "CPU_to_GPU")].increments == [1.5]
assert prom_metrics.histogram_transfer_size[(0, "CPU_to_GPU")].observed == [16, 8]
assert prom_metrics.counter_kv_bytes[(0, "GPU_to_CPU")].increments == [3]
assert prom_metrics.counter_kv_transfer_time[(0, "GPU_to_CPU")].increments == [0.3]
assert prom_metrics.histogram_transfer_size[(0, "GPU_to_CPU")].observed == [1, 2]
def test_prom_metrics_observes_manager_gauge_and_histogram():
metric_definitions = {
PENDING_STORES: OffloadingGaugeMetadata(
documentation="Number of currently pending KV offload stores.",
),
LOOKUP_LATENCY: OffloadingHistogramMetadata(
documentation="KV offload lookup latency.",
buckets=(0.1, 1.0),
),
}
with patch.object(
CPUOffloadingSpec, "build_metric_definitions", return_value=metric_definitions
):
prom_metrics = OffloadPromMetrics(
vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
prom_metrics.observe(
{
_StatsKey.TYPES: {
PENDING_STORES: _MetricType.GAUGE,
LOOKUP_LATENCY: _MetricType.HISTOGRAM,
},
_StatsKey.DATA: {
PENDING_STORES: 5,
LOOKUP_LATENCY: [0.2, 0.4],
},
}
)
gauge = prom_metrics.offloading_metrics[(0, PENDING_STORES)]
histogram = prom_metrics.offloading_metrics[(0, LOOKUP_LATENCY)]
assert gauge.set_values == [5]
assert histogram.observed == [0.2, 0.4]
histogram_def = prom_metrics._offloading_metric_defs[LOOKUP_LATENCY]
assert histogram_def.kwargs["buckets"] == (0.1, 1.0)
def test_prom_metrics_uses_configured_manager_metrics():
prom_metrics = OffloadPromMetrics(
vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
assert STORES_SKIPPED not in prom_metrics._offloading_metric_metadata
def test_aggregate_into_empty_stats():
"""Aggregating non-empty stats into a fresh (empty) stats object works."""
empty = OffloadingConnectorStats()
assert empty.is_empty()
non_empty = OffloadingConnectorStats(
data={
_StatsKey.TYPES: {
LOAD_BYTES: _MetricType.COUNTER,
LOAD_SIZE: _MetricType.HISTOGRAM,
PENDING_STORES: _MetricType.GAUGE,
},
_StatsKey.DATA: {
LOAD_BYTES: 42,
LOAD_SIZE: [10, 20],
PENDING_STORES: 3,
},
},
)
result = empty.aggregate(non_empty)
assert result is empty
values = result.data[_StatsKey.DATA]
assert values[LOAD_BYTES] == 42
assert values[LOAD_SIZE] == [10, 20]
assert values[PENDING_STORES] == 3
def test_prom_metrics_multi_engine_routing():
"""Metrics are routed to the correct engine index."""
prom_metrics = OffloadPromMetrics(
vllm_config=_FakeVllmConfig(), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"], 1: ["model", "1"]},
)
prom_metrics.observe(
{
_StatsKey.TYPES: {LOAD_BYTES: _MetricType.COUNTER},
_StatsKey.DATA: {LOAD_BYTES: 100},
},
engine_idx=1,
)
engine0 = prom_metrics.offloading_metrics[(0, LOAD_BYTES)]
engine1 = prom_metrics.offloading_metrics[(1, LOAD_BYTES)]
assert engine0.increments == []
assert engine1.increments == [100]
def test_prom_metrics_rejects_undeclared_metric():
"""observe() asserts if a metric was never declared in metadata."""
prom_metrics = OffloadPromMetrics(
vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
with pytest.raises(AssertionError):
prom_metrics.observe(
{
_StatsKey.TYPES: {"unknown:metric": _MetricType.COUNTER},
_StatsKey.DATA: {"unknown:metric": 1},
}
)
@@ -4,7 +4,9 @@
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
DirectionalTransferStats,
OffloadingWorkerMetadata,
TransferStats,
)
pytestmark = pytest.mark.cpu_test
@@ -30,3 +32,22 @@ def test_aggregate_multiple_workers():
meta3 = OffloadingWorkerMetadata(completed_jobs={42: 1, 43: 1, 8: 1})
result = meta1.aggregate(meta2).aggregate(meta3)
assert result.completed_jobs == {42: 3, 43: 2, 7: 2, 8: 2}
def test_aggregate_transfer_stats():
meta1 = OffloadingWorkerMetadata(
transfer_stats=TransferStats(
load=DirectionalTransferStats(bytes=10, time=0.5, sizes=[10])
)
)
meta2 = OffloadingWorkerMetadata(
transfer_stats=TransferStats(
load=DirectionalTransferStats(bytes=20, time=1.0, sizes=[20, 30])
)
)
result = meta1.aggregate(meta2)
assert result.transfer_stats.load.bytes == 30
assert result.transfer_stats.load.time == 1.5
assert result.transfer_stats.load.sizes == [10, 20, 30]
@@ -5,6 +5,7 @@ import time
import msgspec
import msgspec.msgpack
import prometheus_client
import pytest
import zmq
from tqdm import tqdm
@@ -303,6 +304,167 @@ def test_cpu_offloading(
del llm
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
def test_cpu_offloading_metrics() -> None:
"""Verify that offloading Prometheus metrics (new flat and deprecated
labeled) are emitted after stores and loads."""
extra_config: dict = {
"cpu_bytes_to_use": 500 << 20,
"block_size": CPU_BLOCK_SIZES,
}
kv_transfer_config = KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
kv_connector_extra_config=extra_config,
)
llm = LLM(
model="meta-llama/Llama-3.2-1B-Instruct",
max_model_len=4096,
gpu_memory_utilization=0.5,
kv_transfer_config=kv_transfer_config,
disable_log_stats=False,
)
try:
prompt_token_ids = list(range(500))
# First generate: cold run, triggers a store to CPU.
# Use max_tokens>1 so the request is still producing output
# tokens when the async store completes and stats get drained.
# (The LLMEngine only records stats on steps with request outputs.)
llm.generate(
[TokensPrompt(prompt_token_ids=prompt_token_ids)],
SamplingParams(max_tokens=10),
use_tqdm=False,
)
# Wait for the async offload to finish, then reset GPU prefix cache
# so the next generate must load from CPU.
_wait_for_prefix_cache_reset(llm)
# Second generate: triggers a load from CPU.
# Send a short filler alongside the load prompt so there's always
# a request producing output tokens when the load stats get drained
# (the LLMEngine only records stats on steps with request outputs).
filler = TokensPrompt(prompt_token_ids=[0])
llm.generate(
[filler, TokensPrompt(prompt_token_ids=prompt_token_ids)],
SamplingParams(max_tokens=50),
use_tqdm=False,
)
# Metric helpers.
registry = prometheus_client.REGISTRY
def _get_counter_value(
name: str, labels: dict[str, str] | None = None
) -> float:
total = 0.0
for metric in registry.collect():
if metric.name == name:
for sample in metric.samples:
if sample.name != name + "_total":
continue
if labels and not all(
sample.labels.get(k) == v for k, v in labels.items()
):
continue
total += sample.value
return total
def _get_histogram_count(
name: str, labels: dict[str, str] | None = None
) -> float:
total = 0.0
for metric in registry.collect():
if metric.name == name:
for sample in metric.samples:
if sample.name != name + "_count":
continue
if labels and not all(
sample.labels.get(k) == v for k, v in labels.items()
):
continue
total += sample.value
return total
# Stats are drained asynchronously — if the transfer finishes
# after the last engine step for that generate() call, the metrics
# won't appear until a subsequent step. Retry with dummy generates
# to force additional stats drains.
deadline = time.monotonic() + _RESET_CACHE_TIMEOUT
while time.monotonic() < deadline:
store_bytes = _get_counter_value("vllm:kv_offload_store_bytes")
load_bytes = _get_counter_value("vllm:kv_offload_load_bytes")
if store_bytes > 0 and load_bytes > 0:
break
llm.generate(
[TokensPrompt(prompt_token_ids=[0])],
SamplingParams(max_tokens=1),
use_tqdm=False,
)
# New flat counter metrics
store_bytes = _get_counter_value("vllm:kv_offload_store_bytes")
assert store_bytes > 0, f"Expected store_bytes > 0, got {store_bytes}"
load_bytes = _get_counter_value("vllm:kv_offload_load_bytes")
assert load_bytes > 0, f"Expected load_bytes > 0, got {load_bytes}"
store_time = _get_counter_value("vllm:kv_offload_store_time")
assert store_time > 0, f"Expected store_time > 0, got {store_time}"
load_time = _get_counter_value("vllm:kv_offload_load_time")
assert load_time > 0, f"Expected load_time > 0, got {load_time}"
# New flat histogram metrics
store_size_count = _get_histogram_count("vllm:kv_offload_store_size")
assert store_size_count > 0, (
f"Expected store_size histogram observations > 0, got {store_size_count}"
)
load_size_count = _get_histogram_count("vllm:kv_offload_load_size")
assert load_size_count > 0, (
f"Expected load_size histogram observations > 0, got {load_size_count}"
)
# Deprecated labeled metrics — verify per transfer_type label.
load_label = {"transfer_type": "CPU_to_GPU"}
store_label = {"transfer_type": "GPU_to_CPU"}
dep_load_bytes = _get_counter_value("vllm:kv_offload_total_bytes", load_label)
assert dep_load_bytes > 0, (
f"Expected deprecated load bytes > 0, got {dep_load_bytes}"
)
dep_store_bytes = _get_counter_value("vllm:kv_offload_total_bytes", store_label)
assert dep_store_bytes > 0, (
f"Expected deprecated store bytes > 0, got {dep_store_bytes}"
)
dep_load_time = _get_counter_value("vllm:kv_offload_total_time", load_label)
assert dep_load_time > 0, (
f"Expected deprecated load time > 0, got {dep_load_time}"
)
dep_store_time = _get_counter_value("vllm:kv_offload_total_time", store_label)
assert dep_store_time > 0, (
f"Expected deprecated store time > 0, got {dep_store_time}"
)
dep_load_size = _get_histogram_count("vllm:kv_offload_size", load_label)
assert dep_load_size > 0, (
f"Expected deprecated load size observations > 0, got {dep_load_size}"
)
dep_store_size = _get_histogram_count("vllm:kv_offload_size", store_label)
assert dep_store_size > 0, (
f"Expected deprecated store size observations > 0, got {dep_store_size}"
)
# Flat and deprecated metrics must be consistent (dual-write).
assert store_bytes == dep_store_bytes
assert load_bytes == dep_load_bytes
assert store_time == dep_store_time
assert load_time == dep_load_time
assert store_size_count == dep_store_size
assert load_size_count == dep_load_size
finally:
del llm
def test_tiering_offloading() -> None:
"""Tests OffloadingConnector with TieringOffloadingSpec."""
extra_config: dict = {
+48 -7
View File
@@ -18,6 +18,8 @@ from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
def make_req_context(
req_id: str = "", kv_transfer_params: dict | None = None
@@ -29,6 +31,22 @@ def make_req_context(
_EMPTY_REQ_CTX = make_req_context()
def make_cpu_manager(
num_blocks: int = 4,
cache_policy: str = "lru",
enable_events: bool = False,
store_threshold: int = 0,
max_tracker_size: int = 64_000,
) -> CPUOffloadingManager:
return CPUOffloadingManager(
num_blocks=num_blocks,
cache_policy=cache_policy,
enable_events=enable_events,
store_threshold=store_threshold,
max_tracker_size=max_tracker_size,
)
@dataclass
class ExpectedPrepareStoreOutput:
keys_to_store: list[int]
@@ -110,7 +128,7 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
candidate to make room for [3, 4, 5]
- After complete_store([2, 3, 4, 5]), block 2 must still be present.
"""
manager = CPUOffloadingManager(
manager = make_cpu_manager(
num_blocks=4,
cache_policy=eviction_policy,
enable_events=True,
@@ -144,14 +162,37 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is True
def test_filter_reused_manager_reports_stores_skipped_counter():
manager = make_cpu_manager(
num_blocks=4,
cache_policy="lru",
store_threshold=2,
)
prepare_store_output = manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX)
verify_store_output(
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[],
store_block_ids=[],
evicted_keys=[],
),
)
stats = manager.get_stats()
assert stats is not None
assert stats.reduce()[STORES_SKIPPED] == 3
stats = manager.get_stats()
assert stats is not None
assert stats.reduce()[STORES_SKIPPED] == 0
def test_cpu_manager():
"""
Tests CPUOffloadingManager with lru policy.
"""
# initialize a CPU manager with a capacity of 4 blocks
cpu_manager = CPUOffloadingManager(
num_blocks=4, cache_policy="lru", enable_events=True
)
cpu_manager = make_cpu_manager(num_blocks=4, cache_policy="lru", enable_events=True)
# prepare store [1, 2]
prepare_store_output = cpu_manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -264,7 +305,7 @@ def test_cpu_manager():
def test_prepare_load_preserves_key_order():
"""block_ids[i] must correspond to keys[i] (co-indexed invariant)."""
manager = CPUOffloadingManager(num_blocks=4, cache_policy="lru")
manager = make_cpu_manager(num_blocks=4, cache_policy="lru")
key_a, key_b, key_c = to_key(0), to_key(1), to_key(2)
@@ -305,7 +346,7 @@ class TestARCPolicy:
def _make_manager(
self, num_blocks: int = 4, enable_events: bool = True
) -> tuple[CPUOffloadingManager, ARCCachePolicy]:
manager = CPUOffloadingManager(
manager = make_cpu_manager(
num_blocks=num_blocks,
cache_policy="arc",
enable_events=enable_events,
@@ -605,7 +646,7 @@ def test_filter_reused_manager():
"""
Tests CPUOffloadingManager reuse filtering (store_threshold=2).
"""
manager = CPUOffloadingManager(
manager = make_cpu_manager(
num_blocks=4,
cache_policy="lru",
enable_events=True,