[HiSparse] Report host-pool block residency as HiSparse metrics (#59485)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-09 16:45:34 -06:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 783ac12071
commit 4c43254d52
7 changed files with 147 additions and 6 deletions
@@ -3,7 +3,7 @@
import contextlib
import ctypes
from types import SimpleNamespace
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
import pytest
import torch
@@ -25,6 +25,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
HiSparseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
from vllm.v1.hisparse import runtime as runtime_module
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.hisparse.runtime import HiSparseCacheHandle
@@ -33,6 +34,7 @@ from vllm.v1.hisparse.types import (
SparseKVPageTransfer,
SparseKVRowMirror,
)
from vllm.v1.metrics.stats import KVCacheEvictionEvent
from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector
@@ -100,6 +102,46 @@ def test_scheduler_stats_report_host_pool_usage():
assert stats.data["host_cache_usage_perc"] == [expected]
def test_hisparse_host_residency_reported_as_hisparse_metrics():
"""Host blocks sample into HiSparse's own residency metrics."""
from tests.v1.core.test_prefix_caching import (
make_hisparse_kv_cache_config,
make_kv_cache_manager,
)
from tests.v1.core.utils import create_requests
collector = KVCacheMetricsCollector(sample_rate=1.0)
manager = make_kv_cache_manager(
make_hisparse_kv_cache_config(2, 2),
max_model_len=128,
enable_caching=True,
hash_block_size=16,
metrics_collector=collector,
)
coordinator = get_hisparse_coordinator(manager)
assert coordinator.host_manager is not None
scheduler = HiSparseConnectorScheduler(async_speculative=False)
scheduler.bind_coordinator(coordinator)
device = manager.block_pool
host = coordinator.host_manager.block_pool
request = create_requests(1, num_tokens=16)[0]
for pool, birth in ((device, 1), (host, 2)):
with patch("time.monotonic_ns", return_value=birth * 10**9):
block = pool.get_new_blocks(1)[0]
assert block.block_id == 1
pool.cache_full_blocks(request, [block], 0, 1, 16, 0)
with patch("time.monotonic_ns", return_value=4_000_000_000):
host.free_blocks([host.blocks[1]])
host.evict_blocks({1})
with patch("time.monotonic_ns", return_value=6_000_000_000):
device.evict_blocks({1})
assert collector.drain_events() == [KVCacheEvictionEvent(5.0, 5.0, ())]
stats = scheduler.get_kv_connector_stats()
assert stats.data["host_block_lifetime_seconds"] == [2.0]
assert stats.data["host_block_idle_before_evict_seconds"] == [2.0]
def test_no_forward_enqueues_deferred_hisparse_transfers():
"""A zero-token step must still enqueue deferred post-forward transfers."""
connector = object.__new__(ActiveKVConnector)
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnector,
)
@@ -82,7 +84,8 @@ def test_build_kv_connector_stats_round_trip():
assert rebuilt.reduce() == stats.reduce()
def test_prom_metrics_observe_host_usage_gauges():
@pytest.mark.parametrize("kv_cache_metrics", [True, False])
def test_prom_metrics_observe_host_metrics(kv_cache_metrics: bool):
from types import SimpleNamespace
from typing import Any
@@ -91,12 +94,14 @@ def test_prom_metrics_observe_host_usage_gauges():
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
HiSparsePromMetrics,
)
from vllm.v1.metrics.stats import KVCacheEvictionEvent
class _FakeMetric:
def __init__(self, **kwargs: Any):
self.kwargs = kwargs
self.increments: list[int | float] = []
self.set_values: list[int | float] = []
self.observed: list[float] = []
def labels(self, *labelvalues: object) -> "_FakeMetric":
return self
@@ -107,6 +112,9 @@ def test_prom_metrics_observe_host_usage_gauges():
def set(self, value: int | float) -> None:
self.set_values.append(value)
def observe(self, value: float) -> None:
self.observed.append(value)
created: dict[str, _FakeMetric] = {}
class _NamedFake(_FakeMetric):
@@ -115,7 +123,12 @@ def test_prom_metrics_observe_host_usage_gauges():
created[kwargs["name"]] = self
prom = HiSparsePromMetrics(
vllm_config=SimpleNamespace(kv_transfer_config=None),
vllm_config=SimpleNamespace(
kv_transfer_config=None,
observability_config=SimpleNamespace(
kv_cache_metrics=kv_cache_metrics, custom_histogram_buckets=None
),
),
metric_types={Gauge: _NamedFake, Counter: _NamedFake, Histogram: _NamedFake},
labelnames=["model_name"],
per_engine_labelvalues={0: ["model"]},
@@ -125,9 +138,21 @@ def test_prom_metrics_observe_host_usage_gauges():
stats.record_snapshot(hits=3, misses=2, host_to_device_bytes=32)
stats.record_host_usage(usage=0.5, pending_page_transfers=1)
stats.record_host_usage(usage=0.75, pending_page_transfers=3)
prom.observe(stats.to_dict())
stats.record_host_evictions([KVCacheEvictionEvent(5.0, 2.0, (1.0, 2.0))])
later = HiSparseKVConnectorStats()
later.record_host_evictions([KVCacheEvictionEvent(3.0, 3.0, ())])
prom.observe(stats.aggregate(later).to_dict())
assert created["vllm:hisparse_cache_hits"].increments == [3]
assert created["vllm:hisparse_host_to_device_bytes"].increments == [32]
assert created["vllm:hisparse_host_cache_usage_perc"].set_values == [0.75]
assert created["vllm:hisparse_pending_page_transfers"].set_values == [3]
if not kv_cache_metrics:
assert not any("host_block" in name for name in created)
return
assert created["vllm:hisparse_host_block_lifetime_seconds"].observed == [5.0, 3.0]
assert created["vllm:hisparse_host_block_idle_before_evict_seconds"].observed == [
2.0,
3.0,
]
assert created["vllm:hisparse_host_block_reuse_gap_seconds"].observed == [1.0, 2.0]
+3
View File
@@ -94,6 +94,9 @@ def test_hisparse_worker_get_kv_connector_stats_reads_completed_snapshot(monkeyp
"host_to_device_bytes": [64],
"host_cache_usage_perc": [],
"pending_page_transfers": [],
"host_block_lifetime_seconds": [],
"host_block_idle_before_evict_seconds": [],
"host_block_reuse_gap_seconds": [],
}