[4/N] Expose HiSparse cache metrics via KV connector stats (#56061)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Nicolò Lucchesi
2026-09-12 10:47:55 -07:00
committed by GitHub
co-authored by Lucas Wilkinson OpenAI Codex
parent dff76bc3e8
commit 29332cf936
9 changed files with 308 additions and 8 deletions
+19 -1
View File
@@ -203,6 +203,7 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel(
int32_t* __restrict__ miss_mask, // [num_rows, top_k] or nullptr
int32_t* __restrict__ device_global_indices, // [max_rows, region_stride]
int16_t* __restrict__ lru_slots, // [max_rows, hot_size]
unsigned long long* __restrict__ stats, // [2] hits,misses or nullptr
const int32_t* __restrict__ request_state_indices, // [num_requests] or
// nullptr
const int32_t request_state_count, const int64_t host_rows,
@@ -552,6 +553,11 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel(
if (swap_counts != nullptr && tid == 0) {
swap_counts[batch_row] = total_misses;
}
if (stats != nullptr && tid == 0) {
atomicAdd(&stats[0], static_cast<unsigned long long>(total_hits));
atomicAdd(&stats[1], static_cast<unsigned long long>(total_misses));
}
// Phase 4: write back the LRU order: stale evictables at the front,
// freshly loaded misses next, then hits at MRU.
const int total_evictable = hot_size - total_hits;
@@ -779,6 +785,7 @@ void hisparse_resolve_residency(
std::optional<torch::stable::Tensor> const& request_state_indices,
int64_t region_stride,
std::optional<torch::stable::Tensor> const& miss_mask,
std::optional<torch::stable::Tensor> const& stats,
std::optional<torch::stable::Tensor> const& attention_indices,
int64_t attention_block_stride,
std::optional<torch::stable::Tensor> const& request_ids,
@@ -997,6 +1004,17 @@ void hisparse_resolve_residency(
miss_mask_ptr = mm.mutable_data_ptr<int32_t>();
}
unsigned long long* stats_ptr = nullptr;
if (stats.has_value()) {
auto const& st = stats.value();
STD_TORCH_CHECK(
st.is_cuda() && st.is_contiguous() && st.dim() == 1 &&
st.scalar_type() == torch::headeronly::ScalarType::UInt64 &&
st.numel() == 2,
"stats must be a contiguous two-element uint64 CUDA tensor");
stats_ptr = static_cast<unsigned long long*>(st.mutable_data_ptr());
}
int32_t* attention_indices_ptr = nullptr;
if (attention_indices.has_value()) {
auto const& indices = attention_indices.value();
@@ -1046,7 +1064,7 @@ void hisparse_resolve_residency(
swap_counts_ptr, hot_indices.mutable_data_ptr<int32_t>(),
attention_indices_ptr, miss_mask_ptr,
device_global_indices.mutable_data_ptr<int32_t>(),
lru_slots.mutable_data_ptr<int16_t>(), request_state_ptr,
lru_slots.mutable_data_ptr<int16_t>(), stats_ptr, request_state_ptr,
request_state_count, host_rows, hot_block_table.stride(0), hot_block_size,
top_k, hot_size, hash_size, region_stride, attention_block_stride,
source_bt_stride, source_num_reqs, source_num_blocks,
+1
View File
@@ -673,6 +673,7 @@ void hisparse_resolve_residency(
std::optional<torch::stable::Tensor> const& request_state_indices,
int64_t region_stride,
std::optional<torch::stable::Tensor> const& miss_mask,
std::optional<torch::stable::Tensor> const& stats,
std::optional<torch::stable::Tensor> const& attention_indices,
int64_t attention_block_stride,
std::optional<torch::stable::Tensor> const& request_ids,
+7 -6
View File
@@ -999,16 +999,17 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) {
" Tensor? request_state_indices,"
" int region_stride,"
" Tensor(a!)? miss_mask=None,"
" Tensor(b!)? attention_indices=None,"
" Tensor(b!)? stats=None,"
" Tensor(c!)? attention_indices=None,"
" int attention_block_stride=0,"
" Tensor? request_ids=None,"
" Tensor? source_block_table=None,"
" int source_block_size=0,"
" Tensor(c!)? resolved_global_indices=None,"
" Tensor(d!)? valid_counts=None,"
" Tensor(e!)? swap_host_physical_rows=None,"
" Tensor(f!)? swap_device_physical_rows=None,"
" Tensor(g!)? swap_counts=None,"
" Tensor(d!)? resolved_global_indices=None,"
" Tensor(e!)? valid_counts=None,"
" Tensor(f!)? swap_host_physical_rows=None,"
" Tensor(g!)? swap_device_physical_rows=None,"
" Tensor(h!)? swap_counts=None,"
" Tensor? resident_block_table=None,"
" int resident_block_size=0,"
" int resident_null_block=0) -> ()");
@@ -0,0 +1,62 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnector,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
HiSparseKVConnectorStats,
)
def test_is_empty_on_fresh_stats():
stats = HiSparseKVConnectorStats()
assert stats.is_empty()
def test_record_snapshot_and_reduce():
stats = HiSparseKVConnectorStats()
stats.record_snapshot(hits=7, misses=3, host_to_device_bytes=48)
stats.record_snapshot(hits=5, misses=1, host_to_device_bytes=16)
assert not stats.is_empty()
reduced = stats.reduce()
assert reduced["HiSparse hot-buffer hits"] == 12
assert reduced["HiSparse hot-buffer misses"] == 4
assert reduced["HiSparse host-to-device bytes"] == 64
def test_aggregate_extends_snapshot_deltas():
first = HiSparseKVConnectorStats()
first.record_snapshot(hits=7, misses=3, host_to_device_bytes=48)
second = HiSparseKVConnectorStats()
second.record_snapshot(hits=5, misses=1, host_to_device_bytes=16)
first.aggregate(second)
assert first.data == {
"cache_hits": [7, 5],
"cache_misses": [3, 1],
"host_to_device_bytes": [48, 16],
}
def test_aggregate_skips_empty_stats():
stats = HiSparseKVConnectorStats()
stats.record_snapshot(hits=2, misses=1, host_to_device_bytes=16)
stats.aggregate(HiSparseKVConnectorStats())
assert stats.data["cache_hits"] == [2]
def test_build_kv_connector_stats_round_trip():
stats = HiSparseKVConnectorStats()
stats.record_snapshot(hits=12, misses=4, host_to_device_bytes=64)
payload = stats.to_dict()
rebuilt = HiSparseConnector.build_kv_connector_stats(data=payload)
assert rebuilt is not None
assert isinstance(rebuilt, HiSparseKVConnectorStats)
assert rebuilt.reduce() == stats.reduce()
+38 -1
View File
@@ -48,10 +48,47 @@ def _make_hisparse_worker() -> HiSparseConnectorWorker:
worker.dma_stream = None
worker.shared_host_region = None
worker._completed_host_copy_dst_ids = []
worker._slot_mapping_staging = None
worker._metrics_calls = 0
worker._metrics_event = MagicMock()
worker._metrics_pending = False
worker.leader_runtimes = []
return worker
def test_hisparse_worker_get_kv_connector_stats_reads_completed_snapshot(monkeypatch):
worker = _make_hisparse_worker()
worker._metrics_calls = hisparse_worker_module._METRICS_INTERVAL - 1
worker._metrics_pending = False
worker._metrics_event = MagicMock()
worker._metrics_event.query.return_value = True
compute_stream = MagicMock()
group = SimpleNamespace(
swap_stats=torch.tensor([12, 4], dtype=torch.uint64),
swap_stats_host=torch.empty(2, dtype=torch.uint64),
stats_row_bytes=16,
copy_stream=MagicMock(),
)
worker.leader_runtimes = [SimpleNamespace(index_group=group)]
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
monkeypatch.setattr(
hisparse_worker_module, "current_stream", lambda: compute_stream
)
assert worker.get_kv_connector_stats() is None
compute_stream.wait_stream.assert_called_once_with(group.copy_stream)
group.copy_stream.wait_stream.assert_called_once_with(compute_stream)
worker._metrics_event.record.assert_called_once_with()
assert group.swap_stats.tolist() == [0, 0]
stats = worker.get_kv_connector_stats()
assert stats is not None
assert stats.data == {
"cache_hits": [12],
"cache_misses": [4],
"host_to_device_bytes": [64],
}
def test_hisparse_row_mirrors_follow_runner_request_order():
first = SparseKVRowMirror((1,), 10, 1)
second = SparseKVRowMirror((2,), 20, 1)
@@ -17,9 +17,19 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorWorkerMetadata,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
HiSparseKVConnectorStats,
HiSparsePromMetrics,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
HiSparseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.core.sched.output import SchedulerOutput
@@ -244,6 +254,33 @@ class HiSparseConnector(KVConnectorBase_V1, SupportsHMA):
assert self.connector_worker is not None
self.connector_worker.reset_hot_state()
def get_kv_connector_stats(self) -> KVConnectorStats | None:
if self.connector_worker is None:
return None
return self.connector_worker.get_kv_connector_stats()
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
HiSparseKVConnectorStats(data=data)
if data is not None
else HiSparseKVConnectorStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return HiSparsePromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
@@ -0,0 +1,97 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stats and Prometheus metrics for the HiSparse connector."""
from dataclasses import dataclass
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.v1.metrics.utils import create_metric_per_engine
_HISPARSE_COUNTERS: tuple[tuple[str, str], ...] = (
("cache_hits", "Number of HiSparse device hot-buffer hits."),
("cache_misses", "Number of HiSparse device hot-buffer misses."),
(
"host_to_device_bytes",
"Bytes transferred from host KV storage to HiSparse hot buffers.",
),
)
@dataclass
class HiSparseKVConnectorStats(KVConnectorStats):
"""Container for HiSparse hot-buffer residency metrics.
Each list entry is the delta recorded over one device counter snapshot
interval, so a list can hold multiple snapshots per logging interval.
"""
def __post_init__(self):
if not self.data:
# Empty container init, no data is passed in.
self.reset()
def reset(self):
# Must be serializable
self.data: dict[str, list[int]] = {
"cache_hits": [],
"cache_misses": [],
"host_to_device_bytes": [],
}
def record_snapshot(self, hits: int, misses: int, host_to_device_bytes: int):
self.data["cache_hits"].append(hits)
self.data["cache_misses"].append(misses)
self.data["host_to_device_bytes"].append(host_to_device_bytes)
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if not other.is_empty():
for k, v in other.data.items():
accumulator = self.data[k]
assert isinstance(accumulator, list)
accumulator.extend(v)
return self
def reduce(self) -> dict[str, int | float]:
# Compute compact representative stats suitable for CLI logging.
return {
"HiSparse hot-buffer hits": sum(self.data["cache_hits"]),
"HiSparse hot-buffer misses": sum(self.data["cache_misses"]),
"HiSparse host-to-device bytes": sum(self.data["host_to_device_bytes"]),
}
def is_empty(self) -> bool:
return all(len(values) == 0 for values in self.data.values())
class HiSparsePromMetrics(KVConnectorPromMetrics):
def __init__(
self,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
self.hisparse_counters: dict[str, Any] = {}
for name, documentation in _HISPARSE_COUNTERS:
counter = self._counter_cls(
name=f"vllm:hisparse_{name}",
documentation=documentation,
labelnames=labelnames,
)
self.hisparse_counters[name] = create_metric_per_engine(
counter, self.per_engine_labelvalues
)
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
for name, counter in self.hisparse_counters.items():
for value in transfer_stats_data.get(name, []):
counter[engine_idx].inc(value)
@@ -15,6 +15,9 @@ import torch
from vllm import _custom_ops as ops
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
HiSparseKVConnectorStats,
)
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tp_group,
@@ -68,6 +71,9 @@ def _allocate_dma_descriptors(size: int) -> _DMADescriptors:
)
_METRICS_INTERVAL = 2000
def _get_hisparse_cache(
forward_context: dict[str, Any], layer_name: str
) -> HiSparseCacheHandle:
@@ -311,6 +317,9 @@ class HiSparseConnectorWorker:
self._pending_transfer_events: deque[tuple[torch.Event, tuple[int, ...]]] = (
deque()
)
self._metrics_calls = 0
self._metrics_event = torch.Event()
self._metrics_pending = False
self._init_dma()
if self.is_host_writer:
for layer_index, handle in enumerate(cache_handles):
@@ -496,6 +505,36 @@ class HiSparseConnectorWorker:
for runtime in self.leader_runtimes:
runtime.reset_hot_state()
def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None:
stats = None
if self._metrics_pending and self._metrics_event.query():
stats = HiSparseKVConnectorStats()
for runtime in self.leader_runtimes:
group = runtime.index_group
hits, misses = group.swap_stats_host.tolist()
if hits or misses:
stats.record_snapshot(hits, misses, misses * group.stats_row_bytes)
self._metrics_pending = False
if stats.is_empty():
stats = None
self._metrics_calls += 1
if (
self._metrics_calls % _METRICS_INTERVAL == 0
and not self._metrics_pending
and not torch.cuda.is_current_stream_capturing()
):
compute_stream = current_stream()
for runtime in self.leader_runtimes:
group = runtime.index_group
compute_stream.wait_stream(group.copy_stream)
group.swap_stats_host.copy_(group.swap_stats, non_blocking=True)
group.swap_stats.zero_()
group.copy_stream.wait_stream(compute_stream)
self._metrics_event.record()
self._metrics_pending = True
return stats
def _release_completed_dma_descriptors(self) -> None:
pending = self._pending_dma_descriptors
while pending and pending[0][0].query():
+8
View File
@@ -464,6 +464,11 @@ class HiSparseIndexGroup:
)
self.logical_topk_ready = logical_topk_ready
self.followers: list[HiSparseRuntime] = []
self.swap_stats = torch.zeros(2, dtype=torch.uint64, device=device)
self.swap_stats_host = torch.empty(
2, dtype=torch.uint64, device="cpu", pin_memory=True
)
self.stats_row_bytes = 0
class HiSparseRuntime:
@@ -526,6 +531,7 @@ class HiSparseRuntime:
else:
index_group.followers.append(self)
self.index_group = index_group
index_group.stats_row_bytes += row_bytes
self.eager_host_mirror = config.eager_host_mirror
self.resident_source_index = -1
@@ -639,6 +645,7 @@ class HiSparseRuntime:
compute_stream.wait_stream(group.copy_stream)
group.device_global_indices.fill_(-1)
group.lru_slots.copy_(group.lru_init.expand_as(group.lru_slots))
group.swap_stats.zero_()
group.copy_stream.wait_stream(compute_stream)
def invalidate_sorted_slots(
@@ -741,6 +748,7 @@ class HiSparseRuntime:
request_state_indices,
self.region_stride,
None,
group.swap_stats,
physical_topk_indices,
hot.attention_block_stride,
req_id_per_token[:num_tokens].contiguous(),