[4/N] Expose HiSparse cache metrics via KV connector stats (#56061)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Nicolò Lucchesi
2026-09-12 10:47:55 -07:00
committed by GitHub
co-authored by Lucas Wilkinson OpenAI Codex
parent dff76bc3e8
commit 29332cf936
9 changed files with 308 additions and 8 deletions
@@ -17,9 +17,19 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorWorkerMetadata,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
HiSparseKVConnectorStats,
HiSparsePromMetrics,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
HiSparseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.core.sched.output import SchedulerOutput
@@ -244,6 +254,33 @@ class HiSparseConnector(KVConnectorBase_V1, SupportsHMA):
assert self.connector_worker is not None
self.connector_worker.reset_hot_state()
def get_kv_connector_stats(self) -> KVConnectorStats | None:
if self.connector_worker is None:
return None
return self.connector_worker.get_kv_connector_stats()
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
HiSparseKVConnectorStats(data=data)
if data is not None
else HiSparseKVConnectorStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return HiSparsePromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
@@ -0,0 +1,97 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stats and Prometheus metrics for the HiSparse connector."""
from dataclasses import dataclass
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.v1.metrics.utils import create_metric_per_engine
_HISPARSE_COUNTERS: tuple[tuple[str, str], ...] = (
("cache_hits", "Number of HiSparse device hot-buffer hits."),
("cache_misses", "Number of HiSparse device hot-buffer misses."),
(
"host_to_device_bytes",
"Bytes transferred from host KV storage to HiSparse hot buffers.",
),
)
@dataclass
class HiSparseKVConnectorStats(KVConnectorStats):
"""Container for HiSparse hot-buffer residency metrics.
Each list entry is the delta recorded over one device counter snapshot
interval, so a list can hold multiple snapshots per logging interval.
"""
def __post_init__(self):
if not self.data:
# Empty container init, no data is passed in.
self.reset()
def reset(self):
# Must be serializable
self.data: dict[str, list[int]] = {
"cache_hits": [],
"cache_misses": [],
"host_to_device_bytes": [],
}
def record_snapshot(self, hits: int, misses: int, host_to_device_bytes: int):
self.data["cache_hits"].append(hits)
self.data["cache_misses"].append(misses)
self.data["host_to_device_bytes"].append(host_to_device_bytes)
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if not other.is_empty():
for k, v in other.data.items():
accumulator = self.data[k]
assert isinstance(accumulator, list)
accumulator.extend(v)
return self
def reduce(self) -> dict[str, int | float]:
# Compute compact representative stats suitable for CLI logging.
return {
"HiSparse hot-buffer hits": sum(self.data["cache_hits"]),
"HiSparse hot-buffer misses": sum(self.data["cache_misses"]),
"HiSparse host-to-device bytes": sum(self.data["host_to_device_bytes"]),
}
def is_empty(self) -> bool:
return all(len(values) == 0 for values in self.data.values())
class HiSparsePromMetrics(KVConnectorPromMetrics):
def __init__(
self,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
self.hisparse_counters: dict[str, Any] = {}
for name, documentation in _HISPARSE_COUNTERS:
counter = self._counter_cls(
name=f"vllm:hisparse_{name}",
documentation=documentation,
labelnames=labelnames,
)
self.hisparse_counters[name] = create_metric_per_engine(
counter, self.per_engine_labelvalues
)
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
for name, counter in self.hisparse_counters.items():
for value in transfer_stats_data.get(name, []):
counter[engine_idx].inc(value)
@@ -15,6 +15,9 @@ import torch
from vllm import _custom_ops as ops
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
HiSparseKVConnectorStats,
)
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tp_group,
@@ -68,6 +71,9 @@ def _allocate_dma_descriptors(size: int) -> _DMADescriptors:
)
_METRICS_INTERVAL = 2000
def _get_hisparse_cache(
forward_context: dict[str, Any], layer_name: str
) -> HiSparseCacheHandle:
@@ -311,6 +317,9 @@ class HiSparseConnectorWorker:
self._pending_transfer_events: deque[tuple[torch.Event, tuple[int, ...]]] = (
deque()
)
self._metrics_calls = 0
self._metrics_event = torch.Event()
self._metrics_pending = False
self._init_dma()
if self.is_host_writer:
for layer_index, handle in enumerate(cache_handles):
@@ -496,6 +505,36 @@ class HiSparseConnectorWorker:
for runtime in self.leader_runtimes:
runtime.reset_hot_state()
def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None:
stats = None
if self._metrics_pending and self._metrics_event.query():
stats = HiSparseKVConnectorStats()
for runtime in self.leader_runtimes:
group = runtime.index_group
hits, misses = group.swap_stats_host.tolist()
if hits or misses:
stats.record_snapshot(hits, misses, misses * group.stats_row_bytes)
self._metrics_pending = False
if stats.is_empty():
stats = None
self._metrics_calls += 1
if (
self._metrics_calls % _METRICS_INTERVAL == 0
and not self._metrics_pending
and not torch.cuda.is_current_stream_capturing()
):
compute_stream = current_stream()
for runtime in self.leader_runtimes:
group = runtime.index_group
compute_stream.wait_stream(group.copy_stream)
group.swap_stats_host.copy_(group.swap_stats, non_blocking=True)
group.swap_stats.zero_()
group.copy_stream.wait_stream(compute_stream)
self._metrics_event.record()
self._metrics_pending = True
return stats
def _release_completed_dma_descriptors(self) -> None:
pending = self._pending_dma_descriptors
while pending and pending[0][0].query():