[4/N] Expose HiSparse cache metrics via KV connector stats (#56061)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
OpenAI Codex
parent
dff76bc3e8
commit
29332cf936
@@ -17,9 +17,19 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorWorkerMetadata,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
|
||||
HiSparseKVConnectorStats,
|
||||
HiSparsePromMetrics,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
|
||||
HiSparseConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
@@ -244,6 +254,33 @@ class HiSparseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.reset_hot_state()
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
if self.connector_worker is None:
|
||||
return None
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
HiSparseKVConnectorStats(data=data)
|
||||
if data is not None
|
||||
else HiSparseKVConnectorStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return HiSparsePromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
|
||||
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stats and Prometheus metrics for the HiSparse connector."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.v1.metrics.utils import create_metric_per_engine
|
||||
|
||||
_HISPARSE_COUNTERS: tuple[tuple[str, str], ...] = (
|
||||
("cache_hits", "Number of HiSparse device hot-buffer hits."),
|
||||
("cache_misses", "Number of HiSparse device hot-buffer misses."),
|
||||
(
|
||||
"host_to_device_bytes",
|
||||
"Bytes transferred from host KV storage to HiSparse hot buffers.",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HiSparseKVConnectorStats(KVConnectorStats):
|
||||
"""Container for HiSparse hot-buffer residency metrics.
|
||||
|
||||
Each list entry is the delta recorded over one device counter snapshot
|
||||
interval, so a list can hold multiple snapshots per logging interval.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
# Empty container init, no data is passed in.
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
# Must be serializable
|
||||
self.data: dict[str, list[int]] = {
|
||||
"cache_hits": [],
|
||||
"cache_misses": [],
|
||||
"host_to_device_bytes": [],
|
||||
}
|
||||
|
||||
def record_snapshot(self, hits: int, misses: int, host_to_device_bytes: int):
|
||||
self.data["cache_hits"].append(hits)
|
||||
self.data["cache_misses"].append(misses)
|
||||
self.data["host_to_device_bytes"].append(host_to_device_bytes)
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if not other.is_empty():
|
||||
for k, v in other.data.items():
|
||||
accumulator = self.data[k]
|
||||
assert isinstance(accumulator, list)
|
||||
accumulator.extend(v)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
# Compute compact representative stats suitable for CLI logging.
|
||||
return {
|
||||
"HiSparse hot-buffer hits": sum(self.data["cache_hits"]),
|
||||
"HiSparse hot-buffer misses": sum(self.data["cache_misses"]),
|
||||
"HiSparse host-to-device bytes": sum(self.data["host_to_device_bytes"]),
|
||||
}
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return all(len(values) == 0 for values in self.data.values())
|
||||
|
||||
|
||||
class HiSparsePromMetrics(KVConnectorPromMetrics):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
):
|
||||
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
|
||||
|
||||
self.hisparse_counters: dict[str, Any] = {}
|
||||
for name, documentation in _HISPARSE_COUNTERS:
|
||||
counter = self._counter_cls(
|
||||
name=f"vllm:hisparse_{name}",
|
||||
documentation=documentation,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.hisparse_counters[name] = create_metric_per_engine(
|
||||
counter, self.per_engine_labelvalues
|
||||
)
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
for name, counter in self.hisparse_counters.items():
|
||||
for value in transfer_stats_data.get(name, []):
|
||||
counter[engine_idx].inc(value)
|
||||
@@ -15,6 +15,9 @@ import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
|
||||
HiSparseKVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tp_group,
|
||||
@@ -68,6 +71,9 @@ def _allocate_dma_descriptors(size: int) -> _DMADescriptors:
|
||||
)
|
||||
|
||||
|
||||
_METRICS_INTERVAL = 2000
|
||||
|
||||
|
||||
def _get_hisparse_cache(
|
||||
forward_context: dict[str, Any], layer_name: str
|
||||
) -> HiSparseCacheHandle:
|
||||
@@ -311,6 +317,9 @@ class HiSparseConnectorWorker:
|
||||
self._pending_transfer_events: deque[tuple[torch.Event, tuple[int, ...]]] = (
|
||||
deque()
|
||||
)
|
||||
self._metrics_calls = 0
|
||||
self._metrics_event = torch.Event()
|
||||
self._metrics_pending = False
|
||||
self._init_dma()
|
||||
if self.is_host_writer:
|
||||
for layer_index, handle in enumerate(cache_handles):
|
||||
@@ -496,6 +505,36 @@ class HiSparseConnectorWorker:
|
||||
for runtime in self.leader_runtimes:
|
||||
runtime.reset_hot_state()
|
||||
|
||||
def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None:
|
||||
stats = None
|
||||
if self._metrics_pending and self._metrics_event.query():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
for runtime in self.leader_runtimes:
|
||||
group = runtime.index_group
|
||||
hits, misses = group.swap_stats_host.tolist()
|
||||
if hits or misses:
|
||||
stats.record_snapshot(hits, misses, misses * group.stats_row_bytes)
|
||||
self._metrics_pending = False
|
||||
if stats.is_empty():
|
||||
stats = None
|
||||
|
||||
self._metrics_calls += 1
|
||||
if (
|
||||
self._metrics_calls % _METRICS_INTERVAL == 0
|
||||
and not self._metrics_pending
|
||||
and not torch.cuda.is_current_stream_capturing()
|
||||
):
|
||||
compute_stream = current_stream()
|
||||
for runtime in self.leader_runtimes:
|
||||
group = runtime.index_group
|
||||
compute_stream.wait_stream(group.copy_stream)
|
||||
group.swap_stats_host.copy_(group.swap_stats, non_blocking=True)
|
||||
group.swap_stats.zero_()
|
||||
group.copy_stream.wait_stream(compute_stream)
|
||||
self._metrics_event.record()
|
||||
self._metrics_pending = True
|
||||
return stats
|
||||
|
||||
def _release_completed_dma_descriptors(self) -> None:
|
||||
pending = self._pending_dma_descriptors
|
||||
while pending and pending[0][0].query():
|
||||
|
||||
Reference in New Issue
Block a user