[4/N] Expose HiSparse cache metrics via KV connector stats (#56061)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
OpenAI Codex
parent
dff76bc3e8
commit
29332cf936
@@ -203,6 +203,7 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel(
|
||||
int32_t* __restrict__ miss_mask, // [num_rows, top_k] or nullptr
|
||||
int32_t* __restrict__ device_global_indices, // [max_rows, region_stride]
|
||||
int16_t* __restrict__ lru_slots, // [max_rows, hot_size]
|
||||
unsigned long long* __restrict__ stats, // [2] hits,misses or nullptr
|
||||
const int32_t* __restrict__ request_state_indices, // [num_requests] or
|
||||
// nullptr
|
||||
const int32_t request_state_count, const int64_t host_rows,
|
||||
@@ -552,6 +553,11 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel(
|
||||
if (swap_counts != nullptr && tid == 0) {
|
||||
swap_counts[batch_row] = total_misses;
|
||||
}
|
||||
if (stats != nullptr && tid == 0) {
|
||||
atomicAdd(&stats[0], static_cast<unsigned long long>(total_hits));
|
||||
atomicAdd(&stats[1], static_cast<unsigned long long>(total_misses));
|
||||
}
|
||||
|
||||
// Phase 4: write back the LRU order: stale evictables at the front,
|
||||
// freshly loaded misses next, then hits at MRU.
|
||||
const int total_evictable = hot_size - total_hits;
|
||||
@@ -779,6 +785,7 @@ void hisparse_resolve_residency(
|
||||
std::optional<torch::stable::Tensor> const& request_state_indices,
|
||||
int64_t region_stride,
|
||||
std::optional<torch::stable::Tensor> const& miss_mask,
|
||||
std::optional<torch::stable::Tensor> const& stats,
|
||||
std::optional<torch::stable::Tensor> const& attention_indices,
|
||||
int64_t attention_block_stride,
|
||||
std::optional<torch::stable::Tensor> const& request_ids,
|
||||
@@ -997,6 +1004,17 @@ void hisparse_resolve_residency(
|
||||
miss_mask_ptr = mm.mutable_data_ptr<int32_t>();
|
||||
}
|
||||
|
||||
unsigned long long* stats_ptr = nullptr;
|
||||
if (stats.has_value()) {
|
||||
auto const& st = stats.value();
|
||||
STD_TORCH_CHECK(
|
||||
st.is_cuda() && st.is_contiguous() && st.dim() == 1 &&
|
||||
st.scalar_type() == torch::headeronly::ScalarType::UInt64 &&
|
||||
st.numel() == 2,
|
||||
"stats must be a contiguous two-element uint64 CUDA tensor");
|
||||
stats_ptr = static_cast<unsigned long long*>(st.mutable_data_ptr());
|
||||
}
|
||||
|
||||
int32_t* attention_indices_ptr = nullptr;
|
||||
if (attention_indices.has_value()) {
|
||||
auto const& indices = attention_indices.value();
|
||||
@@ -1046,7 +1064,7 @@ void hisparse_resolve_residency(
|
||||
swap_counts_ptr, hot_indices.mutable_data_ptr<int32_t>(),
|
||||
attention_indices_ptr, miss_mask_ptr,
|
||||
device_global_indices.mutable_data_ptr<int32_t>(),
|
||||
lru_slots.mutable_data_ptr<int16_t>(), request_state_ptr,
|
||||
lru_slots.mutable_data_ptr<int16_t>(), stats_ptr, request_state_ptr,
|
||||
request_state_count, host_rows, hot_block_table.stride(0), hot_block_size,
|
||||
top_k, hot_size, hash_size, region_stride, attention_block_stride,
|
||||
source_bt_stride, source_num_reqs, source_num_blocks,
|
||||
|
||||
@@ -673,6 +673,7 @@ void hisparse_resolve_residency(
|
||||
std::optional<torch::stable::Tensor> const& request_state_indices,
|
||||
int64_t region_stride,
|
||||
std::optional<torch::stable::Tensor> const& miss_mask,
|
||||
std::optional<torch::stable::Tensor> const& stats,
|
||||
std::optional<torch::stable::Tensor> const& attention_indices,
|
||||
int64_t attention_block_stride,
|
||||
std::optional<torch::stable::Tensor> const& request_ids,
|
||||
|
||||
@@ -999,16 +999,17 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) {
|
||||
" Tensor? request_state_indices,"
|
||||
" int region_stride,"
|
||||
" Tensor(a!)? miss_mask=None,"
|
||||
" Tensor(b!)? attention_indices=None,"
|
||||
" Tensor(b!)? stats=None,"
|
||||
" Tensor(c!)? attention_indices=None,"
|
||||
" int attention_block_stride=0,"
|
||||
" Tensor? request_ids=None,"
|
||||
" Tensor? source_block_table=None,"
|
||||
" int source_block_size=0,"
|
||||
" Tensor(c!)? resolved_global_indices=None,"
|
||||
" Tensor(d!)? valid_counts=None,"
|
||||
" Tensor(e!)? swap_host_physical_rows=None,"
|
||||
" Tensor(f!)? swap_device_physical_rows=None,"
|
||||
" Tensor(g!)? swap_counts=None,"
|
||||
" Tensor(d!)? resolved_global_indices=None,"
|
||||
" Tensor(e!)? valid_counts=None,"
|
||||
" Tensor(f!)? swap_host_physical_rows=None,"
|
||||
" Tensor(g!)? swap_device_physical_rows=None,"
|
||||
" Tensor(h!)? swap_counts=None,"
|
||||
" Tensor? resident_block_table=None,"
|
||||
" int resident_block_size=0,"
|
||||
" int resident_null_block=0) -> ()");
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnector,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
|
||||
HiSparseKVConnectorStats,
|
||||
)
|
||||
|
||||
|
||||
def test_is_empty_on_fresh_stats():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
assert stats.is_empty()
|
||||
|
||||
|
||||
def test_record_snapshot_and_reduce():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_snapshot(hits=7, misses=3, host_to_device_bytes=48)
|
||||
stats.record_snapshot(hits=5, misses=1, host_to_device_bytes=16)
|
||||
assert not stats.is_empty()
|
||||
|
||||
reduced = stats.reduce()
|
||||
assert reduced["HiSparse hot-buffer hits"] == 12
|
||||
assert reduced["HiSparse hot-buffer misses"] == 4
|
||||
assert reduced["HiSparse host-to-device bytes"] == 64
|
||||
|
||||
|
||||
def test_aggregate_extends_snapshot_deltas():
|
||||
first = HiSparseKVConnectorStats()
|
||||
first.record_snapshot(hits=7, misses=3, host_to_device_bytes=48)
|
||||
second = HiSparseKVConnectorStats()
|
||||
second.record_snapshot(hits=5, misses=1, host_to_device_bytes=16)
|
||||
|
||||
first.aggregate(second)
|
||||
|
||||
assert first.data == {
|
||||
"cache_hits": [7, 5],
|
||||
"cache_misses": [3, 1],
|
||||
"host_to_device_bytes": [48, 16],
|
||||
}
|
||||
|
||||
|
||||
def test_aggregate_skips_empty_stats():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_snapshot(hits=2, misses=1, host_to_device_bytes=16)
|
||||
|
||||
stats.aggregate(HiSparseKVConnectorStats())
|
||||
|
||||
assert stats.data["cache_hits"] == [2]
|
||||
|
||||
|
||||
def test_build_kv_connector_stats_round_trip():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_snapshot(hits=12, misses=4, host_to_device_bytes=64)
|
||||
payload = stats.to_dict()
|
||||
|
||||
rebuilt = HiSparseConnector.build_kv_connector_stats(data=payload)
|
||||
|
||||
assert rebuilt is not None
|
||||
assert isinstance(rebuilt, HiSparseKVConnectorStats)
|
||||
assert rebuilt.reduce() == stats.reduce()
|
||||
@@ -48,10 +48,47 @@ def _make_hisparse_worker() -> HiSparseConnectorWorker:
|
||||
worker.dma_stream = None
|
||||
worker.shared_host_region = None
|
||||
worker._completed_host_copy_dst_ids = []
|
||||
worker._slot_mapping_staging = None
|
||||
worker._metrics_calls = 0
|
||||
worker._metrics_event = MagicMock()
|
||||
worker._metrics_pending = False
|
||||
worker.leader_runtimes = []
|
||||
return worker
|
||||
|
||||
|
||||
def test_hisparse_worker_get_kv_connector_stats_reads_completed_snapshot(monkeypatch):
|
||||
worker = _make_hisparse_worker()
|
||||
worker._metrics_calls = hisparse_worker_module._METRICS_INTERVAL - 1
|
||||
worker._metrics_pending = False
|
||||
worker._metrics_event = MagicMock()
|
||||
worker._metrics_event.query.return_value = True
|
||||
compute_stream = MagicMock()
|
||||
group = SimpleNamespace(
|
||||
swap_stats=torch.tensor([12, 4], dtype=torch.uint64),
|
||||
swap_stats_host=torch.empty(2, dtype=torch.uint64),
|
||||
stats_row_bytes=16,
|
||||
copy_stream=MagicMock(),
|
||||
)
|
||||
worker.leader_runtimes = [SimpleNamespace(index_group=group)]
|
||||
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
hisparse_worker_module, "current_stream", lambda: compute_stream
|
||||
)
|
||||
|
||||
assert worker.get_kv_connector_stats() is None
|
||||
compute_stream.wait_stream.assert_called_once_with(group.copy_stream)
|
||||
group.copy_stream.wait_stream.assert_called_once_with(compute_stream)
|
||||
worker._metrics_event.record.assert_called_once_with()
|
||||
assert group.swap_stats.tolist() == [0, 0]
|
||||
|
||||
stats = worker.get_kv_connector_stats()
|
||||
assert stats is not None
|
||||
assert stats.data == {
|
||||
"cache_hits": [12],
|
||||
"cache_misses": [4],
|
||||
"host_to_device_bytes": [64],
|
||||
}
|
||||
|
||||
|
||||
def test_hisparse_row_mirrors_follow_runner_request_order():
|
||||
first = SparseKVRowMirror((1,), 10, 1)
|
||||
second = SparseKVRowMirror((2,), 20, 1)
|
||||
|
||||
@@ -17,9 +17,19 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorWorkerMetadata,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
|
||||
HiSparseKVConnectorStats,
|
||||
HiSparsePromMetrics,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
|
||||
HiSparseConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
@@ -244,6 +254,33 @@ class HiSparseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.reset_hot_state()
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
if self.connector_worker is None:
|
||||
return None
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
HiSparseKVConnectorStats(data=data)
|
||||
if data is not None
|
||||
else HiSparseKVConnectorStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return HiSparsePromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
|
||||
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stats and Prometheus metrics for the HiSparse connector."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.v1.metrics.utils import create_metric_per_engine
|
||||
|
||||
_HISPARSE_COUNTERS: tuple[tuple[str, str], ...] = (
|
||||
("cache_hits", "Number of HiSparse device hot-buffer hits."),
|
||||
("cache_misses", "Number of HiSparse device hot-buffer misses."),
|
||||
(
|
||||
"host_to_device_bytes",
|
||||
"Bytes transferred from host KV storage to HiSparse hot buffers.",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HiSparseKVConnectorStats(KVConnectorStats):
|
||||
"""Container for HiSparse hot-buffer residency metrics.
|
||||
|
||||
Each list entry is the delta recorded over one device counter snapshot
|
||||
interval, so a list can hold multiple snapshots per logging interval.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
# Empty container init, no data is passed in.
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
# Must be serializable
|
||||
self.data: dict[str, list[int]] = {
|
||||
"cache_hits": [],
|
||||
"cache_misses": [],
|
||||
"host_to_device_bytes": [],
|
||||
}
|
||||
|
||||
def record_snapshot(self, hits: int, misses: int, host_to_device_bytes: int):
|
||||
self.data["cache_hits"].append(hits)
|
||||
self.data["cache_misses"].append(misses)
|
||||
self.data["host_to_device_bytes"].append(host_to_device_bytes)
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if not other.is_empty():
|
||||
for k, v in other.data.items():
|
||||
accumulator = self.data[k]
|
||||
assert isinstance(accumulator, list)
|
||||
accumulator.extend(v)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
# Compute compact representative stats suitable for CLI logging.
|
||||
return {
|
||||
"HiSparse hot-buffer hits": sum(self.data["cache_hits"]),
|
||||
"HiSparse hot-buffer misses": sum(self.data["cache_misses"]),
|
||||
"HiSparse host-to-device bytes": sum(self.data["host_to_device_bytes"]),
|
||||
}
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return all(len(values) == 0 for values in self.data.values())
|
||||
|
||||
|
||||
class HiSparsePromMetrics(KVConnectorPromMetrics):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
):
|
||||
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
|
||||
|
||||
self.hisparse_counters: dict[str, Any] = {}
|
||||
for name, documentation in _HISPARSE_COUNTERS:
|
||||
counter = self._counter_cls(
|
||||
name=f"vllm:hisparse_{name}",
|
||||
documentation=documentation,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.hisparse_counters[name] = create_metric_per_engine(
|
||||
counter, self.per_engine_labelvalues
|
||||
)
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
for name, counter in self.hisparse_counters.items():
|
||||
for value in transfer_stats_data.get(name, []):
|
||||
counter[engine_idx].inc(value)
|
||||
@@ -15,6 +15,9 @@ import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
|
||||
HiSparseKVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tp_group,
|
||||
@@ -68,6 +71,9 @@ def _allocate_dma_descriptors(size: int) -> _DMADescriptors:
|
||||
)
|
||||
|
||||
|
||||
_METRICS_INTERVAL = 2000
|
||||
|
||||
|
||||
def _get_hisparse_cache(
|
||||
forward_context: dict[str, Any], layer_name: str
|
||||
) -> HiSparseCacheHandle:
|
||||
@@ -311,6 +317,9 @@ class HiSparseConnectorWorker:
|
||||
self._pending_transfer_events: deque[tuple[torch.Event, tuple[int, ...]]] = (
|
||||
deque()
|
||||
)
|
||||
self._metrics_calls = 0
|
||||
self._metrics_event = torch.Event()
|
||||
self._metrics_pending = False
|
||||
self._init_dma()
|
||||
if self.is_host_writer:
|
||||
for layer_index, handle in enumerate(cache_handles):
|
||||
@@ -496,6 +505,36 @@ class HiSparseConnectorWorker:
|
||||
for runtime in self.leader_runtimes:
|
||||
runtime.reset_hot_state()
|
||||
|
||||
def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None:
|
||||
stats = None
|
||||
if self._metrics_pending and self._metrics_event.query():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
for runtime in self.leader_runtimes:
|
||||
group = runtime.index_group
|
||||
hits, misses = group.swap_stats_host.tolist()
|
||||
if hits or misses:
|
||||
stats.record_snapshot(hits, misses, misses * group.stats_row_bytes)
|
||||
self._metrics_pending = False
|
||||
if stats.is_empty():
|
||||
stats = None
|
||||
|
||||
self._metrics_calls += 1
|
||||
if (
|
||||
self._metrics_calls % _METRICS_INTERVAL == 0
|
||||
and not self._metrics_pending
|
||||
and not torch.cuda.is_current_stream_capturing()
|
||||
):
|
||||
compute_stream = current_stream()
|
||||
for runtime in self.leader_runtimes:
|
||||
group = runtime.index_group
|
||||
compute_stream.wait_stream(group.copy_stream)
|
||||
group.swap_stats_host.copy_(group.swap_stats, non_blocking=True)
|
||||
group.swap_stats.zero_()
|
||||
group.copy_stream.wait_stream(compute_stream)
|
||||
self._metrics_event.record()
|
||||
self._metrics_pending = True
|
||||
return stats
|
||||
|
||||
def _release_completed_dma_descriptors(self) -> None:
|
||||
pending = self._pending_dma_descriptors
|
||||
while pending and pending[0][0].query():
|
||||
|
||||
@@ -464,6 +464,11 @@ class HiSparseIndexGroup:
|
||||
)
|
||||
self.logical_topk_ready = logical_topk_ready
|
||||
self.followers: list[HiSparseRuntime] = []
|
||||
self.swap_stats = torch.zeros(2, dtype=torch.uint64, device=device)
|
||||
self.swap_stats_host = torch.empty(
|
||||
2, dtype=torch.uint64, device="cpu", pin_memory=True
|
||||
)
|
||||
self.stats_row_bytes = 0
|
||||
|
||||
|
||||
class HiSparseRuntime:
|
||||
@@ -526,6 +531,7 @@ class HiSparseRuntime:
|
||||
else:
|
||||
index_group.followers.append(self)
|
||||
self.index_group = index_group
|
||||
index_group.stats_row_bytes += row_bytes
|
||||
|
||||
self.eager_host_mirror = config.eager_host_mirror
|
||||
self.resident_source_index = -1
|
||||
@@ -639,6 +645,7 @@ class HiSparseRuntime:
|
||||
compute_stream.wait_stream(group.copy_stream)
|
||||
group.device_global_indices.fill_(-1)
|
||||
group.lru_slots.copy_(group.lru_init.expand_as(group.lru_slots))
|
||||
group.swap_stats.zero_()
|
||||
group.copy_stream.wait_stream(compute_stream)
|
||||
|
||||
def invalidate_sorted_slots(
|
||||
@@ -741,6 +748,7 @@ class HiSparseRuntime:
|
||||
request_state_indices,
|
||||
self.region_stride,
|
||||
None,
|
||||
group.swap_stats,
|
||||
physical_topk_indices,
|
||||
hot.attention_block_stride,
|
||||
req_id_per_token[:num_tokens].contiguous(),
|
||||
|
||||
Reference in New Issue
Block a user