diff --git a/csrc/libtorch_stable/hisparse_kernels.cu b/csrc/libtorch_stable/hisparse_kernels.cu index 63d8457627..97ce9246cc 100644 --- a/csrc/libtorch_stable/hisparse_kernels.cu +++ b/csrc/libtorch_stable/hisparse_kernels.cu @@ -203,6 +203,7 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel( int32_t* __restrict__ miss_mask, // [num_rows, top_k] or nullptr int32_t* __restrict__ device_global_indices, // [max_rows, region_stride] int16_t* __restrict__ lru_slots, // [max_rows, hot_size] + unsigned long long* __restrict__ stats, // [2] hits,misses or nullptr const int32_t* __restrict__ request_state_indices, // [num_requests] or // nullptr const int32_t request_state_count, const int64_t host_rows, @@ -552,6 +553,11 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel( if (swap_counts != nullptr && tid == 0) { swap_counts[batch_row] = total_misses; } + if (stats != nullptr && tid == 0) { + atomicAdd(&stats[0], static_cast(total_hits)); + atomicAdd(&stats[1], static_cast(total_misses)); + } + // Phase 4: write back the LRU order: stale evictables at the front, // freshly loaded misses next, then hits at MRU. const int total_evictable = hot_size - total_hits; @@ -779,6 +785,7 @@ void hisparse_resolve_residency( std::optional const& request_state_indices, int64_t region_stride, std::optional const& miss_mask, + std::optional const& stats, std::optional const& attention_indices, int64_t attention_block_stride, std::optional const& request_ids, @@ -997,6 +1004,17 @@ void hisparse_resolve_residency( miss_mask_ptr = mm.mutable_data_ptr(); } + unsigned long long* stats_ptr = nullptr; + if (stats.has_value()) { + auto const& st = stats.value(); + STD_TORCH_CHECK( + st.is_cuda() && st.is_contiguous() && st.dim() == 1 && + st.scalar_type() == torch::headeronly::ScalarType::UInt64 && + st.numel() == 2, + "stats must be a contiguous two-element uint64 CUDA tensor"); + stats_ptr = static_cast(st.mutable_data_ptr()); + } + int32_t* attention_indices_ptr = nullptr; if (attention_indices.has_value()) { auto const& indices = attention_indices.value(); @@ -1046,7 +1064,7 @@ void hisparse_resolve_residency( swap_counts_ptr, hot_indices.mutable_data_ptr(), attention_indices_ptr, miss_mask_ptr, device_global_indices.mutable_data_ptr(), - lru_slots.mutable_data_ptr(), request_state_ptr, + lru_slots.mutable_data_ptr(), stats_ptr, request_state_ptr, request_state_count, host_rows, hot_block_table.stride(0), hot_block_size, top_k, hot_size, hash_size, region_stride, attention_block_stride, source_bt_stride, source_num_reqs, source_num_blocks, diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index 6893c6913d..68a24ca434 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -673,6 +673,7 @@ void hisparse_resolve_residency( std::optional const& request_state_indices, int64_t region_stride, std::optional const& miss_mask, + std::optional const& stats, std::optional const& attention_indices, int64_t attention_block_stride, std::optional const& request_ids, diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index cb6f463396..110f8595ee 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -999,16 +999,17 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) { " Tensor? request_state_indices," " int region_stride," " Tensor(a!)? miss_mask=None," - " Tensor(b!)? attention_indices=None," + " Tensor(b!)? stats=None," + " Tensor(c!)? attention_indices=None," " int attention_block_stride=0," " Tensor? request_ids=None," " Tensor? source_block_table=None," " int source_block_size=0," - " Tensor(c!)? resolved_global_indices=None," - " Tensor(d!)? valid_counts=None," - " Tensor(e!)? swap_host_physical_rows=None," - " Tensor(f!)? swap_device_physical_rows=None," - " Tensor(g!)? swap_counts=None," + " Tensor(d!)? resolved_global_indices=None," + " Tensor(e!)? valid_counts=None," + " Tensor(f!)? swap_host_physical_rows=None," + " Tensor(g!)? swap_device_physical_rows=None," + " Tensor(h!)? swap_counts=None," " Tensor? resident_block_table=None," " int resident_block_size=0," " int resident_null_block=0) -> ()"); diff --git a/tests/v1/kv_connector/unit/test_hisparse_stats.py b/tests/v1/kv_connector/unit/test_hisparse_stats.py new file mode 100644 index 0000000000..09904d0a3c --- /dev/null +++ b/tests/v1/kv_connector/unit/test_hisparse_stats.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import ( + HiSparseConnector, +) +from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import ( + HiSparseKVConnectorStats, +) + + +def test_is_empty_on_fresh_stats(): + stats = HiSparseKVConnectorStats() + assert stats.is_empty() + + +def test_record_snapshot_and_reduce(): + stats = HiSparseKVConnectorStats() + stats.record_snapshot(hits=7, misses=3, host_to_device_bytes=48) + stats.record_snapshot(hits=5, misses=1, host_to_device_bytes=16) + assert not stats.is_empty() + + reduced = stats.reduce() + assert reduced["HiSparse hot-buffer hits"] == 12 + assert reduced["HiSparse hot-buffer misses"] == 4 + assert reduced["HiSparse host-to-device bytes"] == 64 + + +def test_aggregate_extends_snapshot_deltas(): + first = HiSparseKVConnectorStats() + first.record_snapshot(hits=7, misses=3, host_to_device_bytes=48) + second = HiSparseKVConnectorStats() + second.record_snapshot(hits=5, misses=1, host_to_device_bytes=16) + + first.aggregate(second) + + assert first.data == { + "cache_hits": [7, 5], + "cache_misses": [3, 1], + "host_to_device_bytes": [48, 16], + } + + +def test_aggregate_skips_empty_stats(): + stats = HiSparseKVConnectorStats() + stats.record_snapshot(hits=2, misses=1, host_to_device_bytes=16) + + stats.aggregate(HiSparseKVConnectorStats()) + + assert stats.data["cache_hits"] == [2] + + +def test_build_kv_connector_stats_round_trip(): + stats = HiSparseKVConnectorStats() + stats.record_snapshot(hits=12, misses=4, host_to_device_bytes=64) + payload = stats.to_dict() + + rebuilt = HiSparseConnector.build_kv_connector_stats(data=payload) + + assert rebuilt is not None + assert isinstance(rebuilt, HiSparseKVConnectorStats) + assert rebuilt.reduce() == stats.reduce() diff --git a/tests/v1/worker/test_utils.py b/tests/v1/worker/test_utils.py index 86c8688906..8422a591c3 100644 --- a/tests/v1/worker/test_utils.py +++ b/tests/v1/worker/test_utils.py @@ -48,10 +48,47 @@ def _make_hisparse_worker() -> HiSparseConnectorWorker: worker.dma_stream = None worker.shared_host_region = None worker._completed_host_copy_dst_ids = [] - worker._slot_mapping_staging = None + worker._metrics_calls = 0 + worker._metrics_event = MagicMock() + worker._metrics_pending = False + worker.leader_runtimes = [] return worker +def test_hisparse_worker_get_kv_connector_stats_reads_completed_snapshot(monkeypatch): + worker = _make_hisparse_worker() + worker._metrics_calls = hisparse_worker_module._METRICS_INTERVAL - 1 + worker._metrics_pending = False + worker._metrics_event = MagicMock() + worker._metrics_event.query.return_value = True + compute_stream = MagicMock() + group = SimpleNamespace( + swap_stats=torch.tensor([12, 4], dtype=torch.uint64), + swap_stats_host=torch.empty(2, dtype=torch.uint64), + stats_row_bytes=16, + copy_stream=MagicMock(), + ) + worker.leader_runtimes = [SimpleNamespace(index_group=group)] + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + monkeypatch.setattr( + hisparse_worker_module, "current_stream", lambda: compute_stream + ) + + assert worker.get_kv_connector_stats() is None + compute_stream.wait_stream.assert_called_once_with(group.copy_stream) + group.copy_stream.wait_stream.assert_called_once_with(compute_stream) + worker._metrics_event.record.assert_called_once_with() + assert group.swap_stats.tolist() == [0, 0] + + stats = worker.get_kv_connector_stats() + assert stats is not None + assert stats.data == { + "cache_hits": [12], + "cache_misses": [4], + "host_to_device_bytes": [64], + } + + def test_hisparse_row_mirrors_follow_runner_request_order(): first = SparseKVRowMirror((1,), 10, 1) second = SparseKVRowMirror((2,), 20, 1) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/connector.py index 338fd5cc6e..5acb7e3bbd 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/connector.py @@ -17,9 +17,19 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorWorkerMetadata, SupportsHMA, ) +from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import ( + HiSparseKVConnectorStats, + HiSparsePromMetrics, +) from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import ( HiSparseConnectorWorker, ) +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, +) from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy from vllm.v1.core.sched.output import SchedulerOutput @@ -244,6 +254,33 @@ class HiSparseConnector(KVConnectorBase_V1, SupportsHMA): assert self.connector_worker is not None self.connector_worker.reset_hot_state() + def get_kv_connector_stats(self) -> KVConnectorStats | None: + if self.connector_worker is None: + return None + return self.connector_worker.get_kv_connector_stats() + + @classmethod + def build_kv_connector_stats( + cls, data: dict[str, Any] | None = None + ) -> KVConnectorStats | None: + return ( + HiSparseKVConnectorStats(data=data) + if data is not None + else HiSparseKVConnectorStats() + ) + + @classmethod + def build_prom_metrics( + cls, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ) -> KVConnectorPromMetrics: + return HiSparsePromMetrics( + vllm_config, metric_types, labelnames, per_engine_labelvalues + ) + def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None: assert self.connector_worker is not None metadata = self._get_connector_metadata() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/stats.py b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/stats.py new file mode 100644 index 0000000000..f2abf51fc4 --- /dev/null +++ b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/stats.py @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Stats and Prometheus metrics for the HiSparse connector.""" + +from dataclasses import dataclass +from typing import Any + +from vllm.config import VllmConfig +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, +) +from vllm.v1.metrics.utils import create_metric_per_engine + +_HISPARSE_COUNTERS: tuple[tuple[str, str], ...] = ( + ("cache_hits", "Number of HiSparse device hot-buffer hits."), + ("cache_misses", "Number of HiSparse device hot-buffer misses."), + ( + "host_to_device_bytes", + "Bytes transferred from host KV storage to HiSparse hot buffers.", + ), +) + + +@dataclass +class HiSparseKVConnectorStats(KVConnectorStats): + """Container for HiSparse hot-buffer residency metrics. + + Each list entry is the delta recorded over one device counter snapshot + interval, so a list can hold multiple snapshots per logging interval. + """ + + def __post_init__(self): + if not self.data: + # Empty container init, no data is passed in. + self.reset() + + def reset(self): + # Must be serializable + self.data: dict[str, list[int]] = { + "cache_hits": [], + "cache_misses": [], + "host_to_device_bytes": [], + } + + def record_snapshot(self, hits: int, misses: int, host_to_device_bytes: int): + self.data["cache_hits"].append(hits) + self.data["cache_misses"].append(misses) + self.data["host_to_device_bytes"].append(host_to_device_bytes) + + def aggregate(self, other: KVConnectorStats) -> KVConnectorStats: + if not other.is_empty(): + for k, v in other.data.items(): + accumulator = self.data[k] + assert isinstance(accumulator, list) + accumulator.extend(v) + return self + + def reduce(self) -> dict[str, int | float]: + # Compute compact representative stats suitable for CLI logging. + return { + "HiSparse hot-buffer hits": sum(self.data["cache_hits"]), + "HiSparse hot-buffer misses": sum(self.data["cache_misses"]), + "HiSparse host-to-device bytes": sum(self.data["host_to_device_bytes"]), + } + + def is_empty(self) -> bool: + return all(len(values) == 0 for values in self.data.values()) + + +class HiSparsePromMetrics(KVConnectorPromMetrics): + def __init__( + self, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ): + super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues) + + self.hisparse_counters: dict[str, Any] = {} + for name, documentation in _HISPARSE_COUNTERS: + counter = self._counter_cls( + name=f"vllm:hisparse_{name}", + documentation=documentation, + labelnames=labelnames, + ) + self.hisparse_counters[name] = create_metric_per_engine( + counter, self.per_engine_labelvalues + ) + + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): + for name, counter in self.hisparse_counters.items(): + for value in transfer_stats_data.get(name, []): + counter[engine_idx].inc(value) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py index 1f723e5344..69721803be 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py @@ -15,6 +15,9 @@ import torch from vllm import _custom_ops as ops from vllm.config import VllmConfig +from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import ( + HiSparseKVConnectorStats, +) from vllm.distributed.parallel_state import ( get_tensor_model_parallel_rank, get_tp_group, @@ -68,6 +71,9 @@ def _allocate_dma_descriptors(size: int) -> _DMADescriptors: ) +_METRICS_INTERVAL = 2000 + + def _get_hisparse_cache( forward_context: dict[str, Any], layer_name: str ) -> HiSparseCacheHandle: @@ -311,6 +317,9 @@ class HiSparseConnectorWorker: self._pending_transfer_events: deque[tuple[torch.Event, tuple[int, ...]]] = ( deque() ) + self._metrics_calls = 0 + self._metrics_event = torch.Event() + self._metrics_pending = False self._init_dma() if self.is_host_writer: for layer_index, handle in enumerate(cache_handles): @@ -496,6 +505,36 @@ class HiSparseConnectorWorker: for runtime in self.leader_runtimes: runtime.reset_hot_state() + def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None: + stats = None + if self._metrics_pending and self._metrics_event.query(): + stats = HiSparseKVConnectorStats() + for runtime in self.leader_runtimes: + group = runtime.index_group + hits, misses = group.swap_stats_host.tolist() + if hits or misses: + stats.record_snapshot(hits, misses, misses * group.stats_row_bytes) + self._metrics_pending = False + if stats.is_empty(): + stats = None + + self._metrics_calls += 1 + if ( + self._metrics_calls % _METRICS_INTERVAL == 0 + and not self._metrics_pending + and not torch.cuda.is_current_stream_capturing() + ): + compute_stream = current_stream() + for runtime in self.leader_runtimes: + group = runtime.index_group + compute_stream.wait_stream(group.copy_stream) + group.swap_stats_host.copy_(group.swap_stats, non_blocking=True) + group.swap_stats.zero_() + group.copy_stream.wait_stream(compute_stream) + self._metrics_event.record() + self._metrics_pending = True + return stats + def _release_completed_dma_descriptors(self) -> None: pending = self._pending_dma_descriptors while pending and pending[0][0].query(): diff --git a/vllm/v1/hisparse/runtime.py b/vllm/v1/hisparse/runtime.py index 96235f1570..3dd8c017f5 100644 --- a/vllm/v1/hisparse/runtime.py +++ b/vllm/v1/hisparse/runtime.py @@ -464,6 +464,11 @@ class HiSparseIndexGroup: ) self.logical_topk_ready = logical_topk_ready self.followers: list[HiSparseRuntime] = [] + self.swap_stats = torch.zeros(2, dtype=torch.uint64, device=device) + self.swap_stats_host = torch.empty( + 2, dtype=torch.uint64, device="cpu", pin_memory=True + ) + self.stats_row_bytes = 0 class HiSparseRuntime: @@ -526,6 +531,7 @@ class HiSparseRuntime: else: index_group.followers.append(self) self.index_group = index_group + index_group.stats_row_bytes += row_bytes self.eager_host_mirror = config.eager_host_mirror self.resident_source_index = -1 @@ -639,6 +645,7 @@ class HiSparseRuntime: compute_stream.wait_stream(group.copy_stream) group.device_global_indices.fill_(-1) group.lru_slots.copy_(group.lru_init.expand_as(group.lru_slots)) + group.swap_stats.zero_() group.copy_stream.wait_stream(compute_stream) def invalidate_sorted_slots( @@ -741,6 +748,7 @@ class HiSparseRuntime: request_state_indices, self.region_stride, None, + group.swap_stats, physical_topk_indices, hot.attention_block_stride, req_id_per_token[:num_tokens].contiguous(),