[4/N] Expose HiSparse cache metrics via KV connector stats (#56061)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
OpenAI Codex
parent
dff76bc3e8
commit
29332cf936
@@ -0,0 +1,62 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnector,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
|
||||
HiSparseKVConnectorStats,
|
||||
)
|
||||
|
||||
|
||||
def test_is_empty_on_fresh_stats():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
assert stats.is_empty()
|
||||
|
||||
|
||||
def test_record_snapshot_and_reduce():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_snapshot(hits=7, misses=3, host_to_device_bytes=48)
|
||||
stats.record_snapshot(hits=5, misses=1, host_to_device_bytes=16)
|
||||
assert not stats.is_empty()
|
||||
|
||||
reduced = stats.reduce()
|
||||
assert reduced["HiSparse hot-buffer hits"] == 12
|
||||
assert reduced["HiSparse hot-buffer misses"] == 4
|
||||
assert reduced["HiSparse host-to-device bytes"] == 64
|
||||
|
||||
|
||||
def test_aggregate_extends_snapshot_deltas():
|
||||
first = HiSparseKVConnectorStats()
|
||||
first.record_snapshot(hits=7, misses=3, host_to_device_bytes=48)
|
||||
second = HiSparseKVConnectorStats()
|
||||
second.record_snapshot(hits=5, misses=1, host_to_device_bytes=16)
|
||||
|
||||
first.aggregate(second)
|
||||
|
||||
assert first.data == {
|
||||
"cache_hits": [7, 5],
|
||||
"cache_misses": [3, 1],
|
||||
"host_to_device_bytes": [48, 16],
|
||||
}
|
||||
|
||||
|
||||
def test_aggregate_skips_empty_stats():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_snapshot(hits=2, misses=1, host_to_device_bytes=16)
|
||||
|
||||
stats.aggregate(HiSparseKVConnectorStats())
|
||||
|
||||
assert stats.data["cache_hits"] == [2]
|
||||
|
||||
|
||||
def test_build_kv_connector_stats_round_trip():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_snapshot(hits=12, misses=4, host_to_device_bytes=64)
|
||||
payload = stats.to_dict()
|
||||
|
||||
rebuilt = HiSparseConnector.build_kv_connector_stats(data=payload)
|
||||
|
||||
assert rebuilt is not None
|
||||
assert isinstance(rebuilt, HiSparseKVConnectorStats)
|
||||
assert rebuilt.reduce() == stats.reduce()
|
||||
@@ -48,10 +48,47 @@ def _make_hisparse_worker() -> HiSparseConnectorWorker:
|
||||
worker.dma_stream = None
|
||||
worker.shared_host_region = None
|
||||
worker._completed_host_copy_dst_ids = []
|
||||
worker._slot_mapping_staging = None
|
||||
worker._metrics_calls = 0
|
||||
worker._metrics_event = MagicMock()
|
||||
worker._metrics_pending = False
|
||||
worker.leader_runtimes = []
|
||||
return worker
|
||||
|
||||
|
||||
def test_hisparse_worker_get_kv_connector_stats_reads_completed_snapshot(monkeypatch):
|
||||
worker = _make_hisparse_worker()
|
||||
worker._metrics_calls = hisparse_worker_module._METRICS_INTERVAL - 1
|
||||
worker._metrics_pending = False
|
||||
worker._metrics_event = MagicMock()
|
||||
worker._metrics_event.query.return_value = True
|
||||
compute_stream = MagicMock()
|
||||
group = SimpleNamespace(
|
||||
swap_stats=torch.tensor([12, 4], dtype=torch.uint64),
|
||||
swap_stats_host=torch.empty(2, dtype=torch.uint64),
|
||||
stats_row_bytes=16,
|
||||
copy_stream=MagicMock(),
|
||||
)
|
||||
worker.leader_runtimes = [SimpleNamespace(index_group=group)]
|
||||
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
hisparse_worker_module, "current_stream", lambda: compute_stream
|
||||
)
|
||||
|
||||
assert worker.get_kv_connector_stats() is None
|
||||
compute_stream.wait_stream.assert_called_once_with(group.copy_stream)
|
||||
group.copy_stream.wait_stream.assert_called_once_with(compute_stream)
|
||||
worker._metrics_event.record.assert_called_once_with()
|
||||
assert group.swap_stats.tolist() == [0, 0]
|
||||
|
||||
stats = worker.get_kv_connector_stats()
|
||||
assert stats is not None
|
||||
assert stats.data == {
|
||||
"cache_hits": [12],
|
||||
"cache_misses": [4],
|
||||
"host_to_device_bytes": [64],
|
||||
}
|
||||
|
||||
|
||||
def test_hisparse_row_mirrors_follow_runner_request_order():
|
||||
first = SparseKVRowMirror((1,), 10, 1)
|
||||
second = SparseKVRowMirror((2,), 20, 1)
|
||||
|
||||
Reference in New Issue
Block a user