[Mooncake] Add metrics for MooncakeStoreConnector operations (#43392)

This commit is contained in:
Dao007forever
2026-05-23 13:34:40 -07:00
committed by GitHub
parent 4438b6e7dc
commit 819c610f9b
5 changed files with 507 additions and 4 deletions
@@ -14,6 +14,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
MooncakeStoreConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
MooncakeStoreConnectorStats,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
@@ -129,6 +132,49 @@ def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
assert connector.get_kv_connector_kv_cache_events() is None
def test_get_kv_connector_stats_delegates_to_worker():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
expected_stats = MooncakeStoreConnectorStats()
expected_stats.record_operation("save_put", 0.01, 2, num_bytes=1024)
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"connector.MooncakeStoreWorker"
) as mock_worker_cls,
):
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
mock_worker_cls.return_value.get_kv_connector_stats.return_value = expected_stats
stats = connector.get_kv_connector_stats()
assert stats is expected_stats
mock_worker_cls.return_value.get_kv_connector_stats.assert_called_once_with()
def test_build_kv_connector_stats_reconstructs_mooncake_stats():
stats = mooncake_store_connector.MooncakeStoreConnector.build_kv_connector_stats(
{
"save_put": [
{
"duration_seconds": 0.02,
"num_keys": 4,
"num_bytes": 2048,
"status": "ok",
"num_failed_keys": 0,
}
]
}
)
assert isinstance(stats, MooncakeStoreConnectorStats)
assert stats.data["save_put"][0]["num_bytes"] == 2048
def test_get_kv_connector_kv_cache_events_wraps_worker_events():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
@@ -28,6 +28,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
LoadSpec,
ReqMeta,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
MooncakeStoreConnectorStats,
)
def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator:
@@ -417,6 +420,24 @@ def test_store_sending_thread_skips_request_during_cpu_pressure():
assert store.batch_put_from_multi_buffers.call_count == 3
def test_store_sending_thread_records_mooncake_metrics():
store = MagicMock()
store.batch_is_exist.return_value = [0, 0]
store.batch_put_from_multi_buffers.return_value = [256, 256]
thread = _make_store_sending_thread(store)
stats = MooncakeStoreConnectorStats()
thread._record_operation_cb = stats.record_operation
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
assert len(stats.data["save_exists"]) == 1
assert stats.data["save_exists"][0]["num_keys"] == 2
assert len(stats.data["save_put"]) == 1
assert stats.data["save_put"][0]["num_bytes"] == 512
assert stats.data["save_put"][0]["status"] == "ok"
def test_store_sending_thread_only_skips_on_no_available_handle():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
@@ -548,6 +569,29 @@ def test_recv_thread_logs_tier_summary_when_enabled(monkeypatch, caplog_vllm):
)
def test_recv_thread_records_partial_failure_metrics(monkeypatch):
monkeypatch.delenv("VLLM_MOONCAKE_STORE_TIER_LOG", raising=False)
store = MagicMock()
store.batch_get_into_multi_buffers.return_value = [256, -10]
thread = _make_store_recving_thread(store, disk_offload_buffer_budget_bytes=None)
stats = MooncakeStoreConnectorStats()
thread._record_operation_cb = stats.record_operation
req = _make_load_req(
"req-a",
[b"a0", b"a1"],
token_len=32,
)
thread._handle_request(req)
assert len(stats.data["load_get"]) == 1
assert stats.data["load_get"][0]["num_keys"] == 2
assert stats.data["load_get"][0]["num_bytes"] == 512
assert stats.data["load_get"][0]["status"] == "partial_failure"
assert stats.data["load_get"][0]["num_failed_keys"] == 1
def test_recv_thread_uses_ratio_scaled_budget_for_first_pass_split():
store = MagicMock()
store.batch_get_into_multi_buffers.side_effect = [
@@ -1025,6 +1069,8 @@ def _make_bare_worker(
worker.disk_offload_buffer_budget_bytes = None
worker.store_replicate_config = SimpleNamespace()
worker._kv_connector_stats_lock = threading.Lock()
worker.kv_connector_stats = MooncakeStoreConnectorStats()
spec = FullAttentionSpec(
block_size=block_size, num_kv_heads=8, head_size=64, dtype=None
@@ -1373,3 +1419,59 @@ def test_topology_embedded_cpu_only(tmp_path, monkeypatch):
assert w.store_replicate_config.preferred_segment == ""
# No disk budget — enable_offload was absent (defaults to False).
assert w.disk_offload_buffer_budget_bytes is None
# ---------------------------------------------------------------------------
# Stats/metrics tests (PR-35 port)
# ---------------------------------------------------------------------------
def test_mooncake_store_stats_aggregate_reduce():
stats = MooncakeStoreConnectorStats()
stats.record_operation("save_put", 0.01, 2, num_bytes=128)
other = MooncakeStoreConnectorStats()
other.record_operation(
"save_put",
0.03,
1,
num_bytes=64,
status="error",
num_failed_keys=1,
)
reduced = stats.aggregate(other).reduce()
assert reduced["save_put_count"] == 2
assert reduced["save_put_total_keys"] == 3
assert reduced["save_put_total_bytes"] == 192
assert reduced["save_put_failed_keys"] == 1
assert reduced["save_put_error_count"] == 1
def test_worker_get_kv_connector_stats_resets_after_read():
worker = _make_bare_worker()
worker._record_kv_connector_operation(
"save_put",
0.01,
2,
num_bytes=128,
)
stats = worker.get_kv_connector_stats()
assert isinstance(stats, MooncakeStoreConnectorStats)
assert stats.data["save_put"][0]["num_bytes"] == 128
assert worker.get_kv_connector_stats() is None
def test_lookup_records_mooncake_metrics():
worker = _make_bare_worker()
worker.store.batch_is_exist.return_value = [1, 1]
result = worker.lookup(32, [b"a0", b"a1"])
stats = worker.get_kv_connector_stats()
assert result == 32
assert isinstance(stats, MooncakeStoreConnectorStats)
assert len(stats.data["lookup_exists"]) == 1
assert stats.data["lookup_exists"][0]["num_keys"] == 2