From 819c610f9bd0f473573c4a9826b229aebad8b301 Mon Sep 17 00:00:00 2001 From: Dao007forever Date: Sat, 23 May 2026 13:34:40 -0700 Subject: [PATCH] [Mooncake] Add metrics for MooncakeStoreConnector operations (#43392) --- .../unit/test_mooncake_store_connector.py | 46 +++++ .../unit/test_mooncake_store_worker.py | 102 ++++++++++ .../v1/mooncake/store/connector.py | 34 ++++ .../kv_connector/v1/mooncake/store/metrics.py | 189 ++++++++++++++++++ .../kv_connector/v1/mooncake/store/worker.py | 140 ++++++++++++- 5 files changed, 507 insertions(+), 4 deletions(-) create mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/metrics.py diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_connector.py b/tests/v1/kv_connector/unit/test_mooncake_store_connector.py index 620b874531..2f717c3e2b 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_connector.py @@ -14,6 +14,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import ( from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( MooncakeStoreConnectorMetadata, ) +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import ( + MooncakeStoreConnectorStats, +) from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheConfig, @@ -129,6 +132,49 @@ def test_get_kv_connector_kv_cache_events_returns_none_when_empty(): assert connector.get_kv_connector_kv_cache_events() is None +def test_get_kv_connector_stats_delegates_to_worker(): + vllm_config = _make_vllm_config() + kv_cache_config = _make_kv_cache_config() + expected_stats = MooncakeStoreConnectorStats() + expected_stats.record_operation("save_put", 0.01, 2, num_bytes=1024) + + with ( + set_current_vllm_config(vllm_config), + patch( + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store." + "connector.MooncakeStoreWorker" + ) as mock_worker_cls, + ): + connector = mooncake_store_connector.MooncakeStoreConnector( + vllm_config, KVConnectorRole.WORKER, kv_cache_config + ) + + mock_worker_cls.return_value.get_kv_connector_stats.return_value = expected_stats + stats = connector.get_kv_connector_stats() + + assert stats is expected_stats + mock_worker_cls.return_value.get_kv_connector_stats.assert_called_once_with() + + +def test_build_kv_connector_stats_reconstructs_mooncake_stats(): + stats = mooncake_store_connector.MooncakeStoreConnector.build_kv_connector_stats( + { + "save_put": [ + { + "duration_seconds": 0.02, + "num_keys": 4, + "num_bytes": 2048, + "status": "ok", + "num_failed_keys": 0, + } + ] + } + ) + + assert isinstance(stats, MooncakeStoreConnectorStats) + assert stats.data["save_put"][0]["num_bytes"] == 2048 + + def test_get_kv_connector_kv_cache_events_wraps_worker_events(): vllm_config = _make_vllm_config() kv_cache_config = _make_kv_cache_config() diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 201c0af26f..a617b6074e 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -28,6 +28,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( LoadSpec, ReqMeta, ) +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import ( + MooncakeStoreConnectorStats, +) def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator: @@ -417,6 +420,24 @@ def test_store_sending_thread_skips_request_during_cpu_pressure(): assert store.batch_put_from_multi_buffers.call_count == 3 +def test_store_sending_thread_records_mooncake_metrics(): + store = MagicMock() + store.batch_is_exist.return_value = [0, 0] + store.batch_put_from_multi_buffers.return_value = [256, 256] + thread = _make_store_sending_thread(store) + stats = MooncakeStoreConnectorStats() + thread._record_operation_cb = stats.record_operation + + thread.add_stored_request("req-a") + thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"])) + + assert len(stats.data["save_exists"]) == 1 + assert stats.data["save_exists"][0]["num_keys"] == 2 + assert len(stats.data["save_put"]) == 1 + assert stats.data["save_put"][0]["num_bytes"] == 512 + assert stats.data["save_put"][0]["status"] == "ok" + + def test_store_sending_thread_only_skips_on_no_available_handle(): store = MagicMock() store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) @@ -548,6 +569,29 @@ def test_recv_thread_logs_tier_summary_when_enabled(monkeypatch, caplog_vllm): ) +def test_recv_thread_records_partial_failure_metrics(monkeypatch): + monkeypatch.delenv("VLLM_MOONCAKE_STORE_TIER_LOG", raising=False) + store = MagicMock() + store.batch_get_into_multi_buffers.return_value = [256, -10] + thread = _make_store_recving_thread(store, disk_offload_buffer_budget_bytes=None) + stats = MooncakeStoreConnectorStats() + thread._record_operation_cb = stats.record_operation + + req = _make_load_req( + "req-a", + [b"a0", b"a1"], + token_len=32, + ) + + thread._handle_request(req) + + assert len(stats.data["load_get"]) == 1 + assert stats.data["load_get"][0]["num_keys"] == 2 + assert stats.data["load_get"][0]["num_bytes"] == 512 + assert stats.data["load_get"][0]["status"] == "partial_failure" + assert stats.data["load_get"][0]["num_failed_keys"] == 1 + + def test_recv_thread_uses_ratio_scaled_budget_for_first_pass_split(): store = MagicMock() store.batch_get_into_multi_buffers.side_effect = [ @@ -1025,6 +1069,8 @@ def _make_bare_worker( worker.disk_offload_buffer_budget_bytes = None worker.store_replicate_config = SimpleNamespace() + worker._kv_connector_stats_lock = threading.Lock() + worker.kv_connector_stats = MooncakeStoreConnectorStats() spec = FullAttentionSpec( block_size=block_size, num_kv_heads=8, head_size=64, dtype=None @@ -1373,3 +1419,59 @@ def test_topology_embedded_cpu_only(tmp_path, monkeypatch): assert w.store_replicate_config.preferred_segment == "" # No disk budget — enable_offload was absent (defaults to False). assert w.disk_offload_buffer_budget_bytes is None + + +# --------------------------------------------------------------------------- +# Stats/metrics tests (PR-35 port) +# --------------------------------------------------------------------------- + + +def test_mooncake_store_stats_aggregate_reduce(): + stats = MooncakeStoreConnectorStats() + stats.record_operation("save_put", 0.01, 2, num_bytes=128) + other = MooncakeStoreConnectorStats() + other.record_operation( + "save_put", + 0.03, + 1, + num_bytes=64, + status="error", + num_failed_keys=1, + ) + + reduced = stats.aggregate(other).reduce() + + assert reduced["save_put_count"] == 2 + assert reduced["save_put_total_keys"] == 3 + assert reduced["save_put_total_bytes"] == 192 + assert reduced["save_put_failed_keys"] == 1 + assert reduced["save_put_error_count"] == 1 + + +def test_worker_get_kv_connector_stats_resets_after_read(): + worker = _make_bare_worker() + worker._record_kv_connector_operation( + "save_put", + 0.01, + 2, + num_bytes=128, + ) + + stats = worker.get_kv_connector_stats() + + assert isinstance(stats, MooncakeStoreConnectorStats) + assert stats.data["save_put"][0]["num_bytes"] == 128 + assert worker.get_kv_connector_stats() is None + + +def test_lookup_records_mooncake_metrics(): + worker = _make_bare_worker() + worker.store.batch_is_exist.return_value = [1, 1] + + result = worker.lookup(32, [b"a0", b"a1"]) + stats = worker.get_kv_connector_stats() + + assert result == 32 + assert isinstance(stats, MooncakeStoreConnectorStats) + assert len(stats.data["lookup_exists"]) == 1 + assert stats.data["lookup_exists"][0]["num_keys"] == 2 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py index 5d6136c587..bf0be63d5e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py @@ -28,6 +28,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorRole, SupportsHMA, ) +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, +) from vllm.forward_context import ForwardContext from vllm.logger import init_logger from vllm.v1.attention.backend import AttentionMetadata @@ -38,6 +44,7 @@ from vllm.v1.outputs import KVConnectorOutput from vllm.v1.request import Request from .data import MooncakeStoreConnectorMetadata +from .metrics import MooncakeStoreConnectorStats, MooncakeStorePromMetrics from .scheduler import MooncakeStoreScheduler from .worker import MooncakeStoreWorker @@ -274,3 +281,30 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): kv_events = MooncakeStoreKVEvents(num_workers=1) kv_events.add_events(events) return kv_events + + def get_kv_connector_stats(self) -> KVConnectorStats | None: + if self.connector_worker is None: + return None + return self.connector_worker.get_kv_connector_stats() + + @classmethod + def build_kv_connector_stats( + cls, data: dict[str, Any] | None = None + ) -> KVConnectorStats | None: + return ( + MooncakeStoreConnectorStats(data=data) + if data is not None + else MooncakeStoreConnectorStats() + ) + + @classmethod + def build_prom_metrics( + cls, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ) -> KVConnectorPromMetrics: + return MooncakeStorePromMetrics( + vllm_config, metric_types, labelnames, per_engine_labelvalues + ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/metrics.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/metrics.py new file mode 100644 index 0000000000..8dc949cf32 --- /dev/null +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/metrics.py @@ -0,0 +1,189 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Per-operation telemetry for MooncakeStoreConnector. + +Records one row per Mooncake RPC (``save_exists``, ``save_put``, ``load_get``, +``lookup_exists``) with duration, key/byte counts, status, and failed-key +count. Exposed to the logger via ``KVConnectorLogging`` and to Prometheus +via ``MooncakeStorePromMetrics``. +""" + +from dataclasses import dataclass +from statistics import fmean +from typing import Any + +from vllm.config import VllmConfig +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, +) + + +def _nearest_rank_percentile(values: list[float], percentile: float) -> float: + if not values: + return 0.0 + sorted_values = sorted(values) + rank = max( + 0, min(len(sorted_values) - 1, int(percentile * len(sorted_values) - 1e-12)) + ) + return sorted_values[rank] + + +@dataclass +class MooncakeStoreConnectorStats(KVConnectorStats): + """Serializable Mooncake store communication telemetry.""" + + def __post_init__(self): + if not self.data: + self.reset() + + def reset(self): + self.data: dict[str, list[dict[str, int | float | str]]] = {} + + def is_empty(self) -> bool: + return not self.data + + def aggregate(self, other: KVConnectorStats) -> KVConnectorStats: + if other.is_empty(): + return self + for operation, records in other.data.items(): + self.data.setdefault(operation, []).extend(records) + return self + + def reduce(self) -> dict[str, int | float]: + reduced: dict[str, int | float] = {} + for operation, records in sorted(self.data.items()): + if not records: + continue + durations = [float(record["duration_seconds"]) for record in records] + reduced[f"{operation}_count"] = len(records) + reduced[f"{operation}_avg_ms"] = round(fmean(durations) * 1e3, 3) + reduced[f"{operation}_p90_ms"] = round( + _nearest_rank_percentile(durations, 0.9) * 1e3, 3 + ) + reduced[f"{operation}_total_keys"] = sum( + int(record["num_keys"]) for record in records + ) + reduced[f"{operation}_total_bytes"] = sum( + int(record["num_bytes"]) for record in records + ) + reduced[f"{operation}_failed_keys"] = sum( + int(record["num_failed_keys"]) for record in records + ) + reduced[f"{operation}_error_count"] = sum( + 1 for record in records if record["status"] == "error" + ) + return reduced + + def record_operation( + self, + operation: str, + duration_seconds: float, + num_keys: int, + *, + num_bytes: int = 0, + status: str = "ok", + num_failed_keys: int = 0, + ) -> None: + self.data.setdefault(operation, []).append( + { + "duration_seconds": duration_seconds, + "num_keys": num_keys, + "num_bytes": num_bytes, + "status": status, + "num_failed_keys": num_failed_keys, + } + ) + + +class MooncakeStorePromMetrics(KVConnectorPromMetrics): + """Prometheus metrics for Mooncake store communication.""" + + def __init__( + self, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ): + super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues) + metric_labelnames = labelnames + ["operation", "status"] + self._metric_cache: dict[tuple[int, str, str], dict[str, PromMetric]] = {} + + self._histogram_operation_time = self._histogram_cls( + name="vllm:mooncake_store_operation_time_seconds", + documentation="Histogram of Mooncake store communication time.", + buckets=[ + 1e-3, + 5e-3, + 1e-2, + 5e-2, + 1e-1, + 2e-1, + 3e-1, + 4e-1, + 5e-1, + 7.5e-1, + 1.0, + 1.5, + 2.0, + 3.0, + 4.0, + ], + labelnames=metric_labelnames, + ) + self._counter_operation_calls = self._counter_cls( + name="vllm:mooncake_store_operation_total", + documentation="Number of Mooncake store communication operations.", + labelnames=metric_labelnames, + ) + self._counter_operation_keys = self._counter_cls( + name="vllm:mooncake_store_operation_keys_total", + documentation="Number of Mooncake store keys touched by operations.", + labelnames=metric_labelnames, + ) + self._counter_operation_bytes = self._counter_cls( + name="vllm:mooncake_store_operation_bytes_total", + documentation="Number of bytes transferred by Mooncake store operations.", + labelnames=metric_labelnames, + ) + self._counter_failed_keys = self._counter_cls( + name="vllm:mooncake_store_operation_failed_keys_total", + documentation="Number of Mooncake store keys that failed in operations.", + labelnames=metric_labelnames, + ) + + def _get_metrics( + self, + engine_idx: int, + operation: str, + status: str, + ) -> dict[str, PromMetric]: + cache_key = (engine_idx, operation, status) + if cache_key not in self._metric_cache: + label_values = self.per_engine_labelvalues[engine_idx] + [operation, status] + self._metric_cache[cache_key] = { + "time": self._histogram_operation_time.labels(*label_values), + "calls": self._counter_operation_calls.labels(*label_values), + "keys": self._counter_operation_keys.labels(*label_values), + "bytes": self._counter_operation_bytes.labels(*label_values), + "failed_keys": self._counter_failed_keys.labels(*label_values), + } + return self._metric_cache[cache_key] + + def observe(self, transfer_stats_data: dict[str, Any] | None, engine_idx: int = 0): + if not transfer_stats_data: + return + for operation, records in transfer_stats_data.items(): + assert isinstance(records, list) + for record in records: + assert isinstance(record, dict) + status = str(record["status"]) + metrics = self._get_metrics(engine_idx, operation, status) + metrics["time"].observe(float(record["duration_seconds"])) + metrics["calls"].inc() + metrics["keys"].inc(int(record["num_keys"])) + metrics["bytes"].inc(int(record["num_bytes"])) + metrics["failed_keys"].inc(int(record["num_failed_keys"])) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 2183cc730c..501c706e21 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -16,7 +16,9 @@ import os import queue import socket import threading +import time from collections import defaultdict +from collections.abc import Callable from dataclasses import dataclass from typing import Any, Literal @@ -58,6 +60,8 @@ from vllm.v1.core.kv_cache_utils import ( from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder +from .metrics import MooncakeStoreConnectorStats + logger = init_logger(__name__) DEFAULT_GLOBAL_SEGMENT_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB @@ -178,6 +182,10 @@ def _estimate_disk_offload_staging_bytes(size_list: list[int]) -> int: return _align_up(data_size, _DIRECT_IO_ALIGNMENT) + _DIRECT_IO_PADDING_BYTES +def _sum_batch_bytes(sizes: list[list[int]]) -> int: + return sum(sum(size) for size in sizes) + + def _get_usable_disk_offload_buffer_budget_bytes(raw_budget_bytes: int) -> int: return max(1, int(raw_budget_bytes * envs.VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO)) @@ -337,6 +345,7 @@ class KVTransferThread(threading.Thread): tp_rank: int, ready_event: threading.Event, name: str, + record_operation: Callable[..., None] | None = None, ): super().__init__(daemon=True, name=name) self.store = store @@ -344,6 +353,7 @@ class KVTransferThread(threading.Thread): self.block_size = block_size self.tp_rank = tp_rank self.token_databases = token_databases + self._record_operation_cb = record_operation self.done_task_lock = threading.Lock() self.request_queue: queue.Queue[Any] = queue.Queue() self.finished_requests: set[str] = set() @@ -379,6 +389,27 @@ class KVTransferThread(threading.Thread): def _handle_request(self, req_meta: Any): pass + def _record_operation( + self, + operation: str, + start_time: float, + num_keys: int, + *, + num_bytes: int = 0, + status: str = "ok", + num_failed_keys: int = 0, + ) -> None: + if self._record_operation_cb is None: + return + self._record_operation_cb( + operation=operation, + duration_seconds=time.perf_counter() - start_time, + num_keys=num_keys, + num_bytes=num_bytes, + status=status, + num_failed_keys=num_failed_keys, + ) + def update_kv_event(self, events: list[BlockStored]): with self.kv_event_lock: self.kv_events.extend(events) @@ -405,6 +436,7 @@ class KVCacheStoreSendingThread(KVTransferThread): ready_event: threading.Event, enable_kv_event: bool = False, replicate_config: Any = None, + record_operation: Callable[..., None] | None = None, ): super().__init__( store, @@ -413,6 +445,7 @@ class KVCacheStoreSendingThread(KVTransferThread): tp_rank, ready_event, name="KVCacheStoreSendingThread", + record_operation=record_operation, ) self.put_step = put_step self.coord = coord @@ -520,7 +553,23 @@ class KVCacheStoreSendingThread(KVTransferThread): return # Check which blocks already exist (dedup) - exists_states = self.store.batch_is_exist(keys) + save_exists_start = time.perf_counter() + try: + exists_states = self.store.batch_is_exist(keys) + except Exception: + self._record_operation( + "save_exists", + save_exists_start, + len(keys), + status="error", + num_failed_keys=len(keys), + ) + raise + self._record_operation( + "save_exists", + save_exists_start, + len(keys), + ) missing_indices = [i for i, exists in enumerate(exists_states) if exists != 1] if not missing_indices: @@ -574,6 +623,8 @@ class KVCacheStoreSendingThread(KVTransferThread): if current_event is not None: current_event.synchronize() + batch_bytes = _sum_batch_bytes(sizes) + put_start = time.perf_counter() try: res = self.store.batch_put_from_multi_buffers( keys, @@ -582,9 +633,15 @@ class KVCacheStoreSendingThread(KVTransferThread): self.replicate_config, ) failed = [i for i, v in enumerate(res) if v < 0] + self._record_operation( + "save_put", + put_start, + len(keys), + num_bytes=batch_bytes, + status="partial_failure" if failed else "ok", + num_failed_keys=len(failed), + ) if failed: - # Compute total bytes attempted for this batch - total_bytes = sum(sum(s) if isinstance(s, list) else s for s in sizes) failed_codes = set(res[i] for i in failed) logger.warning( "batch_put failed: %d/%d keys failed " @@ -593,7 +650,7 @@ class KVCacheStoreSendingThread(KVTransferThread): len(failed), len(keys), failed_codes, - total_bytes, + batch_bytes, len(keys), keys[0] if keys else "N/A", ) @@ -614,6 +671,14 @@ class KVCacheStoreSendingThread(KVTransferThread): "successful store batch" ) except Exception as e: + self._record_operation( + "save_put", + put_start, + len(keys), + num_bytes=batch_bytes, + status="error", + num_failed_keys=len(keys), + ) logger.error("Failed to put key %s, error: %s", keys, e) if self.enable_kv_event and stored_events: @@ -635,6 +700,7 @@ class KVCacheStoreRecvingThread(KVTransferThread): tp_rank: int, ready_event: threading.Event, disk_offload_buffer_budget_bytes: int | None = None, + record_operation: Callable[..., None] | None = None, ): super().__init__( store, @@ -643,6 +709,7 @@ class KVCacheStoreRecvingThread(KVTransferThread): tp_rank, ready_event, name="KVCacheStoreRecvingThread", + record_operation=record_operation, ) self.disk_offload_buffer_budget_bytes = disk_offload_buffer_budget_bytes self.usable_disk_offload_buffer_budget_bytes = ( @@ -721,12 +788,16 @@ class KVCacheStoreRecvingThread(KVTransferThread): return current_batch_keys: list[str] = key_list_c + batch_bytes = 0 try: for batch_keys, batch_addrs, batch_sizes in load_batches: current_batch_keys = batch_keys + batch_bytes = _sum_batch_bytes(batch_sizes) tiers_by_key: dict[str, str] | None = None if envs.VLLM_MOONCAKE_STORE_TIER_LOG: tiers_by_key = _get_replica_tiers_by_key(self.store, batch_keys) + # Reset so the recorded RPC duration excludes tier lookup. + load_get_start = time.perf_counter() res = self.store.batch_get_into_multi_buffers( batch_keys, batch_addrs, batch_sizes ) @@ -739,6 +810,14 @@ class KVCacheStoreRecvingThread(KVTransferThread): for key, value in zip(batch_keys, res, strict=True) if value < 0 ] + self._record_operation( + "load_get", + load_get_start, + len(batch_keys), + num_bytes=batch_bytes, + status="partial_failure" if failed else "ok", + num_failed_keys=len(failed), + ) if failed: logger.warning( "Failed to get %d Mooncake keys from sub-batch " @@ -749,6 +828,14 @@ class KVCacheStoreRecvingThread(KVTransferThread): ) break except Exception as e: + self._record_operation( + "load_get", + load_get_start, + len(current_batch_keys), + num_bytes=batch_bytes, + status="error", + num_failed_keys=len(current_batch_keys), + ) logger.warning( "Failed to get Mooncake sub-batch %s, error: %s", current_batch_keys[:3], @@ -921,6 +1008,8 @@ class MooncakeStoreWorker: self.kv_send_thread: KVCacheStoreSendingThread | None = None self.kv_recv_thread: KVCacheStoreRecvingThread | None = None self.finished_store_req: set[str] = set() + self._kv_connector_stats_lock = threading.Lock() + self.kv_connector_stats = MooncakeStoreConnectorStats() self._kv_cache_config = kv_cache_config # Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to @@ -1057,6 +1146,7 @@ class MooncakeStoreWorker: ready_event_sending, self.enable_kv_events, self.store_replicate_config, + record_operation=self._record_kv_connector_operation, ) self.kv_send_thread.start() @@ -1069,6 +1159,7 @@ class MooncakeStoreWorker: self.tp_rank, ready_event_recving, disk_offload_buffer_budget_bytes=self.disk_offload_buffer_budget_bytes, + record_operation=self._record_kv_connector_operation, ) self.kv_recv_thread.start() ready_event_recving.wait() @@ -1155,6 +1246,34 @@ class MooncakeStoreWorker: ) return done_sending, done_recving + def _record_kv_connector_operation( + self, + operation: str, + duration_seconds: float, + num_keys: int, + *, + num_bytes: int = 0, + status: str = "ok", + num_failed_keys: int = 0, + ) -> None: + with self._kv_connector_stats_lock: + self.kv_connector_stats.record_operation( + operation=operation, + duration_seconds=duration_seconds, + num_keys=num_keys, + num_bytes=num_bytes, + status=status, + num_failed_keys=num_failed_keys, + ) + + def get_kv_connector_stats(self) -> MooncakeStoreConnectorStats | None: + with self._kv_connector_stats_lock: + if self.kv_connector_stats.is_empty(): + return None + kv_connector_stats = self.kv_connector_stats + self.kv_connector_stats = MooncakeStoreConnectorStats() + return kv_connector_stats + def _get_and_clear_finished_sending( self, finished_req_ids: set[str], @@ -1216,9 +1335,22 @@ class MooncakeStoreWorker: if not candidate_keys: return 0 + lookup_start = time.perf_counter() try: res = self.store.batch_is_exist(candidate_keys) + self._record_kv_connector_operation( + "lookup_exists", + time.perf_counter() - lookup_start, + len(candidate_keys), + ) except Exception as e: + self._record_kv_connector_operation( + "lookup_exists", + time.perf_counter() - lookup_start, + len(candidate_keys), + status="error", + num_failed_keys=len(candidate_keys), + ) logger.error("Remote connection failed in lookup: %s", e) return 0