[Mooncake] Add metrics for MooncakeStoreConnector operations (#43392)
This commit is contained in:
@@ -14,6 +14,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
MooncakeStoreConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
|
||||
MooncakeStoreConnectorStats,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
@@ -129,6 +132,49 @@ def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
|
||||
assert connector.get_kv_connector_kv_cache_events() is None
|
||||
|
||||
|
||||
def test_get_kv_connector_stats_delegates_to_worker():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
expected_stats = MooncakeStoreConnectorStats()
|
||||
expected_stats.record_operation("save_put", 0.01, 2, num_bytes=1024)
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker_cls,
|
||||
):
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
|
||||
mock_worker_cls.return_value.get_kv_connector_stats.return_value = expected_stats
|
||||
stats = connector.get_kv_connector_stats()
|
||||
|
||||
assert stats is expected_stats
|
||||
mock_worker_cls.return_value.get_kv_connector_stats.assert_called_once_with()
|
||||
|
||||
|
||||
def test_build_kv_connector_stats_reconstructs_mooncake_stats():
|
||||
stats = mooncake_store_connector.MooncakeStoreConnector.build_kv_connector_stats(
|
||||
{
|
||||
"save_put": [
|
||||
{
|
||||
"duration_seconds": 0.02,
|
||||
"num_keys": 4,
|
||||
"num_bytes": 2048,
|
||||
"status": "ok",
|
||||
"num_failed_keys": 0,
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(stats, MooncakeStoreConnectorStats)
|
||||
assert stats.data["save_put"][0]["num_bytes"] == 2048
|
||||
|
||||
|
||||
def test_get_kv_connector_kv_cache_events_wraps_worker_events():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
|
||||
@@ -28,6 +28,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
LoadSpec,
|
||||
ReqMeta,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
|
||||
MooncakeStoreConnectorStats,
|
||||
)
|
||||
|
||||
|
||||
def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator:
|
||||
@@ -417,6 +420,24 @@ def test_store_sending_thread_skips_request_during_cpu_pressure():
|
||||
assert store.batch_put_from_multi_buffers.call_count == 3
|
||||
|
||||
|
||||
def test_store_sending_thread_records_mooncake_metrics():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [0, 0]
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
thread = _make_store_sending_thread(store)
|
||||
stats = MooncakeStoreConnectorStats()
|
||||
thread._record_operation_cb = stats.record_operation
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert len(stats.data["save_exists"]) == 1
|
||||
assert stats.data["save_exists"][0]["num_keys"] == 2
|
||||
assert len(stats.data["save_put"]) == 1
|
||||
assert stats.data["save_put"][0]["num_bytes"] == 512
|
||||
assert stats.data["save_put"][0]["status"] == "ok"
|
||||
|
||||
|
||||
def test_store_sending_thread_only_skips_on_no_available_handle():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
@@ -548,6 +569,29 @@ def test_recv_thread_logs_tier_summary_when_enabled(monkeypatch, caplog_vllm):
|
||||
)
|
||||
|
||||
|
||||
def test_recv_thread_records_partial_failure_metrics(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_MOONCAKE_STORE_TIER_LOG", raising=False)
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [256, -10]
|
||||
thread = _make_store_recving_thread(store, disk_offload_buffer_budget_bytes=None)
|
||||
stats = MooncakeStoreConnectorStats()
|
||||
thread._record_operation_cb = stats.record_operation
|
||||
|
||||
req = _make_load_req(
|
||||
"req-a",
|
||||
[b"a0", b"a1"],
|
||||
token_len=32,
|
||||
)
|
||||
|
||||
thread._handle_request(req)
|
||||
|
||||
assert len(stats.data["load_get"]) == 1
|
||||
assert stats.data["load_get"][0]["num_keys"] == 2
|
||||
assert stats.data["load_get"][0]["num_bytes"] == 512
|
||||
assert stats.data["load_get"][0]["status"] == "partial_failure"
|
||||
assert stats.data["load_get"][0]["num_failed_keys"] == 1
|
||||
|
||||
|
||||
def test_recv_thread_uses_ratio_scaled_budget_for_first_pass_split():
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.side_effect = [
|
||||
@@ -1025,6 +1069,8 @@ def _make_bare_worker(
|
||||
|
||||
worker.disk_offload_buffer_budget_bytes = None
|
||||
worker.store_replicate_config = SimpleNamespace()
|
||||
worker._kv_connector_stats_lock = threading.Lock()
|
||||
worker.kv_connector_stats = MooncakeStoreConnectorStats()
|
||||
|
||||
spec = FullAttentionSpec(
|
||||
block_size=block_size, num_kv_heads=8, head_size=64, dtype=None
|
||||
@@ -1373,3 +1419,59 @@ def test_topology_embedded_cpu_only(tmp_path, monkeypatch):
|
||||
assert w.store_replicate_config.preferred_segment == ""
|
||||
# No disk budget — enable_offload was absent (defaults to False).
|
||||
assert w.disk_offload_buffer_budget_bytes is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stats/metrics tests (PR-35 port)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_mooncake_store_stats_aggregate_reduce():
|
||||
stats = MooncakeStoreConnectorStats()
|
||||
stats.record_operation("save_put", 0.01, 2, num_bytes=128)
|
||||
other = MooncakeStoreConnectorStats()
|
||||
other.record_operation(
|
||||
"save_put",
|
||||
0.03,
|
||||
1,
|
||||
num_bytes=64,
|
||||
status="error",
|
||||
num_failed_keys=1,
|
||||
)
|
||||
|
||||
reduced = stats.aggregate(other).reduce()
|
||||
|
||||
assert reduced["save_put_count"] == 2
|
||||
assert reduced["save_put_total_keys"] == 3
|
||||
assert reduced["save_put_total_bytes"] == 192
|
||||
assert reduced["save_put_failed_keys"] == 1
|
||||
assert reduced["save_put_error_count"] == 1
|
||||
|
||||
|
||||
def test_worker_get_kv_connector_stats_resets_after_read():
|
||||
worker = _make_bare_worker()
|
||||
worker._record_kv_connector_operation(
|
||||
"save_put",
|
||||
0.01,
|
||||
2,
|
||||
num_bytes=128,
|
||||
)
|
||||
|
||||
stats = worker.get_kv_connector_stats()
|
||||
|
||||
assert isinstance(stats, MooncakeStoreConnectorStats)
|
||||
assert stats.data["save_put"][0]["num_bytes"] == 128
|
||||
assert worker.get_kv_connector_stats() is None
|
||||
|
||||
|
||||
def test_lookup_records_mooncake_metrics():
|
||||
worker = _make_bare_worker()
|
||||
worker.store.batch_is_exist.return_value = [1, 1]
|
||||
|
||||
result = worker.lookup(32, [b"a0", b"a1"])
|
||||
stats = worker.get_kv_connector_stats()
|
||||
|
||||
assert result == 32
|
||||
assert isinstance(stats, MooncakeStoreConnectorStats)
|
||||
assert len(stats.data["lookup_exists"]) == 1
|
||||
assert stats.data["lookup_exists"][0]["num_keys"] == 2
|
||||
|
||||
@@ -28,6 +28,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
@@ -38,6 +44,7 @@ from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
from .data import MooncakeStoreConnectorMetadata
|
||||
from .metrics import MooncakeStoreConnectorStats, MooncakeStorePromMetrics
|
||||
from .scheduler import MooncakeStoreScheduler
|
||||
from .worker import MooncakeStoreWorker
|
||||
|
||||
@@ -274,3 +281,30 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
kv_events = MooncakeStoreKVEvents(num_workers=1)
|
||||
kv_events.add_events(events)
|
||||
return kv_events
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
if self.connector_worker is None:
|
||||
return None
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
MooncakeStoreConnectorStats(data=data)
|
||||
if data is not None
|
||||
else MooncakeStoreConnectorStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return MooncakeStorePromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Per-operation telemetry for MooncakeStoreConnector.
|
||||
|
||||
Records one row per Mooncake RPC (``save_exists``, ``save_put``, ``load_get``,
|
||||
``lookup_exists``) with duration, key/byte counts, status, and failed-key
|
||||
count. Exposed to the logger via ``KVConnectorLogging`` and to Prometheus
|
||||
via ``MooncakeStorePromMetrics``.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from statistics import fmean
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
|
||||
|
||||
def _nearest_rank_percentile(values: list[float], percentile: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
sorted_values = sorted(values)
|
||||
rank = max(
|
||||
0, min(len(sorted_values) - 1, int(percentile * len(sorted_values) - 1e-12))
|
||||
)
|
||||
return sorted_values[rank]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MooncakeStoreConnectorStats(KVConnectorStats):
|
||||
"""Serializable Mooncake store communication telemetry."""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.data: dict[str, list[dict[str, int | float | str]]] = {}
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return not self.data
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if other.is_empty():
|
||||
return self
|
||||
for operation, records in other.data.items():
|
||||
self.data.setdefault(operation, []).extend(records)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
reduced: dict[str, int | float] = {}
|
||||
for operation, records in sorted(self.data.items()):
|
||||
if not records:
|
||||
continue
|
||||
durations = [float(record["duration_seconds"]) for record in records]
|
||||
reduced[f"{operation}_count"] = len(records)
|
||||
reduced[f"{operation}_avg_ms"] = round(fmean(durations) * 1e3, 3)
|
||||
reduced[f"{operation}_p90_ms"] = round(
|
||||
_nearest_rank_percentile(durations, 0.9) * 1e3, 3
|
||||
)
|
||||
reduced[f"{operation}_total_keys"] = sum(
|
||||
int(record["num_keys"]) for record in records
|
||||
)
|
||||
reduced[f"{operation}_total_bytes"] = sum(
|
||||
int(record["num_bytes"]) for record in records
|
||||
)
|
||||
reduced[f"{operation}_failed_keys"] = sum(
|
||||
int(record["num_failed_keys"]) for record in records
|
||||
)
|
||||
reduced[f"{operation}_error_count"] = sum(
|
||||
1 for record in records if record["status"] == "error"
|
||||
)
|
||||
return reduced
|
||||
|
||||
def record_operation(
|
||||
self,
|
||||
operation: str,
|
||||
duration_seconds: float,
|
||||
num_keys: int,
|
||||
*,
|
||||
num_bytes: int = 0,
|
||||
status: str = "ok",
|
||||
num_failed_keys: int = 0,
|
||||
) -> None:
|
||||
self.data.setdefault(operation, []).append(
|
||||
{
|
||||
"duration_seconds": duration_seconds,
|
||||
"num_keys": num_keys,
|
||||
"num_bytes": num_bytes,
|
||||
"status": status,
|
||||
"num_failed_keys": num_failed_keys,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class MooncakeStorePromMetrics(KVConnectorPromMetrics):
|
||||
"""Prometheus metrics for Mooncake store communication."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
):
|
||||
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
|
||||
metric_labelnames = labelnames + ["operation", "status"]
|
||||
self._metric_cache: dict[tuple[int, str, str], dict[str, PromMetric]] = {}
|
||||
|
||||
self._histogram_operation_time = self._histogram_cls(
|
||||
name="vllm:mooncake_store_operation_time_seconds",
|
||||
documentation="Histogram of Mooncake store communication time.",
|
||||
buckets=[
|
||||
1e-3,
|
||||
5e-3,
|
||||
1e-2,
|
||||
5e-2,
|
||||
1e-1,
|
||||
2e-1,
|
||||
3e-1,
|
||||
4e-1,
|
||||
5e-1,
|
||||
7.5e-1,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
],
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
self._counter_operation_calls = self._counter_cls(
|
||||
name="vllm:mooncake_store_operation_total",
|
||||
documentation="Number of Mooncake store communication operations.",
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
self._counter_operation_keys = self._counter_cls(
|
||||
name="vllm:mooncake_store_operation_keys_total",
|
||||
documentation="Number of Mooncake store keys touched by operations.",
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
self._counter_operation_bytes = self._counter_cls(
|
||||
name="vllm:mooncake_store_operation_bytes_total",
|
||||
documentation="Number of bytes transferred by Mooncake store operations.",
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
self._counter_failed_keys = self._counter_cls(
|
||||
name="vllm:mooncake_store_operation_failed_keys_total",
|
||||
documentation="Number of Mooncake store keys that failed in operations.",
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
|
||||
def _get_metrics(
|
||||
self,
|
||||
engine_idx: int,
|
||||
operation: str,
|
||||
status: str,
|
||||
) -> dict[str, PromMetric]:
|
||||
cache_key = (engine_idx, operation, status)
|
||||
if cache_key not in self._metric_cache:
|
||||
label_values = self.per_engine_labelvalues[engine_idx] + [operation, status]
|
||||
self._metric_cache[cache_key] = {
|
||||
"time": self._histogram_operation_time.labels(*label_values),
|
||||
"calls": self._counter_operation_calls.labels(*label_values),
|
||||
"keys": self._counter_operation_keys.labels(*label_values),
|
||||
"bytes": self._counter_operation_bytes.labels(*label_values),
|
||||
"failed_keys": self._counter_failed_keys.labels(*label_values),
|
||||
}
|
||||
return self._metric_cache[cache_key]
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any] | None, engine_idx: int = 0):
|
||||
if not transfer_stats_data:
|
||||
return
|
||||
for operation, records in transfer_stats_data.items():
|
||||
assert isinstance(records, list)
|
||||
for record in records:
|
||||
assert isinstance(record, dict)
|
||||
status = str(record["status"])
|
||||
metrics = self._get_metrics(engine_idx, operation, status)
|
||||
metrics["time"].observe(float(record["duration_seconds"]))
|
||||
metrics["calls"].inc()
|
||||
metrics["keys"].inc(int(record["num_keys"]))
|
||||
metrics["bytes"].inc(int(record["num_bytes"]))
|
||||
metrics["failed_keys"].inc(int(record["num_failed_keys"]))
|
||||
@@ -16,7 +16,9 @@ import os
|
||||
import queue
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal
|
||||
|
||||
@@ -58,6 +60,8 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec
|
||||
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
|
||||
|
||||
from .metrics import MooncakeStoreConnectorStats
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
DEFAULT_GLOBAL_SEGMENT_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
|
||||
@@ -178,6 +182,10 @@ def _estimate_disk_offload_staging_bytes(size_list: list[int]) -> int:
|
||||
return _align_up(data_size, _DIRECT_IO_ALIGNMENT) + _DIRECT_IO_PADDING_BYTES
|
||||
|
||||
|
||||
def _sum_batch_bytes(sizes: list[list[int]]) -> int:
|
||||
return sum(sum(size) for size in sizes)
|
||||
|
||||
|
||||
def _get_usable_disk_offload_buffer_budget_bytes(raw_budget_bytes: int) -> int:
|
||||
return max(1, int(raw_budget_bytes * envs.VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO))
|
||||
|
||||
@@ -337,6 +345,7 @@ class KVTransferThread(threading.Thread):
|
||||
tp_rank: int,
|
||||
ready_event: threading.Event,
|
||||
name: str,
|
||||
record_operation: Callable[..., None] | None = None,
|
||||
):
|
||||
super().__init__(daemon=True, name=name)
|
||||
self.store = store
|
||||
@@ -344,6 +353,7 @@ class KVTransferThread(threading.Thread):
|
||||
self.block_size = block_size
|
||||
self.tp_rank = tp_rank
|
||||
self.token_databases = token_databases
|
||||
self._record_operation_cb = record_operation
|
||||
self.done_task_lock = threading.Lock()
|
||||
self.request_queue: queue.Queue[Any] = queue.Queue()
|
||||
self.finished_requests: set[str] = set()
|
||||
@@ -379,6 +389,27 @@ class KVTransferThread(threading.Thread):
|
||||
def _handle_request(self, req_meta: Any):
|
||||
pass
|
||||
|
||||
def _record_operation(
|
||||
self,
|
||||
operation: str,
|
||||
start_time: float,
|
||||
num_keys: int,
|
||||
*,
|
||||
num_bytes: int = 0,
|
||||
status: str = "ok",
|
||||
num_failed_keys: int = 0,
|
||||
) -> None:
|
||||
if self._record_operation_cb is None:
|
||||
return
|
||||
self._record_operation_cb(
|
||||
operation=operation,
|
||||
duration_seconds=time.perf_counter() - start_time,
|
||||
num_keys=num_keys,
|
||||
num_bytes=num_bytes,
|
||||
status=status,
|
||||
num_failed_keys=num_failed_keys,
|
||||
)
|
||||
|
||||
def update_kv_event(self, events: list[BlockStored]):
|
||||
with self.kv_event_lock:
|
||||
self.kv_events.extend(events)
|
||||
@@ -405,6 +436,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
ready_event: threading.Event,
|
||||
enable_kv_event: bool = False,
|
||||
replicate_config: Any = None,
|
||||
record_operation: Callable[..., None] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
store,
|
||||
@@ -413,6 +445,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
tp_rank,
|
||||
ready_event,
|
||||
name="KVCacheStoreSendingThread",
|
||||
record_operation=record_operation,
|
||||
)
|
||||
self.put_step = put_step
|
||||
self.coord = coord
|
||||
@@ -520,7 +553,23 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
return
|
||||
|
||||
# Check which blocks already exist (dedup)
|
||||
exists_states = self.store.batch_is_exist(keys)
|
||||
save_exists_start = time.perf_counter()
|
||||
try:
|
||||
exists_states = self.store.batch_is_exist(keys)
|
||||
except Exception:
|
||||
self._record_operation(
|
||||
"save_exists",
|
||||
save_exists_start,
|
||||
len(keys),
|
||||
status="error",
|
||||
num_failed_keys=len(keys),
|
||||
)
|
||||
raise
|
||||
self._record_operation(
|
||||
"save_exists",
|
||||
save_exists_start,
|
||||
len(keys),
|
||||
)
|
||||
missing_indices = [i for i, exists in enumerate(exists_states) if exists != 1]
|
||||
|
||||
if not missing_indices:
|
||||
@@ -574,6 +623,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
if current_event is not None:
|
||||
current_event.synchronize()
|
||||
|
||||
batch_bytes = _sum_batch_bytes(sizes)
|
||||
put_start = time.perf_counter()
|
||||
try:
|
||||
res = self.store.batch_put_from_multi_buffers(
|
||||
keys,
|
||||
@@ -582,9 +633,15 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
self.replicate_config,
|
||||
)
|
||||
failed = [i for i, v in enumerate(res) if v < 0]
|
||||
self._record_operation(
|
||||
"save_put",
|
||||
put_start,
|
||||
len(keys),
|
||||
num_bytes=batch_bytes,
|
||||
status="partial_failure" if failed else "ok",
|
||||
num_failed_keys=len(failed),
|
||||
)
|
||||
if failed:
|
||||
# Compute total bytes attempted for this batch
|
||||
total_bytes = sum(sum(s) if isinstance(s, list) else s for s in sizes)
|
||||
failed_codes = set(res[i] for i in failed)
|
||||
logger.warning(
|
||||
"batch_put failed: %d/%d keys failed "
|
||||
@@ -593,7 +650,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
len(failed),
|
||||
len(keys),
|
||||
failed_codes,
|
||||
total_bytes,
|
||||
batch_bytes,
|
||||
len(keys),
|
||||
keys[0] if keys else "N/A",
|
||||
)
|
||||
@@ -614,6 +671,14 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
"successful store batch"
|
||||
)
|
||||
except Exception as e:
|
||||
self._record_operation(
|
||||
"save_put",
|
||||
put_start,
|
||||
len(keys),
|
||||
num_bytes=batch_bytes,
|
||||
status="error",
|
||||
num_failed_keys=len(keys),
|
||||
)
|
||||
logger.error("Failed to put key %s, error: %s", keys, e)
|
||||
|
||||
if self.enable_kv_event and stored_events:
|
||||
@@ -635,6 +700,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
tp_rank: int,
|
||||
ready_event: threading.Event,
|
||||
disk_offload_buffer_budget_bytes: int | None = None,
|
||||
record_operation: Callable[..., None] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
store,
|
||||
@@ -643,6 +709,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
tp_rank,
|
||||
ready_event,
|
||||
name="KVCacheStoreRecvingThread",
|
||||
record_operation=record_operation,
|
||||
)
|
||||
self.disk_offload_buffer_budget_bytes = disk_offload_buffer_budget_bytes
|
||||
self.usable_disk_offload_buffer_budget_bytes = (
|
||||
@@ -721,12 +788,16 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
return
|
||||
|
||||
current_batch_keys: list[str] = key_list_c
|
||||
batch_bytes = 0
|
||||
try:
|
||||
for batch_keys, batch_addrs, batch_sizes in load_batches:
|
||||
current_batch_keys = batch_keys
|
||||
batch_bytes = _sum_batch_bytes(batch_sizes)
|
||||
tiers_by_key: dict[str, str] | None = None
|
||||
if envs.VLLM_MOONCAKE_STORE_TIER_LOG:
|
||||
tiers_by_key = _get_replica_tiers_by_key(self.store, batch_keys)
|
||||
# Reset so the recorded RPC duration excludes tier lookup.
|
||||
load_get_start = time.perf_counter()
|
||||
res = self.store.batch_get_into_multi_buffers(
|
||||
batch_keys, batch_addrs, batch_sizes
|
||||
)
|
||||
@@ -739,6 +810,14 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
for key, value in zip(batch_keys, res, strict=True)
|
||||
if value < 0
|
||||
]
|
||||
self._record_operation(
|
||||
"load_get",
|
||||
load_get_start,
|
||||
len(batch_keys),
|
||||
num_bytes=batch_bytes,
|
||||
status="partial_failure" if failed else "ok",
|
||||
num_failed_keys=len(failed),
|
||||
)
|
||||
if failed:
|
||||
logger.warning(
|
||||
"Failed to get %d Mooncake keys from sub-batch "
|
||||
@@ -749,6 +828,14 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
)
|
||||
break
|
||||
except Exception as e:
|
||||
self._record_operation(
|
||||
"load_get",
|
||||
load_get_start,
|
||||
len(current_batch_keys),
|
||||
num_bytes=batch_bytes,
|
||||
status="error",
|
||||
num_failed_keys=len(current_batch_keys),
|
||||
)
|
||||
logger.warning(
|
||||
"Failed to get Mooncake sub-batch %s, error: %s",
|
||||
current_batch_keys[:3],
|
||||
@@ -921,6 +1008,8 @@ class MooncakeStoreWorker:
|
||||
self.kv_send_thread: KVCacheStoreSendingThread | None = None
|
||||
self.kv_recv_thread: KVCacheStoreRecvingThread | None = None
|
||||
self.finished_store_req: set[str] = set()
|
||||
self._kv_connector_stats_lock = threading.Lock()
|
||||
self.kv_connector_stats = MooncakeStoreConnectorStats()
|
||||
|
||||
self._kv_cache_config = kv_cache_config
|
||||
# Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to
|
||||
@@ -1057,6 +1146,7 @@ class MooncakeStoreWorker:
|
||||
ready_event_sending,
|
||||
self.enable_kv_events,
|
||||
self.store_replicate_config,
|
||||
record_operation=self._record_kv_connector_operation,
|
||||
)
|
||||
self.kv_send_thread.start()
|
||||
|
||||
@@ -1069,6 +1159,7 @@ class MooncakeStoreWorker:
|
||||
self.tp_rank,
|
||||
ready_event_recving,
|
||||
disk_offload_buffer_budget_bytes=self.disk_offload_buffer_budget_bytes,
|
||||
record_operation=self._record_kv_connector_operation,
|
||||
)
|
||||
self.kv_recv_thread.start()
|
||||
ready_event_recving.wait()
|
||||
@@ -1155,6 +1246,34 @@ class MooncakeStoreWorker:
|
||||
)
|
||||
return done_sending, done_recving
|
||||
|
||||
def _record_kv_connector_operation(
|
||||
self,
|
||||
operation: str,
|
||||
duration_seconds: float,
|
||||
num_keys: int,
|
||||
*,
|
||||
num_bytes: int = 0,
|
||||
status: str = "ok",
|
||||
num_failed_keys: int = 0,
|
||||
) -> None:
|
||||
with self._kv_connector_stats_lock:
|
||||
self.kv_connector_stats.record_operation(
|
||||
operation=operation,
|
||||
duration_seconds=duration_seconds,
|
||||
num_keys=num_keys,
|
||||
num_bytes=num_bytes,
|
||||
status=status,
|
||||
num_failed_keys=num_failed_keys,
|
||||
)
|
||||
|
||||
def get_kv_connector_stats(self) -> MooncakeStoreConnectorStats | None:
|
||||
with self._kv_connector_stats_lock:
|
||||
if self.kv_connector_stats.is_empty():
|
||||
return None
|
||||
kv_connector_stats = self.kv_connector_stats
|
||||
self.kv_connector_stats = MooncakeStoreConnectorStats()
|
||||
return kv_connector_stats
|
||||
|
||||
def _get_and_clear_finished_sending(
|
||||
self,
|
||||
finished_req_ids: set[str],
|
||||
@@ -1216,9 +1335,22 @@ class MooncakeStoreWorker:
|
||||
if not candidate_keys:
|
||||
return 0
|
||||
|
||||
lookup_start = time.perf_counter()
|
||||
try:
|
||||
res = self.store.batch_is_exist(candidate_keys)
|
||||
self._record_kv_connector_operation(
|
||||
"lookup_exists",
|
||||
time.perf_counter() - lookup_start,
|
||||
len(candidate_keys),
|
||||
)
|
||||
except Exception as e:
|
||||
self._record_kv_connector_operation(
|
||||
"lookup_exists",
|
||||
time.perf_counter() - lookup_start,
|
||||
len(candidate_keys),
|
||||
status="error",
|
||||
num_failed_keys=len(candidate_keys),
|
||||
)
|
||||
logger.error("Remote connection failed in lookup: %s", e)
|
||||
return 0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user