[Mooncake] Add metrics for MooncakeStoreConnector operations (#43392)

This commit is contained in:
Dao007forever
2026-05-23 13:34:40 -07:00
committed by GitHub
parent 4438b6e7dc
commit 819c610f9b
5 changed files with 507 additions and 4 deletions
@@ -14,6 +14,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
MooncakeStoreConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
MooncakeStoreConnectorStats,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
@@ -129,6 +132,49 @@ def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
assert connector.get_kv_connector_kv_cache_events() is None
def test_get_kv_connector_stats_delegates_to_worker():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
expected_stats = MooncakeStoreConnectorStats()
expected_stats.record_operation("save_put", 0.01, 2, num_bytes=1024)
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"connector.MooncakeStoreWorker"
) as mock_worker_cls,
):
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
mock_worker_cls.return_value.get_kv_connector_stats.return_value = expected_stats
stats = connector.get_kv_connector_stats()
assert stats is expected_stats
mock_worker_cls.return_value.get_kv_connector_stats.assert_called_once_with()
def test_build_kv_connector_stats_reconstructs_mooncake_stats():
stats = mooncake_store_connector.MooncakeStoreConnector.build_kv_connector_stats(
{
"save_put": [
{
"duration_seconds": 0.02,
"num_keys": 4,
"num_bytes": 2048,
"status": "ok",
"num_failed_keys": 0,
}
]
}
)
assert isinstance(stats, MooncakeStoreConnectorStats)
assert stats.data["save_put"][0]["num_bytes"] == 2048
def test_get_kv_connector_kv_cache_events_wraps_worker_events():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
@@ -28,6 +28,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
LoadSpec,
ReqMeta,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
MooncakeStoreConnectorStats,
)
def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator:
@@ -417,6 +420,24 @@ def test_store_sending_thread_skips_request_during_cpu_pressure():
assert store.batch_put_from_multi_buffers.call_count == 3
def test_store_sending_thread_records_mooncake_metrics():
store = MagicMock()
store.batch_is_exist.return_value = [0, 0]
store.batch_put_from_multi_buffers.return_value = [256, 256]
thread = _make_store_sending_thread(store)
stats = MooncakeStoreConnectorStats()
thread._record_operation_cb = stats.record_operation
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
assert len(stats.data["save_exists"]) == 1
assert stats.data["save_exists"][0]["num_keys"] == 2
assert len(stats.data["save_put"]) == 1
assert stats.data["save_put"][0]["num_bytes"] == 512
assert stats.data["save_put"][0]["status"] == "ok"
def test_store_sending_thread_only_skips_on_no_available_handle():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
@@ -548,6 +569,29 @@ def test_recv_thread_logs_tier_summary_when_enabled(monkeypatch, caplog_vllm):
)
def test_recv_thread_records_partial_failure_metrics(monkeypatch):
monkeypatch.delenv("VLLM_MOONCAKE_STORE_TIER_LOG", raising=False)
store = MagicMock()
store.batch_get_into_multi_buffers.return_value = [256, -10]
thread = _make_store_recving_thread(store, disk_offload_buffer_budget_bytes=None)
stats = MooncakeStoreConnectorStats()
thread._record_operation_cb = stats.record_operation
req = _make_load_req(
"req-a",
[b"a0", b"a1"],
token_len=32,
)
thread._handle_request(req)
assert len(stats.data["load_get"]) == 1
assert stats.data["load_get"][0]["num_keys"] == 2
assert stats.data["load_get"][0]["num_bytes"] == 512
assert stats.data["load_get"][0]["status"] == "partial_failure"
assert stats.data["load_get"][0]["num_failed_keys"] == 1
def test_recv_thread_uses_ratio_scaled_budget_for_first_pass_split():
store = MagicMock()
store.batch_get_into_multi_buffers.side_effect = [
@@ -1025,6 +1069,8 @@ def _make_bare_worker(
worker.disk_offload_buffer_budget_bytes = None
worker.store_replicate_config = SimpleNamespace()
worker._kv_connector_stats_lock = threading.Lock()
worker.kv_connector_stats = MooncakeStoreConnectorStats()
spec = FullAttentionSpec(
block_size=block_size, num_kv_heads=8, head_size=64, dtype=None
@@ -1373,3 +1419,59 @@ def test_topology_embedded_cpu_only(tmp_path, monkeypatch):
assert w.store_replicate_config.preferred_segment == ""
# No disk budget — enable_offload was absent (defaults to False).
assert w.disk_offload_buffer_budget_bytes is None
# ---------------------------------------------------------------------------
# Stats/metrics tests (PR-35 port)
# ---------------------------------------------------------------------------
def test_mooncake_store_stats_aggregate_reduce():
stats = MooncakeStoreConnectorStats()
stats.record_operation("save_put", 0.01, 2, num_bytes=128)
other = MooncakeStoreConnectorStats()
other.record_operation(
"save_put",
0.03,
1,
num_bytes=64,
status="error",
num_failed_keys=1,
)
reduced = stats.aggregate(other).reduce()
assert reduced["save_put_count"] == 2
assert reduced["save_put_total_keys"] == 3
assert reduced["save_put_total_bytes"] == 192
assert reduced["save_put_failed_keys"] == 1
assert reduced["save_put_error_count"] == 1
def test_worker_get_kv_connector_stats_resets_after_read():
worker = _make_bare_worker()
worker._record_kv_connector_operation(
"save_put",
0.01,
2,
num_bytes=128,
)
stats = worker.get_kv_connector_stats()
assert isinstance(stats, MooncakeStoreConnectorStats)
assert stats.data["save_put"][0]["num_bytes"] == 128
assert worker.get_kv_connector_stats() is None
def test_lookup_records_mooncake_metrics():
worker = _make_bare_worker()
worker.store.batch_is_exist.return_value = [1, 1]
result = worker.lookup(32, [b"a0", b"a1"])
stats = worker.get_kv_connector_stats()
assert result == 32
assert isinstance(stats, MooncakeStoreConnectorStats)
assert len(stats.data["lookup_exists"]) == 1
assert stats.data["lookup_exists"][0]["num_keys"] == 2
@@ -28,6 +28,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorRole,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
@@ -38,6 +44,7 @@ from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
from .data import MooncakeStoreConnectorMetadata
from .metrics import MooncakeStoreConnectorStats, MooncakeStorePromMetrics
from .scheduler import MooncakeStoreScheduler
from .worker import MooncakeStoreWorker
@@ -274,3 +281,30 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
kv_events = MooncakeStoreKVEvents(num_workers=1)
kv_events.add_events(events)
return kv_events
def get_kv_connector_stats(self) -> KVConnectorStats | None:
if self.connector_worker is None:
return None
return self.connector_worker.get_kv_connector_stats()
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
MooncakeStoreConnectorStats(data=data)
if data is not None
else MooncakeStoreConnectorStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return MooncakeStorePromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
@@ -0,0 +1,189 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Per-operation telemetry for MooncakeStoreConnector.
Records one row per Mooncake RPC (``save_exists``, ``save_put``, ``load_get``,
``lookup_exists``) with duration, key/byte counts, status, and failed-key
count. Exposed to the logger via ``KVConnectorLogging`` and to Prometheus
via ``MooncakeStorePromMetrics``.
"""
from dataclasses import dataclass
from statistics import fmean
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
def _nearest_rank_percentile(values: list[float], percentile: float) -> float:
if not values:
return 0.0
sorted_values = sorted(values)
rank = max(
0, min(len(sorted_values) - 1, int(percentile * len(sorted_values) - 1e-12))
)
return sorted_values[rank]
@dataclass
class MooncakeStoreConnectorStats(KVConnectorStats):
"""Serializable Mooncake store communication telemetry."""
def __post_init__(self):
if not self.data:
self.reset()
def reset(self):
self.data: dict[str, list[dict[str, int | float | str]]] = {}
def is_empty(self) -> bool:
return not self.data
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if other.is_empty():
return self
for operation, records in other.data.items():
self.data.setdefault(operation, []).extend(records)
return self
def reduce(self) -> dict[str, int | float]:
reduced: dict[str, int | float] = {}
for operation, records in sorted(self.data.items()):
if not records:
continue
durations = [float(record["duration_seconds"]) for record in records]
reduced[f"{operation}_count"] = len(records)
reduced[f"{operation}_avg_ms"] = round(fmean(durations) * 1e3, 3)
reduced[f"{operation}_p90_ms"] = round(
_nearest_rank_percentile(durations, 0.9) * 1e3, 3
)
reduced[f"{operation}_total_keys"] = sum(
int(record["num_keys"]) for record in records
)
reduced[f"{operation}_total_bytes"] = sum(
int(record["num_bytes"]) for record in records
)
reduced[f"{operation}_failed_keys"] = sum(
int(record["num_failed_keys"]) for record in records
)
reduced[f"{operation}_error_count"] = sum(
1 for record in records if record["status"] == "error"
)
return reduced
def record_operation(
self,
operation: str,
duration_seconds: float,
num_keys: int,
*,
num_bytes: int = 0,
status: str = "ok",
num_failed_keys: int = 0,
) -> None:
self.data.setdefault(operation, []).append(
{
"duration_seconds": duration_seconds,
"num_keys": num_keys,
"num_bytes": num_bytes,
"status": status,
"num_failed_keys": num_failed_keys,
}
)
class MooncakeStorePromMetrics(KVConnectorPromMetrics):
"""Prometheus metrics for Mooncake store communication."""
def __init__(
self,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
metric_labelnames = labelnames + ["operation", "status"]
self._metric_cache: dict[tuple[int, str, str], dict[str, PromMetric]] = {}
self._histogram_operation_time = self._histogram_cls(
name="vllm:mooncake_store_operation_time_seconds",
documentation="Histogram of Mooncake store communication time.",
buckets=[
1e-3,
5e-3,
1e-2,
5e-2,
1e-1,
2e-1,
3e-1,
4e-1,
5e-1,
7.5e-1,
1.0,
1.5,
2.0,
3.0,
4.0,
],
labelnames=metric_labelnames,
)
self._counter_operation_calls = self._counter_cls(
name="vllm:mooncake_store_operation_total",
documentation="Number of Mooncake store communication operations.",
labelnames=metric_labelnames,
)
self._counter_operation_keys = self._counter_cls(
name="vllm:mooncake_store_operation_keys_total",
documentation="Number of Mooncake store keys touched by operations.",
labelnames=metric_labelnames,
)
self._counter_operation_bytes = self._counter_cls(
name="vllm:mooncake_store_operation_bytes_total",
documentation="Number of bytes transferred by Mooncake store operations.",
labelnames=metric_labelnames,
)
self._counter_failed_keys = self._counter_cls(
name="vllm:mooncake_store_operation_failed_keys_total",
documentation="Number of Mooncake store keys that failed in operations.",
labelnames=metric_labelnames,
)
def _get_metrics(
self,
engine_idx: int,
operation: str,
status: str,
) -> dict[str, PromMetric]:
cache_key = (engine_idx, operation, status)
if cache_key not in self._metric_cache:
label_values = self.per_engine_labelvalues[engine_idx] + [operation, status]
self._metric_cache[cache_key] = {
"time": self._histogram_operation_time.labels(*label_values),
"calls": self._counter_operation_calls.labels(*label_values),
"keys": self._counter_operation_keys.labels(*label_values),
"bytes": self._counter_operation_bytes.labels(*label_values),
"failed_keys": self._counter_failed_keys.labels(*label_values),
}
return self._metric_cache[cache_key]
def observe(self, transfer_stats_data: dict[str, Any] | None, engine_idx: int = 0):
if not transfer_stats_data:
return
for operation, records in transfer_stats_data.items():
assert isinstance(records, list)
for record in records:
assert isinstance(record, dict)
status = str(record["status"])
metrics = self._get_metrics(engine_idx, operation, status)
metrics["time"].observe(float(record["duration_seconds"]))
metrics["calls"].inc()
metrics["keys"].inc(int(record["num_keys"]))
metrics["bytes"].inc(int(record["num_bytes"]))
metrics["failed_keys"].inc(int(record["num_failed_keys"]))
@@ -16,7 +16,9 @@ import os
import queue
import socket
import threading
import time
from collections import defaultdict
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any, Literal
@@ -58,6 +60,8 @@ from vllm.v1.core.kv_cache_utils import (
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
from .metrics import MooncakeStoreConnectorStats
logger = init_logger(__name__)
DEFAULT_GLOBAL_SEGMENT_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
@@ -178,6 +182,10 @@ def _estimate_disk_offload_staging_bytes(size_list: list[int]) -> int:
return _align_up(data_size, _DIRECT_IO_ALIGNMENT) + _DIRECT_IO_PADDING_BYTES
def _sum_batch_bytes(sizes: list[list[int]]) -> int:
return sum(sum(size) for size in sizes)
def _get_usable_disk_offload_buffer_budget_bytes(raw_budget_bytes: int) -> int:
return max(1, int(raw_budget_bytes * envs.VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO))
@@ -337,6 +345,7 @@ class KVTransferThread(threading.Thread):
tp_rank: int,
ready_event: threading.Event,
name: str,
record_operation: Callable[..., None] | None = None,
):
super().__init__(daemon=True, name=name)
self.store = store
@@ -344,6 +353,7 @@ class KVTransferThread(threading.Thread):
self.block_size = block_size
self.tp_rank = tp_rank
self.token_databases = token_databases
self._record_operation_cb = record_operation
self.done_task_lock = threading.Lock()
self.request_queue: queue.Queue[Any] = queue.Queue()
self.finished_requests: set[str] = set()
@@ -379,6 +389,27 @@ class KVTransferThread(threading.Thread):
def _handle_request(self, req_meta: Any):
pass
def _record_operation(
self,
operation: str,
start_time: float,
num_keys: int,
*,
num_bytes: int = 0,
status: str = "ok",
num_failed_keys: int = 0,
) -> None:
if self._record_operation_cb is None:
return
self._record_operation_cb(
operation=operation,
duration_seconds=time.perf_counter() - start_time,
num_keys=num_keys,
num_bytes=num_bytes,
status=status,
num_failed_keys=num_failed_keys,
)
def update_kv_event(self, events: list[BlockStored]):
with self.kv_event_lock:
self.kv_events.extend(events)
@@ -405,6 +436,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
ready_event: threading.Event,
enable_kv_event: bool = False,
replicate_config: Any = None,
record_operation: Callable[..., None] | None = None,
):
super().__init__(
store,
@@ -413,6 +445,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
tp_rank,
ready_event,
name="KVCacheStoreSendingThread",
record_operation=record_operation,
)
self.put_step = put_step
self.coord = coord
@@ -520,7 +553,23 @@ class KVCacheStoreSendingThread(KVTransferThread):
return
# Check which blocks already exist (dedup)
exists_states = self.store.batch_is_exist(keys)
save_exists_start = time.perf_counter()
try:
exists_states = self.store.batch_is_exist(keys)
except Exception:
self._record_operation(
"save_exists",
save_exists_start,
len(keys),
status="error",
num_failed_keys=len(keys),
)
raise
self._record_operation(
"save_exists",
save_exists_start,
len(keys),
)
missing_indices = [i for i, exists in enumerate(exists_states) if exists != 1]
if not missing_indices:
@@ -574,6 +623,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
if current_event is not None:
current_event.synchronize()
batch_bytes = _sum_batch_bytes(sizes)
put_start = time.perf_counter()
try:
res = self.store.batch_put_from_multi_buffers(
keys,
@@ -582,9 +633,15 @@ class KVCacheStoreSendingThread(KVTransferThread):
self.replicate_config,
)
failed = [i for i, v in enumerate(res) if v < 0]
self._record_operation(
"save_put",
put_start,
len(keys),
num_bytes=batch_bytes,
status="partial_failure" if failed else "ok",
num_failed_keys=len(failed),
)
if failed:
# Compute total bytes attempted for this batch
total_bytes = sum(sum(s) if isinstance(s, list) else s for s in sizes)
failed_codes = set(res[i] for i in failed)
logger.warning(
"batch_put failed: %d/%d keys failed "
@@ -593,7 +650,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
len(failed),
len(keys),
failed_codes,
total_bytes,
batch_bytes,
len(keys),
keys[0] if keys else "N/A",
)
@@ -614,6 +671,14 @@ class KVCacheStoreSendingThread(KVTransferThread):
"successful store batch"
)
except Exception as e:
self._record_operation(
"save_put",
put_start,
len(keys),
num_bytes=batch_bytes,
status="error",
num_failed_keys=len(keys),
)
logger.error("Failed to put key %s, error: %s", keys, e)
if self.enable_kv_event and stored_events:
@@ -635,6 +700,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
tp_rank: int,
ready_event: threading.Event,
disk_offload_buffer_budget_bytes: int | None = None,
record_operation: Callable[..., None] | None = None,
):
super().__init__(
store,
@@ -643,6 +709,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
tp_rank,
ready_event,
name="KVCacheStoreRecvingThread",
record_operation=record_operation,
)
self.disk_offload_buffer_budget_bytes = disk_offload_buffer_budget_bytes
self.usable_disk_offload_buffer_budget_bytes = (
@@ -721,12 +788,16 @@ class KVCacheStoreRecvingThread(KVTransferThread):
return
current_batch_keys: list[str] = key_list_c
batch_bytes = 0
try:
for batch_keys, batch_addrs, batch_sizes in load_batches:
current_batch_keys = batch_keys
batch_bytes = _sum_batch_bytes(batch_sizes)
tiers_by_key: dict[str, str] | None = None
if envs.VLLM_MOONCAKE_STORE_TIER_LOG:
tiers_by_key = _get_replica_tiers_by_key(self.store, batch_keys)
# Reset so the recorded RPC duration excludes tier lookup.
load_get_start = time.perf_counter()
res = self.store.batch_get_into_multi_buffers(
batch_keys, batch_addrs, batch_sizes
)
@@ -739,6 +810,14 @@ class KVCacheStoreRecvingThread(KVTransferThread):
for key, value in zip(batch_keys, res, strict=True)
if value < 0
]
self._record_operation(
"load_get",
load_get_start,
len(batch_keys),
num_bytes=batch_bytes,
status="partial_failure" if failed else "ok",
num_failed_keys=len(failed),
)
if failed:
logger.warning(
"Failed to get %d Mooncake keys from sub-batch "
@@ -749,6 +828,14 @@ class KVCacheStoreRecvingThread(KVTransferThread):
)
break
except Exception as e:
self._record_operation(
"load_get",
load_get_start,
len(current_batch_keys),
num_bytes=batch_bytes,
status="error",
num_failed_keys=len(current_batch_keys),
)
logger.warning(
"Failed to get Mooncake sub-batch %s, error: %s",
current_batch_keys[:3],
@@ -921,6 +1008,8 @@ class MooncakeStoreWorker:
self.kv_send_thread: KVCacheStoreSendingThread | None = None
self.kv_recv_thread: KVCacheStoreRecvingThread | None = None
self.finished_store_req: set[str] = set()
self._kv_connector_stats_lock = threading.Lock()
self.kv_connector_stats = MooncakeStoreConnectorStats()
self._kv_cache_config = kv_cache_config
# Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to
@@ -1057,6 +1146,7 @@ class MooncakeStoreWorker:
ready_event_sending,
self.enable_kv_events,
self.store_replicate_config,
record_operation=self._record_kv_connector_operation,
)
self.kv_send_thread.start()
@@ -1069,6 +1159,7 @@ class MooncakeStoreWorker:
self.tp_rank,
ready_event_recving,
disk_offload_buffer_budget_bytes=self.disk_offload_buffer_budget_bytes,
record_operation=self._record_kv_connector_operation,
)
self.kv_recv_thread.start()
ready_event_recving.wait()
@@ -1155,6 +1246,34 @@ class MooncakeStoreWorker:
)
return done_sending, done_recving
def _record_kv_connector_operation(
self,
operation: str,
duration_seconds: float,
num_keys: int,
*,
num_bytes: int = 0,
status: str = "ok",
num_failed_keys: int = 0,
) -> None:
with self._kv_connector_stats_lock:
self.kv_connector_stats.record_operation(
operation=operation,
duration_seconds=duration_seconds,
num_keys=num_keys,
num_bytes=num_bytes,
status=status,
num_failed_keys=num_failed_keys,
)
def get_kv_connector_stats(self) -> MooncakeStoreConnectorStats | None:
with self._kv_connector_stats_lock:
if self.kv_connector_stats.is_empty():
return None
kv_connector_stats = self.kv_connector_stats
self.kv_connector_stats = MooncakeStoreConnectorStats()
return kv_connector_stats
def _get_and_clear_finished_sending(
self,
finished_req_ids: set[str],
@@ -1216,9 +1335,22 @@ class MooncakeStoreWorker:
if not candidate_keys:
return 0
lookup_start = time.perf_counter()
try:
res = self.store.batch_is_exist(candidate_keys)
self._record_kv_connector_operation(
"lookup_exists",
time.perf_counter() - lookup_start,
len(candidate_keys),
)
except Exception as e:
self._record_kv_connector_operation(
"lookup_exists",
time.perf_counter() - lookup_start,
len(candidate_keys),
status="error",
num_failed_keys=len(candidate_keys),
)
logger.error("Remote connection failed in lookup: %s", e)
return 0