[P/D][Mooncake] Add KVConnectorStats for transfer observability (#40414)
Signed-off-by: Zhewen Li <zhewenli@inferact.ai> Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
This commit is contained in:
@@ -31,10 +31,14 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
|
||||
MooncakeBootstrapServer,
|
||||
RegisterWorkerPayload,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.stats import (
|
||||
MooncakeKVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
@@ -457,6 +461,25 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def wait_for_save(self):
|
||||
pass
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""Return worker-local transfer stats since the last call.
|
||||
|
||||
Note the P/D asymmetry: because Mooncake is P-push (P calls
|
||||
batch_transfer_sync_write), P records successful transfer latency,
|
||||
bytes, and descriptor counts, while D only records failures
|
||||
(recv/ZMQ errors). Aggregated NIXL-style dashboards will find
|
||||
successful-transfer metrics on the P worker, not D.
|
||||
"""
|
||||
if self.connector_worker is None:
|
||||
return None
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return MooncakeKVConnectorStats(data=data or {})
|
||||
|
||||
|
||||
class MooncakeConnectorScheduler:
|
||||
"""Implementation of Scheduler side methods"""
|
||||
@@ -816,6 +839,8 @@ class MooncakeConnectorWorker:
|
||||
self.finished_sending_reqs: set[ReqId] = set()
|
||||
self.finished_recving_reqs: set[ReqId] = set()
|
||||
|
||||
self.xfer_stats = MooncakeKVConnectorStats()
|
||||
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self.model_config = vllm_config.model_config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
@@ -1340,11 +1365,23 @@ class MooncakeConnectorWorker:
|
||||
ret_value = self.engine.batch_transfer_sync_write(
|
||||
remote_session, src_ptrs, dst_ptrs, lengths
|
||||
)
|
||||
duration = time.perf_counter() - start_time
|
||||
if ret_value == 0:
|
||||
logger.debug(
|
||||
"Sending to %s done, took %s",
|
||||
self.xfer_stats.record_transfer(
|
||||
duration_s=duration,
|
||||
total_bytes=sum(lengths),
|
||||
num_descs=len(src_ptrs),
|
||||
)
|
||||
logger.debug("Sending to %s done, took %s", remote_session, duration)
|
||||
else:
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
logger.warning(
|
||||
"Sending to %s failed (ret=%s) after %s (%d descriptors, %d bytes)",
|
||||
remote_session,
|
||||
time.perf_counter() - start_time,
|
||||
ret_value,
|
||||
duration,
|
||||
len(src_ptrs),
|
||||
sum(lengths),
|
||||
)
|
||||
return ret_value
|
||||
|
||||
@@ -1445,6 +1482,7 @@ class MooncakeConnectorWorker:
|
||||
send_meta.p_req_id,
|
||||
envs.VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT,
|
||||
)
|
||||
self.xfer_stats.record_kv_expired_req()
|
||||
finished_sending_reqs.add(send_meta.p_req_id)
|
||||
expired_transfer_id.append(transfer_id)
|
||||
|
||||
@@ -1485,6 +1523,13 @@ class MooncakeConnectorWorker:
|
||||
|
||||
return finished_sending_reqs or None, finished_recving_reqs or None
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""Return transfer stats collected since the last call, or None
|
||||
if nothing has been recorded in this interval."""
|
||||
if self.xfer_stats.is_empty():
|
||||
return None
|
||||
return self.xfer_stats.clone_and_reset()
|
||||
|
||||
async def receive_kv_from_single_worker(
|
||||
self,
|
||||
worker_addr: str,
|
||||
@@ -1531,6 +1576,7 @@ class MooncakeConnectorWorker:
|
||||
req_ids,
|
||||
response.err_msg,
|
||||
)
|
||||
self.xfer_stats.record_failed_recv()
|
||||
return
|
||||
self.process_pulling_result(response, pull_metas)
|
||||
if response.status == MooncakeXferResponseStatus.FINISH:
|
||||
@@ -1539,6 +1585,7 @@ class MooncakeConnectorWorker:
|
||||
logger.debug("ZMQ context terminated, exiting Mooncake receiver thread.")
|
||||
except Exception as e:
|
||||
logger.error("MooncakeXferMetadata transfer failed for %s: %s", req_ids, e)
|
||||
self.xfer_stats.record_failed_recv()
|
||||
return
|
||||
|
||||
def process_pulling_result(
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stats container for the Mooncake connector."""
|
||||
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorStats,
|
||||
)
|
||||
|
||||
# TODO(mooncake-stats): add MooncakePromMetrics (mirror NixlPromMetrics)
|
||||
# and wire it via MooncakeConnector.build_prom_metrics in a follow-up PR.
|
||||
|
||||
|
||||
@dataclass
|
||||
class MooncakeKVConnectorStats(KVConnectorStats):
|
||||
"""Container for Mooncake KV transfer performance metrics.
|
||||
|
||||
`_lock` serializes record_* against clone_and_reset so each row's
|
||||
appends are atomic and column lengths stay aligned. Writers run on
|
||||
the sender pool / receiver loop / sender loop; reader runs on the
|
||||
main worker thread.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
self._lock = threading.Lock()
|
||||
if not self.data:
|
||||
self.reset()
|
||||
|
||||
# threading.Lock is not picklable; strip it from the wire form and
|
||||
# rebuild a fresh per-process lock on the receiver side.
|
||||
def __getstate__(self) -> dict[str, Any]:
|
||||
state = self.__dict__.copy()
|
||||
state.pop("_lock", None)
|
||||
return state
|
||||
|
||||
def __setstate__(self, state: dict[str, Any]) -> None:
|
||||
self.__dict__.update(state)
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def reset(self):
|
||||
self.data: dict[str, list[float | int]] = {
|
||||
"transfer_duration": [],
|
||||
"bytes_transferred": [],
|
||||
"num_descriptors": [],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_recvs": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
|
||||
def record_transfer(self, duration_s: float, total_bytes: int, num_descs: int):
|
||||
with self._lock:
|
||||
self.data["transfer_duration"].append(duration_s)
|
||||
self.data["bytes_transferred"].append(total_bytes)
|
||||
self.data["num_descriptors"].append(num_descs)
|
||||
|
||||
# Failure counters store a list of 1s so a future Prom counter can iterate
|
||||
# with .inc(list_item), mirroring NIXL's NixlPromMetrics.observe.
|
||||
def record_failed_transfer(self):
|
||||
with self._lock:
|
||||
self.data["num_failed_transfers"].append(1)
|
||||
|
||||
def record_failed_recv(self):
|
||||
with self._lock:
|
||||
self.data["num_failed_recvs"].append(1)
|
||||
|
||||
def record_kv_expired_req(self):
|
||||
with self._lock:
|
||||
self.data["num_kv_expired_reqs"].append(1)
|
||||
|
||||
def clone_and_reset(self) -> "MooncakeKVConnectorStats":
|
||||
# Copy lists under the lock for length alignment; return a fresh
|
||||
# instance so the snapshot has its own _lock.
|
||||
with self._lock:
|
||||
snapshot_data: dict[str, list[float | int]] = {
|
||||
k: list(v) for k, v in self.data.items()
|
||||
}
|
||||
self.reset()
|
||||
return MooncakeKVConnectorStats(data=snapshot_data)
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return (
|
||||
self.num_successful_transfers == 0
|
||||
and len(self.data["num_failed_transfers"]) == 0
|
||||
and len(self.data["num_failed_recvs"]) == 0
|
||||
and len(self.data["num_kv_expired_reqs"]) == 0
|
||||
)
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if not other.is_empty():
|
||||
for k, v in other.data.items():
|
||||
accumulator = self.data[k]
|
||||
assert isinstance(accumulator, list)
|
||||
accumulator.extend(v)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
num_failed_transfers = len(self.data["num_failed_transfers"])
|
||||
num_failed_recvs = len(self.data["num_failed_recvs"])
|
||||
num_kv_expired_reqs = len(self.data["num_kv_expired_reqs"])
|
||||
|
||||
if self.num_successful_transfers == 0:
|
||||
return {
|
||||
"Num successful transfers": 0,
|
||||
"Avg xfer time (ms)": 0,
|
||||
"P90 xfer time (ms)": 0,
|
||||
"Avg MB per transfer": 0,
|
||||
"Throughput (MB/s)": 0,
|
||||
"Avg number of descriptors": 0,
|
||||
"Num failed transfers": num_failed_transfers,
|
||||
"Num failed recvs": num_failed_recvs,
|
||||
"Num KV expired reqs": num_kv_expired_reqs,
|
||||
}
|
||||
|
||||
xfer_time = np.asarray(self.data["transfer_duration"])
|
||||
mb = np.asarray(self.data["bytes_transferred"]) / 2**20
|
||||
descs = np.asarray(self.data["num_descriptors"], dtype=np.uint32)
|
||||
n = len(descs)
|
||||
assert n == self.num_successful_transfers
|
||||
|
||||
total_mb = mb.sum()
|
||||
avg_mb = total_mb / n
|
||||
total_time_seconds = xfer_time.sum()
|
||||
throughput_mb_s = (
|
||||
total_mb / total_time_seconds if total_time_seconds > 0 else 0.0
|
||||
)
|
||||
|
||||
return {
|
||||
"Num successful transfers": n,
|
||||
"Avg xfer time (ms)": round(xfer_time.mean() * 1e3, 3),
|
||||
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
|
||||
"Avg MB per transfer": round(avg_mb, 3),
|
||||
"Throughput (MB/s)": round(throughput_mb_s, 3),
|
||||
"Avg number of descriptors": round(descs.mean(), 1),
|
||||
"Num failed transfers": num_failed_transfers,
|
||||
"Num failed recvs": num_failed_recvs,
|
||||
"Num KV expired reqs": num_kv_expired_reqs,
|
||||
}
|
||||
|
||||
@property
|
||||
def num_successful_transfers(self) -> int:
|
||||
return len(self.data["transfer_duration"])
|
||||
Reference in New Issue
Block a user